Merge pull request #539 from grafana/arve/sync-prometheus

Sync with latest Prometheus upstream

.golangci.yml

@@ -13,6 +13,7 @@ output:
 linters:
   enable:
     - depguard
+    - errorlint
     - gocritic
     - gofumpt
     - goimports
@@ -31,6 +32,24 @@ issues:
     - path: _test.go
       linters:
         - errcheck
+    - path: discovery/
+      linters:
+        - errorlint
+    - path: scrape/
+      linters:
+        - errorlint
+    - path: storage/
+      linters:
+        - errorlint
+    - path: tsdb/
+      linters:
+        - errorlint
+    - path: util/
+      linters:
+        - errorlint
+    - path: web/
+      linters:
+        - errorlint
 
 linters-settings:
   depguard:

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 2.47.1 / 2023-10-04
+
+* [BUGFIX] Fix duplicate sample detection at chunk size limit #12874
+
 ## 2.47.0 / 2023-09-06
 
 This release adds an experimental OpenTelemetry (OTLP) Ingestion feature,

README.md

@@ -14,6 +14,7 @@ examples and guides.</p>
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/486/badge)](https://bestpractices.coreinfrastructure.org/projects/486)
 [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/prometheus/prometheus)
 [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/prometheus.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:prometheus)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus/badge)](https://api.securityscorecards.dev/projects/github.com/prometheus/prometheus)
 
 </div>
 

RELEASE.md

@@ -52,7 +52,8 @@ Release cadence of first pre-releases being cut is 6 weeks.
 | v2.45 LTS      | 2023-05-31                                 | Jesus Vazquez (Github: @jesusvazquez)       |
 | v2.46          | 2023-07-12                                 | Julien Pivotto (GitHub: @roidelapluie)      |
 | v2.47          | 2023-08-23                                 | Bryan Boreham (GitHub: @bboreham)           |
-| v2.48          | 2023-10-04                                 | **searching for volunteer**                 |
+| v2.48          | 2023-10-04                                 | Levi Harrison (GitHub: @LeviHarrison)       |
+| v2.49          | 2023-11-15                                 | **searching for volunteer**                 |
 
 If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice.
 

VERSION

@@ -1 +1 @@
-2.47.0
+2.47.1

cmd/prometheus/main_test.go

@@ -498,10 +498,9 @@ func TestDocumentation(t *testing.T) {
 	cmd.Stdout = &stdout
 
 	if err := cmd.Run(); err != nil {
-		if exitError, ok := err.(*exec.ExitError); ok {
-			if exitError.ExitCode() != 0 {
-				fmt.Println("Command failed with non-zero exit code")
-			}
+		var exitError *exec.ExitError
+		if errors.As(err, &exitError) && exitError.ExitCode() != 0 {
+			fmt.Println("Command failed with non-zero exit code")
 		}
 	}
 

cmd/promtool/main.go

@@ -733,30 +733,7 @@ func CheckRules(ls lintConfig, files ...string) int {
 	failed := false
 	hasErrors := false
 	if len(files) == 0 {
-		fmt.Println("Checking standard input")
-		data, err := io.ReadAll(os.Stdin)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, "  FAILED:", err)
-			return failureExitCode
-		}
-		rgs, errs := rulefmt.Parse(data)
-		for _, e := range errs {
-			fmt.Fprintln(os.Stderr, e.Error())
-			return failureExitCode
-		}
-		if n, errs := checkRuleGroups(rgs, ls); errs != nil {
-			fmt.Fprintln(os.Stderr, "  FAILED:")
-			for _, e := range errs {
-				fmt.Fprintln(os.Stderr, e.Error())
-			}
-			failed = true
-			for _, err := range errs {
-				hasErrors = hasErrors || !errors.Is(err, lintError)
-			}
-		} else {
-			fmt.Printf("  SUCCESS: %d rules found\n", n)
-		}
-		fmt.Println()
+		failed, hasErrors = checkRulesFromStdin(ls)
 	} else {
 		failed, hasErrors = checkRules(files, ls)
 	}
@@ -771,6 +748,44 @@ func CheckRules(ls lintConfig, files ...string) int {
 	return successExitCode
 }
 
+// checkRulesFromStdin validates rule from stdin.
+func checkRulesFromStdin(ls lintConfig) (bool, bool) {
+	failed := false
+	hasErrors := false
+	fmt.Println("Checking standard input")
+	data, err := io.ReadAll(os.Stdin)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "  FAILED:", err)
+		return true, true
+	}
+	rgs, errs := rulefmt.Parse(data)
+	if errs != nil {
+		failed = true
+		fmt.Fprintln(os.Stderr, "  FAILED:")
+		for _, e := range errs {
+			fmt.Fprintln(os.Stderr, e.Error())
+			hasErrors = hasErrors || !errors.Is(e, lintError)
+		}
+		if hasErrors {
+			return failed, hasErrors
+		}
+	}
+	if n, errs := checkRuleGroups(rgs, ls); errs != nil {
+		fmt.Fprintln(os.Stderr, "  FAILED:")
+		for _, e := range errs {
+			fmt.Fprintln(os.Stderr, e.Error())
+		}
+		failed = true
+		for _, err := range errs {
+			hasErrors = hasErrors || !errors.Is(err, lintError)
+		}
+	} else {
+		fmt.Printf("  SUCCESS: %d rules found\n", n)
+	}
+	fmt.Println()
+	return failed, hasErrors
+}
+
 // checkRules validates rule files.
 func checkRules(files []string, ls lintConfig) (bool, bool) {
 	failed := false
@@ -780,7 +795,14 @@ func checkRules(files []string, ls lintConfig) (bool, bool) {
 		rgs, errs := rulefmt.ParseFile(f)
 		if errs != nil {
 			failed = true
-			continue
+			fmt.Fprintln(os.Stderr, "  FAILED:")
+			for _, e := range errs {
+				fmt.Fprintln(os.Stderr, e.Error())
+				hasErrors = hasErrors || !errors.Is(e, lintError)
+			}
+			if hasErrors {
+				continue
+			}
 		}
 		if n, errs := checkRuleGroups(rgs, ls); errs != nil {
 			fmt.Fprintln(os.Stderr, "  FAILED:")

cmd/promtool/main_test.go

@@ -450,10 +450,9 @@ func TestDocumentation(t *testing.T) {
 	cmd.Stdout = &stdout
 
 	if err := cmd.Run(); err != nil {
-		if exitError, ok := err.(*exec.ExitError); ok {
-			if exitError.ExitCode() != 0 {
-				fmt.Println("Command failed with non-zero exit code")
-			}
+		var exitError *exec.ExitError
+		if errors.As(err, &exitError) && exitError.ExitCode() != 0 {
+			fmt.Println("Command failed with non-zero exit code")
 		}
 	}
 
@@ -464,3 +463,88 @@ func TestDocumentation(t *testing.T) {
 
 	require.Equal(t, string(expectedContent), generatedContent, "Generated content does not match documentation. Hint: run `make cli-documentation`.")
 }
+
+func TestCheckRules(t *testing.T) {
+	t.Run("rules-good", func(t *testing.T) {
+		data, err := os.ReadFile("./testdata/rules.yml")
+		require.NoError(t, err)
+		r, w, err := os.Pipe()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		_, err = w.Write(data)
+		if err != nil {
+			t.Error(err)
+		}
+		w.Close()
+
+		// Restore stdin right after the test.
+		defer func(v *os.File) { os.Stdin = v }(os.Stdin)
+		os.Stdin = r
+
+		exitCode := CheckRules(newLintConfig(lintOptionDuplicateRules, false))
+		require.Equal(t, successExitCode, exitCode, "")
+	})
+
+	t.Run("rules-bad", func(t *testing.T) {
+		data, err := os.ReadFile("./testdata/rules-bad.yml")
+		require.NoError(t, err)
+		r, w, err := os.Pipe()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		_, err = w.Write(data)
+		if err != nil {
+			t.Error(err)
+		}
+		w.Close()
+
+		// Restore stdin right after the test.
+		defer func(v *os.File) { os.Stdin = v }(os.Stdin)
+		os.Stdin = r
+
+		exitCode := CheckRules(newLintConfig(lintOptionDuplicateRules, false))
+		require.Equal(t, failureExitCode, exitCode, "")
+	})
+
+	t.Run("rules-lint-fatal", func(t *testing.T) {
+		data, err := os.ReadFile("./testdata/prometheus-rules.lint.yml")
+		require.NoError(t, err)
+		r, w, err := os.Pipe()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		_, err = w.Write(data)
+		if err != nil {
+			t.Error(err)
+		}
+		w.Close()
+
+		// Restore stdin right after the test.
+		defer func(v *os.File) { os.Stdin = v }(os.Stdin)
+		os.Stdin = r
+
+		exitCode := CheckRules(newLintConfig(lintOptionDuplicateRules, true))
+		require.Equal(t, lintErrExitCode, exitCode, "")
+	})
+}
+
+func TestCheckRulesWithRuleFiles(t *testing.T) {
+	t.Run("rules-good", func(t *testing.T) {
+		exitCode := CheckRules(newLintConfig(lintOptionDuplicateRules, false), "./testdata/rules.yml")
+		require.Equal(t, successExitCode, exitCode, "")
+	})
+
+	t.Run("rules-bad", func(t *testing.T) {
+		exitCode := CheckRules(newLintConfig(lintOptionDuplicateRules, false), "./testdata/rules-bad.yml")
+		require.Equal(t, failureExitCode, exitCode, "")
+	})
+
+	t.Run("rules-lint-fatal", func(t *testing.T) {
+		exitCode := CheckRules(newLintConfig(lintOptionDuplicateRules, true), "./testdata/prometheus-rules.lint.yml")
+		require.Equal(t, lintErrExitCode, exitCode, "")
+	})
+}

cmd/promtool/testdata/rules-bad.yml

@@ -0,0 +1,28 @@
+# This is the rules file.
+
+groups:
+  - name: alerts
+    rules:
+      - alert: InstanceDown
+        expr: up == 0
+        for: 5m
+        labels:
+          severity: page
+        annotations:
+          summary: "Instance {{ $label.foo }} down"
+          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
+      - alert: AlwaysFiring
+        expr: 1
+
+  - name: rules
+    rules:
+      - record: job:test:count_over_time1m
+        expr: sum without(instance) (count_over_time(test[1m]))
+
+      # A recording rule that doesn't depend on input series.
+      - record: fixed_data
+        expr: 1
+
+      # Subquery with default resolution test.
+      - record: suquery_interval_test
+        expr: count_over_time(up[5m:])

cmd/promtool/tsdb.go

@@ -459,7 +459,16 @@ func analyzeBlock(ctx context.Context, path, blockID string, limit int, runExten
 	postingInfos := []postingInfo{}
 
 	printInfo := func(postingInfos []postingInfo) {
-		slices.SortFunc(postingInfos, func(a, b postingInfo) bool { return a.metric > b.metric })
+		slices.SortFunc(postingInfos, func(a, b postingInfo) int {
+			switch {
+			case b.metric < a.metric:
+				return -1
+			case b.metric > a.metric:
+				return 1
+			default:
+				return 0
+			}
+		})
 
 		for i, pc := range postingInfos {
 			if i >= limit {

cmd/promtool/unittest.go

@@ -241,7 +241,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i
 				g.Eval(suite.Context(), ts)
 				for _, r := range g.Rules() {
 					if r.LastError() != nil {
-						evalErrs = append(evalErrs, fmt.Errorf("    rule: %s, time: %s, err: %v",
+						evalErrs = append(evalErrs, fmt.Errorf("    rule: %s, time: %s, err: %w",
 							r.Name(), ts.Sub(time.Unix(0, 0).UTC()), r.LastError()))
 					}
 				}

discovery/legacymanager/manager.go

@@ -140,11 +140,9 @@ type Manager struct {
 // Run starts the background processing
 func (m *Manager) Run() error {
 	go m.sender()
-	for range m.ctx.Done() {
-		m.cancelDiscoverers()
-		return m.ctx.Err()
-	}
-	return nil
+	<-m.ctx.Done()
+	m.cancelDiscoverers()
+	return m.ctx.Err()
 }
 
 // SyncCh returns a read only channel used by all the clients to receive target updates.

discovery/manager.go

@@ -180,11 +180,9 @@ func (m *Manager) Providers() []*Provider {
 // Run starts the background processing.
 func (m *Manager) Run() error {
 	go m.sender()
-	for range m.ctx.Done() {
-		m.cancelDiscoverers()
-		return m.ctx.Err()
-	}
-	return nil
+	<-m.ctx.Done()
+	m.cancelDiscoverers()
+	return m.ctx.Err()
 }
 
 // SyncCh returns a read only channel used by all the clients to receive target updates.

docs/configuration/configuration.md

@@ -3537,7 +3537,13 @@ azuread:
 
   # Azure User-assigned Managed identity.
   [ managed_identity:
-      [ client_id: <string> ]  
+      [ client_id: <string> ] ]  
+
+  # Azure OAuth.
+  [ oauth:
+      [ client_id: <string> ]
+      [ client_secret: <string> ]
+      [ tenant_id: <string> ] ]
 
 # Configures the remote write request's TLS settings.
 tls_config:

docs/configuration/recording_rules.md

@@ -147,3 +147,7 @@ by the rule are discarded, and if it's an alerting rule, _all_ alerts for
 the rule, active, pending, or inactive, are cleared as well. The event will be
 recorded as an error in the evaluation, and as such no stale markers are
 written.
+
+# Failed rule evaluations due to slow evaluation
+
+If a rule group hasn't finished evaluating before its next evaluation is supposed to start (as defined by the `evaluation_interval`), the next evaluation will be skipped. Subsequent evaluations of the rule group will continue to be skipped until the initial evaluation either completes or times out. When this happens, there will be a gap in the metric produced by the recording rule. The `rule_group_iterations_missed_total` metric will be incremented for each missed iteration of the rule group. 

docs/configuration/template_reference.md

@@ -86,6 +86,7 @@ versions.
 | args          | []interface{} | map[string]interface{} | This converts a list of objects to a map with keys arg0, arg1 etc. This is intended to allow multiple arguments to be passed to templates. |
 | tmpl          | string, []interface{} | nothing  | Like the built-in `template`, but allows non-literals as the template name. Note that the result is assumed to be safe, and will not be auto-escaped. Only available in consoles. |
 | safeHtml      | string        | string  | Marks string as HTML not requiring auto-escaping. |
+| externalURL   | _none_        | string  | The external URL under which Prometheus is externally reachable. |
 | pathPrefix    | _none_        | string  | The external URL [path](https://pkg.go.dev/net/url#URL) for use in console templates. |
 
 ## Template type differences

docs/querying/functions.md

@@ -14,7 +14,7 @@ vector, which if not provided it will default to the value of the expression
 _Notes about the experimental native histograms:_
 
 * Ingesting native histograms has to be enabled via a [feature
-  flag](../../feature_flags.md#native-histograms). As long as no native histograms
+  flag](../feature_flags.md#native-histograms). As long as no native histograms
   have been ingested into the TSDB, all functions will behave as usual.
 * Functions that do not explicitly mention native histograms in their
   documentation (see below) will ignore histogram samples.