app/vmalert: follow-up after 6c322b4a00 (#4214)

6c322b4a00

Signed-off-by: hagen1778 <roman@victoriametrics.com>

app/vmalert/README.md

@@ -144,6 +144,15 @@ params:
 headers:
   [ <string>, ...]
 
+# Optional list of HTTP headers in form `header-name: value`
+# applied for all alert notifications sent to notifiers 
+# generated by rules of this group.
+# For example:
+#  notifier_headers:
+#    - "TenantID: foo"
+notifier_headers:
+        [ <string>, ...]
+
 # Optional list of labels added to every rule within a group.
 # It has priority over the external labels.
 # Labels are commonly used for adding environment

app/vmalert/config/config.go

@@ -36,7 +36,7 @@ type Group struct {
 	Params url.Values `yaml:"params"`
 	// Headers contains optional HTTP headers added to each rule request
 	Headers []Header `yaml:"headers,omitempty"`
-	// NotifierHeaders contains optional HTTP headers added to each alert request which will send to notifier
+	// NotifierHeaders contains optional HTTP headers sent to notifiers for generated notifications
 	NotifierHeaders []Header `yaml:"notifier_headers,omitempty"`
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]interface{} `yaml:",inline"`

app/vmalert/config/config_test.go

@@ -538,6 +538,24 @@ rules:
 `)
 	})
 
+	t.Run("`notifier_headers` change", func(t *testing.T) {
+		f(t, `
+name: TestGroup
+notifier_headers:
+  - "TenantID: foo"
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`, `
+name: TestGroup
+notifier_headers:
+  - "TenantID: bar"
+rules:
+  - alert: foo
+    expr: sum by(job) (up == 1)
+`)
+	})
+
 	t.Run("`debug` change", func(t *testing.T) {
 		f(t, `
 name: TestGroup

app/vmalert/config/testdata/rules/rules2-good.rules

@@ -5,6 +5,8 @@ groups:
     limit: 1000
     headers:
       - "MyHeader: foo"
+    notifier_headers:
+      - "MyHeader: foo"
     params:
       denyPartialResponse: ["true"]
     rules:

app/vmalert/group.go

@@ -300,9 +300,8 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 	e := &executor{
 		rw:                       rw,
 		notifiers:                nts,
+		notifierHeaders:          g.NotifierHeaders,
 		previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
-
-		notifierHeaders: g.NotifierHeaders,
 	}
 
 	evalTS := time.Now()
@@ -379,6 +378,8 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
 			// ensure that staleness is tracked or existing rules only
 			e.purgeStaleSeries(g.Rules)
 
+			e.notifierHeaders = g.NotifierHeaders
+
 			if g.Interval != ng.Interval {
 				g.Interval = ng.Interval
 				t.Stop()
@@ -412,8 +413,10 @@ func getResolveDuration(groupInterval, delta, maxDuration time.Duration) time.Du
 }
 
 type executor struct {
-	notifiers func() []notifier.Notifier
-	rw        *remotewrite.Client
+	notifiers       func() []notifier.Notifier
+	notifierHeaders map[string]string
+
+	rw *remotewrite.Client
 
 	previouslySentSeriesToRWMu sync.Mutex
 	// previouslySentSeriesToRW stores series sent to RW on previous iteration
@@ -421,8 +424,6 @@ type executor struct {
 	// where `ruleID` is ID of the Rule within a Group
 	// and `ruleLabels` is []prompb.Label marshalled to a string
 	previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label
-
-	notifierHeaders map[string]string
 }
 
 func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration, limit int) chan error {

app/vmalert/notifier/alertmanager.go

@@ -51,16 +51,16 @@ func (am *AlertManager) Close() {
 func (am AlertManager) Addr() string { return am.addr }
 
 // Send an alert or resolve message
-func (am *AlertManager) Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) error {
+func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[string]string) error {
 	am.metrics.alertsSent.Add(len(alerts))
-	err := am.send(ctx, alerts, notifierHeaders)
+	err := am.send(ctx, alerts, headers)
 	if err != nil {
 		am.metrics.alertsSendErrors.Add(len(alerts))
 	}
 	return err
 }
 
-func (am *AlertManager) send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) error {
+func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[string]string) error {
 	b := &bytes.Buffer{}
 	writeamRequest(b, alerts, am.argFunc, am.relabelConfigs)
 
@@ -69,7 +69,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, notifierHeader
 		return err
 	}
 	req.Header.Set("Content-Type", "application/json")
-	for key, value := range notifierHeaders {
+	for key, value := range headers {
 		req.Header.Set(key, value)
 	}
 

app/vmalert/notifier/alertmanager_test.go

@@ -25,6 +25,7 @@ func TestAlertManager_Addr(t *testing.T) {
 
 func TestAlertManager_Send(t *testing.T) {
 	const baUser, baPass = "foo", "bar"
+	const headerKey, headerValue = "TenantID", "foo"
 	mux := http.NewServeMux()
 	mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
 		t.Errorf("should not be called")
@@ -73,6 +74,11 @@ func TestAlertManager_Send(t *testing.T) {
 			if a[0].EndAt.IsZero() {
 				t.Errorf("expected non-zero end time")
 			}
+		case 3:
+			if r.Header.Get(headerKey) != headerValue {
+				t.Errorf("expected header %q to be set to %q; got %q instead",
+					headerKey, headerValue, r.Header.Get(headerKey))
+			}
 		}
 	})
 	srv := httptest.NewServer(mux)
@@ -108,4 +114,7 @@ func TestAlertManager_Send(t *testing.T) {
 	if c != 2 {
 		t.Errorf("expected 2 calls(count from zero) to server got %d", c)
 	}
+	if err := am.Send(context.Background(), nil, map[string]string{headerKey: headerValue}); err != nil {
+		t.Errorf("unexpected error %s", err)
+	}
 }

docs/CHANGELOG.md

@@ -33,6 +33,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy requests for unauthorized users. See [this doc](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4083).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to specify default route (`default_url`) for processing non-matched requests. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4084).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to filter incoming requests by IP. See [these docs](https://docs.victoriametrics.com/vmauth.html#ip-filters) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3491).
+* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): support configuring of custom HTTP headers sent to notifiers on the Group level. See  [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3260).
 
 * BUGFIX: reduce the probability of sudden increase in the number of small parts on systems with small number of CPU cores.
 * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): fix performance issue when migrating data from VictoriaMetrics according to [these docs](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Add the ability to speed up the data migration via `--vm-native-disable-retries` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4092).

docs/vmalert.md

@@ -148,6 +148,15 @@ params:
 headers:
   [ <string>, ...]
 
+# Optional list of HTTP headers in form `header-name: value`
+# applied for all alert notifications sent to notifiers 
+# generated by rules of this group.
+# For example:
+#  notifier_headers:
+#    - "TenantID: foo"
+notifier_headers:
+        [ <string>, ...]
+
 # Optional list of labels added to every rule within a group.
 # It has priority over the external labels.
 # Labels are commonly used for adding environment