diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 1e9c489e4..749a161d2 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -144,6 +144,15 @@ params: headers: [ , ...] +# Optional list of HTTP headers in form `header-name: value` +# applied for all alert notifications sent to notifiers +# generated by rules of this group. +# For example: +# notifier_headers: +# - "TenantID: foo" +notifier_headers: + [ , ...] + # Optional list of labels added to every rule within a group. # It has priority over the external labels. # Labels are commonly used for adding environment diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index 16fb5bab1..0fc201ef9 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -36,7 +36,7 @@ type Group struct { Params url.Values `yaml:"params"` // Headers contains optional HTTP headers added to each rule request Headers []Header `yaml:"headers,omitempty"` - // NotifierHeaders contains optional HTTP headers added to each alert request which will send to notifier + // NotifierHeaders contains optional HTTP headers sent to notifiers for generated notifications NotifierHeaders []Header `yaml:"notifier_headers,omitempty"` // Catches all undefined fields and must be empty after parsing. XXX map[string]interface{} `yaml:",inline"` diff --git a/app/vmalert/config/config_test.go b/app/vmalert/config/config_test.go index 694e6ff0b..71eee2822 100644 --- a/app/vmalert/config/config_test.go +++ b/app/vmalert/config/config_test.go @@ -538,6 +538,24 @@ rules: `) }) + t.Run("`notifier_headers` change", func(t *testing.T) { + f(t, ` +name: TestGroup +notifier_headers: + - "TenantID: foo" +rules: + - alert: foo + expr: sum by(job) (up == 1) +`, ` +name: TestGroup +notifier_headers: + - "TenantID: bar" +rules: + - alert: foo + expr: sum by(job) (up == 1) +`) + }) + t.Run("`debug` change", func(t *testing.T) { f(t, ` name: TestGroup diff --git a/app/vmalert/config/testdata/rules/rules2-good.rules b/app/vmalert/config/testdata/rules/rules2-good.rules index 7b17282a0..6fb2d4a2c 100644 --- a/app/vmalert/config/testdata/rules/rules2-good.rules +++ b/app/vmalert/config/testdata/rules/rules2-good.rules @@ -5,6 +5,8 @@ groups: limit: 1000 headers: - "MyHeader: foo" + notifier_headers: + - "MyHeader: foo" params: denyPartialResponse: ["true"] rules: diff --git a/app/vmalert/group.go b/app/vmalert/group.go index acab27d64..1aae05b5d 100644 --- a/app/vmalert/group.go +++ b/app/vmalert/group.go @@ -300,9 +300,8 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r e := &executor{ rw: rw, notifiers: nts, + notifierHeaders: g.NotifierHeaders, previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label), - - notifierHeaders: g.NotifierHeaders, } evalTS := time.Now() @@ -379,6 +378,8 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r // ensure that staleness is tracked or existing rules only e.purgeStaleSeries(g.Rules) + e.notifierHeaders = g.NotifierHeaders + if g.Interval != ng.Interval { g.Interval = ng.Interval t.Stop() @@ -412,8 +413,10 @@ func getResolveDuration(groupInterval, delta, maxDuration time.Duration) time.Du } type executor struct { - notifiers func() []notifier.Notifier - rw *remotewrite.Client + notifiers func() []notifier.Notifier + notifierHeaders map[string]string + + rw *remotewrite.Client previouslySentSeriesToRWMu sync.Mutex // previouslySentSeriesToRW stores series sent to RW on previous iteration @@ -421,8 +424,6 @@ type executor struct { // where `ruleID` is ID of the Rule within a Group // and `ruleLabels` is []prompb.Label marshalled to a string previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label - - notifierHeaders map[string]string } func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration, limit int) chan error { diff --git a/app/vmalert/notifier/alertmanager.go b/app/vmalert/notifier/alertmanager.go index d30d34d7c..527ad17f8 100644 --- a/app/vmalert/notifier/alertmanager.go +++ b/app/vmalert/notifier/alertmanager.go @@ -51,16 +51,16 @@ func (am *AlertManager) Close() { func (am AlertManager) Addr() string { return am.addr } // Send an alert or resolve message -func (am *AlertManager) Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) error { +func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[string]string) error { am.metrics.alertsSent.Add(len(alerts)) - err := am.send(ctx, alerts, notifierHeaders) + err := am.send(ctx, alerts, headers) if err != nil { am.metrics.alertsSendErrors.Add(len(alerts)) } return err } -func (am *AlertManager) send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) error { +func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[string]string) error { b := &bytes.Buffer{} writeamRequest(b, alerts, am.argFunc, am.relabelConfigs) @@ -69,7 +69,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, notifierHeader return err } req.Header.Set("Content-Type", "application/json") - for key, value := range notifierHeaders { + for key, value := range headers { req.Header.Set(key, value) } diff --git a/app/vmalert/notifier/alertmanager_test.go b/app/vmalert/notifier/alertmanager_test.go index 9bc79009f..5f3ed8c6e 100644 --- a/app/vmalert/notifier/alertmanager_test.go +++ b/app/vmalert/notifier/alertmanager_test.go @@ -25,6 +25,7 @@ func TestAlertManager_Addr(t *testing.T) { func TestAlertManager_Send(t *testing.T) { const baUser, baPass = "foo", "bar" + const headerKey, headerValue = "TenantID", "foo" mux := http.NewServeMux() mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) { t.Errorf("should not be called") @@ -73,6 +74,11 @@ func TestAlertManager_Send(t *testing.T) { if a[0].EndAt.IsZero() { t.Errorf("expected non-zero end time") } + case 3: + if r.Header.Get(headerKey) != headerValue { + t.Errorf("expected header %q to be set to %q; got %q instead", + headerKey, headerValue, r.Header.Get(headerKey)) + } } }) srv := httptest.NewServer(mux) @@ -108,4 +114,7 @@ func TestAlertManager_Send(t *testing.T) { if c != 2 { t.Errorf("expected 2 calls(count from zero) to server got %d", c) } + if err := am.Send(context.Background(), nil, map[string]string{headerKey: headerValue}); err != nil { + t.Errorf("unexpected error %s", err) + } } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 3edf8882d..fdc96b107 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -32,6 +32,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add the metric relabel playground feature to the vmui. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3807). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy requests for unauthorized users. See [this doc](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4083). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to specify default route (`default_url`) for processing non-matched requests. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4084). +* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): support configuring of custom HTTP headers sent to notifiers on the Group level. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3260). * BUGFIX: reduce the probability of sudden increase in the number of small parts on systems with small number of CPU cores. * BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): fix performance issue when migrating data from VictoriaMetrics according to [these docs](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Add the ability to speed up the data migration via `--vm-native-disable-retries` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4092). diff --git a/docs/vmalert.md b/docs/vmalert.md index abf2232eb..e354cfbfd 100644 --- a/docs/vmalert.md +++ b/docs/vmalert.md @@ -148,6 +148,15 @@ params: headers: [ , ...] +# Optional list of HTTP headers in form `header-name: value` +# applied for all alert notifications sent to notifiers +# generated by rules of this group. +# For example: +# notifier_headers: +# - "TenantID: foo" +notifier_headers: + [ , ...] + # Optional list of labels added to every rule within a group. # It has priority over the external labels. # Labels are commonly used for adding environment