mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-15 00:13:30 +01:00
app/vmalert: follow-up after 6c322b4a00
(#4214)
6c322b4a00
Signed-off-by: hagen1778 <roman@victoriametrics.com>
This commit is contained in:
parent
4b0db17bec
commit
1c3bf0d0d8
@ -144,6 +144,15 @@ params:
|
||||
headers:
|
||||
[ <string>, ...]
|
||||
|
||||
# Optional list of HTTP headers in form `header-name: value`
|
||||
# applied for all alert notifications sent to notifiers
|
||||
# generated by rules of this group.
|
||||
# For example:
|
||||
# notifier_headers:
|
||||
# - "TenantID: foo"
|
||||
notifier_headers:
|
||||
[ <string>, ...]
|
||||
|
||||
# Optional list of labels added to every rule within a group.
|
||||
# It has priority over the external labels.
|
||||
# Labels are commonly used for adding environment
|
||||
|
@ -36,7 +36,7 @@ type Group struct {
|
||||
Params url.Values `yaml:"params"`
|
||||
// Headers contains optional HTTP headers added to each rule request
|
||||
Headers []Header `yaml:"headers,omitempty"`
|
||||
// NotifierHeaders contains optional HTTP headers added to each alert request which will send to notifier
|
||||
// NotifierHeaders contains optional HTTP headers sent to notifiers for generated notifications
|
||||
NotifierHeaders []Header `yaml:"notifier_headers,omitempty"`
|
||||
// Catches all undefined fields and must be empty after parsing.
|
||||
XXX map[string]interface{} `yaml:",inline"`
|
||||
|
@ -538,6 +538,24 @@ rules:
|
||||
`)
|
||||
})
|
||||
|
||||
t.Run("`notifier_headers` change", func(t *testing.T) {
|
||||
f(t, `
|
||||
name: TestGroup
|
||||
notifier_headers:
|
||||
- "TenantID: foo"
|
||||
rules:
|
||||
- alert: foo
|
||||
expr: sum by(job) (up == 1)
|
||||
`, `
|
||||
name: TestGroup
|
||||
notifier_headers:
|
||||
- "TenantID: bar"
|
||||
rules:
|
||||
- alert: foo
|
||||
expr: sum by(job) (up == 1)
|
||||
`)
|
||||
})
|
||||
|
||||
t.Run("`debug` change", func(t *testing.T) {
|
||||
f(t, `
|
||||
name: TestGroup
|
||||
|
@ -5,6 +5,8 @@ groups:
|
||||
limit: 1000
|
||||
headers:
|
||||
- "MyHeader: foo"
|
||||
notifier_headers:
|
||||
- "MyHeader: foo"
|
||||
params:
|
||||
denyPartialResponse: ["true"]
|
||||
rules:
|
||||
|
@ -300,9 +300,8 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
|
||||
e := &executor{
|
||||
rw: rw,
|
||||
notifiers: nts,
|
||||
notifierHeaders: g.NotifierHeaders,
|
||||
previouslySentSeriesToRW: make(map[uint64]map[string][]prompbmarshal.Label),
|
||||
|
||||
notifierHeaders: g.NotifierHeaders,
|
||||
}
|
||||
|
||||
evalTS := time.Now()
|
||||
@ -379,6 +378,8 @@ func (g *Group) start(ctx context.Context, nts func() []notifier.Notifier, rw *r
|
||||
// ensure that staleness is tracked or existing rules only
|
||||
e.purgeStaleSeries(g.Rules)
|
||||
|
||||
e.notifierHeaders = g.NotifierHeaders
|
||||
|
||||
if g.Interval != ng.Interval {
|
||||
g.Interval = ng.Interval
|
||||
t.Stop()
|
||||
@ -412,8 +413,10 @@ func getResolveDuration(groupInterval, delta, maxDuration time.Duration) time.Du
|
||||
}
|
||||
|
||||
type executor struct {
|
||||
notifiers func() []notifier.Notifier
|
||||
rw *remotewrite.Client
|
||||
notifiers func() []notifier.Notifier
|
||||
notifierHeaders map[string]string
|
||||
|
||||
rw *remotewrite.Client
|
||||
|
||||
previouslySentSeriesToRWMu sync.Mutex
|
||||
// previouslySentSeriesToRW stores series sent to RW on previous iteration
|
||||
@ -421,8 +424,6 @@ type executor struct {
|
||||
// where `ruleID` is ID of the Rule within a Group
|
||||
// and `ruleLabels` is []prompb.Label marshalled to a string
|
||||
previouslySentSeriesToRW map[uint64]map[string][]prompbmarshal.Label
|
||||
|
||||
notifierHeaders map[string]string
|
||||
}
|
||||
|
||||
func (e *executor) execConcurrently(ctx context.Context, rules []Rule, ts time.Time, concurrency int, resolveDuration time.Duration, limit int) chan error {
|
||||
|
@ -51,16 +51,16 @@ func (am *AlertManager) Close() {
|
||||
func (am AlertManager) Addr() string { return am.addr }
|
||||
|
||||
// Send an alert or resolve message
|
||||
func (am *AlertManager) Send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) error {
|
||||
func (am *AlertManager) Send(ctx context.Context, alerts []Alert, headers map[string]string) error {
|
||||
am.metrics.alertsSent.Add(len(alerts))
|
||||
err := am.send(ctx, alerts, notifierHeaders)
|
||||
err := am.send(ctx, alerts, headers)
|
||||
if err != nil {
|
||||
am.metrics.alertsSendErrors.Add(len(alerts))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (am *AlertManager) send(ctx context.Context, alerts []Alert, notifierHeaders map[string]string) error {
|
||||
func (am *AlertManager) send(ctx context.Context, alerts []Alert, headers map[string]string) error {
|
||||
b := &bytes.Buffer{}
|
||||
writeamRequest(b, alerts, am.argFunc, am.relabelConfigs)
|
||||
|
||||
@ -69,7 +69,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert, notifierHeader
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
for key, value := range notifierHeaders {
|
||||
for key, value := range headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
|
@ -25,6 +25,7 @@ func TestAlertManager_Addr(t *testing.T) {
|
||||
|
||||
func TestAlertManager_Send(t *testing.T) {
|
||||
const baUser, baPass = "foo", "bar"
|
||||
const headerKey, headerValue = "TenantID", "foo"
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/", func(_ http.ResponseWriter, _ *http.Request) {
|
||||
t.Errorf("should not be called")
|
||||
@ -73,6 +74,11 @@ func TestAlertManager_Send(t *testing.T) {
|
||||
if a[0].EndAt.IsZero() {
|
||||
t.Errorf("expected non-zero end time")
|
||||
}
|
||||
case 3:
|
||||
if r.Header.Get(headerKey) != headerValue {
|
||||
t.Errorf("expected header %q to be set to %q; got %q instead",
|
||||
headerKey, headerValue, r.Header.Get(headerKey))
|
||||
}
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
@ -108,4 +114,7 @@ func TestAlertManager_Send(t *testing.T) {
|
||||
if c != 2 {
|
||||
t.Errorf("expected 2 calls(count from zero) to server got %d", c)
|
||||
}
|
||||
if err := am.Send(context.Background(), nil, map[string]string{headerKey: headerValue}); err != nil {
|
||||
t.Errorf("unexpected error %s", err)
|
||||
}
|
||||
}
|
||||
|
@ -33,6 +33,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy requests for unauthorized users. See [this doc](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4083).
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to specify default route (`default_url`) for processing non-matched requests. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4084).
|
||||
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to filter incoming requests by IP. See [these docs](https://docs.victoriametrics.com/vmauth.html#ip-filters) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3491).
|
||||
* FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): support configuring of custom HTTP headers sent to notifiers on the Group level. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3260).
|
||||
|
||||
* BUGFIX: reduce the probability of sudden increase in the number of small parts on systems with small number of CPU cores.
|
||||
* BUGFIX: [vmctl](https://docs.victoriametrics.com/vmctl.html): fix performance issue when migrating data from VictoriaMetrics according to [these docs](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Add the ability to speed up the data migration via `--vm-native-disable-retries` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4092).
|
||||
|
@ -148,6 +148,15 @@ params:
|
||||
headers:
|
||||
[ <string>, ...]
|
||||
|
||||
# Optional list of HTTP headers in form `header-name: value`
|
||||
# applied for all alert notifications sent to notifiers
|
||||
# generated by rules of this group.
|
||||
# For example:
|
||||
# notifier_headers:
|
||||
# - "TenantID: foo"
|
||||
notifier_headers:
|
||||
[ <string>, ...]
|
||||
|
||||
# Optional list of labels added to every rule within a group.
|
||||
# It has priority over the external labels.
|
||||
# Labels are commonly used for adding environment
|
||||
|
Loading…
Reference in New Issue
Block a user