mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-23 12:31:07 +01:00
vmalert: do not send message to alertmanager when alert has no label … (#6823)
…pair `alert_relabel_configs` in [notifier config](https://docs.victoriametrics.com/vmalert/#notifier-configuration-file) can drop alert labels when used to filter different tenant alert message to different notifier. alertmanager would report error like `msg="Failed to validate alerts" err="at least one label pair required"` in this case, but the rest of the alerts inside one request would still be valid in alertmanager, so it's not severe.
This commit is contained in:
Hui Wang
GitHub
parent
f9a8c09fe8
commit
ae4d376e41
@ -187,7 +187,7 @@ func templateAnnotation(dst io.Writer, text string, data tplData, tmpl *textTpl.
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a Alert) toPromLabels(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
|
||||
func (a Alert) applyRelabelingIfNeeded(relabelCfg *promrelabel.ParsedConfigs) []prompbmarshal.Label {
|
||||
var labels []prompbmarshal.Label
|
||||
for k, v := range a.Labels {
|
||||
labels = append(labels, prompbmarshal.Label{
|
||||
|
||||
@ -187,7 +187,7 @@ func TestAlert_toPromLabels(t *testing.T) {
|
||||
fn := func(labels map[string]string, exp []prompbmarshal.Label, relabel *promrelabel.ParsedConfigs) {
|
||||
t.Helper()
|
||||
a := Alert{Labels: labels}
|
||||
got := a.toPromLabels(relabel)
|
||||
got := a.applyRelabelingIfNeeded(relabel)
|
||||
if !reflect.DeepEqual(got, exp) {
|
||||
t.Fatalf("expected to have: \n%v;\ngot:\n%v",
|
||||
exp, got)
|
||||
|
||||
@ -8,6 +8,8 @@
|
||||
{% func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) %}
|
||||
[
|
||||
{% for i, alert := range alerts %}
|
||||
{% code lbls := alert.applyRelabelingIfNeeded(relabelCfg) %}
|
||||
{% if len(lbls) == 0 %} {% continue %} {% endif %}
|
||||
{
|
||||
"startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %},
|
||||
"generatorURL": {%q= generatorURL(alert) %},
|
||||
@ -15,7 +17,6 @@
|
||||
"endsAt":{%q= alert.End.Format(time.RFC3339Nano) %},
|
||||
{% endif %}
|
||||
"labels": {
|
||||
{% code lbls := alert.toPromLabels(relabelCfg) %}
|
||||
{% code ll := len(lbls) %}
|
||||
{% for idx, l := range lbls %}
|
||||
{%q= l.Name %}:{%q= l.Value %}{% if idx != ll-1 %}, {% endif %}
|
||||
|
||||
@ -30,111 +30,117 @@ func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL fun
|
||||
qw422016.N().S(`[`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:10
|
||||
for i, alert := range alerts {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:10
|
||||
qw422016.N().S(`{"startsAt":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
qw422016.N().S(`,"generatorURL":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:13
|
||||
qw422016.N().Q(generatorURL(alert))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:13
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:14
|
||||
if !alert.End.IsZero() {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:14
|
||||
qw422016.N().S(`"endsAt":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:15
|
||||
qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:15
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:16
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:16
|
||||
qw422016.N().S(`"labels": {`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:18
|
||||
lbls := alert.toPromLabels(relabelCfg)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:11
|
||||
lbls := alert.applyRelabelingIfNeeded(relabelCfg)
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:19
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
if len(lbls) == 0 {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
continue
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:12
|
||||
qw422016.N().S(`{"startsAt":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:14
|
||||
qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:14
|
||||
qw422016.N().S(`,"generatorURL":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:15
|
||||
qw422016.N().Q(generatorURL(alert))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:15
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:16
|
||||
if !alert.End.IsZero() {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:16
|
||||
qw422016.N().S(`"endsAt":`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:17
|
||||
qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:17
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:18
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:18
|
||||
qw422016.N().S(`"labels": {`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:20
|
||||
ll := len(lbls)
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:20
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
for idx, l := range lbls {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
qw422016.N().Q(l.Name)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
qw422016.N().Q(l.Value)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
if idx != ll-1 {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:21
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:23
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:22
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:23
|
||||
qw422016.N().S(`},"annotations": {`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:25
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:26
|
||||
c := len(alert.Annotations)
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:26
|
||||
for k, v := range alert.Annotations {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:27
|
||||
for k, v := range alert.Annotations {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
c = c - 1
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
qw422016.N().Q(k)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
qw422016.N().S(`:`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
qw422016.N().Q(v)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
if c > 0 {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:28
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:30
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:29
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:30
|
||||
qw422016.N().S(`}}`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:32
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
if i != len(alerts)-1 {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:32
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
qw422016.N().S(`,`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:32
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:34
|
||||
}
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:33
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:34
|
||||
qw422016.N().S(`]`)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
}
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
qw422016 := qt422016.AcquireWriter(qq422016)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
streamamRequest(qw422016, alerts, generatorURL, relabelCfg)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
qt422016.ReleaseWriter(qw422016)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
}
|
||||
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
func amRequest(alerts []Alert, generatorURL func(Alert) string, relabelCfg *promrelabel.ParsedConfigs) string {
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
qb422016 := qt422016.AcquireByteBuffer()
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
writeamRequest(qb422016, alerts, generatorURL, relabelCfg)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
qs422016 := string(qb422016.B)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
qt422016.ReleaseByteBuffer(qb422016)
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
return qs422016
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:35
|
||||
//line app/vmalert/notifier/alertmanager_request.qtpl:36
|
||||
}
|
||||
|
||||
@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
|
||||
)
|
||||
|
||||
func TestAlertManager_Addr(t *testing.T) {
|
||||
@ -76,12 +77,33 @@ func TestAlertManager_Send(t *testing.T) {
|
||||
if a[0].EndAt.IsZero() {
|
||||
t.Fatalf("expected non-zero end time")
|
||||
}
|
||||
if len(a[0].Labels) != 1 {
|
||||
t.Fatalf("expected 1 labels got %d", len(a[0].Labels))
|
||||
}
|
||||
if len(a[0].Annotations) != 2 {
|
||||
t.Fatalf("expected 2 annotations got %d", len(a[0].Annotations))
|
||||
}
|
||||
if r.Header.Get(headerKey) != "bar" {
|
||||
t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
|
||||
}
|
||||
case 3:
|
||||
if r.Header.Get(headerKey) != headerValue {
|
||||
t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, headerValue, r.Header.Get(headerKey))
|
||||
var a []struct {
|
||||
Labels map[string]string `json:"labels"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&a); err != nil {
|
||||
t.Fatalf("can not unmarshal data into alert %s", err)
|
||||
}
|
||||
if len(a) != 1 {
|
||||
t.Fatalf("expected 1 alert in array got %d", len(a))
|
||||
}
|
||||
if len(a[0].Labels) != 3 {
|
||||
t.Fatalf("expected 3 labels got %d", len(a[0].Labels))
|
||||
}
|
||||
if a[0].Labels["env"] != "prod" {
|
||||
t.Fatalf("expected env label to be prod during relabeling, got %s", a[0].Labels["env"])
|
||||
}
|
||||
if r.Header.Get(headerKey) != "bar" {
|
||||
t.Fatalf("expected header %q to be set to %q; got %q instead", headerKey, "bar", r.Header.Get(headerKey))
|
||||
}
|
||||
}
|
||||
})
|
||||
@ -95,31 +117,58 @@ func TestAlertManager_Send(t *testing.T) {
|
||||
},
|
||||
Headers: []string{fmt.Sprintf("%s:%s", headerKey, headerValue)},
|
||||
}
|
||||
parsedConfigs, err := promrelabel.ParseRelabelConfigsData([]byte(`
|
||||
- action: drop
|
||||
if: '{tenant="0"}'
|
||||
regex: ".*"
|
||||
- target_label: "env"
|
||||
replacement: "prod"
|
||||
if: '{tenant="1"}'
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error when parse relabeling config: %s", err)
|
||||
}
|
||||
am, err := NewAlertManager(srv.URL+alertManagerPath, func(alert Alert) string {
|
||||
return strconv.FormatUint(alert.GroupID, 10) + "/" + strconv.FormatUint(alert.ID, 10)
|
||||
}, aCfg, nil, 0)
|
||||
}, aCfg, parsedConfigs, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %s", err)
|
||||
}
|
||||
if err := am.Send(context.Background(), []Alert{{}, {}}, nil); err == nil {
|
||||
|
||||
if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil {
|
||||
t.Fatalf("expected connection error got nil")
|
||||
}
|
||||
if err := am.Send(context.Background(), []Alert{}, nil); err == nil {
|
||||
|
||||
if err := am.Send(context.Background(), []Alert{{Labels: map[string]string{"a": "b"}}}, nil); err == nil {
|
||||
t.Fatalf("expected wrong http code error got nil")
|
||||
}
|
||||
|
||||
if err := am.Send(context.Background(), []Alert{{
|
||||
GroupID: 0,
|
||||
Name: "alert0",
|
||||
Start: time.Now().UTC(),
|
||||
End: time.Now().UTC(),
|
||||
Annotations: map[string]string{"a": "b", "c": "d", "e": "f"},
|
||||
Labels: map[string]string{"alertname": "alert0"},
|
||||
Annotations: map[string]string{"a": "b", "c": "d"},
|
||||
}}, map[string]string{headerKey: "bar"}); err != nil {
|
||||
t.Fatalf("unexpected error %s", err)
|
||||
}
|
||||
if c != 2 {
|
||||
t.Fatalf("expected 2 calls(count from zero) to server got %d", c)
|
||||
}
|
||||
if err := am.Send(context.Background(), nil, map[string]string{headerKey: headerValue}); err != nil {
|
||||
|
||||
if err := am.Send(context.Background(), []Alert{
|
||||
// drop tenant0 alert message during relabeling
|
||||
{
|
||||
Name: "alert1",
|
||||
Labels: map[string]string{"rule": "test", "tenant": "0"},
|
||||
},
|
||||
{
|
||||
Name: "alert2",
|
||||
Labels: map[string]string{"rule": "test", "tenant": "1"},
|
||||
},
|
||||
}, map[string]string{headerKey: "bar"}); err != nil {
|
||||
t.Fatalf("unexpected error %s", err)
|
||||
}
|
||||
|
||||
if c != 3 {
|
||||
t.Fatalf("expected 3 calls(count from zero) to server got %d", c)
|
||||
}
|
||||
}
|
||||
|
||||
@ -35,6 +35,8 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
|
||||
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): fix metric `vm_object_references{type="indexdb"}`. Previously, it was overcounted.
|
||||
* BUGFIX: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): properly ingest stale NaN samples. Previously it could be dropped if series didn't exist at storage node. See this issue [https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5069] for details.
|
||||
|
||||
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): do not send notifications without labels to Alertmanager. Such notifications are rejected by Alertmanager anyway. Before, vmalert could send alert notifications even if no label-value pairs left after applying `alert_relabel_configs` from [notifier config](https://docs.victoriametrics.com/vmalert/#notifier-configuration-file).
|
||||
|
||||
## [v1.103.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.103.0)
|
||||
|
||||
Released at 2024-08-28
|
||||
|
||||
@ -198,6 +198,7 @@ headers:
|
||||
# Optional list of HTTP headers in form `header-name: value`
|
||||
# applied for all alert notifications sent to notifiers
|
||||
# generated by rules of this group.
|
||||
# It has higher priority over headers defined in notifier config.
|
||||
# For example:
|
||||
# notifier_headers:
|
||||
# - "TenantID: foo"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user