2020-05-10 18:58:17 +02:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"sort"
|
|
|
|
"testing"
|
|
|
|
"time"
|
|
|
|
|
2020-06-01 12:46:37 +02:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
|
2020-05-10 18:58:17 +02:00
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
|
|
|
|
)
|
|
|
|
|
|
|
|
func TestUpdateWith(t *testing.T) {
|
|
|
|
testCases := []struct {
|
|
|
|
name string
|
2020-06-01 12:46:37 +02:00
|
|
|
currentRules []Rule
|
|
|
|
// rules must be sorted by ID
|
|
|
|
newRules []Rule
|
2020-05-10 18:58:17 +02:00
|
|
|
}{
|
|
|
|
{
|
|
|
|
"new rule",
|
2020-06-01 12:46:37 +02:00
|
|
|
[]Rule{},
|
|
|
|
[]Rule{&AlertingRule{Name: "bar"}},
|
2020-05-10 18:58:17 +02:00
|
|
|
},
|
|
|
|
{
|
2020-06-01 12:46:37 +02:00
|
|
|
"update alerting rule",
|
|
|
|
[]Rule{&AlertingRule{
|
2020-05-10 18:58:17 +02:00
|
|
|
Name: "foo",
|
|
|
|
Expr: "up > 0",
|
|
|
|
For: time.Second,
|
|
|
|
Labels: map[string]string{
|
|
|
|
"bar": "baz",
|
|
|
|
},
|
|
|
|
Annotations: map[string]string{
|
|
|
|
"summary": "{{ $value|humanize }}",
|
|
|
|
"description": "{{$labels}}",
|
|
|
|
},
|
|
|
|
}},
|
2020-06-01 12:46:37 +02:00
|
|
|
[]Rule{&AlertingRule{
|
|
|
|
Name: "foo",
|
2020-05-10 18:58:17 +02:00
|
|
|
Expr: "up > 10",
|
|
|
|
For: time.Second,
|
|
|
|
Labels: map[string]string{
|
|
|
|
"baz": "bar",
|
|
|
|
},
|
|
|
|
Annotations: map[string]string{
|
|
|
|
"summary": "none",
|
|
|
|
},
|
|
|
|
}},
|
|
|
|
},
|
2020-06-01 12:46:37 +02:00
|
|
|
{
|
|
|
|
"update recording rule",
|
|
|
|
[]Rule{&RecordingRule{
|
|
|
|
Name: "foo",
|
|
|
|
Expr: "max(up)",
|
|
|
|
Labels: map[string]string{
|
|
|
|
"bar": "baz",
|
|
|
|
},
|
|
|
|
}},
|
|
|
|
[]Rule{&RecordingRule{
|
|
|
|
Name: "foo",
|
|
|
|
Expr: "min(up)",
|
|
|
|
Labels: map[string]string{
|
|
|
|
"baz": "bar",
|
|
|
|
},
|
|
|
|
}},
|
|
|
|
},
|
2020-05-10 18:58:17 +02:00
|
|
|
{
|
|
|
|
"empty rule",
|
2020-06-01 12:46:37 +02:00
|
|
|
[]Rule{&AlertingRule{Name: "foo"}, &RecordingRule{Name: "bar"}},
|
|
|
|
[]Rule{},
|
2020-05-10 18:58:17 +02:00
|
|
|
},
|
|
|
|
{
|
|
|
|
"multiple rules",
|
2020-06-01 12:46:37 +02:00
|
|
|
[]Rule{
|
|
|
|
&AlertingRule{Name: "bar"},
|
|
|
|
&AlertingRule{Name: "baz"},
|
|
|
|
&RecordingRule{Name: "foo"},
|
|
|
|
},
|
|
|
|
[]Rule{
|
|
|
|
&AlertingRule{Name: "baz"},
|
|
|
|
&RecordingRule{Name: "foo"},
|
|
|
|
},
|
2020-05-15 08:55:22 +02:00
|
|
|
},
|
|
|
|
{
|
|
|
|
"replace rule",
|
2020-06-01 12:46:37 +02:00
|
|
|
[]Rule{&AlertingRule{Name: "foo1"}},
|
|
|
|
[]Rule{&AlertingRule{Name: "foo2"}},
|
2020-05-15 08:55:22 +02:00
|
|
|
},
|
|
|
|
{
|
|
|
|
"replace multiple rules",
|
2020-06-01 12:46:37 +02:00
|
|
|
[]Rule{
|
|
|
|
&AlertingRule{Name: "foo1"},
|
|
|
|
&RecordingRule{Name: "foo2"},
|
|
|
|
&AlertingRule{Name: "foo3"},
|
|
|
|
},
|
|
|
|
[]Rule{
|
|
|
|
&AlertingRule{Name: "foo3"},
|
|
|
|
&AlertingRule{Name: "foo4"},
|
|
|
|
&RecordingRule{Name: "foo5"},
|
|
|
|
},
|
2020-05-10 18:58:17 +02:00
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, tc := range testCases {
|
|
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
|
|
g := &Group{Rules: tc.currentRules}
|
2020-06-01 12:46:37 +02:00
|
|
|
err := g.updateWith(&Group{Rules: tc.newRules})
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2020-05-10 18:58:17 +02:00
|
|
|
|
|
|
|
if len(g.Rules) != len(tc.newRules) {
|
|
|
|
t.Fatalf("expected to have %d rules; got: %d",
|
|
|
|
len(g.Rules), len(tc.newRules))
|
|
|
|
}
|
2020-05-15 08:55:22 +02:00
|
|
|
sort.Slice(g.Rules, func(i, j int) bool {
|
2020-06-01 12:46:37 +02:00
|
|
|
return g.Rules[i].ID() < g.Rules[j].ID()
|
2020-05-15 08:55:22 +02:00
|
|
|
})
|
2020-05-10 18:58:17 +02:00
|
|
|
for i, r := range g.Rules {
|
|
|
|
got, want := r, tc.newRules[i]
|
2020-06-01 12:46:37 +02:00
|
|
|
if got.ID() != want.ID() {
|
|
|
|
t.Fatalf("expected to have rule %q; got %q", want, got)
|
2020-05-10 18:58:17 +02:00
|
|
|
}
|
2020-06-01 12:46:37 +02:00
|
|
|
if err := compareRules(t, got, want); err != nil {
|
|
|
|
t.Fatalf("comparsion error: %s", err)
|
2020-05-10 18:58:17 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestGroupStart(t *testing.T) {
|
|
|
|
// TODO: make parsing from string instead of file
|
2020-06-06 22:27:09 +02:00
|
|
|
groups, err := config.Parse([]string{"config/testdata/rules1-good.rules"}, true, true)
|
2020-05-10 18:58:17 +02:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("failed to parse rules: %s", err)
|
|
|
|
}
|
2020-06-01 12:46:37 +02:00
|
|
|
const evalInterval = time.Millisecond
|
|
|
|
g := newGroup(groups[0], evalInterval)
|
2020-06-09 14:21:20 +02:00
|
|
|
g.Concurrency = 2
|
2020-05-10 18:58:17 +02:00
|
|
|
|
|
|
|
fn := &fakeNotifier{}
|
|
|
|
fs := &fakeQuerier{}
|
|
|
|
|
|
|
|
const inst1, inst2, job = "foo", "bar", "baz"
|
|
|
|
m1 := metricWithLabels(t, "instance", inst1, "job", job)
|
|
|
|
m2 := metricWithLabels(t, "instance", inst2, "job", job)
|
|
|
|
|
2020-06-01 12:46:37 +02:00
|
|
|
r := g.Rules[0].(*AlertingRule)
|
|
|
|
alert1, err := r.newAlert(m1, time.Now())
|
2020-05-10 18:58:17 +02:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("faield to create alert: %s", err)
|
|
|
|
}
|
|
|
|
alert1.State = notifier.StateFiring
|
|
|
|
alert1.ID = hash(m1)
|
|
|
|
|
2020-06-01 12:46:37 +02:00
|
|
|
alert2, err := r.newAlert(m2, time.Now())
|
2020-05-10 18:58:17 +02:00
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("faield to create alert: %s", err)
|
|
|
|
}
|
|
|
|
alert2.State = notifier.StateFiring
|
|
|
|
alert2.ID = hash(m2)
|
|
|
|
|
|
|
|
finished := make(chan struct{})
|
|
|
|
fs.add(m1)
|
|
|
|
fs.add(m2)
|
|
|
|
go func() {
|
2020-06-01 12:46:37 +02:00
|
|
|
g.start(context.Background(), fs, fn, nil)
|
2020-05-10 18:58:17 +02:00
|
|
|
close(finished)
|
|
|
|
}()
|
|
|
|
|
|
|
|
// wait for multiple evals
|
|
|
|
time.Sleep(20 * evalInterval)
|
|
|
|
|
|
|
|
gotAlerts := fn.getAlerts()
|
|
|
|
expectedAlerts := []notifier.Alert{*alert1, *alert2}
|
|
|
|
compareAlerts(t, expectedAlerts, gotAlerts)
|
|
|
|
|
|
|
|
// reset previous data
|
|
|
|
fs.reset()
|
|
|
|
// and set only one datapoint for response
|
|
|
|
fs.add(m1)
|
|
|
|
|
|
|
|
// wait for multiple evals
|
|
|
|
time.Sleep(20 * evalInterval)
|
|
|
|
|
|
|
|
gotAlerts = fn.getAlerts()
|
|
|
|
expectedAlerts = []notifier.Alert{*alert1}
|
|
|
|
compareAlerts(t, expectedAlerts, gotAlerts)
|
|
|
|
|
|
|
|
g.close()
|
|
|
|
<-finished
|
|
|
|
}
|