2020-05-10 18:58:17 +02:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"math/rand"
|
2020-06-01 12:46:37 +02:00
|
|
|
"net/url"
|
|
|
|
"os"
|
2020-05-10 18:58:17 +02:00
|
|
|
"strings"
|
|
|
|
"sync"
|
|
|
|
"testing"
|
|
|
|
"time"
|
2020-06-01 12:46:37 +02:00
|
|
|
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
|
2020-05-10 18:58:17 +02:00
|
|
|
)
|
|
|
|
|
2020-06-01 12:46:37 +02:00
|
|
|
func TestMain(m *testing.M) {
|
|
|
|
u, _ := url.Parse("https://victoriametrics.com/path")
|
|
|
|
notifier.InitTemplateFunc(u)
|
|
|
|
os.Exit(m.Run())
|
|
|
|
}
|
|
|
|
|
2020-05-10 18:58:17 +02:00
|
|
|
func TestManagerUpdateError(t *testing.T) {
|
|
|
|
m := &manager{groups: make(map[uint64]*Group)}
|
|
|
|
path := []string{"foo/bar"}
|
2020-06-06 22:27:09 +02:00
|
|
|
err := m.update(context.Background(), path, true, true, false)
|
2020-05-10 18:58:17 +02:00
|
|
|
if err == nil {
|
|
|
|
t.Fatalf("expected to have err; got nil instead")
|
|
|
|
}
|
|
|
|
expErr := "no groups found"
|
|
|
|
if !strings.Contains(err.Error(), expErr) {
|
|
|
|
t.Fatalf("expected to got err %s; got %s", expErr, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// TestManagerUpdateConcurrent supposed to test concurrent
|
|
|
|
// execution of configuration update.
|
|
|
|
// Should be executed with -race flag
|
|
|
|
func TestManagerUpdateConcurrent(t *testing.T) {
|
2020-05-17 16:12:09 +02:00
|
|
|
m := &manager{
|
|
|
|
groups: make(map[uint64]*Group),
|
|
|
|
storage: &fakeQuerier{},
|
|
|
|
notifier: &fakeNotifier{},
|
|
|
|
}
|
2020-05-10 18:58:17 +02:00
|
|
|
paths := []string{
|
2020-06-01 12:46:37 +02:00
|
|
|
"config/testdata/dir/rules0-good.rules",
|
|
|
|
"config/testdata/dir/rules0-bad.rules",
|
|
|
|
"config/testdata/dir/rules1-good.rules",
|
|
|
|
"config/testdata/dir/rules1-bad.rules",
|
|
|
|
"config/testdata/rules0-good.rules",
|
|
|
|
"config/testdata/rules1-good.rules",
|
|
|
|
"config/testdata/rules2-good.rules",
|
2020-05-10 18:58:17 +02:00
|
|
|
}
|
2020-05-17 16:12:09 +02:00
|
|
|
*evaluationInterval = time.Millisecond
|
2020-06-06 22:27:09 +02:00
|
|
|
if err := m.start(context.Background(), []string{paths[0]}, true, true); err != nil {
|
2020-05-17 16:12:09 +02:00
|
|
|
t.Fatalf("failed to start: %s", err)
|
|
|
|
}
|
2020-05-10 18:58:17 +02:00
|
|
|
|
2020-05-17 16:12:09 +02:00
|
|
|
const workers = 500
|
|
|
|
const iterations = 10
|
2020-05-10 18:58:17 +02:00
|
|
|
wg := sync.WaitGroup{}
|
2020-05-17 16:12:09 +02:00
|
|
|
wg.Add(workers)
|
|
|
|
for i := 0; i < workers; i++ {
|
2020-05-10 18:58:17 +02:00
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
2020-05-17 16:12:09 +02:00
|
|
|
for i := 0; i < iterations; i++ {
|
|
|
|
rnd := rand.Intn(len(paths))
|
|
|
|
path := []string{paths[rnd]}
|
2020-06-06 22:27:09 +02:00
|
|
|
_ = m.update(context.Background(), path, true, true, false)
|
2020-05-10 18:58:17 +02:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
}
|
|
|
|
|
|
|
|
// TestManagerUpdate tests sequential configuration
|
|
|
|
// updates.
|
|
|
|
func TestManagerUpdate(t *testing.T) {
|
2020-06-01 12:46:37 +02:00
|
|
|
const defaultEvalInterval = time.Second * 30
|
|
|
|
currentEvalInterval := *evaluationInterval
|
|
|
|
*evaluationInterval = defaultEvalInterval
|
|
|
|
defer func() {
|
|
|
|
*evaluationInterval = currentEvalInterval
|
|
|
|
}()
|
|
|
|
|
|
|
|
var (
|
|
|
|
VMRows = &AlertingRule{
|
|
|
|
Name: "VMRows",
|
|
|
|
Expr: "vm_rows > 0",
|
|
|
|
For: 10 * time.Second,
|
|
|
|
Labels: map[string]string{
|
|
|
|
"label": "bar",
|
|
|
|
"host": "{{ $labels.instance }}",
|
|
|
|
},
|
|
|
|
Annotations: map[string]string{
|
|
|
|
"summary": "{{ $value|humanize }}",
|
|
|
|
"description": "{{$labels}}",
|
|
|
|
},
|
|
|
|
}
|
|
|
|
Conns = &AlertingRule{
|
|
|
|
Name: "Conns",
|
|
|
|
Expr: "sum(vm_tcplistener_conns) by(instance) > 1",
|
|
|
|
Annotations: map[string]string{
|
|
|
|
"summary": "Too high connection number for {{$labels.instance}}",
|
|
|
|
"description": "It is {{ $value }} connections for {{$labels.instance}}",
|
|
|
|
},
|
|
|
|
}
|
|
|
|
ExampleAlertAlwaysFiring = &AlertingRule{
|
|
|
|
Name: "ExampleAlertAlwaysFiring",
|
|
|
|
Expr: "sum by(job) (up == 1)",
|
|
|
|
}
|
|
|
|
)
|
|
|
|
|
2020-05-10 18:58:17 +02:00
|
|
|
testCases := []struct {
|
|
|
|
name string
|
|
|
|
initPath string
|
|
|
|
updatePath string
|
|
|
|
want []*Group
|
|
|
|
}{
|
|
|
|
{
|
|
|
|
name: "update good rules",
|
2020-06-01 12:46:37 +02:00
|
|
|
initPath: "config/testdata/rules0-good.rules",
|
|
|
|
updatePath: "config/testdata/dir/rules1-good.rules",
|
2020-05-10 18:58:17 +02:00
|
|
|
want: []*Group{
|
|
|
|
{
|
2020-06-01 12:46:37 +02:00
|
|
|
File: "config/testdata/dir/rules1-good.rules",
|
|
|
|
Name: "duplicatedGroupDiffFiles",
|
|
|
|
Interval: defaultEvalInterval,
|
|
|
|
Rules: []Rule{
|
|
|
|
&AlertingRule{
|
|
|
|
Name: "VMRows",
|
|
|
|
Expr: "vm_rows > 0",
|
|
|
|
For: 5 * time.Minute,
|
|
|
|
Labels: map[string]string{"label": "bar"},
|
|
|
|
Annotations: map[string]string{
|
|
|
|
"summary": "{{ $value }}",
|
|
|
|
"description": "{{$labels}}",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
2020-05-10 18:58:17 +02:00
|
|
|
},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "update good rules from 1 to 2 groups",
|
2020-06-01 12:46:37 +02:00
|
|
|
initPath: "config/testdata/dir/rules1-good.rules",
|
|
|
|
updatePath: "config/testdata/rules0-good.rules",
|
2020-05-10 18:58:17 +02:00
|
|
|
want: []*Group{
|
|
|
|
{
|
2020-06-01 12:46:37 +02:00
|
|
|
File: "config/testdata/rules0-good.rules",
|
|
|
|
Name: "groupGorSingleAlert",
|
|
|
|
Rules: []Rule{VMRows},
|
|
|
|
Interval: defaultEvalInterval,
|
|
|
|
},
|
2020-05-10 18:58:17 +02:00
|
|
|
{
|
2020-06-01 12:46:37 +02:00
|
|
|
File: "config/testdata/rules0-good.rules",
|
|
|
|
Interval: defaultEvalInterval,
|
|
|
|
Name: "TestGroup", Rules: []Rule{
|
|
|
|
Conns,
|
|
|
|
ExampleAlertAlwaysFiring,
|
2020-05-10 18:58:17 +02:00
|
|
|
}},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
name: "update with one bad rule file",
|
2020-06-01 12:46:37 +02:00
|
|
|
initPath: "config/testdata/rules0-good.rules",
|
|
|
|
updatePath: "config/testdata/dir/rules2-bad.rules",
|
2020-05-10 18:58:17 +02:00
|
|
|
want: []*Group{
|
|
|
|
{
|
2020-06-01 12:46:37 +02:00
|
|
|
File: "config/testdata/rules0-good.rules",
|
|
|
|
Name: "groupGorSingleAlert",
|
|
|
|
Interval: defaultEvalInterval,
|
|
|
|
Rules: []Rule{VMRows},
|
|
|
|
},
|
2020-05-10 18:58:17 +02:00
|
|
|
{
|
2020-06-01 12:46:37 +02:00
|
|
|
File: "config/testdata/rules0-good.rules",
|
|
|
|
Interval: defaultEvalInterval,
|
|
|
|
Name: "TestGroup", Rules: []Rule{
|
|
|
|
Conns,
|
|
|
|
ExampleAlertAlwaysFiring,
|
2020-05-10 18:58:17 +02:00
|
|
|
}},
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
|
|
ctx, cancel := context.WithCancel(context.TODO())
|
2020-05-17 16:12:09 +02:00
|
|
|
m := &manager{groups: make(map[uint64]*Group), storage: &fakeQuerier{}}
|
2020-05-10 18:58:17 +02:00
|
|
|
path := []string{tc.initPath}
|
2020-06-06 22:27:09 +02:00
|
|
|
if err := m.update(ctx, path, true, true, false); err != nil {
|
2020-05-10 18:58:17 +02:00
|
|
|
t.Fatalf("failed to complete initial rules update: %s", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
path = []string{tc.updatePath}
|
2020-06-06 22:27:09 +02:00
|
|
|
_ = m.update(ctx, path, true, true, false)
|
2020-05-10 18:58:17 +02:00
|
|
|
if len(tc.want) != len(m.groups) {
|
|
|
|
t.Fatalf("\nwant number of groups: %d;\ngot: %d ", len(tc.want), len(m.groups))
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, wantG := range tc.want {
|
|
|
|
gotG, ok := m.groups[wantG.ID()]
|
|
|
|
if !ok {
|
|
|
|
t.Fatalf("expected to have group %q", wantG.Name)
|
|
|
|
}
|
2020-06-01 12:46:37 +02:00
|
|
|
compareGroups(t, wantG, gotG)
|
2020-05-10 18:58:17 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
cancel()
|
|
|
|
m.close()
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|