[vmalert] config parser (#393)

* [vmalert] config parser

* make linter be happy

* fix test

* fix sprintf add test for rule validation

app/vmalert/common/alert.go

@@ -1,21 +1,17 @@
-package provider
+package common
 
 import (
 	"bytes"
+	"fmt"
+	"io"
 	"strings"
 	"text/template"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
-// AlertProvider is common interface for alert manager provider
-type AlertProvider interface {
-	Send(alerts []Alert) error
-}
-
 // Alert the triggered alert
 type Alert struct {
 	Group       string
@@ -37,8 +33,9 @@ type alertTplData struct {
 const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $externalLabels := .ExternalLabels }}`
 
 // AlertsFromMetrics converts metrics to alerts by alert Rule
-func AlertsFromMetrics(metrics []datasource.Metric, group string, rule config.Rule, start, end time.Time) []Alert {
+func AlertsFromMetrics(metrics []datasource.Metric, group string, rule Rule, start, end time.Time) []Alert {
 	alerts := make([]Alert, 0, len(metrics))
+	var err error
 	for i, m := range metrics {
 		a := Alert{
 			Group: group,
@@ -49,7 +46,10 @@ func AlertsFromMetrics(metrics []datasource.Metric, group string, rule config.Ru
 		}
 		tplData := alertTplData{Value: m.Value, ExternalLabels: make(map[string]string)}
 		tplData.Labels, a.Labels = mergeLabels(metrics[i].Labels, rule.Labels)
-		a.Annotations = templateAnnotations(rule.Annotations, tplHeader, tplData)
+		a.Annotations, err = templateAnnotations(rule.Annotations, tplHeader, tplData)
+		if err != nil {
+			logger.Errorf("%s", err)
+		}
 		alerts = append(alerts, a)
 	}
 	return alerts
@@ -74,9 +74,10 @@ func mergeLabels(ml []datasource.Label, rl map[string]string) (map[string]string
 	return set, sl
 }
 
-func templateAnnotations(annotations map[string]string, header string, data alertTplData) map[string]string {
+func templateAnnotations(annotations map[string]string, header string, data alertTplData) (map[string]string, error) {
 	var builder strings.Builder
 	var buf bytes.Buffer
+	eg := errGroup{}
 	r := make(map[string]string, len(annotations))
 	for key, text := range annotations {
 		r[key] = text
@@ -85,17 +86,48 @@ func templateAnnotations(annotations map[string]string, header string, data aler
 		builder.Grow(len(header) + len(text))
 		builder.WriteString(header)
 		builder.WriteString(text)
-		// todo add template helper func from Prometheus
-		tpl, err := template.New("").Option("missingkey=zero").Parse(builder.String())
-		if err != nil {
-			logger.Errorf("error parsing annotation template %q for %q:%s", text, key, err)
-			continue
-		}
-		if err = tpl.Execute(&buf, data); err != nil {
-			logger.Errorf("error evaluating annotation template %s for %s:%s", text, key, err)
+		if err := templateAnnotation(&buf, builder.String(), data); err != nil {
+			eg.errs = append(eg.errs, fmt.Sprintf("key %s, template %s:%s", key, text, err))
 			continue
 		}
 		r[key] = buf.String()
 	}
-	return r
+	return r, eg.err()
+}
+
+// ValidateAnnotations validate annotations for possible template error, uses empty data for template population
+func ValidateAnnotations(annotations map[string]string) error {
+	_, err := templateAnnotations(annotations, tplHeader, alertTplData{
+		Labels:         map[string]string{},
+		ExternalLabels: map[string]string{},
+		Value:          0,
+	})
+	return err
+}
+
+func templateAnnotation(dst io.Writer, text string, data alertTplData) error {
+	// todo add template helper func from Prometheus
+	tpl, err := template.New("").Option("missingkey=zero").Parse(text)
+	if err != nil {
+		return fmt.Errorf("error parsing annotation:%w", err)
+	}
+	if err = tpl.Execute(dst, data); err != nil {
+		return fmt.Errorf("error evaluating annotation template:%w", err)
+	}
+	return nil
+}
+
+type errGroup struct {
+	errs []string
+}
+
+func (eg *errGroup) err() error {
+	if eg == nil || len(eg.errs) == 0 {
+		return nil
+	}
+	return eg
+}
+
+func (eg *errGroup) Error() string {
+	return fmt.Sprintf("errors:%s", strings.Join(eg.errs, "\n"))
 }

app/vmalert/common/alert_test.go

@@ -1,4 +1,4 @@
-package provider
+package common
 
 import (
 	"reflect"
@@ -6,7 +6,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 )
 
@@ -30,7 +29,7 @@ func TestAlertsFromMetrics(t *testing.T) {
 			Value:     30,
 		},
 	}
-	rule := config.Rule{
+	rule := Rule{
 		Name: "alertname",
 		Expr: "up==0",
 		Labels: map[string]string{

app/vmalert/common/rule.go

@@ -0,0 +1,38 @@
+package common
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/metricsql"
+)
+
+// Rule is basic alert entity
+type Rule struct {
+	Name        string            `yaml:"alert"`
+	Expr        string            `yaml:"expr"`
+	For         time.Duration     `yaml:"for"`
+	Labels      map[string]string `yaml:"labels"`
+	Annotations map[string]string `yaml:"annotations"`
+}
+
+// Validate validates rule
+func (r Rule) Validate() error {
+	if r.Name == "" {
+		return errors.New("rule name can not be empty")
+	}
+	if r.Expr == "" {
+		return fmt.Errorf("rule %s expression can not be empty", r.Name)
+	}
+	if _, err := metricsql.Parse(r.Expr); err != nil {
+		return fmt.Errorf("rule %s invalid expression: %w", r.Name, err)
+	}
+	return nil
+}
+
+// Group grouping array of alert
+type Group struct {
+	Name  string
+	Rules []Rule
+}

app/vmalert/common/rule_test.go

@@ -0,0 +1,18 @@
+package common
+
+import "testing"
+
+func TestRule_Validate(t *testing.T) {
+	if err := (Rule{}).Validate(); err == nil {
+		t.Errorf("exptected empty name error")
+	}
+	if err := (Rule{Name: "alert"}).Validate(); err == nil {
+		t.Errorf("exptected empty expr error")
+	}
+	if err := (Rule{Name: "alert", Expr: "test{"}).Validate(); err == nil {
+		t.Errorf("exptected invalid expr error")
+	}
+	if err := (Rule{Name: "alert", Expr: "test>0"}).Validate(); err != nil {
+		t.Errorf("exptected valid rule got %s", err)
+	}
+}

app/vmalert/config/parser.go

@@ -1,38 +1,64 @@
 package config
 
-import "time"
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
 
-// Rule is basic alert entity
-type Rule struct {
-	Name        string
-	Expr        string
-	For         time.Duration
-	Labels      map[string]string
-	Annotations map[string]string
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/common"
+	"gopkg.in/yaml.v2"
+)
+
+// Parse parses rule configs from given file patterns
+func Parse(pathPatterns []string, validateAnnotations bool) ([]common.Group, error) {
+	var fp []string
+	for _, pattern := range pathPatterns {
+		matches, err := filepath.Glob(pattern)
+		if err != nil {
+			return nil, fmt.Errorf("error reading file patther %s:%v", pattern, err)
+		}
+		fp = append(fp, matches...)
+	}
+	var groups []common.Group
+	for _, file := range fp {
+		groupsNames := map[string]struct{}{}
+		gr, err := parseFile(file)
+		if err != nil {
+			return nil, fmt.Errorf("file %s: %w", file, err)
+		}
+		for _, group := range gr {
+			if _, ok := groupsNames[group.Name]; ok {
+				return nil, fmt.Errorf("one file can not contain groups with the same name %s, filepath:%s", file, group.Name)
+			}
+			groupsNames[group.Name] = struct{}{}
+			for _, rule := range group.Rules {
+				if err = rule.Validate(); err != nil {
+					return nil, fmt.Errorf("invalid rule filepath:%s, group %s:%w", file, group.Name, err)
+				}
+				if validateAnnotations {
+					if err = common.ValidateAnnotations(rule.Annotations); err != nil {
+						return nil, fmt.Errorf("invalida annotations filepath:%s, group %s:%w", file, group.Name, err)
+					}
+				}
+			}
+		}
+		groups = append(groups, gr...)
+	}
+	if len(groups) < 1 {
+		return nil, fmt.Errorf("no groups found in %s", strings.Join(pathPatterns, ";"))
+	}
+	return groups, nil
 }
 
-// Group grouping array of alert
-type Group struct {
-	Name  string
-	Rules []Rule
-}
-
-// Parse parses config from given file
-func Parse(filepath string) ([]Group, error) {
-	return []Group{{
-		Name: "foobar",
-		Rules: []Rule{{
-			Name: "vmrowsalert",
-			Expr: "vm_rows",
-			For:  1 * time.Second,
-			Labels: map[string]string{
-				"alert_label":  "value1",
-				"alert_label2": "value2",
-			},
-			Annotations: map[string]string{
-				"summary":     "{{ $value }}",
-				"description": "LABELS: {{ $labels }}",
-			},
-		}},
-	}}, nil
+func parseFile(path string) ([]common.Group, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("error reading alert rule file: %w", err)
+	}
+	g := struct {
+		Groups []common.Group `yaml:"groups"`
+	}{}
+	err = yaml.Unmarshal(data, &g)
+	return g.Groups, err
 }

app/vmalert/config/parser_test.go

@@ -0,0 +1,26 @@
+package config
+
+import (
+	"testing"
+)
+
+func TestParseGood(t *testing.T) {
+	if _, err := Parse([]string{"testdata/*good.rules", "testdata/dir/*good.*"}, true); err != nil {
+		t.Errorf("error parsing files %s", err)
+	}
+}
+
+func TestParseBad(t *testing.T) {
+	if _, err := Parse([]string{"testdata/rules0-bad.rules"}, true); err == nil {
+		t.Errorf("expected syntaxt error")
+	}
+	if _, err := Parse([]string{"testdata/dir/rules0-bad.rules"}, true); err == nil {
+		t.Errorf("expected template annotation error")
+	}
+	if _, err := Parse([]string{"testdata/dir/rules1-bad.rules"}, true); err == nil {
+		t.Errorf("expected same group error")
+	}
+	if _, err := Parse([]string{"testdata/*.yaml"}, true); err == nil {
+		t.Errorf("expected empty group")
+	}
+}

app/vmalert/config/testdata/dir/rules0-bad.rules

@@ -0,0 +1,19 @@
+groups:
+  - name: group
+    rules:
+      - alert: InvalidAnnotations
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }"
+          description: "{{$labels}}"
+      - alert: UnkownAnnotationsFunction
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ value|humanize }}"
+          description: "{{$labels}}"

app/vmalert/config/testdata/dir/rules0-good.rules

@@ -0,0 +1,13 @@
+groups:
+  - name: duplicatedGroupDiffFiles
+    rules:
+      - alert: VMRows
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+
+

app/vmalert/config/testdata/dir/rules1-bad.rules

@@ -0,0 +1,22 @@
+groups:
+  - name: sameGroup
+    rules:
+      - alert: alert
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+  - name: sameGroup
+    rules:
+      - alert: alert
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+

app/vmalert/config/testdata/dir/rules1-good.rules

@@ -0,0 +1,13 @@
+groups:
+  - name: duplicatedGroupDiffFiles
+    rules:
+      - alert: VMRows
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+
+

app/vmalert/config/testdata/rules0-bad.rules

@@ -0,0 +1,28 @@
+groups:
+  - name: group
+    rules:
+      - alert: InvalidExpr
+        for: 5m
+        expr: vm_rows{ > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+      - alert: EmptyExpr
+        for: 5m
+        expr: ""
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+      - alert: ""
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: foo
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+

app/vmalert/config/testdata/rules0-good.rules

@@ -0,0 +1,12 @@
+groups:
+  - name: groupGorSingleAlert
+    rules:
+      - alert: VMRows
+        for: 5m
+        expr: vm_rows > 0
+        labels:
+          label: bar
+        annotations:
+          summary: "{{ $value }}"
+          description: "{{$labels}}"
+

app/vmalert/main.go

@@ -9,38 +9,46 @@ import (
 	"strings"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/provider"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 )
 
 var (
-	configPath     = flag.String("config", "config.yaml", "Path to alert configuration file")
-	httpListenAddr = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
-
-	datasourceURL      = flag.String("datasource.url", "", "Victoria Metrics or VMSelect url. Required parameter. e.g. http://127.0.0.1:8428")
-	basicAuthUsername  = flag.String("datasource.basicAuth.username", "", "Optional basic auth username to use for -datasource.url")
-	basicAuthPassword  = flag.String("datasource.basicAuth.password", "", "Optional basic auth password to use for -datasource.url")
-	evaluationInterval = flag.Duration("evaluationInterval", 1*time.Minute, "How often to evaluate the rules. Default 1m")
-	providerURL        = flag.String("provider.url", "", "Prometheus alertmanager url. Required parameter. e.g. http://127.0.0.1:9093")
+	rulePath = flagutil.NewArray("rule", `Path to file with alert rules, accepts patterns. 
+Flag can be specified multiple time. 
+Examples:
+ -rule /path/to/file. Path to single file with alerting rules
+ -rule dir/*.yaml -rule /*.yaml. Paths to all yaml files in relative dir folder and absolute yaml file in a root.`)
+	validateAlertAnnotations = flag.Bool("rule.validateAnnotations", true, "Indicates to validate annotation templates")
+	httpListenAddr           = flag.String("httpListenAddr", ":8880", "Address to listen for http connections")
+	datasourceURL            = flag.String("datasource.url", "", "Victoria Metrics or VMSelect url. Required parameter. e.g. http://127.0.0.1:8428")
+	basicAuthUsername        = flag.String("datasource.basicAuth.username", "", "Optional basic auth username to use for -datasource.url")
+	basicAuthPassword        = flag.String("datasource.basicAuth.password", "", "Optional basic auth password to use for -datasource.url")
+	evaluationInterval       = flag.Duration("evaluationInterval", 1*time.Minute, "How often to evaluate the rules. Default 1m")
+	providerURL              = flag.String("provider.url", "", "Prometheus alertmanager url. Required parameter. e.g. http://127.0.0.1:9093")
 )
 
 func main() {
 	envflag.Parse()
 	buildinfo.Init()
 	logger.Init()
+	checkFlags()
 	ctx, cancel := context.WithCancel(context.Background())
 
-	logger.Infof("reading alert rules configuration file from %s", *configPath)
-	alertGroups, err := config.Parse(*configPath)
+	logger.Infof("reading alert rules configuration file from %s", strings.Join(*rulePath, ";"))
+	alertGroups, err := config.Parse(*rulePath, *validateAlertAnnotations)
 	if err != nil {
-		logger.Fatalf("Cannot parse configuration file %s", err)
+		logger.Fatalf("Cannot parse configuration file: %s", err)
 	}
+
 	addr := getWebServerAddr(*httpListenAddr, false)
 	w := &watchdog{
 		storage: datasource.NewVMStorage(*datasourceURL, *basicAuthUsername, *basicAuthPassword, &http.Client{}),
@@ -49,7 +57,7 @@ func main() {
 		}, &http.Client{}),
 	}
 	for id := range alertGroups {
-		go func(group config.Group) {
+		go func(group common.Group) {
 			w.run(ctx, group, *evaluationInterval)
 		}(alertGroups[id])
 	}
@@ -72,11 +80,12 @@ type watchdog struct {
 	alertProvider provider.AlertProvider
 }
 
-func (w *watchdog) run(ctx context.Context, a config.Group, evaluationInterval time.Duration) {
+func (w *watchdog) run(ctx context.Context, a common.Group, evaluationInterval time.Duration) {
+	logger.Infof("watchdog for %s has been run", a.Name)
 	t := time.NewTicker(evaluationInterval)
 	var metrics []datasource.Metric
 	var err error
-	var alerts []provider.Alert
+	var alerts []common.Alert
 	defer t.Stop()
 	for {
 		select {
@@ -92,7 +101,7 @@ func (w *watchdog) run(ctx context.Context, a config.Group, evaluationInterval t
 					continue
 				}
 				// todo define alert end time
-				alerts = provider.AlertsFromMetrics(metrics, a.Name, r, start, time.Time{})
+				alerts = common.AlertsFromMetrics(metrics, a.Name, r, start, time.Time{})
 				// todo save to storage
 				if err := w.alertProvider.Send(alerts); err != nil {
 					logger.Errorf("error sending alerts %s", err)
@@ -134,3 +143,14 @@ func getWebServerAddr(httpListenAddr string, isSecure bool) string {
 func (w *watchdog) stop() {
 	panic("not implemented")
 }
+
+func checkFlags() {
+	if *providerURL == "" {
+		flag.PrintDefaults()
+		logger.Fatalf("provider.url is empty")
+	}
+	if *datasourceURL == "" {
+		flag.PrintDefaults()
+		logger.Fatalf("datasource.url is empty")
+	}
+}

app/vmalert/provider/alert_manager_request.qtpl

@@ -1,9 +1,10 @@
 {% import (
     "time"
+    "github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/common"
 ) %}
 {% stripspace %}
 
-{% func amRequest(alerts []Alert, generatorURL func(string, string) string) %}
+{% func amRequest(alerts []common.Alert, generatorURL func(string, string) string) %}
 [
 {% for i, alert := range alerts %}
 {

app/vmalert/provider/alert_manager_request.qtpl.go

@@ -6,125 +6,126 @@ package provider
 
 //line app/vmalert/provider/alert_manager_request.qtpl:1
 import (
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/common"
 	"time"
 )
 
-//line app/vmalert/provider/alert_manager_request.qtpl:6
+//line app/vmalert/provider/alert_manager_request.qtpl:7
 import (
 	qtio422016 "io"
 
 	qt422016 "github.com/valyala/quicktemplate"
 )
 
-//line app/vmalert/provider/alert_manager_request.qtpl:6
+//line app/vmalert/provider/alert_manager_request.qtpl:7
 var (
 	_ = qtio422016.Copy
 	_ = qt422016.AcquireByteBuffer
 )
 
-//line app/vmalert/provider/alert_manager_request.qtpl:6
-func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
-//line app/vmalert/provider/alert_manager_request.qtpl:6
+//line app/vmalert/provider/alert_manager_request.qtpl:7
+func streamamRequest(qw422016 *qt422016.Writer, alerts []common.Alert, generatorURL func(string, string) string) {
+//line app/vmalert/provider/alert_manager_request.qtpl:7
 	qw422016.N().S(`[`)
-//line app/vmalert/provider/alert_manager_request.qtpl:8
+//line app/vmalert/provider/alert_manager_request.qtpl:9
 	for i, alert := range alerts {
-//line app/vmalert/provider/alert_manager_request.qtpl:8
+//line app/vmalert/provider/alert_manager_request.qtpl:9
 		qw422016.N().S(`{"startsAt":`)
-//line app/vmalert/provider/alert_manager_request.qtpl:10
+//line app/vmalert/provider/alert_manager_request.qtpl:11
 		qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
-//line app/vmalert/provider/alert_manager_request.qtpl:10
+//line app/vmalert/provider/alert_manager_request.qtpl:11
 		qw422016.N().S(`,"generatorURL":`)
-//line app/vmalert/provider/alert_manager_request.qtpl:11
+//line app/vmalert/provider/alert_manager_request.qtpl:12
 		qw422016.N().Q(generatorURL(alert.Group, alert.Name))
-//line app/vmalert/provider/alert_manager_request.qtpl:11
+//line app/vmalert/provider/alert_manager_request.qtpl:12
 		qw422016.N().S(`,`)
-//line app/vmalert/provider/alert_manager_request.qtpl:12
+//line app/vmalert/provider/alert_manager_request.qtpl:13
 		if !alert.End.IsZero() {
-//line app/vmalert/provider/alert_manager_request.qtpl:12
+//line app/vmalert/provider/alert_manager_request.qtpl:13
 			qw422016.N().S(`"endsAt":`)
-//line app/vmalert/provider/alert_manager_request.qtpl:13
+//line app/vmalert/provider/alert_manager_request.qtpl:14
 			qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
-//line app/vmalert/provider/alert_manager_request.qtpl:13
+//line app/vmalert/provider/alert_manager_request.qtpl:14
 			qw422016.N().S(`,`)
-//line app/vmalert/provider/alert_manager_request.qtpl:14
+//line app/vmalert/provider/alert_manager_request.qtpl:15
 		}
-//line app/vmalert/provider/alert_manager_request.qtpl:14
+//line app/vmalert/provider/alert_manager_request.qtpl:15
 		qw422016.N().S(`"labels": {"alertname":`)
-//line app/vmalert/provider/alert_manager_request.qtpl:16
+//line app/vmalert/provider/alert_manager_request.qtpl:17
 		qw422016.N().Q(alert.Name)
-//line app/vmalert/provider/alert_manager_request.qtpl:17
+//line app/vmalert/provider/alert_manager_request.qtpl:18
 		for _, v := range alert.Labels {
-//line app/vmalert/provider/alert_manager_request.qtpl:17
+//line app/vmalert/provider/alert_manager_request.qtpl:18
 			qw422016.N().S(`,`)
-//line app/vmalert/provider/alert_manager_request.qtpl:18
+//line app/vmalert/provider/alert_manager_request.qtpl:19
 			qw422016.N().Q(v.Name)
-//line app/vmalert/provider/alert_manager_request.qtpl:18
+//line app/vmalert/provider/alert_manager_request.qtpl:19
 			qw422016.N().S(`:`)
-//line app/vmalert/provider/alert_manager_request.qtpl:18
+//line app/vmalert/provider/alert_manager_request.qtpl:19
 			qw422016.N().Q(v.Value)
-//line app/vmalert/provider/alert_manager_request.qtpl:19
+//line app/vmalert/provider/alert_manager_request.qtpl:20
 		}
-//line app/vmalert/provider/alert_manager_request.qtpl:19
+//line app/vmalert/provider/alert_manager_request.qtpl:20
 		qw422016.N().S(`},"annotations": {`)
-//line app/vmalert/provider/alert_manager_request.qtpl:22
+//line app/vmalert/provider/alert_manager_request.qtpl:23
 		c := len(alert.Annotations)
 
-//line app/vmalert/provider/alert_manager_request.qtpl:23
-		for k, v := range alert.Annotations {
 //line app/vmalert/provider/alert_manager_request.qtpl:24
+		for k, v := range alert.Annotations {
+//line app/vmalert/provider/alert_manager_request.qtpl:25
 			c = c - 1
 
-//line app/vmalert/provider/alert_manager_request.qtpl:25
+//line app/vmalert/provider/alert_manager_request.qtpl:26
 			qw422016.N().Q(k)
-//line app/vmalert/provider/alert_manager_request.qtpl:25
+//line app/vmalert/provider/alert_manager_request.qtpl:26
 			qw422016.N().S(`:`)
-//line app/vmalert/provider/alert_manager_request.qtpl:25
+//line app/vmalert/provider/alert_manager_request.qtpl:26
 			qw422016.N().Q(v)
-//line app/vmalert/provider/alert_manager_request.qtpl:25
+//line app/vmalert/provider/alert_manager_request.qtpl:26
 			if c > 0 {
-//line app/vmalert/provider/alert_manager_request.qtpl:25
+//line app/vmalert/provider/alert_manager_request.qtpl:26
 				qw422016.N().S(`,`)
-//line app/vmalert/provider/alert_manager_request.qtpl:25
+//line app/vmalert/provider/alert_manager_request.qtpl:26
 			}
-//line app/vmalert/provider/alert_manager_request.qtpl:26
+//line app/vmalert/provider/alert_manager_request.qtpl:27
 		}
-//line app/vmalert/provider/alert_manager_request.qtpl:26
+//line app/vmalert/provider/alert_manager_request.qtpl:27
 		qw422016.N().S(`}}`)
-//line app/vmalert/provider/alert_manager_request.qtpl:29
+//line app/vmalert/provider/alert_manager_request.qtpl:30
 		if i != len(alerts)-1 {
-//line app/vmalert/provider/alert_manager_request.qtpl:29
+//line app/vmalert/provider/alert_manager_request.qtpl:30
 			qw422016.N().S(`,`)
-//line app/vmalert/provider/alert_manager_request.qtpl:29
+//line app/vmalert/provider/alert_manager_request.qtpl:30
 		}
-//line app/vmalert/provider/alert_manager_request.qtpl:30
+//line app/vmalert/provider/alert_manager_request.qtpl:31
 	}
-//line app/vmalert/provider/alert_manager_request.qtpl:30
+//line app/vmalert/provider/alert_manager_request.qtpl:31
 	qw422016.N().S(`]`)
-//line app/vmalert/provider/alert_manager_request.qtpl:32
+//line app/vmalert/provider/alert_manager_request.qtpl:33
 }
 
-//line app/vmalert/provider/alert_manager_request.qtpl:32
-func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
-//line app/vmalert/provider/alert_manager_request.qtpl:32
+//line app/vmalert/provider/alert_manager_request.qtpl:33
+func writeamRequest(qq422016 qtio422016.Writer, alerts []common.Alert, generatorURL func(string, string) string) {
+//line app/vmalert/provider/alert_manager_request.qtpl:33
 	qw422016 := qt422016.AcquireWriter(qq422016)
-//line app/vmalert/provider/alert_manager_request.qtpl:32
+//line app/vmalert/provider/alert_manager_request.qtpl:33
 	streamamRequest(qw422016, alerts, generatorURL)
-//line app/vmalert/provider/alert_manager_request.qtpl:32
+//line app/vmalert/provider/alert_manager_request.qtpl:33
 	qt422016.ReleaseWriter(qw422016)
-//line app/vmalert/provider/alert_manager_request.qtpl:32
+//line app/vmalert/provider/alert_manager_request.qtpl:33
 }
 
-//line app/vmalert/provider/alert_manager_request.qtpl:32
-func amRequest(alerts []Alert, generatorURL func(string, string) string) string {
-//line app/vmalert/provider/alert_manager_request.qtpl:32
+//line app/vmalert/provider/alert_manager_request.qtpl:33
+func amRequest(alerts []common.Alert, generatorURL func(string, string) string) string {
+//line app/vmalert/provider/alert_manager_request.qtpl:33
 	qb422016 := qt422016.AcquireByteBuffer()
-//line app/vmalert/provider/alert_manager_request.qtpl:32
+//line app/vmalert/provider/alert_manager_request.qtpl:33
 	writeamRequest(qb422016, alerts, generatorURL)
-//line app/vmalert/provider/alert_manager_request.qtpl:32
+//line app/vmalert/provider/alert_manager_request.qtpl:33
 	qs422016 := string(qb422016.B)
-//line app/vmalert/provider/alert_manager_request.qtpl:32
+//line app/vmalert/provider/alert_manager_request.qtpl:33
 	qt422016.ReleaseByteBuffer(qb422016)
-//line app/vmalert/provider/alert_manager_request.qtpl:32
+//line app/vmalert/provider/alert_manager_request.qtpl:33
 	return qs422016
-//line app/vmalert/provider/alert_manager_request.qtpl:32
+//line app/vmalert/provider/alert_manager_request.qtpl:33
 }

app/vmalert/provider/alertmanager.go

@@ -8,11 +8,17 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/common"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 )
 
 const alertsPath = "/api/v2/alerts"
 
+// AlertProvider is common interface for alert manager provider
+type AlertProvider interface {
+	Send(alerts []common.Alert) error
+}
+
 var pool = sync.Pool{New: func() interface{} {
 	return &bytes.Buffer{}
 }}
@@ -37,7 +43,7 @@ func NewAlertManager(alertManagerURL string, fn AlertURLGenerator, c *http.Clien
 }
 
 // Send an alert or resolve message
-func (am *AlertManager) Send(alerts []Alert) error {
+func (am *AlertManager) Send(alerts []common.Alert) error {
 	b := pool.Get().(*bytes.Buffer)
 	b.Reset()
 	defer pool.Put(b)

app/vmalert/provider/alertmanager_test.go

@@ -6,6 +6,8 @@ import (
 	"net/http/httptest"
 	"testing"
 	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/common"
 )
 
 func TestAlertManager_Send(t *testing.T) {
@@ -59,13 +61,13 @@ func TestAlertManager_Send(t *testing.T) {
 	am := NewAlertManager(srv.URL, func(group, name string) string {
 		return group + name
 	}, srv.Client())
-	if err := am.Send([]Alert{{}, {}}); err == nil {
+	if err := am.Send([]common.Alert{{}, {}}); err == nil {
 		t.Error("expected connection error got nil")
 	}
-	if err := am.Send([]Alert{}); err == nil {
+	if err := am.Send([]common.Alert{}); err == nil {
 		t.Error("expected wrong http code error got nil")
 	}
-	if err := am.Send([]Alert{{
+	if err := am.Send([]common.Alert{{
 		Group:       "group0",
 		Name:        "alert0",
 		Start:       time.Now().UTC(),

lib/flagutil/array.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 )
 
-// NewArray returns new Array with the given name and descprition.
+// NewArray returns new Array with the given name and description.
 func NewArray(name, description string) *Array {
 	var a Array
 	flag.Var(&a, name, description)