2020-04-06 13:44:03 +02:00
|
|
|
package notifier
|
2020-02-16 19:59:02 +01:00
|
|
|
|
2020-03-13 11:19:31 +01:00
|
|
|
import (
|
2020-03-27 17:31:16 +01:00
|
|
|
"bytes"
|
2020-03-29 00:48:30 +01:00
|
|
|
"fmt"
|
|
|
|
"io"
|
2020-03-27 17:31:16 +01:00
|
|
|
"strings"
|
|
|
|
"text/template"
|
2020-03-13 11:19:31 +01:00
|
|
|
"time"
|
2020-06-29 21:21:03 +02:00
|
|
|
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/utils"
|
2020-03-13 11:19:31 +01:00
|
|
|
)
|
2020-02-16 19:59:02 +01:00
|
|
|
|
2020-03-13 11:19:31 +01:00
|
|
|
// Alert the triggered alert
|
2020-04-06 13:44:03 +02:00
|
|
|
// TODO: Looks like alert name isn't unique
|
2020-03-13 11:19:31 +01:00
|
|
|
type Alert struct {
|
2021-09-13 14:48:18 +02:00
|
|
|
// GroupID contains the ID of the parent rules group
|
|
|
|
GroupID uint64
|
|
|
|
// Name represents Alert name
|
|
|
|
Name string
|
|
|
|
// Labels is the list of label-value pairs attached to the Alert
|
|
|
|
Labels map[string]string
|
|
|
|
// Annotations is the list of annotations generated on Alert evaluation
|
2020-03-13 11:19:31 +01:00
|
|
|
Annotations map[string]string
|
2021-09-13 14:48:18 +02:00
|
|
|
// State represents the current state of the Alert
|
|
|
|
State AlertState
|
|
|
|
// Expr contains expression that was executed to generate the Alert
|
|
|
|
Expr string
|
|
|
|
// Start defines the moment of time when Alert has triggered
|
2020-03-13 11:19:31 +01:00
|
|
|
Start time.Time
|
2021-09-13 14:48:18 +02:00
|
|
|
// End defines the moment of time when Alert supposed to expire
|
|
|
|
End time.Time
|
|
|
|
// Value stores the value returned from evaluating expression from Expr field
|
2020-03-13 11:19:31 +01:00
|
|
|
Value float64
|
2021-09-13 14:48:18 +02:00
|
|
|
// ID is the unique identifer for the Alert
|
|
|
|
ID uint64
|
2021-10-22 11:30:38 +02:00
|
|
|
// Restored is true if Alert was restored after restart
|
|
|
|
Restored bool
|
2020-03-13 11:19:31 +01:00
|
|
|
}
|
|
|
|
|
2020-04-06 13:44:03 +02:00
|
|
|
// AlertState type indicates the Alert state
|
|
|
|
type AlertState int
|
|
|
|
|
|
|
|
const (
|
|
|
|
// StateInactive is the state of an alert that is neither firing nor pending.
|
|
|
|
StateInactive AlertState = iota
|
|
|
|
// StatePending is the state of an alert that has been active for less than
|
|
|
|
// the configured threshold duration.
|
|
|
|
StatePending
|
|
|
|
// StateFiring is the state of an alert that has been active for longer than
|
|
|
|
// the configured threshold duration.
|
|
|
|
StateFiring
|
|
|
|
)
|
|
|
|
|
2020-04-11 11:40:24 +02:00
|
|
|
// String stringer for AlertState
|
|
|
|
func (as AlertState) String() string {
|
|
|
|
switch as {
|
|
|
|
case StateFiring:
|
|
|
|
return "firing"
|
|
|
|
case StatePending:
|
|
|
|
return "pending"
|
|
|
|
}
|
|
|
|
return "inactive"
|
|
|
|
}
|
|
|
|
|
2020-12-19 13:10:59 +01:00
|
|
|
// AlertTplData is used to execute templating
|
|
|
|
type AlertTplData struct {
|
2020-04-06 13:44:03 +02:00
|
|
|
Labels map[string]string
|
|
|
|
Value float64
|
2020-05-18 10:55:16 +02:00
|
|
|
Expr string
|
2020-03-27 17:31:16 +01:00
|
|
|
}
|
|
|
|
|
2020-05-18 10:55:16 +02:00
|
|
|
const tplHeader = `{{ $value := .Value }}{{ $labels := .Labels }}{{ $expr := .Expr }}`
|
2020-03-27 17:31:16 +01:00
|
|
|
|
2020-12-19 13:10:59 +01:00
|
|
|
// ExecTemplate executes the Alert template for given
|
2020-04-06 13:44:03 +02:00
|
|
|
// map of annotations.
|
2020-12-14 19:11:45 +01:00
|
|
|
// Every alert could have a different datasource, so function
|
|
|
|
// requires a queryFunction as an argument.
|
|
|
|
func (a *Alert) ExecTemplate(q QueryFn, annotations map[string]string) (map[string]string, error) {
|
2020-12-19 13:10:59 +01:00
|
|
|
tplData := AlertTplData{Value: a.Value, Labels: a.Labels, Expr: a.Expr}
|
2020-12-14 19:11:45 +01:00
|
|
|
return templateAnnotations(annotations, tplData, funcsWithQuery(q))
|
2020-03-13 11:19:31 +01:00
|
|
|
}
|
|
|
|
|
2020-12-19 13:10:59 +01:00
|
|
|
// ExecTemplate executes the given template for given annotations map.
|
|
|
|
func ExecTemplate(q QueryFn, annotations map[string]string, tpl AlertTplData) (map[string]string, error) {
|
|
|
|
return templateAnnotations(annotations, tpl, funcsWithQuery(q))
|
|
|
|
}
|
|
|
|
|
2020-04-26 13:15:04 +02:00
|
|
|
// ValidateTemplates validate annotations for possible template error, uses empty data for template population
|
|
|
|
func ValidateTemplates(annotations map[string]string) error {
|
2020-12-19 13:10:59 +01:00
|
|
|
_, err := templateAnnotations(annotations, AlertTplData{
|
2020-04-06 13:44:03 +02:00
|
|
|
Labels: map[string]string{},
|
|
|
|
Value: 0,
|
2020-12-14 19:11:45 +01:00
|
|
|
}, tmplFunc)
|
2020-04-06 13:44:03 +02:00
|
|
|
return err
|
2020-03-27 17:31:16 +01:00
|
|
|
}
|
|
|
|
|
2020-12-19 13:10:59 +01:00
|
|
|
func templateAnnotations(annotations map[string]string, data AlertTplData, funcs template.FuncMap) (map[string]string, error) {
|
2020-03-27 17:31:16 +01:00
|
|
|
var builder strings.Builder
|
|
|
|
var buf bytes.Buffer
|
2020-06-29 21:21:03 +02:00
|
|
|
eg := new(utils.ErrGroup)
|
2020-03-27 17:31:16 +01:00
|
|
|
r := make(map[string]string, len(annotations))
|
|
|
|
for key, text := range annotations {
|
|
|
|
buf.Reset()
|
|
|
|
builder.Reset()
|
2020-12-14 19:11:45 +01:00
|
|
|
builder.Grow(len(tplHeader) + len(text))
|
|
|
|
builder.WriteString(tplHeader)
|
2020-03-27 17:31:16 +01:00
|
|
|
builder.WriteString(text)
|
2020-12-14 19:11:45 +01:00
|
|
|
if err := templateAnnotation(&buf, builder.String(), data, funcs); err != nil {
|
2021-07-28 18:26:20 +02:00
|
|
|
r[key] = text
|
2020-06-30 21:58:18 +02:00
|
|
|
eg.Add(fmt.Errorf("key %q, template %q: %w", key, text, err))
|
2020-03-13 11:19:31 +01:00
|
|
|
continue
|
|
|
|
}
|
2020-03-27 17:31:16 +01:00
|
|
|
r[key] = buf.String()
|
2020-03-13 11:19:31 +01:00
|
|
|
}
|
2020-06-29 21:21:03 +02:00
|
|
|
return r, eg.Err()
|
2020-03-29 00:48:30 +01:00
|
|
|
}
|
|
|
|
|
2020-12-19 13:10:59 +01:00
|
|
|
func templateAnnotation(dst io.Writer, text string, data AlertTplData, funcs template.FuncMap) error {
|
2020-12-14 19:11:45 +01:00
|
|
|
t := template.New("").Funcs(funcs).Option("missingkey=zero")
|
|
|
|
tpl, err := t.Parse(text)
|
2020-03-29 00:48:30 +01:00
|
|
|
if err != nil {
|
2020-06-01 12:46:37 +02:00
|
|
|
return fmt.Errorf("error parsing annotation: %w", err)
|
2020-03-29 00:48:30 +01:00
|
|
|
}
|
|
|
|
if err = tpl.Execute(dst, data); err != nil {
|
2020-06-01 12:46:37 +02:00
|
|
|
return fmt.Errorf("error evaluating annotation template: %w", err)
|
2020-03-29 00:48:30 +01:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|