Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2025-01-20 07:19:17 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

[vmalert] add webserver (#410)

Browse Source 
* [vmalert] add webserver
This commit is contained in:
kreedom 2020-04-11 12:40:24 +03:00 committed by GitHub
parent 830d5fb1e0
commit 90de3086b3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 230 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/vmalert/main.go
View File

@ -61,7 +61,7 @@ func main() {
w := &watchdog{
storage: datasource.NewVMStorage(*datasourceURL, *basicAuthUsername, *basicAuthPassword, &http.Client{}),
alertProvider: notifier.NewAlertManager(*providerURL, func(group, name string) string {
return strings.Replace(fmt.Sprintf("%s/%s/%s/status", eu, group, name), "//", "/", -1)
return fmt.Sprintf("%s/api/v1/%s/%s/status", eu, group, name)
}, &http.Client{}),
}
wg := sync.WaitGroup{}
@ -73,9 +73,7 @@ func main() {
}(groups[i])
}
go httpserver.Serve(*httpListenAddr, func(w http.ResponseWriter, r *http.Request) bool {
panic("not implemented")
})
go httpserver.Serve(*httpListenAddr, (&requestHandler{groups: groups}).handler)
sig := procutil.WaitForSigterm()
logger.Infof("service received signal %s", sig)

12
app/vmalert/notifier/alert.go
View File

@ -21,6 +21,7 @@ type Alert struct {
Start time.Time
End time.Time
Value float64
ID uint64
}
// AlertState type indicates the Alert state
@ -37,6 +38,17 @@ const (
StateFiring
)
// String stringer for AlertState
func (as AlertState) String() string {
switch as {
case StateFiring:
return "firing"
case StatePending:
return "pending"
}
return "inactive"
}
type alertTplData struct {
Labels map[string]string
Value float64

2
app/vmalert/notifier/alertmanager.go
View File

@ -37,7 +37,7 @@ func (am *AlertManager) Send(alerts []Alert) error {
}
// AlertURLGenerator returns URL to single alert by given name
type AlertURLGenerator func(group, name string) string
type AlertURLGenerator func(group, id string) string
const alertManagerPath = "/api/v2/alerts"

3
app/vmalert/notifier/alertmanager_request.qtpl
View File

@ -1,4 +1,5 @@
{% import (
"strconv"
"time"
) %}
{% stripspace %}
@ -8,7 +9,7 @@
{% for i, alert := range alerts %}
{
"startsAt":{%q= alert.Start.Format(time.RFC3339Nano) %},
"generatorURL": {%q= generatorURL(alert.Group, alert.Name) %},
"generatorURL": {%q= generatorURL(alert.Group, strconv.FormatUint(alert.ID, 10)) %},
{% if !alert.End.IsZero() %}
"endsAt":{%q= alert.End.Format(time.RFC3339Nano) %},
{% endif %}

113
app/vmalert/notifier/alertmanager_request.qtpl.go
View File

@ -1,130 +1,131 @@
// Code generated by qtc from "alertmanager_request.qtpl". DO NOT EDIT.
// See https://github.com/valyala/quicktemplate for details.
//line notifier/alertmanager_request.qtpl:1
//line app/vmalert/notifier/alertmanager_request.qtpl:1
package notifier
//line notifier/alertmanager_request.qtpl:1
//line app/vmalert/notifier/alertmanager_request.qtpl:1
import (
"strconv"
"time"
)
//line notifier/alertmanager_request.qtpl:6
//line app/vmalert/notifier/alertmanager_request.qtpl:7
import (
qtio422016 "io"
qt422016 "github.com/valyala/quicktemplate"
)
//line notifier/alertmanager_request.qtpl:6
//line app/vmalert/notifier/alertmanager_request.qtpl:7
var (
_ = qtio422016.Copy
_ = qt422016.AcquireByteBuffer
)
//line notifier/alertmanager_request.qtpl:6
//line app/vmalert/notifier/alertmanager_request.qtpl:7
func streamamRequest(qw422016 *qt422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
//line notifier/alertmanager_request.qtpl:6
//line app/vmalert/notifier/alertmanager_request.qtpl:7
qw422016.N().S(`[`)
//line notifier/alertmanager_request.qtpl:8
//line app/vmalert/notifier/alertmanager_request.qtpl:9
for i, alert := range alerts {
//line notifier/alertmanager_request.qtpl:8
//line app/vmalert/notifier/alertmanager_request.qtpl:9
qw422016.N().S(`{"startsAt":`)
//line notifier/alertmanager_request.qtpl:10
//line app/vmalert/notifier/alertmanager_request.qtpl:11
qw422016.N().Q(alert.Start.Format(time.RFC3339Nano))
//line notifier/alertmanager_request.qtpl:10
//line app/vmalert/notifier/alertmanager_request.qtpl:11
qw422016.N().S(`,"generatorURL":`)
//line notifier/alertmanager_request.qtpl:11
qw422016.N().Q(generatorURL(alert.Group, alert.Name))
//line notifier/alertmanager_request.qtpl:11
//line app/vmalert/notifier/alertmanager_request.qtpl:12
qw422016.N().Q(generatorURL(alert.Group, strconv.FormatUint(alert.ID, 10)))
//line app/vmalert/notifier/alertmanager_request.qtpl:12
qw422016.N().S(`,`)
//line notifier/alertmanager_request.qtpl:12
//line app/vmalert/notifier/alertmanager_request.qtpl:13
if !alert.End.IsZero() {
//line notifier/alertmanager_request.qtpl:12
//line app/vmalert/notifier/alertmanager_request.qtpl:13
qw422016.N().S(`"endsAt":`)
//line notifier/alertmanager_request.qtpl:13
//line app/vmalert/notifier/alertmanager_request.qtpl:14
qw422016.N().Q(alert.End.Format(time.RFC3339Nano))
//line notifier/alertmanager_request.qtpl:13
//line app/vmalert/notifier/alertmanager_request.qtpl:14
qw422016.N().S(`,`)
//line notifier/alertmanager_request.qtpl:14
//line app/vmalert/notifier/alertmanager_request.qtpl:15
}
//line notifier/alertmanager_request.qtpl:14
//line app/vmalert/notifier/alertmanager_request.qtpl:15
qw422016.N().S(`"labels": {"alertname":`)
//line notifier/alertmanager_request.qtpl:16
//line app/vmalert/notifier/alertmanager_request.qtpl:17
qw422016.N().Q(alert.Name)
//line notifier/alertmanager_request.qtpl:17
//line app/vmalert/notifier/alertmanager_request.qtpl:18
for k, v := range alert.Labels {
//line notifier/alertmanager_request.qtpl:17
//line app/vmalert/notifier/alertmanager_request.qtpl:18
qw422016.N().S(`,`)
//line notifier/alertmanager_request.qtpl:18
//line app/vmalert/notifier/alertmanager_request.qtpl:19
qw422016.N().Q(k)
//line notifier/alertmanager_request.qtpl:18
//line app/vmalert/notifier/alertmanager_request.qtpl:19
qw422016.N().S(`:`)
//line notifier/alertmanager_request.qtpl:18
//line app/vmalert/notifier/alertmanager_request.qtpl:19
qw422016.N().Q(v)
//line notifier/alertmanager_request.qtpl:19
//line app/vmalert/notifier/alertmanager_request.qtpl:20
}
//line notifier/alertmanager_request.qtpl:19
//line app/vmalert/notifier/alertmanager_request.qtpl:20
qw422016.N().S(`},"annotations": {`)
//line notifier/alertmanager_request.qtpl:22
//line app/vmalert/notifier/alertmanager_request.qtpl:23
c := len(alert.Annotations)
//line notifier/alertmanager_request.qtpl:23
//line app/vmalert/notifier/alertmanager_request.qtpl:24
for k, v := range alert.Annotations {
//line notifier/alertmanager_request.qtpl:24
//line app/vmalert/notifier/alertmanager_request.qtpl:25
c = c - 1
//line notifier/alertmanager_request.qtpl:25
//line app/vmalert/notifier/alertmanager_request.qtpl:26
qw422016.N().Q(k)
//line notifier/alertmanager_request.qtpl:25
//line app/vmalert/notifier/alertmanager_request.qtpl:26
qw422016.N().S(`:`)
//line notifier/alertmanager_request.qtpl:25
//line app/vmalert/notifier/alertmanager_request.qtpl:26
qw422016.N().Q(v)
//line notifier/alertmanager_request.qtpl:25
//line app/vmalert/notifier/alertmanager_request.qtpl:26
if c > 0 {
//line notifier/alertmanager_request.qtpl:25
//line app/vmalert/notifier/alertmanager_request.qtpl:26
qw422016.N().S(`,`)
//line notifier/alertmanager_request.qtpl:25
//line app/vmalert/notifier/alertmanager_request.qtpl:26
}
//line notifier/alertmanager_request.qtpl:26
//line app/vmalert/notifier/alertmanager_request.qtpl:27
}
//line notifier/alertmanager_request.qtpl:26
//line app/vmalert/notifier/alertmanager_request.qtpl:27
qw422016.N().S(`}}`)
//line notifier/alertmanager_request.qtpl:29
//line app/vmalert/notifier/alertmanager_request.qtpl:30
if i != len(alerts)-1 {
//line notifier/alertmanager_request.qtpl:29
//line app/vmalert/notifier/alertmanager_request.qtpl:30
qw422016.N().S(`,`)
//line notifier/alertmanager_request.qtpl:29
//line app/vmalert/notifier/alertmanager_request.qtpl:30
}
//line notifier/alertmanager_request.qtpl:30
//line app/vmalert/notifier/alertmanager_request.qtpl:31
}
//line notifier/alertmanager_request.qtpl:30
//line app/vmalert/notifier/alertmanager_request.qtpl:31
qw422016.N().S(`]`)
//line notifier/alertmanager_request.qtpl:32
//line app/vmalert/notifier/alertmanager_request.qtpl:33
}
//line notifier/alertmanager_request.qtpl:32
//line app/vmalert/notifier/alertmanager_request.qtpl:33
func writeamRequest(qq422016 qtio422016.Writer, alerts []Alert, generatorURL func(string, string) string) {
//line notifier/alertmanager_request.qtpl:32
//line app/vmalert/notifier/alertmanager_request.qtpl:33
qw422016 := qt422016.AcquireWriter(qq422016)
//line notifier/alertmanager_request.qtpl:32
//line app/vmalert/notifier/alertmanager_request.qtpl:33
streamamRequest(qw422016, alerts, generatorURL)
//line notifier/alertmanager_request.qtpl:32
//line app/vmalert/notifier/alertmanager_request.qtpl:33
qt422016.ReleaseWriter(qw422016)
//line notifier/alertmanager_request.qtpl:32
//line app/vmalert/notifier/alertmanager_request.qtpl:33
}
//line notifier/alertmanager_request.qtpl:32
//line app/vmalert/notifier/alertmanager_request.qtpl:33
func amRequest(alerts []Alert, generatorURL func(string, string) string) string {
//line notifier/alertmanager_request.qtpl:32
//line app/vmalert/notifier/alertmanager_request.qtpl:33
qb422016 := qt422016.AcquireByteBuffer()
//line notifier/alertmanager_request.qtpl:32
//line app/vmalert/notifier/alertmanager_request.qtpl:33
writeamRequest(qb422016, alerts, generatorURL)
//line notifier/alertmanager_request.qtpl:32
//line app/vmalert/notifier/alertmanager_request.qtpl:33
qs422016 := string(qb422016.B)
//line notifier/alertmanager_request.qtpl:32
//line app/vmalert/notifier/alertmanager_request.qtpl:33
qt422016.ReleaseByteBuffer(qb422016)
//line notifier/alertmanager_request.qtpl:32
//line app/vmalert/notifier/alertmanager_request.qtpl:33
return qs422016
//line notifier/alertmanager_request.qtpl:32
//line app/vmalert/notifier/alertmanager_request.qtpl:33
}

4
app/vmalert/notifier/alertmanager_test.go
View File

@ -40,7 +40,7 @@ func TestAlertManager_Send(t *testing.T) {
if len(a) != 1 {
t.Errorf("expected 1 alert in array got %d", len(a))
}
if a[0].GeneratorURL != "group0alert0" {
if a[0].GeneratorURL != "group0" {
t.Errorf("exptected alert0 as generatorURL got %s", a[0].GeneratorURL)
}
if a[0].Labels["alertname"] != "alert0" {
@ -66,7 +66,7 @@ func TestAlertManager_Send(t *testing.T) {
t.Error("expected wrong http code error got nil")
}
if err := am.Send([]Alert{{
Group: "group0",
Group: "group",
Name: "alert0",
Start: time.Now().UTC(),
End: time.Now().UTC(),

33
app/vmalert/rule.go
View File

@ -21,6 +21,15 @@ type Group struct {
Rules []*Rule
}
// ActiveAlerts returns list of active alert for all rules
func (g *Group) ActiveAlerts() []*notifier.Alert {
var list []*notifier.Alert
for i := range g.Rules {
list = append(list, g.Rules[i].listActiveAlerts()...)
}
return list
}
// Rule is basic alert entity
type Rule struct {
Name string `yaml:"alert"`
@ -61,7 +70,6 @@ func (r *Rule) Validate() error {
// Based on the Querier results Rule maintains notifier.Alerts
func (r *Rule) Exec(ctx context.Context, q datasource.Querier) error {
metrics, err := q.Query(ctx, r.Expr)
r.mu.Lock()
defer r.mu.Unlock()
@ -91,6 +99,7 @@ func (r *Rule) Exec(ctx context.Context, q datasource.Querier) error {
r.lastExecError = err
return fmt.Errorf("failed to create alert: %s", err)
}
a.ID = h
a.State = notifier.StatePending
r.alerts[h] = a
}
@ -119,7 +128,7 @@ func (r *Rule) Exec(ctx context.Context, q datasource.Querier) error {
// notifier.Notifier.
// See for reference https://prometheus.io/docs/alerting/clients/
// TODO: add tests for endAt value
func (r *Rule) Send(ctx context.Context, ap notifier.Notifier) error {
func (r *Rule) Send(_ context.Context, ap notifier.Notifier) error {
// copy alerts to new list to avoid locks
var alertsCopy []notifier.Alert
r.mu.Lock()
@ -178,3 +187,23 @@ func (r *Rule) newAlert(m datasource.Metric) (*notifier.Alert, error) {
a.Annotations, err = a.ExecTemplate(r.Annotations)
return a, err
}
func (r *Rule) listActiveAlerts() []*notifier.Alert {
r.mu.Lock()
defer r.mu.Unlock()
var list []*notifier.Alert
for _, a := range r.alerts {
a := a
if a.State == notifier.StateFiring {
list = append(list, a)
}
}
return list
}
// Alert returns single alert by its id(hash)
func (r *Rule) Alert(id uint64) *notifier.Alert {
r.mu.Lock()
defer r.mu.Unlock()
return r.alerts[id]
}

2
app/vmalert/testdata/rules0-good.rules vendored
View File

@ -2,7 +2,7 @@ groups:
- name: groupGorSingleAlert
rules:
- alert: VMRows
for: 5m
for: 10s
expr: vm_rows > 0
labels:
label: bar

122
app/vmalert/web.go Normal file
View File

@ -0,0 +1,122 @@
package main
import (
"encoding/json"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
)
// apiAlert has info for an alert.
type apiAlert struct {
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
State string `json:"state"`
ActiveAt time.Time `json:"activeAt"`
Value string `json:"value"`
}
type requestHandler struct {
groups []Group
}
func (rh *requestHandler) handler(w http.ResponseWriter, r *http.Request) bool {
resph := responseHandler{w}
switch r.URL.Path {
default:
if strings.HasSuffix(r.URL.Path, "/status") {
resph.handle(rh.alert(r.URL.Path))
return true
}
return false
case "/api/v1/alerts":
resph.handle(rh.listActiveAlerts())
return true
}
}
func (rh *requestHandler) listActiveAlerts() ([]byte, error) {
type listAlertsResponse struct {
Data struct {
Alerts []apiAlert `json:"alerts"`
} `json:"data"`
Status string `json:"status"`
}
lr := listAlertsResponse{Status: "success"}
for _, g := range rh.groups {
alerts := g.ActiveAlerts()
for i := range alerts {
alert := alerts[i]
lr.Data.Alerts = append(lr.Data.Alerts, apiAlert{
Labels: alert.Labels,
Annotations: alert.Annotations,
State: alert.State.String(),
ActiveAt: alert.Start,
Value: strconv.FormatFloat(alert.Value, 'e', -1, 64),
})
}
}
b, err := json.Marshal(lr)
if err != nil {
return nil, &httpserver.ErrorWithStatusCode{
Err: fmt.Errorf(`error encoding list of active alerts: %s`, err),
StatusCode: http.StatusInternalServerError,
}
}
return b, nil
}
func (rh *requestHandler) alert(path string) ([]byte, error) {
parts := strings.SplitN(strings.TrimPrefix(path, "/api/v1/"), "/", 3)
if len(parts) != 3 {
return nil, &httpserver.ErrorWithStatusCode{
Err: fmt.Errorf(`path %q cointains /status suffix but doesn't match pattern "/group/alert/status"`, path),
StatusCode: http.StatusBadRequest,
}
}
group := strings.TrimRight(parts[0], "/")
idStr := strings.TrimRight(parts[1], "/")
id, err := strconv.ParseUint(idStr, 10, 0)
if err != nil {
return nil, &httpserver.ErrorWithStatusCode{
Err: fmt.Errorf(`cannot parse int from %s"`, idStr),
StatusCode: http.StatusBadRequest,
}
}
for _, g := range rh.groups {
if g.Name == group {
for i := range g.Rules {
if alert := g.Rules[i].Alert(id); alert != nil {
return json.Marshal(apiAlert{
Labels: alert.Labels,
Annotations: alert.Annotations,
State: alert.State.String(),
ActiveAt: alert.Start,
Value: strconv.FormatFloat(alert.Value, 'e', -1, 64),
})
}
}
}
}
return nil, &httpserver.ErrorWithStatusCode{
Err: fmt.Errorf(`cannot find alert %s in %s"`, idStr, group),
StatusCode: http.StatusNotFound,
}
}
// responseHandler wrapper on http.ResponseWriter with sugar
type responseHandler struct{ http.ResponseWriter }
func (w responseHandler) handle(b []byte, err error) {
if err != nil {
httpserver.Errorf(w, "%s", err)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(b)
}