From beee24eceecfa5d70832c331eaedf180745ac3db Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Sat, 22 May 2021 22:26:01 +0100 Subject: [PATCH] vmalert: support `extra_filter_labels` setting per-group (#1319) The new setting `extra_filter_labels` may be assigned to group. If it is, then all rules within a group will automatically filter for configured labels. The feature is well-described here https://docs.victoriametrics.com#prometheus-querying-api-enhancements New setting is compatible only with VM datasource. --- app/vmalert/README.md | 6 +++ app/vmalert/alerting.go | 2 + app/vmalert/config/config.go | 4 ++ app/vmalert/config/testdata/rules2-good.rules | 2 + app/vmalert/datasource/vm.go | 9 +++++ app/vmalert/datasource/vm_test.go | 13 ++++++ app/vmalert/group.go | 40 +++++++++++-------- app/vmalert/manager.go | 14 ++++--- app/vmalert/recording.go | 4 +- app/vmalert/web_types.go | 17 ++++---- 10 files changed, 78 insertions(+), 33 deletions(-) diff --git a/app/vmalert/README.md b/app/vmalert/README.md index 05ac55c0c7..b1b9bc9bc6 100644 --- a/app/vmalert/README.md +++ b/app/vmalert/README.md @@ -85,6 +85,12 @@ name: # By default "prometheus" rule type is used. [ type: ] +# Optional list of label filters applied to every rule's +# request withing a group. Is compatible only with VM datasource. +# See more details at https://docs.victoriametrics.com#prometheus-querying-api-enhancements +extra_filter_labels: + [ : ... ] + rules: [ - ... ] ``` diff --git a/app/vmalert/alerting.go b/app/vmalert/alerting.go index 75c80f2de8..fa9537a565 100644 --- a/app/vmalert/alerting.go +++ b/app/vmalert/alerting.go @@ -65,6 +65,7 @@ func newAlertingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rule q: qb.BuildWithParams(datasource.QuerierParams{ DataSourceType: &cfg.Type, EvaluationInterval: group.Interval, + ExtraLabels: group.ExtraFilterLabels, }), alerts: make(map[uint64]*notifier.Alert), metrics: &alertingRuleMetrics{}, @@ -250,6 +251,7 @@ func (ar *AlertingRule) UpdateWith(r Rule) error { ar.For = nr.For ar.Labels = nr.Labels ar.Annotations = nr.Annotations + ar.q = nr.q return nil } diff --git a/app/vmalert/config/config.go b/app/vmalert/config/config.go index f639febedc..9efc16ab5a 100644 --- a/app/vmalert/config/config.go +++ b/app/vmalert/config/config.go @@ -29,6 +29,10 @@ type Group struct { Interval time.Duration `yaml:"interval,omitempty"` Rules []Rule `yaml:"rules"` Concurrency int `yaml:"concurrency"` + // ExtraFilterLabels is a list label filters applied to every rule + // request withing a group. Is compatible only with VM datasources. + // See https://docs.victoriametrics.com#prometheus-querying-api-enhancements + ExtraFilterLabels map[string]string `yaml:"extra_filter_labels"` // Checksum stores the hash of yaml definition for this group. // May be used to detect any changes like rules re-ordering etc. Checksum string diff --git a/app/vmalert/config/testdata/rules2-good.rules b/app/vmalert/config/testdata/rules2-good.rules index 00e4cb8ee5..d1a1c4260a 100644 --- a/app/vmalert/config/testdata/rules2-good.rules +++ b/app/vmalert/config/testdata/rules2-good.rules @@ -2,6 +2,8 @@ groups: - name: TestGroup interval: 2s concurrency: 2 + extra_filter_labels: + job: victoriametrics rules: - alert: Conns expr: sum(vm_tcplistener_conns) by(instance) > 1 diff --git a/app/vmalert/datasource/vm.go b/app/vmalert/datasource/vm.go index 8b59b389c3..7ffb020a62 100644 --- a/app/vmalert/datasource/vm.go +++ b/app/vmalert/datasource/vm.go @@ -85,6 +85,7 @@ type VMStorage struct { dataSourceType Type evaluationInterval time.Duration + extraLabels []string } const queryPath = "/api/v1/query" @@ -97,6 +98,8 @@ const graphitePrefix = "/graphite" type QuerierParams struct { DataSourceType *Type EvaluationInterval time.Duration + // see https://docs.victoriametrics.com/#prometheus-querying-api-enhancements + ExtraLabels map[string]string } // Clone makes clone of VMStorage, shares http client. @@ -119,6 +122,9 @@ func (s *VMStorage) ApplyParams(params QuerierParams) *VMStorage { s.dataSourceType = *params.DataSourceType } s.evaluationInterval = params.EvaluationInterval + for k, v := range params.ExtraLabels { + s.extraLabels = append(s.extraLabels, fmt.Sprintf("%s=%s", k, v)) + } return s } @@ -222,6 +228,9 @@ func (s *VMStorage) setPrometheusReqParams(r *http.Request, query string, timest if s.roundDigits != "" { q.Set("round_digits", s.roundDigits) } + for _, l := range s.extraLabels { + q.Add("extra_label", l) + } r.URL.RawQuery = q.Encode() } diff --git a/app/vmalert/datasource/vm_test.go b/app/vmalert/datasource/vm_test.go index 9a004bcc56..c7e684aea4 100644 --- a/app/vmalert/datasource/vm_test.go +++ b/app/vmalert/datasource/vm_test.go @@ -253,6 +253,19 @@ func TestPrepareReq(t *testing.T) { checkEqualString(t, exp, r.URL.RawQuery) }, }, + { + "extra labels", + &VMStorage{ + extraLabels: []string{ + "env=prod", + "query=es=cape", + }, + }, + func(t *testing.T, r *http.Request) { + exp := fmt.Sprintf("extra_label=env%%3Dprod&extra_label=query%%3Des%%3Dcape&query=%s&time=%d", query, timestamp.Unix()) + checkEqualString(t, exp, r.URL.RawQuery) + }, + }, } for _, tc := range testCases { diff --git a/app/vmalert/group.go b/app/vmalert/group.go index f9ded5d861..efe07cdb47 100644 --- a/app/vmalert/group.go +++ b/app/vmalert/group.go @@ -18,14 +18,15 @@ import ( // Group is an entity for grouping rules type Group struct { - mu sync.RWMutex - Name string - File string - Rules []Rule - Type datasource.Type - Interval time.Duration - Concurrency int - Checksum string + mu sync.RWMutex + Name string + File string + Rules []Rule + Type datasource.Type + Interval time.Duration + Concurrency int + Checksum string + ExtraFilterLabels map[string]string doneCh chan struct{} finishedCh chan struct{} @@ -51,15 +52,17 @@ func newGroupMetrics(name, file string) *groupMetrics { func newGroup(cfg config.Group, qb datasource.QuerierBuilder, defaultInterval time.Duration, labels map[string]string) *Group { g := &Group{ - Type: cfg.Type, - Name: cfg.Name, - File: cfg.File, - Interval: cfg.Interval, - Concurrency: cfg.Concurrency, - Checksum: cfg.Checksum, - doneCh: make(chan struct{}), - finishedCh: make(chan struct{}), - updateCh: make(chan *Group), + Type: cfg.Type, + Name: cfg.Name, + File: cfg.File, + Interval: cfg.Interval, + Concurrency: cfg.Concurrency, + Checksum: cfg.Checksum, + ExtraFilterLabels: cfg.ExtraFilterLabels, + + doneCh: make(chan struct{}), + finishedCh: make(chan struct{}), + updateCh: make(chan *Group), } g.metrics = newGroupMetrics(g.Name, g.File) if g.Interval == 0 { @@ -115,6 +118,8 @@ func (g *Group) Restore(ctx context.Context, qb datasource.QuerierBuilder, lookb if rr.For < 1 { continue } + // ignore g.ExtraFilterLabels on purpose, so it + // won't affect the restore procedure. q := qb.BuildWithParams(datasource.QuerierParams{}) if err := rr.Restore(ctx, q, lookback, labels); err != nil { return fmt.Errorf("error while restoring rule %q: %w", rule, err) @@ -163,6 +168,7 @@ func (g *Group) updateWith(newGroup *Group) error { } g.Type = newGroup.Type g.Concurrency = newGroup.Concurrency + g.ExtraFilterLabels = newGroup.ExtraFilterLabels g.Checksum = newGroup.Checksum g.Rules = newRules return nil diff --git a/app/vmalert/manager.go b/app/vmalert/manager.go index 4d9b0595ea..99a00afe50 100644 --- a/app/vmalert/manager.go +++ b/app/vmalert/manager.go @@ -147,12 +147,14 @@ func (g *Group) toAPI() APIGroup { ag := APIGroup{ // encode as string to avoid rounding - ID: fmt.Sprintf("%d", g.ID()), - Name: g.Name, - Type: g.Type.String(), - File: g.File, - Interval: g.Interval.String(), - Concurrency: g.Concurrency, + ID: fmt.Sprintf("%d", g.ID()), + + Name: g.Name, + Type: g.Type.String(), + File: g.File, + Interval: g.Interval.String(), + Concurrency: g.Concurrency, + ExtraFilterLabels: g.ExtraFilterLabels, } for _, r := range g.Rules { switch v := r.(type) { diff --git a/app/vmalert/recording.go b/app/vmalert/recording.go index 5e00c34db3..b8112357fa 100644 --- a/app/vmalert/recording.go +++ b/app/vmalert/recording.go @@ -66,6 +66,7 @@ func newRecordingRule(qb datasource.QuerierBuilder, group *Group, cfg config.Rul q: qb.BuildWithParams(datasource.QuerierParams{ DataSourceType: &cfg.Type, EvaluationInterval: group.Interval, + ExtraLabels: group.ExtraFilterLabels, }), } @@ -151,8 +152,6 @@ func (rr *RecordingRule) toTimeSeries(m datasource.Metric, timestamp time.Time) } // UpdateWith copies all significant fields. -// alerts state isn't copied since -// it should be updated in next 2 Execs func (rr *RecordingRule) UpdateWith(r Rule) error { nr, ok := r.(*RecordingRule) if !ok { @@ -160,6 +159,7 @@ func (rr *RecordingRule) UpdateWith(r Rule) error { } rr.Expr = nr.Expr rr.Labels = nr.Labels + rr.q = nr.q return nil } diff --git a/app/vmalert/web_types.go b/app/vmalert/web_types.go index ca3cbdd05d..8586fb64c4 100644 --- a/app/vmalert/web_types.go +++ b/app/vmalert/web_types.go @@ -20,14 +20,15 @@ type APIAlert struct { // APIGroup represents Group for WEB view type APIGroup struct { - Name string `json:"name"` - Type string `json:"type"` - ID string `json:"id"` - File string `json:"file"` - Interval string `json:"interval"` - Concurrency int `json:"concurrency"` - AlertingRules []APIAlertingRule `json:"alerting_rules"` - RecordingRules []APIRecordingRule `json:"recording_rules"` + Name string `json:"name"` + Type string `json:"type"` + ID string `json:"id"` + File string `json:"file"` + Interval string `json:"interval"` + Concurrency int `json:"concurrency"` + ExtraFilterLabels map[string]string `json:"extra_filter_labels"` + AlertingRules []APIAlertingRule `json:"alerting_rules"` + RecordingRules []APIRecordingRule `json:"recording_rules"` } // APIAlertingRule represents AlertingRule for WEB view