From 3a444bb7bb319c6c6478edb7ca853c08a172e5cd Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 23 Jun 2020 17:17:58 +0300 Subject: [PATCH] lib/promrelabel: add support for `keep_if_equal` and `drop_if_equal` actions to relabel configs These actions may be useful for filtering out unneeded targets and/or metrics if they contain equal label values. For example, the following rule would leave the target only if __meta_kubernetes_annotation_prometheus_io_port equals __meta_kubernetes_pod_container_port_number: - action: keep_if_equal source_labels: [__meta_kubernetes_annotation_prometheus_io_port, __meta_kubernetes_pod_container_port_number] --- app/vmagent/README.md | 10 +++ docs/vmagent.md | 10 +++ lib/promrelabel/config.go | 8 ++ lib/promrelabel/config_test.go | 34 +++++++- lib/promrelabel/relabel.go | 39 +++++++++ lib/promrelabel/relabel_test.go | 86 +++++++++++++++++++ .../testdata/relabel_configs_valid.yml | 4 + 7 files changed, 189 insertions(+), 2 deletions(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index ddf7ca7152..cf2f1d4443 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -170,6 +170,8 @@ Additionally it provides the following extra actions: * `replace_all`: replaces all the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the result in the `target_label`. * `labelmap_all`: replaces all the occurences of `regex` in all the label names with the `replacement`. +* `keep_if_equal`: keeps the entry if all label values from `source_labels` are equal. +* `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal. The relabeling can be defined in the following places: @@ -210,6 +212,14 @@ either via `vmagent` itself or via Prometheus, so the exported metrics could be The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set. If you don't want to send all the data from the directory to remote storage, simply stop `vmagent` and delete the directory. +* If you see `skipping duplicate scrape target with identical labels` errors when scraping Kubernetes pods, then it is likely these pods listen multiple ports. + Just add the following relabeling rule to `relabel_configs` section in order to filter out targets with unneeded ports: + +```yml +- action: keep_if_equal + source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_container_port_number] +``` + ### How to build from sources diff --git a/docs/vmagent.md b/docs/vmagent.md index ddf7ca7152..cf2f1d4443 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -170,6 +170,8 @@ Additionally it provides the following extra actions: * `replace_all`: replaces all the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the result in the `target_label`. * `labelmap_all`: replaces all the occurences of `regex` in all the label names with the `replacement`. +* `keep_if_equal`: keeps the entry if all label values from `source_labels` are equal. +* `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal. The relabeling can be defined in the following places: @@ -210,6 +212,14 @@ either via `vmagent` itself or via Prometheus, so the exported metrics could be The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set. If you don't want to send all the data from the directory to remote storage, simply stop `vmagent` and delete the directory. +* If you see `skipping duplicate scrape target with identical labels` errors when scraping Kubernetes pods, then it is likely these pods listen multiple ports. + Just add the following relabeling rule to `relabel_configs` section in order to filter out targets with unneeded ports: + +```yml +- action: keep_if_equal + source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_container_port_number] +``` + ### How to build from sources diff --git a/lib/promrelabel/config.go b/lib/promrelabel/config.go index 282c48c08a..5a5d602861 100644 --- a/lib/promrelabel/config.go +++ b/lib/promrelabel/config.go @@ -92,6 +92,14 @@ func parseRelabelConfig(dst []ParsedRelabelConfig, rc *RelabelConfig) ([]ParsedR if targetLabel == "" { return dst, fmt.Errorf("missing `target_label` for `action=replace`") } + case "keep_if_equal": + if len(sourceLabels) < 2 { + return dst, fmt.Errorf("`source_labels` must contain at least two entries for `action=keep_if_equal`; got %q", sourceLabels) + } + case "drop_if_equal": + if len(sourceLabels) < 2 { + return dst, fmt.Errorf("`source_labels` must contain at least two entries for `action=drop_if_equal`; got %q", sourceLabels) + } case "keep": if len(sourceLabels) == 0 { return dst, fmt.Errorf("missing `source_labels` for `action=keep`") diff --git a/lib/promrelabel/config_test.go b/lib/promrelabel/config_test.go index 206c345198..3a1ae1477c 100644 --- a/lib/promrelabel/config_test.go +++ b/lib/promrelabel/config_test.go @@ -11,8 +11,8 @@ func TestLoadRelabelConfigsSuccess(t *testing.T) { if err != nil { t.Fatalf("cannot load relabel configs from %q: %s", path, err) } - if len(prcs) != 7 { - t.Fatalf("unexpected number of relabel configs loaded from %q; got %d; want %d", path, len(prcs), 7) + if len(prcs) != 9 { + t.Fatalf("unexpected number of relabel configs loaded from %q; got %d; want %d", path, len(prcs), 9) } } @@ -117,6 +117,36 @@ func TestParseRelabelConfigsFailure(t *testing.T) { }, }) }) + t.Run("keep_if_equal-missing-source-labels", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "keep_if_equal", + }, + }) + }) + t.Run("keep_if_equal-single-source-label", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "keep_if_equal", + SourceLabels: []string{"foo"}, + }, + }) + }) + t.Run("drop_if_equal-missing-source-labels", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "drop_if_equal", + }, + }) + }) + t.Run("drop_if_equal-single-source-label", func(t *testing.T) { + f([]RelabelConfig{ + { + Action: "drop_if_equal", + SourceLabels: []string{"foo"}, + }, + }) + }) t.Run("drop-missing-source-labels", func(t *testing.T) { f([]RelabelConfig{ { diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go index 211fd77a6f..8432f8cfff 100644 --- a/lib/promrelabel/relabel.go +++ b/lib/promrelabel/relabel.go @@ -148,6 +148,30 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par relabelBufPool.Put(bb) valueStr := prc.Regex.ReplaceAllString(sourceStr, prc.Replacement) return setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr) + case "keep_if_equal": + // Keep the entry if all the label values in source_labels are equal. + // For example: + // + // - source_labels: [foo, bar] + // action: keep_if_equal + // + // Would leave the entry if `foo` value equals `bar` value + if areEqualLabelValues(src, prc.SourceLabels) { + return labels + } + return labels[:labelsOffset] + case "drop_if_equal": + // Drop the entry if all the label values in source_labels are equal. + // For example: + // + // - source_labels: [foo, bar] + // action: drop_if_equal + // + // Would drop the entry if `foo` value equals `bar` value. + if areEqualLabelValues(src, prc.SourceLabels) { + return labels[:labelsOffset] + } + return labels case "keep": bb := relabelBufPool.Get() bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator) @@ -249,6 +273,21 @@ func (prc *ParsedRelabelConfig) expandCaptureGroups(template, source string, mat var relabelBufPool bytesutil.ByteBufferPool +func areEqualLabelValues(labels []prompbmarshal.Label, labelNames []string) bool { + if len(labelNames) < 2 { + logger.Panicf("BUG: expecting at least 2 labelNames; got %d", len(labelNames)) + return false + } + labelValue := GetLabelValueByName(labels, labelNames[0]) + for _, labelName := range labelNames[1:] { + v := GetLabelValueByName(labels, labelName) + if v != labelValue { + return false + } + } + return true +} + func concatLabelValues(dst []byte, labels []prompbmarshal.Label, labelNames []string, separator string) []byte { if len(labelNames) == 0 { return dst diff --git a/lib/promrelabel/relabel_test.go b/lib/promrelabel/relabel_test.go index 2cbfbee8bf..f26a51372e 100644 --- a/lib/promrelabel/relabel_test.go +++ b/lib/promrelabel/relabel_test.go @@ -336,6 +336,92 @@ func TestApplyRelabelConfigs(t *testing.T) { }, }) }) + t.Run("keep_if_equal-miss", func(t *testing.T) { + f([]ParsedRelabelConfig{ + { + Action: "keep_if_equal", + SourceLabels: []string{"foo", "bar"}, + }, + }, nil, true, nil) + f([]ParsedRelabelConfig{ + { + Action: "keep_if_equal", + SourceLabels: []string{"xxx", "bar"}, + }, + }, []prompbmarshal.Label{ + { + Name: "xxx", + Value: "yyy", + }, + }, true, []prompbmarshal.Label{}) + }) + t.Run("keep_if_equal-hit", func(t *testing.T) { + f([]ParsedRelabelConfig{ + { + Action: "keep_if_equal", + SourceLabels: []string{"xxx", "bar"}, + }, + }, []prompbmarshal.Label{ + { + Name: "xxx", + Value: "yyy", + }, + { + Name: "bar", + Value: "yyy", + }, + }, true, []prompbmarshal.Label{ + { + Name: "bar", + Value: "yyy", + }, + { + Name: "xxx", + Value: "yyy", + }, + }) + }) + t.Run("drop_if_equal-miss", func(t *testing.T) { + f([]ParsedRelabelConfig{ + { + Action: "drop_if_equal", + SourceLabels: []string{"foo", "bar"}, + }, + }, nil, true, nil) + f([]ParsedRelabelConfig{ + { + Action: "drop_if_equal", + SourceLabels: []string{"xxx", "bar"}, + }, + }, []prompbmarshal.Label{ + { + Name: "xxx", + Value: "yyy", + }, + }, true, []prompbmarshal.Label{ + { + Name: "xxx", + Value: "yyy", + }, + }) + }) + t.Run("drop_if_equal-hit", func(t *testing.T) { + f([]ParsedRelabelConfig{ + { + Action: "drop_if_equal", + SourceLabels: []string{"xxx", "bar"}, + }, + }, []prompbmarshal.Label{ + { + Name: "xxx", + Value: "yyy", + }, + { + Name: "bar", + Value: "yyy", + }, + }, true, []prompbmarshal.Label{}) + }) t.Run("keep-miss", func(t *testing.T) { f([]ParsedRelabelConfig{ { diff --git a/lib/promrelabel/testdata/relabel_configs_valid.yml b/lib/promrelabel/testdata/relabel_configs_valid.yml index 7ebe8ef3ad..5b4a8828ec 100644 --- a/lib/promrelabel/testdata/relabel_configs_valid.yml +++ b/lib/promrelabel/testdata/relabel_configs_valid.yml @@ -18,3 +18,7 @@ - action: labelmap_all regex: "\\." replacement: ":" +- action: keep_if_equal + source_labels: [foo, bar] +- action: drop_if_equal + source_labels: [foo, bar]