lib/promrelabel: add support for keep_if_equal and drop_if_equal actions to relabel configs

These actions may be useful for filtering out unneeded targets and/or metrics if they contain equal label values.
For example, the following rule would leave the target only if __meta_kubernetes_annotation_prometheus_io_port
equals __meta_kubernetes_pod_container_port_number:

  - action: keep_if_equal
    source_labels: [__meta_kubernetes_annotation_prometheus_io_port, __meta_kubernetes_pod_container_port_number]

app/vmagent/README.md

@@ -170,6 +170,8 @@ Additionally it provides the following extra actions:
 
 * `replace_all`: replaces all the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the result in the `target_label`.
 * `labelmap_all`: replaces all the occurences of `regex` in all the label names with the `replacement`.
+* `keep_if_equal`: keeps the entry if all label values from `source_labels` are equal.
+* `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal.
 
 The relabeling can be defined in the following places:
 
@@ -210,6 +212,14 @@ either via `vmagent` itself or via Prometheus, so the exported metrics could be
   The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
   If you don't want to send all the data from the directory to remote storage, simply stop `vmagent` and delete the directory.
 
+* If you see `skipping duplicate scrape target with identical labels` errors when scraping Kubernetes pods, then it is likely these pods listen multiple ports.
+  Just add the following relabeling rule to `relabel_configs` section in order to filter out targets with unneeded ports:
+
+```yml
+- action: keep_if_equal
+  source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_container_port_number]
+```
+
 
 ### How to build from sources
 

docs/vmagent.md

@@ -170,6 +170,8 @@ Additionally it provides the following extra actions:
 
 * `replace_all`: replaces all the occurences of `regex` in the values of `source_labels` with the `replacement` and stores the result in the `target_label`.
 * `labelmap_all`: replaces all the occurences of `regex` in all the label names with the `replacement`.
+* `keep_if_equal`: keeps the entry if all label values from `source_labels` are equal.
+* `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal.
 
 The relabeling can be defined in the following places:
 
@@ -210,6 +212,14 @@ either via `vmagent` itself or via Prometheus, so the exported metrics could be
   The directory can grow large when remote storage is unavailable for extended periods of time and if `-remoteWrite.maxDiskUsagePerURL` isn't set.
   If you don't want to send all the data from the directory to remote storage, simply stop `vmagent` and delete the directory.
 
+* If you see `skipping duplicate scrape target with identical labels` errors when scraping Kubernetes pods, then it is likely these pods listen multiple ports.
+  Just add the following relabeling rule to `relabel_configs` section in order to filter out targets with unneeded ports:
+
+```yml
+- action: keep_if_equal
+  source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port, __meta_kubernetes_pod_container_port_number]
+```
+
 
 ### How to build from sources
 

lib/promrelabel/config.go

@@ -92,6 +92,14 @@ func parseRelabelConfig(dst []ParsedRelabelConfig, rc *RelabelConfig) ([]ParsedR
 		if targetLabel == "" {
 			return dst, fmt.Errorf("missing `target_label` for `action=replace`")
 		}
+	case "keep_if_equal":
+		if len(sourceLabels) < 2 {
+			return dst, fmt.Errorf("`source_labels` must contain at least two entries for `action=keep_if_equal`; got %q", sourceLabels)
+		}
+	case "drop_if_equal":
+		if len(sourceLabels) < 2 {
+			return dst, fmt.Errorf("`source_labels` must contain at least two entries for `action=drop_if_equal`; got %q", sourceLabels)
+		}
 	case "keep":
 		if len(sourceLabels) == 0 {
 			return dst, fmt.Errorf("missing `source_labels` for `action=keep`")

lib/promrelabel/config_test.go

@@ -11,8 +11,8 @@ func TestLoadRelabelConfigsSuccess(t *testing.T) {
 	if err != nil {
 		t.Fatalf("cannot load relabel configs from %q: %s", path, err)
 	}
-	if len(prcs) != 7 {
+	if len(prcs) != 9 {
-		t.Fatalf("unexpected number of relabel configs loaded from %q; got %d; want %d", path, len(prcs), 7)
+		t.Fatalf("unexpected number of relabel configs loaded from %q; got %d; want %d", path, len(prcs), 9)
 	}
 }
 
@@ -117,6 +117,36 @@ func TestParseRelabelConfigsFailure(t *testing.T) {
 			},
 		})
 	})
+	t.Run("keep_if_equal-missing-source-labels", func(t *testing.T) {
+		f([]RelabelConfig{
+			{
+				Action: "keep_if_equal",
+			},
+		})
+	})
+	t.Run("keep_if_equal-single-source-label", func(t *testing.T) {
+		f([]RelabelConfig{
+			{
+				Action:       "keep_if_equal",
+				SourceLabels: []string{"foo"},
+			},
+		})
+	})
+	t.Run("drop_if_equal-missing-source-labels", func(t *testing.T) {
+		f([]RelabelConfig{
+			{
+				Action: "drop_if_equal",
+			},
+		})
+	})
+	t.Run("drop_if_equal-single-source-label", func(t *testing.T) {
+		f([]RelabelConfig{
+			{
+				Action:       "drop_if_equal",
+				SourceLabels: []string{"foo"},
+			},
+		})
+	})
 	t.Run("drop-missing-source-labels", func(t *testing.T) {
 		f([]RelabelConfig{
 			{

lib/promrelabel/relabel.go

@@ -148,6 +148,30 @@ func applyRelabelConfig(labels []prompbmarshal.Label, labelsOffset int, prc *Par
 		relabelBufPool.Put(bb)
 		valueStr := prc.Regex.ReplaceAllString(sourceStr, prc.Replacement)
 		return setLabelValue(labels, labelsOffset, prc.TargetLabel, valueStr)
+	case "keep_if_equal":
+		// Keep the entry if all the label values in source_labels are equal.
+		// For example:
+		//
+		//   - source_labels: [foo, bar]
+		//     action: keep_if_equal
+		//
+		// Would leave the entry if `foo` value equals `bar` value
+		if areEqualLabelValues(src, prc.SourceLabels) {
+			return labels
+		}
+		return labels[:labelsOffset]
+	case "drop_if_equal":
+		// Drop the entry if all the label values in source_labels are equal.
+		// For example:
+		//
+		//   - source_labels: [foo, bar]
+		//     action: drop_if_equal
+		//
+		// Would drop the entry if `foo` value equals `bar` value.
+		if areEqualLabelValues(src, prc.SourceLabels) {
+			return labels[:labelsOffset]
+		}
+		return labels
 	case "keep":
 		bb := relabelBufPool.Get()
 		bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator)
@@ -249,6 +273,21 @@ func (prc *ParsedRelabelConfig) expandCaptureGroups(template, source string, mat
 
 var relabelBufPool bytesutil.ByteBufferPool
 
+func areEqualLabelValues(labels []prompbmarshal.Label, labelNames []string) bool {
+	if len(labelNames) < 2 {
+		logger.Panicf("BUG: expecting at least 2 labelNames; got %d", len(labelNames))
+		return false
+	}
+	labelValue := GetLabelValueByName(labels, labelNames[0])
+	for _, labelName := range labelNames[1:] {
+		v := GetLabelValueByName(labels, labelName)
+		if v != labelValue {
+			return false
+		}
+	}
+	return true
+}
+
 func concatLabelValues(dst []byte, labels []prompbmarshal.Label, labelNames []string, separator string) []byte {
 	if len(labelNames) == 0 {
 		return dst

lib/promrelabel/relabel_test.go

@@ -336,6 +336,92 @@ func TestApplyRelabelConfigs(t *testing.T) {
 			},
 		})
 	})
+	t.Run("keep_if_equal-miss", func(t *testing.T) {
+		f([]ParsedRelabelConfig{
+			{
+				Action:       "keep_if_equal",
+				SourceLabels: []string{"foo", "bar"},
+			},
+		}, nil, true, nil)
+		f([]ParsedRelabelConfig{
+			{
+				Action:       "keep_if_equal",
+				SourceLabels: []string{"xxx", "bar"},
+			},
+		}, []prompbmarshal.Label{
+			{
+				Name:  "xxx",
+				Value: "yyy",
+			},
+		}, true, []prompbmarshal.Label{})
+	})
+	t.Run("keep_if_equal-hit", func(t *testing.T) {
+		f([]ParsedRelabelConfig{
+			{
+				Action:       "keep_if_equal",
+				SourceLabels: []string{"xxx", "bar"},
+			},
+		}, []prompbmarshal.Label{
+			{
+				Name:  "xxx",
+				Value: "yyy",
+			},
+			{
+				Name:  "bar",
+				Value: "yyy",
+			},
+		}, true, []prompbmarshal.Label{
+			{
+				Name:  "bar",
+				Value: "yyy",
+			},
+			{
+				Name:  "xxx",
+				Value: "yyy",
+			},
+		})
+	})
+	t.Run("drop_if_equal-miss", func(t *testing.T) {
+		f([]ParsedRelabelConfig{
+			{
+				Action:       "drop_if_equal",
+				SourceLabels: []string{"foo", "bar"},
+			},
+		}, nil, true, nil)
+		f([]ParsedRelabelConfig{
+			{
+				Action:       "drop_if_equal",
+				SourceLabels: []string{"xxx", "bar"},
+			},
+		}, []prompbmarshal.Label{
+			{
+				Name:  "xxx",
+				Value: "yyy",
+			},
+		}, true, []prompbmarshal.Label{
+			{
+				Name:  "xxx",
+				Value: "yyy",
+			},
+		})
+	})
+	t.Run("drop_if_equal-hit", func(t *testing.T) {
+		f([]ParsedRelabelConfig{
+			{
+				Action:       "drop_if_equal",
+				SourceLabels: []string{"xxx", "bar"},
+			},
+		}, []prompbmarshal.Label{
+			{
+				Name:  "xxx",
+				Value: "yyy",
+			},
+			{
+				Name:  "bar",
+				Value: "yyy",
+			},
+		}, true, []prompbmarshal.Label{})
+	})
 	t.Run("keep-miss", func(t *testing.T) {
 		f([]ParsedRelabelConfig{
 			{

lib/promrelabel/testdata/relabel_configs_valid.yml

@@ -18,3 +18,7 @@
 - action: labelmap_all
   regex: "\\."
   replacement: ":"
+- action: keep_if_equal
+  source_labels: [foo, bar]
+- action: drop_if_equal
+  source_labels: [foo, bar]