From 1c7f402598fc7d03f2f389be986a97175e515a8f Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sun, 21 Aug 2022 22:46:54 +0300 Subject: [PATCH] app/vmagent: add ability to construct a label from multiple existing labels by referring them in the `replacement` field during relabeling For example: - target_label: composite-label replacement: {{source_label1}}-{{source_label2}} --- app/vmagent/README.md | 128 +-- docs/CHANGELOG.md | 1 + docs/vmagent.md | 128 +-- lib/promrelabel/config.go | 8 +- lib/promrelabel/if_expression_test.go | 9 +- lib/promrelabel/relabel.go | 42 +- lib/promrelabel/relabel_test.go | 1377 +++---------------------- 7 files changed, 350 insertions(+), 1343 deletions(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 7ae2666cb3..efff2fb4d5 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -178,7 +178,7 @@ See [the list of supported service discovery types for Prometheus scrape targets ## scrape_config enhancements -`vmagent` supports the following additional options in `scrape_configs` section: +`vmagent` supports the following additional options in [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) section: * `headers` - a list of HTTP headers to send to scrape target with each scrape request. This can be used when the scrape target needs custom authorization and authentication. For example: @@ -199,9 +199,12 @@ scrape_configs: * `relabel_debug: true` for enabling debug logging during relabeling of the discovered targets. See [these docs](#relabeling). * `metric_relabel_debug: true` for enabling debug logging during relabeling of the scraped metrics. See [these docs](#relabeling). +See [scrape_configs docs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) for more details on all the supported options. + + ## Loading scrape configs from multiple files -`vmagent` supports loading scrape configs from multiple files specified in the `scrape_config_files` section of `-promscrape.config` file. For example, the following `-promscrape.config` instructs `vmagent` loading scrape configs from all the `*.yml` files under `configs` directory, from `single_scrape_config.yml` local file and from `https://config-server/scrape_config.yml` url: +`vmagent` supports loading [scrape configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) from multiple files specified in the `scrape_config_files` section of `-promscrape.config` file. For example, the following `-promscrape.config` instructs `vmagent` loading scrape configs from all the `*.yml` files under `configs` directory, from `single_scrape_config.yml` local file and from `https://config-server/scrape_config.yml` url: ```yml scrape_config_files: @@ -210,7 +213,7 @@ scrape_config_files: - https://config-server/scrape_config.yml ``` -Every referred file can contain arbitrary number of [supported scrape configs](#how-to-collect-metrics-in-prometheus-format). There is no need in specifying top-level `scrape_configs` section in these files. For example: +Every referred file can contain arbitrary number of [supported scrape configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). There is no need in specifying top-level `scrape_configs` section in these files. For example: ```yml - job_name: foo @@ -338,85 +341,94 @@ The following articles contain useful information about Prometheus relabeling: ## Relabeling enhancements -VictoriaMetrics provides the following additional relabeling actions on top of standard actions from the [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config): +* The `replacement` option can refer arbitrary labels via {% raw %}`{{label_name}}`{% endraw %} placeholders. Such placeholders are substituted with the corresponding label value. For example, the following relabeling rule sets `instance-job` label value to `host123-foo` when applied to the metric with `{instance="host123",job="foo"}` labels: -* `replace_all` replaces all of the occurrences of `regex` in the values of `source_labels` with the `replacement` and stores the results in the `target_label`. For example, the following relabeling config replaces all the occurrences of `-` char in metric names with `_` char (e.g. `foo-bar-baz` metric name is transformed into `foo_bar_baz`): + {% raw %} + ```yaml + - target_label: "instance-job" + replacement: "{{instance}}-{{job}}" + ``` + {% endraw %} + +* An optional `if` filter can be used for conditional relabeling. The `if` filter may contain arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). For example, the following relabeling rule drops metrics, which don't match `foo{bar="baz"}` series selector, while leaving the rest of metrics: ```yaml - - action: replace_all - source_labels: ["__name__"] - target_label: "__name__" - regex: "-" - replacement: "_" + - action: keep + if: 'foo{bar="baz"}' ``` -* `labelmap_all` replaces all of the occurrences of `regex` in all the label names with the `replacement`. For example, the following relabeling config replaces all the occurrences of `-` char in all the label names with `_` char (e.g. `foo-bar-baz` label name is transformed into `foo_bar_baz`): + This is equivalent to less clear Prometheus-compatible relabeling rule: ```yaml - - action: labelmap_all - regex: "-" - replacement: "_" + - action: keep + source_labels: [__name__, bar] + regex: 'foo;baz' ``` -* `keep_if_equal`: keeps the entry if all the label values from `source_labels` are equal, while dropping all the other entries. For example, the following relabeling config keeps targets if they contain equal values for `instance` and `host` labels, while dropping all the other targets: - - ```yaml - - action: keep_if_equal - source_labels: ["instance", "host"] - ``` - -* `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal, while keeping all the other entries. For example, the following relabeling config drops targets if they contain equal values for `instance` and `host` labels, while keeping all the other targets: - - ```yaml - - action: drop_if_equal - source_labels: ["instance", "host"] - ``` - -* `keep_metrics`: keeps all the metrics with names matching the given `regex`, while dropping all the other metrics. For example, the following relabeling config keeps metrics with `fo` and `bar` names, while dropping all the other metrics: +* The `regex` value can be split into multiple lines for improved readability and maintainability. These lines are automatically joined with `|` char when parsed. For example, the following configs are equivalent: ```yaml - action: keep_metrics - regex: "foo|bar" + regex: "metric_a|metric_b|foo_.+" ``` -* `drop_metrics`: drops all the metrics with names matching the given `regex`, while keeping all the other metrics. For example, the following relabeling config drops metrics with `foo` and `bar` names, while leaving all the other metrics: - ```yaml - - action: drop_metrics - regex: "foo|bar" + - action: keep_metrics + regex: + - "metric_a" + - "metric_b" + - "foo_.+" ``` -* `graphite`: applies Graphite-style relabeling to metric name. See [these docs](#graphite-relabeling) for details. +* VictoriaMetrics provides the following additional relabeling actions on top of standard actions from the [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config): -The `regex` value can be split into multiple lines for improved readability and maintainability. These lines are automatically joined with `|` char when parsed. For example, the following configs are equivalent: + * `replace_all` replaces all of the occurrences of `regex` in the values of `source_labels` with the `replacement` and stores the results in the `target_label`. For example, the following relabeling config replaces all the occurrences of `-` char in metric names with `_` char (e.g. `foo-bar-baz` metric name is transformed into `foo_bar_baz`): -```yaml -- action: keep_metrics - regex: "metric_a|metric_b|foo_.+" -``` + ```yaml + - action: replace_all + source_labels: ["__name__"] + target_label: "__name__" + regex: "-" + replacement: "_" + ``` -```yaml -- action: keep_metrics - regex: - - "metric_a" - - "metric_b" - - "foo_.+" -``` + * `labelmap_all` replaces all of the occurrences of `regex` in all the label names with the `replacement`. For example, the following relabeling config replaces all the occurrences of `-` char in all the label names with `_` char (e.g. `foo-bar-baz` label name is transformed into `foo_bar_baz`): -VictoriaMetrics components support an optional `if` filter in relabeling configs, which can be used for conditional relabeling. The `if` filter may contain arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). For example, the following relabeling rule drops metrics, which don't match `foo{bar="baz"}` series selector, while leaving the rest of metrics: + ```yaml + - action: labelmap_all + regex: "-" + replacement: "_" + ``` -```yaml -- action: keep - if: 'foo{bar="baz"}' -``` + * `keep_if_equal`: keeps the entry if all the label values from `source_labels` are equal, while dropping all the other entries. For example, the following relabeling config keeps targets if they contain equal values for `instance` and `host` labels, while dropping all the other targets: -This is equivalent to less clear Prometheus-compatible relabeling rule: + ```yaml + - action: keep_if_equal + source_labels: ["instance", "host"] + ``` -```yaml -- action: keep - source_labels: [__name__, bar] - regex: 'foo;baz' -``` + * `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal, while keeping all the other entries. For example, the following relabeling config drops targets if they contain equal values for `instance` and `host` labels, while keeping all the other targets: + + ```yaml + - action: drop_if_equal + source_labels: ["instance", "host"] + ``` + + * `keep_metrics`: keeps all the metrics with names matching the given `regex`, while dropping all the other metrics. For example, the following relabeling config keeps metrics with `fo` and `bar` names, while dropping all the other metrics: + + ```yaml + - action: keep_metrics + regex: "foo|bar" + ``` + + * `drop_metrics`: drops all the metrics with names matching the given `regex`, while keeping all the other metrics. For example, the following relabeling config drops metrics with `foo` and `bar` names, while leaving all the other metrics: + + ```yaml + - action: drop_metrics + regex: "foo|bar" + ``` + + * `graphite`: applies Graphite-style relabeling to metric name. See [these docs](#graphite-relabeling) for details. ## Graphite relabeling diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 39fecebaad..787376239c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -24,6 +24,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: return shorter error messages to Grafana and to other clients requesting [/api/v1/query](https://docs.victoriametrics.com/keyConcepts.html#instant-query) and [/api/v1/query_range](https://docs.victoriametrics.com/keyConcepts.html#range-query) endpoints. This should simplify reading these errors by humans. The long error message with full context is still written to logs. * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): reduce the amounts of logging at `vmstorage` when `vmselect` connects/disconnects to `vmstorage`. * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): improve performance for heavy queries on systems with many CPU cores. +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add ability to use {% raw %}`{{label_name}}`{% endraw %} placeholders in the `replacement` option of relabeling rules. This simplifies constructing label values from multiple existing label values. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling-enhancements) for details. * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): generate additional per-target metrics - `scrape_series_limit`, `scrape_series_current` and `scrape_series_limit_samples_dropped` if series limit is set according to [these docs](https://docs.victoriametrics.com/vmagent.html#cardinality-limiter). This simplifies alerting on targets with the exceeded series limit. See [these docs](https://docs.victoriametrics.com/vmagent.html#automatically-generated-metrics) for details on these metrics. * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for MX record types in [dns_sd_configs](https://docs.victoriametrics.com/sd_configs.html#dns_sd_configs) in the same way as Prometheus 2.38 [does](https://github.com/prometheus/prometheus/pull/10099). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `__meta_kubernetes_service_port_number` meta-label for `role: service` in [kubernetes_sd_configs](https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs) in the same way as Prometheus 2.38 [does](https://github.com/prometheus/prometheus/pull/11002). diff --git a/docs/vmagent.md b/docs/vmagent.md index 510a1a99e2..ad20d4cd2d 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -182,7 +182,7 @@ See [the list of supported service discovery types for Prometheus scrape targets ## scrape_config enhancements -`vmagent` supports the following additional options in `scrape_configs` section: +`vmagent` supports the following additional options in [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) section: * `headers` - a list of HTTP headers to send to scrape target with each scrape request. This can be used when the scrape target needs custom authorization and authentication. For example: @@ -203,9 +203,12 @@ scrape_configs: * `relabel_debug: true` for enabling debug logging during relabeling of the discovered targets. See [these docs](#relabeling). * `metric_relabel_debug: true` for enabling debug logging during relabeling of the scraped metrics. See [these docs](#relabeling). +See [scrape_configs docs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) for more details on all the supported options. + + ## Loading scrape configs from multiple files -`vmagent` supports loading scrape configs from multiple files specified in the `scrape_config_files` section of `-promscrape.config` file. For example, the following `-promscrape.config` instructs `vmagent` loading scrape configs from all the `*.yml` files under `configs` directory, from `single_scrape_config.yml` local file and from `https://config-server/scrape_config.yml` url: +`vmagent` supports loading [scrape configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) from multiple files specified in the `scrape_config_files` section of `-promscrape.config` file. For example, the following `-promscrape.config` instructs `vmagent` loading scrape configs from all the `*.yml` files under `configs` directory, from `single_scrape_config.yml` local file and from `https://config-server/scrape_config.yml` url: ```yml scrape_config_files: @@ -214,7 +217,7 @@ scrape_config_files: - https://config-server/scrape_config.yml ``` -Every referred file can contain arbitrary number of [supported scrape configs](#how-to-collect-metrics-in-prometheus-format). There is no need in specifying top-level `scrape_configs` section in these files. For example: +Every referred file can contain arbitrary number of [supported scrape configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). There is no need in specifying top-level `scrape_configs` section in these files. For example: ```yml - job_name: foo @@ -342,85 +345,94 @@ The following articles contain useful information about Prometheus relabeling: ## Relabeling enhancements -VictoriaMetrics provides the following additional relabeling actions on top of standard actions from the [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config): +* The `replacement` option can refer arbitrary labels via {% raw %}`{{label_name}}`{% endraw %} placeholders. Such placeholders are substituted with the corresponding label value. For example, the following relabeling rule sets `instance-job` label value to `host123-foo` when applied to the metric with `{instance="host123",job="foo"}` labels: -* `replace_all` replaces all of the occurrences of `regex` in the values of `source_labels` with the `replacement` and stores the results in the `target_label`. For example, the following relabeling config replaces all the occurrences of `-` char in metric names with `_` char (e.g. `foo-bar-baz` metric name is transformed into `foo_bar_baz`): + {% raw %} + ```yaml + - target_label: "instance-job" + replacement: "{{instance}}-{{job}}" + ``` + {% endraw %} + +* An optional `if` filter can be used for conditional relabeling. The `if` filter may contain arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). For example, the following relabeling rule drops metrics, which don't match `foo{bar="baz"}` series selector, while leaving the rest of metrics: ```yaml - - action: replace_all - source_labels: ["__name__"] - target_label: "__name__" - regex: "-" - replacement: "_" + - action: keep + if: 'foo{bar="baz"}' ``` -* `labelmap_all` replaces all of the occurrences of `regex` in all the label names with the `replacement`. For example, the following relabeling config replaces all the occurrences of `-` char in all the label names with `_` char (e.g. `foo-bar-baz` label name is transformed into `foo_bar_baz`): + This is equivalent to less clear Prometheus-compatible relabeling rule: ```yaml - - action: labelmap_all - regex: "-" - replacement: "_" + - action: keep + source_labels: [__name__, bar] + regex: 'foo;baz' ``` -* `keep_if_equal`: keeps the entry if all the label values from `source_labels` are equal, while dropping all the other entries. For example, the following relabeling config keeps targets if they contain equal values for `instance` and `host` labels, while dropping all the other targets: - - ```yaml - - action: keep_if_equal - source_labels: ["instance", "host"] - ``` - -* `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal, while keeping all the other entries. For example, the following relabeling config drops targets if they contain equal values for `instance` and `host` labels, while keeping all the other targets: - - ```yaml - - action: drop_if_equal - source_labels: ["instance", "host"] - ``` - -* `keep_metrics`: keeps all the metrics with names matching the given `regex`, while dropping all the other metrics. For example, the following relabeling config keeps metrics with `fo` and `bar` names, while dropping all the other metrics: +* The `regex` value can be split into multiple lines for improved readability and maintainability. These lines are automatically joined with `|` char when parsed. For example, the following configs are equivalent: ```yaml - action: keep_metrics - regex: "foo|bar" + regex: "metric_a|metric_b|foo_.+" ``` -* `drop_metrics`: drops all the metrics with names matching the given `regex`, while keeping all the other metrics. For example, the following relabeling config drops metrics with `foo` and `bar` names, while leaving all the other metrics: - ```yaml - - action: drop_metrics - regex: "foo|bar" + - action: keep_metrics + regex: + - "metric_a" + - "metric_b" + - "foo_.+" ``` -* `graphite`: applies Graphite-style relabeling to metric name. See [these docs](#graphite-relabeling) for details. +* VictoriaMetrics provides the following additional relabeling actions on top of standard actions from the [Prometheus relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config): -The `regex` value can be split into multiple lines for improved readability and maintainability. These lines are automatically joined with `|` char when parsed. For example, the following configs are equivalent: + * `replace_all` replaces all of the occurrences of `regex` in the values of `source_labels` with the `replacement` and stores the results in the `target_label`. For example, the following relabeling config replaces all the occurrences of `-` char in metric names with `_` char (e.g. `foo-bar-baz` metric name is transformed into `foo_bar_baz`): -```yaml -- action: keep_metrics - regex: "metric_a|metric_b|foo_.+" -``` + ```yaml + - action: replace_all + source_labels: ["__name__"] + target_label: "__name__" + regex: "-" + replacement: "_" + ``` -```yaml -- action: keep_metrics - regex: - - "metric_a" - - "metric_b" - - "foo_.+" -``` + * `labelmap_all` replaces all of the occurrences of `regex` in all the label names with the `replacement`. For example, the following relabeling config replaces all the occurrences of `-` char in all the label names with `_` char (e.g. `foo-bar-baz` label name is transformed into `foo_bar_baz`): -VictoriaMetrics components support an optional `if` filter in relabeling configs, which can be used for conditional relabeling. The `if` filter may contain arbitrary [time series selector](https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors). For example, the following relabeling rule drops metrics, which don't match `foo{bar="baz"}` series selector, while leaving the rest of metrics: + ```yaml + - action: labelmap_all + regex: "-" + replacement: "_" + ``` -```yaml -- action: keep - if: 'foo{bar="baz"}' -``` + * `keep_if_equal`: keeps the entry if all the label values from `source_labels` are equal, while dropping all the other entries. For example, the following relabeling config keeps targets if they contain equal values for `instance` and `host` labels, while dropping all the other targets: -This is equivalent to less clear Prometheus-compatible relabeling rule: + ```yaml + - action: keep_if_equal + source_labels: ["instance", "host"] + ``` -```yaml -- action: keep - source_labels: [__name__, bar] - regex: 'foo;baz' -``` + * `drop_if_equal`: drops the entry if all the label values from `source_labels` are equal, while keeping all the other entries. For example, the following relabeling config drops targets if they contain equal values for `instance` and `host` labels, while keeping all the other targets: + + ```yaml + - action: drop_if_equal + source_labels: ["instance", "host"] + ``` + + * `keep_metrics`: keeps all the metrics with names matching the given `regex`, while dropping all the other metrics. For example, the following relabeling config keeps metrics with `fo` and `bar` names, while dropping all the other metrics: + + ```yaml + - action: keep_metrics + regex: "foo|bar" + ``` + + * `drop_metrics`: drops all the metrics with names matching the given `regex`, while keeping all the other metrics. For example, the following relabeling config drops metrics with `foo` and `bar` names, while leaving all the other metrics: + + ```yaml + - action: drop_metrics + regex: "foo|bar" + ``` + + * `graphite`: applies Graphite-style relabeling to metric name. See [these docs](#graphite-relabeling) for details. ## Graphite relabeling diff --git a/lib/promrelabel/config.go b/lib/promrelabel/config.go index 830cf7cfeb..bd31e99b17 100644 --- a/lib/promrelabel/config.go +++ b/lib/promrelabel/config.go @@ -343,8 +343,10 @@ func parseRelabelConfig(rc *RelabelConfig) (*parsedRelabelConfig, error) { graphiteMatchTemplate: graphiteMatchTemplate, graphiteLabelRules: graphiteLabelRules, - regexOriginal: regexOriginalCompiled, - hasCaptureGroupInTargetLabel: strings.Contains(targetLabel, "$"), - hasCaptureGroupInReplacement: strings.Contains(replacement, "$"), + regexOriginal: regexOriginalCompiled, + + hasCaptureGroupInTargetLabel: strings.Contains(targetLabel, "$"), + hasCaptureGroupInReplacement: strings.Contains(replacement, "$"), + hasLabelReferenceInReplacement: strings.Contains(replacement, "{{"), }, nil } diff --git a/lib/promrelabel/if_expression_test.go b/lib/promrelabel/if_expression_test.go index ce9f202908..18b654f4c5 100644 --- a/lib/promrelabel/if_expression_test.go +++ b/lib/promrelabel/if_expression_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "strings" "testing" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" @@ -188,6 +189,12 @@ func TestIfExpressionMismatch(t *testing.T) { } func parseMetricWithLabels(metricWithLabels string) ([]prompbmarshal.Label, error) { + stripDummyMetric := false + if strings.HasPrefix(metricWithLabels, "{") { + // Add a dummy metric name, since the parser needs it + metricWithLabels = "dummy_metric" + metricWithLabels + stripDummyMetric = true + } // add a value to metricWithLabels, so it could be parsed by prometheus protocol parser. s := metricWithLabels + " 123" var rows prometheus.Rows @@ -203,7 +210,7 @@ func parseMetricWithLabels(metricWithLabels string) ([]prompbmarshal.Label, erro } r := rows.Rows[0] var lfs []prompbmarshal.Label - if r.Metric != "" { + if !stripDummyMetric { lfs = append(lfs, prompbmarshal.Label{ Name: "__name__", Value: r.Metric, diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go index 9c0516639b..1d6a926229 100644 --- a/lib/promrelabel/relabel.go +++ b/lib/promrelabel/relabel.go @@ -28,9 +28,11 @@ type parsedRelabelConfig struct { graphiteMatchTemplate *graphiteMatchTemplate graphiteLabelRules []graphiteLabelRule - regexOriginal *regexp.Regexp - hasCaptureGroupInTargetLabel bool - hasCaptureGroupInReplacement bool + regexOriginal *regexp.Regexp + + hasCaptureGroupInTargetLabel bool + hasCaptureGroupInReplacement bool + hasLabelReferenceInReplacement bool } // String returns human-readable representation for prc. @@ -172,10 +174,16 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset return labels case "replace": // Store `replacement` at `target_label` if the `regex` matches `source_labels` joined with `separator` + replacement := prc.Replacement bb := relabelBufPool.Get() + if prc.hasLabelReferenceInReplacement { + // Fill {{labelName}} references in the replacement + bb.B = fillLabelReferences(bb.B[:0], replacement, labels[labelsOffset:]) + replacement = string(bb.B) + } bb.B = concatLabelValues(bb.B[:0], src, prc.SourceLabels, prc.Separator) if prc.Regex == defaultRegexForRelabelConfig && !prc.hasCaptureGroupInTargetLabel { - if prc.Replacement == "$1" { + if replacement == "$1" { // Fast path for the rule that copies source label values to destination: // - source_labels: [...] // target_label: foobar @@ -188,7 +196,7 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset // - target_label: foobar // replacement: something-here relabelBufPool.Put(bb) - labels = setLabelValue(labels, labelsOffset, prc.TargetLabel, prc.Replacement) + labels = setLabelValue(labels, labelsOffset, prc.TargetLabel, replacement) return labels } } @@ -203,7 +211,7 @@ func (prc *parsedRelabelConfig) apply(labels []prompbmarshal.Label, labelsOffset if prc.hasCaptureGroupInTargetLabel { nameStr = prc.expandCaptureGroups(nameStr, sourceStr, match) } - valueStr := prc.expandCaptureGroups(prc.Replacement, sourceStr, match) + valueStr := prc.expandCaptureGroups(replacement, sourceStr, match) relabelBufPool.Put(bb) return setLabelValue(labels, labelsOffset, nameStr, valueStr) case "replace_all": @@ -539,3 +547,25 @@ func labelsToString(labels []prompbmarshal.Label) string { b = append(b, '}') return string(b) } + +func fillLabelReferences(dst []byte, replacement string, labels []prompbmarshal.Label) []byte { + s := replacement + for len(s) > 0 { + n := strings.Index(s, "{{") + if n < 0 { + return append(dst, s...) + } + dst = append(dst, s[:n]...) + s = s[n+2:] + n = strings.Index(s, "}}") + if n < 0 { + dst = append(dst, "{{"...) + return append(dst, s...) + } + labelName := s[:n] + s = s[n+2:] + labelValue := GetLabelValueByName(labels, labelName) + dst = append(dst, labelValue...) + } + return dst +} diff --git a/lib/promrelabel/relabel_test.go b/lib/promrelabel/relabel_test.go index 68ec973bfc..52b501bc5b 100644 --- a/lib/promrelabel/relabel_test.go +++ b/lib/promrelabel/relabel_test.go @@ -1,7 +1,6 @@ package promrelabel import ( - "reflect" "testing" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" @@ -55,96 +54,50 @@ func TestLabelsToString(t *testing.T) { } func TestApplyRelabelConfigs(t *testing.T) { - f := func(config string, labels []prompbmarshal.Label, isFinalize bool, resultExpected []prompbmarshal.Label) { + f := func(config, metric string, isFinalize bool, resultExpected string) { t.Helper() pcs, err := ParseRelabelConfigsData([]byte(config), false) if err != nil { t.Fatalf("cannot parse %q: %s", config, err) } - result := pcs.Apply(labels, 0, isFinalize) - if !reflect.DeepEqual(result, resultExpected) { - t.Fatalf("unexpected result; got\n%v\nwant\n%v", result, resultExpected) + labels, err := parseMetricWithLabels(metric) + if err != nil { + t.Fatalf("cannot parse %s: %s", metric, err) + } + resultLabels := pcs.Apply(labels, 0, isFinalize) + result := labelsToString(resultLabels) + if result != resultExpected { + t.Fatalf("unexpected result; got\n%s\nwant\n%s", result, resultExpected) } } t.Run("empty_relabel_configs", func(t *testing.T) { - f("", nil, false, nil) - f("", nil, true, nil) - f("", []prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - }, false, []prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - }) - f("", []prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - { - Name: "__name__", - Value: "xxx", - }, - { - Name: "__aaa", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "xxx", - }, - { - Name: "foo", - Value: "bar", - }, - }) + f("", `{}`, false, `{}`) + f("", `{}`, true, `{}`) + f("", `{foo="bar"}`, false, `{foo="bar"}`) + f("", `xxx{foo="bar",__aaa="yyy"}`, false, `xxx{__aaa="yyy",foo="bar"}`) + f("", `xxx{foo="bar",__aaa="yyy"}`, true, `xxx{foo="bar"}`) }) t.Run("replace-miss", func(t *testing.T) { f(` - action: replace target_label: bar -`, nil, false, []prompbmarshal.Label{}) +`, `{}`, false, `{}`) f(` - action: replace source_labels: ["foo"] target_label: bar -`, nil, false, []prompbmarshal.Label{}) +`, `{}`, false, `{}`) f(` - action: replace source_labels: ["foo"] target_label: "bar" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }, false, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy"}`, false, `{xxx="yyy"}`) f(` - action: replace source_labels: ["foo"] target_label: "bar" regex: ".+" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }, false, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy"}`, false, `{xxx="yyy"}`) }) t.Run("replace-if-miss", func(t *testing.T) { f(` @@ -153,17 +106,7 @@ func TestApplyRelabelConfigs(t *testing.T) { source_labels: ["xxx", "foo"] target_label: "bar" replacement: "a-$1-b" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }, false, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy"}`, false, `{xxx="yyy"}`) }) t.Run("replace-hit", func(t *testing.T) { f(` @@ -171,21 +114,7 @@ func TestApplyRelabelConfigs(t *testing.T) { source_labels: ["xxx", "foo"] target_label: "bar" replacement: "a-$1-b" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }, false, []prompbmarshal.Label{ - { - Name: "bar", - Value: "a-yyy;-b", - }, - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy"}`, false, `{bar="a-yyy;-b",xxx="yyy"}`) }) t.Run("replace-if-hit", func(t *testing.T) { f(` @@ -194,21 +123,7 @@ func TestApplyRelabelConfigs(t *testing.T) { source_labels: ["xxx", "foo"] target_label: "bar" replacement: "a-$1-b" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }, false, []prompbmarshal.Label{ - { - Name: "bar", - Value: "a-yyy;-b", - }, - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy"}`, false, `{bar="a-yyy;-b",xxx="yyy"}`) }) t.Run("replace-remove-label-value-hit", func(t *testing.T) { f(` @@ -217,21 +132,7 @@ func TestApplyRelabelConfigs(t *testing.T) { target_label: "foo" regex: "xxx" replacement: "" -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "xxx", - }, - { - Name: "bar", - Value: "baz", - }, - }, false, []prompbmarshal.Label{ - { - Name: "bar", - Value: "baz", - }, - }) +`, `{foo="xxx",bar="baz"}`, false, `{bar="baz"}`) }) t.Run("replace-remove-label-value-miss", func(t *testing.T) { f(` @@ -240,25 +141,7 @@ func TestApplyRelabelConfigs(t *testing.T) { target_label: "foo" regex: "xxx" replacement: "" -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - { - Name: "bar", - Value: "baz", - }, - }, false, []prompbmarshal.Label{ - { - Name: "bar", - Value: "baz", - }, - { - Name: "foo", - Value: "yyy", - }, - }) +`, `{foo="yyy",bar="baz"}`, false, `{bar="baz",foo="yyy"}`) }) t.Run("replace-hit-remove-label", func(t *testing.T) { f(` @@ -267,21 +150,7 @@ func TestApplyRelabelConfigs(t *testing.T) { regex: "yyy;.+" target_label: "foo" replacement: "" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - { - Name: "foo", - Value: "bar", - }, - }, false, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy",foo="bar"}`, false, `{xxx="yyy"}`) }) t.Run("replace-miss-remove-label", func(t *testing.T) { f(` @@ -290,25 +159,7 @@ func TestApplyRelabelConfigs(t *testing.T) { regex: "yyy;.+" target_label: "foo" replacement: "" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyyz", - }, - { - Name: "foo", - Value: "bar", - }, - }, false, []prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - { - Name: "xxx", - Value: "yyyz", - }, - }) +`, `{xxx="yyyz",foo="bar"}`, false, `{foo="bar",xxx="yyyz"}`) }) t.Run("replace-hit-target-label-with-capture-group", func(t *testing.T) { f(` @@ -316,64 +167,30 @@ func TestApplyRelabelConfigs(t *testing.T) { source_labels: ["xxx", "foo"] target_label: "bar-$1" replacement: "a-$1-b" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }, false, []prompbmarshal.Label{ - { - Name: "bar-yyy;", - Value: "a-yyy;-b", - }, - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy"}`, false, `{bar-yyy;="a-yyy;-b",xxx="yyy"}`) }) t.Run("replace_all-miss", func(t *testing.T) { f(` - action: replace_all source_labels: [foo] target_label: "bar" -`, nil, false, []prompbmarshal.Label{}) +`, `{}`, false, `{}`) f(` - action: replace_all source_labels: ["foo"] target_label: "bar" -`, nil, false, []prompbmarshal.Label{}) +`, `{}`, false, `{}`) f(` - action: replace_all source_labels: ["foo"] target_label: "bar" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }, false, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy"}`, false, `{xxx="yyy"}`) f(` - action: replace_all source_labels: ["foo"] target_label: "bar" regex: ".+" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }, false, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy"}`, false, `{xxx="yyy"}`) }) t.Run("replace_all-if-miss", func(t *testing.T) { f(` @@ -383,17 +200,7 @@ func TestApplyRelabelConfigs(t *testing.T) { target_label: "xxx" regex: "-" replacement: "." -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "a-b-c", - }, - }, false, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "a-b-c", - }, - }) +`, `{xxx="a-b-c"}`, false, `{xxx="a-b-c"}`) }) t.Run("replace_all-hit", func(t *testing.T) { f(` @@ -402,17 +209,7 @@ func TestApplyRelabelConfigs(t *testing.T) { target_label: "xxx" regex: "-" replacement: "." -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "a-b-c", - }, - }, false, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "a.b.c", - }, - }) +`, `{xxx="a-b-c"}`, false, `{xxx="a.b.c"}`) }) t.Run("replace_all-if-hit", func(t *testing.T) { f(` @@ -422,17 +219,7 @@ func TestApplyRelabelConfigs(t *testing.T) { target_label: "xxx" regex: "-" replacement: "." -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "a-b-c", - }, - }, false, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "a.b.c", - }, - }) +`, `{xxx="a-b-c"}`, false, `{xxx="a.b.c"}`) }) t.Run("replace_all-regex-hit", func(t *testing.T) { f(` @@ -441,17 +228,7 @@ func TestApplyRelabelConfigs(t *testing.T) { target_label: "xxx" regex: "(;)" replacement: "-$1-" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "y;y", - }, - }, false, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "y-;-y-;-", - }, - }) +`, `{xxx="y;y"}`, false, `{xxx="y-;-y-;-"}`) }) t.Run("replace-add-multi-labels", func(t *testing.T) { f(` @@ -463,33 +240,7 @@ func TestApplyRelabelConfigs(t *testing.T) { source_labels: ["bar"] target_label: "zar" replacement: "b-$1" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - { - Name: "instance", - Value: "a.bc", - }, - }, true, []prompbmarshal.Label{ - { - Name: "bar", - Value: "a-yyy", - }, - { - Name: "instance", - Value: "a.bc", - }, - { - Name: "xxx", - Value: "yyy", - }, - { - Name: "zar", - Value: "b-a-yyy", - }, - }) +`, `{xxx="yyy",instance="a.bc"}`, true, `{bar="a-yyy",instance="a.bc",xxx="yyy",zar="b-a-yyy"}`) }) t.Run("replace-self", func(t *testing.T) { f(` @@ -497,181 +248,84 @@ func TestApplyRelabelConfigs(t *testing.T) { source_labels: ["foo"] target_label: "foo" replacement: "a-$1" -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "aaxx", - }, - }, true, []prompbmarshal.Label{ - { - Name: "foo", - Value: "a-aaxx", - }, - }) +`, `{foo="aaxx"}`, true, `{foo="a-aaxx"}`) }) t.Run("replace-missing-source", func(t *testing.T) { f(` - action: replace target_label: foo replacement: "foobar" -`, []prompbmarshal.Label{}, true, []prompbmarshal.Label{ - { - Name: "foo", - Value: "foobar", - }, - }) +`, `{}`, true, `{foo="foobar"}`) }) t.Run("keep_if_equal-miss", func(t *testing.T) { f(` - action: keep_if_equal source_labels: ["foo", "bar"] -`, nil, true, nil) +`, `{}`, true, `{}`) f(` - action: keep_if_equal source_labels: ["xxx", "bar"] -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{}) +`, `{xxx="yyy"}`, true, `{}`) }) t.Run("keep_if_equal-hit", func(t *testing.T) { f(` - action: keep_if_equal source_labels: ["xxx", "bar"] -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - { - Name: "bar", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{ - { - Name: "bar", - Value: "yyy", - }, - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy",bar="yyy"}`, true, `{bar="yyy",xxx="yyy"}`) }) t.Run("drop_if_equal-miss", func(t *testing.T) { f(` - action: drop_if_equal source_labels: ["foo", "bar"] -`, nil, true, nil) +`, `{}`, true, `{}`) f(` - action: drop_if_equal source_labels: ["xxx", "bar"] -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy"}`, true, `{xxx="yyy"}`) }) t.Run("drop_if_equal-hit", func(t *testing.T) { f(` - action: drop_if_equal source_labels: [xxx, bar] -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - { - Name: "bar", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{}) +`, `{xxx="yyy",bar="yyy"}`, true, `{}`) }) t.Run("keep-miss", func(t *testing.T) { f(` - action: keep source_labels: [foo] regex: ".+" -`, nil, true, nil) +`, `{}`, true, `{}`) f(` - action: keep source_labels: [foo] regex: ".+" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{}) +`, `{xxx="yyy"}`, true, `{}`) }) t.Run("keep-if-miss", func(t *testing.T) { f(` - action: keep if: '{foo="bar"}' -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }, false, []prompbmarshal.Label{}) +`, `{foo="yyy"}`, false, `{}`) }) t.Run("keep-if-hit", func(t *testing.T) { f(` - action: keep if: '{foo="yyy"}' -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }, false, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }) +`, `{foo="yyy"}`, false, `{foo="yyy"}`) }) t.Run("keep-hit", func(t *testing.T) { f(` - action: keep source_labels: [foo] regex: "yyy" -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }, false, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }) +`, `{foo="yyy"}`, false, `{foo="yyy"}`) }) t.Run("keep-hit-regexp", func(t *testing.T) { f(` - action: keep source_labels: ["foo"] regex: ".+" -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }, false, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }) +`, `{foo="yyy"}`, false, `{foo="yyy"}`) }) t.Run("keep_metrics-miss", func(t *testing.T) { f(` @@ -679,39 +333,19 @@ func TestApplyRelabelConfigs(t *testing.T) { regex: - foo - bar -`, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "xxx", - }, - }, true, []prompbmarshal.Label{}) +`, `xxx`, true, `{}`) }) t.Run("keep_metrics-if-miss", func(t *testing.T) { f(` - action: keep_metrics if: 'bar' -`, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "foo", - }, - }, true, []prompbmarshal.Label{}) +`, `foo`, true, `{}`) }) t.Run("keep_metrics-if-hit", func(t *testing.T) { f(` - action: keep_metrics if: 'foo' -`, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "foo", - }, - }, true, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "foo", - }, - }) +`, `foo`, true, `foo`) }) t.Run("keep_metrics-hit", func(t *testing.T) { f(` @@ -719,90 +353,45 @@ func TestApplyRelabelConfigs(t *testing.T) { regex: - foo - bar -`, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "foo", - }, - }, true, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "foo", - }, - }) +`, `foo`, true, `foo`) }) t.Run("drop-miss", func(t *testing.T) { f(` - action: drop source_labels: [foo] regex: ".+" -`, nil, false, nil) +`, `{}`, false, `{}`) f(` - action: drop source_labels: [foo] regex: ".+" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy"}`, true, `{xxx="yyy"}`) }) t.Run("drop-if-miss", func(t *testing.T) { f(` - action: drop if: '{foo="bar"}' -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }) +`, `{foo="yyy"}`, true, `{foo="yyy"}`) }) t.Run("drop-if-hit", func(t *testing.T) { f(` - action: drop if: '{foo="yyy"}' -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{}) +`, `{foo="yyy"}`, true, `{}`) }) t.Run("drop-hit", func(t *testing.T) { f(` - action: drop source_labels: [foo] regex: yyy -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{}) +`, `{foo="yyy"}`, true, `{}`) }) t.Run("drop-hit-regexp", func(t *testing.T) { f(` - action: drop source_labels: [foo] regex: ".+" -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{}) +`, `{foo="yyy"}`, true, `{}`) }) t.Run("drop_metrics-miss", func(t *testing.T) { f(` @@ -810,44 +399,19 @@ func TestApplyRelabelConfigs(t *testing.T) { regex: - foo - bar -`, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "xxx", - }, - }, true, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "xxx", - }, - }) +`, `xxx`, true, `xxx`) }) t.Run("drop_metrics-if-miss", func(t *testing.T) { f(` - action: drop_metrics if: bar -`, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "foo", - }, - }, true, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "foo", - }, - }) +`, `foo`, true, `foo`) }) t.Run("drop_metrics-if-hit", func(t *testing.T) { f(` - action: drop_metrics if: foo -`, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "foo", - }, - }, true, []prompbmarshal.Label{}) +`, `foo`, true, `{}`) }) t.Run("drop_metrics-hit", func(t *testing.T) { f(` @@ -855,12 +419,7 @@ func TestApplyRelabelConfigs(t *testing.T) { regex: - foo - bar -`, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "foo", - }, - }, true, []prompbmarshal.Label{}) +`, `foo`, true, `{}`) }) t.Run("hashmod-miss", func(t *testing.T) { f(` @@ -868,21 +427,7 @@ func TestApplyRelabelConfigs(t *testing.T) { source_labels: [foo] target_label: aaa modulus: 123 -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }, false, []prompbmarshal.Label{ - { - Name: "aaa", - Value: "81", - }, - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy"}`, false, `{aaa="81",xxx="yyy"}`) }) t.Run("hashmod-if-miss", func(t *testing.T) { f(` @@ -891,17 +436,7 @@ func TestApplyRelabelConfigs(t *testing.T) { source_labels: [foo] target_label: aaa modulus: 123 -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }) +`, `{foo="yyy"}`, true, `{foo="yyy"}`) }) t.Run("hashmod-if-hit", func(t *testing.T) { f(` @@ -910,21 +445,7 @@ func TestApplyRelabelConfigs(t *testing.T) { source_labels: [foo] target_label: aaa modulus: 123 -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{ - { - Name: "aaa", - Value: "73", - }, - { - Name: "foo", - Value: "yyy", - }, - }) +`, `{foo="yyy"}`, true, `{aaa="73",foo="yyy"}`) }) t.Run("hashmod-hit", func(t *testing.T) { f(` @@ -932,21 +453,7 @@ func TestApplyRelabelConfigs(t *testing.T) { source_labels: [foo] target_label: aaa modulus: 123 -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - }, true, []prompbmarshal.Label{ - { - Name: "aaa", - Value: "73", - }, - { - Name: "foo", - Value: "yyy", - }, - }) +`, `{foo="yyy"}`, true, `{aaa="73",foo="yyy"}`) }) t.Run("labelmap-copy-label-if-miss", func(t *testing.T) { f(` @@ -954,25 +461,7 @@ func TestApplyRelabelConfigs(t *testing.T) { if: '{foo="yyy",foobar="aab"}' regex: "foo" replacement: "bar" -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }, true, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }) +`, `{foo="yyy",foobar="aaa"}`, true, `{foo="yyy",foobar="aaa"}`) }) t.Run("labelmap-copy-label-if-hit", func(t *testing.T) { f(` @@ -980,143 +469,33 @@ func TestApplyRelabelConfigs(t *testing.T) { if: '{foo="yyy",foobar="aaa"}' regex: "foo" replacement: "bar" -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }, true, []prompbmarshal.Label{ - { - Name: "bar", - Value: "yyy", - }, - { - Name: "foo", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }) +`, `{foo="yyy",foobar="aaa"}`, true, `{bar="yyy",foo="yyy",foobar="aaa"}`) }) t.Run("labelmap-copy-label", func(t *testing.T) { f(` - action: labelmap regex: "foo" replacement: "bar" -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }, true, []prompbmarshal.Label{ - { - Name: "bar", - Value: "yyy", - }, - { - Name: "foo", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }) +`, `{foo="yyy",foobar="aaa"}`, true, `{bar="yyy",foo="yyy",foobar="aaa"}`) }) t.Run("labelmap-remove-prefix-dot-star", func(t *testing.T) { f(` - action: labelmap regex: "foo(.*)" -`, []prompbmarshal.Label{ - { - Name: "xoo", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }, true, []prompbmarshal.Label{ - { - Name: "bar", - Value: "aaa", - }, - { - Name: "foobar", - Value: "aaa", - }, - { - Name: "xoo", - Value: "yyy", - }, - }) +`, `{xoo="yyy",foobar="aaa"}`, true, `{bar="aaa",foobar="aaa",xoo="yyy"}`) }) t.Run("labelmap-remove-prefix-dot-plus", func(t *testing.T) { f(` - action: labelmap regex: "foo(.+)" -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }, true, []prompbmarshal.Label{ - { - Name: "bar", - Value: "aaa", - }, - { - Name: "foo", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }) +`, `{foo="yyy",foobar="aaa"}`, true, `{bar="aaa",foo="yyy",foobar="aaa"}`) }) t.Run("labelmap-regex", func(t *testing.T) { f(` - action: labelmap regex: "foo(.+)" replacement: "$1-x" -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }, true, []prompbmarshal.Label{ - { - Name: "bar-x", - Value: "aaa", - }, - { - Name: "foo", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }) +`, `{foo="yyy",foobar="aaa"}`, true, `{bar-x="aaa",foo="yyy",foobar="aaa"}`) }) t.Run("labelmap_all-if-miss", func(t *testing.T) { f(` @@ -1124,25 +503,7 @@ func TestApplyRelabelConfigs(t *testing.T) { if: foobar regex: "\\." replacement: "-" -`, []prompbmarshal.Label{ - { - Name: "foo.bar.baz", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }, true, []prompbmarshal.Label{ - { - Name: "foo.bar.baz", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }) +`, `{foo.bar.baz="yyy",foobar="aaa"}`, true, `{foo.bar.baz="yyy",foobar="aaa"}`) }) t.Run("labelmap_all-if-hit", func(t *testing.T) { f(` @@ -1150,473 +511,119 @@ func TestApplyRelabelConfigs(t *testing.T) { if: '{foo.bar.baz="yyy"}' regex: "\\." replacement: "-" -`, []prompbmarshal.Label{ - { - Name: "foo.bar.baz", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }, true, []prompbmarshal.Label{ - { - Name: "foo-bar-baz", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }) +`, `{foo.bar.baz="yyy",foobar="aaa"}`, true, `{foo-bar-baz="yyy",foobar="aaa"}`) }) t.Run("labelmap_all", func(t *testing.T) { f(` - action: labelmap_all regex: "\\." replacement: "-" -`, []prompbmarshal.Label{ - { - Name: "foo.bar.baz", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }, true, []prompbmarshal.Label{ - { - Name: "foo-bar-baz", - Value: "yyy", - }, - { - Name: "foobar", - Value: "aaa", - }, - }) +`, `{foo.bar.baz="yyy",foobar="aaa"}`, true, `{foo-bar-baz="yyy",foobar="aaa"}`) }) t.Run("labelmap_all-regexp", func(t *testing.T) { f(` - action: labelmap_all regex: "ba(.)" replacement: "${1}ss" -`, []prompbmarshal.Label{ - { - Name: "foo.bar.baz", - Value: "yyy", - }, - { - Name: "foozar", - Value: "aaa", - }, - }, true, []prompbmarshal.Label{ - { - Name: "foo.rss.zss", - Value: "yyy", - }, - { - Name: "foozar", - Value: "aaa", - }, - }) +`, `{foo.bar.baz="yyy",foozar="aaa"}`, true, `{foo.rss.zss="yyy",foozar="aaa"}`) }) t.Run("labeldrop", func(t *testing.T) { f(` - action: labeldrop regex: dropme -`, []prompbmarshal.Label{ - { - Name: "aaa", - Value: "bbb", - }, - }, true, []prompbmarshal.Label{ - { - Name: "aaa", - Value: "bbb", - }, - }) +`, `{aaa="bbb"}`, true, `{aaa="bbb"}`) // if-miss f(` - action: labeldrop if: foo regex: dropme -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - { - Name: "dropme", - Value: "aaa", - }, - { - Name: "foo", - Value: "bar", - }, - }, false, []prompbmarshal.Label{ - { - Name: "dropme", - Value: "aaa", - }, - { - Name: "foo", - Value: "bar", - }, - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy",dropme="aaa",foo="bar"}`, false, `{dropme="aaa",foo="bar",xxx="yyy"}`) // if-hit f(` - action: labeldrop if: '{xxx="yyy"}' regex: dropme -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - { - Name: "dropme", - Value: "aaa", - }, - { - Name: "foo", - Value: "bar", - }, - }, false, []prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy",dropme="aaa",foo="bar"}`, false, `{foo="bar",xxx="yyy"}`) f(` - action: labeldrop regex: dropme -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - { - Name: "dropme", - Value: "aaa", - }, - { - Name: "foo", - Value: "bar", - }, - }, false, []prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy",dropme="aaa",foo="bar"}`, false, `{foo="bar",xxx="yyy"}`) // regex in single quotes f(` - action: labeldrop regex: 'dropme' -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - { - Name: "dropme", - Value: "aaa", - }, - }, false, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy",dropme="aaa"}`, false, `{xxx="yyy"}`) // regex in double quotes f(` - action: labeldrop regex: "dropme" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - { - Name: "dropme", - Value: "aaa", - }, - }, false, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy",dropme="aaa"}`, false, `{xxx="yyy"}`) }) t.Run("labeldrop-prefix", func(t *testing.T) { f(` - action: labeldrop regex: "dropme.*" -`, []prompbmarshal.Label{ - { - Name: "aaa", - Value: "bbb", - }, - }, true, []prompbmarshal.Label{ - { - Name: "aaa", - Value: "bbb", - }, - }) +`, `{aaa="bbb"}`, true, `{aaa="bbb"}`) f(` - action: labeldrop regex: "dropme(.+)" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - { - Name: "dropme-please", - Value: "aaa", - }, - { - Name: "foo", - Value: "bar", - }, - }, false, []prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy",dropme-please="aaa",foo="bar"}`, false, `{foo="bar",xxx="yyy"}`) }) t.Run("labeldrop-regexp", func(t *testing.T) { f(` - action: labeldrop regex: ".*dropme.*" -`, []prompbmarshal.Label{ - { - Name: "aaa", - Value: "bbb", - }, - }, true, []prompbmarshal.Label{ - { - Name: "aaa", - Value: "bbb", - }, - }) +`, `{aaa="bbb"}`, true, `{aaa="bbb"}`) f(` - action: labeldrop regex: ".*dropme.*" -`, []prompbmarshal.Label{ - { - Name: "xxx", - Value: "yyy", - }, - { - Name: "dropme-please", - Value: "aaa", - }, - { - Name: "foo", - Value: "bar", - }, - }, false, []prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - { - Name: "xxx", - Value: "yyy", - }, - }) +`, `{xxx="yyy",dropme-please="aaa",foo="bar"}`, false, `{foo="bar",xxx="yyy"}`) }) t.Run("labelkeep", func(t *testing.T) { f(` - action: labelkeep regex: "keepme" -`, []prompbmarshal.Label{ - { - Name: "keepme", - Value: "aaa", - }, - }, true, []prompbmarshal.Label{ - { - Name: "keepme", - Value: "aaa", - }, - }) +`, `{keepme="aaa"}`, true, `{keepme="aaa"}`) // if-miss f(` - action: labelkeep if: '{aaaa="awefx"}' regex: keepme -`, []prompbmarshal.Label{ - { - Name: "keepme", - Value: "aaa", - }, - { - Name: "aaaa", - Value: "awef", - }, - { - Name: "keepme-aaa", - Value: "234", - }, - }, false, []prompbmarshal.Label{ - { - Name: "aaaa", - Value: "awef", - }, - { - Name: "keepme", - Value: "aaa", - }, - { - Name: "keepme-aaa", - Value: "234", - }, - }) +`, `{keepme="aaa",aaaa="awef",keepme-aaa="234"}`, false, `{aaaa="awef",keepme="aaa",keepme-aaa="234"}`) // if-hit f(` - action: labelkeep if: '{aaaa="awef"}' regex: keepme -`, []prompbmarshal.Label{ - { - Name: "keepme", - Value: "aaa", - }, - { - Name: "aaaa", - Value: "awef", - }, - { - Name: "keepme-aaa", - Value: "234", - }, - }, false, []prompbmarshal.Label{ - { - Name: "keepme", - Value: "aaa", - }, - }) +`, `{keepme="aaa",aaaa="awef",keepme-aaa="234"}`, false, `{keepme="aaa"}`) f(` - action: labelkeep regex: keepme -`, []prompbmarshal.Label{ - { - Name: "keepme", - Value: "aaa", - }, - { - Name: "aaaa", - Value: "awef", - }, - { - Name: "keepme-aaa", - Value: "234", - }, - }, false, []prompbmarshal.Label{ - { - Name: "keepme", - Value: "aaa", - }, - }) +`, `{keepme="aaa",aaaa="awef",keepme-aaa="234"}`, false, `{keepme="aaa"}`) }) t.Run("labelkeep-regexp", func(t *testing.T) { f(` - action: labelkeep regex: "keepme.*" -`, []prompbmarshal.Label{ - { - Name: "keepme", - Value: "aaa", - }, - }, true, []prompbmarshal.Label{ - { - Name: "keepme", - Value: "aaa", - }, - }) +`, `{keepme="aaa"}`, true, `{keepme="aaa"}`) f(` - action: labelkeep regex: "keepme.*" -`, []prompbmarshal.Label{ - { - Name: "keepme", - Value: "aaa", - }, - { - Name: "aaaa", - Value: "awef", - }, - { - Name: "keepme-aaa", - Value: "234", - }, - }, false, []prompbmarshal.Label{ - { - Name: "keepme", - Value: "aaa", - }, - { - Name: "keepme-aaa", - Value: "234", - }, - }) +`, `{keepme="aaa",aaaa="awef",keepme-aaa="234"}`, false, `{keepme="aaa",keepme-aaa="234"}`) }) t.Run("upper-lower-case", func(t *testing.T) { f(` - action: uppercase source_labels: ["foo"] target_label: foo -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - }, true, []prompbmarshal.Label{ - { - Name: "foo", - Value: "BAR", - }, - }) +`, `{foo="bar"}`, true, `{foo="BAR"}`) f(` - action: lowercase source_labels: ["foo", "bar"] target_label: baz - action: labeldrop regex: foo|bar -`, []prompbmarshal.Label{ - { - Name: "foo", - Value: "BaR", - }, - { - Name: "bar", - Value: "fOO", - }, - }, true, []prompbmarshal.Label{ - { - Name: "baz", - Value: "bar;foo", - }, - }) +`, `{foo="BaR",bar="fOO"}`, true, `{baz="bar;foo"}`) f(` - action: lowercase source_labels: ["foo"] @@ -1624,17 +631,7 @@ func TestApplyRelabelConfigs(t *testing.T) { - action: uppercase source_labels: ["bar"] target_label: baz -`, []prompbmarshal.Label{ - { - Name: "qux", - Value: "quux", - }, - }, true, []prompbmarshal.Label{ - { - Name: "qux", - Value: "quux", - }, - }) +`, `{qux="quux"}`, true, `{qux="quux"}`) }) t.Run("graphite-match", func(t *testing.T) { f(` @@ -1643,21 +640,7 @@ func TestApplyRelabelConfigs(t *testing.T) { labels: __name__: aaa job: ${1}-zz -`, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "foo.bar.baz", - }, - }, true, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "aaa", - }, - { - Name: "job", - Value: "bar-zz", - }, - }) +`, `foo.bar.baz`, true, `aaa{job="bar-zz"}`) }) t.Run("graphite-mismatch", func(t *testing.T) { f(` @@ -1666,120 +649,80 @@ func TestApplyRelabelConfigs(t *testing.T) { labels: __name__: aaa job: ${1}-zz -`, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "foo.bar.bazz", - }, - }, true, []prompbmarshal.Label{ - { - Name: "__name__", - Value: "foo.bar.bazz", - }, - }) +`, `foo.bar.bazz`, true, `foo.bar.bazz`) + }) + t.Run("replacement-with-label-refs", func(t *testing.T) { + // no regex + f(` +- target_label: abc + replacement: "{{__name__}}.{{foo}}" +`, `qwe{foo="bar",baz="aaa"}`, true, `qwe{abc="qwe.bar",baz="aaa",foo="bar"}`) + // with regex + f(` +- target_label: abc + replacement: "{{__name__}}.{{foo}}.$1" + source_labels: [baz] + regex: "a(.+)" +`, `qwe{foo="bar",baz="aaa"}`, true, `qwe{abc="qwe.bar.aa",baz="aaa",foo="bar"}`) }) } func TestFinalizeLabels(t *testing.T) { - f := func(labels, resultExpected []prompbmarshal.Label) { + f := func(metric, resultExpected string) { t.Helper() - result := FinalizeLabels(nil, labels) - if !reflect.DeepEqual(result, resultExpected) { - t.Fatalf("unexpected result; got\n%v\nwant\n%v", result, resultExpected) + labels, err := parseMetricWithLabels(metric) + if err != nil { + t.Fatalf("cannot parse %s: %s", metric, err) + } + resultLabels := FinalizeLabels(nil, labels) + result := labelsToString(resultLabels) + if result != resultExpected { + t.Fatalf("unexpected result; got\n%s\nwant\n%s", result, resultExpected) } } - f(nil, nil) - f([]prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - { - Name: "__aaa", - Value: "ass", - }, - { - Name: "instance", - Value: "foo.com", - }, - }, []prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - { - Name: "instance", - Value: "foo.com", - }, - }) - f([]prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - { - Name: "instance", - Value: "ass", - }, - { - Name: "__address__", - Value: "foo.com", - }, - }, []prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - { - Name: "instance", - Value: "ass", - }, - }) + f(`{}`, `{}`) + f(`{foo="bar",__aaa="ass",instance="foo.com"}`, `{foo="bar",instance="foo.com"}`) + f(`{foo="bar",instance="ass",__address__="foo.com"}`, `{foo="bar",instance="ass"}`) + f(`{foo="bar",abc="def",__address__="foo.com"}`, `{abc="def",foo="bar"}`) } func TestRemoveMetaLabels(t *testing.T) { - f := func(labels, resultExpected []prompbmarshal.Label) { + f := func(metric, resultExpected string) { t.Helper() - result := RemoveMetaLabels(nil, labels) - if !reflect.DeepEqual(result, resultExpected) { - t.Fatalf("unexpected result of RemoveMetaLabels;\ngot\n%v\nwant\n%v", result, resultExpected) + labels, err := parseMetricWithLabels(metric) + if err != nil { + t.Fatalf("cannot parse %s: %s", metric, err) + } + resultLabels := RemoveMetaLabels(nil, labels) + result := labelsToString(resultLabels) + if result != resultExpected { + t.Fatalf("unexpected result of RemoveMetaLabels;\ngot\n%s\nwant\n%s", result, resultExpected) } } - f(nil, nil) - f([]prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - }, []prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - }) - f([]prompbmarshal.Label{ - { - Name: "__meta_foo", - Value: "bar", - }, - }, nil) - f([]prompbmarshal.Label{ - { - Name: "__meta_foo", - Value: "bdffr", - }, - { - Name: "foo", - Value: "bar", - }, - { - Name: "__meta_xxx", - Value: "basd", - }, - }, []prompbmarshal.Label{ - { - Name: "foo", - Value: "bar", - }, - }) + f(`{}`, `{}`) + f(`{foo="bar"}`, `{foo="bar"}`) + f(`{__meta_foo="bar"}`, `{}`) + f(`{__meta_foo="bdffr",foo="bar",__meta_xxx="basd"}`, `{foo="bar"}`) +} + +func TestFillLabelReferences(t *testing.T) { + f := func(replacement, metric, resultExpected string) { + t.Helper() + labels, err := parseMetricWithLabels(metric) + if err != nil { + t.Fatalf("cannot parse %s: %s", metric, err) + } + result := fillLabelReferences(nil, replacement, labels) + if string(result) != resultExpected { + t.Fatalf("unexpected result; got\n%q\nwant\n%q", result, resultExpected) + } + } + f(``, `foo{bar="baz"}`, ``) + f(`abc`, `foo{bar="baz"}`, `abc`) + f(`foo{{bar`, `foo{bar="baz"}`, `foo{{bar`) + f(`foo-$1`, `foo{bar="baz"}`, `foo-$1`) + f(`foo{{bar}}`, `foo{bar="baz"}`, `foobaz`) + f(`{{bar}}`, `foo{bar="baz"}`, `baz`) + f(`{{bar}}-aa`, `foo{bar="baz"}`, `baz-aa`) + f(`{{bar}}-aa{{__name__}}.{{bar}}{{non-existing-label}}`, `foo{bar="baz"}`, `baz-aafoo.baz`) }