diff --git a/README.md b/README.md index a08e5779e..d628f0226 100644 --- a/README.md +++ b/README.md @@ -1762,6 +1762,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed -relabelConfig string Optional path to a file with relabeling rules, which are applied to all the ingested metrics. See https://docs.victoriametrics.com/#relabeling for details + -relabelDebug + Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, then the metrics aren't sent to storage. This is useful for debugging the relabeling configs -retentionPeriod value Data with timestamps outside the retentionPeriod is automatically deleted The following optional suffixes are supported: h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 1) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 1ea295f63..2db301f19 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -219,10 +219,10 @@ and also provides the following actions: The relabeling can be defined in the following places: -* At the `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to target labels. -* At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to all the scraped metrics in the given `scrape_config`. -* At the `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage. -* At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. +* At the `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to target labels. This relabeling can be debugged by passing `relabel_debug: true` option to the corresponding `scrape_config` section. In this case `vmagent` logs target labels before and after the relabeling and then drops the logged target. +* At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to all the scraped metrics in the given `scrape_config`. This relabeling can be debugged by passing `metric_relabel_debug: true` option to the corresponding `scrape_config` section. In this case `vmagent` logs metrics before and after the relabeling and then drops the logged metrics. +* At the `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage. +* At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`. You can read more about relabeling in the following articles: @@ -721,6 +721,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Supports array of values separated by comma or specified via multiple flags. -remoteWrite.relabelConfig string Optional path to file with relabel_config entries. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details + -remoteWrite.relabelDebug + Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs -remoteWrite.roundDigits array Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics Supports array of values separated by comma or specified via multiple flags. @@ -755,6 +757,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . -remoteWrite.urlRelabelConfig array Optional path to relabel config for the corresponding -remoteWrite.url Supports an array of values separated by comma or specified via multiple flags. + -remoteWrite.urlRelabelDebug array + Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. This is useful for debugging the relabeling configs + Supports array of values separated by comma or specified via multiple flags. -sortLabels Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit -tls diff --git a/app/vmagent/remotewrite/relabel.go b/app/vmagent/remotewrite/relabel.go index ce30eaab9..f2e87847b 100644 --- a/app/vmagent/remotewrite/relabel.go +++ b/app/vmagent/remotewrite/relabel.go @@ -17,8 +17,12 @@ var ( "Pass multiple -remoteWrite.label flags in order to add multiple labels to metrics before sending them to remote storage") relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. These entries are applied to all the metrics "+ "before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details") + relabelDebugGlobal = flag.Bool("remoteWrite.relabelDebug", false, "Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. "+ + "If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs") relabelConfigPaths = flagutil.NewArray("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url") - relabelDebug = flag.Bool("remoteWrite.relabelDebug", false, "Show relabel results of via -remoteWrite.relabelConfig specified rules and skip its submission.") + relabelDebug = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+ + "If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. "+ + "This is useful for debugging the relabeling configs") ) var labelsGlobal []prompbmarshal.Label @@ -32,8 +36,7 @@ func CheckRelabelConfigs() error { func loadRelabelConfigs() (*relabelConfigs, error) { var rcs relabelConfigs if *relabelConfigPathGlobal != "" { - global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal) - global.RelabelDebug = *relabelDebug + global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal, *relabelDebugGlobal) if err != nil { return nil, fmt.Errorf("cannot load -remoteWrite.relabelConfig=%q: %w", *relabelConfigPathGlobal, err) } @@ -49,11 +52,10 @@ func loadRelabelConfigs() (*relabelConfigs, error) { // Skip empty relabel config. continue } - prc, err := promrelabel.LoadRelabelConfigs(path) + prc, err := promrelabel.LoadRelabelConfigs(path, relabelDebug.GetOptionalArg(i)) if err != nil { return nil, fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %w", path, err) } - prc.RelabelDebug = *relabelDebug rcs.perURL[i] = prc } return &rcs, nil @@ -103,11 +105,7 @@ func (rctx *relabelCtx) applyRelabeling(tss []prompbmarshal.TimeSeries, extraLab labels = append(labels, *extraLabel) } } - labels = pcs.Apply(labels, labelsLen, true, pcs.RelabelDebug) - if (pcs.RelabelDebug) { - // simulate "all labels dropped" to avoid submission - labels = labels[:labelsLen] - } + labels = pcs.Apply(labels, labelsLen, true) if len(labels) == labelsLen { // Drop the current time series, since relabeling removed all the labels. continue diff --git a/app/vminsert/relabel/relabel.go b/app/vminsert/relabel/relabel.go index 665f7c9ec..064ba8333 100644 --- a/app/vminsert/relabel/relabel.go +++ b/app/vminsert/relabel/relabel.go @@ -14,8 +14,12 @@ import ( "github.com/VictoriaMetrics/metrics" ) -var relabelConfig = flag.String("relabelConfig", "", "Optional path to a file with relabeling rules, which are applied to all the ingested metrics. "+ - "See https://docs.victoriametrics.com/#relabeling for details") +var ( + relabelConfig = flag.String("relabelConfig", "", "Optional path to a file with relabeling rules, which are applied to all the ingested metrics. "+ + "See https://docs.victoriametrics.com/#relabeling for details") + relabelDebug = flag.Bool("relabelDebug", false, "Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, "+ + "then the metrics aren't sent to storage. This is useful for debugging the relabeling configs") +) // Init must be called after flag.Parse and before using the relabel package. func Init() { @@ -52,7 +56,7 @@ func loadRelabelConfig() (*promrelabel.ParsedConfigs, error) { if len(*relabelConfig) == 0 { return nil, nil } - pcs, err := promrelabel.LoadRelabelConfigs(*relabelConfig) + pcs, err := promrelabel.LoadRelabelConfigs(*relabelConfig, *relabelDebug) if err != nil { return nil, fmt.Errorf("error when reading -relabelConfig=%q: %w", *relabelConfig, err) } @@ -101,7 +105,7 @@ func (ctx *Ctx) ApplyRelabeling(labels []prompb.Label) []prompb.Label { } // Apply relabeling - tmpLabels = pcs.Apply(tmpLabels, 0, true, pcs.RelabelDebug) + tmpLabels = pcs.Apply(tmpLabels, 0, true) ctx.tmpLabels = tmpLabels if len(tmpLabels) == 0 { metricsDropped.Inc() diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 240f03958..9071e1657 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -11,6 +11,7 @@ sort: 15 * FEATURE: vmauth: add ability to specify mutliple `url_prefix` entries for balancing the load among multiple `vmselect` and/or `vminsert` nodes in a cluster. See [these docs](https://docs.victoriametrics.com/vmauth.html#load-balancing). * FEATURE: vminsert: add `-disableRerouting` command-line flag for forcibly disabling the rerouting. This should help resolving [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/791) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1054) issues. * FEATURE: allow building VictoriaMetrics components for Solaris / SmartOS. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1322). +* FEATURE: vmagent: add ability to debug relabeling rules. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1343). * BUGFIX: reduce CPU usage by up to 2x during querying a database with big number of active daily time series. The issue has been introduced in `v1.59.0`. * BUGFIX: vmagent: properly apply auth and tls configs in `eureka_sd_configs`. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1350). diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 657bdf15b..65999e0d3 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -1766,6 +1766,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed -relabelConfig string Optional path to a file with relabeling rules, which are applied to all the ingested metrics. See https://docs.victoriametrics.com/#relabeling for details + -relabelDebug + Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, then the metrics aren't sent to storage. This is useful for debugging the relabeling configs -retentionPeriod value Data with timestamps outside the retentionPeriod is automatically deleted The following optional suffixes are supported: h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months (default 1) diff --git a/docs/vmagent.md b/docs/vmagent.md index d5d3bb35f..a82980747 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -223,11 +223,10 @@ and also provides the following actions: The relabeling can be defined in the following places: -* At the `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to target labels. -* At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to all the scraped metrics in the given `scrape_config`. -* By setting the `relabel_debug` property of a `scrape_configs` target section in the `-promscrape.config` file to `true`, one is able to instruct the vmagent to just log the metric before and after relabeling and skip its submission to servers. This way it is much easier to understand and check scraped relabeled metrics before poisoning servers with it. -* At the `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage. -* At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. +* At the `scrape_config -> relabel_configs` section in `-promscrape.config` file. This relabeling is applied to target labels. This relabeling can be debugged by passing `relabel_debug: true` option to the corresponding `scrape_config` section. In this case `vmagent` logs target labels before and after the relabeling and then drops the logged target. +* At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is applied to all the scraped metrics in the given `scrape_config`. This relabeling can be debugged by passing `metric_relabel_debug: true` option to the corresponding `scrape_config` section. In this case `vmagent` logs metrics before and after the relabeling and then drops the logged metrics. +* At the `-remoteWrite.relabelConfig` file. This relabeling is aplied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage. +* At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`. You can read more about relabeling in the following articles: @@ -726,6 +725,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Supports array of values separated by comma or specified via multiple flags. -remoteWrite.relabelConfig string Optional path to file with relabel_config entries. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details + -remoteWrite.relabelDebug + Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs -remoteWrite.roundDigits array Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics Supports array of values separated by comma or specified via multiple flags. @@ -760,6 +761,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . -remoteWrite.urlRelabelConfig array Optional path to relabel config for the corresponding -remoteWrite.url Supports an array of values separated by comma or specified via multiple flags. + -remoteWrite.urlRelabelDebug array + Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. This is useful for debugging the relabeling configs + Supports array of values separated by comma or specified via multiple flags. -sortLabels Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit -tls diff --git a/lib/promrelabel/config.go b/lib/promrelabel/config.go index 170ad2619..3c9e13660 100644 --- a/lib/promrelabel/config.go +++ b/lib/promrelabel/config.go @@ -25,8 +25,8 @@ type RelabelConfig struct { // ParsedConfigs represents parsed relabel configs. type ParsedConfigs struct { - prcs []*parsedRelabelConfig - RelabelDebug bool + prcs []*parsedRelabelConfig + relabelDebug bool } // Len returns the number of relabel configs in pcs. @@ -44,19 +44,20 @@ func (pcs *ParsedConfigs) String() string { } var sb strings.Builder for _, prc := range pcs.prcs { - fmt.Fprintf(&sb, "%s", prc.String()) + fmt.Fprintf(&sb, "%s,", prc.String()) } + fmt.Fprintf(&sb, "relabelDebug=%v", pcs.relabelDebug) return sb.String() } // LoadRelabelConfigs loads relabel configs from the given path. -func LoadRelabelConfigs(path string) (*ParsedConfigs, error) { +func LoadRelabelConfigs(path string, relabelDebug bool) (*ParsedConfigs, error) { data, err := ioutil.ReadFile(path) if err != nil { return nil, fmt.Errorf("cannot read `relabel_configs` from %q: %w", path, err) } data = envtemplate.Replace(data) - pcs, err := ParseRelabelConfigsData(data) + pcs, err := ParseRelabelConfigsData(data, relabelDebug) if err != nil { return nil, fmt.Errorf("cannot unmarshal `relabel_configs` from %q: %w", path, err) } @@ -64,16 +65,16 @@ func LoadRelabelConfigs(path string) (*ParsedConfigs, error) { } // ParseRelabelConfigsData parses relabel configs from the given data. -func ParseRelabelConfigsData(data []byte) (*ParsedConfigs, error) { +func ParseRelabelConfigsData(data []byte, relabelDebug bool) (*ParsedConfigs, error) { var rcs []RelabelConfig if err := yaml.UnmarshalStrict(data, &rcs); err != nil { return nil, err } - return ParseRelabelConfigs(rcs) + return ParseRelabelConfigs(rcs, relabelDebug) } // ParseRelabelConfigs parses rcs to dst. -func ParseRelabelConfigs(rcs []RelabelConfig) (*ParsedConfigs, error) { +func ParseRelabelConfigs(rcs []RelabelConfig, relabelDebug bool) (*ParsedConfigs, error) { if len(rcs) == 0 { return nil, nil } @@ -86,7 +87,8 @@ func ParseRelabelConfigs(rcs []RelabelConfig) (*ParsedConfigs, error) { prcs[i] = prc } return &ParsedConfigs{ - prcs: prcs, + prcs: prcs, + relabelDebug: relabelDebug, }, nil } diff --git a/lib/promrelabel/config_test.go b/lib/promrelabel/config_test.go index f514d4d9a..a590c3f1e 100644 --- a/lib/promrelabel/config_test.go +++ b/lib/promrelabel/config_test.go @@ -7,7 +7,7 @@ import ( func TestLoadRelabelConfigsSuccess(t *testing.T) { path := "testdata/relabel_configs_valid.yml" - pcs, err := LoadRelabelConfigs(path) + pcs, err := LoadRelabelConfigs(path, false) if err != nil { t.Fatalf("cannot load relabel configs from %q: %s", path, err) } @@ -19,7 +19,7 @@ func TestLoadRelabelConfigsSuccess(t *testing.T) { func TestLoadRelabelConfigsFailure(t *testing.T) { f := func(path string) { t.Helper() - rcs, err := LoadRelabelConfigs(path) + rcs, err := LoadRelabelConfigs(path, false) if err == nil { t.Fatalf("expecting non-nil error") } @@ -38,7 +38,7 @@ func TestLoadRelabelConfigsFailure(t *testing.T) { func TestParseRelabelConfigsSuccess(t *testing.T) { f := func(rcs []RelabelConfig, pcsExpected *ParsedConfigs) { t.Helper() - pcs, err := ParseRelabelConfigs(rcs) + pcs, err := ParseRelabelConfigs(rcs, false) if err != nil { t.Fatalf("unexected error: %s", err) } @@ -72,7 +72,7 @@ func TestParseRelabelConfigsSuccess(t *testing.T) { func TestParseRelabelConfigsFailure(t *testing.T) { f := func(rcs []RelabelConfig) { t.Helper() - pcs, err := ParseRelabelConfigs(rcs) + pcs, err := ParseRelabelConfigs(rcs, false) if err == nil { t.Fatalf("expecting non-nil error") } diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go index 2829efca6..60abc845c 100644 --- a/lib/promrelabel/relabel.go +++ b/lib/promrelabel/relabel.go @@ -35,35 +35,16 @@ func (prc *parsedRelabelConfig) String() string { prc.SourceLabels, prc.Separator, prc.TargetLabel, prc.Regex.String(), prc.Modulus, prc.Replacement, prc.Action) } -func labelsToString(labels []prompbmarshal.Label) string { - var b []byte - b = append(b, '{') - mname := "" - for _, label := range labels { - if (label.Name == "__name__") { - mname = label.Value - continue - } - b = append(b, label.Name...) - b = append(b, '=') - b = strconv.AppendQuote(b, label.Value) - b = append(b, ',') - } - if b[len(b)-1] == ',' { - b[len(b)-1] = '}' - return mname + string(b) - } - return mname -} - // Apply applies pcs to labels starting from the labelsOffset. // // If isFinalize is set, then FinalizeLabels is called on the labels[labelsOffset:]. // // The returned labels at labels[labelsOffset:] are sorted. -func (pcs *ParsedConfigs) Apply(labels []prompbmarshal.Label, labelsOffset int, isFinalize bool, relabelDebug bool) []prompbmarshal.Label { - var inStr, outStr string +func (pcs *ParsedConfigs) Apply(labels []prompbmarshal.Label, labelsOffset int, isFinalize bool) []prompbmarshal.Label { + var inStr string + relabelDebug := false if pcs != nil { + relabelDebug = pcs.relabelDebug if relabelDebug { inStr = labelsToString(labels[labelsOffset:]) } @@ -71,8 +52,8 @@ func (pcs *ParsedConfigs) Apply(labels []prompbmarshal.Label, labelsOffset int, tmp := prc.apply(labels, labelsOffset) if len(tmp) == labelsOffset { // All the labels have been removed. - if relabelDebug { - logger.Infof("\nRelabel In: %s\nRelabel Out: DROPPED - all labels removed.", inStr) + if pcs.relabelDebug { + logger.Infof("\nRelabel In: %s\nRelabel Out: DROPPED - all labels removed", inStr) } return tmp } @@ -80,24 +61,24 @@ func (pcs *ParsedConfigs) Apply(labels []prompbmarshal.Label, labelsOffset int, } } labels = removeEmptyLabels(labels, labelsOffset) - if relabelDebug { - outStr = labelsToString(labels[labelsOffset:]) - if inStr == outStr { - logger.Infof("\nRelabel In: %s\nRelabel Out: KEPT AS IS - no change.", inStr) - } else { - logger.Infof("\nRelabel In: %s\nRelabel Out: %s", inStr, outStr) - } - // lib/promscrape/config.go::getScrapeWork() would drop the entire - // target, if this func does not return a single label. So we can't do - // a simple 'labels = labels[:labelsOffset]' here. - // BTW: Other callees would just drop the related ts, yes, e.g. - // app/vmagent/remotewrite/relabel.go::applyRelabeling() and - // lib/promscrape/scrapework.go::addRowToTimeseries() - } if isFinalize { labels = FinalizeLabels(labels[:labelsOffset], labels[labelsOffset:]) } SortLabels(labels[labelsOffset:]) + if relabelDebug { + if len(labels) == labelsOffset { + logger.Infof("\nRelabel In: %s\nRelabel Out: DROPPED - all labels removed", inStr) + return labels + } + outStr := labelsToString(labels[labelsOffset:]) + if inStr == outStr { + logger.Infof("\nRelabel In: %s\nRelabel Out: KEPT AS IS - no change", inStr) + } else { + logger.Infof("\nRelabel In: %s\nRelabel Out: %s", inStr, outStr) + } + // Drop labels + labels = labels[:labelsOffset] + } return labels } @@ -454,3 +435,33 @@ func CleanLabels(labels []prompbmarshal.Label) { label.Value = "" } } + +func labelsToString(labels []prompbmarshal.Label) string { + labelsCopy := append([]prompbmarshal.Label{}, labels...) + SortLabels(labelsCopy) + mname := "" + for _, label := range labelsCopy { + if label.Name == "__name__" { + mname = label.Value + break + } + } + if len(labelsCopy) <= 1 { + return mname + } + b := []byte(mname) + b = append(b, '{') + for i, label := range labelsCopy { + if label.Name == "__name__" { + continue + } + b = append(b, label.Name...) + b = append(b, '=') + b = strconv.AppendQuote(b, label.Value) + if i+1 < len(labelsCopy) { + b = append(b, ',') + } + } + b = append(b, '}') + return string(b) +} diff --git a/lib/promrelabel/relabel_test.go b/lib/promrelabel/relabel_test.go index 742219921..1eb5d67e4 100644 --- a/lib/promrelabel/relabel_test.go +++ b/lib/promrelabel/relabel_test.go @@ -10,7 +10,7 @@ import ( func TestApplyRelabelConfigs(t *testing.T) { f := func(config string, labels []prompbmarshal.Label, isFinalize bool, resultExpected []prompbmarshal.Label) { t.Helper() - pcs, err := ParseRelabelConfigsData([]byte(config)) + pcs, err := ParseRelabelConfigsData([]byte(config), false) if err != nil { t.Fatalf("cannot parse %q: %s", config, err) } diff --git a/lib/promrelabel/relabel_timing_test.go b/lib/promrelabel/relabel_timing_test.go index bdaf05986..cddb84e12 100644 --- a/lib/promrelabel/relabel_timing_test.go +++ b/lib/promrelabel/relabel_timing_test.go @@ -840,7 +840,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) { } func mustParseRelabelConfigs(config string) *ParsedConfigs { - pcs, err := ParseRelabelConfigsData([]byte(config)) + pcs, err := ParseRelabelConfigsData([]byte(config), false) if err != nil { panic(fmt.Errorf("unexpected error: %w", err)) } diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 75d5456ef..e5a088ebf 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -104,7 +104,6 @@ type ScrapeConfig struct { ProxyURL proxy.URL `yaml:"proxy_url,omitempty"` RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"` MetricRelabelConfigs []promrelabel.RelabelConfig `yaml:"metric_relabel_configs,omitempty"` - RelabelDebug bool `yaml:"relabel_debug,omitempty"` SampleLimit int `yaml:"sample_limit,omitempty"` StaticConfigs []StaticConfig `yaml:"static_configs,omitempty"` @@ -119,6 +118,8 @@ type ScrapeConfig struct { GCESDConfigs []gce.SDConfig `yaml:"gce_sd_configs,omitempty"` // These options are supported only by lib/promscrape. + RelabelDebug bool `yaml:"relabel_debug,omitempty"` + MetricRelabelDebug bool `yaml:"metric_relabel_debug,omitempty"` DisableCompression bool `yaml:"disable_compression,omitempty"` DisableKeepAlive bool `yaml:"disable_keepalive,omitempty"` StreamParse bool `yaml:"stream_parse,omitempty"` @@ -574,15 +575,14 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf if err != nil { return nil, fmt.Errorf("cannot parse proxy auth config for `job_name` %q: %w", jobName, err) } - relabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.RelabelConfigs) + relabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.RelabelConfigs, sc.RelabelDebug) if err != nil { return nil, fmt.Errorf("cannot parse `relabel_configs` for `job_name` %q: %w", jobName, err) } - metricRelabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.MetricRelabelConfigs) + metricRelabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.MetricRelabelConfigs, sc.MetricRelabelDebug) if err != nil { return nil, fmt.Errorf("cannot parse `metric_relabel_configs` for `job_name` %q: %w", jobName, err) } - metricRelabelConfigs.RelabelDebug = sc.RelabelDebug swc := &scrapeWorkConfig{ scrapeInterval: scrapeInterval, scrapeTimeout: scrapeTimeout, @@ -822,7 +822,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel // Reduce memory usage by interning all the strings in originalLabels. internLabelStrings(originalLabels) } - labels = swc.relabelConfigs.Apply(labels, 0, false, swc.metricRelabelConfigs.RelabelDebug) + labels = swc.relabelConfigs.Apply(labels, 0, false) labels = promrelabel.RemoveMetaLabels(labels[:0], labels) // Remove references to already deleted labels, so GC could clean strings for label name and label value past len(labels). // This should reduce memory usage when relabeling creates big number of temporary labels with long names and/or values. diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go index 5a0ad9cd3..792234429 100644 --- a/lib/promscrape/scrapework.go +++ b/lib/promscrape/scrapework.go @@ -512,10 +512,7 @@ func (sw *scrapeWork) addRowToTimeseries(wc *writeRequestCtx, r *parser.Row, tim labelsLen := len(wc.labels) wc.labels = appendLabels(wc.labels, r.Metric, r.Tags, sw.Config.Labels, sw.Config.HonorLabels) if needRelabel { - wc.labels = sw.Config.MetricRelabelConfigs.Apply(wc.labels, labelsLen, true, sw.Config.MetricRelabelConfigs.RelabelDebug) - if sw.Config.MetricRelabelConfigs.RelabelDebug { - return // avoid submission - } + wc.labels = sw.Config.MetricRelabelConfigs.Apply(wc.labels, labelsLen, true) } else { wc.labels = promrelabel.FinalizeLabels(wc.labels[:labelsLen], wc.labels[labelsLen:]) promrelabel.SortLabels(wc.labels[labelsLen:]) diff --git a/lib/promscrape/scrapework_test.go b/lib/promscrape/scrapework_test.go index 843e8dde1..7d2eeced6 100644 --- a/lib/promscrape/scrapework_test.go +++ b/lib/promscrape/scrapework_test.go @@ -435,7 +435,7 @@ func timeseriesToString(ts *prompbmarshal.TimeSeries) string { } func mustParseRelabelConfigs(config string) *promrelabel.ParsedConfigs { - pcs, err := promrelabel.ParseRelabelConfigsData([]byte(config)) + pcs, err := promrelabel.ParseRelabelConfigsData([]byte(config), false) if err != nil { panic(fmt.Errorf("cannot parse %q: %w", config, err)) }