From 97b41e727c4db2a4b62c95227bccdaaf9fb59663 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sat, 10 Dec 2022 02:09:21 -0800 Subject: [PATCH] lib/promscrape: implement target-level and metric-level relabel debugging Target-level debugging is performed by clicking the 'debug' link at the corresponding target on either http://vmagent:8429/targets page or on http://vmagent:8428/service-discovery page. Metric-level debugging is perfromed at http://vmagent:8429/metric-relabel-debug page. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3407 See https://docs.victoriametrics.com/vmagent.html#relabel-debug --- README.md | 5 +- app/vmagent/README.md | 45 ++- app/vmagent/main.go | 21 +- app/vmagent/remotewrite/relabel.go | 9 +- app/vmalert/notifier/alert_test.go | 2 +- app/vmalert/notifier/config.go | 4 +- app/vminsert/relabel/relabel.go | 4 +- app/vmselect/main.go | 7 + docs/CHANGELOG.md | 3 + docs/Cluster-VictoriaMetrics.md | 3 +- docs/README.md | 2 - docs/Single-server-VictoriaMetrics.md | 2 - docs/operator/api.MD | 2 - docs/relabeling.md | 4 + docs/sd_configs.md | 8 - docs/vmagent.md | 45 ++- lib/promrelabel/config.go | 39 +- lib/promrelabel/config_test.go | 27 +- lib/promrelabel/if_expression_test.go | 4 +- lib/promrelabel/relabel.go | 105 ++++-- lib/promrelabel/relabel_test.go | 94 ++++- lib/promrelabel/relabel_timing_test.go | 2 +- lib/promscrape/config.go | 11 +- lib/promscrape/config_test.go | 2 +- lib/promscrape/relabel_debug.go | 133 +++++++ lib/promscrape/relabel_debug.qtpl | 152 ++++++++ lib/promscrape/relabel_debug.qtpl.go | 360 ++++++++++++++++++ lib/promscrape/scraper.go | 4 +- lib/promscrape/scrapework.go | 24 +- lib/promscrape/scrapework_test.go | 6 +- lib/promscrape/targetstatus.go | 114 ++++-- lib/promscrape/targetstatus.qtpl | 18 +- lib/promscrape/targetstatus.qtpl.go | 494 +++++++++++++------------ lib/promutils/labels.go | 26 +- lib/promutils/labels_test.go | 4 +- 35 files changed, 1326 insertions(+), 459 deletions(-) create mode 100644 lib/promscrape/relabel_debug.go create mode 100644 lib/promscrape/relabel_debug.qtpl create mode 100644 lib/promscrape/relabel_debug.qtpl.go diff --git a/README.md b/README.md index 37ca8e8b6..4e5519667 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ Additionally, all the VictoriaMetrics components allow setting flag values via e ## Automatic vmstorage discovery -`vminsert` and `vmselect` components in [entrprise version of VictoriaMetrics](https://docs.victoriametrics.com/enterprise.html) support +`vminsert` and `vmselect` components in [enterprise version of VictoriaMetrics](https://docs.victoriametrics.com/enterprise.html) support the following approaches for automatic discovery of `vmstorage` nodes: - file-based discovery - put the list of `vmstorage` nodes into a file - one node address per each line - and then pass `-storageNode=file:/path/to/file-with-vmstorage-list` @@ -319,6 +319,7 @@ See [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html - `api/v1/status/active_queries` - for currently executed active queries. Note that every `vmselect` maintains an independent list of active queries, which is returned in the response. - `api/v1/status/top_queries` - for listing the most frequently executed queries and queries taking the most duration. + - `metric-relabel-debug` - for debugging [relabeling rules](https://docs.victoriametrics.com/relabeling.html). - URLs for [Graphite Metrics API](https://graphite-api.readthedocs.io/en/latest/api.html#the-metrics-api): `http://:8481/select//graphite/`, where: - `` is an arbitrary number identifying data namespace for query (aka tenant) @@ -866,8 +867,6 @@ Below is the output for `/path/to/vminsert -help`: Supports an array of values separated by comma or specified via multiple flags. -relabelConfig string Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal - -relabelDebug - Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, then the metrics aren't sent to storage. This is useful for debugging the relabeling configs -replicationFactor int Replication factor for the ingested data, i.e. how many copies to make among distinct -storageNode instances. Note that vmselect must run with -dedup.minScrapeInterval=1ms for data de-duplication when replicationFactor is greater than 1. Higher values for -dedup.minScrapeInterval at vmselect is OK (default 1) -rpc.disableCompression diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 2fc8fb3ca..03b09af21 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -245,8 +245,6 @@ scrape_configs: * `scrape_align_interval: duration` for aligning scrapes to the given interval instead of using random offset in the range `[0 ... scrape_interval]` for scraping each target. The random offset helps spreading scrapes evenly in time. * `scrape_offset: duration` for specifying the exact offset for scraping instead of using random offset in the range `[0 ... scrape_interval]`. -* `relabel_debug: true` for enabling debug logging during relabeling of the discovered targets. See [these docs](#relabeling). -* `metric_relabel_debug: true` for enabling debug logging during relabeling of the scraped metrics. See [these docs](#relabeling). See [scrape_configs docs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) for more details on all the supported options. @@ -419,26 +417,25 @@ with [additional enhancements](#relabeling-enhancements). The relabeling can be This relabeling is used for modifying labels in discovered targets and for dropping unneded targets. See [relabeling cookbook](https://docs.victoriametrics.com/relabeling.html) for details. - This relabeling can be debugged by passing `relabel_debug: true` option to the corresponding `scrape_config` section. - In this case `vmagent` logs target labels before and after the relabeling and then drops the logged target. + This relabeling can be debugged by clicking the `debug` link at the corresponding target on the `http://vmagent:8429/targets` page + or on the `http://vmagent:8429/service-discovery` page. See [these docs](#relabel-debug) for details. * At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is used for modifying labels in scraped metrics and for dropping unneeded metrics. See [relabeling cookbook](https://docs.victoriametrics.com/relabeling.html) for details. - This relabeling can be debugged by passing `metric_relabel_debug: true` option to the corresponding `scrape_config` section. - In this case `vmagent` logs metrics before and after the relabeling and then drops the logged metrics. + This relabeling can be debugged via `http://vmagent:8429/metric-relabel-debug` page. See [these docs](#relabel-debug) for details. * At the `-remoteWrite.relabelConfig` file. This relabeling is used for modifying labels for all the collected metrics - (inluding [metrics obtained via push-based protocols](#how-to-push-data-to-vmagent)) and for dropping unneeded metrics + (including [metrics obtained via push-based protocols](#how-to-push-data-to-vmagent)) and for dropping unneeded metrics before sending them to all the configured `-remoteWrite.url` addresses. - This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. - In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage. + + This relabeling can be debugged via `http://vmagent:8429/metric-relabel-debug` page. See [these docs](#relabel-debug) for details. * At the `-remoteWrite.urlRelabelConfig` files. This relabeling is used for modifying labels for metrics and for dropping unneeded metrics before sending them to a particular `-remoteWrite.url`. - This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. - In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`. + + This relabeling can be debugged via `http://vmagent:8429/metric-relabel-debug` page. See [these docs](#relabel-debug) for details. All the files with relabeling configs can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values. @@ -453,9 +450,6 @@ The following articles contain useful information about Prometheus relabeling: * [Extracting labels from legacy metric names](https://www.robustperception.io/extracting-labels-from-legacy-metric-names) * [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs) -[This relabeler playground](https://relabeler.promlabs.com/) can help debugging issues related to relabeling. - - ## Relabeling enhancements `vmagent` provides the following enhancements on top of Prometheus-compatible relabeling: @@ -597,6 +591,24 @@ Important notes about `action: graphite` relabeling rules: The `action: graphite` relabeling rules are easier to write and maintain than `action: replace` for labels extraction from Graphite-style metric names. Additionally, the `action: graphite` relabeling rules usually work much faster than the equivalent `action: replace` rules. +## Relabel debug + +`vmagent` provides the following tools for debugging target-level and metric-level relabeling: + +- Target-level relabeling (e.g. `relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs)) + can be performed by navigating to `http://vmagent:8429/targets` page and clicking the `debug` link at the target, which must be debugged. + The opened page will show step-by-step results for the actual relabeling rules applied to the target labels. + + The `http://vmagent:8429/targets` page shows only active targets. If you need to understand why some target + is dropped during the relabeling, then navigate to `http://vmagent:8428/service-discovery` page, find the dropped target + and click the `debug` link there. The opened page will show step-by-step results for the actual relabeling rules, + which resulted to target drop. + +- Metric-level relabeling (e.g. `metric_relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) + and all the relabeling, which can be set up via `-relabelConfig`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` + command-line flags) can be performed by navigating to `http://vmagent:8429/metric-relabel-debug` page + and submitting there relabeling rules together with the metric to be relabeled. + The page will show step-by-step results for the entered relabeling rules executed against the entered metric. ## Prometheus staleness markers @@ -1428,8 +1440,6 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Supports array of values separated by comma or specified via multiple flags. -remoteWrite.relabelConfig string Optional path to file with relabel_config entries. The path can point either to local file or to http url. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details - -remoteWrite.relabelDebug - Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs -remoteWrite.roundDigits array Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics Supports array of values separated by comma or specified via multiple flags. @@ -1464,9 +1474,6 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . -remoteWrite.urlRelabelConfig array Optional path to relabel config for the corresponding -remoteWrite.url. The path can point either to local file or to http url Supports an array of values separated by comma or specified via multiple flags. - -remoteWrite.urlRelabelDebug array - Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. This is useful for debugging the relabeling configs - Supports array of values separated by comma or specified via multiple flags. -sortLabels Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit -tls diff --git a/app/vmagent/main.go b/app/vmagent/main.go index 4209603b1..32c2f9094 100644 --- a/app/vmagent/main.go +++ b/app/vmagent/main.go @@ -207,6 +207,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool { httpserver.WriteAPIHelp(w, [][2]string{ {"targets", "status for discovered active targets"}, {"service-discovery", "labels before and after relabeling for discovered targets"}, + {"metric-relabel-debug", "debug metric relabeling"}, {"api/v1/targets", "advanced information about discovered targets in JSON format"}, {"config", "-promscrape.config contents"}, {"metrics", "available service metrics"}, @@ -325,6 +326,18 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool { promscrapeServiceDiscoveryRequests.Inc() promscrape.WriteServiceDiscovery(w, r) return true + case "/prometheus/metric-relabel-debug", "/metric-relabel-debug": + promscrapeMetricRelabelDebugRequests.Inc() + promscrape.WriteMetricRelabelDebug(w, r) + return true + case "/prometheus/target-relabel-debug", "/target-relabel-debug": + promscrapeTargetRelabelDebugRequests.Inc() + if err := promscrape.WriteTargetRelabelDebug(w, r); err != nil { + promscrapeTargetRelabelDebugErrors.Inc() + httpserver.Errorf(w, r, "%s", err) + return true + } + return true case "/prometheus/api/v1/targets", "/api/v1/targets": promscrapeAPIV1TargetsRequests.Inc() w.Header().Set("Content-Type", "application/json") @@ -546,7 +559,13 @@ var ( promscrapeTargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/targets"}`) promscrapeServiceDiscoveryRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/service-discovery"}`) - promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`) + + promscrapeMetricRelabelDebugRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/metric-relabel-debug"}`) + + promscrapeTargetRelabelDebugRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target-relabel-debug"}`) + promscrapeTargetRelabelDebugErrors = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target-relabel-debug"}`) + + promscrapeAPIV1TargetsRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/api/v1/targets"}`) promscrapeTargetResponseRequests = metrics.NewCounter(`vmagent_http_requests_total{path="/target_response"}`) promscrapeTargetResponseErrors = metrics.NewCounter(`vmagent_http_request_errors_total{path="/target_response"}`) diff --git a/app/vmagent/remotewrite/relabel.go b/app/vmagent/remotewrite/relabel.go index 5ae9b0c1f..1b796e24c 100644 --- a/app/vmagent/remotewrite/relabel.go +++ b/app/vmagent/remotewrite/relabel.go @@ -18,13 +18,8 @@ var ( relabelConfigPathGlobal = flag.String("remoteWrite.relabelConfig", "", "Optional path to file with relabel_config entries. "+ "The path can point either to local file or to http url. These entries are applied to all the metrics "+ "before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details") - relabelDebugGlobal = flag.Bool("remoteWrite.relabelDebug", false, "Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. "+ - "If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs") relabelConfigPaths = flagutil.NewArrayString("remoteWrite.urlRelabelConfig", "Optional path to relabel config for the corresponding -remoteWrite.url. "+ "The path can point either to local file or to http url") - relabelDebug = flagutil.NewArrayBool("remoteWrite.urlRelabelDebug", "Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. "+ - "If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. "+ - "This is useful for debugging the relabeling configs") usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+ "in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+ @@ -42,7 +37,7 @@ func CheckRelabelConfigs() error { func loadRelabelConfigs() (*relabelConfigs, error) { var rcs relabelConfigs if *relabelConfigPathGlobal != "" { - global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal, *relabelDebugGlobal) + global, err := promrelabel.LoadRelabelConfigs(*relabelConfigPathGlobal) if err != nil { return nil, fmt.Errorf("cannot load -remoteWrite.relabelConfig=%q: %w", *relabelConfigPathGlobal, err) } @@ -58,7 +53,7 @@ func loadRelabelConfigs() (*relabelConfigs, error) { // Skip empty relabel config. continue } - prc, err := promrelabel.LoadRelabelConfigs(path, relabelDebug.GetOptionalArg(i)) + prc, err := promrelabel.LoadRelabelConfigs(path) if err != nil { return nil, fmt.Errorf("cannot load relabel configs from -remoteWrite.urlRelabelConfig=%q: %w", path, err) } diff --git a/app/vmalert/notifier/alert_test.go b/app/vmalert/notifier/alert_test.go index d93c9c455..7032bceab 100644 --- a/app/vmalert/notifier/alert_test.go +++ b/app/vmalert/notifier/alert_test.go @@ -239,7 +239,7 @@ func TestAlert_toPromLabels(t *testing.T) { replacement: "aaa" - action: labeldrop regex: "env.*" -`), false) +`)) if err != nil { t.Fatalf("unexpected error: %s", err) } diff --git a/app/vmalert/notifier/config.go b/app/vmalert/notifier/config.go index 23c6b1198..ac60a4965 100644 --- a/app/vmalert/notifier/config.go +++ b/app/vmalert/notifier/config.go @@ -83,12 +83,12 @@ func (cfg *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { if cfg.Timeout.Duration() == 0 { cfg.Timeout = promutils.NewDuration(time.Second * 10) } - rCfg, err := promrelabel.ParseRelabelConfigs(cfg.RelabelConfigs, false) + rCfg, err := promrelabel.ParseRelabelConfigs(cfg.RelabelConfigs) if err != nil { return fmt.Errorf("failed to parse relabeling config: %w", err) } cfg.parsedRelabelConfigs = rCfg - arCfg, err := promrelabel.ParseRelabelConfigs(cfg.AlertRelabelConfigs, false) + arCfg, err := promrelabel.ParseRelabelConfigs(cfg.AlertRelabelConfigs) if err != nil { return fmt.Errorf("failed to parse alert relabeling config: %w", err) } diff --git a/app/vminsert/relabel/relabel.go b/app/vminsert/relabel/relabel.go index 5c8db252e..601eb4968 100644 --- a/app/vminsert/relabel/relabel.go +++ b/app/vminsert/relabel/relabel.go @@ -19,8 +19,6 @@ var ( relabelConfig = flag.String("relabelConfig", "", "Optional path to a file with relabeling rules, which are applied to all the ingested metrics. "+ "The path can point either to local file or to http url. "+ "See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal") - relabelDebug = flag.Bool("relabelDebug", false, "Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, "+ - "then the metrics aren't sent to storage. This is useful for debugging the relabeling configs") usePromCompatibleNaming = flag.Bool("usePromCompatibleNaming", false, "Whether to replace characters unsupported by Prometheus with underscores "+ "in the ingested metric names and label names. For example, foo.bar{a.b='c'} is transformed into foo_bar{a_b='c'} during data ingestion if this flag is set. "+ @@ -77,7 +75,7 @@ func loadRelabelConfig() (*promrelabel.ParsedConfigs, error) { if len(*relabelConfig) == 0 { return nil, nil } - pcs, err := promrelabel.LoadRelabelConfigs(*relabelConfig, *relabelDebug) + pcs, err := promrelabel.LoadRelabelConfigs(*relabelConfig) if err != nil { return nil, fmt.Errorf("error when reading -relabelConfig=%q: %w", *relabelConfig, err) } diff --git a/app/vmselect/main.go b/app/vmselect/main.go index 56a31a5d4..40431151f 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -27,6 +27,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape" "github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics" "github.com/VictoriaMetrics/VictoriaMetrics/lib/querytracer" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" @@ -568,6 +569,10 @@ func selectHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW return true } return true + case "prometheus/metric-relabel-debug", "metric-relabel-debug": + promscrapeMetricRelabelDebugRequests.Inc() + promscrape.WriteMetricRelabelDebug(w, r) + return true case "prometheus/api/v1/rules", "prometheus/rules": rulesRequests.Inc() if len(*vmalertProxyURL) > 0 { @@ -739,6 +744,8 @@ var ( graphiteFunctionsRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/graphite/functions"}`) + promscrapeMetricRelabelDebugRequests = metrics.NewCounter(`vm_http_requests_total{path="select/{}/prometheus/metric-relabel-debug"}`) + vmalertRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/vmalert"}`) rulesRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/rules"}`) alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/alerts"}`) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 661cf837c..9a3689522 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -19,6 +19,9 @@ The following tip changes can be tested by building VictoriaMetrics components f **Update note 2:** this release splits `type="indexdb"` metrics into `type="indexdb/inmemory"` and `type="indexdb/file"` metrics. This may break old dashboards and alerting rules, which contain [label filter](https://docs.victoriametrics.com/keyConcepts.html#filtering) on `{type="indexdb"}`. Such label filter must be substituted with `{type=~"indexdb.*"}`, so it matches `indexdb` from the previous releases and `indexdb/inmemory` + `indexdb/file` from new releases. It is recommended upgrading to the latest available dashboards and alerting rules mentioned in [these docs](https://docs.victoriametrics.com/#monitoring), since they already contain fixed label filters. +**Update note 3:** this release deprecates `relabel_debug` and `metric_relabel_debug` config options in [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). The `-relabelDebug`, `-remoteWrite.relabelDebug` and `-remoteWrite.urlRelabelDebug` command-line options are also deprecated. Use more powerful target-level relabel debugging and metric-level relabel debugging instead as documented [here](https://docs.victoriametrics.com/vmagent.html#relabel-debug). + +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): provide enhanced target-level and metric-level relabel debugging. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabel-debug) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3407). * FEATURE: leave a sample with the biggest value for identical timestamps per each `-dedup.minScrapeInterval` discrete interval when the [deduplication](https://docs.victoriametrics.com/#deduplication) is enabled. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3333). * FEATURE: add `-inmemoryDataFlushInterval` command-line flag, which can be used for controlling the frequency of in-memory data flush to disk. The data flush frequency can be reduced when VictoriaMetrics stores data to low-end flash device with limited number of write cycles (for example, on Raspberry PI). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3337). * FEATURE: expose additional metrics for `indexdb` and `storage` parts stored in memory and for `indexdb` parts stored in files (see [storage docs](https://docs.victoriametrics.com/#storage) for technical details): diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index f7eeda0f9..0f9bb11d2 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -323,6 +323,7 @@ See [troubleshooting docs](https://docs.victoriametrics.com/Troubleshooting.html - `api/v1/status/active_queries` - for currently executed active queries. Note that every `vmselect` maintains an independent list of active queries, which is returned in the response. - `api/v1/status/top_queries` - for listing the most frequently executed queries and queries taking the most duration. + - `metric-relabel-debug` - for debugging [relabeling rules](https://docs.victoriametrics.com/relabeling.html). - URLs for [Graphite Metrics API](https://graphite-api.readthedocs.io/en/latest/api.html#the-metrics-api): `http://:8481/select//graphite/`, where: - `` is an arbitrary number identifying data namespace for query (aka tenant) @@ -870,8 +871,6 @@ Below is the output for `/path/to/vminsert -help`: Supports an array of values separated by comma or specified via multiple flags. -relabelConfig string Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal - -relabelDebug - Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, then the metrics aren't sent to storage. This is useful for debugging the relabeling configs -replicationFactor int Replication factor for the ingested data, i.e. how many copies to make among distinct -storageNode instances. Note that vmselect must run with -dedup.minScrapeInterval=1ms for data de-duplication when replicationFactor is greater than 1. Higher values for -dedup.minScrapeInterval at vmselect is OK (default 1) -rpc.disableCompression diff --git a/docs/README.md b/docs/README.md index 720620ad5..eedee6b44 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2304,8 +2304,6 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Supports an array of values separated by comma or specified via multiple flags. -relabelConfig string Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal - -relabelDebug - Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, then the metrics aren't sent to storage. This is useful for debugging the relabeling configs -retentionFilter array Retention filter in the format 'filter:retention'. For example, '{env="dev"}:3d' configures the retention for time series with env="dev" label to 3 days. See https://docs.victoriametrics.com/#retention-filters for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 3e917836c..43f8f0fe6 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -2307,8 +2307,6 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Supports an array of values separated by comma or specified via multiple flags. -relabelConfig string Optional path to a file with relabeling rules, which are applied to all the ingested metrics. The path can point either to local file or to http url. See https://docs.victoriametrics.com/#relabeling for details. The config is reloaded on SIGHUP signal - -relabelDebug - Whether to log metrics before and after relabeling with -relabelConfig. If the -relabelDebug is enabled, then the metrics aren't sent to storage. This is useful for debugging the relabeling configs -retentionFilter array Retention filter in the format 'filter:retention'. For example, '{env="dev"}:3d' configures the retention for time series with env="dev" label to 3 days. See https://docs.victoriametrics.com/#retention-filters for details. This flag is available only in VictoriaMetrics enterprise. See https://docs.victoriametrics.com/enterprise.html Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/operator/api.MD b/docs/operator/api.MD index cd1e9f863..379c00707 100644 --- a/docs/operator/api.MD +++ b/docs/operator/api.MD @@ -1371,8 +1371,6 @@ VMScrapeParams defines scrape target configuration that compatible only with Vic | Field | Description | Scheme | Required | | ----- | ----------- | ------ | -------- | -| relabel_debug | | *bool | false | -| metric_relabel_debug | | *bool | false | | disable_compression | | *bool | false | | disable_keep_alive | | *bool | false | | no_stale_markers | | *bool | false | diff --git a/docs/relabeling.md b/docs/relabeling.md index 1b295a63a..3e5df86a6 100644 --- a/docs/relabeling.md +++ b/docs/relabeling.md @@ -446,6 +446,8 @@ See also [useful tips for target relabeling](#useful-tips-for-target-relabeling) ## Useful tips for target relabeling +* Target relabelig can be debugged by clicking the `debug` link for the needed target on the `http://vmagent:8429/target` + or on the `http://vmagent:8429/service-discovery` pages. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabel-debug). * Every discovered target contains a set of meta-labels, which start with `__meta_` prefix. The specific sets of labels per each supported service discovery option are listed [here](https://docs.victoriametrics.com/sd_configs.html#prometheus-service-discovery). @@ -462,6 +464,8 @@ See also [useful tips for target relabeling](#useful-tips-for-target-relabeling) ## Useful tips for metric relabeling +* Metric relabeling can be debugged at `http://vmagent:8429/metric-relabel-debug` page. + See [these docs](https://docs.victoriametrics.com/vmagent.html#relabel-debug). * All the labels, which start with `__` prefix, are automatically removed from metrics after the relabeling. So it is common practice to store temporary labels with names startigh with `__` during metrics relabeling. * All the target-level labels are automatically added to all the metrics scraped from targets, diff --git a/docs/sd_configs.md b/docs/sd_configs.md index 2562c6b1b..9125d890c 100644 --- a/docs/sd_configs.md +++ b/docs/sd_configs.md @@ -1181,14 +1181,6 @@ scrape_configs: # By default the limit is disabled. # sample_limit: - # relabel_debug enables debugging for relabel_configs if set to true. - # See https://docs.victoriametrics.com/vmagent.html#relabeling - # relabel_debug: - - # metric_relabel_debug enables debugging for metric_relabel_configs if set to true. - # See https://docs.victoriametrics.com/vmagent.html#relabeling - # metric_relabel_debug: - # disable_compression allows disabling HTTP compression for responses received from scrape targets. # By default scrape targets are queried with `Accept-Encoding: gzip` http request header, # so targets could send compressed responses in order to save network bandwidth. diff --git a/docs/vmagent.md b/docs/vmagent.md index 8d7164aa5..c1558e594 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -249,8 +249,6 @@ scrape_configs: * `scrape_align_interval: duration` for aligning scrapes to the given interval instead of using random offset in the range `[0 ... scrape_interval]` for scraping each target. The random offset helps spreading scrapes evenly in time. * `scrape_offset: duration` for specifying the exact offset for scraping instead of using random offset in the range `[0 ... scrape_interval]`. -* `relabel_debug: true` for enabling debug logging during relabeling of the discovered targets. See [these docs](#relabeling). -* `metric_relabel_debug: true` for enabling debug logging during relabeling of the scraped metrics. See [these docs](#relabeling). See [scrape_configs docs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) for more details on all the supported options. @@ -423,26 +421,25 @@ with [additional enhancements](#relabeling-enhancements). The relabeling can be This relabeling is used for modifying labels in discovered targets and for dropping unneded targets. See [relabeling cookbook](https://docs.victoriametrics.com/relabeling.html) for details. - This relabeling can be debugged by passing `relabel_debug: true` option to the corresponding `scrape_config` section. - In this case `vmagent` logs target labels before and after the relabeling and then drops the logged target. + This relabeling can be debugged by clicking the `debug` link at the corresponding target on the `http://vmagent:8429/targets` page + or on the `http://vmagent:8429/service-discovery` page. See [these docs](#relabel-debug) for details. * At the `scrape_config -> metric_relabel_configs` section in `-promscrape.config` file. This relabeling is used for modifying labels in scraped metrics and for dropping unneeded metrics. See [relabeling cookbook](https://docs.victoriametrics.com/relabeling.html) for details. - This relabeling can be debugged by passing `metric_relabel_debug: true` option to the corresponding `scrape_config` section. - In this case `vmagent` logs metrics before and after the relabeling and then drops the logged metrics. + This relabeling can be debugged via `http://vmagent:8429/metric-relabel-debug` page. See [these docs](#relabel-debug) for details. * At the `-remoteWrite.relabelConfig` file. This relabeling is used for modifying labels for all the collected metrics - (inluding [metrics obtained via push-based protocols](#how-to-push-data-to-vmagent)) and for dropping unneeded metrics + (including [metrics obtained via push-based protocols](#how-to-push-data-to-vmagent)) and for dropping unneeded metrics before sending them to all the configured `-remoteWrite.url` addresses. - This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. - In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage. + + This relabeling can be debugged via `http://vmagent:8429/metric-relabel-debug` page. See [these docs](#relabel-debug) for details. * At the `-remoteWrite.urlRelabelConfig` files. This relabeling is used for modifying labels for metrics and for dropping unneeded metrics before sending them to a particular `-remoteWrite.url`. - This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. - In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`. + + This relabeling can be debugged via `http://vmagent:8429/metric-relabel-debug` page. See [these docs](#relabel-debug) for details. All the files with relabeling configs can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values. @@ -457,9 +454,6 @@ The following articles contain useful information about Prometheus relabeling: * [Extracting labels from legacy metric names](https://www.robustperception.io/extracting-labels-from-legacy-metric-names) * [relabel_configs vs metric_relabel_configs](https://www.robustperception.io/relabel_configs-vs-metric_relabel_configs) -[This relabeler playground](https://relabeler.promlabs.com/) can help debugging issues related to relabeling. - - ## Relabeling enhancements `vmagent` provides the following enhancements on top of Prometheus-compatible relabeling: @@ -601,6 +595,24 @@ Important notes about `action: graphite` relabeling rules: The `action: graphite` relabeling rules are easier to write and maintain than `action: replace` for labels extraction from Graphite-style metric names. Additionally, the `action: graphite` relabeling rules usually work much faster than the equivalent `action: replace` rules. +## Relabel debug + +`vmagent` provides the following tools for debugging target-level and metric-level relabeling: + +- Target-level relabeling (e.g. `relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs)) + can be performed by navigating to `http://vmagent:8429/targets` page and clicking the `debug` link at the target, which must be debugged. + The opened page will show step-by-step results for the actual relabeling rules applied to the target labels. + + The `http://vmagent:8429/targets` page shows only active targets. If you need to understand why some target + is dropped during the relabeling, then navigate to `http://vmagent:8428/service-discovery` page, find the dropped target + and click the `debug` link there. The opened page will show step-by-step results for the actual relabeling rules, + which resulted to target drop. + +- Metric-level relabeling (e.g. `metric_relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs) + and all the relabeling, which can be set up via `-relabelConfig`, `-remoteWrite.relabelConfig` and `-remoteWrite.urlRelabelConfig` + command-line flags) can be performed by navigating to `http://vmagent:8429/metric-relabel-debug` page + and submitting there relabeling rules together with the metric to be relabeled. + The page will show step-by-step results for the entered relabeling rules executed against the entered metric. ## Prometheus staleness markers @@ -1432,8 +1444,6 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Supports array of values separated by comma or specified via multiple flags. -remoteWrite.relabelConfig string Optional path to file with relabel_config entries. The path can point either to local file or to http url. These entries are applied to all the metrics before sending them to -remoteWrite.url. See https://docs.victoriametrics.com/vmagent.html#relabeling for details - -remoteWrite.relabelDebug - Whether to log metrics before and after relabeling with -remoteWrite.relabelConfig. If the -remoteWrite.relabelDebug is enabled, then the metrics aren't sent to remote storage. This is useful for debugging the relabeling configs -remoteWrite.roundDigits array Round metric values to this number of decimal digits after the point before writing them to remote storage. Examples: -remoteWrite.roundDigits=2 would round 1.236 to 1.24, while -remoteWrite.roundDigits=-1 would round 126.78 to 130. By default digits rounding is disabled. Set it to 100 for disabling it for a particular remote storage. This option may be used for improving data compression for the stored metrics Supports array of values separated by comma or specified via multiple flags. @@ -1468,9 +1478,6 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . -remoteWrite.urlRelabelConfig array Optional path to relabel config for the corresponding -remoteWrite.url. The path can point either to local file or to http url Supports an array of values separated by comma or specified via multiple flags. - -remoteWrite.urlRelabelDebug array - Whether to log metrics before and after relabeling with -remoteWrite.urlRelabelConfig. If the -remoteWrite.urlRelabelDebug is enabled, then the metrics aren't sent to the corresponding -remoteWrite.url. This is useful for debugging the relabeling configs - Supports array of values separated by comma or specified via multiple flags. -sortLabels Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit -tls diff --git a/lib/promrelabel/config.go b/lib/promrelabel/config.go index 11553ed43..fd98dcdb1 100644 --- a/lib/promrelabel/config.go +++ b/lib/promrelabel/config.go @@ -18,14 +18,14 @@ import ( // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config type RelabelConfig struct { + If *IfExpression `yaml:"if,omitempty"` + Action string `yaml:"action,omitempty"` SourceLabels []string `yaml:"source_labels,flow,omitempty"` Separator *string `yaml:"separator,omitempty"` TargetLabel string `yaml:"target_label,omitempty"` Regex *MultiLineRegex `yaml:"regex,omitempty"` Modulus uint64 `yaml:"modulus,omitempty"` Replacement *string `yaml:"replacement,omitempty"` - Action string `yaml:"action,omitempty"` - If *IfExpression `yaml:"if,omitempty"` // Match is used together with Labels for `action: graphite`. For example: // - action: graphite @@ -121,8 +121,7 @@ func (mlr *MultiLineRegex) MarshalYAML() (interface{}, error) { // ParsedConfigs represents parsed relabel configs. type ParsedConfigs struct { - prcs []*parsedRelabelConfig - relabelDebug bool + prcs []*parsedRelabelConfig } // Len returns the number of relabel configs in pcs. @@ -140,14 +139,23 @@ func (pcs *ParsedConfigs) String() string { } var a []string for _, prc := range pcs.prcs { - s := "[" + prc.String() + "]" + s := prc.String() + lines := strings.Split(s, "\n") + lines[0] = "- " + lines[0] + for i := range lines[1:] { + line := &lines[1+i] + if len(*line) > 0 { + *line = " " + *line + } + } + s = strings.Join(lines, "\n") a = append(a, s) } - return fmt.Sprintf("%s, relabelDebug=%v", strings.Join(a, ","), pcs.relabelDebug) + return strings.Join(a, "") } // LoadRelabelConfigs loads relabel configs from the given path. -func LoadRelabelConfigs(path string, relabelDebug bool) (*ParsedConfigs, error) { +func LoadRelabelConfigs(path string) (*ParsedConfigs, error) { data, err := fs.ReadFileOrHTTP(path) if err != nil { return nil, fmt.Errorf("cannot read `relabel_configs` from %q: %w", path, err) @@ -156,7 +164,7 @@ func LoadRelabelConfigs(path string, relabelDebug bool) (*ParsedConfigs, error) if err != nil { return nil, fmt.Errorf("cannot expand environment vars at %q: %w", path, err) } - pcs, err := ParseRelabelConfigsData(data, relabelDebug) + pcs, err := ParseRelabelConfigsData(data) if err != nil { return nil, fmt.Errorf("cannot unmarshal `relabel_configs` from %q: %w", path, err) } @@ -164,16 +172,16 @@ func LoadRelabelConfigs(path string, relabelDebug bool) (*ParsedConfigs, error) } // ParseRelabelConfigsData parses relabel configs from the given data. -func ParseRelabelConfigsData(data []byte, relabelDebug bool) (*ParsedConfigs, error) { +func ParseRelabelConfigsData(data []byte) (*ParsedConfigs, error) { var rcs []RelabelConfig if err := yaml.UnmarshalStrict(data, &rcs); err != nil { return nil, err } - return ParseRelabelConfigs(rcs, relabelDebug) + return ParseRelabelConfigs(rcs) } // ParseRelabelConfigs parses rcs to dst. -func ParseRelabelConfigs(rcs []RelabelConfig, relabelDebug bool) (*ParsedConfigs, error) { +func ParseRelabelConfigs(rcs []RelabelConfig) (*ParsedConfigs, error) { if len(rcs) == 0 { return nil, nil } @@ -186,8 +194,7 @@ func ParseRelabelConfigs(rcs []RelabelConfig, relabelDebug bool) (*ParsedConfigs prcs[i] = prc } return &ParsedConfigs{ - prcs: prcs, - relabelDebug: relabelDebug, + prcs: prcs, }, nil } @@ -350,7 +357,13 @@ func parseRelabelConfig(rc *RelabelConfig) (*parsedRelabelConfig, error) { return nil, fmt.Errorf("`labels` config cannot be applied to `action=%s`; it is applied only to `action=graphite`", action) } } + ruleOriginal, err := yaml.Marshal(rc) + if err != nil { + logger.Panicf("BUG: cannot marshal RelabelConfig: %s", err) + } prc := &parsedRelabelConfig{ + ruleOriginal: string(ruleOriginal), + SourceLabels: sourceLabels, Separator: separator, TargetLabel: targetLabel, diff --git a/lib/promrelabel/config_test.go b/lib/promrelabel/config_test.go index dcc689b27..c1d945156 100644 --- a/lib/promrelabel/config_test.go +++ b/lib/promrelabel/config_test.go @@ -50,7 +50,7 @@ func TestRelabelConfigMarshalUnmarshal(t *testing.T) { f(` - action: keep regex: foobar -`, "- regex: foobar\n action: keep\n") + `, "- action: keep\n regex: foobar\n") f(` - regex: - 'fo.+' @@ -80,7 +80,7 @@ func TestRelabelConfigMarshalUnmarshal(t *testing.T) { func TestLoadRelabelConfigsSuccess(t *testing.T) { path := "testdata/relabel_configs_valid.yml" - pcs, err := LoadRelabelConfigs(path, false) + pcs, err := LoadRelabelConfigs(path) if err != nil { t.Fatalf("cannot load relabel configs from %q: %s", path, err) } @@ -93,7 +93,7 @@ func TestLoadRelabelConfigsSuccess(t *testing.T) { func TestLoadRelabelConfigsFailure(t *testing.T) { f := func(path string) { t.Helper() - rcs, err := LoadRelabelConfigs(path, false) + rcs, err := LoadRelabelConfigs(path) if err == nil { t.Fatalf("expecting non-nil error") } @@ -112,7 +112,7 @@ func TestLoadRelabelConfigsFailure(t *testing.T) { func TestParsedConfigsString(t *testing.T) { f := func(rcs []RelabelConfig, sExpected string) { t.Helper() - pcs, err := ParseRelabelConfigs(rcs, false) + pcs, err := ParseRelabelConfigs(rcs) if err != nil { t.Fatalf("unexpected error: %s", err) } @@ -126,8 +126,7 @@ func TestParsedConfigsString(t *testing.T) { TargetLabel: "foo", SourceLabels: []string{"aaa"}, }, - }, "[SourceLabels=[aaa], Separator=;, TargetLabel=foo, Regex=.*, Modulus=0, Replacement=$1, Action=replace, If=, "+ - "graphiteMatchTemplate=, graphiteLabelRules=[]], relabelDebug=false") + }, "- source_labels: [aaa]\n target_label: foo\n") var ie IfExpression if err := ie.Parse("{foo=~'bar'}"); err != nil { t.Fatalf("unexpected error when parsing if expression: %s", err) @@ -141,8 +140,8 @@ func TestParsedConfigsString(t *testing.T) { }, If: &ie, }, - }, "[SourceLabels=[], Separator=;, TargetLabel=, Regex=.*, Modulus=0, Replacement=$1, Action=graphite, If={foo=~'bar'}, "+ - "graphiteMatchTemplate=foo.*.bar, graphiteLabelRules=[replaceTemplate=$1-zz, targetLabel=job]], relabelDebug=false") + }, "- if: '{foo=~''bar''}'\n action: graphite\n match: foo.*.bar\n labels:\n job: $1-zz\n") + replacement := "foo" f([]RelabelConfig{ { Action: "replace", @@ -150,19 +149,23 @@ func TestParsedConfigsString(t *testing.T) { TargetLabel: "x", If: &ie, }, - }, "[SourceLabels=[foo bar], Separator=;, TargetLabel=x, Regex=.*, Modulus=0, Replacement=$1, Action=replace, If={foo=~'bar'}, "+ - "graphiteMatchTemplate=, graphiteLabelRules=[]], relabelDebug=false") + { + TargetLabel: "x", + Replacement: &replacement, + }, + }, "- if: '{foo=~''bar''}'\n action: replace\n source_labels: [foo, bar]\n target_label: x\n- target_label: x\n replacement: foo\n") } func TestParseRelabelConfigsSuccess(t *testing.T) { f := func(rcs []RelabelConfig, pcsExpected *ParsedConfigs) { t.Helper() - pcs, err := ParseRelabelConfigs(rcs, false) + pcs, err := ParseRelabelConfigs(rcs) if err != nil { t.Fatalf("unexpected error: %s", err) } if pcs != nil { for _, prc := range pcs.prcs { + prc.ruleOriginal = "" prc.stringReplacer = nil prc.submatchReplacer = nil } @@ -198,7 +201,7 @@ func TestParseRelabelConfigsSuccess(t *testing.T) { func TestParseRelabelConfigsFailure(t *testing.T) { f := func(rcs []RelabelConfig) { t.Helper() - pcs, err := ParseRelabelConfigs(rcs, false) + pcs, err := ParseRelabelConfigs(rcs) if err == nil { t.Fatalf("expecting non-nil error") } diff --git a/lib/promrelabel/if_expression_test.go b/lib/promrelabel/if_expression_test.go index 6e71a822f..40962c70a 100644 --- a/lib/promrelabel/if_expression_test.go +++ b/lib/promrelabel/if_expression_test.go @@ -122,7 +122,7 @@ func TestIfExpressionMatch(t *testing.T) { if err := yaml.UnmarshalStrict([]byte(ifExpr), &ie); err != nil { t.Fatalf("unexpected error during unmarshal: %s", err) } - labels := promutils.NewLabelsFromString(metricWithLabels) + labels := promutils.MustNewLabelsFromString(metricWithLabels) if !ie.Match(labels.GetLabels()) { t.Fatalf("unexpected mismatch of ifExpr=%s for %s", ifExpr, metricWithLabels) } @@ -156,7 +156,7 @@ func TestIfExpressionMismatch(t *testing.T) { if err := yaml.UnmarshalStrict([]byte(ifExpr), &ie); err != nil { t.Fatalf("unexpected error during unmarshal: %s", err) } - labels := promutils.NewLabelsFromString(metricWithLabels) + labels := promutils.MustNewLabelsFromString(metricWithLabels) if ie.Match(labels.GetLabels()) { t.Fatalf("unexpected match of ifExpr=%s for %s", ifExpr, metricWithLabels) } diff --git a/lib/promrelabel/relabel.go b/lib/promrelabel/relabel.go index 6d56a3754..125ddd08b 100644 --- a/lib/promrelabel/relabel.go +++ b/lib/promrelabel/relabel.go @@ -18,6 +18,9 @@ import ( // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config type parsedRelabelConfig struct { + // ruleOriginal contains the original relabeling rule for the given prasedRelabelConfig. + ruleOriginal string + SourceLabels []string Separator string TargetLabel string @@ -41,50 +44,78 @@ type parsedRelabelConfig struct { submatchReplacer *bytesutil.FastStringTransformer } +// DebugStep contains debug information about a single relabeling rule step +type DebugStep struct { + // Rule contains string representation of the rule step + Rule string + + // In contains the input labels before the exeuction of the rule step + In string + + // Out contains the output labels after the execution of the rule step + Out string +} + +// String returns human-readable representation for ds +func (ds DebugStep) String() string { + return fmt.Sprintf("rule=%q, in=%s, out=%s", ds.Rule, ds.In, ds.Out) +} + // String returns human-readable representation for prc. func (prc *parsedRelabelConfig) String() string { - return fmt.Sprintf("SourceLabels=%s, Separator=%s, TargetLabel=%s, Regex=%s, Modulus=%d, Replacement=%s, Action=%s, If=%s, graphiteMatchTemplate=%s, graphiteLabelRules=%s", - prc.SourceLabels, prc.Separator, prc.TargetLabel, prc.regexOriginal, prc.Modulus, prc.Replacement, - prc.Action, prc.If, prc.graphiteMatchTemplate, prc.graphiteLabelRules) + return prc.ruleOriginal +} + +// ApplyDebug applies pcs to labels in debug mode. +// +// It returns DebugStep list - one entry per each applied relabeling step. +func (pcs *ParsedConfigs) ApplyDebug(labels []prompbmarshal.Label) ([]prompbmarshal.Label, []DebugStep) { + labels, dss := pcs.applyInternal(labels, 0, true) + return labels, dss } // Apply applies pcs to labels starting from the labelsOffset. func (pcs *ParsedConfigs) Apply(labels []prompbmarshal.Label, labelsOffset int) []prompbmarshal.Label { - var inStr string - relabelDebug := false + labels, _ = pcs.applyInternal(labels, labelsOffset, false) + return labels +} + +func (pcs *ParsedConfigs) applyInternal(labels []prompbmarshal.Label, labelsOffset int, debug bool) ([]prompbmarshal.Label, []DebugStep) { + var dss []DebugStep + inStr := "" + if debug { + inStr = LabelsToString(labels[labelsOffset:]) + } if pcs != nil { - relabelDebug = pcs.relabelDebug - if relabelDebug { - inStr = labelsToString(labels[labelsOffset:]) - } for _, prc := range pcs.prcs { - tmp := prc.apply(labels, labelsOffset) - if len(tmp) == labelsOffset { - // All the labels have been removed. - if pcs.relabelDebug { - logger.Infof("\nRelabel In: %s\nRelabel Out: DROPPED - all labels removed", inStr) - } - return tmp + labels = prc.apply(labels, labelsOffset) + if debug { + outStr := LabelsToString(labels[labelsOffset:]) + dss = append(dss, DebugStep{ + Rule: prc.String(), + In: inStr, + Out: outStr, + }) + inStr = outStr + } + if len(labels) == labelsOffset { + // All the labels have been removed. + return labels, dss } - labels = tmp } } labels = removeEmptyLabels(labels, labelsOffset) - if relabelDebug { - if len(labels) == labelsOffset { - logger.Infof("\nRelabel In: %s\nRelabel Out: DROPPED - all labels removed", inStr) - return labels + if debug { + outStr := LabelsToString(labels[labelsOffset:]) + if outStr != inStr { + dss = append(dss, DebugStep{ + Rule: "remove empty labels", + In: inStr, + Out: outStr, + }) } - outStr := labelsToString(labels[labelsOffset:]) - if inStr == outStr { - logger.Infof("\nRelabel In: %s\nRelabel Out: KEPT AS IS - no change", inStr) - } else { - logger.Infof("\nRelabel In: %s\nRelabel Out: %s", inStr, outStr) - } - // Drop labels - labels = labels[:labelsOffset] } - return labels + return labels, dss } func removeEmptyLabels(labels []prompbmarshal.Label, labelsOffset int) []prompbmarshal.Label { @@ -504,25 +535,27 @@ func CleanLabels(labels []prompbmarshal.Label) { } } -func labelsToString(labels []prompbmarshal.Label) string { +// LabelsToString returns Prometheus string representation for the given labels. +// +// Labels in the returned string are sorted by name, +// while the __name__ label is put in front of {} labels. +func LabelsToString(labels []prompbmarshal.Label) string { labelsCopy := append([]prompbmarshal.Label{}, labels...) SortLabels(labelsCopy) mname := "" - for _, label := range labelsCopy { + for i, label := range labelsCopy { if label.Name == "__name__" { mname = label.Value + labelsCopy = append(labelsCopy[:i], labelsCopy[i+1:]...) break } } - if mname != "" && len(labelsCopy) <= 1 { + if mname != "" && len(labelsCopy) == 0 { return mname } b := []byte(mname) b = append(b, '{') for i, label := range labelsCopy { - if label.Name == "__name__" { - continue - } b = append(b, label.Name...) b = append(b, '=') b = strconv.AppendQuote(b, label.Value) diff --git a/lib/promrelabel/relabel_test.go b/lib/promrelabel/relabel_test.go index 8df056e79..9c34a7ebf 100644 --- a/lib/promrelabel/relabel_test.go +++ b/lib/promrelabel/relabel_test.go @@ -2,6 +2,7 @@ package promrelabel import ( "fmt" + "reflect" "testing" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" @@ -27,7 +28,7 @@ func TestSanitizeName(t *testing.T) { func TestLabelsToString(t *testing.T) { f := func(labels []prompbmarshal.Label, sExpected string) { t.Helper() - s := labelsToString(labels) + s := LabelsToString(labels) if s != sExpected { t.Fatalf("unexpected result;\ngot\n%s\nwant\n%s", s, sExpected) } @@ -71,20 +72,95 @@ func TestLabelsToString(t *testing.T) { }, `xxx{a="bc",foo="bar"}`) } -func TestApplyRelabelConfigs(t *testing.T) { - f := func(config, metric string, isFinalize bool, resultExpected string) { +func TestParsedRelabelConfigsApplyDebug(t *testing.T) { + f := func(config, metric string, dssExpected []DebugStep) { t.Helper() - pcs, err := ParseRelabelConfigsData([]byte(config), false) + pcs, err := ParseRelabelConfigsData([]byte(config)) if err != nil { t.Fatalf("cannot parse %q: %s", config, err) } - labels := promutils.NewLabelsFromString(metric) + labels := promutils.MustNewLabelsFromString(metric) + _, dss := pcs.ApplyDebug(labels.GetLabels()) + if !reflect.DeepEqual(dss, dssExpected) { + t.Fatalf("unexpected result; got\n%s\nwant\n%s", dss, dssExpected) + } + } + + // empty relabel config + f(``, `foo`, nil) + // add label + f(` +- target_label: abc + replacement: xyz +`, `foo{bar="baz"}`, []DebugStep{ + { + Rule: "target_label: abc\nreplacement: xyz\n", + In: `foo{bar="baz"}`, + Out: `foo{abc="xyz",bar="baz"}`, + }, + }) + // drop label + f(` +- target_label: bar + replacement: '' +`, `foo{bar="baz"}`, []DebugStep{ + { + Rule: "target_label: bar\nreplacement: \"\"\n", + In: `foo{bar="baz"}`, + Out: `foo{bar=""}`, + }, + { + Rule: "remove empty labels", + In: `foo{bar=""}`, + Out: `foo`, + }, + }) + // drop metric + f(` +- action: drop + source_labels: [bar] + regex: baz +`, `foo{bar="baz",abc="def"}`, []DebugStep{ + { + Rule: "action: drop\nsource_labels: [bar]\nregex: baz\n", + In: `foo{abc="def",bar="baz"}`, + Out: `{}`, + }, + }) + // Multiple steps + f(` +- action: labeldrop + regex: "foo.*" +- target_label: foobar + replacement: "abc" +`, `m{foo="x",foobc="123",a="b"}`, []DebugStep{ + { + Rule: "action: labeldrop\nregex: foo.*\n", + In: `m{a="b",foo="x",foobc="123"}`, + Out: `m{a="b"}`, + }, + { + Rule: "target_label: foobar\nreplacement: abc\n", + In: `m{a="b"}`, + Out: `m{a="b",foobar="abc"}`, + }, + }) +} + +func TestParsedRelabelConfigsApply(t *testing.T) { + f := func(config, metric string, isFinalize bool, resultExpected string) { + t.Helper() + pcs, err := ParseRelabelConfigsData([]byte(config)) + if err != nil { + t.Fatalf("cannot parse %q: %s", config, err) + } + labels := promutils.MustNewLabelsFromString(metric) resultLabels := pcs.Apply(labels.GetLabels(), 0) if isFinalize { resultLabels = FinalizeLabels(resultLabels[:0], resultLabels) } SortLabels(resultLabels) - result := labelsToString(resultLabels) + result := LabelsToString(resultLabels) if result != resultExpected { t.Fatalf("unexpected result; got\n%s\nwant\n%s", result, resultExpected) } @@ -726,9 +802,9 @@ func TestApplyRelabelConfigs(t *testing.T) { func TestFinalizeLabels(t *testing.T) { f := func(metric, resultExpected string) { t.Helper() - labels := promutils.NewLabelsFromString(metric) + labels := promutils.MustNewLabelsFromString(metric) resultLabels := FinalizeLabels(nil, labels.GetLabels()) - result := labelsToString(resultLabels) + result := LabelsToString(resultLabels) if result != resultExpected { t.Fatalf("unexpected result; got\n%s\nwant\n%s", result, resultExpected) } @@ -742,7 +818,7 @@ func TestFinalizeLabels(t *testing.T) { func TestFillLabelReferences(t *testing.T) { f := func(replacement, metric, resultExpected string) { t.Helper() - labels := promutils.NewLabelsFromString(metric) + labels := promutils.MustNewLabelsFromString(metric) result := fillLabelReferences(nil, replacement, labels.GetLabels()) if string(result) != resultExpected { t.Fatalf("unexpected result; got\n%q\nwant\n%q", result, resultExpected) diff --git a/lib/promrelabel/relabel_timing_test.go b/lib/promrelabel/relabel_timing_test.go index ab78f2063..31682e787 100644 --- a/lib/promrelabel/relabel_timing_test.go +++ b/lib/promrelabel/relabel_timing_test.go @@ -1155,7 +1155,7 @@ func BenchmarkApplyRelabelConfigs(b *testing.B) { } func mustParseRelabelConfigs(config string) *ParsedConfigs { - pcs, err := ParseRelabelConfigsData([]byte(config), false) + pcs, err := ParseRelabelConfigsData([]byte(config)) if err != nil { panic(fmt.Errorf("unexpected error: %w", err)) } diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 4591c9fc8..d4513f792 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -267,8 +267,6 @@ type ScrapeConfig struct { YandexCloudSDConfigs []yandexcloud.SDConfig `yaml:"yandexcloud_sd_configs,omitempty"` // These options are supported only by lib/promscrape. - RelabelDebug bool `yaml:"relabel_debug,omitempty"` - MetricRelabelDebug bool `yaml:"metric_relabel_debug,omitempty"` DisableCompression bool `yaml:"disable_compression,omitempty"` DisableKeepAlive bool `yaml:"disable_keepalive,omitempty"` StreamParse bool `yaml:"stream_parse,omitempty"` @@ -928,11 +926,11 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf if err != nil { return nil, fmt.Errorf("cannot parse proxy auth config for `job_name` %q: %w", jobName, err) } - relabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.RelabelConfigs, sc.RelabelDebug) + relabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.RelabelConfigs) if err != nil { return nil, fmt.Errorf("cannot parse `relabel_configs` for `job_name` %q: %w", jobName, err) } - metricRelabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.MetricRelabelConfigs, sc.MetricRelabelDebug) + metricRelabelConfigs, err := promrelabel.ParseRelabelConfigs(sc.MetricRelabelConfigs) if err != nil { return nil, fmt.Errorf("cannot parse `metric_relabel_configs` for `job_name` %q: %w", jobName, err) } @@ -1188,7 +1186,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel } if labels.Len() == 0 { // Drop target without labels. - droppedTargetsMap.Register(originalLabels) + droppedTargetsMap.Register(originalLabels, swc.relabelConfigs) return nil, nil } // See https://www.robustperception.io/life-of-a-label @@ -1203,7 +1201,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel address := labels.Get("__address__") if len(address) == 0 { // Drop target without scrape address. - droppedTargetsMap.Register(originalLabels) + droppedTargetsMap.Register(originalLabels, swc.relabelConfigs) return nil, nil } // Usability extension to Prometheus behavior: extract optional scheme and metricsPath from __address__. @@ -1314,6 +1312,7 @@ func (swc *scrapeWorkConfig) getScrapeWork(target string, extraLabels, metaLabel ProxyURL: swc.proxyURL, ProxyAuthConfig: swc.proxyAuthConfig, AuthConfig: swc.authConfig, + RelabelConfigs: swc.relabelConfigs, MetricRelabelConfigs: swc.metricRelabelConfigs, SampleLimit: swc.sampleLimit, DisableCompression: swc.disableCompression, diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go index b6c1f3bd4..40586756c 100644 --- a/lib/promscrape/config_test.go +++ b/lib/promscrape/config_test.go @@ -104,7 +104,6 @@ scrape_configs: static_configs: - targets: - foo - relabel_debug: true scrape_align_interval: 1h30m0s proxy_bearer_token_file: file.txt proxy_headers: @@ -721,6 +720,7 @@ scrape_config_files: func resetNonEssentialFields(sws []*ScrapeWork) { for _, sw := range sws { sw.OriginalLabels = nil + sw.RelabelConfigs = nil sw.MetricRelabelConfigs = nil } } diff --git a/lib/promscrape/relabel_debug.go b/lib/promscrape/relabel_debug.go new file mode 100644 index 000000000..a52157615 --- /dev/null +++ b/lib/promscrape/relabel_debug.go @@ -0,0 +1,133 @@ +package promscrape + +import ( + "fmt" + "net/http" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" +) + +// WriteMetricRelabelDebug serves requests to /metric-relabel-debug page +func WriteMetricRelabelDebug(w http.ResponseWriter, r *http.Request) { + metric := r.FormValue("metric") + relabelConfigs := r.FormValue("relabel_configs") + if metric == "" { + metric = "{}" + } + labels, err := promutils.NewLabelsFromString(metric) + if err != nil { + err = fmt.Errorf("cannot parse metric: %s", err) + WriteMetricRelabelDebugSteps(w, nil, metric, relabelConfigs, err) + return + } + + pcs, err := promrelabel.ParseRelabelConfigsData([]byte(relabelConfigs)) + if err != nil { + err = fmt.Errorf("cannot parse relabel configs: %s", err) + WriteMetricRelabelDebugSteps(w, nil, metric, relabelConfigs, err) + return + } + + // The metric relabeling below must be in sync with the code at scrapeWork.addRowToTimeseries + + // Apply relabeling + labelsResult, dss := pcs.ApplyDebug(labels.GetLabels()) + + // Remove labels with __ prefix + inStr := promrelabel.LabelsToString(labelsResult) + labelsResult = promrelabel.FinalizeLabels(labelsResult[:0], labelsResult) + outStr := promrelabel.LabelsToString(labelsResult) + if inStr != outStr { + dss = append(dss, promrelabel.DebugStep{ + Rule: "remove labels with __ prefix", + In: inStr, + Out: outStr, + }) + } + + // There is no need in labels' sorting, since promrelabel.LabelsToString() automatically sorts labels. + + WriteMetricRelabelDebugSteps(w, dss, metric, relabelConfigs, nil) +} + +// WriteTargetRelabelDebug generates response for /target-relabel-debug page +func WriteTargetRelabelDebug(w http.ResponseWriter, r *http.Request) error { + targetID := r.FormValue("id") + relabelConfigs, labels, ok := getRelabelContextByTargetID(targetID) + if !ok { + return fmt.Errorf("cannot find target for id=%s", targetID) + } + + // The target relabeling below must be in sync with the code at scrapeWorkConfig.getScrapeWork + + // Prevent from modifying the original labels + labels = labels.Clone() + + // Apply relabeling + labelsResult, dss := relabelConfigs.ApplyDebug(labels.GetLabels()) + + // Remove labels with __meta_ prefix + inStr := promrelabel.LabelsToString(labelsResult) + labels.Labels = labelsResult + labels.RemoveMetaLabels() + outStr := promrelabel.LabelsToString(labels.Labels) + if inStr != outStr { + dss = append(dss, promrelabel.DebugStep{ + Rule: "remove labels with __meta_ prefix", + In: inStr, + Out: outStr, + }) + } + + // Add missing instance label + if labels.Get("instance") == "" { + address := labels.Get("__address__") + if address != "" { + inStr = outStr + labels.Add("instance", address) + outStr = promrelabel.LabelsToString(labels.Labels) + dss = append(dss, promrelabel.DebugStep{ + Rule: "add missing instance label from __address__ label", + In: inStr, + Out: outStr, + }) + } + } + + // Remove labels with __ prefix + inStr = outStr + labels.RemoveLabelsWithDoubleUnderscorePrefix() + outStr = promrelabel.LabelsToString(labels.Labels) + if inStr != outStr { + dss = append(dss, promrelabel.DebugStep{ + Rule: "remove labels with __ prefix", + In: inStr, + Out: outStr, + }) + } + + // There is no need in labels' sorting, since promrelabel.LabelsToString() automatically sorts labels. + + WriteTargetRelabelDebugSteps(w, dss) + return nil +} + +func getChangedLabelNames(in, out *promutils.Labels) map[string]struct{} { + inMap := in.ToMap() + outMap := out.ToMap() + changed := make(map[string]struct{}) + for k, v := range outMap { + inV, ok := inMap[k] + if !ok || inV != v { + changed[k] = struct{}{} + } + } + for k, v := range inMap { + outV, ok := outMap[k] + if !ok || outV != v { + changed[k] = struct{}{} + } + } + return changed +} diff --git a/lib/promscrape/relabel_debug.qtpl b/lib/promscrape/relabel_debug.qtpl new file mode 100644 index 000000000..7ee19fcd8 --- /dev/null +++ b/lib/promscrape/relabel_debug.qtpl @@ -0,0 +1,152 @@ +{% import ( + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" +) %} + +{% stripspace %} + +{% func MetricRelabelDebugSteps(dss []promrelabel.DebugStep, metric, relabelConfigs string, err error) %} + + + + {%= commonHeader() %} + Metric relabel debug + + + {%= navbar() %} +
+ Relabeling docs
+ {% if err != nil %} + {%= errorNotification(err) %} + {% endif %} + +
+
+
+ Relabel configs:
+ +
+ +
+ Metric with labels:
+ +
+ +
+
+ +
+
+ {%= relabelDebugSteps(dss) %} +
+
+
+ + +{% endfunc %} + +{% func TargetRelabelDebugSteps(dss []promrelabel.DebugStep) %} + + + + {%= commonHeader() %} + Target relabel debug + + + {%= navbar() %} +
+ Relabeling docs
+
+
+ {%= relabelDebugSteps(dss) %} +
+
+
+ + +{% endfunc %} + +{% func relabelDebugSteps(dss []promrelabel.DebugStep) %} +{% if len(dss) > 0 %} +
+ Original labels: {%= mustFormatLabels(dss[0].In) %} +
+{% endif %} + + + + + + + + + + {% for i, ds := range dss %} + {% code + inLabels := promutils.MustNewLabelsFromString(ds.In) + outLabels := promutils.MustNewLabelsFromString(ds.Out) + changedLabels := getChangedLabelNames(inLabels, outLabels) + %} + + + + + + + {% endfor %} + +
StepRelabeling RuleInput LabelsOutput labels +
{%d i %}
{%s ds.Rule %}
 +
+ {%= labelsWithHighlight(inLabels, changedLabels, "red") %} +
+ 		+
+ {%= labelsWithHighlight(outLabels, changedLabels, "blue") %} +
+
+{% if len(dss) > 0 %} +
+ Resulting labels: {%= mustFormatLabels(dss[len(dss)-1].Out) %} +
+{% endif %} +{% endfunc %} + +{% func labelsWithHighlight(labels *promutils.Labels, highlight map[string]struct{}, color string) %} + {% code + labelsList := labels.GetLabels() + metricName := "" + for i, label := range labelsList { + if label.Name == "__name__" { + metricName = label.Value + labelsList = append(labelsList[:i], labelsList[i+1:]...) + break + } + } + %} + {% if metricName != "" %} + {% if _, ok := highlight["__name__"]; ok %} + {%s metricName %} + {% else %} + {%s metricName %} + {% endif %} + {% if len(labelsList) == 0 %}{% return %}{% endif %} + {% endif %} +{ + {% for i, label := range labelsList %} + {% if _, ok := highlight[label.Name]; ok %} + {%s label.Name %}={%q label.Value %} + {% else %} + {%s label.Name %}={%q label.Value %} + {% endif %} + {% if i < len(labelsList)-1 %},{% space %}{% endif %} + {% endfor %} +} +{% endfunc %} + +{% func mustFormatLabels(s string) %} + {% code labels := promutils.MustNewLabelsFromString(s) %} + {%= labelsWithHighlight(labels, nil, "") %} +{% endfunc %} + +{% endstripspace %} diff --git a/lib/promscrape/relabel_debug.qtpl.go b/lib/promscrape/relabel_debug.qtpl.go new file mode 100644 index 000000000..9a7b778af --- /dev/null +++ b/lib/promscrape/relabel_debug.qtpl.go @@ -0,0 +1,360 @@ +// Code generated by qtc from "relabel_debug.qtpl". DO NOT EDIT. +// See https://github.com/valyala/quicktemplate for details. + +//line lib/promscrape/relabel_debug.qtpl:1 +package promscrape + +//line lib/promscrape/relabel_debug.qtpl:1 +import ( + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils" +) + +//line lib/promscrape/relabel_debug.qtpl:8 +import ( + qtio422016 "io" + + qt422016 "github.com/valyala/quicktemplate" +) + +//line lib/promscrape/relabel_debug.qtpl:8 +var ( + _ = qtio422016.Copy + _ = qt422016.AcquireByteBuffer +) + +//line lib/promscrape/relabel_debug.qtpl:8 +func StreamMetricRelabelDebugSteps(qw422016 *qt422016.Writer, dss []promrelabel.DebugStep, metric, relabelConfigs string, err error) { +//line lib/promscrape/relabel_debug.qtpl:8 + qw422016.N().S(``) +//line lib/promscrape/relabel_debug.qtpl:12 + streamcommonHeader(qw422016) +//line lib/promscrape/relabel_debug.qtpl:12 + qw422016.N().S(`Metric relabel debug`) +//line lib/promscrape/relabel_debug.qtpl:16 + streamnavbar(qw422016) +//line lib/promscrape/relabel_debug.qtpl:16 + qw422016.N().S(`
Relabeling docs
`) +//line lib/promscrape/relabel_debug.qtpl:19 + if err != nil { +//line lib/promscrape/relabel_debug.qtpl:20 + streamerrorNotification(qw422016, err) +//line lib/promscrape/relabel_debug.qtpl:21 + } +//line lib/promscrape/relabel_debug.qtpl:21 + qw422016.N().S(`
Relabel configs:
Metric with labels:
`) +//line lib/promscrape/relabel_debug.qtpl:40 + streamrelabelDebugSteps(qw422016, dss) +//line lib/promscrape/relabel_debug.qtpl:40 + qw422016.N().S(`
`) +//line lib/promscrape/relabel_debug.qtpl:46 +} + +//line lib/promscrape/relabel_debug.qtpl:46 +func WriteMetricRelabelDebugSteps(qq422016 qtio422016.Writer, dss []promrelabel.DebugStep, metric, relabelConfigs string, err error) { +//line lib/promscrape/relabel_debug.qtpl:46 + qw422016 := qt422016.AcquireWriter(qq422016) +//line lib/promscrape/relabel_debug.qtpl:46 + StreamMetricRelabelDebugSteps(qw422016, dss, metric, relabelConfigs, err) +//line lib/promscrape/relabel_debug.qtpl:46 + qt422016.ReleaseWriter(qw422016) +//line lib/promscrape/relabel_debug.qtpl:46 +} + +//line lib/promscrape/relabel_debug.qtpl:46 +func MetricRelabelDebugSteps(dss []promrelabel.DebugStep, metric, relabelConfigs string, err error) string { +//line lib/promscrape/relabel_debug.qtpl:46 + qb422016 := qt422016.AcquireByteBuffer() +//line lib/promscrape/relabel_debug.qtpl:46 + WriteMetricRelabelDebugSteps(qb422016, dss, metric, relabelConfigs, err) +//line lib/promscrape/relabel_debug.qtpl:46 + qs422016 := string(qb422016.B) +//line lib/promscrape/relabel_debug.qtpl:46 + qt422016.ReleaseByteBuffer(qb422016) +//line lib/promscrape/relabel_debug.qtpl:46 + return qs422016 +//line lib/promscrape/relabel_debug.qtpl:46 +} + +//line lib/promscrape/relabel_debug.qtpl:48 +func StreamTargetRelabelDebugSteps(qw422016 *qt422016.Writer, dss []promrelabel.DebugStep) { +//line lib/promscrape/relabel_debug.qtpl:48 + qw422016.N().S(``) +//line lib/promscrape/relabel_debug.qtpl:52 + streamcommonHeader(qw422016) +//line lib/promscrape/relabel_debug.qtpl:52 + qw422016.N().S(`Target relabel debug`) +//line lib/promscrape/relabel_debug.qtpl:56 + streamnavbar(qw422016) +//line lib/promscrape/relabel_debug.qtpl:56 + qw422016.N().S(`
Relabeling docs
`) +//line lib/promscrape/relabel_debug.qtpl:61 + streamrelabelDebugSteps(qw422016, dss) +//line lib/promscrape/relabel_debug.qtpl:61 + qw422016.N().S(`
`) +//line lib/promscrape/relabel_debug.qtpl:67 +} + +//line lib/promscrape/relabel_debug.qtpl:67 +func WriteTargetRelabelDebugSteps(qq422016 qtio422016.Writer, dss []promrelabel.DebugStep) { +//line lib/promscrape/relabel_debug.qtpl:67 + qw422016 := qt422016.AcquireWriter(qq422016) +//line lib/promscrape/relabel_debug.qtpl:67 + StreamTargetRelabelDebugSteps(qw422016, dss) +//line lib/promscrape/relabel_debug.qtpl:67 + qt422016.ReleaseWriter(qw422016) +//line lib/promscrape/relabel_debug.qtpl:67 +} + +//line lib/promscrape/relabel_debug.qtpl:67 +func TargetRelabelDebugSteps(dss []promrelabel.DebugStep) string { +//line lib/promscrape/relabel_debug.qtpl:67 + qb422016 := qt422016.AcquireByteBuffer() +//line lib/promscrape/relabel_debug.qtpl:67 + WriteTargetRelabelDebugSteps(qb422016, dss) +//line lib/promscrape/relabel_debug.qtpl:67 + qs422016 := string(qb422016.B) +//line lib/promscrape/relabel_debug.qtpl:67 + qt422016.ReleaseByteBuffer(qb422016) +//line lib/promscrape/relabel_debug.qtpl:67 + return qs422016 +//line lib/promscrape/relabel_debug.qtpl:67 +} + +//line lib/promscrape/relabel_debug.qtpl:69 +func streamrelabelDebugSteps(qw422016 *qt422016.Writer, dss []promrelabel.DebugStep) { +//line lib/promscrape/relabel_debug.qtpl:70 + if len(dss) > 0 { +//line lib/promscrape/relabel_debug.qtpl:70 + qw422016.N().S(`
Original labels: `) +//line lib/promscrape/relabel_debug.qtpl:72 + streammustFormatLabels(qw422016, dss[0].In) +//line lib/promscrape/relabel_debug.qtpl:72 + qw422016.N().S(`
`) +//line lib/promscrape/relabel_debug.qtpl:74 + } +//line lib/promscrape/relabel_debug.qtpl:74 + qw422016.N().S(``) +//line lib/promscrape/relabel_debug.qtpl:85 + for i, ds := range dss { +//line lib/promscrape/relabel_debug.qtpl:87 + inLabels := promutils.MustNewLabelsFromString(ds.In) + outLabels := promutils.MustNewLabelsFromString(ds.Out) + changedLabels := getChangedLabelNames(inLabels, outLabels) + +//line lib/promscrape/relabel_debug.qtpl:90 + qw422016.N().S(``) +//line lib/promscrape/relabel_debug.qtpl:105 + } +//line lib/promscrape/relabel_debug.qtpl:105 + qw422016.N().S(`
StepRelabeling RuleInput LabelsOutput labels
`) +//line lib/promscrape/relabel_debug.qtpl:92 + qw422016.N().D(i) +//line lib/promscrape/relabel_debug.qtpl:92 + qw422016.N().S(`
`)
+//line lib/promscrape/relabel_debug.qtpl:93
+		qw422016.E().S(ds.Rule)
+//line lib/promscrape/relabel_debug.qtpl:93
+		qw422016.N().S(`
`) +//line lib/promscrape/relabel_debug.qtpl:96 + streamlabelsWithHighlight(qw422016, inLabels, changedLabels, "red") +//line lib/promscrape/relabel_debug.qtpl:96 + qw422016.N().S(`
`) +//line lib/promscrape/relabel_debug.qtpl:101 + streamlabelsWithHighlight(qw422016, outLabels, changedLabels, "blue") +//line lib/promscrape/relabel_debug.qtpl:101 + qw422016.N().S(`
`) +//line lib/promscrape/relabel_debug.qtpl:108 + if len(dss) > 0 { +//line lib/promscrape/relabel_debug.qtpl:108 + qw422016.N().S(`
Resulting labels: `) +//line lib/promscrape/relabel_debug.qtpl:110 + streammustFormatLabels(qw422016, dss[len(dss)-1].Out) +//line lib/promscrape/relabel_debug.qtpl:110 + qw422016.N().S(`
`) +//line lib/promscrape/relabel_debug.qtpl:112 + } +//line lib/promscrape/relabel_debug.qtpl:113 +} + +//line lib/promscrape/relabel_debug.qtpl:113 +func writerelabelDebugSteps(qq422016 qtio422016.Writer, dss []promrelabel.DebugStep) { +//line lib/promscrape/relabel_debug.qtpl:113 + qw422016 := qt422016.AcquireWriter(qq422016) +//line lib/promscrape/relabel_debug.qtpl:113 + streamrelabelDebugSteps(qw422016, dss) +//line lib/promscrape/relabel_debug.qtpl:113 + qt422016.ReleaseWriter(qw422016) +//line lib/promscrape/relabel_debug.qtpl:113 +} + +//line lib/promscrape/relabel_debug.qtpl:113 +func relabelDebugSteps(dss []promrelabel.DebugStep) string { +//line lib/promscrape/relabel_debug.qtpl:113 + qb422016 := qt422016.AcquireByteBuffer() +//line lib/promscrape/relabel_debug.qtpl:113 + writerelabelDebugSteps(qb422016, dss) +//line lib/promscrape/relabel_debug.qtpl:113 + qs422016 := string(qb422016.B) +//line lib/promscrape/relabel_debug.qtpl:113 + qt422016.ReleaseByteBuffer(qb422016) +//line lib/promscrape/relabel_debug.qtpl:113 + return qs422016 +//line lib/promscrape/relabel_debug.qtpl:113 +} + +//line lib/promscrape/relabel_debug.qtpl:115 +func streamlabelsWithHighlight(qw422016 *qt422016.Writer, labels *promutils.Labels, highlight map[string]struct{}, color string) { +//line lib/promscrape/relabel_debug.qtpl:117 + labelsList := labels.GetLabels() + metricName := "" + for i, label := range labelsList { + if label.Name == "__name__" { + metricName = label.Value + labelsList = append(labelsList[:i], labelsList[i+1:]...) + break + } + } + +//line lib/promscrape/relabel_debug.qtpl:127 + if metricName != "" { +//line lib/promscrape/relabel_debug.qtpl:128 + if _, ok := highlight["__name__"]; ok { +//line lib/promscrape/relabel_debug.qtpl:128 + qw422016.N().S(``) +//line lib/promscrape/relabel_debug.qtpl:129 + qw422016.E().S(metricName) +//line lib/promscrape/relabel_debug.qtpl:129 + qw422016.N().S(``) +//line lib/promscrape/relabel_debug.qtpl:130 + } else { +//line lib/promscrape/relabel_debug.qtpl:131 + qw422016.E().S(metricName) +//line lib/promscrape/relabel_debug.qtpl:132 + } +//line lib/promscrape/relabel_debug.qtpl:133 + if len(labelsList) == 0 { +//line lib/promscrape/relabel_debug.qtpl:133 + return +//line lib/promscrape/relabel_debug.qtpl:133 + } +//line lib/promscrape/relabel_debug.qtpl:134 + } +//line lib/promscrape/relabel_debug.qtpl:134 + qw422016.N().S(`{`) +//line lib/promscrape/relabel_debug.qtpl:136 + for i, label := range labelsList { +//line lib/promscrape/relabel_debug.qtpl:137 + if _, ok := highlight[label.Name]; ok { +//line lib/promscrape/relabel_debug.qtpl:137 + qw422016.N().S(``) +//line lib/promscrape/relabel_debug.qtpl:138 + qw422016.E().S(label.Name) +//line lib/promscrape/relabel_debug.qtpl:138 + qw422016.N().S(`=`) +//line lib/promscrape/relabel_debug.qtpl:138 + qw422016.E().Q(label.Value) +//line lib/promscrape/relabel_debug.qtpl:138 + qw422016.N().S(``) +//line lib/promscrape/relabel_debug.qtpl:139 + } else { +//line lib/promscrape/relabel_debug.qtpl:140 + qw422016.E().S(label.Name) +//line lib/promscrape/relabel_debug.qtpl:140 + qw422016.N().S(`=`) +//line lib/promscrape/relabel_debug.qtpl:140 + qw422016.E().Q(label.Value) +//line lib/promscrape/relabel_debug.qtpl:141 + } +//line lib/promscrape/relabel_debug.qtpl:142 + if i < len(labelsList)-1 { +//line lib/promscrape/relabel_debug.qtpl:142 + qw422016.N().S(`,`) +//line lib/promscrape/relabel_debug.qtpl:142 + qw422016.N().S(` `) +//line lib/promscrape/relabel_debug.qtpl:142 + } +//line lib/promscrape/relabel_debug.qtpl:143 + } +//line lib/promscrape/relabel_debug.qtpl:143 + qw422016.N().S(`}`) +//line lib/promscrape/relabel_debug.qtpl:145 +} + +//line lib/promscrape/relabel_debug.qtpl:145 +func writelabelsWithHighlight(qq422016 qtio422016.Writer, labels *promutils.Labels, highlight map[string]struct{}, color string) { +//line lib/promscrape/relabel_debug.qtpl:145 + qw422016 := qt422016.AcquireWriter(qq422016) +//line lib/promscrape/relabel_debug.qtpl:145 + streamlabelsWithHighlight(qw422016, labels, highlight, color) +//line lib/promscrape/relabel_debug.qtpl:145 + qt422016.ReleaseWriter(qw422016) +//line lib/promscrape/relabel_debug.qtpl:145 +} + +//line lib/promscrape/relabel_debug.qtpl:145 +func labelsWithHighlight(labels *promutils.Labels, highlight map[string]struct{}, color string) string { +//line lib/promscrape/relabel_debug.qtpl:145 + qb422016 := qt422016.AcquireByteBuffer() +//line lib/promscrape/relabel_debug.qtpl:145 + writelabelsWithHighlight(qb422016, labels, highlight, color) +//line lib/promscrape/relabel_debug.qtpl:145 + qs422016 := string(qb422016.B) +//line lib/promscrape/relabel_debug.qtpl:145 + qt422016.ReleaseByteBuffer(qb422016) +//line lib/promscrape/relabel_debug.qtpl:145 + return qs422016 +//line lib/promscrape/relabel_debug.qtpl:145 +} + +//line lib/promscrape/relabel_debug.qtpl:147 +func streammustFormatLabels(qw422016 *qt422016.Writer, s string) { +//line lib/promscrape/relabel_debug.qtpl:148 + labels := promutils.MustNewLabelsFromString(s) + +//line lib/promscrape/relabel_debug.qtpl:149 + streamlabelsWithHighlight(qw422016, labels, nil, "") +//line lib/promscrape/relabel_debug.qtpl:150 +} + +//line lib/promscrape/relabel_debug.qtpl:150 +func writemustFormatLabels(qq422016 qtio422016.Writer, s string) { +//line lib/promscrape/relabel_debug.qtpl:150 + qw422016 := qt422016.AcquireWriter(qq422016) +//line lib/promscrape/relabel_debug.qtpl:150 + streammustFormatLabels(qw422016, s) +//line lib/promscrape/relabel_debug.qtpl:150 + qt422016.ReleaseWriter(qw422016) +//line lib/promscrape/relabel_debug.qtpl:150 +} + +//line lib/promscrape/relabel_debug.qtpl:150 +func mustFormatLabels(s string) string { +//line lib/promscrape/relabel_debug.qtpl:150 + qb422016 := qt422016.AcquireByteBuffer() +//line lib/promscrape/relabel_debug.qtpl:150 + writemustFormatLabels(qb422016, s) +//line lib/promscrape/relabel_debug.qtpl:150 + qs422016 := string(qb422016.B) +//line lib/promscrape/relabel_debug.qtpl:150 + qt422016.ReleaseByteBuffer(qb422016) +//line lib/promscrape/relabel_debug.qtpl:150 + return qs422016 +//line lib/promscrape/relabel_debug.qtpl:150 +} diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go index a6b59070b..70daa217f 100644 --- a/lib/promscrape/scraper.go +++ b/lib/promscrape/scraper.go @@ -363,9 +363,9 @@ func (sg *scraperGroup) update(sws []*ScrapeWork) { "make sure service discovery and relabeling is set up properly; "+ "see also https://docs.victoriametrics.com/vmagent.html#troubleshooting; "+ "original labels for target1: %s; original labels for target2: %s", - sw.ScrapeURL, sw.LabelsString(), originalLabels.String(), sw.OriginalLabels.String()) + sw.ScrapeURL, sw.Labels.String(), originalLabels.String(), sw.OriginalLabels.String()) } - droppedTargetsMap.Register(sw.OriginalLabels) + droppedTargetsMap.Register(sw.OriginalLabels, sw.RelabelConfigs) continue } swsMap[key] = sw.OriginalLabels diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go index 52c39ddd9..340afddde 100644 --- a/lib/promscrape/scrapework.go +++ b/lib/promscrape/scrapework.go @@ -101,6 +101,9 @@ type ScrapeWork struct { // Auth config AuthConfig *promauth.Config + // Optional `relabel_configs`. + RelabelConfigs *promrelabel.ParsedConfigs + // Optional `metric_relabel_configs`. MetricRelabelConfigs *promrelabel.ParsedConfigs @@ -147,15 +150,17 @@ func (sw *ScrapeWork) canSwitchToStreamParseMode() bool { // It can be used for comparing for equality for two ScrapeWork objects. func (sw *ScrapeWork) key() string { // Do not take into account OriginalLabels, since they can be changed with relabeling. + // Do not take into account RelabelConfigs, since it is already applied to Labels. // Take into account JobNameOriginal in order to capture the case when the original job_name is changed via relabeling. key := fmt.Sprintf("JobNameOriginal=%s, ScrapeURL=%s, ScrapeInterval=%s, ScrapeTimeout=%s, HonorLabels=%v, HonorTimestamps=%v, DenyRedirects=%v, Labels=%s, "+ "ExternalLabels=%s, "+ - "ProxyURL=%s, ProxyAuthConfig=%s, AuthConfig=%s, MetricRelabelConfigs=%s, SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v, "+ + "ProxyURL=%s, ProxyAuthConfig=%s, AuthConfig=%s, MetricRelabelConfigs=%q, "+ + "SampleLimit=%d, DisableCompression=%v, DisableKeepAlive=%v, StreamParse=%v, "+ "ScrapeAlignInterval=%s, ScrapeOffset=%s, SeriesLimit=%d, NoStaleMarkers=%v", - sw.jobNameOriginal, sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels, sw.HonorTimestamps, sw.DenyRedirects, sw.LabelsString(), + sw.jobNameOriginal, sw.ScrapeURL, sw.ScrapeInterval, sw.ScrapeTimeout, sw.HonorLabels, sw.HonorTimestamps, sw.DenyRedirects, sw.Labels.String(), sw.ExternalLabels.String(), - sw.ProxyURL.String(), sw.ProxyAuthConfig.String(), - sw.AuthConfig.String(), sw.MetricRelabelConfigs.String(), sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse, + sw.ProxyURL.String(), sw.ProxyAuthConfig.String(), sw.AuthConfig.String(), sw.MetricRelabelConfigs.String(), + sw.SampleLimit, sw.DisableCompression, sw.DisableKeepAlive, sw.StreamParse, sw.ScrapeAlignInterval, sw.ScrapeOffset, sw.SeriesLimit, sw.NoStaleMarkers) return key } @@ -165,11 +170,6 @@ func (sw *ScrapeWork) Job() string { return sw.Labels.Get("job") } -// LabelsString returns labels in Prometheus format for the given sw. -func (sw *ScrapeWork) LabelsString() string { - return sw.Labels.String() -} - type scrapeWork struct { // Config for the scrape. Config *ScrapeWork @@ -281,7 +281,7 @@ func (sw *scrapeWork) run(stopCh <-chan struct{}, globalStopCh <-chan struct{}) // scrapes replicated targets at different time offsets. This guarantees that the deduplication consistently leaves samples // received from the same vmagent replica. // See https://docs.victoriametrics.com/vmagent.html#scraping-big-number-of-targets - key := fmt.Sprintf("clusterName=%s, clusterMemberID=%d, ScrapeURL=%s, Labels=%s", *clusterName, clusterMemberID, sw.Config.ScrapeURL, sw.Config.LabelsString()) + key := fmt.Sprintf("clusterName=%s, clusterMemberID=%d, ScrapeURL=%s, Labels=%s", *clusterName, clusterMemberID, sw.Config.ScrapeURL, sw.Config.Labels.String()) h := xxhash.Sum64(bytesutil.ToUnsafeBytes(key)) randSleep = uint64(float64(scrapeInterval) * (float64(h) / (1 << 64))) sleepOffset := uint64(time.Now().UnixNano()) % uint64(scrapeInterval) @@ -348,7 +348,7 @@ func (sw *scrapeWork) logError(s string) { if !*suppressScrapeErrors { logger.ErrorfSkipframes(1, "error when scraping %q from job %q with labels %s: %s; "+ "scrape errors can be disabled by -promscrape.suppressScrapeErrors command-line flag", - sw.Config.ScrapeURL, sw.Config.Job(), sw.Config.LabelsString(), s) + sw.Config.ScrapeURL, sw.Config.Job(), sw.Config.Labels.String(), s) } } @@ -362,7 +362,7 @@ func (sw *scrapeWork) scrapeAndLogError(scrapeTimestamp, realTimestamp int64) { sw.errsSuppressedCount++ return } - err = fmt.Errorf("cannot scrape %q (job %q, labels %s): %w", sw.Config.ScrapeURL, sw.Config.Job(), sw.Config.LabelsString(), err) + err = fmt.Errorf("cannot scrape %q (job %q, labels %s): %w", sw.Config.ScrapeURL, sw.Config.Job(), sw.Config.Labels.String(), err) if sw.errsSuppressedCount > 0 { err = fmt.Errorf("%w; %d similar errors suppressed during the last %.1f seconds", err, sw.errsSuppressedCount, d.Seconds()) } diff --git a/lib/promscrape/scrapework_test.go b/lib/promscrape/scrapework_test.go index 20ab892f4..b94ba1dd1 100644 --- a/lib/promscrape/scrapework_test.go +++ b/lib/promscrape/scrapework_test.go @@ -40,8 +40,8 @@ func TestIsAutoMetric(t *testing.T) { func TestAppendExtraLabels(t *testing.T) { f := func(sourceLabels, extraLabels string, honorLabels bool, resultExpected string) { t.Helper() - src := promutils.NewLabelsFromString(sourceLabels) - extra := promutils.NewLabelsFromString(extraLabels) + src := promutils.MustNewLabelsFromString(sourceLabels) + extra := promutils.MustNewLabelsFromString(extraLabels) var labels promutils.Labels labels.Labels = appendExtraLabels(src.GetLabels(), extra.GetLabels(), 0, honorLabels) result := labels.String() @@ -794,7 +794,7 @@ func timeseriesToString(ts *prompbmarshal.TimeSeries) string { } func mustParseRelabelConfigs(config string) *promrelabel.ParsedConfigs { - pcs, err := promrelabel.ParseRelabelConfigsData([]byte(config), false) + pcs, err := promrelabel.ParseRelabelConfigsData([]byte(config)) if err != nil { panic(fmt.Errorf("cannot parse %q: %w", config, err)) } diff --git a/lib/promscrape/targetstatus.go b/lib/promscrape/targetstatus.go index 8e28497e6..bf220f7c4 100644 --- a/lib/promscrape/targetstatus.go +++ b/lib/promscrape/targetstatus.go @@ -147,15 +147,16 @@ func (tsm *targetStatusMap) getScrapeWorkByTargetID(targetID string) *scrapeWork tsm.mu.Lock() defer tsm.mu.Unlock() for sw := range tsm.m { - if getTargetID(sw) == targetID { + // The target is uniquely identified by a pointer to its original labels. + if getLabelsID(sw.Config.OriginalLabels) == targetID { return sw } } return nil } -func getTargetID(sw *scrapeWork) string { - return fmt.Sprintf("%016x", uintptr(unsafe.Pointer(sw))) +func getLabelsID(labels *promutils.Labels) string { + return fmt.Sprintf("%016x", uintptr(unsafe.Pointer(labels))) } // StatusByGroup returns the number of targets with status==up @@ -254,36 +255,36 @@ type droppedTargets struct { type droppedTarget struct { originalLabels *promutils.Labels + relabelConfigs *promrelabel.ParsedConfigs deadline uint64 } -func (dt *droppedTargets) getTargetsLabels() []*promutils.Labels { +func (dt *droppedTargets) getTargetsList() []droppedTarget { dt.mu.Lock() - dtls := make([]*promutils.Labels, 0, len(dt.m)) + dts := make([]droppedTarget, 0, len(dt.m)) for _, v := range dt.m { - dtls = append(dtls, v.originalLabels) + dts = append(dts, v) } dt.mu.Unlock() // Sort discovered targets by __address__ label, so they stay in consistent order across calls - sort.Slice(dtls, func(i, j int) bool { - addr1 := dtls[i].Get("__address__") - addr2 := dtls[j].Get("__address__") + sort.Slice(dts, func(i, j int) bool { + addr1 := dts[i].originalLabels.Get("__address__") + addr2 := dts[j].originalLabels.Get("__address__") return addr1 < addr2 }) - return dtls + return dts } -func (dt *droppedTargets) Register(originalLabels *promutils.Labels) { - // It is better to have hash collisions instead of spending additional CPU on promLabelsString() call. +func (dt *droppedTargets) Register(originalLabels *promutils.Labels, relabelConfigs *promrelabel.ParsedConfigs) { + // It is better to have hash collisions instead of spending additional CPU on originalLabels.String() call. key := labelsHash(originalLabels) currentTime := fasttime.UnixTimestamp() dt.mu.Lock() - if k, ok := dt.m[key]; ok { - k.deadline = currentTime + 10*60 - dt.m[key] = k - } else if len(dt.m) < *maxDroppedTargets { + _, ok := dt.m[key] + if ok || len(dt.m) < *maxDroppedTargets { dt.m[key] = droppedTarget{ originalLabels: originalLabels, + relabelConfigs: relabelConfigs, deadline: currentTime + 10*60, } } @@ -318,13 +319,13 @@ var xxhashPool = &sync.Pool{ // WriteDroppedTargetsJSON writes `droppedTargets` contents to w according to https://prometheus.io/docs/prometheus/latest/querying/api/#targets func (dt *droppedTargets) WriteDroppedTargetsJSON(w io.Writer) { - dtls := dt.getTargetsLabels() + dts := dt.getTargetsList() fmt.Fprintf(w, `[`) - for i, labels := range dtls { + for i, dt := range dts { fmt.Fprintf(w, `{"discoveredLabels":`) - writeLabelsJSON(w, labels) + writeLabelsJSON(w, dt.originalLabels) fmt.Fprintf(w, `}`) - if i+1 < len(dtls) { + if i+1 < len(dts) { fmt.Fprintf(w, `,`) } } @@ -385,12 +386,12 @@ func (tsm *targetStatusMap) getTargetsStatusByJob(filter *requestFilter) *target // Do not show empty jobs if target filters are set. emptyJobs = nil } - dtls := droppedTargetsMap.getTargetsLabels() + dts := droppedTargetsMap.getTargetsList() return &targetsStatusResult{ - jobTargetsStatuses: jts, - droppedTargetsLabels: dtls, - emptyJobs: emptyJobs, - err: err, + jobTargetsStatuses: jts, + droppedTargets: dts, + emptyJobs: emptyJobs, + err: err, } } @@ -497,16 +498,16 @@ func getRequestFilter(r *http.Request) *requestFilter { } type targetsStatusResult struct { - jobTargetsStatuses []*jobTargetsStatuses - droppedTargetsLabels []*promutils.Labels - emptyJobs []string - err error + jobTargetsStatuses []*jobTargetsStatuses + droppedTargets []droppedTarget + emptyJobs []string + err error } type targetLabels struct { - up bool - discoveredLabels *promutils.Labels - labels *promutils.Labels + up bool + originalLabels *promutils.Labels + labels *promutils.Labels } type targetLabelsByJob struct { jobName string @@ -515,6 +516,43 @@ type targetLabelsByJob struct { droppedTargets int } +func getRelabelContextByTargetID(targetID string) (*promrelabel.ParsedConfigs, *promutils.Labels, bool) { + var relabelConfigs *promrelabel.ParsedConfigs + var labels *promutils.Labels + found := false + + // Search for relabel context in tsmGlobal (aka active targets) + tsmGlobal.mu.Lock() + for sw := range tsmGlobal.m { + // The target is uniquely identified by a pointer to its original labels. + if getLabelsID(sw.Config.OriginalLabels) == targetID { + relabelConfigs = sw.Config.RelabelConfigs + labels = sw.Config.OriginalLabels + found = true + break + } + } + tsmGlobal.mu.Unlock() + + if found { + return relabelConfigs, labels, true + } + + // Search for relabel context in droppedTargetsMap (aka deleted targets) + droppedTargetsMap.mu.Lock() + for _, dt := range droppedTargetsMap.m { + if getLabelsID(dt.originalLabels) == targetID { + relabelConfigs = dt.relabelConfigs + labels = dt.originalLabels + found = true + break + } + } + droppedTargetsMap.mu.Unlock() + + return relabelConfigs, labels, found +} + func (tsr *targetsStatusResult) getTargetLabelsByJob() []*targetLabelsByJob { byJob := make(map[string]*targetLabelsByJob) for _, jts := range tsr.jobTargetsStatuses { @@ -529,14 +567,14 @@ func (tsr *targetsStatusResult) getTargetLabelsByJob() []*targetLabelsByJob { } m.activeTargets++ m.targets = append(m.targets, targetLabels{ - up: ts.up, - discoveredLabels: ts.sw.Config.OriginalLabels, - labels: ts.sw.Config.Labels, + up: ts.up, + originalLabels: ts.sw.Config.OriginalLabels, + labels: ts.sw.Config.Labels, }) } } - for _, labels := range tsr.droppedTargetsLabels { - jobName := labels.Get("job") + for _, dt := range tsr.droppedTargets { + jobName := dt.originalLabels.Get("job") m := byJob[jobName] if m == nil { m = &targetLabelsByJob{ @@ -546,7 +584,7 @@ func (tsr *targetsStatusResult) getTargetLabelsByJob() []*targetLabelsByJob { } m.droppedTargets++ m.targets = append(m.targets, targetLabels{ - discoveredLabels: labels, + originalLabels: dt.originalLabels, }) } a := make([]*targetLabelsByJob, 0, len(byJob)) diff --git a/lib/promscrape/targetstatus.qtpl b/lib/promscrape/targetstatus.qtpl index 53bc8b767..39ee1456c 100644 --- a/lib/promscrape/targetstatus.qtpl +++ b/lib/promscrape/targetstatus.qtpl @@ -103,7 +103,7 @@ {% func navbar() %}
- + VictoriaMetrics @@ -225,6 +225,7 @@ Endpoint State Labels + Debug relabeling Scrapes Errors Last Scrape @@ -237,7 +238,8 @@ {% for _, ts := range jts.targetsStatus %} {% code endpoint := ts.sw.Config.ScrapeURL - targetID := getTargetID(ts.sw) + // The target is uniquely identified by a pointer to its original labels. + targetID := getLabelsID(ts.sw.Config.OriginalLabels) lastScrapeDuration := ts.getDurationFromLastScrape() %} @@ -263,6 +265,9 @@ {%= formatLabels(ts.sw.Config.OriginalLabels) %}
+ + debug + {%d ts.scrapesTotal %} {%d ts.scrapesFailed %} @@ -304,8 +309,9 @@ Status - Discovered Labels + Discovered Labels Target Labels + Debug relabeling @@ -330,11 +336,15 @@ {% endif %} - {%= formatLabels(t.discoveredLabels) %} + {%= formatLabels(t.originalLabels) %} {%= formatLabels(t.labels) %} + + {% code targetID := getLabelsID(t.originalLabels) %} + debug + {% endfor %} diff --git a/lib/promscrape/targetstatus.qtpl.go b/lib/promscrape/targetstatus.qtpl.go index a560409cf..36bab50d2 100644 --- a/lib/promscrape/targetstatus.qtpl.go +++ b/lib/promscrape/targetstatus.qtpl.go @@ -355,7 +355,7 @@ func commonHeader() string { //line lib/promscrape/targetstatus.qtpl:103 func streamnavbar(qw422016 *qt422016.Writer) { //line lib/promscrape/targetstatus.qtpl:103 - qw422016.N().S(`
VictoriaMetrics
`) + qw422016.N().S(`
VictoriaMetrics
`) //line lib/promscrape/targetstatus.qtpl:112 } @@ -633,336 +633,350 @@ func streamscrapeJobTargets(qw422016 *qt422016.Writer, num int, jts *jobTargetsS //line lib/promscrape/targetstatus.qtpl:221 qw422016.N().D(num) //line lib/promscrape/targetstatus.qtpl:221 - qw422016.N().S(`" class="scrape-job table-responsive">`) -//line lib/promscrape/targetstatus.qtpl:237 + qw422016.N().S(`" class="scrape-job table-responsive">
EndpointStateLabelsScrapesErrorsLast ScrapeDurationSamplesLast error
`) +//line lib/promscrape/targetstatus.qtpl:238 for _, ts := range jts.targetsStatus { -//line lib/promscrape/targetstatus.qtpl:239 +//line lib/promscrape/targetstatus.qtpl:240 endpoint := ts.sw.Config.ScrapeURL - targetID := getTargetID(ts.sw) + // The target is uniquely identified by a pointer to its original labels. + targetID := getLabelsID(ts.sw.Config.OriginalLabels) lastScrapeDuration := ts.getDurationFromLastScrape() -//line lib/promscrape/targetstatus.qtpl:242 +//line lib/promscrape/targetstatus.qtpl:244 qw422016.N().S(``) -//line lib/promscrape/targetstatus.qtpl:278 +//line lib/promscrape/targetstatus.qtpl:283 } -//line lib/promscrape/targetstatus.qtpl:278 +//line lib/promscrape/targetstatus.qtpl:283 qw422016.N().S(`
EndpointStateLabelsDebug relabelingScrapesErrorsLast ScrapeDurationSamplesLast error
`) -//line lib/promscrape/targetstatus.qtpl:245 +//line lib/promscrape/targetstatus.qtpl:247 qw422016.E().S(endpoint) -//line lib/promscrape/targetstatus.qtpl:245 +//line lib/promscrape/targetstatus.qtpl:247 qw422016.N().S(` (response)`) -//line lib/promscrape/targetstatus.qtpl:251 +//line lib/promscrape/targetstatus.qtpl:253 if ts.up { -//line lib/promscrape/targetstatus.qtpl:251 +//line lib/promscrape/targetstatus.qtpl:253 qw422016.N().S(`UP`) -//line lib/promscrape/targetstatus.qtpl:253 +//line lib/promscrape/targetstatus.qtpl:255 } else { -//line lib/promscrape/targetstatus.qtpl:253 +//line lib/promscrape/targetstatus.qtpl:255 qw422016.N().S(`DOWN`) -//line lib/promscrape/targetstatus.qtpl:255 +//line lib/promscrape/targetstatus.qtpl:257 } -//line lib/promscrape/targetstatus.qtpl:255 +//line lib/promscrape/targetstatus.qtpl:257 qw422016.N().S(`
`) -//line lib/promscrape/targetstatus.qtpl:260 +//line lib/promscrape/targetstatus.qtpl:262 streamformatLabels(qw422016, ts.sw.Config.Labels) -//line lib/promscrape/targetstatus.qtpl:260 +//line lib/promscrape/targetstatus.qtpl:262 qw422016.N().S(`
`) -//line lib/promscrape/targetstatus.qtpl:266 - qw422016.N().D(ts.scrapesTotal) -//line lib/promscrape/targetstatus.qtpl:266 - qw422016.N().S(``) -//line lib/promscrape/targetstatus.qtpl:267 - qw422016.N().D(ts.scrapesFailed) -//line lib/promscrape/targetstatus.qtpl:267 - qw422016.N().S(``) +//line lib/promscrape/targetstatus.qtpl:265 + qw422016.N().S(`debug`) //line lib/promscrape/targetstatus.qtpl:271 - } else { + qw422016.N().D(ts.scrapesTotal) //line lib/promscrape/targetstatus.qtpl:271 - qw422016.N().S(`none`) -//line lib/promscrape/targetstatus.qtpl:273 - } -//line lib/promscrape/targetstatus.qtpl:273 - qw422016.N().S(``) -//line lib/promscrape/targetstatus.qtpl:274 - qw422016.N().D(int(ts.scrapeDuration)) -//line lib/promscrape/targetstatus.qtpl:274 - qw422016.N().S(`ms`) -//line lib/promscrape/targetstatus.qtpl:275 - qw422016.N().D(ts.samplesScraped) -//line lib/promscrape/targetstatus.qtpl:275 qw422016.N().S(``) +//line lib/promscrape/targetstatus.qtpl:272 + qw422016.N().D(ts.scrapesFailed) +//line lib/promscrape/targetstatus.qtpl:272 + qw422016.N().S(``) +//line lib/promscrape/targetstatus.qtpl:274 + if lastScrapeDuration < 365*24*time.Hour { +//line lib/promscrape/targetstatus.qtpl:275 + qw422016.N().D(int(lastScrapeDuration.Milliseconds())) +//line lib/promscrape/targetstatus.qtpl:275 + qw422016.N().S(`ms ago`) //line lib/promscrape/targetstatus.qtpl:276 - if ts.err != nil { -//line lib/promscrape/targetstatus.qtpl:276 - qw422016.E().S(ts.err.Error()) + } else { //line lib/promscrape/targetstatus.qtpl:276 + qw422016.N().S(`none`) +//line lib/promscrape/targetstatus.qtpl:278 } -//line lib/promscrape/targetstatus.qtpl:276 +//line lib/promscrape/targetstatus.qtpl:278 + qw422016.N().S(``) +//line lib/promscrape/targetstatus.qtpl:279 + qw422016.N().D(int(ts.scrapeDuration)) +//line lib/promscrape/targetstatus.qtpl:279 + qw422016.N().S(`ms`) +//line lib/promscrape/targetstatus.qtpl:280 + qw422016.N().D(ts.samplesScraped) +//line lib/promscrape/targetstatus.qtpl:280 + qw422016.N().S(``) +//line lib/promscrape/targetstatus.qtpl:281 + if ts.err != nil { +//line lib/promscrape/targetstatus.qtpl:281 + qw422016.E().S(ts.err.Error()) +//line lib/promscrape/targetstatus.qtpl:281 + } +//line lib/promscrape/targetstatus.qtpl:281 qw422016.N().S(`
`) -//line lib/promscrape/targetstatus.qtpl:284 +//line lib/promscrape/targetstatus.qtpl:289 } -//line lib/promscrape/targetstatus.qtpl:284 +//line lib/promscrape/targetstatus.qtpl:289 func writescrapeJobTargets(qq422016 qtio422016.Writer, num int, jts *jobTargetsStatuses) { -//line lib/promscrape/targetstatus.qtpl:284 +//line lib/promscrape/targetstatus.qtpl:289 qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promscrape/targetstatus.qtpl:284 +//line lib/promscrape/targetstatus.qtpl:289 streamscrapeJobTargets(qw422016, num, jts) -//line lib/promscrape/targetstatus.qtpl:284 +//line lib/promscrape/targetstatus.qtpl:289 qt422016.ReleaseWriter(qw422016) -//line lib/promscrape/targetstatus.qtpl:284 +//line lib/promscrape/targetstatus.qtpl:289 } -//line lib/promscrape/targetstatus.qtpl:284 +//line lib/promscrape/targetstatus.qtpl:289 func scrapeJobTargets(num int, jts *jobTargetsStatuses) string { -//line lib/promscrape/targetstatus.qtpl:284 +//line lib/promscrape/targetstatus.qtpl:289 qb422016 := qt422016.AcquireByteBuffer() -//line lib/promscrape/targetstatus.qtpl:284 +//line lib/promscrape/targetstatus.qtpl:289 writescrapeJobTargets(qb422016, num, jts) -//line lib/promscrape/targetstatus.qtpl:284 +//line lib/promscrape/targetstatus.qtpl:289 qs422016 := string(qb422016.B) -//line lib/promscrape/targetstatus.qtpl:284 +//line lib/promscrape/targetstatus.qtpl:289 qt422016.ReleaseByteBuffer(qb422016) -//line lib/promscrape/targetstatus.qtpl:284 +//line lib/promscrape/targetstatus.qtpl:289 return qs422016 -//line lib/promscrape/targetstatus.qtpl:284 +//line lib/promscrape/targetstatus.qtpl:289 } -//line lib/promscrape/targetstatus.qtpl:286 +//line lib/promscrape/targetstatus.qtpl:291 func streamdiscoveredTargets(qw422016 *qt422016.Writer, tsr *targetsStatusResult) { -//line lib/promscrape/targetstatus.qtpl:287 +//line lib/promscrape/targetstatus.qtpl:292 tljs := tsr.getTargetLabelsByJob() -//line lib/promscrape/targetstatus.qtpl:287 +//line lib/promscrape/targetstatus.qtpl:292 qw422016.N().S(`
`) -//line lib/promscrape/targetstatus.qtpl:290 +//line lib/promscrape/targetstatus.qtpl:295 for i, tlj := range tljs { -//line lib/promscrape/targetstatus.qtpl:291 +//line lib/promscrape/targetstatus.qtpl:296 streamdiscoveredJobTargets(qw422016, i, tlj) -//line lib/promscrape/targetstatus.qtpl:292 +//line lib/promscrape/targetstatus.qtpl:297 } -//line lib/promscrape/targetstatus.qtpl:292 +//line lib/promscrape/targetstatus.qtpl:297 qw422016.N().S(`
`) -//line lib/promscrape/targetstatus.qtpl:295 +//line lib/promscrape/targetstatus.qtpl:300 } -//line lib/promscrape/targetstatus.qtpl:295 +//line lib/promscrape/targetstatus.qtpl:300 func writediscoveredTargets(qq422016 qtio422016.Writer, tsr *targetsStatusResult) { -//line lib/promscrape/targetstatus.qtpl:295 +//line lib/promscrape/targetstatus.qtpl:300 qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promscrape/targetstatus.qtpl:295 +//line lib/promscrape/targetstatus.qtpl:300 streamdiscoveredTargets(qw422016, tsr) -//line lib/promscrape/targetstatus.qtpl:295 +//line lib/promscrape/targetstatus.qtpl:300 qt422016.ReleaseWriter(qw422016) -//line lib/promscrape/targetstatus.qtpl:295 +//line lib/promscrape/targetstatus.qtpl:300 } -//line lib/promscrape/targetstatus.qtpl:295 +//line lib/promscrape/targetstatus.qtpl:300 func discoveredTargets(tsr *targetsStatusResult) string { -//line lib/promscrape/targetstatus.qtpl:295 +//line lib/promscrape/targetstatus.qtpl:300 qb422016 := qt422016.AcquireByteBuffer() -//line lib/promscrape/targetstatus.qtpl:295 +//line lib/promscrape/targetstatus.qtpl:300 writediscoveredTargets(qb422016, tsr) -//line lib/promscrape/targetstatus.qtpl:295 +//line lib/promscrape/targetstatus.qtpl:300 qs422016 := string(qb422016.B) -//line lib/promscrape/targetstatus.qtpl:295 +//line lib/promscrape/targetstatus.qtpl:300 qt422016.ReleaseByteBuffer(qb422016) -//line lib/promscrape/targetstatus.qtpl:295 +//line lib/promscrape/targetstatus.qtpl:300 return qs422016 -//line lib/promscrape/targetstatus.qtpl:295 +//line lib/promscrape/targetstatus.qtpl:300 } -//line lib/promscrape/targetstatus.qtpl:297 +//line lib/promscrape/targetstatus.qtpl:302 func streamdiscoveredJobTargets(qw422016 *qt422016.Writer, num int, tlj *targetLabelsByJob) { -//line lib/promscrape/targetstatus.qtpl:297 +//line lib/promscrape/targetstatus.qtpl:302 qw422016.N().S(`

`) -//line lib/promscrape/targetstatus.qtpl:299 +//line lib/promscrape/targetstatus.qtpl:304 qw422016.E().S(tlj.jobName) -//line lib/promscrape/targetstatus.qtpl:299 +//line lib/promscrape/targetstatus.qtpl:304 qw422016.N().S(` `) -//line lib/promscrape/targetstatus.qtpl:299 +//line lib/promscrape/targetstatus.qtpl:304 qw422016.N().S(`(`) -//line lib/promscrape/targetstatus.qtpl:299 +//line lib/promscrape/targetstatus.qtpl:304 qw422016.N().D(tlj.activeTargets) -//line lib/promscrape/targetstatus.qtpl:299 +//line lib/promscrape/targetstatus.qtpl:304 qw422016.N().S(`/`) -//line lib/promscrape/targetstatus.qtpl:299 +//line lib/promscrape/targetstatus.qtpl:304 qw422016.N().D(tlj.activeTargets + tlj.droppedTargets) -//line lib/promscrape/targetstatus.qtpl:299 +//line lib/promscrape/targetstatus.qtpl:304 qw422016.N().S(` `) -//line lib/promscrape/targetstatus.qtpl:299 +//line lib/promscrape/targetstatus.qtpl:304 qw422016.N().S(`active)`) -//line lib/promscrape/targetstatus.qtpl:300 +//line lib/promscrape/targetstatus.qtpl:305 streamshowHideScrapeJobButtons(qw422016, num) -//line lib/promscrape/targetstatus.qtpl:300 +//line lib/promscrape/targetstatus.qtpl:305 qw422016.N().S(`

`) -//line lib/promscrape/targetstatus.qtpl:312 +//line lib/promscrape/targetstatus.qtpl:307 + qw422016.N().S(`" class="scrape-job table-responsive">
StatusDiscovered LabelsTarget Labels
`) +//line lib/promscrape/targetstatus.qtpl:318 for _, t := range tlj.targets { -//line lib/promscrape/targetstatus.qtpl:312 +//line lib/promscrape/targetstatus.qtpl:318 qw422016.N().S(` 0 { -//line lib/promscrape/targetstatus.qtpl:316 - qw422016.N().S(`class="alert alert-danger"`) -//line lib/promscrape/targetstatus.qtpl:318 - } else { -//line lib/promscrape/targetstatus.qtpl:318 - qw422016.N().S(`class="alert alert-warning"`) //line lib/promscrape/targetstatus.qtpl:320 + if !t.up { +//line lib/promscrape/targetstatus.qtpl:321 + qw422016.N().S(` `) +//line lib/promscrape/targetstatus.qtpl:321 + qw422016.N().S(`role="alert"`) +//line lib/promscrape/targetstatus.qtpl:321 + qw422016.N().S(` `) +//line lib/promscrape/targetstatus.qtpl:322 + if t.labels.Len() > 0 { +//line lib/promscrape/targetstatus.qtpl:322 + qw422016.N().S(`class="alert alert-danger"`) +//line lib/promscrape/targetstatus.qtpl:324 + } else { +//line lib/promscrape/targetstatus.qtpl:324 + qw422016.N().S(`class="alert alert-warning"`) +//line lib/promscrape/targetstatus.qtpl:326 } -//line lib/promscrape/targetstatus.qtpl:321 +//line lib/promscrape/targetstatus.qtpl:327 } -//line lib/promscrape/targetstatus.qtpl:321 +//line lib/promscrape/targetstatus.qtpl:327 qw422016.N().S(`>`) -//line lib/promscrape/targetstatus.qtpl:339 +//line lib/promscrape/targetstatus.qtpl:342 + qw422016.N().S(``) +//line lib/promscrape/targetstatus.qtpl:349 } -//line lib/promscrape/targetstatus.qtpl:339 +//line lib/promscrape/targetstatus.qtpl:349 qw422016.N().S(`
StatusDiscovered LabelsTarget LabelsDebug relabeling
`) -//line lib/promscrape/targetstatus.qtpl:324 +//line lib/promscrape/targetstatus.qtpl:330 if t.up { -//line lib/promscrape/targetstatus.qtpl:324 +//line lib/promscrape/targetstatus.qtpl:330 qw422016.N().S(`UP`) -//line lib/promscrape/targetstatus.qtpl:326 +//line lib/promscrape/targetstatus.qtpl:332 } else if t.labels.Len() > 0 { -//line lib/promscrape/targetstatus.qtpl:326 +//line lib/promscrape/targetstatus.qtpl:332 qw422016.N().S(`DOWN`) -//line lib/promscrape/targetstatus.qtpl:328 +//line lib/promscrape/targetstatus.qtpl:334 } else { -//line lib/promscrape/targetstatus.qtpl:328 +//line lib/promscrape/targetstatus.qtpl:334 qw422016.N().S(`DROPPED`) -//line lib/promscrape/targetstatus.qtpl:330 +//line lib/promscrape/targetstatus.qtpl:336 } -//line lib/promscrape/targetstatus.qtpl:330 - qw422016.N().S(``) -//line lib/promscrape/targetstatus.qtpl:333 - streamformatLabels(qw422016, t.discoveredLabels) -//line lib/promscrape/targetstatus.qtpl:333 - qw422016.N().S(``) //line lib/promscrape/targetstatus.qtpl:336 + qw422016.N().S(``) +//line lib/promscrape/targetstatus.qtpl:339 + streamformatLabels(qw422016, t.originalLabels) +//line lib/promscrape/targetstatus.qtpl:339 + qw422016.N().S(``) +//line lib/promscrape/targetstatus.qtpl:342 streamformatLabels(qw422016, t.labels) -//line lib/promscrape/targetstatus.qtpl:336 - qw422016.N().S(`
`) +//line lib/promscrape/targetstatus.qtpl:345 + targetID := getLabelsID(t.originalLabels) + +//line lib/promscrape/targetstatus.qtpl:345 + qw422016.N().S(`debug
`) -//line lib/promscrape/targetstatus.qtpl:343 +//line lib/promscrape/targetstatus.qtpl:353 } -//line lib/promscrape/targetstatus.qtpl:343 +//line lib/promscrape/targetstatus.qtpl:353 func writediscoveredJobTargets(qq422016 qtio422016.Writer, num int, tlj *targetLabelsByJob) { -//line lib/promscrape/targetstatus.qtpl:343 +//line lib/promscrape/targetstatus.qtpl:353 qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promscrape/targetstatus.qtpl:343 +//line lib/promscrape/targetstatus.qtpl:353 streamdiscoveredJobTargets(qw422016, num, tlj) -//line lib/promscrape/targetstatus.qtpl:343 +//line lib/promscrape/targetstatus.qtpl:353 qt422016.ReleaseWriter(qw422016) -//line lib/promscrape/targetstatus.qtpl:343 +//line lib/promscrape/targetstatus.qtpl:353 } -//line lib/promscrape/targetstatus.qtpl:343 +//line lib/promscrape/targetstatus.qtpl:353 func discoveredJobTargets(num int, tlj *targetLabelsByJob) string { -//line lib/promscrape/targetstatus.qtpl:343 +//line lib/promscrape/targetstatus.qtpl:353 qb422016 := qt422016.AcquireByteBuffer() -//line lib/promscrape/targetstatus.qtpl:343 +//line lib/promscrape/targetstatus.qtpl:353 writediscoveredJobTargets(qb422016, num, tlj) -//line lib/promscrape/targetstatus.qtpl:343 +//line lib/promscrape/targetstatus.qtpl:353 qs422016 := string(qb422016.B) -//line lib/promscrape/targetstatus.qtpl:343 +//line lib/promscrape/targetstatus.qtpl:353 qt422016.ReleaseByteBuffer(qb422016) -//line lib/promscrape/targetstatus.qtpl:343 +//line lib/promscrape/targetstatus.qtpl:353 return qs422016 -//line lib/promscrape/targetstatus.qtpl:343 +//line lib/promscrape/targetstatus.qtpl:353 } -//line lib/promscrape/targetstatus.qtpl:345 +//line lib/promscrape/targetstatus.qtpl:355 func streamshowHideScrapeJobButtons(qw422016 *qt422016.Writer, num int) { -//line lib/promscrape/targetstatus.qtpl:345 +//line lib/promscrape/targetstatus.qtpl:355 qw422016.N().S(``) -//line lib/promscrape/targetstatus.qtpl:354 +//line lib/promscrape/targetstatus.qtpl:364 } -//line lib/promscrape/targetstatus.qtpl:354 +//line lib/promscrape/targetstatus.qtpl:364 func writeshowHideScrapeJobButtons(qq422016 qtio422016.Writer, num int) { -//line lib/promscrape/targetstatus.qtpl:354 +//line lib/promscrape/targetstatus.qtpl:364 qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promscrape/targetstatus.qtpl:354 +//line lib/promscrape/targetstatus.qtpl:364 streamshowHideScrapeJobButtons(qw422016, num) -//line lib/promscrape/targetstatus.qtpl:354 +//line lib/promscrape/targetstatus.qtpl:364 qt422016.ReleaseWriter(qw422016) -//line lib/promscrape/targetstatus.qtpl:354 +//line lib/promscrape/targetstatus.qtpl:364 } -//line lib/promscrape/targetstatus.qtpl:354 +//line lib/promscrape/targetstatus.qtpl:364 func showHideScrapeJobButtons(num int) string { -//line lib/promscrape/targetstatus.qtpl:354 +//line lib/promscrape/targetstatus.qtpl:364 qb422016 := qt422016.AcquireByteBuffer() -//line lib/promscrape/targetstatus.qtpl:354 +//line lib/promscrape/targetstatus.qtpl:364 writeshowHideScrapeJobButtons(qb422016, num) -//line lib/promscrape/targetstatus.qtpl:354 +//line lib/promscrape/targetstatus.qtpl:364 qs422016 := string(qb422016.B) -//line lib/promscrape/targetstatus.qtpl:354 +//line lib/promscrape/targetstatus.qtpl:364 qt422016.ReleaseByteBuffer(qb422016) -//line lib/promscrape/targetstatus.qtpl:354 +//line lib/promscrape/targetstatus.qtpl:364 return qs422016 -//line lib/promscrape/targetstatus.qtpl:354 +//line lib/promscrape/targetstatus.qtpl:364 } -//line lib/promscrape/targetstatus.qtpl:356 +//line lib/promscrape/targetstatus.qtpl:366 func streamqueryArgs(qw422016 *qt422016.Writer, filter *requestFilter, override map[string]string) { -//line lib/promscrape/targetstatus.qtpl:358 +//line lib/promscrape/targetstatus.qtpl:368 showOnlyUnhealthy := "false" if filter.showOnlyUnhealthy { showOnlyUnhealthy = "true" @@ -980,126 +994,126 @@ func streamqueryArgs(qw422016 *qt422016.Writer, filter *requestFilter, override qa[k] = []string{v} } -//line lib/promscrape/targetstatus.qtpl:375 +//line lib/promscrape/targetstatus.qtpl:385 qw422016.E().S(qa.Encode()) -//line lib/promscrape/targetstatus.qtpl:376 +//line lib/promscrape/targetstatus.qtpl:386 } -//line lib/promscrape/targetstatus.qtpl:376 +//line lib/promscrape/targetstatus.qtpl:386 func writequeryArgs(qq422016 qtio422016.Writer, filter *requestFilter, override map[string]string) { -//line lib/promscrape/targetstatus.qtpl:376 +//line lib/promscrape/targetstatus.qtpl:386 qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promscrape/targetstatus.qtpl:376 +//line lib/promscrape/targetstatus.qtpl:386 streamqueryArgs(qw422016, filter, override) -//line lib/promscrape/targetstatus.qtpl:376 +//line lib/promscrape/targetstatus.qtpl:386 qt422016.ReleaseWriter(qw422016) -//line lib/promscrape/targetstatus.qtpl:376 +//line lib/promscrape/targetstatus.qtpl:386 } -//line lib/promscrape/targetstatus.qtpl:376 +//line lib/promscrape/targetstatus.qtpl:386 func queryArgs(filter *requestFilter, override map[string]string) string { -//line lib/promscrape/targetstatus.qtpl:376 +//line lib/promscrape/targetstatus.qtpl:386 qb422016 := qt422016.AcquireByteBuffer() -//line lib/promscrape/targetstatus.qtpl:376 +//line lib/promscrape/targetstatus.qtpl:386 writequeryArgs(qb422016, filter, override) -//line lib/promscrape/targetstatus.qtpl:376 +//line lib/promscrape/targetstatus.qtpl:386 qs422016 := string(qb422016.B) -//line lib/promscrape/targetstatus.qtpl:376 +//line lib/promscrape/targetstatus.qtpl:386 qt422016.ReleaseByteBuffer(qb422016) -//line lib/promscrape/targetstatus.qtpl:376 +//line lib/promscrape/targetstatus.qtpl:386 return qs422016 -//line lib/promscrape/targetstatus.qtpl:376 +//line lib/promscrape/targetstatus.qtpl:386 } -//line lib/promscrape/targetstatus.qtpl:378 +//line lib/promscrape/targetstatus.qtpl:388 func streamformatLabels(qw422016 *qt422016.Writer, labels *promutils.Labels) { -//line lib/promscrape/targetstatus.qtpl:379 +//line lib/promscrape/targetstatus.qtpl:389 labelsList := labels.GetLabels() -//line lib/promscrape/targetstatus.qtpl:379 +//line lib/promscrape/targetstatus.qtpl:389 qw422016.N().S(`{`) -//line lib/promscrape/targetstatus.qtpl:381 +//line lib/promscrape/targetstatus.qtpl:391 for i, label := range labelsList { -//line lib/promscrape/targetstatus.qtpl:382 +//line lib/promscrape/targetstatus.qtpl:392 qw422016.E().S(label.Name) -//line lib/promscrape/targetstatus.qtpl:382 +//line lib/promscrape/targetstatus.qtpl:392 qw422016.N().S(`=`) -//line lib/promscrape/targetstatus.qtpl:382 +//line lib/promscrape/targetstatus.qtpl:392 qw422016.E().Q(label.Value) -//line lib/promscrape/targetstatus.qtpl:383 +//line lib/promscrape/targetstatus.qtpl:393 if i+1 < len(labelsList) { -//line lib/promscrape/targetstatus.qtpl:383 +//line lib/promscrape/targetstatus.qtpl:393 qw422016.N().S(`,`) -//line lib/promscrape/targetstatus.qtpl:383 +//line lib/promscrape/targetstatus.qtpl:393 qw422016.N().S(` `) -//line lib/promscrape/targetstatus.qtpl:383 +//line lib/promscrape/targetstatus.qtpl:393 } -//line lib/promscrape/targetstatus.qtpl:384 +//line lib/promscrape/targetstatus.qtpl:394 } -//line lib/promscrape/targetstatus.qtpl:384 +//line lib/promscrape/targetstatus.qtpl:394 qw422016.N().S(`}`) -//line lib/promscrape/targetstatus.qtpl:386 +//line lib/promscrape/targetstatus.qtpl:396 } -//line lib/promscrape/targetstatus.qtpl:386 +//line lib/promscrape/targetstatus.qtpl:396 func writeformatLabels(qq422016 qtio422016.Writer, labels *promutils.Labels) { -//line lib/promscrape/targetstatus.qtpl:386 +//line lib/promscrape/targetstatus.qtpl:396 qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promscrape/targetstatus.qtpl:386 +//line lib/promscrape/targetstatus.qtpl:396 streamformatLabels(qw422016, labels) -//line lib/promscrape/targetstatus.qtpl:386 +//line lib/promscrape/targetstatus.qtpl:396 qt422016.ReleaseWriter(qw422016) -//line lib/promscrape/targetstatus.qtpl:386 +//line lib/promscrape/targetstatus.qtpl:396 } -//line lib/promscrape/targetstatus.qtpl:386 +//line lib/promscrape/targetstatus.qtpl:396 func formatLabels(labels *promutils.Labels) string { -//line lib/promscrape/targetstatus.qtpl:386 +//line lib/promscrape/targetstatus.qtpl:396 qb422016 := qt422016.AcquireByteBuffer() -//line lib/promscrape/targetstatus.qtpl:386 +//line lib/promscrape/targetstatus.qtpl:396 writeformatLabels(qb422016, labels) -//line lib/promscrape/targetstatus.qtpl:386 +//line lib/promscrape/targetstatus.qtpl:396 qs422016 := string(qb422016.B) -//line lib/promscrape/targetstatus.qtpl:386 +//line lib/promscrape/targetstatus.qtpl:396 qt422016.ReleaseByteBuffer(qb422016) -//line lib/promscrape/targetstatus.qtpl:386 +//line lib/promscrape/targetstatus.qtpl:396 return qs422016 -//line lib/promscrape/targetstatus.qtpl:386 +//line lib/promscrape/targetstatus.qtpl:396 } -//line lib/promscrape/targetstatus.qtpl:388 +//line lib/promscrape/targetstatus.qtpl:398 func streamerrorNotification(qw422016 *qt422016.Writer, err error) { -//line lib/promscrape/targetstatus.qtpl:388 +//line lib/promscrape/targetstatus.qtpl:398 qw422016.N().S(``) -//line lib/promscrape/targetstatus.qtpl:396 +//line lib/promscrape/targetstatus.qtpl:406 } -//line lib/promscrape/targetstatus.qtpl:396 +//line lib/promscrape/targetstatus.qtpl:406 func writeerrorNotification(qq422016 qtio422016.Writer, err error) { -//line lib/promscrape/targetstatus.qtpl:396 +//line lib/promscrape/targetstatus.qtpl:406 qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promscrape/targetstatus.qtpl:396 +//line lib/promscrape/targetstatus.qtpl:406 streamerrorNotification(qw422016, err) -//line lib/promscrape/targetstatus.qtpl:396 +//line lib/promscrape/targetstatus.qtpl:406 qt422016.ReleaseWriter(qw422016) -//line lib/promscrape/targetstatus.qtpl:396 +//line lib/promscrape/targetstatus.qtpl:406 } -//line lib/promscrape/targetstatus.qtpl:396 +//line lib/promscrape/targetstatus.qtpl:406 func errorNotification(err error) string { -//line lib/promscrape/targetstatus.qtpl:396 +//line lib/promscrape/targetstatus.qtpl:406 qb422016 := qt422016.AcquireByteBuffer() -//line lib/promscrape/targetstatus.qtpl:396 +//line lib/promscrape/targetstatus.qtpl:406 writeerrorNotification(qb422016, err) -//line lib/promscrape/targetstatus.qtpl:396 +//line lib/promscrape/targetstatus.qtpl:406 qs422016 := string(qb422016.B) -//line lib/promscrape/targetstatus.qtpl:396 +//line lib/promscrape/targetstatus.qtpl:406 qt422016.ReleaseByteBuffer(qb422016) -//line lib/promscrape/targetstatus.qtpl:396 +//line lib/promscrape/targetstatus.qtpl:406 return qs422016 -//line lib/promscrape/targetstatus.qtpl:396 +//line lib/promscrape/targetstatus.qtpl:406 } diff --git a/lib/promutils/labels.go b/lib/promutils/labels.go index f5b7163de..c16ba5100 100644 --- a/lib/promutils/labels.go +++ b/lib/promutils/labels.go @@ -35,7 +35,7 @@ func NewLabelsFromMap(m map[string]string) *Labels { // MarshalYAML implements yaml.Marshaler interface. func (x *Labels) MarshalYAML() (interface{}, error) { - m := x.toMap() + m := x.ToMap() return m, nil } @@ -51,7 +51,7 @@ func (x *Labels) UnmarshalYAML(unmarshal func(interface{}) error) error { // MarshalJSON returns JSON respresentation for x. func (x *Labels) MarshalJSON() ([]byte, error) { - m := x.toMap() + m := x.ToMap() return json.Marshal(m) } @@ -74,7 +74,8 @@ func (x *Labels) InitFromMap(m map[string]string) { x.Sort() } -func (x *Labels) toMap() map[string]string { +// ToMap returns a map for the given labels x. +func (x *Labels) ToMap() map[string]string { labels := x.GetLabels() m := make(map[string]string, len(labels)) for _, label := range labels { @@ -293,10 +294,21 @@ func PutLabels(x *Labels) { var labelsPool sync.Pool +// MustNewLabelsFromString creates labels from s, which can have the form `metric{labels}`. +// +// This function must be used only in tests. Use NewLabelsFromString in production code. +func MustNewLabelsFromString(metricWithLabels string) *Labels { + labels, err := NewLabelsFromString(metricWithLabels) + if err != nil { + logger.Panicf("BUG: cannot parse %q: %s", metricWithLabels, err) + } + return labels +} + // NewLabelsFromString creates labels from s, which can have the form `metric{labels}`. // // This function must be used only in tests -func NewLabelsFromString(metricWithLabels string) *Labels { +func NewLabelsFromString(metricWithLabels string) (*Labels, error) { stripDummyMetric := false if strings.HasPrefix(metricWithLabels, "{") { // Add a dummy metric name, since the parser needs it @@ -311,10 +323,10 @@ func NewLabelsFromString(metricWithLabels string) *Labels { err = fmt.Errorf("error during metric parse: %s", s) }) if err != nil { - logger.Panicf("BUG: cannot parse %q: %s", metricWithLabels, err) + return nil, err } if len(rows.Rows) != 1 { - logger.Panicf("BUG: unexpected number of rows parsed; got %d; want 1", len(rows.Rows)) + return nil, fmt.Errorf("unexpected number of rows parsed; got %d; want 1", len(rows.Rows)) } r := rows.Rows[0] var x Labels @@ -324,5 +336,5 @@ func NewLabelsFromString(metricWithLabels string) *Labels { for _, tag := range r.Tags { x.Add(tag.Key, tag.Value) } - return &x + return &x, nil } diff --git a/lib/promutils/labels_test.go b/lib/promutils/labels_test.go index f218d5f86..c57e4dec5 100644 --- a/lib/promutils/labels_test.go +++ b/lib/promutils/labels_test.go @@ -147,7 +147,7 @@ func TestLabelsAddFrom(t *testing.T) { func TestLabelsRemoveMetaLabels(t *testing.T) { f := func(metric, resultExpected string) { t.Helper() - labels := NewLabelsFromString(metric) + labels := MustNewLabelsFromString(metric) labels.RemoveMetaLabels() result := labels.String() if result != resultExpected { @@ -163,7 +163,7 @@ func TestLabelsRemoveMetaLabels(t *testing.T) { func TestLabelsRemoveLabelsWithDoubleUnderscorePrefix(t *testing.T) { f := func(metric, resultExpected string) { t.Helper() - labels := NewLabelsFromString(metric) + labels := MustNewLabelsFromString(metric) labels.RemoveLabelsWithDoubleUnderscorePrefix() result := labels.String() if result != resultExpected {