From 8ce5b095b7b3f8531dc55de36c22618162717bb0 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 28 Nov 2022 18:37:06 -0800 Subject: [PATCH] lib/promscrape: add `exported_` prefix to metric names exported by scrape targets if they clash with automatically generated metrics Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3406 --- app/vmagent/README.md | 6 +++- docs/CHANGELOG.md | 2 ++ docs/vmagent.md | 6 +++- lib/promscrape/scrapework.go | 24 ++++++++++++- lib/promscrape/scrapework_test.go | 43 ++++++++++++++++++++++++ lib/promscrape/scrapework_timing_test.go | 42 +++++++++++++++++++++++ 6 files changed, 120 insertions(+), 3 deletions(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 456bd267b1..e4da010bb4 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -317,7 +317,8 @@ Extra labels can be added to metrics collected by `vmagent` via the following me ## Automatically generated metrics -`vmagent` automatically generates the following metrics per each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format): +`vmagent` automatically generates the following metrics per each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format) +and attaches target-specific `instance` and `job` labels to these metrics: * `up` - this metric exposes `1` value on successful scrape and `0` value on unsuccessful scrape. This allows monitoring failing scrapes with the following [MetricsQL query](https://docs.victoriametrics.com/MetricsQL.html): @@ -405,6 +406,9 @@ Extra labels can be added to metrics collected by `vmagent` via the following me sum_over_time(scrape_series_limit_samples_dropped[1h]) > 0 ``` +If the target exports metrics with names clashing with the automatically generated metric names, then `vmagent` automatically +adds `exported_` prefix to these metric names, so they don't clash with automatically generated metric names. + ## Relabeling diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 58812c8de9..4f6931ec71 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,8 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `exported_` prefix to metric names exported by scrape targets if these metric names clash with [automatically generated metrics](https://docs.victoriametrics.com/vmagent.html#automatically-generated-metrics) such as `up`, `scrape_samples_scraped`, etc. This prevents from corruption of automatically generated metrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3406). + ## [v1.84.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.84.0) diff --git a/docs/vmagent.md b/docs/vmagent.md index 70b35ba74b..87318c0a0f 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -321,7 +321,8 @@ Extra labels can be added to metrics collected by `vmagent` via the following me ## Automatically generated metrics -`vmagent` automatically generates the following metrics per each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format): +`vmagent` automatically generates the following metrics per each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format) +and attaches target-specific `instance` and `job` labels to these metrics: * `up` - this metric exposes `1` value on successful scrape and `0` value on unsuccessful scrape. This allows monitoring failing scrapes with the following [MetricsQL query](https://docs.victoriametrics.com/MetricsQL.html): @@ -409,6 +410,9 @@ Extra labels can be added to metrics collected by `vmagent` via the following me sum_over_time(scrape_series_limit_samples_dropped[1h]) > 0 ``` +If the target exports metrics with names clashing with the automatically generated metric names, then `vmagent` automatically +adds `exported_` prefix to these metric names, so they don't clash with automatically generated metric names. + ## Relabeling diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go index fd17e903f8..97c6f460a4 100644 --- a/lib/promscrape/scrapework.go +++ b/lib/promscrape/scrapework.go @@ -811,6 +811,18 @@ type autoMetrics struct { seriesLimitSamplesDropped int } +func isAutoMetric(s string) bool { + switch s { + case "up", "scrape_duration_seconds", "scrape_samples_scraped", + "scrape_samples_post_metric_relabeling", "scrape_series_added", + "scrape_timeout_seconds", "scrape_samples_limit", + "scrape_series_limit_samples_dropped", "scrape_series_limit", + "scrape_series_current": + return true + } + return false +} + func (sw *scrapeWork) addAutoMetrics(am *autoMetrics, wc *writeRequestCtx, timestamp int64) { sw.addAutoTimeseries(wc, "up", float64(am.up), timestamp) sw.addAutoTimeseries(wc, "scrape_duration_seconds", am.scrapeDurationSeconds, timestamp) @@ -842,8 +854,16 @@ func (sw *scrapeWork) addAutoTimeseries(wc *writeRequestCtx, name string, value } func (sw *scrapeWork) addRowToTimeseries(wc *writeRequestCtx, r *parser.Row, timestamp int64, needRelabel bool) { + metric := r.Metric + if needRelabel && isAutoMetric(metric) { + bb := bbPool.Get() + bb.B = append(bb.B, "exported_"...) + bb.B = append(bb.B, metric...) + metric = bytesutil.InternString(bytesutil.ToUnsafeString(bb.B)) + bbPool.Put(bb) + } labelsLen := len(wc.labels) - wc.labels = appendLabels(wc.labels, r.Metric, r.Tags, sw.Config.Labels, sw.Config.HonorLabels) + wc.labels = appendLabels(wc.labels, metric, r.Tags, sw.Config.Labels, sw.Config.HonorLabels) if needRelabel { wc.labels = sw.Config.MetricRelabelConfigs.Apply(wc.labels, labelsLen) } @@ -870,6 +890,8 @@ func (sw *scrapeWork) addRowToTimeseries(wc *writeRequestCtx, r *parser.Row, tim }) } +var bbPool bytesutil.ByteBufferPool + func appendLabels(dst []prompbmarshal.Label, metric string, src []parser.Tag, extraLabels []prompbmarshal.Label, honorLabels bool) []prompbmarshal.Label { dstLen := len(dst) dst = append(dst, prompbmarshal.Label{ diff --git a/lib/promscrape/scrapework_test.go b/lib/promscrape/scrapework_test.go index 226f42e4aa..f068044652 100644 --- a/lib/promscrape/scrapework_test.go +++ b/lib/promscrape/scrapework_test.go @@ -12,6 +12,30 @@ import ( parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus" ) +func TestIsAutoMetric(t *testing.T) { + f := func(metric string, resultExpected bool) { + t.Helper() + result := isAutoMetric(metric) + if result != resultExpected { + t.Fatalf("unexpected result for isAutoMetric(%q); got %v; want %v", metric, result, resultExpected) + } + } + f("up", true) + f("scrape_duration_seconds", true) + f("scrape_samples_scraped", true) + f("scrape_samples_post_metric_relabeling", true) + f("scrape_series_added", true) + f("scrape_timeout_seconds", true) + f("scrape_samples_limit", true) + f("scrape_series_limit_samples_dropped", true) + f("scrape_series_limit", true) + f("scrape_series_current", true) + + f("foobar", false) + f("exported_up", false) + f("upx", false) +} + func TestAppendExtraLabels(t *testing.T) { f := func(sourceLabels, extraLabels string, honorLabels bool, resultExpected string) { t.Helper() @@ -385,6 +409,25 @@ func TestScrapeWorkScrapeInternalSuccess(t *testing.T) { scrape_series_added{job="xx",instance="foo.com"} 4 123 scrape_timeout_seconds{job="xx",instance="foo.com"} 42 123 `) + // Scrape metrics with names clashing with auto metrics + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3406 + f(` + up{bar="baz"} 34.44 + bar{a="b",c="d"} -3e4 + scrape_series_added 3.435 + `, &ScrapeWork{ + ScrapeTimeout: time.Second * 42, + }, ` + exported_up{bar="baz"} 34.44 123 + exported_scrape_series_added 3.435 123 + bar{a="b",c="d"} -3e4 123 + up 1 123 + scrape_samples_scraped 3 123 + scrape_duration_seconds 0 123 + scrape_samples_post_metric_relabeling 3 123 + scrape_series_added 3 123 + scrape_timeout_seconds 42 123 + `) // Scrape success with the given SampleLimit. f(` foo{bar="baz"} 34.44 diff --git a/lib/promscrape/scrapework_timing_test.go b/lib/promscrape/scrapework_timing_test.go index 91e8fc9648..bd83eb6ae1 100644 --- a/lib/promscrape/scrapework_timing_test.go +++ b/lib/promscrape/scrapework_timing_test.go @@ -8,6 +8,48 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" ) +func BenchmarkIsAutoMetricMiss(b *testing.B) { + metrics := []string{ + "process_cpu_seconds_total", + "process_resident_memory_bytes", + "vm_tcplistener_read_calls_total", + "http_requests_total", + "node_cpu_seconds_total", + } + b.ReportAllocs() + b.SetBytes(1) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + for _, metric := range metrics { + if isAutoMetric(metric) { + panic(fmt.Errorf("BUG: %q mustn't be detected as auto metric", metric)) + } + } + } + }) +} + +func BenchmarkIsAutoMetricHit(b *testing.B) { + metrics := []string{ + "up", + "scrape_duration_seconds", + "scrape_series_current", + "scrape_samples_scraped", + "scrape_series_added", + } + b.ReportAllocs() + b.SetBytes(1) + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + for _, metric := range metrics { + if !isAutoMetric(metric) { + panic(fmt.Errorf("BUG: %q must be detected as auto metric", metric)) + } + } + } + }) +} + func BenchmarkScrapeWorkScrapeInternal(b *testing.B) { data := ` vm_tcplistener_accepts_total{name="http", addr=":80"} 1443