lib/promscrape: add exported_ prefix to metric names exported by scrape targets if they clash with automatically generated metrics

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3406

app/vmagent/README.md

@@ -317,7 +317,8 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
 
 ## Automatically generated metrics
 
-`vmagent` automatically generates the following metrics per each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format):
+`vmagent` automatically generates the following metrics per each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format)
+and attaches target-specific `instance` and `job` labels to these metrics:
 
 * `up` - this metric exposes `1` value on successful scrape and `0` value on unsuccessful scrape. This allows monitoring
   failing scrapes with the following [MetricsQL query](https://docs.victoriametrics.com/MetricsQL.html):
@@ -405,6 +406,9 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
   sum_over_time(scrape_series_limit_samples_dropped[1h]) > 0
   ```
 
+If the target exports metrics with names clashing with the automatically generated metric names, then `vmagent` automatically
+adds `exported_` prefix to these metric names, so they don't clash with automatically generated metric names.
+
 
 ## Relabeling
 

docs/CHANGELOG.md

@@ -15,6 +15,8 @@ The following tip changes can be tested by building VictoriaMetrics components f
 
 ## tip
 
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `exported_` prefix to metric names exported by scrape targets if these metric names clash with [automatically generated metrics](https://docs.victoriametrics.com/vmagent.html#automatically-generated-metrics) such as `up`, `scrape_samples_scraped`, etc. This prevents from corruption of automatically generated metrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3406).
+
 
 ## [v1.84.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.84.0)
 

docs/vmagent.md

@@ -321,7 +321,8 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
 
 ## Automatically generated metrics
 
-`vmagent` automatically generates the following metrics per each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format):
+`vmagent` automatically generates the following metrics per each scrape of every [Prometheus-compatible target](#how-to-collect-metrics-in-prometheus-format)
+and attaches target-specific `instance` and `job` labels to these metrics:
 
 * `up` - this metric exposes `1` value on successful scrape and `0` value on unsuccessful scrape. This allows monitoring
   failing scrapes with the following [MetricsQL query](https://docs.victoriametrics.com/MetricsQL.html):
@@ -409,6 +410,9 @@ Extra labels can be added to metrics collected by `vmagent` via the following me
   sum_over_time(scrape_series_limit_samples_dropped[1h]) > 0
   ```
 
+If the target exports metrics with names clashing with the automatically generated metric names, then `vmagent` automatically
+adds `exported_` prefix to these metric names, so they don't clash with automatically generated metric names.
+
 
 ## Relabeling
 

lib/promscrape/scrapework.go

@@ -811,6 +811,18 @@ type autoMetrics struct {
 	seriesLimitSamplesDropped int
 }
 
+func isAutoMetric(s string) bool {
+	switch s {
+	case "up", "scrape_duration_seconds", "scrape_samples_scraped",
+		"scrape_samples_post_metric_relabeling", "scrape_series_added",
+		"scrape_timeout_seconds", "scrape_samples_limit",
+		"scrape_series_limit_samples_dropped", "scrape_series_limit",
+		"scrape_series_current":
+		return true
+	}
+	return false
+}
+
 func (sw *scrapeWork) addAutoMetrics(am *autoMetrics, wc *writeRequestCtx, timestamp int64) {
 	sw.addAutoTimeseries(wc, "up", float64(am.up), timestamp)
 	sw.addAutoTimeseries(wc, "scrape_duration_seconds", am.scrapeDurationSeconds, timestamp)
@@ -842,8 +854,16 @@ func (sw *scrapeWork) addAutoTimeseries(wc *writeRequestCtx, name string, value
 }
 
 func (sw *scrapeWork) addRowToTimeseries(wc *writeRequestCtx, r *parser.Row, timestamp int64, needRelabel bool) {
+	metric := r.Metric
+	if needRelabel && isAutoMetric(metric) {
+		bb := bbPool.Get()
+		bb.B = append(bb.B, "exported_"...)
+		bb.B = append(bb.B, metric...)
+		metric = bytesutil.InternString(bytesutil.ToUnsafeString(bb.B))
+		bbPool.Put(bb)
+	}
 	labelsLen := len(wc.labels)
-	wc.labels = appendLabels(wc.labels, r.Metric, r.Tags, sw.Config.Labels, sw.Config.HonorLabels)
+	wc.labels = appendLabels(wc.labels, metric, r.Tags, sw.Config.Labels, sw.Config.HonorLabels)
 	if needRelabel {
 		wc.labels = sw.Config.MetricRelabelConfigs.Apply(wc.labels, labelsLen)
 	}
@@ -870,6 +890,8 @@ func (sw *scrapeWork) addRowToTimeseries(wc *writeRequestCtx, r *parser.Row, tim
 	})
 }
 
+var bbPool bytesutil.ByteBufferPool
+
 func appendLabels(dst []prompbmarshal.Label, metric string, src []parser.Tag, extraLabels []prompbmarshal.Label, honorLabels bool) []prompbmarshal.Label {
 	dstLen := len(dst)
 	dst = append(dst, prompbmarshal.Label{

lib/promscrape/scrapework_test.go

@@ -12,6 +12,30 @@ import (
 	parser "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus"
 )
 
+func TestIsAutoMetric(t *testing.T) {
+	f := func(metric string, resultExpected bool) {
+		t.Helper()
+		result := isAutoMetric(metric)
+		if result != resultExpected {
+			t.Fatalf("unexpected result for isAutoMetric(%q); got %v; want %v", metric, result, resultExpected)
+		}
+	}
+	f("up", true)
+	f("scrape_duration_seconds", true)
+	f("scrape_samples_scraped", true)
+	f("scrape_samples_post_metric_relabeling", true)
+	f("scrape_series_added", true)
+	f("scrape_timeout_seconds", true)
+	f("scrape_samples_limit", true)
+	f("scrape_series_limit_samples_dropped", true)
+	f("scrape_series_limit", true)
+	f("scrape_series_current", true)
+
+	f("foobar", false)
+	f("exported_up", false)
+	f("upx", false)
+}
+
 func TestAppendExtraLabels(t *testing.T) {
 	f := func(sourceLabels, extraLabels string, honorLabels bool, resultExpected string) {
 		t.Helper()
@@ -385,6 +409,25 @@ func TestScrapeWorkScrapeInternalSuccess(t *testing.T) {
 		scrape_series_added{job="xx",instance="foo.com"} 4 123
 		scrape_timeout_seconds{job="xx",instance="foo.com"} 42 123
 	`)
+	// Scrape metrics with names clashing with auto metrics
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3406
+	f(`
+		up{bar="baz"} 34.44
+		bar{a="b",c="d"} -3e4
+		scrape_series_added 3.435
+	`, &ScrapeWork{
+		ScrapeTimeout: time.Second * 42,
+	}, `
+		exported_up{bar="baz"} 34.44 123
+		exported_scrape_series_added 3.435 123
+		bar{a="b",c="d"} -3e4 123
+		up 1 123
+		scrape_samples_scraped 3 123
+		scrape_duration_seconds 0 123
+		scrape_samples_post_metric_relabeling 3 123
+		scrape_series_added 3 123
+		scrape_timeout_seconds 42 123
+	`)
 	// Scrape success with the given SampleLimit.
 	f(`
 		foo{bar="baz"} 34.44

lib/promscrape/scrapework_timing_test.go

@@ -8,6 +8,48 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 )
 
+func BenchmarkIsAutoMetricMiss(b *testing.B) {
+	metrics := []string{
+		"process_cpu_seconds_total",
+		"process_resident_memory_bytes",
+		"vm_tcplistener_read_calls_total",
+		"http_requests_total",
+		"node_cpu_seconds_total",
+	}
+	b.ReportAllocs()
+	b.SetBytes(1)
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			for _, metric := range metrics {
+				if isAutoMetric(metric) {
+					panic(fmt.Errorf("BUG: %q mustn't be detected as auto metric", metric))
+				}
+			}
+		}
+	})
+}
+
+func BenchmarkIsAutoMetricHit(b *testing.B) {
+	metrics := []string{
+		"up",
+		"scrape_duration_seconds",
+		"scrape_series_current",
+		"scrape_samples_scraped",
+		"scrape_series_added",
+	}
+	b.ReportAllocs()
+	b.SetBytes(1)
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			for _, metric := range metrics {
+				if !isAutoMetric(metric) {
+					panic(fmt.Errorf("BUG: %q must be detected as auto metric", metric))
+				}
+			}
+		}
+	})
+}
+
 func BenchmarkScrapeWorkScrapeInternal(b *testing.B) {
 	data := `
 vm_tcplistener_accepts_total{name="http", addr=":80"} 1443