app/vmagent/remotewrite: fix data race when extra labels are added to samples before sending them to multiple remote storage systems

See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4972

app/vmagent/remotewrite/relabel.go

@@ -127,9 +127,6 @@ func (rctx *relabelCtx) appendExtraLabels(tss []prompbmarshal.TimeSeries, extraL
 		labels = append(labels, ts.Labels...)
 		for j := range extraLabels {
 			extraLabel := extraLabels[j]
-			if *usePromCompatibleNaming {
-				extraLabel.Name = promrelabel.SanitizeLabelName(extraLabel.Name)
-			}
 			tmp := promrelabel.GetLabelByName(labels[labelsLen:], extraLabel.Name)
 			if tmp != nil {
 				tmp.Value = extraLabel.Value

app/vmagent/remotewrite/relabel_test.go

@@ -40,6 +40,7 @@ func TestApplyRelabeling(t *testing.T) {
 
 func TestAppendExtraLabels(t *testing.T) {
 	f := func(extraLabels []prompbmarshal.Label, sTss, sExpTss string) {
+		t.Helper()
 		rctx := &relabelCtx{}
 		tss, expTss := parseSeries(sTss), parseSeries(sExpTss)
 		rctx.appendExtraLabels(tss, extraLabels)
@@ -55,7 +56,7 @@ func TestAppendExtraLabels(t *testing.T) {
 
 	oldVal := *usePromCompatibleNaming
 	*usePromCompatibleNaming = true
-	f([]prompbmarshal.Label{{Name: "foo.bar", Value: "baz"}}, "up", `up{foo_bar="baz"}`)
+	f([]prompbmarshal.Label{{Name: "foo.bar", Value: "baz"}}, "up", `up{foo.bar="baz"}`)
 	*usePromCompatibleNaming = oldVal
 }
 

app/vmagent/remotewrite/remotewrite.go

@@ -719,15 +719,26 @@ func dropAggregatedSeries(src []prompbmarshal.TimeSeries, matchIdxs []byte, drop
 }
 
 func (rwctx *remoteWriteCtx) pushInternal(tss []prompbmarshal.TimeSeries) {
+	var rctx *relabelCtx
+	var v *[]prompbmarshal.TimeSeries
 	if len(labelsGlobal) > 0 {
-		rctx := getRelabelCtx()
+		// Make a copy of tss before adding extra labels in order to prevent
-		defer putRelabelCtx(rctx)
+		// from affecting time series for other remoteWrite.url configs.
+		rctx = getRelabelCtx()
+		v = tssPool.Get().(*[]prompbmarshal.TimeSeries)
+		tss = append(*v, tss...)
 		rctx.appendExtraLabels(tss, labelsGlobal)
 	}
 
 	pss := rwctx.pss
 	idx := atomic.AddUint64(&rwctx.pssNextIdx, 1) % uint64(len(pss))
 	pss[idx].Push(tss)
+
+	if rctx != nil {
+		*v = prompbmarshal.ResetTimeSeries(tss)
+		tssPool.Put(v)
+		putRelabelCtx(rctx)
+	}
 }
 
 func (rwctx *remoteWriteCtx) reinitStreamAggr() {

docs/CHANGELOG.md

@@ -52,6 +52,7 @@ ssue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4825) and [these
 * BUGFIX: properly build production armv5 binaries for `GOARCH=arm`. This has been broken after the upgrading of Go builder to Go1.21.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4965).
 * BUGFIX: [vmselect](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): return `503 Service Unavailable` status code when [partial responses](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#cluster-availability) are denied and some of `vmstorage` nodes are temporarily unavailable. Previously `422 Unprocessable Entiry` status code was mistakenly returned in this case, which could prevent from automatic recovery by re-sending the request to healthy cluster replica in another availability zone.
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix the bug when Group's `params` fields with multiple values were overriding each other instead of adding up. The bug was introduced in [this commit](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/eccecdf177115297fa1dc4d42d38e23de9a9f2cb) starting from [v1.91.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.91.1). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4908).
+* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): fix possble corruption of labels in the collected samples if `-remoteWrite.label` is set toghether with multiple `-remoteWrite.url` options. The bug has been introduced in [v1.93.1](#v1931). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4972).
 
 ## [v1.93.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.93.3)