From 5aaaa686a4061df5870ba872a415511bb8a8e645 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sat, 11 Sep 2021 10:51:04 +0300 Subject: [PATCH] lib/promscrape: send stale markers for disappeared metrics like Prometheus does --- app/vmagent/README.md | 10 +- docs/CHANGELOG.md | 1 + lib/promscrape/scrapework.go | 35 +-- lib/protoparser/prometheus/parser.go | 215 ++++++++++++++++++ lib/protoparser/prometheus/parser_test.go | 85 +++++++ .../prometheus/parser_timing_test.go | 110 +++++++++ 6 files changed, 440 insertions(+), 16 deletions(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 3695228953..fae337dc57 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -292,7 +292,15 @@ You can read more about relabeling in the following articles: ## Prometheus staleness markers -Starting from [v1.64.0](https://docs.victoriametrics.com/CHANGELOG.html#v1640), `vmagent` sends [Prometheus staleness markers](https://www.robustperception.io/staleness-and-promql) for scraped metrics when the scrape target is removed from the list of targets. Prometheus staleness markers aren't sent in [stream parsing mode](#stream-parsing-mode) or if `-promscrape.noStaleMarkers` command-line is set. +`vmagent` sends [Prometheus staleness markers](https://www.robustperception.io/staleness-and-promql) to `-remoteWrite.url` in the following cases: + +* If they are passed to `vmagent` via [Prometheus remote_write protocol](#prometheus-remote_write-proxy). +* If the metric disappears from the list of scraped metrics, then stale marker is sent to this particular metrics. +* If the scrape target becomes temporarily unavailable, then stale markers are sent for all the metrics scraped from this target. +* If the scrape target is removed from the list of targets, then stale markers are sent for all the metrics scraped from this target. +* Stale markers are sent for all the scraped metrics on graceful shutdown of `vmagent`. + +Prometheus staleness markers aren't sent in [stream parsing mode](#stream-parsing-mode) or if `-promscrape.noStaleMarkers` command-line is set. ## Stream parsing mode diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 559d8ef21d..3ce5714a0a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,7 @@ sort: 15 ## tip * FEATURE: vmalert: add web UI with the list of alerting groups, alerts and alert statuses. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1602). +* FEATURE: vmagent: send stale markers for disappeared metrics like Prometheus does. Previously stale markers were sent only when the scrape target disappears or when it becomes temporarily unavailable. See [these docs](https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers) for details. * FEATURE: vmagent: add ability to set `series_limit` option for a particular scrape target via `__series_limit__` label. This allows setting the limit on the number of time series on a per-target basis. See [these docs](https://docs.victoriametrics.com/vmagent.html#cardinality-limiter) for details. * FEATURE: vmagent: add ability to set `stream_parse` option for a particular scrape target via `__stream_parse__` label. This allows managing the stream parsing mode on a per-target basis. See [these docs](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode) for details. * FEATURE: add new relabeling actions: `keep_metrics` and `drop_metrics`. This simplifies metrics filtering by metric names. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details. diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go index ea19d516b3..68b342b315 100644 --- a/lib/promscrape/scrapework.go +++ b/lib/promscrape/scrapework.go @@ -248,7 +248,7 @@ func (sw *scrapeWork) run(stopCh <-chan struct{}) { select { case <-stopCh: t := time.Now().UnixNano() / 1e6 - sw.sendStaleMarkersForLastScrape(t, true) + sw.sendStaleSeries("", t, true) if sw.seriesLimiter != nil { sw.seriesLimiter.MustStop() } @@ -344,9 +344,8 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error tsmGlobal.Update(sw.Config, sw.ScrapeGroup, up == 1, realTimestamp, int64(duration*1000), samplesScraped, err) if up == 0 { bodyString = "" - sw.sendStaleMarkersForLastScrape(scrapeTimestamp, false) } - sw.updateLastScrape(bodyString) + sw.sendStaleSeries(bodyString, scrapeTimestamp, false) return err } @@ -519,23 +518,29 @@ func (sw *scrapeWork) updateSeriesAdded(wc *writeRequestCtx) { wc.writeRequest.Timeseries = dstSeries } -func (sw *scrapeWork) updateLastScrape(response string) { +func (sw *scrapeWork) sendStaleSeries(currScrape string, timestamp int64, addAutoSeries bool) { if *noStaleMarkers { return } - sw.lastScrape = append(sw.lastScrape[:0], response...) -} - -func (sw *scrapeWork) sendStaleMarkersForLastScrape(timestamp int64, addAutoSeries bool) { - bodyString := bytesutil.ToUnsafeString(sw.lastScrape) - if len(bodyString) == 0 && !addAutoSeries { + lastScrape := bytesutil.ToUnsafeString(sw.lastScrape) + if parser.AreIdenticalSeriesFast(lastScrape, currScrape) { + // Fast path: the current scrape contains the same set of series as the previous scrape. return } + // Slow path: the current scrape contains different set of series than the previous scrape. + // Detect missing series in the current scrape and send stale markers for them. + bodyString := lastScrape + if currScrape != "" { + bodyString = parser.GetDiffWithStaleRows(lastScrape, currScrape) + } wc := writeRequestCtxPool.Get(sw.prevLabelsLen) - wc.rows.UnmarshalWithErrLogger(bodyString, sw.logError) - srcRows := wc.rows.Rows - for i := range srcRows { - sw.addRowToTimeseries(wc, &srcRows[i], timestamp, true) + defer writeRequestCtxPool.Put(wc) + if bodyString != "" { + wc.rows.Unmarshal(bodyString) + srcRows := wc.rows.Rows + for i := range srcRows { + sw.addRowToTimeseries(wc, &srcRows[i], timestamp, true) + } } if addAutoSeries { sw.addAutoTimeseries(wc, "up", 0, timestamp) @@ -556,7 +561,7 @@ func (sw *scrapeWork) sendStaleMarkersForLastScrape(timestamp int64, addAutoSeri } } sw.pushData(&wc.writeRequest) - writeRequestCtxPool.Put(wc) + sw.lastScrape = append(sw.lastScrape[:0], currScrape...) } func (sw *scrapeWork) finalizeSeriesAdded(lastScrapeSize int) int { diff --git a/lib/protoparser/prometheus/parser.go b/lib/protoparser/prometheus/parser.go index 69c61cf5ce..dd250e1aa4 100644 --- a/lib/protoparser/prometheus/parser.go +++ b/lib/protoparser/prometheus/parser.go @@ -2,6 +2,7 @@ package prometheus import ( "fmt" + "strconv" "strings" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" @@ -364,3 +365,217 @@ func prevBackslashesCount(s string) int { } return n } + +// GetDiffWithStaleRows returns rows from s1, which are missing in s2. +// +// The returned rows have default value 0 and have no timestamps. +func GetDiffWithStaleRows(s1, s2 string) string { + var r1, r2 Rows + r1.Unmarshal(s1) + r2.Unmarshal(s2) + rows1 := r1.Rows + rows2 := r2.Rows + m := make(map[string]bool, len(rows2)) + for i := range rows2 { + r := &rows2[i] + key := marshalMetricNameWithTags(r) + m[key] = true + } + var diff []byte + for i := range rows1 { + r := &rows1[i] + key := marshalMetricNameWithTags(r) + if !m[key] { + logger.Infof("missing %s", key) + diff = append(diff, key...) + diff = append(diff, " 0\n"...) + } else { + logger.Infof("found %s", key) + } + } + return string(diff) +} + +func marshalMetricNameWithTags(r *Row) string { + if len(r.Tags) == 0 { + return r.Metric + } + var b []byte + b = append(b, r.Metric...) + b = append(b, '{') + for i, t := range r.Tags { + b = append(b, t.Key...) + b = append(b, '=') + b = strconv.AppendQuote(b, t.Value) + if i+1 < len(r.Tags) { + b = append(b, ',') + } + } + b = append(b, '}') + return string(b) +} + +// AreIdenticalSeriesFast returns true if s1 and s2 contains identical Prometheus series with possible different values. +// +// This function is optimized for speed. +func AreIdenticalSeriesFast(s1, s2 string) bool { + for { + if len(s1) == 0 { + // The last byte on the line reached. + return len(s2) == 0 + } + if len(s2) == 0 { + // The last byte on s2 reached, while s1 has non-empty contents. + return false + } + + // Extract the next pair of lines from s1 and s2. + var x1, x2 string + n1 := strings.IndexByte(s1, '\n') + if n1 < 0 { + x1 = s1 + s1 = "" + } else { + x1 = s1[:n1] + s1 = s1[n1+1:] + } + if n := strings.IndexByte(x1, '#'); n >= 0 { + // Drop comment. + x1 = x1[:n] + } + n2 := strings.IndexByte(s2, '\n') + if n2 < 0 { + if n1 >= 0 { + return false + } + x2 = s2 + s2 = "" + } else { + if n1 < 0 { + return false + } + x2 = s2[:n2] + s2 = s2[n2+1:] + } + if n := strings.IndexByte(x2, '#'); n >= 0 { + // Drop comment. + x2 = x2[:n] + } + + // Skip whitespaces in front of lines + for len(x1) > 0 && x1[0] == ' ' { + if len(x2) == 0 || x2[0] != ' ' { + return false + } + x1 = x1[1:] + x2 = x2[1:] + } + if len(x1) == 0 { + // The last byte on x1 reached. + if len(x2) != 0 { + return false + } + continue + } + if len(x2) == 0 { + // The last byte on x2 reached, while x1 has non-empty contents. + return false + } + // Compare metric names + n := strings.IndexByte(x1, ' ') + if n < 0 { + // Invalid Prometheus line - it must contain at least a single space between metric name and value + return false + } + n++ + if n > len(x2) || x1[:n] != x2[:n] { + // Metric names mismatch + return false + } + x1 = x1[n:] + x2 = x2[n:] + + // The space could belong to metric name in the following cases: + // foo {bar="baz"} 1 + // foo{ bar="baz"} 2 + // foo{bar="baz", aa="b"} 3 + // foo{bar="b az"} 4 + // foo 5 + // Continue comparing the remaining parts until space or newline. + for { + n1 := strings.IndexByte(x1, ' ') + if n1 < 0 { + // Fast path. + // Treat x1 as a value. + // Skip values at x1 and x2. + n2 := strings.IndexByte(x2, ' ') + if n2 >= 0 { + // x2 contains additional parts. + return false + } + break + } + n1++ + // Slow path. + // The x1[:n1] can be either a part of metric name or a value if timestamp is present: + // foo 12 34 + if isNumeric(x1[:n1-1]) { + // Skip numeric part (most likely a value before timestamp) in x1 and x2 + n2 := strings.IndexByte(x2, ' ') + if n2 < 0 { + // x2 contains less parts than x1 + return false + } + n2++ + if !isNumeric(x2[:n2-1]) { + // x1 contains numeric part, while x2 contains non-numeric part + return false + } + x1 = x1[n1:] + x2 = x2[n2:] + } else { + // The non-numeric part from x1 must match the corresponding part from x2. + if n1 > len(x2) || x1[:n1] != x2[:n1] { + // Parts mismatch + return false + } + x1 = x1[n1:] + x2 = x2[n1:] + } + } + } +} + +func isNumeric(s string) bool { + for i := 0; i < len(s); i++ { + if numericChars[s[i]] { + continue + } + if i == 0 && s == "NaN" || s == "nan" || s == "Inf" || s == "inf" { + return true + } + if i == 1 && (s[0] == '-' || s[0] == '+') && (s[1:] == "Inf" || s[1:] == "inf") { + return true + } + return false + } + return true +} + +var numericChars = [256]bool{ + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + '-': true, + '+': true, + 'e': true, + 'E': true, + '.': true, +} diff --git a/lib/protoparser/prometheus/parser_test.go b/lib/protoparser/prometheus/parser_test.go index eef1497130..27812c9c49 100644 --- a/lib/protoparser/prometheus/parser_test.go +++ b/lib/protoparser/prometheus/parser_test.go @@ -6,6 +6,87 @@ import ( "testing" ) +func TestGetDiffWithStaleRows(t *testing.T) { + f := func(s1, s2, resultExpected string) { + t.Helper() + result := GetDiffWithStaleRows(s1, s2) + if result != resultExpected { + t.Fatalf("unexpected result for GetDiffWithStaleRows(%q, %q); got %q; want %q", s1, s2, result, resultExpected) + } + } + f("", "", "") + f("", "foo 1", "") + f(" ", "foo 1", "") + f("foo 123", "", "foo 0\n") + f("foo 123", "bar 3", "foo 0\n") + f("foo 123", "bar 3\nfoo 344", "") + f("foo{x=\"y\", z=\"a a a\"} 123", "bar 3\nfoo{x=\"y\", z=\"b b b\"} 344", "foo{x=\"y\",z=\"a a a\"} 0\n") + f("foo{bar=\"baz\"} 123\nx 3.4 5\ny 5 6", "x 34 342", "foo{bar=\"baz\"} 0\ny 0\n") +} + +func TestAreIdenticalSeriesFast(t *testing.T) { + f := func(s1, s2 string, resultExpected bool) { + t.Helper() + result := AreIdenticalSeriesFast(s1, s2) + if result != resultExpected { + t.Fatalf("unexpected result for AreIdenticalSeries(%q, %q); got %v; want %v", s1, s2, result, resultExpected) + } + } + f("", "", true) + f("", "a 1", false) // different number of metrics + f(" ", " a 1", false) // different number of metrics + f("a 1", "", false) // different number of metrics + f(" a 1", " ", false) // different number of metrics + f("foo", "foo", false) // missing value + f("foo 1", "foo 1", true) + f("foo 1", "foo 2", true) + f("foo 1 ", "foo 2 ", true) + f("foo 1 ", "foo 2 ", false) // different number of spaces + f("foo 1 ", "foo 2 ", false) // different number of spaces + f("foo nan", "foo -inf", true) + f("foo 1 # coment x", "foo 2 #comment y", true) + f(" foo 1", " foo 1", true) + f(" foo 1", " foo 1", false) // different number of spaces in front of metric + f(" foo 1", " foo 1", false) // different number of spaces in front of metric + f("foo 1", "bar 1", false) // different metric name + f("foo 1", "fooo 1", false) // different metric name + f("foo 123", "foo 32.32", true) + f(`foo{bar="x"} -3.3e-6`, `foo{bar="x"} 23343`, true) + f(`foo{} 1`, `foo{} 234`, true) + f(`foo {x="y x" } 234`, `foo {x="y x" } 43.342`, true) + f(`foo {x="y x"} 234`, `foo{x="y x"} 43.342`, false) // different spaces + f("foo 2\nbar 3", "foo 34.43\nbar -34.3", true) + f("foo 2\nbar 3", "foo 34.43\nbarz -34.3", false) // different metric names + f("\nfoo 13\n", "\nfoo 3.4\n", true) + f("\nfoo 13", "\nfoo 3.4\n", false) // different number of blank lines + f("\nfoo 13\n", "\nfoo 3.4", false) // different number of blank lines + f("\n\nfoo 1", "\n\nfoo 34.43", true) + f("\n\nfoo 3434\n", "\n\nfoo 43\n", true) + f("\nfoo 1", "\n\nfoo 34.43", false) // different number of blank lines + f("#foo{bar}", "#baz", true) + f("", "#baz", false) // different number of comments + f("#foo{bar}", "", false) // different number of comments + f("#foo{bar}", "bar 3", false) // different number of comments + f("foo{bar} 2", "#bar 3", false) // different number of comments + f("#foo\n", "#bar", false) // different number of blank lines + f("#foo{bar}\n#baz", "#baz\n#xdsfds dsf", true) + f("# foo\nbar 234\nbaz{x=\"y\", z=\"\"} 3", "# foo\nbar 3.3\nbaz{x=\"y\", z=\"\"} 4323", true) + f("# foo\nbar 234\nbaz{x=\"z\", z=\"\"} 3", "# foo\nbar 3.3\nbaz{x=\"y\", z=\"\"} 4323", false) // different label value + f("foo {bar=\"xfdsdsffdsa\"} 1", "foo {x=\"y\"} 2", false) // different labels + f("foo {x=\"z\"} 1", "foo {x=\"y\"} 2", false) // different label value + + // Lines with timestamps + f("foo 1 2", "foo 234 4334", true) + f("foo 2", "foo 3 4", false) // missing timestamp + f("foo 2 1", "foo 3", false) // missing timestamp + f("foo{bar=\"b az\"} 2 5", "foo{bar=\"b az\"} +6.3 7.43", true) + f("foo{bar=\"b az\"} 2 5 # comment ss ", "foo{bar=\"b az\"} +6.3 7.43 # comment as ", true) + f("foo{bar=\"b az\"} 2 5 #comment", "foo{bar=\"b az\"} +6.3 7.43 #comment {foo=\"bar\"} 21.44", true) + f("foo{bar=\"b az\"} +Inf 5", "foo{bar=\"b az\"} NaN 7.43", true) + f("foo{bar=\"b az\"} +Inf 5", "foo{bar=\"b az\"} nan 7.43", true) + f("foo{bar=\"b az\"} +Inf 5", "foo{bar=\"b az\"} nansf 7.43", false) // invalid value +} + func TestPrevBackslashesCount(t *testing.T) { f := func(s string, nExpected int) { t.Helper() @@ -105,6 +186,10 @@ func TestRowsUnmarshalFailure(t *testing.T) { // empty metric name f(`{foo="bar"}`) + // Invalid quotes for label value + f(`{foo='bar'} 23`) + f("{foo=`bar`} 23") + // Missing value f("aaa") f(" aaa") diff --git a/lib/protoparser/prometheus/parser_timing_test.go b/lib/protoparser/prometheus/parser_timing_test.go index 893e829b11..fa650ce9bd 100644 --- a/lib/protoparser/prometheus/parser_timing_test.go +++ b/lib/protoparser/prometheus/parser_timing_test.go @@ -5,6 +5,116 @@ import ( "testing" ) +func BenchmarkAreIdenticalSeriesFast(b *testing.B) { + b.Run("identical-series-no-timestamps", func(b *testing.B) { + s := ` +# HELP machine_cpu_cores Number of logical CPU cores. +# TYPE machine_cpu_cores gauge +machine_cpu_cores{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 4 +# HELP machine_cpu_physical_cores Number of physical CPU cores. +# TYPE machine_cpu_physical_cores gauge +machine_cpu_physical_cores{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 2 +# HELP machine_cpu_sockets Number of CPU sockets. +# TYPE machine_cpu_sockets gauge +machine_cpu_sockets{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 1 +# HELP machine_memory_bytes Amount of memory installed on the machine. +# TYPE machine_memory_bytes gauge +machine_memory_bytes{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 1.6706146304e+10 +# HELP machine_nvm_avg_power_budget_watts NVM power budget. +# TYPE machine_nvm_avg_power_budget_watts gauge +machine_nvm_avg_power_budget_watts{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 0 +# HELP machine_nvm_capacity NVM capacity value labeled by NVM mode (memory mode or app direct mode). +# TYPE machine_nvm_capacity gauge +machine_nvm_capacity{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",mode="app_direct_mode",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 0 +machine_nvm_capacity{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",mode="memory_mode",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 0 +# HELP machine_scrape_error 1 if there was an error while getting machine metrics, 0 otherwise. +# TYPE machine_scrape_error gauge +machine_scrape_error 0 +` + benchmarkAreIdenticalSeriesFast(b, s, s, true) + }) + b.Run("different-series-no-timestamps", func(b *testing.B) { + s := ` +# HELP machine_cpu_cores Number of logical CPU cores. +# TYPE machine_cpu_cores gauge +machine_cpu_cores{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 4 +# HELP machine_cpu_physical_cores Number of physical CPU cores. +# TYPE machine_cpu_physical_cores gauge +machine_cpu_physical_cores{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 2 +# HELP machine_cpu_sockets Number of CPU sockets. +# TYPE machine_cpu_sockets gauge +machine_cpu_sockets{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 1 +# HELP machine_memory_bytes Amount of memory installed on the machine. +# TYPE machine_memory_bytes gauge +machine_memory_bytes{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 1.6706146304e+10 +# HELP machine_nvm_avg_power_budget_watts NVM power budget. +# TYPE machine_nvm_avg_power_budget_watts gauge +machine_nvm_avg_power_budget_watts{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 0 +# HELP machine_nvm_capacity NVM capacity value labeled by NVM mode (memory mode or app direct mode). +# TYPE machine_nvm_capacity gauge +machine_nvm_capacity{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",mode="app_direct_mode",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 0 +machine_nvm_capacity{boot_id="a1b49bdb-4c2a-4943-9ab3-363a316e9260",machine_id="857143c2dbea4a179223627cf9f47d06",mode="memory_mode",system_uuid="03a75ec7-5105-421a-8b8a-3d7190f6e890"} 0 +# HELP machine_scrape_error 1 if there was an error while getting machine metrics, 0 otherwise. +# TYPE machine_scrape_error gauge +machine_scrape_error 0 +` + benchmarkAreIdenticalSeriesFast(b, s, s+"\nfoo 1", false) + }) + b.Run("identical-series-with-timestamps", func(b *testing.B) { + s := ` +container_ulimits_soft{container="",id="/kubelet/kubepods/burstable/pod48ea6dbad93797db01928fb7884b8154/49d928b5e3e3398730c9ce9de02171bb139b5bf2f485b153d9a293114a5762a3",image="sha256:0184c1613d92931126feb4c548e5da11015513b9e4c104e7305ee8b53b50a9da",name="49d928b5e3e3398730c9ce9de02171bb139b5bf2f485b153d9a293114a5762a3",namespace="kube-system",pod="kube-apiserver-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113856793 +container_ulimits_soft{container="",id="/kubelet/kubepods/burstable/pod69cd289b4ed80ced4f95a59ff60fa102/602a9be3cad5ca8aa57bdbb4a947ddd3b1b229b6e54c7acbb6906de061d51d05",image="sha256:0184c1613d92931126feb4c548e5da11015513b9e4c104e7305ee8b53b50a9da",name="602a9be3cad5ca8aa57bdbb4a947ddd3b1b229b6e54c7acbb6906de061d51d05",namespace="kube-system",pod="kube-scheduler-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113855488 +container_ulimits_soft{container="",id="/kubelet/kubepods/burstable/pod86744a0c8ef8da0d937493e4ed918cda/2f1a3706328f86337864f7c2c7100aabf9cabf03fef5518e883380977372d53f",image="sha256:0184c1613d92931126feb4c548e5da11015513b9e4c104e7305ee8b53b50a9da",name="2f1a3706328f86337864f7c2c7100aabf9cabf03fef5518e883380977372d53f",namespace="kube-system",pod="kube-controller-manager-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113858430 +container_ulimits_soft{container="",id="/kubelet/kubepods/burstable/poda4a6a8d4c9c0100deb8dc3a1d3adfa32/a84ce063fb5cab82bb938151e9fa1e98ad875c3cf5dad88d797d4c65c6229c13",image="sha256:0184c1613d92931126feb4c548e5da11015513b9e4c104e7305ee8b53b50a9da",name="a84ce063fb5cab82bb938151e9fa1e98ad875c3cf5dad88d797d4c65c6229c13",namespace="kube-system",pod="etcd-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113850216 +container_ulimits_soft{container="",id="/kubelet/kubepods/poda922c399-764c-4614-8a2d-84bdd6765ffc/ec6b156815cc77c389fe08a4be82603514c8929a9827b8ba27f9cb9c0b57b067",image="sha256:0184c1613d92931126feb4c548e5da11015513b9e4c104e7305ee8b53b50a9da",name="ec6b156815cc77c389fe08a4be82603514c8929a9827b8ba27f9cb9c0b57b067",namespace="kube-system",pod="kindnet-nj4p9",ulimit="max_open_files"} 1.048576e+06 1631113865193 +container_ulimits_soft{container="etcd",id="/docker/6b7c234cfe92a0924e54e2a51d9607a5893a38ed14c7161f324863eeaa2fb985/kubelet/kubepods/burstable/poda4a6a8d4c9c0100deb8dc3a1d3adfa32/0cd86529af0ca0e389ed657b2c0a20f03275cf6d9e0cd52fe4c1f90b96037de7",image="k8s.gcr.io/etcd:3.4.13-0",name="0cd86529af0ca0e389ed657b2c0a20f03275cf6d9e0cd52fe4c1f90b96037de7",namespace="kube-system",pod="etcd-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113855044 +container_ulimits_soft{container="etcd",id="/kubelet/kubepods/burstable/poda4a6a8d4c9c0100deb8dc3a1d3adfa32/0cd86529af0ca0e389ed657b2c0a20f03275cf6d9e0cd52fe4c1f90b96037de7",image="k8s.gcr.io/etcd:3.4.13-0",name="0cd86529af0ca0e389ed657b2c0a20f03275cf6d9e0cd52fe4c1f90b96037de7",namespace="kube-system",pod="etcd-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113867411 +container_ulimits_soft{container="kindnet-cni",id="/docker/6b7c234cfe92a0924e54e2a51d9607a5893a38ed14c7161f324863eeaa2fb985/kubelet/kubepods/poda922c399-764c-4614-8a2d-84bdd6765ffc/b38094619c14a9f921e2d10fb0f84433bea774aeb223ba19dade527e1c46de22",image="docker.io/kindest/kindnetd:v20210119-d5ef916d",name="b38094619c14a9f921e2d10fb0f84433bea774aeb223ba19dade527e1c46de22",namespace="kube-system",pod="kindnet-nj4p9",ulimit="max_open_files"} 1.048576e+06 1631113868404 +container_ulimits_soft{container="kindnet-cni",id="/kubelet/kubepods/poda922c399-764c-4614-8a2d-84bdd6765ffc/b38094619c14a9f921e2d10fb0f84433bea774aeb223ba19dade527e1c46de22",image="docker.io/kindest/kindnetd:v20210119-d5ef916d",name="b38094619c14a9f921e2d10fb0f84433bea774aeb223ba19dade527e1c46de22",namespace="kube-system",pod="kindnet-nj4p9",ulimit="max_open_files"} 1.048576e+06 1631113862176 +container_ulimits_soft{container="kube-apiserver",id="/docker/6b7c234cfe92a0924e54e2a51d9607a5893a38ed14c7161f324863eeaa2fb985/kubelet/kubepods/burstable/pod48ea6dbad93797db01928fb7884b8154/4026cf5500d96c6e274a2607b507891abc21f7b1577e29c9400cfb0f0ce5d8aa",image="k8s.gcr.io/kube-apiserver:v1.20.2",name="4026cf5500d96c6e274a2607b507891abc21f7b1577e29c9400cfb0f0ce5d8aa",namespace="kube-system",pod="kube-apiserver-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113865919 +container_ulimits_soft{container="kube-apiserver",id="/kubelet/kubepods/burstable/pod48ea6dbad93797db01928fb7884b8154/4026cf5500d96c6e274a2607b507891abc21f7b1577e29c9400cfb0f0ce5d8aa",image="k8s.gcr.io/kube-apiserver:v1.20.2",name="4026cf5500d96c6e274a2607b507891abc21f7b1577e29c9400cfb0f0ce5d8aa",namespace="kube-system",pod="kube-apiserver-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113863531 +container_ulimits_soft{container="kube-controller-manager",id="/docker/6b7c234cfe92a0924e54e2a51d9607a5893a38ed14c7161f324863eeaa2fb985/kubelet/kubepods/burstable/pod86744a0c8ef8da0d937493e4ed918cda/04b0948ab58f83013fed7611f0ffadb13ff7336561c91606644848f60405771b",image="k8s.gcr.io/kube-controller-manager:v1.20.2",name="04b0948ab58f83013fed7611f0ffadb13ff7336561c91606644848f60405771b",namespace="kube-system",pod="kube-controller-manager-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113868172 +container_ulimits_soft{container="kube-controller-manager",id="/kubelet/kubepods/burstable/pod86744a0c8ef8da0d937493e4ed918cda/04b0948ab58f83013fed7611f0ffadb13ff7336561c91606644848f60405771b",image="k8s.gcr.io/kube-controller-manager:v1.20.2",name="04b0948ab58f83013fed7611f0ffadb13ff7336561c91606644848f60405771b",namespace="kube-system",pod="kube-controller-manager-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113860485 +container_ulimits_soft{container="kube-scheduler",id="/docker/6b7c234cfe92a0924e54e2a51d9607a5893a38ed14c7161f324863eeaa2fb985/kubelet/kubepods/burstable/pod69cd289b4ed80ced4f95a59ff60fa102/d9627625c8d60d859f2a13f9ed66c77c9767368e18eb5669fe1a85d600e43f9b",image="k8s.gcr.io/kube-scheduler:v1.20.2",name="d9627625c8d60d859f2a13f9ed66c77c9767368e18eb5669fe1a85d600e43f9b",namespace="kube-system",pod="kube-scheduler-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113857794 +container_ulimits_soft{container="kube-scheduler",id="/kubelet/kubepods/burstable/pod69cd289b4ed80ced4f95a59ff60fa102/d9627625c8d60d859f2a13f9ed66c77c9767368e18eb5669fe1a85d600e43f9b",image="k8s.gcr.io/kube-scheduler:v1.20.2",name="d9627625c8d60d859f2a13f9ed66c77c9767368e18eb5669fe1a85d600e43f9b",namespace="kube-system",pod="kube-scheduler-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113868640 +` + benchmarkAreIdenticalSeriesFast(b, s, s, true) + }) + b.Run("different-series-with-timestamps", func(b *testing.B) { + s := ` +container_ulimits_soft{container="",id="/kubelet/kubepods/burstable/pod48ea6dbad93797db01928fb7884b8154/49d928b5e3e3398730c9ce9de02171bb139b5bf2f485b153d9a293114a5762a3",image="sha256:0184c1613d92931126feb4c548e5da11015513b9e4c104e7305ee8b53b50a9da",name="49d928b5e3e3398730c9ce9de02171bb139b5bf2f485b153d9a293114a5762a3",namespace="kube-system",pod="kube-apiserver-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113856793 +container_ulimits_soft{container="",id="/kubelet/kubepods/burstable/pod69cd289b4ed80ced4f95a59ff60fa102/602a9be3cad5ca8aa57bdbb4a947ddd3b1b229b6e54c7acbb6906de061d51d05",image="sha256:0184c1613d92931126feb4c548e5da11015513b9e4c104e7305ee8b53b50a9da",name="602a9be3cad5ca8aa57bdbb4a947ddd3b1b229b6e54c7acbb6906de061d51d05",namespace="kube-system",pod="kube-scheduler-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113855488 +container_ulimits_soft{container="",id="/kubelet/kubepods/burstable/pod86744a0c8ef8da0d937493e4ed918cda/2f1a3706328f86337864f7c2c7100aabf9cabf03fef5518e883380977372d53f",image="sha256:0184c1613d92931126feb4c548e5da11015513b9e4c104e7305ee8b53b50a9da",name="2f1a3706328f86337864f7c2c7100aabf9cabf03fef5518e883380977372d53f",namespace="kube-system",pod="kube-controller-manager-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113858430 +container_ulimits_soft{container="",id="/kubelet/kubepods/burstable/poda4a6a8d4c9c0100deb8dc3a1d3adfa32/a84ce063fb5cab82bb938151e9fa1e98ad875c3cf5dad88d797d4c65c6229c13",image="sha256:0184c1613d92931126feb4c548e5da11015513b9e4c104e7305ee8b53b50a9da",name="a84ce063fb5cab82bb938151e9fa1e98ad875c3cf5dad88d797d4c65c6229c13",namespace="kube-system",pod="etcd-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113850216 +container_ulimits_soft{container="",id="/kubelet/kubepods/poda922c399-764c-4614-8a2d-84bdd6765ffc/ec6b156815cc77c389fe08a4be82603514c8929a9827b8ba27f9cb9c0b57b067",image="sha256:0184c1613d92931126feb4c548e5da11015513b9e4c104e7305ee8b53b50a9da",name="ec6b156815cc77c389fe08a4be82603514c8929a9827b8ba27f9cb9c0b57b067",namespace="kube-system",pod="kindnet-nj4p9",ulimit="max_open_files"} 1.048576e+06 1631113865193 +container_ulimits_soft{container="etcd",id="/docker/6b7c234cfe92a0924e54e2a51d9607a5893a38ed14c7161f324863eeaa2fb985/kubelet/kubepods/burstable/poda4a6a8d4c9c0100deb8dc3a1d3adfa32/0cd86529af0ca0e389ed657b2c0a20f03275cf6d9e0cd52fe4c1f90b96037de7",image="k8s.gcr.io/etcd:3.4.13-0",name="0cd86529af0ca0e389ed657b2c0a20f03275cf6d9e0cd52fe4c1f90b96037de7",namespace="kube-system",pod="etcd-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113855044 +container_ulimits_soft{container="etcd",id="/kubelet/kubepods/burstable/poda4a6a8d4c9c0100deb8dc3a1d3adfa32/0cd86529af0ca0e389ed657b2c0a20f03275cf6d9e0cd52fe4c1f90b96037de7",image="k8s.gcr.io/etcd:3.4.13-0",name="0cd86529af0ca0e389ed657b2c0a20f03275cf6d9e0cd52fe4c1f90b96037de7",namespace="kube-system",pod="etcd-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113867411 +container_ulimits_soft{container="kindnet-cni",id="/docker/6b7c234cfe92a0924e54e2a51d9607a5893a38ed14c7161f324863eeaa2fb985/kubelet/kubepods/poda922c399-764c-4614-8a2d-84bdd6765ffc/b38094619c14a9f921e2d10fb0f84433bea774aeb223ba19dade527e1c46de22",image="docker.io/kindest/kindnetd:v20210119-d5ef916d",name="b38094619c14a9f921e2d10fb0f84433bea774aeb223ba19dade527e1c46de22",namespace="kube-system",pod="kindnet-nj4p9",ulimit="max_open_files"} 1.048576e+06 1631113868404 +container_ulimits_soft{container="kindnet-cni",id="/kubelet/kubepods/poda922c399-764c-4614-8a2d-84bdd6765ffc/b38094619c14a9f921e2d10fb0f84433bea774aeb223ba19dade527e1c46de22",image="docker.io/kindest/kindnetd:v20210119-d5ef916d",name="b38094619c14a9f921e2d10fb0f84433bea774aeb223ba19dade527e1c46de22",namespace="kube-system",pod="kindnet-nj4p9",ulimit="max_open_files"} 1.048576e+06 1631113862176 +container_ulimits_soft{container="kube-apiserver",id="/docker/6b7c234cfe92a0924e54e2a51d9607a5893a38ed14c7161f324863eeaa2fb985/kubelet/kubepods/burstable/pod48ea6dbad93797db01928fb7884b8154/4026cf5500d96c6e274a2607b507891abc21f7b1577e29c9400cfb0f0ce5d8aa",image="k8s.gcr.io/kube-apiserver:v1.20.2",name="4026cf5500d96c6e274a2607b507891abc21f7b1577e29c9400cfb0f0ce5d8aa",namespace="kube-system",pod="kube-apiserver-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113865919 +container_ulimits_soft{container="kube-apiserver",id="/kubelet/kubepods/burstable/pod48ea6dbad93797db01928fb7884b8154/4026cf5500d96c6e274a2607b507891abc21f7b1577e29c9400cfb0f0ce5d8aa",image="k8s.gcr.io/kube-apiserver:v1.20.2",name="4026cf5500d96c6e274a2607b507891abc21f7b1577e29c9400cfb0f0ce5d8aa",namespace="kube-system",pod="kube-apiserver-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113863531 +container_ulimits_soft{container="kube-controller-manager",id="/docker/6b7c234cfe92a0924e54e2a51d9607a5893a38ed14c7161f324863eeaa2fb985/kubelet/kubepods/burstable/pod86744a0c8ef8da0d937493e4ed918cda/04b0948ab58f83013fed7611f0ffadb13ff7336561c91606644848f60405771b",image="k8s.gcr.io/kube-controller-manager:v1.20.2",name="04b0948ab58f83013fed7611f0ffadb13ff7336561c91606644848f60405771b",namespace="kube-system",pod="kube-controller-manager-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113868172 +container_ulimits_soft{container="kube-controller-manager",id="/kubelet/kubepods/burstable/pod86744a0c8ef8da0d937493e4ed918cda/04b0948ab58f83013fed7611f0ffadb13ff7336561c91606644848f60405771b",image="k8s.gcr.io/kube-controller-manager:v1.20.2",name="04b0948ab58f83013fed7611f0ffadb13ff7336561c91606644848f60405771b",namespace="kube-system",pod="kube-controller-manager-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113860485 +container_ulimits_soft{container="kube-scheduler",id="/docker/6b7c234cfe92a0924e54e2a51d9607a5893a38ed14c7161f324863eeaa2fb985/kubelet/kubepods/burstable/pod69cd289b4ed80ced4f95a59ff60fa102/d9627625c8d60d859f2a13f9ed66c77c9767368e18eb5669fe1a85d600e43f9b",image="k8s.gcr.io/kube-scheduler:v1.20.2",name="d9627625c8d60d859f2a13f9ed66c77c9767368e18eb5669fe1a85d600e43f9b",namespace="kube-system",pod="kube-scheduler-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113857794 +container_ulimits_soft{container="kube-scheduler",id="/kubelet/kubepods/burstable/pod69cd289b4ed80ced4f95a59ff60fa102/d9627625c8d60d859f2a13f9ed66c77c9767368e18eb5669fe1a85d600e43f9b",image="k8s.gcr.io/kube-scheduler:v1.20.2",name="d9627625c8d60d859f2a13f9ed66c77c9767368e18eb5669fe1a85d600e43f9b",namespace="kube-system",pod="kube-scheduler-kind-control-plane",ulimit="max_open_files"} 1.048576e+06 1631113868640 +` + benchmarkAreIdenticalSeriesFast(b, s, s+"\nfoo 1", false) + }) +} + +func benchmarkAreIdenticalSeriesFast(b *testing.B, s1, s2 string, expectedResult bool) { + b.SetBytes(int64(len(s1))) + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + result := AreIdenticalSeriesFast(s1, s2) + if result != expectedResult { + panic(fmt.Errorf("unexpected result; got %v; want %v", result, expectedResult)) + } + } + }) +} + func BenchmarkRowsUnmarshal(b *testing.B) { s := `cpu_usage{mode="user"} 1.23 cpu_usage{mode="system"} 23.344