From ae6c7edf04b2565206e06d5d56945492f8946f98 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sat, 1 Oct 2022 19:47:35 +0300 Subject: [PATCH] app/vmselect: do not export NaN values for stale metrics at /federate endpoint Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3185 --- app/vmselect/bufferedwriter/bufferedwriter.go | 3 + app/vmselect/prometheus/federate.qtpl | 23 +++++- app/vmselect/prometheus/federate.qtpl.go | 71 +++++++++++-------- docs/CHANGELOG.md | 1 + 4 files changed, 67 insertions(+), 31 deletions(-) diff --git a/app/vmselect/bufferedwriter/bufferedwriter.go b/app/vmselect/bufferedwriter/bufferedwriter.go index 985d98d6da..a06e59ecbe 100644 --- a/app/vmselect/bufferedwriter/bufferedwriter.go +++ b/app/vmselect/bufferedwriter/bufferedwriter.go @@ -54,6 +54,9 @@ func (bw *Writer) reset() { // Write writes p to bw. func (bw *Writer) Write(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } bw.lock.Lock() defer bw.lock.Unlock() if bw.err != nil { diff --git a/app/vmselect/prometheus/federate.qtpl b/app/vmselect/prometheus/federate.qtpl index abd1184575..3e2440cfd6 100644 --- a/app/vmselect/prometheus/federate.qtpl +++ b/app/vmselect/prometheus/federate.qtpl @@ -1,4 +1,6 @@ {% import ( + "math" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" ) %} @@ -7,10 +9,25 @@ // Federate writes rs in /federate format. // See https://prometheus.io/docs/prometheus/latest/federation/ {% func Federate(rs *netstorage.Result) %} - {% if len(rs.Timestamps) == 0 || len(rs.Values) == 0 %}{% return %}{% endif %} + {% code + values := rs.Values + timestamps := rs.Timestamps + %} + {% if len(timestamps) == 0 || len(values) == 0 %}{% return %}{% endif %} + {% code + lastValue := values[len(values)-1] + %} + {% if math.IsNaN(lastValue) %} + {% comment %} + This is most likely a staleness marker. + Return nothing after the staleness marker. + See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3185 + {% endcomment %} + {% return %} + {% endif %} {%= prometheusMetricName(&rs.MetricName) %}{% space %} - {%f= rs.Values[len(rs.Values)-1] %}{% space %} - {%dl= rs.Timestamps[len(rs.Timestamps)-1] %}{% newline %} + {%f= lastValue %}{% space %} + {%dl= timestamps[len(timestamps)-1] %}{% newline %} {% endfunc %} {% endstripspace %} diff --git a/app/vmselect/prometheus/federate.qtpl.go b/app/vmselect/prometheus/federate.qtpl.go index 8fa82513f6..1dd69ca3cd 100644 --- a/app/vmselect/prometheus/federate.qtpl.go +++ b/app/vmselect/prometheus/federate.qtpl.go @@ -6,70 +6,85 @@ package prometheus //line app/vmselect/prometheus/federate.qtpl:1 import ( + "math" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" ) // Federate writes rs in /federate format.// See https://prometheus.io/docs/prometheus/latest/federation/ -//line app/vmselect/prometheus/federate.qtpl:9 +//line app/vmselect/prometheus/federate.qtpl:11 import ( qtio422016 "io" qt422016 "github.com/valyala/quicktemplate" ) -//line app/vmselect/prometheus/federate.qtpl:9 +//line app/vmselect/prometheus/federate.qtpl:11 var ( _ = qtio422016.Copy _ = qt422016.AcquireByteBuffer ) -//line app/vmselect/prometheus/federate.qtpl:9 +//line app/vmselect/prometheus/federate.qtpl:11 func StreamFederate(qw422016 *qt422016.Writer, rs *netstorage.Result) { -//line app/vmselect/prometheus/federate.qtpl:10 - if len(rs.Timestamps) == 0 || len(rs.Values) == 0 { -//line app/vmselect/prometheus/federate.qtpl:10 +//line app/vmselect/prometheus/federate.qtpl:13 + values := rs.Values + timestamps := rs.Timestamps + +//line app/vmselect/prometheus/federate.qtpl:16 + if len(timestamps) == 0 || len(values) == 0 { +//line app/vmselect/prometheus/federate.qtpl:16 return -//line app/vmselect/prometheus/federate.qtpl:10 +//line app/vmselect/prometheus/federate.qtpl:16 } -//line app/vmselect/prometheus/federate.qtpl:11 +//line app/vmselect/prometheus/federate.qtpl:18 + lastValue := values[len(values)-1] + +//line app/vmselect/prometheus/federate.qtpl:20 + if math.IsNaN(lastValue) { +//line app/vmselect/prometheus/federate.qtpl:26 + return +//line app/vmselect/prometheus/federate.qtpl:27 + } +//line app/vmselect/prometheus/federate.qtpl:28 streamprometheusMetricName(qw422016, &rs.MetricName) -//line app/vmselect/prometheus/federate.qtpl:11 +//line app/vmselect/prometheus/federate.qtpl:28 qw422016.N().S(` `) -//line app/vmselect/prometheus/federate.qtpl:12 - qw422016.N().F(rs.Values[len(rs.Values)-1]) -//line app/vmselect/prometheus/federate.qtpl:12 +//line app/vmselect/prometheus/federate.qtpl:29 + qw422016.N().F(lastValue) +//line app/vmselect/prometheus/federate.qtpl:29 qw422016.N().S(` `) -//line app/vmselect/prometheus/federate.qtpl:13 - qw422016.N().DL(rs.Timestamps[len(rs.Timestamps)-1]) -//line app/vmselect/prometheus/federate.qtpl:13 +//line app/vmselect/prometheus/federate.qtpl:30 + qw422016.N().DL(timestamps[len(timestamps)-1]) +//line app/vmselect/prometheus/federate.qtpl:30 qw422016.N().S(` `) -//line app/vmselect/prometheus/federate.qtpl:14 +//line app/vmselect/prometheus/federate.qtpl:31 } -//line app/vmselect/prometheus/federate.qtpl:14 +//line app/vmselect/prometheus/federate.qtpl:31 func WriteFederate(qq422016 qtio422016.Writer, rs *netstorage.Result) { -//line app/vmselect/prometheus/federate.qtpl:14 +//line app/vmselect/prometheus/federate.qtpl:31 qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmselect/prometheus/federate.qtpl:14 +//line app/vmselect/prometheus/federate.qtpl:31 StreamFederate(qw422016, rs) -//line app/vmselect/prometheus/federate.qtpl:14 +//line app/vmselect/prometheus/federate.qtpl:31 qt422016.ReleaseWriter(qw422016) -//line app/vmselect/prometheus/federate.qtpl:14 +//line app/vmselect/prometheus/federate.qtpl:31 } -//line app/vmselect/prometheus/federate.qtpl:14 +//line app/vmselect/prometheus/federate.qtpl:31 func Federate(rs *netstorage.Result) string { -//line app/vmselect/prometheus/federate.qtpl:14 +//line app/vmselect/prometheus/federate.qtpl:31 qb422016 := qt422016.AcquireByteBuffer() -//line app/vmselect/prometheus/federate.qtpl:14 +//line app/vmselect/prometheus/federate.qtpl:31 WriteFederate(qb422016, rs) -//line app/vmselect/prometheus/federate.qtpl:14 +//line app/vmselect/prometheus/federate.qtpl:31 qs422016 := string(qb422016.B) -//line app/vmselect/prometheus/federate.qtpl:14 +//line app/vmselect/prometheus/federate.qtpl:31 qt422016.ReleaseByteBuffer(qb422016) -//line app/vmselect/prometheus/federate.qtpl:14 +//line app/vmselect/prometheus/federate.qtpl:31 return qs422016 -//line app/vmselect/prometheus/federate.qtpl:14 +//line app/vmselect/prometheus/federate.qtpl:31 } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 83b31f7066..6f3ece426d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -49,6 +49,7 @@ See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#m * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `external_labels` from `global` section at `-promscrape.config` after the [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling) is applied to scraped metrics. This aligns with Prometheus behaviour. Previously the `external_labels` were added to scrape targets, so they could be modified during relabeling. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3137). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow specifying per-`-remoteWrite.url` limits for on-disk size for pending data via `-remoteWrite.maxDiskUsagePerURL` command-line flag. Thanks to @rbizos for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3071). +* BUGFIX: do not export stale metrics via [/federate api](https://docs.victoriametrics.com/#federation) after the staleness markers. Previously such metrics were exported with `NaN` values. this could break some setups. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3185). * BUGFIX: [vmauth](https://docs.victoriametrics.com/vmauth.html): properly handle request paths ending with `/` such as `/vmui/`. Previously `vmui` was dropping the traling `/`, which could prevent from using `vmui` via `vmauth`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1752). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly encode query params for aws signed requests, use `%20` instead of `+` as api requires. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3171). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): properly parse relabel config when regex ending with escaped `$`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3131).