diff --git a/app/victoria-metrics/self_scraper.go b/app/victoria-metrics/self_scraper.go index 931e5a517..cc405884d 100644 --- a/app/victoria-metrics/self_scraper.go +++ b/app/victoria-metrics/self_scraper.go @@ -8,6 +8,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage" "github.com/VictoriaMetrics/VictoriaMetrics/lib/appmetrics" "github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompb" "github.com/VictoriaMetrics/VictoriaMetrics/lib/protoparser/prometheus" @@ -49,16 +50,8 @@ func selfScraper(scrapeInterval time.Duration) { var mrs []storage.MetricRow var labels []prompb.Label t := time.NewTicker(scrapeInterval) - var currentTimestamp int64 - for { - select { - case <-selfScraperStopCh: - t.Stop() - logger.Infof("stopped self-scraping `/metrics` page") - return - case currentTime := <-t.C: - currentTimestamp = currentTime.UnixNano() / 1e6 - } + f := func(currentTime time.Time, sendStaleMarkers bool) { + currentTimestamp := currentTime.UnixNano() / 1e6 bb.Reset() appmetrics.WritePrometheusMetrics(&bb) s := bytesutil.ToUnsafeString(bb.B) @@ -83,12 +76,27 @@ func selfScraper(scrapeInterval time.Duration) { mr := &mrs[len(mrs)-1] mr.MetricNameRaw = storage.MarshalMetricNameRaw(mr.MetricNameRaw[:0], labels) mr.Timestamp = currentTimestamp - mr.Value = r.Value + if sendStaleMarkers { + mr.Value = decimal.StaleNaN + } else { + mr.Value = r.Value + } } if err := vmstorage.AddRows(mrs); err != nil { logger.Errorf("cannot store self-scraped metrics: %s", err) } } + for { + select { + case <-selfScraperStopCh: + f(time.Now(), true) + t.Stop() + logger.Infof("stopped self-scraping `/metrics` page") + return + case currentTime := <-t.C: + f(currentTime, false) + } + } } func addLabel(dst []prompb.Label, key, value string) []prompb.Label { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 249eee04e..4ae023ba3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -41,6 +41,7 @@ The sandbox cluster installation is running under the constant load generated by * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly propagate [label filters](https://docs.victoriametrics.com/keyconcepts/#filtering) from multiple arguments passed to [aggregate functions](https://docs.victoriametrics.com/metricsql/#aggregate-functions). For example, `sum({job="foo"}, {job="bar"}) by (job) + a` was improperly optimized to `sum({job="foo"}, {job="bar"}) by (job) + a{job="foo"}` before being executed. This could lead to unexpected results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5604). * BUGFIX: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): properly handle precision errors when calculating [changes](https://docs.victoriametrics.com/metricsql/#changes), [changes_prometheus](https://docs.victoriametrics.com/metricsql/#changes_prometheus), [increases_over_time](https://docs.victoriametrics.com/metricsql/#increases_over_time) and [resets](https://docs.victoriametrics.com/metricsql/#resets) functions. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/767). * BUGFIX: all VictoriaMetrics components: consistently return 200 http status code from [`/-/reload` endpoint](https://docs.victoriametrics.com/vmagent/#configuration-update). Previously [single-node VictoriaMetrics](https://docs.victoriametrics.com/) was returning 204 http status code. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5774). +* BUGFIX: properly store [staleness markers](https://docs.victoriametrics.com/vmagent/#prometheus-staleness-markers) for [self-scraped metrics](https://docs.victoriametrics.com/#monitoring) on single-node VictoriaMetrics shutdown. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/943). * BUGFIX: prevent from possible `too big indexBlockSize` panic when samples with too long label values (~64Kb) are ingested into VictoriaMetrics. * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix the graph dragging for Firefox and Safari. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5764). * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix handling invalid timezone. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5732).