Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-15 00:13:30 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

lib/promscrape: always send stale markers with the real scrape timestamp

Browse Source 
This guarantees that query won't return data just after the series is disappeared.
This commit is contained in:
Aliaksandr Valialkin 2022-06-23 10:55:14 +03:00
parent fc03950efa
commit 71b0dfdefa
No known key found for this signature in database
GPG Key ID: A72BEC6CD3D0DED1
2 changed files with 12 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/CHANGELOG.md
View File

@ -28,6 +28,8 @@ scrape_configs:
- targets: ["host123:8080"] - targets: ["host123:8080"]
``` ```
* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): make sure that [stale markers](https://docs.victoriametrics.com/vmagent.html#prometheus-staleness-markers) are generated with the actual timestamp when unsuccessful scrape occurs. This should prevent from possible time series overlap on scrape target restart in dynmaic envirnoments such as Kubernetes.
## [v1.78.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.0) ## [v1.78.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.0)
Released at 20-06-2022 Released at 20-06-2022

13
lib/promscrape/scrapework.go
View File

@ -332,7 +332,10 @@ func (sw *scrapeWork) run(stopCh <-chan struct{}, globalStopCh <-chan struct{})
// Do not send staleness markers on graceful shutdown as Prometheus does. // Do not send staleness markers on graceful shutdown as Prometheus does.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2013#issuecomment-1006994079 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2013#issuecomment-1006994079
default: default:
// Send staleness markers when the given target disappears. // Send staleness markers to all the metrics scraped last time from the target
// when the given target disappears as Prometheus does.
// Use the current real timestamp for staleness markers, so queries
// stop returning data just after the time the target disappears.
sw.sendStaleSeries(lastScrape, "", t, true) sw.sendStaleSeries(lastScrape, "", t, true)
} }
if sw.seriesLimiter != nil { if sw.seriesLimiter != nil {
@ -491,7 +494,9 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error
} }
// body must be released only after wc is released, since wc refers to body. // body must be released only after wc is released, since wc refers to body.
if !areIdenticalSeries { if !areIdenticalSeries {
sw.sendStaleSeries(lastScrape, bodyString, scrapeTimestamp, false) // Send stale markers for disappeared metrics with the real scrape timestamp
// in order to guarantee that query doesn't return data after this time for the disappeared metrics.
sw.sendStaleSeries(lastScrape, bodyString, realTimestamp, false)
sw.storeLastScrape(body.B) sw.storeLastScrape(body.B)
} }
sw.finalizeLastScrape() sw.finalizeLastScrape()
@ -599,7 +604,9 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error {
wc.reset() wc.reset()
writeRequestCtxPool.Put(wc) writeRequestCtxPool.Put(wc)
if !areIdenticalSeries { if !areIdenticalSeries {
sw.sendStaleSeries(lastScrape, bodyString, scrapeTimestamp, false) // Send stale markers for disappeared metrics with the real scrape timestamp
// in order to guarantee that query doesn't return data after this time for the disappeared metrics.
sw.sendStaleSeries(lastScrape, bodyString, realTimestamp, false)
sw.storeLastScrape(sbr.body) sw.storeLastScrape(sbr.body)
} }
sw.finalizeLastScrape() sw.finalizeLastScrape()