diff --git a/README.md b/README.md index f5e23e32a..025aae1c8 100644 --- a/README.md +++ b/README.md @@ -923,6 +923,9 @@ The most interesting metrics are: If this removes gaps on the graphs, then it is likely data with timestamps older than `-search.cacheTimestampOffset` is ingested into VictoriaMetrics. Make sure that data sources have synchronized time with VictoriaMetrics. + If the gaps are related to irregular intervals between samples, then try adjusting `-search.minStalenessInterval` command-line flag + to value close to the maximum interval between samples. + ### Backfilling VictoriaMetrics accepts historical data in arbitrary order of time via [any supported ingestion method](#how-to-import-time-series-data). diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index 565777e7d..c88a1b8bb 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -33,7 +33,7 @@ var ( "The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. "+ "See also '-search.maxStalenessInterval' flag, which has the same meaining due to historical reasons") maxStalenessInterval = flag.Duration("search.maxStalenessInterval", 0, "The maximum interval for staleness calculations. "+ - "By default it is automatically calculated from the median interval between samples. This flag can be useful for tuning "+ + "By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning "+ "Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. "+ "See also '-search.maxLookback' flag, which has the same meanining due to historical reasons") ) diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index 3dca9cd3a..82d68a779 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -1,6 +1,7 @@ package promql import ( + "flag" "fmt" "math" "strings" @@ -14,6 +15,10 @@ import ( "github.com/valyala/histogram" ) +var minStalenessInterval = flag.Duration("search.minStalenessInterval", 0, "The mimimum interval for staleness calculations. "+ + "This flag could be useful for removing gaps on graphs generated from time series with irregular intervals between samples. "+ + "See also '-search.maxStalenessInterval'") + var rollupFuncs = map[string]newRollupFunc{ // Standard rollup funcs from PromQL. // See funcs accepting range-vector on https://prometheus.io/docs/prometheus/latest/querying/functions/ . @@ -451,6 +456,11 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu if rc.LookbackDelta > 0 && maxPrevInterval > rc.LookbackDelta { maxPrevInterval = rc.LookbackDelta } + if *minStalenessInterval > 0 { + if msi := minStalenessInterval.Milliseconds(); msi > 0 && maxPrevInterval < msi { + maxPrevInterval = msi + } + } window := rc.Window if window <= 0 { window = rc.Step diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index f5e23e32a..025aae1c8 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -923,6 +923,9 @@ The most interesting metrics are: If this removes gaps on the graphs, then it is likely data with timestamps older than `-search.cacheTimestampOffset` is ingested into VictoriaMetrics. Make sure that data sources have synchronized time with VictoriaMetrics. + If the gaps are related to irregular intervals between samples, then try adjusting `-search.minStalenessInterval` command-line flag + to value close to the maximum interval between samples. + ### Backfilling VictoriaMetrics accepts historical data in arbitrary order of time via [any supported ingestion method](#how-to-import-time-series-data).