app/vmselect: add -search.minStalenessInterval command-line flag for removing gaps on graphs built from time series with irregular duration between samples

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/426

README.md

@@ -923,6 +923,9 @@ The most interesting metrics are:
   If this removes gaps on the graphs, then it is likely data with timestamps older than `-search.cacheTimestampOffset`
   is ingested into VictoriaMetrics. Make sure that data sources have synchronized time with VictoriaMetrics.
 
+  If the gaps are related to irregular intervals between samples, then try adjusting `-search.minStalenessInterval` command-line flag
+  to value close to the maximum interval between samples.
+
 ### Backfilling
 
 VictoriaMetrics accepts historical data in arbitrary order of time via [any supported ingestion method](#how-to-import-time-series-data).

app/vmselect/prometheus/prometheus.go

@@ -33,7 +33,7 @@ var (
 		"The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. "+
 		"See also '-search.maxStalenessInterval' flag, which has the same meaining due to historical reasons")
 	maxStalenessInterval = flag.Duration("search.maxStalenessInterval", 0, "The maximum interval for staleness calculations. "+
-		"By default it is automatically calculated from the median interval between samples. This flag can be useful for tuning "+
+		"By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning "+
 		"Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. "+
 		"See also '-search.maxLookback' flag, which has the same meanining due to historical reasons")
 )

app/vmselect/promql/rollup.go

@@ -1,6 +1,7 @@
 package promql
 
 import (
+	"flag"
 	"fmt"
 	"math"
 	"strings"
@@ -14,6 +15,10 @@ import (
 	"github.com/valyala/histogram"
 )
 
+var minStalenessInterval = flag.Duration("search.minStalenessInterval", 0, "The mimimum interval for staleness calculations. "+
+	"This flag could be useful for removing gaps on graphs generated from time series with irregular intervals between samples. "+
+	"See also '-search.maxStalenessInterval'")
+
 var rollupFuncs = map[string]newRollupFunc{
 	// Standard rollup funcs from PromQL.
 	// See funcs accepting range-vector on https://prometheus.io/docs/prometheus/latest/querying/functions/ .
@@ -451,6 +456,11 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu
 	if rc.LookbackDelta > 0 && maxPrevInterval > rc.LookbackDelta {
 		maxPrevInterval = rc.LookbackDelta
 	}
+	if *minStalenessInterval > 0 {
+		if msi := minStalenessInterval.Milliseconds(); msi > 0 && maxPrevInterval < msi {
+			maxPrevInterval = msi
+		}
+	}
 	window := rc.Window
 	if window <= 0 {
 		window = rc.Step

docs/Single-server-VictoriaMetrics.md

@@ -923,6 +923,9 @@ The most interesting metrics are:
   If this removes gaps on the graphs, then it is likely data with timestamps older than `-search.cacheTimestampOffset`
   is ingested into VictoriaMetrics. Make sure that data sources have synchronized time with VictoriaMetrics.
 
+  If the gaps are related to irregular intervals between samples, then try adjusting `-search.minStalenessInterval` command-line flag
+  to value close to the maximum interval between samples.
+
 ### Backfilling
 
 VictoriaMetrics accepts historical data in arbitrary order of time via [any supported ingestion method](#how-to-import-time-series-data).