From 51362f9333f77c194b8a04ac3d6cae281d77d67b Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 22 Jun 2022 14:17:02 +0300 Subject: [PATCH] app/vmselect: add `-search.setLookbackToStep` command-line flag for making the gap filling algorithm similar to InfluxDB data model This option should override `-search.maxStalenessInterval` for most cases when users migrate from InfluxDB to VictoriaMetrics --- README.md | 8 +++++--- app/vmselect/prometheus/prometheus.go | 18 ++++++++++++++++-- docs/CHANGELOG.md | 2 ++ docs/Cluster-VictoriaMetrics.md | 4 +++- docs/README.md | 8 +++++--- docs/Single-server-VictoriaMetrics.md | 8 +++++--- docs/guides/migrate-from-influx.md | 6 ++++-- 7 files changed, 40 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3e8735138..1d507f477 100644 --- a/README.md +++ b/README.md @@ -1599,8 +1599,8 @@ See also more advanced [cardinality limiter in vmagent](https://docs.victoriamet If the gaps are related to irregular intervals between samples, then try adjusting `-search.minStalenessInterval` command-line flag to value close to the maximum interval between samples. -* If you are switching from InfluxDB or TimescaleDB, then take a look at `-search.maxStalenessInterval` command-line flag. - It may be needed in order to suppress default gap filling algorithm used by VictoriaMetrics - by default it assumes +* If you are switching from InfluxDB or TimescaleDB, then it may be needed to set `-search.setLookbackToStep` command-line flag. + This suppresses default gap filling algorithm used by VictoriaMetrics - by default it assumes each time series is continuous instead of discrete, so it fills gaps between real samples with regular intervals. * Metrics and labels leading to [high cardinality](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) or [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) can be determined via [cardinality explorer](#cardinality-explorer) and via [/api/v1/status/tsdb](#tsdb-stats) endpoint. @@ -2108,7 +2108,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -search.maxSeries int The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage (default 100000) -search.maxStalenessInterval duration - The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.maxLookback' flag, which has the same meaning due to historical reasons + The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.setLookbackToStep' flag -search.maxStatusRequestDuration duration The maximum duration for /api/v1/status/* requests (default 5m0s) -search.maxStepForPointsAdjustment duration @@ -2133,6 +2133,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats (default 1ms) -search.resetCacheAuthKey string Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call + -search.setLookbackToStep + Whether to fix lookback interval to 'step' query arg value. If set to true, the query model becomes closer to InfluxDB data model. If set to true, then -search.maxLookback and -search.maxStalenessInterval are ignored -search.treatDotsAsIsInRegexps Whether to treat dots as is in regexp label filters used in queries. For example, foo{bar=~"a.b.c"} will be automatically converted to foo{bar=~"a\\.b\\.c"}, i.e. all the dots in regexp filters will be automatically escaped in order to match only dot char instead of matching any char. Dots in ".+", ".*" and ".{n}" regexps aren't escaped. This option is DEPRECATED in favor of {__graphite__="a.*.c"} syntax for selecting metrics matching the given Graphite metrics filter -selfScrapeInstance string diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index 1c86588d9..b30bdc304 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -38,7 +38,9 @@ var ( maxStalenessInterval = flag.Duration("search.maxStalenessInterval", 0, "The maximum interval for staleness calculations. "+ "By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning "+ "Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. "+ - "See also '-search.maxLookback' flag, which has the same meaning due to historical reasons") + "See also '-search.setLookbackToStep' flag") + setLookbackToStep = flag.Bool("search.setLookbackToStep", false, "Whether to fix lookback interval to 'step' query arg value. "+ + "If set to true, the query model becomes closer to InfluxDB data model. If set to true, then -search.maxLookback and -search.maxStalenessInterval are ignored") maxStepForPointsAdjustment = flag.Duration("search.maxStepForPointsAdjustment", time.Minute, "The maximum step when /api/v1/query_range handler adjusts "+ "points with timestamps closer than -search.latencyOffset to the current time. The adjustment is needed because such points may contain incomplete data") @@ -981,7 +983,19 @@ func getMaxLookback(r *http.Request) (int64, error) { if d == 0 { d = maxStalenessInterval.Milliseconds() } - return searchutils.GetDuration(r, "max_lookback", d) + maxLookback, err := searchutils.GetDuration(r, "max_lookback", d) + if err != nil { + return 0, err + } + d = maxLookback + if *setLookbackToStep { + step, err := searchutils.GetDuration(r, "step", d) + if err != nil { + return 0, err + } + d = step + } + return d, nil } func getTagFilterssFromMatches(matches []string) ([][]storage.TagFilter, error) { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4266addda..f0ff8d37e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,8 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip +* FEATURE: add `-search.setLookbackToStep` command-line flag, which enables InfluxDB-like gap filling during querying. See [these docs](https://docs.victoriametrics.com/guides/migrate-from-influx.html) for details. + ## [v1.78.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.0) diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index 3e86c1bbf..f50323ae2 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -774,7 +774,7 @@ Below is the output for `/path/to/vmselect -help`: -search.maxSeries int The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage (default 100000) -search.maxStalenessInterval duration - The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.maxLookback' flag, which has the same meaning due to historical reasons + The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.setLookbackToStep' flag -search.maxStatusRequestDuration duration The maximum duration for /api/v1/status/* requests (default 5m0s) -search.maxStepForPointsAdjustment duration @@ -793,6 +793,8 @@ Below is the output for `/path/to/vmselect -help`: The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats (default 1ms) -search.resetCacheAuthKey string Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call + -search.setLookbackToStep + Whether to fix lookback interval to 'step' query arg value. If set to true, the query model becomes closer to InfluxDB data model. If set to true, then -search.maxLookback and -search.maxStalenessInterval are ignored -search.treatDotsAsIsInRegexps Whether to treat dots as is in regexp label filters used in queries. For example, foo{bar=~"a.b.c"} will be automatically converted to foo{bar=~"a\\.b\\.c"}, i.e. all the dots in regexp filters will be automatically escaped in order to match only dot char instead of matching any char. Dots in ".+", ".*" and ".{n}" regexps aren't escaped. This option is DEPRECATED in favor of {__graphite__="a.*.c"} syntax for selecting metrics matching the given Graphite metrics filter -selectNode array diff --git a/docs/README.md b/docs/README.md index 3e8735138..1d507f477 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1599,8 +1599,8 @@ See also more advanced [cardinality limiter in vmagent](https://docs.victoriamet If the gaps are related to irregular intervals between samples, then try adjusting `-search.minStalenessInterval` command-line flag to value close to the maximum interval between samples. -* If you are switching from InfluxDB or TimescaleDB, then take a look at `-search.maxStalenessInterval` command-line flag. - It may be needed in order to suppress default gap filling algorithm used by VictoriaMetrics - by default it assumes +* If you are switching from InfluxDB or TimescaleDB, then it may be needed to set `-search.setLookbackToStep` command-line flag. + This suppresses default gap filling algorithm used by VictoriaMetrics - by default it assumes each time series is continuous instead of discrete, so it fills gaps between real samples with regular intervals. * Metrics and labels leading to [high cardinality](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) or [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) can be determined via [cardinality explorer](#cardinality-explorer) and via [/api/v1/status/tsdb](#tsdb-stats) endpoint. @@ -2108,7 +2108,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -search.maxSeries int The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage (default 100000) -search.maxStalenessInterval duration - The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.maxLookback' flag, which has the same meaning due to historical reasons + The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.setLookbackToStep' flag -search.maxStatusRequestDuration duration The maximum duration for /api/v1/status/* requests (default 5m0s) -search.maxStepForPointsAdjustment duration @@ -2133,6 +2133,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats (default 1ms) -search.resetCacheAuthKey string Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call + -search.setLookbackToStep + Whether to fix lookback interval to 'step' query arg value. If set to true, the query model becomes closer to InfluxDB data model. If set to true, then -search.maxLookback and -search.maxStalenessInterval are ignored -search.treatDotsAsIsInRegexps Whether to treat dots as is in regexp label filters used in queries. For example, foo{bar=~"a.b.c"} will be automatically converted to foo{bar=~"a\\.b\\.c"}, i.e. all the dots in regexp filters will be automatically escaped in order to match only dot char instead of matching any char. Dots in ".+", ".*" and ".{n}" regexps aren't escaped. This option is DEPRECATED in favor of {__graphite__="a.*.c"} syntax for selecting metrics matching the given Graphite metrics filter -selfScrapeInstance string diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 882bf2029..eb302a089 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -1603,8 +1603,8 @@ See also more advanced [cardinality limiter in vmagent](https://docs.victoriamet If the gaps are related to irregular intervals between samples, then try adjusting `-search.minStalenessInterval` command-line flag to value close to the maximum interval between samples. -* If you are switching from InfluxDB or TimescaleDB, then take a look at `-search.maxStalenessInterval` command-line flag. - It may be needed in order to suppress default gap filling algorithm used by VictoriaMetrics - by default it assumes +* If you are switching from InfluxDB or TimescaleDB, then it may be needed to set `-search.setLookbackToStep` command-line flag. + This suppresses default gap filling algorithm used by VictoriaMetrics - by default it assumes each time series is continuous instead of discrete, so it fills gaps between real samples with regular intervals. * Metrics and labels leading to [high cardinality](https://docs.victoriametrics.com/FAQ.html#what-is-high-cardinality) or [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate) can be determined via [cardinality explorer](#cardinality-explorer) and via [/api/v1/status/tsdb](#tsdb-stats) endpoint. @@ -2112,7 +2112,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -search.maxSeries int The maximum number of time series, which can be returned from /api/v1/series. This option allows limiting memory usage (default 100000) -search.maxStalenessInterval duration - The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.maxLookback' flag, which has the same meaning due to historical reasons + The maximum interval for staleness calculations. By default it is automatically calculated from the median interval between samples. This flag could be useful for tuning Prometheus data model closer to Influx-style data model. See https://prometheus.io/docs/prometheus/latest/querying/basics/#staleness for details. See also '-search.setLookbackToStep' flag -search.maxStatusRequestDuration duration The maximum duration for /api/v1/status/* requests (default 5m0s) -search.maxStepForPointsAdjustment duration @@ -2137,6 +2137,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The minimum duration for queries to track in query stats at /api/v1/status/top_queries. Queries with lower duration are ignored in query stats (default 1ms) -search.resetCacheAuthKey string Optional authKey for resetting rollup cache via /internal/resetRollupResultCache call + -search.setLookbackToStep + Whether to fix lookback interval to 'step' query arg value. If set to true, the query model becomes closer to InfluxDB data model. If set to true, then -search.maxLookback and -search.maxStalenessInterval are ignored -search.treatDotsAsIsInRegexps Whether to treat dots as is in regexp label filters used in queries. For example, foo{bar=~"a.b.c"} will be automatically converted to foo{bar=~"a\\.b\\.c"}, i.e. all the dots in regexp filters will be automatically escaped in order to match only dot char instead of matching any char. Dots in ".+", ".*" and ".{n}" regexps aren't escaped. This option is DEPRECATED in favor of {__graphite__="a.*.c"} syntax for selecting metrics matching the given Graphite metrics filter -selfScrapeInstance string diff --git a/docs/guides/migrate-from-influx.md b/docs/guides/migrate-from-influx.md index a4965f91f..ccd3ba656 100644 --- a/docs/guides/migrate-from-influx.md +++ b/docs/guides/migrate-from-influx.md @@ -202,8 +202,10 @@ detail [here](https://docs.victoriametrics.com/keyConcepts.html#range-query). In behavior by adding `fill(previous)` to the query. VictoriaMetrics fills the gaps on the graph assuming time series are always continious and not discrete. -To limit the interval on which VictoriaMetrics will try to fill the gaps try setting `-search.maxStalenessInterval` -command-line flag to the value equal to actual resolution between data points (for example, to `10s`). +To limit the interval on which VictoriaMetrics will try to fill the gaps, set `-search.setLookbackToStep` +command-line flag. This limits the gap filling to a single `step` interval passed to +[/api/v1/query_range](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries). +This behavior is close to InfluxDB data model. ### Advanced usage