diff --git a/README.md b/README.md index 2cac9d126..cc615c1ed 100644 --- a/README.md +++ b/README.md @@ -848,6 +848,10 @@ The most interesting metrics are: of data loss stored in the broken parts. In the future, `vmrecover` tool will be created for automatic recovering from such errors. +* If you see gaps on the graphs, try resetting the cache by sending request to `/internal/resetRollupResultCache`. + If this removes gaps on the graphs, then it is likely data with timestamps older than `-search.cacheTimestampOffset` + is ingested into VictoriaMetrics. Make sure that data sources have synchronized time with VictoriaMetrics. + ### Backfilling @@ -858,6 +862,9 @@ It is recommended disabling query cache with `-search.disableCache` command-line historical data with timestamps from the past, since the cache assumes that the data is written with the current timestamps. Query cache can be enabled after the backfilling is complete. +An alternative solution is to query `/internal/resetRollupResultCache` url after backfilling is complete. This will reset +the query cache, which could contain incomplete data cached during the backfilling. + ### Profiling diff --git a/app/vmselect/promql/rollup_result_cache.go b/app/vmselect/promql/rollup_result_cache.go index 8a807a802..89d85e49d 100644 --- a/app/vmselect/promql/rollup_result_cache.go +++ b/app/vmselect/promql/rollup_result_cache.go @@ -18,7 +18,12 @@ import ( "github.com/VictoriaMetrics/metrics" ) -var disableCache = flag.Bool("search.disableCache", false, "Whether to disable response caching. This may be useful during data backfilling") +var ( + disableCache = flag.Bool("search.disableCache", false, "Whether to disable response caching. This may be useful during data backfilling") + cacheTimestampOffset = flag.Duration("search.cacheTimestampOffset", 5*time.Minute, "The maximum duration since the current time for response data, "+ + "which is always queried from the original raw data, without using the response cache. Increase this value if you see gaps in responses "+ + "due to time synchronization issues between VictoriaMetrics and data sources") +) var rollupResultCacheV = &rollupResultCache{ c: workingsetcache.New(1024*1024, time.Hour), // This is a cache for testing. @@ -215,10 +220,10 @@ func (rrc *rollupResultCache) Put(ec *EvalConfig, expr metricsql.Expr, window in return } - // Remove values up to currentTime - step - maxSilenceInterval, + // Remove values up to currentTime - step - cacheTimestampOffset, // since these values may be added later. timestamps := tss[0].Timestamps - deadline := (time.Now().UnixNano() / 1e6) - ec.Step - maxSilenceInterval + deadline := (time.Now().UnixNano() / 1e6) - ec.Step - cacheTimestampOffset.Milliseconds() i := len(timestamps) - 1 for i >= 0 && timestamps[i] > deadline { i--