From b05ff477c8d498d691befb284f919aacbdbee793 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 6 Oct 2022 23:21:45 +0300 Subject: [PATCH] app/vmselect/promql: properly calculate `vm_rows_scanned_per_query` histogram for rollup functions, which take into account only a few samples on the provided lookbehind window --- app/vmselect/promql/rollup.go | 64 +++++++++-- app/vmselect/promql/rollup_test.go | 164 ++++++++++++++--------------- docs/CHANGELOG.md | 1 + 3 files changed, 138 insertions(+), 91 deletions(-) diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index cdc26f0a3f..05ca64216f 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -167,6 +167,8 @@ var rollupFuncsCanAdjustWindow = map[string]bool{ "timestamp": true, } +// rollupFuncsRemoveCounterResets contains functions, which need to call removeCounterResets +// over input samples before calling the corresponding rollup functions. var rollupFuncsRemoveCounterResets = map[string]bool{ "increase": true, "increase_prometheus": true, @@ -177,6 +179,36 @@ var rollupFuncsRemoveCounterResets = map[string]bool{ "rollup_rate": true, } +// rollupFuncsSamplesScannedPerCall contains functions, which scan lower number of samples +// than is passed to the rollup func. +// +// It is expected that the remaining rollupFuncs scan all the samples passed to them. +var rollupFuncsSamplesScannedPerCall = map[string]int{ + "absent_over_time": 1, + "count_over_time": 1, + "default_rollup": 1, + "delta": 2, + "delta_prometheus": 2, + "deriv_fast": 2, + "first_over_time": 1, + "idelta": 2, + "ideriv": 2, + "increase": 2, + "increase_prometheus": 2, + "increase_pure": 2, + "irate": 2, + "lag": 1, + "last_over_time": 1, + "lifetime": 2, + "present_over_time": 1, + "rate": 2, + "scrape_interval": 2, + "tfirst_over_time": 1, + "timestamp": 1, + "timestamp_with_name": 1, + "tlast_over_time": 1, +} + // These functions don't change physical meaning of input time series, // so they don't drop metric name var rollupFuncsKeepMetricName = map[string]bool{ @@ -248,14 +280,17 @@ func getRollupAggrFuncNames(expr metricsql.Expr) ([]string, error) { return aggrFuncNames, nil } -func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, end, step int64, maxPointsPerSeries int, window, lookbackDelta int64, sharedTimestamps []int64) ( +func getRollupConfigs(funcName string, rf rollupFunc, expr metricsql.Expr, start, end, step int64, maxPointsPerSeries int, + window, lookbackDelta int64, sharedTimestamps []int64) ( func(values []float64, timestamps []int64), []*rollupConfig, error) { preFunc := func(values []float64, timestamps []int64) {} - if rollupFuncsRemoveCounterResets[name] { + funcName = strings.ToLower(funcName) + if rollupFuncsRemoveCounterResets[funcName] { preFunc = func(values []float64, timestamps []int64) { removeCounterResets(values) } } + samplesScannedPerCall := rollupFuncsSamplesScannedPerCall[funcName] newRollupConfig := func(rf rollupFunc, tagValue string) *rollupConfig { return &rollupConfig{ TagValue: tagValue, @@ -267,10 +302,11 @@ func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, en MaxPointsPerSeries: maxPointsPerSeries, - MayAdjustWindow: rollupFuncsCanAdjustWindow[name], - LookbackDelta: lookbackDelta, - Timestamps: sharedTimestamps, - isDefaultRollup: name == "default_rollup", + MayAdjustWindow: rollupFuncsCanAdjustWindow[funcName], + LookbackDelta: lookbackDelta, + Timestamps: sharedTimestamps, + isDefaultRollup: funcName == "default_rollup", + samplesScannedPerCall: samplesScannedPerCall, } } appendRollupConfigs := func(dst []*rollupConfig) []*rollupConfig { @@ -280,7 +316,7 @@ func getRollupConfigs(name string, rf rollupFunc, expr metricsql.Expr, start, en return dst } var rcs []*rollupConfig - switch name { + switch funcName { case "rollup": rcs = appendRollupConfigs(rcs) case "rollup_rate", "rollup_deriv": @@ -420,6 +456,11 @@ type rollupConfig struct { // Whether default_rollup is used. isDefaultRollup bool + + // The estimated number of samples scanned per Func call. + // + // If zero, then it is considered that Func scans all the samples passed to it. + samplesScannedPerCall int } func (rc *rollupConfig) getTimestamps() []int64 { @@ -562,7 +603,8 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu ni := 0 nj := 0 f := rc.Func - var samplesScanned uint64 + samplesScanned := uint64(len(values)) + samplesScannedPerCall := uint64(rc.samplesScannedPerCall) for _, tEnd := range rc.Timestamps { tStart := tEnd - window ni = seekFirstTimestampIdxAfter(timestamps[i:], tStart, ni) @@ -594,7 +636,11 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu rfa.currTimestamp = tEnd value := f(rfa) rfa.idx++ - samplesScanned += uint64(len(rfa.values)) + if samplesScannedPerCall > 0 { + samplesScanned += samplesScannedPerCall + } else { + samplesScanned += uint64(len(rfa.values)) + } dstValues = append(dstValues, value) } putRollupFuncArg(rfa) diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go index 806398c1e5..833f32ce9f 100644 --- a/app/vmselect/promql/rollup_test.go +++ b/app/vmselect/promql/rollup_test.go @@ -587,8 +587,8 @@ func TestRollupNoWindowNoPoints(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned != 0 { - t.Fatalf("expecting zero samplesScanned from rollupConfig.Do; got %d", samplesScanned) + if samplesScanned != 12 { + t.Fatalf("expecting 12 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, nan, nan, nan, nan} timestampsExpected := []int64{0, 1, 2, 3, 4} @@ -626,8 +626,8 @@ func TestRollupWindowNoPoints(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned != 0 { - t.Fatalf("expecting zero samplesScanned from rollupConfig.Do; got %d", samplesScanned) + if samplesScanned != 12 { + t.Fatalf("expecting 12 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, nan, nan, nan, nan} timestampsExpected := []int64{0, 1, 2, 3, 4} @@ -644,8 +644,8 @@ func TestRollupWindowNoPoints(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned != 0 { - t.Fatalf("expecting zero samplesScanned from rollupConfig.Do; got %d", samplesScanned) + if samplesScanned != 12 { + t.Fatalf("expecting 12 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, nan, nan, nan} timestampsExpected := []int64{161, 171, 181, 191} @@ -665,8 +665,8 @@ func TestRollupNoWindowPartialPoints(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 15 { + t.Fatalf("expecting 15 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 123, nan, 34, nan, 44} timestampsExpected := []int64{0, 5, 10, 15, 20, 25} @@ -683,8 +683,8 @@ func TestRollupNoWindowPartialPoints(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 16 { + t.Fatalf("expecting 16 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{44, 32, 34, nan} timestampsExpected := []int64{100, 120, 140, 160} @@ -701,8 +701,8 @@ func TestRollupNoWindowPartialPoints(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, nan, 123, 34, 32} timestampsExpected := []int64{-50, 0, 50, 100, 150} @@ -722,8 +722,8 @@ func TestRollupWindowPartialPoints(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 16 { + t.Fatalf("expecting 16 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 123, 123, 34, 34} timestampsExpected := []int64{0, 5, 10, 15, 20} @@ -740,8 +740,8 @@ func TestRollupWindowPartialPoints(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 16 { + t.Fatalf("expecting 16 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{44, 34, 34, nan} timestampsExpected := []int64{100, 120, 140, 160} @@ -758,8 +758,8 @@ func TestRollupWindowPartialPoints(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 15 { + t.Fatalf("expecting 15 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 54, 44, nan} timestampsExpected := []int64{0, 50, 100, 150} @@ -779,8 +779,8 @@ func TestRollupFuncsLookbackDelta(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 18 { + t.Fatalf("expecting 18 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan} timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140} @@ -797,8 +797,8 @@ func TestRollupFuncsLookbackDelta(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 18 { + t.Fatalf("expecting 18 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan} timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140} @@ -815,8 +815,8 @@ func TestRollupFuncsLookbackDelta(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 18 { + t.Fatalf("expecting 18 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan} timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140} @@ -836,8 +836,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 123, 54, 44, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -854,8 +854,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 4, 4, 3, 1} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -872,8 +872,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 21, 12, 32, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -890,8 +890,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 123, 99, 44, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -908,8 +908,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 222, 199, 110, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -926,8 +926,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 21, -9, 22, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -944,8 +944,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, -102, -42, -10, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -962,8 +962,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{123, 33, -87, 0} timestampsExpected := []int64{10, 50, 90, 130} @@ -980,8 +980,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 0.004, 0, 0, 0.03} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -998,8 +998,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 0.031, 0.044, 0.04, 0.01} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1016,8 +1016,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 47 { + t.Fatalf("expecting 47 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 0.031, 0.075, 0.115, 0.125} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1034,8 +1034,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, 0.01} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1052,8 +1052,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 35 { + t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, 0.0125} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1070,8 +1070,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 4, 4, 3, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1088,8 +1088,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 3, 3, 2, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1106,8 +1106,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 16 { + t.Fatalf("expecting 16 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 1, 1, 1, 1, 0} timestampsExpected := []int64{0, 9, 18, 27, 36, 45} @@ -1124,8 +1124,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 2, 2, 1, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1142,8 +1142,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1160,8 +1160,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, -2879.310344827588, 127.87627310448904, -496.5831435079728, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1178,8 +1178,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 14 { + t.Fatalf("expecting 14 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, nan, nan, 0, -8900, 0} timestampsExpected := []int64{0, 4, 8, 12, 16, 20} @@ -1196,8 +1196,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1214,8 +1214,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1232,8 +1232,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 2.148, 1.593, 1.156, 1.36} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1250,8 +1250,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 24 { + t.Fatalf("expecting 24 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 4, 4, 3, 1} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1268,8 +1268,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 35 { + t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 4, 7, 6, 3} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1286,8 +1286,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 35 { + t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 21, 34, 34, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1304,8 +1304,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 35 { + t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, 2775, 5262.5, 3862.5, 1800} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1322,8 +1322,8 @@ func TestRollupFuncsNoWindow(t *testing.T) { } rc.Timestamps = rc.getTimestamps() values, samplesScanned := rc.Do(nil, testValues, testTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 35 { + t.Fatalf("expecting 35 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{nan, -0.86650328627136, -1.1200838283548589, -0.40035755084856683, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} @@ -1348,8 +1348,8 @@ func TestRollupBigNumberOfValues(t *testing.T) { srcTimestamps[i] = int64(i / 2) } values, samplesScanned := rc.Do(nil, srcValues, srcTimestamps) - if samplesScanned == 0 { - t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + if samplesScanned != 22002 { + t.Fatalf("expecting 22002 samplesScanned from rollupConfig.Do; got %d", samplesScanned) } valuesExpected := []float64{1, 4001, 8001, 9999, nan, nan} timestampsExpected := []int64{0, 2000, 4000, 6000, 8000, 10000} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 68cf5a7dc9..89fc215600 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -71,6 +71,7 @@ See [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#m * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): re-evaluate annotations per each alert evaluation. Previously, annotations were evaluated only on alert's value change. This could result in stale annotations in some cases described in [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3119). * BUGFIX: prevent from excessive CPU usage when the storage enters [read-only mode](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#readonly-mode). The previous fix in [v1.81.0](https://docs.victoriametrics.com/CHANGELOG.html#v1810) wasn't complete. * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): change default value for command-line flag `-datasource.queryStep` from `0s` to `5m`. Param `step` is added by vmalert to every rule evaluation request sent to datasource. Before this change, `step` was equal to group's evaluation interval by default. Param `step` for instant queries defines how far VM can look back for the last written data point. The change supposed to improve reliability of the rules evaluation when evaluation interval is lower than scraping interval. +* BUGFIX: properly calculate `vm_rows_scanned_per_query` histogram exported at `/metrics` page of `vmselect` and single-node VictoriaMetrics. Previously it could return misleadingly high numbers for [rollup functions](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions), which scan only a few samples on the provided lookbehind window in square brackets. For example, `increase(m[1d])` always scans only 2 rows (aka `raw samples`) per each returned time series. ## [v1.81.2](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.81.2)