From a43f2d0bc53c6ed14034af650078443590ab45b6 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 28 Jun 2022 19:26:17 +0300 Subject: [PATCH] app/vmselect/promql: show the number of scanned samples in the query trace --- app/vmselect/promql/eval.go | 37 +++-- app/vmselect/promql/rollup.go | 20 ++- app/vmselect/promql/rollup_test.go | 210 +++++++++++++++++++++++------ 3 files changed, 208 insertions(+), 59 deletions(-) diff --git a/app/vmselect/promql/eval.go b/app/vmselect/promql/eval.go index 92836df95..684a83b60 100644 --- a/app/vmselect/promql/eval.go +++ b/app/vmselect/promql/eval.go @@ -8,6 +8,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" @@ -762,27 +763,30 @@ func evalRollupFuncWithSubquery(qt *querytracer.Tracer, ec *EvalConfig, funcName } tss := make([]*timeseries, 0, len(tssSQ)*len(rcs)) var tssLock sync.Mutex + var samplesScannedTotal uint64 keepMetricNames := getKeepMetricNames(expr) doParallel(tssSQ, func(tsSQ *timeseries, values []float64, timestamps []int64) ([]float64, []int64) { values, timestamps = removeNanValues(values[:0], timestamps[:0], tsSQ.Values, tsSQ.Timestamps) preFunc(values, timestamps) for _, rc := range rcs { if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &tsSQ.MetricName); tsm != nil { - rc.DoTimeseriesMap(tsm, values, timestamps) + samplesScanned := rc.DoTimeseriesMap(tsm, values, timestamps) + atomic.AddUint64(&samplesScannedTotal, samplesScanned) tssLock.Lock() tss = tsm.AppendTimeseriesTo(tss) tssLock.Unlock() continue } var ts timeseries - doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &tsSQ.MetricName, values, timestamps, sharedTimestamps) + samplesScanned := doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &tsSQ.MetricName, values, timestamps, sharedTimestamps) + atomic.AddUint64(&samplesScannedTotal, samplesScanned) tssLock.Lock() tss = append(tss, &ts) tssLock.Unlock() } return values, timestamps }) - qt.Printf("rollup %s() over %d series returned by subquery: series=%d", funcName, len(tssSQ), len(tss)) + qt.Printf("rollup %s() over %d series returned by subquery: series=%d, samplesScanned=%d", funcName, len(tssSQ), len(tss), samplesScannedTotal) return tss, nil } @@ -981,8 +985,9 @@ func getRollupMemoryLimiter() *memoryLimiter { func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool, iafc *incrementalAggrFuncContext, rss *netstorage.Results, rcs []*rollupConfig, preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) { - qt = qt.NewChild("rollup %s() with incremental aggregation %s() over %d series", funcName, iafc.ae.Name, rss.Len()) + qt = qt.NewChild("rollup %s() with incremental aggregation %s() over %d series; rollupConfigs=%s", funcName, iafc.ae.Name, rss.Len(), rcs) defer qt.Done() + var samplesScannedTotal uint64 err := rss.RunParallel(qt, func(rs *netstorage.Result, workerID uint) error { rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps) preFunc(rs.Values, rs.Timestamps) @@ -990,14 +995,16 @@ func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string, defer putTimeseries(ts) for _, rc := range rcs { if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &rs.MetricName); tsm != nil { - rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps) + samplesScanned := rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps) for _, ts := range tsm.m { iafc.updateTimeseries(ts, workerID) } + atomic.AddUint64(&samplesScannedTotal, samplesScanned) continue } ts.Reset() - doRollupForTimeseries(funcName, keepMetricNames, rc, ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps) + samplesScanned := doRollupForTimeseries(funcName, keepMetricNames, rc, ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps) + atomic.AddUint64(&samplesScannedTotal, samplesScanned) iafc.updateTimeseries(ts, workerID) // ts.Timestamps points to sharedTimestamps. Zero it, so it can be re-used. @@ -1010,35 +1017,39 @@ func evalRollupWithIncrementalAggregate(qt *querytracer.Tracer, funcName string, return nil, err } tss := iafc.finalizeTimeseries() - qt.Printf("series after aggregation with %s(): %d", iafc.ae.Name, len(tss)) + qt.Printf("series after aggregation with %s(): %d; samplesScanned=%d", iafc.ae.Name, len(tss), samplesScannedTotal) return tss, nil } func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, keepMetricNames bool, rss *netstorage.Results, rcs []*rollupConfig, preFunc func(values []float64, timestamps []int64), sharedTimestamps []int64) ([]*timeseries, error) { - qt = qt.NewChild("rollup %s() over %d series", funcName, rss.Len()) + qt = qt.NewChild("rollup %s() over %d series; rollupConfigs=%s", funcName, rss.Len(), rcs) defer qt.Done() tss := make([]*timeseries, 0, rss.Len()*len(rcs)) var tssLock sync.Mutex + var samplesScannedTotal uint64 err := rss.RunParallel(qt, func(rs *netstorage.Result, workerID uint) error { rs.Values, rs.Timestamps = dropStaleNaNs(funcName, rs.Values, rs.Timestamps) preFunc(rs.Values, rs.Timestamps) for _, rc := range rcs { if tsm := newTimeseriesMap(funcName, keepMetricNames, sharedTimestamps, &rs.MetricName); tsm != nil { - rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps) + samplesScanned := rc.DoTimeseriesMap(tsm, rs.Values, rs.Timestamps) + atomic.AddUint64(&samplesScannedTotal, samplesScanned) tssLock.Lock() tss = tsm.AppendTimeseriesTo(tss) tssLock.Unlock() continue } var ts timeseries - doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps) + samplesScanned := doRollupForTimeseries(funcName, keepMetricNames, rc, &ts, &rs.MetricName, rs.Values, rs.Timestamps, sharedTimestamps) + atomic.AddUint64(&samplesScannedTotal, samplesScanned) tssLock.Lock() tss = append(tss, &ts) tssLock.Unlock() } return nil }) + qt.Printf("samplesScanned=%d", samplesScannedTotal) if err != nil { return nil, err } @@ -1046,7 +1057,7 @@ func evalRollupNoIncrementalAggregate(qt *querytracer.Tracer, funcName string, k } func doRollupForTimeseries(funcName string, keepMetricNames bool, rc *rollupConfig, tsDst *timeseries, mnSrc *storage.MetricName, - valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64) { + valuesSrc []float64, timestampsSrc []int64, sharedTimestamps []int64) uint64 { tsDst.MetricName.CopyFrom(mnSrc) if len(rc.TagValue) > 0 { tsDst.MetricName.AddTag("rollup", rc.TagValue) @@ -1054,9 +1065,11 @@ func doRollupForTimeseries(funcName string, keepMetricNames bool, rc *rollupConf if !keepMetricNames && !rollupFuncsKeepMetricName[funcName] { tsDst.MetricName.ResetMetricGroup() } - tsDst.Values = rc.Do(tsDst.Values[:0], valuesSrc, timestampsSrc) + var samplesScanned uint64 + tsDst.Values, samplesScanned = rc.Do(tsDst.Values[:0], valuesSrc, timestampsSrc) tsDst.Timestamps = sharedTimestamps tsDst.denyReuse = true + return samplesScanned } var bbPool bytesutil.ByteBufferPool diff --git a/app/vmselect/promql/rollup.go b/app/vmselect/promql/rollup.go index 640a35828..b19ac8273 100644 --- a/app/vmselect/promql/rollup.go +++ b/app/vmselect/promql/rollup.go @@ -416,6 +416,12 @@ type rollupConfig struct { isDefaultRollup bool } +func (rc *rollupConfig) String() string { + start := storage.TimestampToHumanReadableFormat(rc.Start) + end := storage.TimestampToHumanReadableFormat(rc.End) + return fmt.Sprintf("timeRange=[%s..%s], step=%d, window=%d, points=%d", start, end, rc.Step, rc.Window, len(rc.Timestamps)) +} + var ( nan = math.NaN() inf = math.Inf(1) @@ -483,18 +489,20 @@ func (tsm *timeseriesMap) GetOrCreateTimeseries(labelName, labelValue string) *t // timestamps must cover time range [rc.Start - rc.Window - maxSilenceInterval ... rc.End]. // // Do cannot be called from concurrent goroutines. -func (rc *rollupConfig) Do(dstValues []float64, values []float64, timestamps []int64) []float64 { +func (rc *rollupConfig) Do(dstValues []float64, values []float64, timestamps []int64) ([]float64, uint64) { return rc.doInternal(dstValues, nil, values, timestamps) } // DoTimeseriesMap calculates rollups for the given timestamps and values and puts them to tsm. -func (rc *rollupConfig) DoTimeseriesMap(tsm *timeseriesMap, values []float64, timestamps []int64) { +func (rc *rollupConfig) DoTimeseriesMap(tsm *timeseriesMap, values []float64, timestamps []int64) uint64 { ts := getTimeseries() - ts.Values = rc.doInternal(ts.Values[:0], tsm, values, timestamps) + var samplesScanned uint64 + ts.Values, samplesScanned = rc.doInternal(ts.Values[:0], tsm, values, timestamps) putTimeseries(ts) + return samplesScanned } -func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, values []float64, timestamps []int64) []float64 { +func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, values []float64, timestamps []int64) ([]float64, uint64) { // Sanity checks. if rc.Step <= 0 { logger.Panicf("BUG: Step must be bigger than 0; got %d", rc.Step) @@ -544,6 +552,7 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu ni := 0 nj := 0 f := rc.Func + var samplesScanned uint64 for _, tEnd := range rc.Timestamps { tStart := tEnd - window ni = seekFirstTimestampIdxAfter(timestamps[i:], tStart, ni) @@ -575,11 +584,12 @@ func (rc *rollupConfig) doInternal(dstValues []float64, tsm *timeseriesMap, valu rfa.currTimestamp = tEnd value := f(rfa) rfa.idx++ + samplesScanned += uint64(len(rfa.values)) dstValues = append(dstValues, value) } putRollupFuncArg(rfa) - return dstValues + return dstValues, samplesScanned } func seekFirstTimestampIdxAfter(timestamps []int64, seekTimestamp int64, nHint int) int { diff --git a/app/vmselect/promql/rollup_test.go b/app/vmselect/promql/rollup_test.go index 67e0934cf..bfc6d670e 100644 --- a/app/vmselect/promql/rollup_test.go +++ b/app/vmselect/promql/rollup_test.go @@ -584,7 +584,10 @@ func TestRollupNoWindowNoPoints(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned != 0 { + t.Fatalf("expecting zero samplesScanned from rollupConfig.Do; got %d", samplesScanned) + } valuesExpected := []float64{nan, nan, nan, nan, nan} timestampsExpected := []int64{0, 1, 2, 3, 4} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -598,7 +601,10 @@ func TestRollupNoWindowNoPoints(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{2, 0, 0, 0, nan, nan, nan, nan} timestampsExpected := []int64{120, 124, 128, 132, 136, 140, 144, 148} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -615,7 +621,10 @@ func TestRollupWindowNoPoints(t *testing.T) { Window: 3, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned != 0 { + t.Fatalf("expecting zero samplesScanned from rollupConfig.Do; got %d", samplesScanned) + } valuesExpected := []float64{nan, nan, nan, nan, nan} timestampsExpected := []int64{0, 1, 2, 3, 4} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -629,7 +638,10 @@ func TestRollupWindowNoPoints(t *testing.T) { Window: 3, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned != 0 { + t.Fatalf("expecting zero samplesScanned from rollupConfig.Do; got %d", samplesScanned) + } valuesExpected := []float64{nan, nan, nan, nan} timestampsExpected := []int64{161, 171, 181, 191} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -646,7 +658,10 @@ func TestRollupNoWindowPartialPoints(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 123, nan, 34, nan, 44} timestampsExpected := []int64{0, 5, 10, 15, 20, 25} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -660,7 +675,10 @@ func TestRollupNoWindowPartialPoints(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{44, 32, 34, nan} timestampsExpected := []int64{100, 120, 140, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -674,7 +692,10 @@ func TestRollupNoWindowPartialPoints(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, nan, 123, 34, 32} timestampsExpected := []int64{-50, 0, 50, 100, 150} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -691,7 +712,10 @@ func TestRollupWindowPartialPoints(t *testing.T) { Window: 8, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 123, 123, 34, 34} timestampsExpected := []int64{0, 5, 10, 15, 20} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -705,7 +729,10 @@ func TestRollupWindowPartialPoints(t *testing.T) { Window: 18, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{44, 34, 34, nan} timestampsExpected := []int64{100, 120, 140, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -719,7 +746,10 @@ func TestRollupWindowPartialPoints(t *testing.T) { Window: 19, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 54, 44, nan} timestampsExpected := []int64{0, 50, 100, 150} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -736,7 +766,10 @@ func TestRollupFuncsLookbackDelta(t *testing.T) { LookbackDelta: 1, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan} timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -750,7 +783,10 @@ func TestRollupFuncsLookbackDelta(t *testing.T) { LookbackDelta: 7, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan} timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -764,7 +800,10 @@ func TestRollupFuncsLookbackDelta(t *testing.T) { LookbackDelta: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{99, nan, 44, nan, 32, 34, nan} timestampsExpected := []int64{80, 90, 100, 110, 120, 130, 140} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -781,7 +820,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 123, 54, 44, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -795,7 +837,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 4, 4, 3, 1} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -809,7 +854,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 21, 12, 32, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -823,7 +871,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 123, 99, 44, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -837,7 +888,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 222, 199, 110, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -851,7 +905,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 21, -9, 22, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -865,7 +922,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, -102, -42, -10, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -879,7 +939,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{123, 33, -87, 0} timestampsExpected := []int64{10, 50, 90, 130} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -893,7 +956,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 0.004, 0, 0, 0.03} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -907,7 +973,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 0.031, 0.044, 0.04, 0.01} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -921,7 +990,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 200, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 0.031, 0.075, 0.115, 0.125} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -935,7 +1007,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 0.010333333333333333, 0.011, 0.013333333333333334, 0.01} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -949,7 +1024,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 80, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 0.010333333333333333, 0.010714285714285714, 0.012, 0.0125} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -963,7 +1041,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 4, 4, 3, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -977,7 +1058,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 3, 3, 2, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -991,7 +1075,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 9, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 1, 1, 1, 1, 0} timestampsExpected := []int64{0, 9, 18, 27, 36, 45} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -1005,7 +1092,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 2, 2, 1, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -1019,7 +1109,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 55.5, 49.75, 36.666666666666664, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -1033,7 +1126,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, -2879.310344827588, 127.87627310448904, -496.5831435079728, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -1047,7 +1143,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, nan, nan, 0, -8900, 0} timestampsExpected := []int64{0, 4, 8, 12, 16, 20} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -1061,7 +1160,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, -1916.6666666666665, -43500, 400, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -1075,7 +1177,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 39.81519810323691, 32.080952292598795, 5.2493385826745405, 0} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -1089,7 +1194,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 2.148, 1.593, 1.156, 1.36} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -1103,7 +1211,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 0, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 4, 4, 3, 1} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -1117,7 +1228,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 80, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 4, 7, 6, 3} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -1131,7 +1245,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 80, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 21, 34, 34, 34} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -1145,7 +1262,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 80, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, 2775, 5262.5, 3678.5714285714284, 2880} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -1159,7 +1279,10 @@ func TestRollupFuncsNoWindow(t *testing.T) { Window: 80, } rc.Timestamps = getTimestamps(rc.Start, rc.End, rc.Step) - values := rc.Do(nil, testValues, testTimestamps) + values, samplesScanned := rc.Do(nil, testValues, testTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{nan, -0.86650328627136, -1.1200838283548589, -0.40035755084856683, nan} timestampsExpected := []int64{0, 40, 80, 120, 160} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected) @@ -1181,7 +1304,10 @@ func TestRollupBigNumberOfValues(t *testing.T) { srcValues[i] = float64(i) srcTimestamps[i] = int64(i / 2) } - values := rc.Do(nil, srcValues, srcTimestamps) + values, samplesScanned := rc.Do(nil, srcValues, srcTimestamps) + if samplesScanned == 0 { + t.Fatalf("expecting non-zero samplesScanned from rollupConfig.Do") + } valuesExpected := []float64{1, 4001, 8001, 9999, nan, nan} timestampsExpected := []int64{0, 2000, 4000, 6000, 8000, 10000} testRowsEqual(t, values, rc.Timestamps, valuesExpected, timestampsExpected)