app/vmselect/promql: consistently replace NaN data points with non-NaN values for range_first and range_last functions

It is expected that range_first and range_last functions return non-nan const value across all the points
if the original series contains at least a single non-NaN value. Previously this rule was violated for NaN data points
in the original series. This could confuse users.

While at it, add tests for series with NaN values across all the range_* and running_* functions, in order to maintain
consistent handling of NaN values across these functions.
This commit is contained in:
Aliaksandr Valialkin 2024-09-23 14:54:39 +02:00
parent 9ebc056861
commit 2a17cddf3d
No known key found for this signature in database
GPG Key ID: 52C003EE2BCDB9EB
3 changed files with 232 additions and 8 deletions

View File

@ -7261,6 +7261,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_trim_outliers(time() > 1200)`, func(t *testing.T) {
t.Parallel()
q := `range_trim_outliers(0.5, time() > 1200)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{nan, nan, nan, 1600, 1800, nan},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_trim_spikes()`, func(t *testing.T) {
t.Parallel()
q := `range_trim_spikes(0.2, time())`
@ -7272,6 +7283,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_trim_spikes(time() > 1200 <= 1800)`, func(t *testing.T) {
t.Parallel()
q := `range_trim_spikes(0.2, time() > 1200 <= 1800)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{nan, nan, nan, 1600, nan, nan},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_trim_zscore()`, func(t *testing.T) {
t.Parallel()
q := `range_trim_zscore(0.9, time())`
@ -7283,6 +7305,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_trim_zscore(time() > 1200 <= 1800)`, func(t *testing.T) {
t.Parallel()
q := `range_trim_zscore(0.9, time() > 1200 <= 1800)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{nan, nan, nan, 1600, nan, nan},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_zscore()`, func(t *testing.T) {
t.Parallel()
q := `round(range_zscore(time()), 0.1)`
@ -7294,6 +7327,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_zscore(time() > 1200 < 1800)`, func(t *testing.T) {
t.Parallel()
q := `round(range_zscore(time() > 1200 < 1800), 0.1)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{nan, nan, -1, 1, nan, nan},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_quantile(0.5)`, func(t *testing.T) {
t.Parallel()
q := `range_quantile(0.5, time())`
@ -7305,6 +7349,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_quantile(0.5, time() > 1200 < 2000)`, func(t *testing.T) {
t.Parallel()
q := `range_quantile(0.5, time() > 1200 < 2000)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1600, 1600, 1600, 1600, 1600, 1600},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_stddev()`, func(t *testing.T) {
t.Parallel()
q := `round(range_stddev(time()),0.01)`
@ -7316,6 +7371,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_stddev(time() > 1200 < 1800)`, func(t *testing.T) {
t.Parallel()
q := `round(range_stddev(time() > 1200 < 1800),0.01)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{100, 100, 100, 100, 100, 100},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_stdvar()`, func(t *testing.T) {
t.Parallel()
q := `round(range_stdvar(time()),0.01)`
@ -7327,6 +7393,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_stdvar(time() > 1200 < 1800)`, func(t *testing.T) {
t.Parallel()
q := `round(range_stdvar(time() > 1200 < 1800),0.01)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{10000, 10000, 10000, 10000, 10000, 10000},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_median()`, func(t *testing.T) {
t.Parallel()
q := `range_median(time())`
@ -7687,6 +7764,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`running_min(abs(1500-time()) < 400 > 100)`, func(t *testing.T) {
t.Parallel()
q := `running_min(abs(1500-time()) < 400 > 100)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{nan, 300, 300, 300, 300, 300},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`running_max(abs(1300-time()))`, func(t *testing.T) {
t.Parallel()
q := `running_max(abs(1300-time()))`
@ -7698,6 +7786,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`running_max(abs(1300-time()) > 300 < 700)`, func(t *testing.T) {
t.Parallel()
q := `running_max(abs(1300-time()) > 300 < 700)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{nan, nan, nan, nan, 500, 500},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`running_sum(1)`, func(t *testing.T) {
t.Parallel()
q := `running_sum(1)`
@ -7720,6 +7819,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`running_sum(time() > 1.2 < 1.8)`, func(t *testing.T) {
t.Parallel()
q := `running_sum(time()/1e3 > 1.2 < 1.8)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{nan, nan, 1.4, 3, 3, 3},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`running_avg(time())`, func(t *testing.T) {
t.Parallel()
q := `running_avg(time())`
@ -7731,6 +7841,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`running_avg(time() > 1200 < 1800)`, func(t *testing.T) {
t.Parallel()
q := `running_avg(time() > 1200 < 1800)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{nan, nan, 1400, 1500, 1500, 1500},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`smooth_exponential(time(), 1)`, func(t *testing.T) {
t.Parallel()
q := `smooth_exponential(time(), 1)`
@ -7811,6 +7932,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_min(time() > 1200 < 1800)`, func(t *testing.T) {
t.Parallel()
q := `range_min(time() > 1200 < 1800)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1400, 1400, 1400, 1400, 1400, 1400},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_normalize(time(),alias(-time(),"negative"))`, func(t *testing.T) {
t.Parallel()
q := `range_normalize(time(),alias(-time(), "negative"))`
@ -7828,6 +7960,23 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r1, r2}
f(q, resultExpected)
})
t.Run(`range_normalize(time() > 1200 < 1800,alias(-(time() > 1400 < 2000),"negative"))`, func(t *testing.T) {
t.Parallel()
q := `range_normalize(time() > 1200 < 1800,alias(-(time() > 1200 < 2000), "negative"))`
r1 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{nan, nan, 0, 1, nan, nan},
Timestamps: timestampsExpected,
}
r2 := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{nan, nan, 1, 0.5, 0, nan},
Timestamps: timestampsExpected,
}
r2.MetricName.MetricGroup = []byte("negative")
resultExpected := []netstorage.Result{r1, r2}
f(q, resultExpected)
})
t.Run(`range_first(time())`, func(t *testing.T) {
t.Parallel()
q := `range_first(time())`
@ -7839,6 +7988,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_first(time() > 1200 < 1800)`, func(t *testing.T) {
t.Parallel()
q := `range_first(time() > 1200 < 1800)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1400, 1400, 1400, 1400, 1400, 1400},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_mad(time())`, func(t *testing.T) {
t.Parallel()
q := `range_mad(time())`
@ -7850,6 +8010,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_mad(time() > 1200 < 1800)`, func(t *testing.T) {
t.Parallel()
q := `range_mad(time() > 1200 < 1800)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{100, 100, 100, 100, 100, 100},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_max(time())`, func(t *testing.T) {
t.Parallel()
q := `range_max(time())`
@ -7861,6 +8032,39 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_max(time() > 1200 < 1800)`, func(t *testing.T) {
t.Parallel()
q := `range_max(time() > 1200 < 1800)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1600, 1600, 1600, 1600, 1600, 1600},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_sum(time())`, func(t *testing.T) {
t.Parallel()
q := `range_sum(time())`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{9000, 9000, 9000, 9000, 9000, 9000},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_sum(time() > 1200 < 1800)`, func(t *testing.T) {
t.Parallel()
q := `range_sum(time() > 1200 < 1800)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{3000, 3000, 3000, 3000, 3000, 3000},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_last(time())`, func(t *testing.T) {
t.Parallel()
q := `range_last(time())`
@ -7872,6 +8076,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_last(time() > 1200 < 1800)`, func(t *testing.T) {
t.Parallel()
q := `range_last(time() > 1200 < 1800)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1600, 1600, 1600, 1600, 1600, 1600},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_linear_regression(time())`, func(t *testing.T) {
t.Parallel()
q := `range_linear_regression(time())`
@ -7894,6 +8109,17 @@ func TestExecSuccess(t *testing.T) {
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_linear_regression(time() > 1200 < 1800)`, func(t *testing.T) {
t.Parallel()
q := `range_linear_regression(time() > 1200 < 1800)`
r := netstorage.Result{
MetricName: metricNameExpected,
Values: []float64{1000, 1200, 1400, 1600, 1800, 2000},
Timestamps: timestampsExpected,
}
resultExpected := []netstorage.Result{r}
f(q, resultExpected)
})
t.Run(`range_linear_regression(100/time())`, func(t *testing.T) {
t.Parallel()
q := `sort_desc(round((

View File

@ -1544,10 +1544,8 @@ func transformRangeFirst(tfa *transformFuncArg) ([]*timeseries, error) {
continue
}
vFirst := values[0]
for i, v := range values {
if math.IsNaN(v) {
continue
}
values = ts.Values
for i := range values {
values[i] = vFirst
}
}
@ -1571,10 +1569,8 @@ func setLastValues(tss []*timeseries) {
continue
}
vLast := values[len(values)-1]
for i, v := range values {
if math.IsNaN(v) {
continue
}
values = ts.Values
for i := range values {
values[i] = vLast
}
}

View File

@ -40,6 +40,8 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert): do not send notifications without labels to Alertmanager. Such notifications are rejected by Alertmanager anyway. Before, vmalert could send alert notifications even if no label-value pairs left after applying `alert_relabel_configs` from [notifier config](https://docs.victoriametrics.com/vmalert/#notifier-configuration-file).
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert/): properly update value of variable `$activeAt` in rules annotation during replay mode. Before, `$activeAt` could have provided incorrect values during replay.
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/metricsql/): properly handle `c1 AND c2` and `c1 OR c1` queries for constants `c1` and `c2`. Previously such queries could return unexpected results. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6637).
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/metricsql/): consistently return the first non-`NaN` value from [`range_first`](https://docs.victoriametrics.com/metricsql/#range_first) function across all the returned data points. Previously `NaN` data points weren't replaced with the first non-`NaN` value.
* BUGFIX: [MetricsQL](https://docs.victoriametrics.com/metricsql/): consistently return the last non-`NaN` value from [`range_last`](https://docs.victoriametrics.com/metricsql/#range_last) function across all the returned data points. Previously `NaN` data points weren't replaced with the last non-`NaN` value.
* BUGFIX: all VictoriaMetrics components: increase default value of `-loggerMaxArgLen` cmd-line flag from 1000 to 5000. This should improve visibility on errors produced by very long queries.
## [v1.103.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.103.0)