From 0e7b2008b2ee5ba56782f192d4e987a295a9f743 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Tue, 14 Jul 2020 12:45:42 +0300
Subject: [PATCH] app/vmselect/prometheus: do not adjust last points in time
 series with timestamps exceeding the current time

Such timestamps usually mean that the query contains `offset`.
See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/625
---
 app/vmselect/prometheus/prometheus.go      | 19 ++++++++++----
 app/vmselect/prometheus/prometheus_test.go | 30 ++++++++++++++++++----
 2 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go
index 783dc9e492..97cefb46fd 100644
--- a/app/vmselect/prometheus/prometheus.go
+++ b/app/vmselect/prometheus/prometheus.go
@@ -878,7 +878,7 @@ func queryRangeHandler(at *auth.Token, w http.ResponseWriter, query string, star
 	}
 	queryOffset := getLatencyOffsetMilliseconds()
 	if ct-queryOffset < end {
-		result = adjustLastPoints(result, ct-queryOffset, end)
+		result = adjustLastPoints(result, ct-queryOffset, ct+step)
 	}
 
 	// Remove NaN values as Prometheus does.
@@ -923,23 +923,32 @@ func removeNaNValuesInplace(tss []netstorage.Result) {
 
 var queryRangeDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/api/v1/query_range"}`)
 
+var nan = math.NaN()
+
 // adjustLastPoints substitutes the last point values on the time range (start..end]
-// with the previous point values, since these points may contain garbage.
+// with the previous point values, since these points may contain incomplete values.
 func adjustLastPoints(tss []netstorage.Result, start, end int64) []netstorage.Result {
 	for i := range tss {
 		ts := &tss[i]
 		values := ts.Values
 		timestamps := ts.Timestamps
 		j := len(timestamps) - 1
+		if j >= 0 && timestamps[j] > end {
+			// It looks like the `offset` is used in the query, which shifts time range beyond the `end`.
+			// Leave such a time series as is, since it is unclear which points may be incomplete in it.
+			// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/625
+			continue
+		}
 		for j >= 0 && timestamps[j] > start {
 			j--
 		}
 		j++
-		if j <= 0 {
-			continue
+		lastValue := nan
+		if j > 0 {
+			lastValue = values[j-1]
 		}
 		for j < len(timestamps) && timestamps[j] <= end {
-			values[j] = values[j-1]
+			values[j] = lastValue
 			j++
 		}
 	}
diff --git a/app/vmselect/prometheus/prometheus_test.go b/app/vmselect/prometheus/prometheus_test.go
index 32bfdd585a..a84028f927 100644
--- a/app/vmselect/prometheus/prometheus_test.go
+++ b/app/vmselect/prometheus/prometheus_test.go
@@ -20,8 +20,6 @@ func TestRemoveNaNValuesInplace(t *testing.T) {
 		}
 	}
 
-	nan := math.NaN()
-
 	f(nil, nil)
 	f([]netstorage.Result{
 		{
@@ -123,14 +121,14 @@ func TestAdjustLastPoints(t *testing.T) {
 				expectedValue := tssExpected[i].Values[j]
 				if math.IsNaN(expectedValue) {
 					if !math.IsNaN(value) {
-						t.Fatalf("unexpected result; got %v; want nan", value)
+						t.Fatalf("unexpected value for time series #%d at position %d; got %v; want nan", i, j, value)
 					}
 				} else if expectedValue != value {
-					t.Fatalf("unexpected result; got %v; want %v", value, expectedValue)
+					t.Fatalf("unexpected value for time series #%d at position %d; got %v; want %v", i, j, value, expectedValue)
 				}
 			}
 			if !reflect.DeepEqual(ts.Timestamps, tssExpected[i].Timestamps) {
-				t.Fatalf("unexpected result; got %v; want %v", tss, tssExpected)
+				t.Fatalf("unexpected timestamps for time series #%d; got %v; want %v", i, tss, tssExpected)
 			}
 		}
 
@@ -239,4 +237,26 @@ func TestAdjustLastPoints(t *testing.T) {
 			Values:     []float64{1, 2, nan},
 		},
 	})
+
+	// Check for timestamps outside the configured time range.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/625
+	f([]netstorage.Result{
+		{
+			Timestamps: []int64{100, 200, 300, 400, 500},
+			Values:     []float64{1, 2, 3, nan, nan},
+		},
+		{
+			Timestamps: []int64{100, 200, 300},
+			Values:     []float64{1, 2, 45},
+		},
+	}, 250, 400, []netstorage.Result{
+		{
+			Timestamps: []int64{100, 200, 300, 400, 500},
+			Values:     []float64{1, 2, 3, nan, nan},
+		},
+		{
+			Timestamps: []int64{100, 200, 300},
+			Values:     []float64{1, 2, 2},
+		},
+	})
 }