From 1e2c5117474dca51d456a36a3661b2db0e921325 Mon Sep 17 00:00:00 2001 From: Artem Navoiev Date: Sun, 6 Oct 2019 23:01:45 +0300 Subject: [PATCH] Add regression test for query apo Part of https://github.com/VictoriaMetrics/VictoriaMetrics/issues/187 cover: - https://github.com/VictoriaMetrics/VictoriaMetrics/issues/153 - https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150 --- app/victoria-metrics/main_test.go | 116 +++++++++++++----- .../graphite/comparison-not-inf-not-nan.json | 14 +++ .../graphite/not-nan-as-missing-data.json | 14 +++ 3 files changed, 111 insertions(+), 33 deletions(-) create mode 100644 app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json create mode 100644 app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json diff --git a/app/victoria-metrics/main_test.go b/app/victoria-metrics/main_test.go index 995ed8221..b6f106fac 100644 --- a/app/victoria-metrics/main_test.go +++ b/app/victoria-metrics/main_test.go @@ -56,13 +56,14 @@ var ( ) type test struct { - Name string `json:"name"` - Data []string `json:"data"` - Query []string `json:"query"` - ResultMetrics []Metric `json:"result_metrics"` - ResultSeries Series `json:"result_series"` - ResultQuery Query `json:"result_query"` - Issue string `json:"issue"` + Name string `json:"name"` + Data []string `json:"data"` + Query []string `json:"query"` + ResultMetrics []Metric `json:"result_metrics"` + ResultSeries Series `json:"result_series"` + ResultQuery Query `json:"result_query"` + ResultQueryRange QueryRange `json:"result_query_range"` + Issue string `json:"issue"` } type Metric struct { @@ -99,6 +100,25 @@ func (r *QueryDataResult) UnmarshalJSON(b []byte) error { return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r)) } +type QueryRange struct { + Status string `json:"status"` + Data QueryRangeData `json:"data"` +} +type QueryRangeData struct { + ResultType string `json:"resultType"` + Result []QueryRangeDataResult `json:"result"` +} + +type QueryRangeDataResult struct { + Metric map[string]string `json:"metric"` + Values [][]interface{} `json:"values"` +} + +func (r *QueryRangeDataResult) UnmarshalJSON(b []byte) error { + type plain QueryRangeDataResult + return json.Unmarshal(testutil.PopulateTimeTpl(b, insertionTime), (*plain)(r)) +} + func TestMain(m *testing.M) { setUp() code := m.Run() @@ -242,19 +262,32 @@ func testRead(t *testing.T) { t.Parallel() for _, q := range test.Query { q = testutil.PopulateTimeTplString(q, insertionTime) + if test.Issue != "" { + test.Issue = "Regression in " + test.Issue + } switch true { case strings.HasPrefix(q, "/api/v1/export"): - checkMetricsResult(t, httpReadMetrics(t, testReadHTTPPath, q), test.ResultMetrics, q, test.Issue) + if err := checkMetricsResult(httpReadMetrics(t, testReadHTTPPath, q), test.ResultMetrics); err != nil { + t.Fatalf("Export. %s fails with error %s.%s", q, err, test.Issue) + } case strings.HasPrefix(q, "/api/v1/series"): s := Series{} httpReadStruct(t, testReadHTTPPath, q, &s) - checkSeriesResult(t, s, test.ResultSeries, q, test.Issue) + if err := checkSeriesResult(s, test.ResultSeries); err != nil { + t.Fatalf("Series. %s fails with error %s.%s", q, err, test.Issue) + } case strings.HasPrefix(q, "/api/v1/query_range"): - t.Fatalf("unsupported read query %s", q) + queryResult := QueryRange{} + httpReadStruct(t, testReadHTTPPath, q, &queryResult) + if err := checkQueryRangeResult(queryResult, test.ResultQueryRange); err != nil { + t.Fatalf("Query Range. %s fails with error %s.%s", q, err, test.Issue) + } case strings.HasPrefix(q, "/api/v1/query"): queryResult := Query{} httpReadStruct(t, testReadHTTPPath, q, &queryResult) - checkQueryResult(t, queryResult, test.ResultQuery, q, test.Issue) + if err := checkQueryResult(queryResult, test.ResultQuery); err != nil { + t.Fatalf("Query. %s fails with error %s.%s", q, err, test.Issue) + } default: t.Fatalf("unsupported read query %s", q) } @@ -334,17 +367,14 @@ func httpReadStruct(t *testing.T, address, query string, dst interface{}) { s.noError(json.NewDecoder(resp.Body).Decode(dst)) } -func checkMetricsResult(t *testing.T, got, want []Metric, query, issue string) { - t.Helper() +func checkMetricsResult(got, want []Metric) error { for _, r := range append([]Metric(nil), got...) { want = removeIfFoundMetrics(r, want) } if len(want) > 0 { - if issue != "" { - issue = "Regression in " + issue - } - t.Fatalf("query: %s. Result metrics %+v not found in %+v.%s", query, want, got, issue) + return fmt.Errorf("exptected metrics %+v not found in %+v", want, got) } + return nil } func removeIfFoundMetrics(r Metric, contains []Metric) []Metric { @@ -358,21 +388,18 @@ func removeIfFoundMetrics(r Metric, contains []Metric) []Metric { return contains } -func checkSeriesResult(t *testing.T, got, want Series, query, issue string) { - t.Helper() +func checkSeriesResult(got, want Series) error { if got.Status != want.Status { - t.Fatalf("query %s. Result Series status mismatch %q - %q. %s", query, want.Status, got.Status, issue) + return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status) } wantData := append([]map[string]string(nil), want.Data...) for _, r := range got.Data { wantData = removeIfFoundSeries(r, wantData) } if len(wantData) > 0 { - if issue != "" { - issue = "Regression in " + issue - } - t.Fatalf("query %s. Result Series %+v not found in %+v.%s", query, wantData, got.Data, issue) + return fmt.Errorf("expected seria(s) %+v not found in %+v", wantData, got.Data) } + return nil } func removeIfFoundSeries(r map[string]string, contains []map[string]string) []map[string]string { @@ -385,25 +412,21 @@ func removeIfFoundSeries(r map[string]string, contains []map[string]string) []ma return contains } -func checkQueryResult(t *testing.T, got, want Query, query, issue string) { - t.Helper() +func checkQueryResult(got, want Query) error { if got.Status != want.Status { - t.Fatalf("query %s. Result Query status mismatch %q - %q. %s", query, want.Status, got.Status, issue) + return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status) } if got.Data.ResultType != want.Data.ResultType { - t.Fatalf("query %s. Result Query Data ResultType status mismatch %q - %q. %s", query, want.Data.ResultType, got.Data.ResultType, issue) + return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType) } - wantData := append([]QueryDataResult(nil), want.Data.Result...) for _, r := range got.Data.Result { wantData = removeIfFoundQueryData(r, wantData) } if len(wantData) > 0 { - if issue != "" { - issue = "Regression in " + issue - } - t.Fatalf("query %s. Result query %+v not found in %+v.%s", query, wantData, got.Data.Result, issue) + return fmt.Errorf("expected query result %+v not found in %+v", wantData, got.Data.Result) } + return nil } func removeIfFoundQueryData(r QueryDataResult, contains []QueryDataResult) []QueryDataResult { @@ -416,6 +439,33 @@ func removeIfFoundQueryData(r QueryDataResult, contains []QueryDataResult) []Que return contains } +func checkQueryRangeResult(got, want QueryRange) error { + if got.Status != want.Status { + return fmt.Errorf("status mismatch %q - %q", want.Status, got.Status) + } + if got.Data.ResultType != want.Data.ResultType { + return fmt.Errorf("result type mismatch %q - %q", want.Data.ResultType, got.Data.ResultType) + } + wantData := append([]QueryRangeDataResult(nil), want.Data.Result...) + for _, r := range got.Data.Result { + wantData = removeIfFoundQueryRangeData(r, wantData) + } + if len(wantData) > 0 { + return fmt.Errorf("expected query range result %+v not found in %+v", wantData, got.Data.Result) + } + return nil +} + +func removeIfFoundQueryRangeData(r QueryRangeDataResult, contains []QueryRangeDataResult) []QueryRangeDataResult { + for i, item := range contains { + if reflect.DeepEqual(r.Metric, item.Metric) && reflect.DeepEqual(r.Values, item.Values) { + contains[i] = contains[len(contains)-1] + return contains[:len(contains)-1] + } + } + return contains +} + type suite struct{ t *testing.T } func newSuite(t *testing.T) *suite { return &suite{t: t} } diff --git a/app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json b/app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json new file mode 100644 index 000000000..4e790bfe3 --- /dev/null +++ b/app/victoria-metrics/testdata/graphite/comparison-not-inf-not-nan.json @@ -0,0 +1,14 @@ +{ + "name": "comparison-not-inf-not-nan", + "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/150", + "data": [ + "not_nan_not_inf;item=x 1 {TIME_S-1m}", + "not_nan_not_inf;item=x 2 {TIME_S-2m}", + "not_nan_not_inf;item=y 1 {TIME_S-1m}", + "not_nan_not_inf;item=y 2 {TIME_S-2m}"], + "query": ["/api/v1/query_range?query=1/(not_nan_not_inf-1)!=inf!=nan&start={TIME_S-2m}&end={TIME_S}&step=60"], + "result_query_range": { + "status":"success", + "data":{"resultType":"matrix", + "result":[{"metric":{"item":"x"},"values":[["{TIME_S-2m}","1"]]},{"metric":{"item":"y"},"values":[["{TIME_S-2m}","1"]]}]}} +} diff --git a/app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json b/app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json new file mode 100644 index 000000000..9c201743a --- /dev/null +++ b/app/victoria-metrics/testdata/graphite/not-nan-as-missing-data.json @@ -0,0 +1,14 @@ +{ + "name": "not-nan-as-missing-data", + "issue": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/153", + "data": [ + "not_nan_as_missing_data;item=x 1 {TIME_S-1m}", + "not_nan_as_missing_data;item=x 2 {TIME_S-2m}", + "not_nan_as_missing_data;item=y 3 {TIME_S-1m}", + "not_nan_as_missing_data;item=y 4 {TIME_S-2m}"], + "query": ["/api/v1/query_range?query=not_nan_as_missing_data>1&start={TIME_S-2m}&end={TIME_S}&step=60"], + "result_query_range": { + "status":"success", + "data":{"resultType":"matrix", + "result":[{"metric":{"__name__":"not_nan_as_missing_data","item":"x"},"values":[["{TIME_S-2m}","2"]]},{"metric":{"__name__":"not_nan_as_missing_data","item":"y"},"values":[["{TIME_S-2m}","4"]]}]}} +}