mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-25 20:00:06 +01:00
44b071296d
### Describe Your Changes Added an ability to query data across multiple tenants. See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1434 Currently, the following endpoints work with multi-tenancy: - /prometheus/api/v1/query - /prometheus/api/v1/query_range - /prometheus/api/v1/series - /prometheus/api/v1/labels - /prometheus/api/v1/label/<label_name>/values - /prometheus/api/v1/status/active_queries - /prometheus/api/v1/status/top_queries - /prometheus/api/v1/status/tsdb - /prometheus/api/v1/export - /prometheus/api/v1/export/csv - /vmui A note regarding VMUI: endpoints such as `active_queries` and `top_queries` have been updated to indicate whether query was a single-tenant or multi-tenant, but UI needs to be updated to display this info. cc: @Loori-R --------- Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com> Signed-off-by: f41gh7 <nik@victoriametrics.com> Co-authored-by: f41gh7 <nik@victoriametrics.com>
572 lines
14 KiB
Go
572 lines
14 KiB
Go
package promql
|
|
|
|
import (
|
|
"fmt"
|
|
"testing"
|
|
|
|
"github.com/VictoriaMetrics/metricsql"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/auth"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
|
|
)
|
|
|
|
func TestRollupResultCacheInitStop(t *testing.T) {
|
|
t.Run("inmemory", func(_ *testing.T) {
|
|
for i := 0; i < 5; i++ {
|
|
InitRollupResultCache("")
|
|
StopRollupResultCache()
|
|
}
|
|
})
|
|
t.Run("file-based", func(_ *testing.T) {
|
|
cacheFilePath := "test-rollup-result-cache"
|
|
for i := 0; i < 3; i++ {
|
|
InitRollupResultCache(cacheFilePath)
|
|
StopRollupResultCache()
|
|
}
|
|
fs.MustRemoveAll(cacheFilePath)
|
|
fs.MustRemoveAll(cacheFilePath + ".key.prefix")
|
|
})
|
|
}
|
|
|
|
func TestRollupResultCache(t *testing.T) {
|
|
InitRollupResultCache("")
|
|
defer StopRollupResultCache()
|
|
|
|
ResetRollupResultCache()
|
|
window := int64(456)
|
|
ec := &EvalConfig{
|
|
Start: 1000,
|
|
End: 2000,
|
|
Step: 200,
|
|
MaxPointsPerSeries: 1e4,
|
|
|
|
AuthTokens: []*auth.Token{{
|
|
AccountID: 333,
|
|
ProjectID: 843,
|
|
}},
|
|
|
|
MayCache: true,
|
|
}
|
|
me := &metricsql.MetricExpr{
|
|
LabelFilterss: [][]metricsql.LabelFilter{
|
|
{
|
|
{
|
|
Label: "aaa",
|
|
Value: "xxx",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
fe := &metricsql.FuncExpr{
|
|
Name: "foo",
|
|
Args: []metricsql.Expr{me},
|
|
}
|
|
ae := &metricsql.AggrFuncExpr{
|
|
Name: "foobar",
|
|
Args: []metricsql.Expr{fe},
|
|
}
|
|
|
|
// Try obtaining an empty value.
|
|
t.Run("empty", func(t *testing.T) {
|
|
tss, newStart := rollupResultCacheV.GetSeries(nil, ec, fe, window)
|
|
if newStart != ec.Start {
|
|
t.Fatalf("unexpected newStart; got %d; want %d", newStart, ec.Start)
|
|
}
|
|
if len(tss) != 0 {
|
|
t.Fatalf("got %d timeseries, while expecting zero", len(tss))
|
|
}
|
|
})
|
|
|
|
// Store timeseries overlapping with start
|
|
t.Run("start-overlap-no-ae", func(t *testing.T) {
|
|
ResetRollupResultCache()
|
|
tss := []*timeseries{
|
|
{
|
|
Timestamps: []int64{800, 1000, 1200},
|
|
Values: []float64{0, 1, 2},
|
|
},
|
|
}
|
|
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss)
|
|
tss, newStart := rollupResultCacheV.GetSeries(nil, ec, fe, window)
|
|
if newStart != 1400 {
|
|
t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1400)
|
|
}
|
|
tssExpected := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200},
|
|
Values: []float64{1, 2},
|
|
},
|
|
}
|
|
testTimeseriesEqual(t, tss, tssExpected)
|
|
})
|
|
t.Run("start-overlap-with-ae", func(t *testing.T) {
|
|
ResetRollupResultCache()
|
|
tss := []*timeseries{
|
|
{
|
|
Timestamps: []int64{800, 1000, 1200},
|
|
Values: []float64{0, 1, 2},
|
|
},
|
|
}
|
|
rollupResultCacheV.PutSeries(nil, ec, ae, window, tss)
|
|
tss, newStart := rollupResultCacheV.GetSeries(nil, ec, ae, window)
|
|
if newStart != 1400 {
|
|
t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1400)
|
|
}
|
|
tssExpected := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200},
|
|
Values: []float64{1, 2},
|
|
},
|
|
}
|
|
testTimeseriesEqual(t, tss, tssExpected)
|
|
})
|
|
|
|
// Store timeseries overlapping with end
|
|
t.Run("end-overlap", func(t *testing.T) {
|
|
ResetRollupResultCache()
|
|
tss := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1800, 2000, 2200, 2400},
|
|
Values: []float64{333, 0, 1, 2},
|
|
},
|
|
}
|
|
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss)
|
|
tss, newStart := rollupResultCacheV.GetSeries(nil, ec, fe, window)
|
|
if newStart != 1000 {
|
|
t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1000)
|
|
}
|
|
if len(tss) != 0 {
|
|
t.Fatalf("got %d timeseries, while expecting zero", len(tss))
|
|
}
|
|
})
|
|
|
|
// Store timeseries covered by [start ... end]
|
|
t.Run("full-cover", func(t *testing.T) {
|
|
ResetRollupResultCache()
|
|
tss := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1200, 1400, 1600},
|
|
Values: []float64{0, 1, 2},
|
|
},
|
|
}
|
|
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss)
|
|
tss, newStart := rollupResultCacheV.GetSeries(nil, ec, fe, window)
|
|
if newStart != 1000 {
|
|
t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1000)
|
|
}
|
|
if len(tss) != 0 {
|
|
t.Fatalf("got %d timeseries, while expecting zero", len(tss))
|
|
}
|
|
})
|
|
|
|
// Store timeseries below start
|
|
t.Run("before-start", func(t *testing.T) {
|
|
ResetRollupResultCache()
|
|
tss := []*timeseries{
|
|
{
|
|
Timestamps: []int64{200, 400, 600},
|
|
Values: []float64{0, 1, 2},
|
|
},
|
|
}
|
|
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss)
|
|
tss, newStart := rollupResultCacheV.GetSeries(nil, ec, fe, window)
|
|
if newStart != 1000 {
|
|
t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1000)
|
|
}
|
|
if len(tss) != 0 {
|
|
t.Fatalf("got %d timeseries, while expecting zero", len(tss))
|
|
}
|
|
})
|
|
|
|
// Store timeseries after end
|
|
t.Run("after-end", func(t *testing.T) {
|
|
ResetRollupResultCache()
|
|
tss := []*timeseries{
|
|
{
|
|
Timestamps: []int64{2200, 2400, 2600},
|
|
Values: []float64{0, 1, 2},
|
|
},
|
|
}
|
|
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss)
|
|
tss, newStart := rollupResultCacheV.GetSeries(nil, ec, fe, window)
|
|
if newStart != 1000 {
|
|
t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1000)
|
|
}
|
|
if len(tss) != 0 {
|
|
t.Fatalf("got %d timeseries, while expecting zero", len(tss))
|
|
}
|
|
})
|
|
|
|
// Store timeseries bigger than the interval [start ... end]
|
|
t.Run("bigger-than-start-end", func(t *testing.T) {
|
|
ResetRollupResultCache()
|
|
tss := []*timeseries{
|
|
{
|
|
Timestamps: []int64{800, 1000, 1200, 1400, 1600, 1800, 2000, 2200},
|
|
Values: []float64{0, 1, 2, 3, 4, 5, 6, 7},
|
|
},
|
|
}
|
|
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss)
|
|
tss, newStart := rollupResultCacheV.GetSeries(nil, ec, fe, window)
|
|
if newStart != 2200 {
|
|
t.Fatalf("unexpected newStart; got %d; want %d", newStart, 2200)
|
|
}
|
|
tssExpected := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200, 1400, 1600, 1800, 2000},
|
|
Values: []float64{1, 2, 3, 4, 5, 6},
|
|
},
|
|
}
|
|
testTimeseriesEqual(t, tss, tssExpected)
|
|
})
|
|
|
|
// Store timeseries matching the interval [start ... end]
|
|
t.Run("start-end-match", func(t *testing.T) {
|
|
ResetRollupResultCache()
|
|
tss := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200, 1400, 1600, 1800, 2000},
|
|
Values: []float64{1, 2, 3, 4, 5, 6},
|
|
},
|
|
}
|
|
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss)
|
|
tss, newStart := rollupResultCacheV.GetSeries(nil, ec, fe, window)
|
|
if newStart != 2200 {
|
|
t.Fatalf("unexpected newStart; got %d; want %d", newStart, 2200)
|
|
}
|
|
tssExpected := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200, 1400, 1600, 1800, 2000},
|
|
Values: []float64{1, 2, 3, 4, 5, 6},
|
|
},
|
|
}
|
|
testTimeseriesEqual(t, tss, tssExpected)
|
|
})
|
|
|
|
// Store big timeseries, so their marshaled size exceeds 64Kb.
|
|
t.Run("big-timeseries", func(t *testing.T) {
|
|
ResetRollupResultCache()
|
|
var tss []*timeseries
|
|
for i := 0; i < 1000; i++ {
|
|
ts := ×eries{
|
|
Timestamps: []int64{1000, 1200, 1400, 1600, 1800, 2000},
|
|
Values: []float64{1, 2, 3, 4, 5, 6},
|
|
}
|
|
ts.MetricName.MetricGroup = []byte(fmt.Sprintf("metric %d", i))
|
|
tss = append(tss, ts)
|
|
}
|
|
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss)
|
|
tssResult, newStart := rollupResultCacheV.GetSeries(nil, ec, fe, window)
|
|
if newStart != 2200 {
|
|
t.Fatalf("unexpected newStart; got %d; want %d", newStart, 2200)
|
|
}
|
|
testTimeseriesEqual(t, tssResult, tss)
|
|
})
|
|
|
|
// Store series with identical naming (they shouldn't be stored)
|
|
t.Run("duplicate-series", func(t *testing.T) {
|
|
ResetRollupResultCache()
|
|
tss := []*timeseries{
|
|
{
|
|
Timestamps: []int64{800, 1000, 1200},
|
|
Values: []float64{0, 1, 2},
|
|
},
|
|
{
|
|
Timestamps: []int64{800, 1000, 1200},
|
|
Values: []float64{0, 1, 2},
|
|
},
|
|
}
|
|
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss)
|
|
tssResult, newStart := rollupResultCacheV.GetSeries(nil, ec, fe, window)
|
|
if newStart != ec.Start {
|
|
t.Fatalf("unexpected newStart; got %d; want %d", newStart, ec.Start)
|
|
}
|
|
if len(tssResult) != 0 {
|
|
t.Fatalf("unexpected non-empty series returned")
|
|
}
|
|
})
|
|
|
|
// Store multiple time series
|
|
t.Run("multi-timeseries", func(t *testing.T) {
|
|
ResetRollupResultCache()
|
|
tss1 := []*timeseries{
|
|
{
|
|
Timestamps: []int64{800, 1000, 1200},
|
|
Values: []float64{0, 1, 2},
|
|
},
|
|
}
|
|
tss2 := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1800, 2000, 2200, 2400},
|
|
Values: []float64{333, 0, 1, 2},
|
|
},
|
|
}
|
|
tss3 := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1200, 1400, 1600},
|
|
Values: []float64{0, 1, 2},
|
|
},
|
|
}
|
|
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss1)
|
|
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss2)
|
|
rollupResultCacheV.PutSeries(nil, ec, fe, window, tss3)
|
|
tss, newStart := rollupResultCacheV.GetSeries(nil, ec, fe, window)
|
|
if newStart != 1400 {
|
|
t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1400)
|
|
}
|
|
tssExpected := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200},
|
|
Values: []float64{1, 2},
|
|
},
|
|
}
|
|
testTimeseriesEqual(t, tss, tssExpected)
|
|
})
|
|
t.Run("multi-tenant cache can be retrieved", func(t *testing.T) {
|
|
ResetRollupResultCache()
|
|
tssGolden := []*timeseries{
|
|
{
|
|
MetricName: storage.MetricName{
|
|
AccountID: 0,
|
|
ProjectID: 0,
|
|
},
|
|
Timestamps: []int64{800, 1000, 1200},
|
|
Values: []float64{0, 1, 2},
|
|
},
|
|
{
|
|
MetricName: storage.MetricName{
|
|
AccountID: 0,
|
|
ProjectID: 1,
|
|
},
|
|
Timestamps: []int64{800, 1000, 1200},
|
|
Values: []float64{0, 1, 2},
|
|
},
|
|
{
|
|
MetricName: storage.MetricName{
|
|
AccountID: 1,
|
|
ProjectID: 1,
|
|
},
|
|
Timestamps: []int64{800, 1000, 1200},
|
|
Values: []float64{0, 1, 2},
|
|
},
|
|
}
|
|
ecL := copyEvalConfig(ec)
|
|
ecL.Start = 800
|
|
ecL.AuthTokens = []*auth.Token{
|
|
{
|
|
AccountID: 0,
|
|
ProjectID: 0,
|
|
},
|
|
{
|
|
AccountID: 0,
|
|
ProjectID: 1,
|
|
},
|
|
{
|
|
AccountID: 1,
|
|
ProjectID: 1,
|
|
},
|
|
}
|
|
ecL.IsMultiTenant = true
|
|
rollupResultCacheV.PutSeries(nil, ecL, fe, window, tssGolden)
|
|
|
|
tss, newStart := rollupResultCacheV.GetSeries(nil, ecL, fe, window)
|
|
if newStart != 1400 {
|
|
t.Fatalf("unexpected newStart; got %d; want %d", newStart, 1400)
|
|
}
|
|
|
|
testTimeseriesEqual(t, tss, tssGolden)
|
|
})
|
|
}
|
|
|
|
func TestMergeSeries(t *testing.T) {
|
|
ec := &EvalConfig{
|
|
Start: 1000,
|
|
End: 2000,
|
|
Step: 200,
|
|
MaxPointsPerSeries: 1e4,
|
|
}
|
|
bStart := int64(1400)
|
|
|
|
t.Run("bStart=ec.Start", func(t *testing.T) {
|
|
a := []*timeseries{}
|
|
b := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200, 1400, 1600, 1800, 2000},
|
|
Values: []float64{1, 2, 3, 4, 5, 6},
|
|
},
|
|
}
|
|
tss, ok := mergeSeries(nil, a, b, 1000, ec)
|
|
if !ok {
|
|
t.Fatalf("unexpected failure to merge series")
|
|
}
|
|
tssExpected := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200, 1400, 1600, 1800, 2000},
|
|
Values: []float64{1, 2, 3, 4, 5, 6},
|
|
},
|
|
}
|
|
testTimeseriesEqual(t, tss, tssExpected)
|
|
})
|
|
t.Run("a-empty", func(t *testing.T) {
|
|
a := []*timeseries{}
|
|
b := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1400, 1600, 1800, 2000},
|
|
Values: []float64{3, 4, 5, 6},
|
|
},
|
|
}
|
|
tss, ok := mergeSeries(nil, a, b, bStart, ec)
|
|
if !ok {
|
|
t.Fatalf("unexpected failure to merge series")
|
|
}
|
|
tssExpected := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200, 1400, 1600, 1800, 2000},
|
|
Values: []float64{nan, nan, 3, 4, 5, 6},
|
|
},
|
|
}
|
|
testTimeseriesEqual(t, tss, tssExpected)
|
|
})
|
|
t.Run("b-empty", func(t *testing.T) {
|
|
a := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200},
|
|
Values: []float64{2, 1},
|
|
},
|
|
}
|
|
b := []*timeseries{}
|
|
tss, ok := mergeSeries(nil, a, b, bStart, ec)
|
|
if !ok {
|
|
t.Fatalf("unexpected failure to merge series")
|
|
}
|
|
tssExpected := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200, 1400, 1600, 1800, 2000},
|
|
Values: []float64{2, 1, nan, nan, nan, nan},
|
|
},
|
|
}
|
|
testTimeseriesEqual(t, tss, tssExpected)
|
|
})
|
|
t.Run("non-empty", func(t *testing.T) {
|
|
a := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200},
|
|
Values: []float64{2, 1},
|
|
},
|
|
}
|
|
b := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1400, 1600, 1800, 2000},
|
|
Values: []float64{3, 4, 5, 6},
|
|
},
|
|
}
|
|
tss, ok := mergeSeries(nil, a, b, bStart, ec)
|
|
if !ok {
|
|
t.Fatalf("unexpected failure to merge series")
|
|
}
|
|
tssExpected := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200, 1400, 1600, 1800, 2000},
|
|
Values: []float64{2, 1, 3, 4, 5, 6},
|
|
},
|
|
}
|
|
testTimeseriesEqual(t, tss, tssExpected)
|
|
})
|
|
t.Run("non-empty-distinct-metric-names", func(t *testing.T) {
|
|
a := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200},
|
|
Values: []float64{2, 1},
|
|
},
|
|
}
|
|
a[0].MetricName.MetricGroup = []byte("bar")
|
|
b := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1400, 1600, 1800, 2000},
|
|
Values: []float64{3, 4, 5, 6},
|
|
},
|
|
}
|
|
b[0].MetricName.MetricGroup = []byte("foo")
|
|
tss, ok := mergeSeries(nil, a, b, bStart, ec)
|
|
if !ok {
|
|
t.Fatalf("unexpected failure to merge series")
|
|
}
|
|
tssExpected := []*timeseries{
|
|
{
|
|
MetricName: storage.MetricName{
|
|
MetricGroup: []byte("foo"),
|
|
},
|
|
Timestamps: []int64{1000, 1200, 1400, 1600, 1800, 2000},
|
|
Values: []float64{nan, nan, 3, 4, 5, 6},
|
|
},
|
|
{
|
|
MetricName: storage.MetricName{
|
|
MetricGroup: []byte("bar"),
|
|
},
|
|
Timestamps: []int64{1000, 1200, 1400, 1600, 1800, 2000},
|
|
Values: []float64{2, 1, nan, nan, nan, nan},
|
|
},
|
|
}
|
|
testTimeseriesEqual(t, tss, tssExpected)
|
|
})
|
|
t.Run("duplicate-series-a", func(t *testing.T) {
|
|
a := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200},
|
|
Values: []float64{2, 1},
|
|
},
|
|
{
|
|
Timestamps: []int64{1000, 1200},
|
|
Values: []float64{3, 3},
|
|
},
|
|
}
|
|
b := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1400, 1600, 1800, 2000},
|
|
Values: []float64{3, 4, 5, 6},
|
|
},
|
|
}
|
|
tss, ok := mergeSeries(nil, a, b, bStart, ec)
|
|
if ok {
|
|
t.Fatalf("expecting failre to merge series")
|
|
}
|
|
testTimeseriesEqual(t, tss, nil)
|
|
})
|
|
t.Run("duplicate-series-b", func(t *testing.T) {
|
|
a := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1000, 1200},
|
|
Values: []float64{1, 2},
|
|
},
|
|
}
|
|
b := []*timeseries{
|
|
{
|
|
Timestamps: []int64{1400, 1600, 1800, 2000},
|
|
Values: []float64{3, 4, 5, 6},
|
|
},
|
|
{
|
|
Timestamps: []int64{1400, 1600, 1800, 2000},
|
|
Values: []float64{13, 14, 15, 16},
|
|
},
|
|
}
|
|
tss, ok := mergeSeries(nil, a, b, bStart, ec)
|
|
if ok {
|
|
t.Fatalf("expecting failre to merge series")
|
|
}
|
|
testTimeseriesEqual(t, tss, nil)
|
|
})
|
|
}
|
|
|
|
func testTimeseriesEqual(t *testing.T, tss, tssExpected []*timeseries) {
|
|
t.Helper()
|
|
if len(tss) != len(tssExpected) {
|
|
t.Fatalf(`unexpected timeseries count; got %d; want %d`, len(tss), len(tssExpected))
|
|
}
|
|
for i, ts := range tss {
|
|
tsExpected := tssExpected[i]
|
|
testMetricNamesEqual(t, &ts.MetricName, &tsExpected.MetricName, true, i)
|
|
testRowsEqual(t, ts.Values, ts.Timestamps, tsExpected.Values, tsExpected.Timestamps)
|
|
}
|
|
}
|