VictoriaMetrics/lib/storage/partition_test.go

160 lines
4.5 KiB
Go
Raw Normal View History

2019-05-22 23:16:55 +02:00
package storage
import (
"math/rand"
"reflect"
"testing"
)
func TestPartitionMaxOutPartRows(t *testing.T) {
pt := &partition{
bigPartsPath: ".",
}
n := pt.maxOutPartRows()
if n < 1e3 {
t.Fatalf("too small number of rows can be created in the current directory: %d", n)
}
}
func TestAppendPartsToMerge(t *testing.T) {
testAppendPartsToMerge(t, 2, []int{}, nil)
testAppendPartsToMerge(t, 2, []int{123}, nil)
testAppendPartsToMerge(t, 2, []int{4, 2}, nil)
testAppendPartsToMerge(t, 2, []int{128, 64, 32, 16, 8, 4, 2, 1}, nil)
testAppendPartsToMerge(t, 4, []int{128, 64, 32, 10, 9, 7, 2, 1}, []int{2, 7, 9, 10})
testAppendPartsToMerge(t, 2, []int{128, 64, 32, 16, 8, 4, 2, 2}, []int{2, 2})
testAppendPartsToMerge(t, 4, []int{128, 64, 32, 16, 8, 4, 2, 2}, []int{2, 2, 4, 8})
testAppendPartsToMerge(t, 2, []int{1, 1}, []int{1, 1})
testAppendPartsToMerge(t, 2, []int{2, 2, 2}, []int{2, 2})
testAppendPartsToMerge(t, 2, []int{4, 2, 4}, []int{4, 4})
testAppendPartsToMerge(t, 2, []int{1, 3, 7, 2}, nil)
testAppendPartsToMerge(t, 3, []int{1, 3, 7, 2}, []int{1, 2, 3})
testAppendPartsToMerge(t, 4, []int{1, 3, 7, 2}, []int{1, 2, 3})
testAppendPartsToMerge(t, 3, []int{11, 1, 10, 100, 10}, []int{10, 10, 11})
}
func TestAppendPartsToMergeManyParts(t *testing.T) {
// Verify that big number of parts are merged into minimal number of parts
// using minimum merges.
var a []int
maxOutPartRows := uint64(0)
for i := 0; i < 1024; i++ {
n := int(rand.NormFloat64() * 1e9)
if n < 0 {
n = -n
}
n++
maxOutPartRows += uint64(n)
a = append(a, n)
}
pws := newTestPartWrappersForRowsCount(a)
iterationsCount := 0
rowsMerged := uint64(0)
for {
pms := appendPartsToMerge(nil, pws, defaultPartsToMerge, maxOutPartRows)
if len(pms) == 0 {
break
}
m := make(map[*partWrapper]bool)
for _, pw := range pms {
m[pw] = true
}
var pwsNew []*partWrapper
rowsCount := uint64(0)
for _, pw := range pws {
if m[pw] {
rowsCount += pw.p.ph.RowsCount
} else {
pwsNew = append(pwsNew, pw)
}
}
pw := &partWrapper{
p: &part{
ph: partHeader{
RowsCount: rowsCount,
},
},
}
rowsMerged += rowsCount
pwsNew = append(pwsNew, pw)
pws = pwsNew
iterationsCount++
}
rowsCount := newTestRowsCountFromPartWrappers(pws)
rowsTotal := uint64(0)
for _, rc := range rowsCount {
rowsTotal += uint64(rc)
}
overhead := float64(rowsMerged) / float64(rowsTotal)
if overhead > 2.96 {
t.Fatalf("too big overhead; rowsCount=%d, iterationsCount=%d, rowsTotal=%d, rowsMerged=%d, overhead=%f",
rowsCount, iterationsCount, rowsTotal, rowsMerged, overhead)
}
if len(rowsCount) > 40 {
t.Fatalf("too many rowsCount %d; rowsCount=%d, iterationsCount=%d, rowsTotal=%d, rowsMerged=%d, overhead=%f",
len(rowsCount), rowsCount, iterationsCount, rowsTotal, rowsMerged, overhead)
}
}
func testAppendPartsToMerge(t *testing.T, maxPartsToMerge int, initialRowsCount, expectedRowsCount []int) {
t.Helper()
pws := newTestPartWrappersForRowsCount(initialRowsCount)
// Verify appending to nil.
pms := appendPartsToMerge(nil, pws, maxPartsToMerge, 1e9)
rowsCount := newTestRowsCountFromPartWrappers(pms)
if !reflect.DeepEqual(rowsCount, expectedRowsCount) {
t.Fatalf("unexpected rowsCount for maxPartsToMerge=%d, initialRowsCount=%d; got\n%d; want\n%d",
maxPartsToMerge, initialRowsCount, rowsCount, expectedRowsCount)
}
// Verify appending to prefix
prefix := []*partWrapper{
&partWrapper{
p: &part{
ph: partHeader{
RowsCount: 1234,
},
},
},
&partWrapper{},
&partWrapper{},
}
pms = appendPartsToMerge(prefix, pws, maxPartsToMerge, 1e9)
if !reflect.DeepEqual(pms[:len(prefix)], prefix) {
t.Fatalf("unexpected prefix for maxPartsToMerge=%d, initialRowsCount=%d; got\n%+v; want\n%+v",
maxPartsToMerge, initialRowsCount, pms[:len(prefix)], prefix)
}
rowsCount = newTestRowsCountFromPartWrappers(pms[len(prefix):])
if !reflect.DeepEqual(rowsCount, expectedRowsCount) {
t.Fatalf("unexpected prefixed rowsCount for maxPartsToMerge=%d, initialRowsCount=%d; got\n%d; want\n%d",
maxPartsToMerge, initialRowsCount, rowsCount, expectedRowsCount)
}
}
func newTestRowsCountFromPartWrappers(pws []*partWrapper) []int {
var rowsCount []int
for _, pw := range pws {
rowsCount = append(rowsCount, int(pw.p.ph.RowsCount))
}
return rowsCount
}
func newTestPartWrappersForRowsCount(rowsCount []int) []*partWrapper {
var pws []*partWrapper
for _, rc := range rowsCount {
pw := &partWrapper{
p: &part{
ph: partHeader{
RowsCount: uint64(rc),
},
},
}
pws = append(pws, pw)
}
return pws
}