mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-15 00:13:30 +01:00
160 lines
4.5 KiB
Go
160 lines
4.5 KiB
Go
|
package storage
|
||
|
|
||
|
import (
|
||
|
"math/rand"
|
||
|
"reflect"
|
||
|
"testing"
|
||
|
)
|
||
|
|
||
|
func TestPartitionMaxOutPartRows(t *testing.T) {
|
||
|
pt := &partition{
|
||
|
bigPartsPath: ".",
|
||
|
}
|
||
|
n := pt.maxOutPartRows()
|
||
|
if n < 1e3 {
|
||
|
t.Fatalf("too small number of rows can be created in the current directory: %d", n)
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func TestAppendPartsToMerge(t *testing.T) {
|
||
|
testAppendPartsToMerge(t, 2, []int{}, nil)
|
||
|
testAppendPartsToMerge(t, 2, []int{123}, nil)
|
||
|
testAppendPartsToMerge(t, 2, []int{4, 2}, nil)
|
||
|
testAppendPartsToMerge(t, 2, []int{128, 64, 32, 16, 8, 4, 2, 1}, nil)
|
||
|
testAppendPartsToMerge(t, 4, []int{128, 64, 32, 10, 9, 7, 2, 1}, []int{2, 7, 9, 10})
|
||
|
testAppendPartsToMerge(t, 2, []int{128, 64, 32, 16, 8, 4, 2, 2}, []int{2, 2})
|
||
|
testAppendPartsToMerge(t, 4, []int{128, 64, 32, 16, 8, 4, 2, 2}, []int{2, 2, 4, 8})
|
||
|
testAppendPartsToMerge(t, 2, []int{1, 1}, []int{1, 1})
|
||
|
testAppendPartsToMerge(t, 2, []int{2, 2, 2}, []int{2, 2})
|
||
|
testAppendPartsToMerge(t, 2, []int{4, 2, 4}, []int{4, 4})
|
||
|
testAppendPartsToMerge(t, 2, []int{1, 3, 7, 2}, nil)
|
||
|
testAppendPartsToMerge(t, 3, []int{1, 3, 7, 2}, []int{1, 2, 3})
|
||
|
testAppendPartsToMerge(t, 4, []int{1, 3, 7, 2}, []int{1, 2, 3})
|
||
|
testAppendPartsToMerge(t, 3, []int{11, 1, 10, 100, 10}, []int{10, 10, 11})
|
||
|
}
|
||
|
|
||
|
func TestAppendPartsToMergeManyParts(t *testing.T) {
|
||
|
// Verify that big number of parts are merged into minimal number of parts
|
||
|
// using minimum merges.
|
||
|
var a []int
|
||
|
maxOutPartRows := uint64(0)
|
||
|
for i := 0; i < 1024; i++ {
|
||
|
n := int(rand.NormFloat64() * 1e9)
|
||
|
if n < 0 {
|
||
|
n = -n
|
||
|
}
|
||
|
n++
|
||
|
maxOutPartRows += uint64(n)
|
||
|
a = append(a, n)
|
||
|
}
|
||
|
pws := newTestPartWrappersForRowsCount(a)
|
||
|
|
||
|
iterationsCount := 0
|
||
|
rowsMerged := uint64(0)
|
||
|
for {
|
||
|
pms := appendPartsToMerge(nil, pws, defaultPartsToMerge, maxOutPartRows)
|
||
|
if len(pms) == 0 {
|
||
|
break
|
||
|
}
|
||
|
m := make(map[*partWrapper]bool)
|
||
|
for _, pw := range pms {
|
||
|
m[pw] = true
|
||
|
}
|
||
|
var pwsNew []*partWrapper
|
||
|
rowsCount := uint64(0)
|
||
|
for _, pw := range pws {
|
||
|
if m[pw] {
|
||
|
rowsCount += pw.p.ph.RowsCount
|
||
|
} else {
|
||
|
pwsNew = append(pwsNew, pw)
|
||
|
}
|
||
|
}
|
||
|
pw := &partWrapper{
|
||
|
p: &part{
|
||
|
ph: partHeader{
|
||
|
RowsCount: rowsCount,
|
||
|
},
|
||
|
},
|
||
|
}
|
||
|
rowsMerged += rowsCount
|
||
|
pwsNew = append(pwsNew, pw)
|
||
|
pws = pwsNew
|
||
|
iterationsCount++
|
||
|
}
|
||
|
rowsCount := newTestRowsCountFromPartWrappers(pws)
|
||
|
rowsTotal := uint64(0)
|
||
|
for _, rc := range rowsCount {
|
||
|
rowsTotal += uint64(rc)
|
||
|
}
|
||
|
overhead := float64(rowsMerged) / float64(rowsTotal)
|
||
|
if overhead > 2.96 {
|
||
|
t.Fatalf("too big overhead; rowsCount=%d, iterationsCount=%d, rowsTotal=%d, rowsMerged=%d, overhead=%f",
|
||
|
rowsCount, iterationsCount, rowsTotal, rowsMerged, overhead)
|
||
|
}
|
||
|
if len(rowsCount) > 40 {
|
||
|
t.Fatalf("too many rowsCount %d; rowsCount=%d, iterationsCount=%d, rowsTotal=%d, rowsMerged=%d, overhead=%f",
|
||
|
len(rowsCount), rowsCount, iterationsCount, rowsTotal, rowsMerged, overhead)
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func testAppendPartsToMerge(t *testing.T, maxPartsToMerge int, initialRowsCount, expectedRowsCount []int) {
|
||
|
t.Helper()
|
||
|
|
||
|
pws := newTestPartWrappersForRowsCount(initialRowsCount)
|
||
|
|
||
|
// Verify appending to nil.
|
||
|
pms := appendPartsToMerge(nil, pws, maxPartsToMerge, 1e9)
|
||
|
rowsCount := newTestRowsCountFromPartWrappers(pms)
|
||
|
if !reflect.DeepEqual(rowsCount, expectedRowsCount) {
|
||
|
t.Fatalf("unexpected rowsCount for maxPartsToMerge=%d, initialRowsCount=%d; got\n%d; want\n%d",
|
||
|
maxPartsToMerge, initialRowsCount, rowsCount, expectedRowsCount)
|
||
|
}
|
||
|
|
||
|
// Verify appending to prefix
|
||
|
prefix := []*partWrapper{
|
||
|
&partWrapper{
|
||
|
p: &part{
|
||
|
ph: partHeader{
|
||
|
RowsCount: 1234,
|
||
|
},
|
||
|
},
|
||
|
},
|
||
|
&partWrapper{},
|
||
|
&partWrapper{},
|
||
|
}
|
||
|
pms = appendPartsToMerge(prefix, pws, maxPartsToMerge, 1e9)
|
||
|
if !reflect.DeepEqual(pms[:len(prefix)], prefix) {
|
||
|
t.Fatalf("unexpected prefix for maxPartsToMerge=%d, initialRowsCount=%d; got\n%+v; want\n%+v",
|
||
|
maxPartsToMerge, initialRowsCount, pms[:len(prefix)], prefix)
|
||
|
}
|
||
|
|
||
|
rowsCount = newTestRowsCountFromPartWrappers(pms[len(prefix):])
|
||
|
if !reflect.DeepEqual(rowsCount, expectedRowsCount) {
|
||
|
t.Fatalf("unexpected prefixed rowsCount for maxPartsToMerge=%d, initialRowsCount=%d; got\n%d; want\n%d",
|
||
|
maxPartsToMerge, initialRowsCount, rowsCount, expectedRowsCount)
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func newTestRowsCountFromPartWrappers(pws []*partWrapper) []int {
|
||
|
var rowsCount []int
|
||
|
for _, pw := range pws {
|
||
|
rowsCount = append(rowsCount, int(pw.p.ph.RowsCount))
|
||
|
}
|
||
|
return rowsCount
|
||
|
}
|
||
|
|
||
|
func newTestPartWrappersForRowsCount(rowsCount []int) []*partWrapper {
|
||
|
var pws []*partWrapper
|
||
|
for _, rc := range rowsCount {
|
||
|
pw := &partWrapper{
|
||
|
p: &part{
|
||
|
ph: partHeader{
|
||
|
RowsCount: uint64(rc),
|
||
|
},
|
||
|
},
|
||
|
}
|
||
|
pws = append(pws, pw)
|
||
|
}
|
||
|
return pws
|
||
|
}
|