mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-16 09:15:26 +01:00
92 lines
2.1 KiB
Go
92 lines
2.1 KiB
Go
|
package logstorage
|
||
|
|
||
|
import (
|
||
|
"math/rand"
|
||
|
"testing"
|
||
|
)
|
||
|
|
||
|
func TestAppendPartsToMergeManyParts(t *testing.T) {
|
||
|
// Verify that big number of parts are merged into minimal number of parts
|
||
|
// using minimum merges.
|
||
|
var sizes []uint64
|
||
|
maxOutSize := uint64(0)
|
||
|
r := rand.New(rand.NewSource(1))
|
||
|
for i := 0; i < 1024; i++ {
|
||
|
n := uint64(uint32(r.NormFloat64() * 1e9))
|
||
|
n++
|
||
|
maxOutSize += n
|
||
|
sizes = append(sizes, n)
|
||
|
}
|
||
|
pws := newTestPartWrappersForSizes(sizes)
|
||
|
|
||
|
iterationsCount := 0
|
||
|
sizeMergedTotal := uint64(0)
|
||
|
for {
|
||
|
pms := appendPartsToMerge(nil, pws, maxOutSize)
|
||
|
if len(pms) == 0 {
|
||
|
break
|
||
|
}
|
||
|
m := make(map[*partWrapper]bool)
|
||
|
for _, pw := range pms {
|
||
|
m[pw] = true
|
||
|
}
|
||
|
var pwsNew []*partWrapper
|
||
|
size := uint64(0)
|
||
|
for _, pw := range pws {
|
||
|
if m[pw] {
|
||
|
size += pw.p.ph.CompressedSizeBytes
|
||
|
} else {
|
||
|
pwsNew = append(pwsNew, pw)
|
||
|
}
|
||
|
}
|
||
|
pw := &partWrapper{
|
||
|
p: &part{
|
||
|
ph: partHeader{
|
||
|
CompressedSizeBytes: size,
|
||
|
},
|
||
|
},
|
||
|
}
|
||
|
sizeMergedTotal += size
|
||
|
pwsNew = append(pwsNew, pw)
|
||
|
pws = pwsNew
|
||
|
iterationsCount++
|
||
|
}
|
||
|
sizes = newTestSizesFromPartWrappers(pws)
|
||
|
sizeTotal := uint64(0)
|
||
|
for _, size := range sizes {
|
||
|
sizeTotal += uint64(size)
|
||
|
}
|
||
|
overhead := float64(sizeMergedTotal) / float64(sizeTotal)
|
||
|
if overhead > 2.1 {
|
||
|
t.Fatalf("too big overhead; sizes=%d, iterationsCount=%d, sizeTotal=%d, sizeMergedTotal=%d, overhead=%f",
|
||
|
sizes, iterationsCount, sizeTotal, sizeMergedTotal, overhead)
|
||
|
}
|
||
|
if len(sizes) > 18 {
|
||
|
t.Fatalf("too many sizes %d; sizes=%d, iterationsCount=%d, sizeTotal=%d, sizeMergedTotal=%d, overhead=%f",
|
||
|
len(sizes), sizes, iterationsCount, sizeTotal, sizeMergedTotal, overhead)
|
||
|
}
|
||
|
}
|
||
|
|
||
|
func newTestSizesFromPartWrappers(pws []*partWrapper) []uint64 {
|
||
|
var sizes []uint64
|
||
|
for _, pw := range pws {
|
||
|
sizes = append(sizes, pw.p.ph.CompressedSizeBytes)
|
||
|
}
|
||
|
return sizes
|
||
|
}
|
||
|
|
||
|
func newTestPartWrappersForSizes(sizes []uint64) []*partWrapper {
|
||
|
var pws []*partWrapper
|
||
|
for _, size := range sizes {
|
||
|
pw := &partWrapper{
|
||
|
p: &part{
|
||
|
ph: partHeader{
|
||
|
CompressedSizeBytes: size,
|
||
|
},
|
||
|
},
|
||
|
}
|
||
|
pws = append(pws, pw)
|
||
|
}
|
||
|
return pws
|
||
|
}
|