mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-30 07:40:06 +01:00
1f28b46ae9
This reverts the following commits: -e0e16a2d36-2ce02a7fe6The reason for revert: the updated logic breaks assumptions made when fixing https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2698 . For example, if a time series stop receiving new samples during the first day after the indexdb rotation, there are chances that the time series won't be registered in the new indexdb. This is OK until the next indexdb rotation, since the time series is registered in the previous indexdb, so it can be found during queries. But the time series will become invisible for search after the next indexdb rotation, while its data is still there. There is also incompletely solved issue with the increased CPU and disk IO resource usage just after the indexdb rotation. There was an attempt to fix it, but it didn't fix it in full, while introducing the issue mentioned above. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1401 TODO: to find out the solution, which simultaneously solves the following issues: - increased memory usage for setups high churn rate and long retention (e.g. what the reverted commit does) - increased CPU and disk IO usage during indexdb rotation ( https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1401 ) - https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2698 Possible solution - to create the new indexdb in one hour before the indexdb rotation and to gradually pre-populate it with the needed index data during the last hour before indexdb rotation. Then the new indexdb will contain all the needed data just after the rotation, so it won't trigger increased CPU and disk IO.
343 lines
9.6 KiB
Go
343 lines
9.6 KiB
Go
package storage
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"regexp"
|
|
"strconv"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func BenchmarkRegexpFilterMatch(b *testing.B) {
|
|
b.ReportAllocs()
|
|
b.RunParallel(func(pb *testing.PB) {
|
|
re := regexp.MustCompile(`.*foo-bar-baz.*`)
|
|
b := []byte("fdsffd foo-bar-baz assd fdsfad dasf dsa")
|
|
for pb.Next() {
|
|
if !re.Match(b) {
|
|
panic("BUG: regexp must match!")
|
|
}
|
|
b[0]++
|
|
}
|
|
})
|
|
}
|
|
|
|
func BenchmarkRegexpFilterMismatch(b *testing.B) {
|
|
b.ReportAllocs()
|
|
b.RunParallel(func(pb *testing.PB) {
|
|
re := regexp.MustCompile(`.*foo-bar-baz.*`)
|
|
b := []byte("fdsffd foo-bar sfddsf assd nmn,mfdsdsakj")
|
|
for pb.Next() {
|
|
if re.Match(b) {
|
|
panic("BUG: regexp mustn't match!")
|
|
}
|
|
b[0]++
|
|
}
|
|
})
|
|
}
|
|
|
|
func BenchmarkIndexDBAddTSIDs(b *testing.B) {
|
|
const recordsPerLoop = 1e3
|
|
|
|
s := newTestStorage()
|
|
defer stopTestStorage(s)
|
|
|
|
dbName := nextIndexDBTableName()
|
|
var isReadOnly uint32
|
|
db := mustOpenIndexDB(dbName, s, 0, &isReadOnly)
|
|
defer func() {
|
|
db.MustClose()
|
|
if err := os.RemoveAll(dbName); err != nil {
|
|
b.Fatalf("cannot remove indexDB: %s", err)
|
|
}
|
|
}()
|
|
|
|
b.ReportAllocs()
|
|
b.SetBytes(recordsPerLoop)
|
|
b.ResetTimer()
|
|
b.RunParallel(func(pb *testing.PB) {
|
|
var mn MetricName
|
|
var tsid TSID
|
|
|
|
// The most common tags.
|
|
mn.Tags = []Tag{
|
|
{
|
|
Key: []byte("job"),
|
|
},
|
|
{
|
|
Key: []byte("instance"),
|
|
},
|
|
}
|
|
|
|
startOffset := 0
|
|
for pb.Next() {
|
|
benchmarkIndexDBAddTSIDs(db, &tsid, &mn, startOffset, recordsPerLoop)
|
|
startOffset += recordsPerLoop
|
|
}
|
|
})
|
|
b.StopTimer()
|
|
}
|
|
|
|
func benchmarkIndexDBAddTSIDs(db *indexDB, tsid *TSID, mn *MetricName, startOffset, recordsPerLoop int) {
|
|
var metricName []byte
|
|
var metricNameRaw []byte
|
|
is := db.getIndexSearch(noDeadline)
|
|
defer db.putIndexSearch(is)
|
|
for i := 0; i < recordsPerLoop; i++ {
|
|
mn.MetricGroup = strconv.AppendUint(mn.MetricGroup[:0], uint64(i+startOffset), 10)
|
|
for j := range mn.Tags {
|
|
mn.Tags[j].Value = strconv.AppendUint(mn.Tags[j].Value[:0], uint64(i*j), 16)
|
|
}
|
|
mn.sortTags()
|
|
metricName = mn.Marshal(metricName[:0])
|
|
metricNameRaw = mn.marshalRaw(metricNameRaw[:0])
|
|
if err := is.GetOrCreateTSIDByName(tsid, metricName, metricNameRaw, 0); err != nil {
|
|
panic(fmt.Errorf("cannot insert record: %w", err))
|
|
}
|
|
}
|
|
}
|
|
|
|
func BenchmarkHeadPostingForMatchers(b *testing.B) {
|
|
// This benchmark is equivalent to https://github.com/prometheus/prometheus/blob/23c0299d85bfeb5d9b59e994861553a25ca578e5/tsdb/head_bench_test.go#L52
|
|
// See https://www.robustperception.io/evaluating-performance-and-correctness for more details.
|
|
s := newTestStorage()
|
|
defer stopTestStorage(s)
|
|
|
|
dbName := nextIndexDBTableName()
|
|
var isReadOnly uint32
|
|
db := mustOpenIndexDB(dbName, s, 0, &isReadOnly)
|
|
defer func() {
|
|
db.MustClose()
|
|
if err := os.RemoveAll(dbName); err != nil {
|
|
b.Fatalf("cannot remove indexDB: %s", err)
|
|
}
|
|
}()
|
|
|
|
// Fill the db with data as in https://github.com/prometheus/prometheus/blob/23c0299d85bfeb5d9b59e994861553a25ca578e5/tsdb/head_bench_test.go#L66
|
|
var mn MetricName
|
|
var metricName []byte
|
|
var metricNameRaw []byte
|
|
var tsid TSID
|
|
is := db.getIndexSearch(noDeadline)
|
|
defer db.putIndexSearch(is)
|
|
addSeries := func(kvs ...string) {
|
|
mn.Reset()
|
|
for i := 0; i < len(kvs); i += 2 {
|
|
mn.AddTag(kvs[i], kvs[i+1])
|
|
}
|
|
mn.sortTags()
|
|
metricName = mn.Marshal(metricName[:0])
|
|
metricNameRaw = mn.marshalRaw(metricNameRaw[:0])
|
|
if err := is.createTSIDByName(&tsid, metricName, metricNameRaw, 0); err != nil {
|
|
b.Fatalf("cannot insert record: %s", err)
|
|
}
|
|
}
|
|
for n := 0; n < 10; n++ {
|
|
ns := strconv.Itoa(n)
|
|
for i := 0; i < 100000; i++ {
|
|
ix := strconv.Itoa(i)
|
|
addSeries("i", ix, "n", ns, "j", "foo")
|
|
// Have some series that won't be matched, to properly test inverted matches.
|
|
addSeries("i", ix, "n", ns, "j", "bar")
|
|
addSeries("i", ix, "n", "0_"+ns, "j", "bar")
|
|
addSeries("i", ix, "n", "1_"+ns, "j", "bar")
|
|
addSeries("i", ix, "n", "2_"+ns, "j", "foo")
|
|
}
|
|
}
|
|
|
|
// Make sure all the items can be searched.
|
|
db.tb.DebugFlush()
|
|
b.ResetTimer()
|
|
|
|
benchSearch := func(b *testing.B, tfs *TagFilters, expectedMetricIDs int) {
|
|
is := db.getIndexSearch(noDeadline)
|
|
defer db.putIndexSearch(is)
|
|
tfss := []*TagFilters{tfs}
|
|
tr := TimeRange{
|
|
MinTimestamp: 0,
|
|
MaxTimestamp: timestampFromTime(time.Now()),
|
|
}
|
|
for i := 0; i < b.N; i++ {
|
|
metricIDs, err := is.searchMetricIDs(nil, tfss, tr, 2e9)
|
|
if err != nil {
|
|
b.Fatalf("unexpected error in searchMetricIDs: %s", err)
|
|
}
|
|
if len(metricIDs) != expectedMetricIDs {
|
|
b.Fatalf("unexpected metricIDs found; got %d; want %d", len(metricIDs), expectedMetricIDs)
|
|
}
|
|
}
|
|
}
|
|
addTagFilter := func(tfs *TagFilters, key, value string, isNegative, isRegexp bool) {
|
|
if err := tfs.Add([]byte(key), []byte(value), isNegative, isRegexp); err != nil {
|
|
b.Fatalf("cannot add tag filter %q=%q, isNegative=%v, isRegexp=%v", key, value, isNegative, isRegexp)
|
|
}
|
|
}
|
|
|
|
b.Run(`n="1"`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "n", "1", false, false)
|
|
benchSearch(b, tfs, 2e5)
|
|
})
|
|
b.Run(`n="1",j="foo"`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "n", "1", false, false)
|
|
addTagFilter(tfs, "j", "foo", false, false)
|
|
benchSearch(b, tfs, 1e5)
|
|
})
|
|
b.Run(`j="foo",n="1"`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "j", "foo", false, false)
|
|
addTagFilter(tfs, "n", "1", false, false)
|
|
benchSearch(b, tfs, 1e5)
|
|
})
|
|
b.Run(`n="1",j!="foo"`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "n", "1", false, false)
|
|
addTagFilter(tfs, "j", "foo", true, false)
|
|
benchSearch(b, tfs, 1e5)
|
|
})
|
|
b.Run(`i=~".*"`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "i", ".*", false, true)
|
|
benchSearch(b, tfs, 0)
|
|
})
|
|
b.Run(`i=~".+"`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "i", ".+", false, true)
|
|
benchSearch(b, tfs, 5e6)
|
|
})
|
|
b.Run(`i=~""`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "i", "", false, true)
|
|
benchSearch(b, tfs, 0)
|
|
})
|
|
b.Run(`i!=""`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "i", "", true, false)
|
|
benchSearch(b, tfs, 5e6)
|
|
})
|
|
b.Run(`n="1",i=~".*",j="foo"`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "n", "1", false, false)
|
|
addTagFilter(tfs, "i", ".*", false, true)
|
|
addTagFilter(tfs, "j", "foo", false, false)
|
|
benchSearch(b, tfs, 1e5)
|
|
})
|
|
b.Run(`n="1",i=~".*",i!="2",j="foo"`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "n", "1", false, false)
|
|
addTagFilter(tfs, "i", ".*", false, true)
|
|
addTagFilter(tfs, "i", "2", true, false)
|
|
addTagFilter(tfs, "j", "foo", false, false)
|
|
benchSearch(b, tfs, 1e5-1)
|
|
})
|
|
b.Run(`n="1",i!=""`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "n", "1", false, false)
|
|
addTagFilter(tfs, "i", "", true, false)
|
|
benchSearch(b, tfs, 2e5)
|
|
})
|
|
b.Run(`n="1",i!="",j="foo"`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "n", "1", false, false)
|
|
addTagFilter(tfs, "i", "", true, false)
|
|
addTagFilter(tfs, "j", "foo", false, false)
|
|
benchSearch(b, tfs, 1e5)
|
|
})
|
|
b.Run(`n="1",i=~".+",j="foo"`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "n", "1", false, false)
|
|
addTagFilter(tfs, "i", ".+", false, true)
|
|
addTagFilter(tfs, "j", "foo", false, false)
|
|
benchSearch(b, tfs, 1e5)
|
|
})
|
|
b.Run(`n="1",i=~"1.+",j="foo"`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "n", "1", false, false)
|
|
addTagFilter(tfs, "i", "1.+", false, true)
|
|
addTagFilter(tfs, "j", "foo", false, false)
|
|
benchSearch(b, tfs, 11110)
|
|
})
|
|
b.Run(`n="1",i=~".+",i!="2",j="foo"`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "n", "1", false, false)
|
|
addTagFilter(tfs, "i", ".+", false, true)
|
|
addTagFilter(tfs, "i", "2", true, false)
|
|
addTagFilter(tfs, "j", "foo", false, false)
|
|
benchSearch(b, tfs, 1e5-1)
|
|
})
|
|
b.Run(`n="1",i=~".+",i!~"2.*",j="foo"`, func(b *testing.B) {
|
|
tfs := NewTagFilters()
|
|
addTagFilter(tfs, "n", "1", false, false)
|
|
addTagFilter(tfs, "i", ".+", false, true)
|
|
addTagFilter(tfs, "i", "2.*", true, true)
|
|
addTagFilter(tfs, "j", "foo", false, false)
|
|
benchSearch(b, tfs, 88889)
|
|
})
|
|
}
|
|
|
|
func BenchmarkIndexDBGetTSIDs(b *testing.B) {
|
|
s := newTestStorage()
|
|
defer stopTestStorage(s)
|
|
|
|
dbName := nextIndexDBTableName()
|
|
var isReadOnly uint32
|
|
db := mustOpenIndexDB(dbName, s, 0, &isReadOnly)
|
|
defer func() {
|
|
db.MustClose()
|
|
if err := os.RemoveAll(dbName); err != nil {
|
|
b.Fatalf("cannot remove indexDB: %s", err)
|
|
}
|
|
}()
|
|
|
|
const recordsPerLoop = 1000
|
|
const recordsCount = 1e5
|
|
|
|
// Fill the db with recordsCount records.
|
|
var mn MetricName
|
|
mn.MetricGroup = []byte("rps")
|
|
for i := 0; i < 2; i++ {
|
|
key := fmt.Sprintf("key_%d", i)
|
|
value := fmt.Sprintf("value_%d", i)
|
|
mn.AddTag(key, value)
|
|
}
|
|
var tsid TSID
|
|
var metricName []byte
|
|
var metricNameRaw []byte
|
|
|
|
is := db.getIndexSearch(noDeadline)
|
|
defer db.putIndexSearch(is)
|
|
for i := 0; i < recordsCount; i++ {
|
|
mn.sortTags()
|
|
metricName = mn.Marshal(metricName[:0])
|
|
metricNameRaw = mn.marshalRaw(metricName[:0])
|
|
if err := is.GetOrCreateTSIDByName(&tsid, metricName, metricNameRaw, 0); err != nil {
|
|
b.Fatalf("cannot insert record: %s", err)
|
|
}
|
|
}
|
|
|
|
b.SetBytes(recordsPerLoop)
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
b.RunParallel(func(pb *testing.PB) {
|
|
var tsidLocal TSID
|
|
var metricNameLocal []byte
|
|
var metricNameLocalRaw []byte
|
|
mnLocal := mn
|
|
is := db.getIndexSearch(noDeadline)
|
|
defer db.putIndexSearch(is)
|
|
for pb.Next() {
|
|
for i := 0; i < recordsPerLoop; i++ {
|
|
mnLocal.sortTags()
|
|
metricNameLocal = mnLocal.Marshal(metricNameLocal[:0])
|
|
metricNameLocalRaw = mnLocal.marshalRaw(metricNameLocalRaw[:0])
|
|
if err := is.GetOrCreateTSIDByName(&tsidLocal, metricNameLocal, metricNameLocalRaw, 0); err != nil {
|
|
panic(fmt.Errorf("cannot obtain tsid: %w", err))
|
|
}
|
|
}
|
|
}
|
|
})
|
|
b.StopTimer()
|
|
}
|