mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-16 17:21:04 +01:00
120f3bc467
- Compare the actual free disk space to the value provided via -storage.minFreeDiskSpaceBytes directly inside the Storage.IsReadOnly(). This should work fast in most cases. This simplifies the logic at lib/storage. - Do not take into account -storage.minFreeDiskSpaceBytes during background merges, since it results in uncontrolled growth of small parts when the free disk space approaches -storage.minFreeDiskSpaceBytes. The background merge logic uses another mechanism for determining whether there is enough disk space for the merge - it reserves the needed disk space before the merge and releases it after the merge. This prevents from out of disk space errors during background merge. - Properly handle corner cases for flushing in-memory data to disk when the storage enters read-only mode. This is better than losing the in-memory data. - Return back Storage.MustAddRows() instead of Storage.AddRows(), since the only case when AddRows() can return error is when the storage is in read-only mode. This case must be handled by the caller by calling Storage.IsReadOnly() before adding rows to the storage. This simplifies the code a bit, since the caller of Storage.MustAddRows() shouldn't handle errors returned by Storage.AddRows(). - Properly store parsed logs to Storage if parts of the request contain invalid log lines. Previously the parsed logs could be lost in this case. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4737 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4945
67 lines
1.5 KiB
Go
67 lines
1.5 KiB
Go
package loki
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/golang/snappy"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logstorage"
|
|
)
|
|
|
|
func BenchmarkParseProtobufRequest(b *testing.B) {
|
|
for _, streams := range []int{5, 10} {
|
|
for _, rows := range []int{100, 1000} {
|
|
for _, labels := range []int{10, 50} {
|
|
b.Run(fmt.Sprintf("streams_%d/rows_%d/labels_%d", streams, rows, labels), func(b *testing.B) {
|
|
benchmarkParseProtobufRequest(b, streams, rows, labels)
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func benchmarkParseProtobufRequest(b *testing.B, streams, rows, labels int) {
|
|
b.ReportAllocs()
|
|
b.SetBytes(int64(streams * rows))
|
|
b.RunParallel(func(pb *testing.PB) {
|
|
body := getProtobufBody(streams, rows, labels)
|
|
for pb.Next() {
|
|
_, err := parseProtobufRequest(body, func(timestamp int64, fields []logstorage.Field) {})
|
|
if err != nil {
|
|
panic(fmt.Errorf("unexpected error: %s", err))
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func getProtobufBody(streams, rows, labels int) []byte {
|
|
var pr PushRequest
|
|
|
|
for i := 0; i < streams; i++ {
|
|
var st Stream
|
|
|
|
st.Labels = `{`
|
|
for j := 0; j < labels; j++ {
|
|
st.Labels += `label_` + strconv.Itoa(j) + `="value_` + strconv.Itoa(j) + `"`
|
|
if j < labels-1 {
|
|
st.Labels += `,`
|
|
}
|
|
}
|
|
st.Labels += `}`
|
|
|
|
for j := 0; j < rows; j++ {
|
|
st.Entries = append(st.Entries, Entry{Timestamp: time.Now(), Line: "value_" + strconv.Itoa(j)})
|
|
}
|
|
|
|
pr.Streams = append(pr.Streams, st)
|
|
}
|
|
|
|
body, _ := pr.Marshal()
|
|
encodedBody := snappy.Encode(nil, body)
|
|
|
|
return encodedBody
|
|
}
|