mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-18 14:40:26 +01:00
cbc637d1dd
- Move the remaining code responsible for stream aggregation initialization from remotewrite.go to streamaggr.go . This improves code maintainability a bit. - Properly shut down streamaggr.Aggregators initialized inside remotewrite.CheckStreamAggrConfigs(). This prevents from potential resource leaks. - Use separate functions for initializing and reloading of global stream aggregation and per-remoteWrite.url stream aggregation. This makes the code easier to read and maintain. This also fixes INFO and ERROR logs emitted by these functions. - Add an ability to specify `name` option in every stream aggregation config. This option is used as `name` label in metrics exposed by stream aggregation at /metrics page. This simplifies investigation of the exposed metrics. - Add `path` label additionally to `name`, `url` and `position` labels at metrics exposed by streaming aggregation. This label should simplify investigation of the exposed metrics. - Remove `match` and `group` labels from metrics exposed by streaming aggregation, since they have little practical applicability: it is hard to use these labels in query filters and aggregation functions. - Rename the metric `vm_streamaggr_flushed_samples_total` to less misleading `vm_streamaggr_output_samples_total` . This metric shows the number of samples generated by the corresponding streaming aggregation rule. This metric has been added in the commit861852f262. See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6462 - Remove the metric `vm_streamaggr_stale_samples_total`, since it is unclear how it can be used in practice. This metric has been added in the commit861852f262. See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6462 - Remove Alias and aggrID fields from streamaggr.Options struct, since these fields aren't related to optional params, which could modify the behaviour of the constructed streaming aggregator. Convert the Alias field to regular argument passed to LoadFromFile() function, since this argument is mandatory. - Pass Options arg to LoadFromFile() function by reference, since this structure is quite big. This also allows passing nil instead of Options when default options are enough. - Add `name`, `path`, `url` and `position` labels to `vm_streamaggr_dedup_state_size_bytes` and `vm_streamaggr_dedup_state_items_count` metrics, so they have consistent set of labels comparing to the rest of streaming aggregation metrics. - Convert aggregator.aggrStates field type from `map[string]aggrState` to `[]aggrOutput`, where `aggrOutput` contains the corresponding `aggrState` plus all the related metrics (currently only `vm_streamaggr_output_samples_total` metric is exposed with the corresponding `output` label per each configured output function). This simplifies and speeds up the code responsible for updating per-output metrics. This is a follow-up for the commit2eb1bc4f81. See https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6604 - Added missing urls to docs ( https://docs.victoriametrics.com/stream-aggregation/ ) in error messages. These urls help users figuring out why VictoriaMetrics or vmagent generates the corresponding error messages. The urls were removed for unknown reason in the commit2eb1bc4f81. - Fix incorrect update for `vm_streamaggr_output_samples_total` metric in flushCtx.appendSeriesWithExtraLabel() function. While at it, reduce memory usage by limiting the maximum number of samples per flush to 10K. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5467 Updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/6268
112 lines
2.6 KiB
Go
112 lines
2.6 KiB
Go
package streamaggr
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
|
)
|
|
|
|
var benchOutputs = []string{
|
|
"avg",
|
|
"count_samples",
|
|
"count_series",
|
|
"histogram_bucket",
|
|
"increase",
|
|
"increase_prometheus",
|
|
"last",
|
|
"max",
|
|
"min",
|
|
"quantiles(0, 0.5, 1)",
|
|
"rate_avg",
|
|
"rate_sum",
|
|
"stddev",
|
|
"stdvar",
|
|
"sum_samples",
|
|
"total",
|
|
"total_prometheus",
|
|
"unique_samples",
|
|
}
|
|
|
|
func BenchmarkAggregatorsPush(b *testing.B) {
|
|
for _, output := range benchOutputs {
|
|
b.Run(fmt.Sprintf("output=%s", output), func(b *testing.B) {
|
|
benchmarkAggregatorsPush(b, output)
|
|
})
|
|
}
|
|
}
|
|
|
|
func BenchmarkAggregatorsFlushInternalSerial(b *testing.B) {
|
|
pushFunc := func(_ []prompbmarshal.TimeSeries) {}
|
|
a := newBenchAggregators(benchOutputs, pushFunc)
|
|
defer a.MustStop()
|
|
_ = a.Push(benchSeries, nil)
|
|
|
|
b.ResetTimer()
|
|
b.ReportAllocs()
|
|
b.SetBytes(int64(len(benchSeries) * len(benchOutputs)))
|
|
for i := 0; i < b.N; i++ {
|
|
for _, aggr := range a.as {
|
|
aggr.flushInternal(pushFunc, false)
|
|
}
|
|
}
|
|
}
|
|
|
|
func benchmarkAggregatorsPush(b *testing.B, output string) {
|
|
pushFunc := func(_ []prompbmarshal.TimeSeries) {}
|
|
a := newBenchAggregators([]string{output}, pushFunc)
|
|
defer a.MustStop()
|
|
|
|
const loops = 100
|
|
|
|
b.ResetTimer()
|
|
b.ReportAllocs()
|
|
b.SetBytes(int64(len(benchSeries) * loops))
|
|
b.RunParallel(func(pb *testing.PB) {
|
|
var matchIdxs []byte
|
|
for pb.Next() {
|
|
for i := 0; i < loops; i++ {
|
|
matchIdxs = a.Push(benchSeries, matchIdxs)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func newBenchAggregators(outputs []string, pushFunc PushFunc) *Aggregators {
|
|
outputsQuoted := make([]string, len(outputs))
|
|
for i := range outputs {
|
|
outputsQuoted[i] = strconv.Quote(outputs[i])
|
|
}
|
|
config := fmt.Sprintf(`
|
|
- match: http_requests_total
|
|
interval: 24h
|
|
by: [job]
|
|
outputs: [%s]
|
|
`, strings.Join(outputsQuoted, ","))
|
|
|
|
a, err := LoadFromData([]byte(config), pushFunc, nil, "some_alias")
|
|
if err != nil {
|
|
panic(fmt.Errorf("unexpected error when initializing aggregators: %s", err))
|
|
}
|
|
return a
|
|
}
|
|
|
|
func newBenchSeries(seriesCount int) []prompbmarshal.TimeSeries {
|
|
a := make([]string, seriesCount)
|
|
for j := 0; j < seriesCount; j++ {
|
|
s := fmt.Sprintf(`http_requests_total{path="/foo/%d",job="foo_%d",instance="bar",pod="pod-123232312",namespace="kube-foo-bar",node="node-123-3434-443",`+
|
|
`some_other_label="foo-bar-baz",environment="prod",label1="value1",label2="value2",label3="value3"} %d`, j, j%100, j*1000)
|
|
a = append(a, s)
|
|
}
|
|
metrics := strings.Join(a, "\n")
|
|
offsetMsecs := time.Now().UnixMilli()
|
|
return prompbmarshal.MustParsePromMetrics(metrics, offsetMsecs)
|
|
}
|
|
|
|
const seriesCount = 10_000
|
|
|
|
var benchSeries = newBenchSeries(seriesCount)
|