mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-25 06:17:36 +01:00
090cb2c9de
* app/vmagent: allow to disabled on-disk queue Previously, it wasn't possible to build data processing pipeline with a chain of vmagents. In case when remoteWrite for the last vmagent in the chain wasn't accessible, it persisted data only when it has enough disk capacity. If disk queue is full, it started to silently drop ingested metrics. New flags allows to disable on-disk persistent and immediatly return an error if remoteWrite is not accessible anymore. It blocks any writes and notify client, that data ingestion isn't possible. Main use case for this feature - use external queue such as kafka for data persistence. https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2110 * adds test, updates readme * apply review suggestions * update docs for vmagent * makes linter happy --------- Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>
72 lines
2.2 KiB
Go
72 lines
2.2 KiB
Go
package remotewrite
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"testing"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
|
)
|
|
|
|
func TestPushWriteRequest(t *testing.T) {
|
|
rowsCounts := []int{1, 10, 100, 1e3, 1e4}
|
|
expectedBlockLensProm := []int{216, 1848, 16424, 169882, 1757876}
|
|
expectedBlockLensVM := []int{138, 492, 3927, 34995, 288476}
|
|
for i, rowsCount := range rowsCounts {
|
|
expectedBlockLenProm := expectedBlockLensProm[i]
|
|
expectedBlockLenVM := expectedBlockLensVM[i]
|
|
t.Run(fmt.Sprintf("%d", rowsCount), func(t *testing.T) {
|
|
testPushWriteRequest(t, rowsCount, expectedBlockLenProm, expectedBlockLenVM)
|
|
})
|
|
}
|
|
}
|
|
|
|
func testPushWriteRequest(t *testing.T, rowsCount, expectedBlockLenProm, expectedBlockLenVM int) {
|
|
f := func(isVMRemoteWrite bool, expectedBlockLen int, tolerancePrc float64) {
|
|
t.Helper()
|
|
wr := newTestWriteRequest(rowsCount, 20)
|
|
pushBlockLen := 0
|
|
pushBlock := func(block []byte) bool {
|
|
if pushBlockLen > 0 {
|
|
panic(fmt.Errorf("BUG: pushBlock called multiple times; pushBlockLen=%d at first call, len(block)=%d at second call", pushBlockLen, len(block)))
|
|
}
|
|
pushBlockLen = len(block)
|
|
return true
|
|
}
|
|
_ = pushWriteRequest(wr, pushBlock, isVMRemoteWrite)
|
|
if math.Abs(float64(pushBlockLen-expectedBlockLen)/float64(expectedBlockLen)*100) > tolerancePrc {
|
|
t.Fatalf("unexpected block len for rowsCount=%d, isVMRemoteWrite=%v; got %d bytes; expecting %d bytes +- %.0f%%",
|
|
rowsCount, isVMRemoteWrite, pushBlockLen, expectedBlockLen, tolerancePrc)
|
|
}
|
|
}
|
|
|
|
// Check Prometheus remote write
|
|
f(false, expectedBlockLenProm, 0)
|
|
|
|
// Check VictoriaMetrics remote write
|
|
f(true, expectedBlockLenVM, 15)
|
|
}
|
|
|
|
func newTestWriteRequest(seriesCount, labelsCount int) *prompbmarshal.WriteRequest {
|
|
var wr prompbmarshal.WriteRequest
|
|
for i := 0; i < seriesCount; i++ {
|
|
var labels []prompbmarshal.Label
|
|
for j := 0; j < labelsCount; j++ {
|
|
labels = append(labels, prompbmarshal.Label{
|
|
Name: fmt.Sprintf("label_%d_%d", i, j),
|
|
Value: fmt.Sprintf("value_%d_%d", i, j),
|
|
})
|
|
}
|
|
wr.Timeseries = append(wr.Timeseries, prompbmarshal.TimeSeries{
|
|
Labels: labels,
|
|
Samples: []prompbmarshal.Sample{
|
|
{
|
|
Value: float64(i),
|
|
Timestamp: 1000 * int64(i),
|
|
},
|
|
},
|
|
})
|
|
}
|
|
return &wr
|
|
}
|