app/vmagent: add -remoteWrite.decimalPlaces command-line flag, which may be used for reducing disk space usage on the remote storage

This commit is contained in:
Aliaksandr Valialkin 2020-07-21 21:55:24 +03:00
parent d3442b40b2
commit 31ae5911a8
3 changed files with 71 additions and 1 deletions

View File

@ -6,6 +6,7 @@ import (
"sync"
"sync/atomic"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@ -30,6 +31,9 @@ var (
"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500000000. "+
"Disk usage is unlimited if the value is set to 0")
decimalPlaces = flag.Int("remoteWrite.decimalPlaces", 0, "The number of significant decimal places to leave in metric values before writing them to remote storage. "+
"See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant decimal places. "+
"This option may be used for increasing on-disk compression level for the stored metrics")
)
var rwctxs []*remoteWriteCtx
@ -118,8 +122,19 @@ func Stop() {
// Push sends wr to remote storage systems set via `-remoteWrite.url`.
//
// Note that wr may be modified by Push due to relabeling.
// Note that wr may be modified by Push due to relabeling and rounding.
func Push(wr *prompbmarshal.WriteRequest) {
if *decimalPlaces > 0 {
// Round values according to decimalPlaces
for i := range wr.Timeseries {
samples := wr.Timeseries[i].Samples
for j := range samples {
s := &samples[j]
s.Value = decimal.Round(s.Value, *decimalPlaces)
}
}
}
var rctx *relabelCtx
rcs := allRelabelConfigs.Load().(*relabelConfigs)
prcsGlobal := rcs.global

View File

@ -256,6 +256,38 @@ func maxUpExponent(v int64) int16 {
}
}
// Round f to value with the given number of significant decimal digits.
func Round(f float64, digits int) float64 {
if digits <= 0 || digits >= 18 {
return f
}
if math.IsNaN(f) || math.IsInf(f, 0) || f == 0 {
return f
}
n := int64(math.Pow10(digits))
isNegative := f < 0
if isNegative {
f = -f
}
v, e := positiveFloatToDecimal(f)
if v > vMax {
v = vMax
}
var rem int64
for v > n {
rem = v % 10
v /= 10
e++
}
if rem >= 5 {
v++
}
if isNegative {
v = -v
}
return ToFloat(v, e)
}
// ToFloat returns f=v*10^e.
func ToFloat(v int64, e int16) float64 {
f := float64(v)

View File

@ -7,6 +7,29 @@ import (
"testing"
)
func TestRoundInplace(t *testing.T) {
f := func(f float64, digits int, resultExpected float64) {
t.Helper()
result := Round(f, digits)
if math.IsNaN(result) {
if !math.IsNaN(resultExpected) {
t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
}
}
if result != resultExpected {
t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
}
}
f(1234, 0, 1234)
f(-12.34, 20, -12.34)
f(12, 1, 10)
f(25, 1, 30)
f(2.5, 1, 3)
f(-0.56, 1, -0.6)
f(1234567, 3, 1230000)
f(-1.234567, 4, -1.235)
}
func TestPositiveFloatToDecimal(t *testing.T) {
f := func(f float64, decimalExpected int64, exponentExpected int16) {
t.Helper()