mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-23 20:37:12 +01:00
app/vmagent: add -remoteWrite.decimalPlaces
command-line flag, which may be used for reducing disk space usage on the remote storage
This commit is contained in:
parent
d3442b40b2
commit
31ae5911a8
@ -6,6 +6,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/decimal"
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
|
||||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||||
@ -30,6 +31,9 @@ var (
|
|||||||
"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
|
"for each -remoteWrite.url. When buffer size reaches the configured maximum, then old data is dropped when adding new data to the buffer. "+
|
||||||
"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500000000. "+
|
"Buffered data is stored in ~500MB chunks, so the minimum practical value for this flag is 500000000. "+
|
||||||
"Disk usage is unlimited if the value is set to 0")
|
"Disk usage is unlimited if the value is set to 0")
|
||||||
|
decimalPlaces = flag.Int("remoteWrite.decimalPlaces", 0, "The number of significant decimal places to leave in metric values before writing them to remote storage. "+
|
||||||
|
"See https://en.wikipedia.org/wiki/Significant_figures . Zero value saves all the significant decimal places. "+
|
||||||
|
"This option may be used for increasing on-disk compression level for the stored metrics")
|
||||||
)
|
)
|
||||||
|
|
||||||
var rwctxs []*remoteWriteCtx
|
var rwctxs []*remoteWriteCtx
|
||||||
@ -118,8 +122,19 @@ func Stop() {
|
|||||||
|
|
||||||
// Push sends wr to remote storage systems set via `-remoteWrite.url`.
|
// Push sends wr to remote storage systems set via `-remoteWrite.url`.
|
||||||
//
|
//
|
||||||
// Note that wr may be modified by Push due to relabeling.
|
// Note that wr may be modified by Push due to relabeling and rounding.
|
||||||
func Push(wr *prompbmarshal.WriteRequest) {
|
func Push(wr *prompbmarshal.WriteRequest) {
|
||||||
|
if *decimalPlaces > 0 {
|
||||||
|
// Round values according to decimalPlaces
|
||||||
|
for i := range wr.Timeseries {
|
||||||
|
samples := wr.Timeseries[i].Samples
|
||||||
|
for j := range samples {
|
||||||
|
s := &samples[j]
|
||||||
|
s.Value = decimal.Round(s.Value, *decimalPlaces)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var rctx *relabelCtx
|
var rctx *relabelCtx
|
||||||
rcs := allRelabelConfigs.Load().(*relabelConfigs)
|
rcs := allRelabelConfigs.Load().(*relabelConfigs)
|
||||||
prcsGlobal := rcs.global
|
prcsGlobal := rcs.global
|
||||||
|
@ -256,6 +256,38 @@ func maxUpExponent(v int64) int16 {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Round f to value with the given number of significant decimal digits.
|
||||||
|
func Round(f float64, digits int) float64 {
|
||||||
|
if digits <= 0 || digits >= 18 {
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
if math.IsNaN(f) || math.IsInf(f, 0) || f == 0 {
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
n := int64(math.Pow10(digits))
|
||||||
|
isNegative := f < 0
|
||||||
|
if isNegative {
|
||||||
|
f = -f
|
||||||
|
}
|
||||||
|
v, e := positiveFloatToDecimal(f)
|
||||||
|
if v > vMax {
|
||||||
|
v = vMax
|
||||||
|
}
|
||||||
|
var rem int64
|
||||||
|
for v > n {
|
||||||
|
rem = v % 10
|
||||||
|
v /= 10
|
||||||
|
e++
|
||||||
|
}
|
||||||
|
if rem >= 5 {
|
||||||
|
v++
|
||||||
|
}
|
||||||
|
if isNegative {
|
||||||
|
v = -v
|
||||||
|
}
|
||||||
|
return ToFloat(v, e)
|
||||||
|
}
|
||||||
|
|
||||||
// ToFloat returns f=v*10^e.
|
// ToFloat returns f=v*10^e.
|
||||||
func ToFloat(v int64, e int16) float64 {
|
func ToFloat(v int64, e int16) float64 {
|
||||||
f := float64(v)
|
f := float64(v)
|
||||||
|
@ -7,6 +7,29 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestRoundInplace(t *testing.T) {
|
||||||
|
f := func(f float64, digits int, resultExpected float64) {
|
||||||
|
t.Helper()
|
||||||
|
result := Round(f, digits)
|
||||||
|
if math.IsNaN(result) {
|
||||||
|
if !math.IsNaN(resultExpected) {
|
||||||
|
t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if result != resultExpected {
|
||||||
|
t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f(1234, 0, 1234)
|
||||||
|
f(-12.34, 20, -12.34)
|
||||||
|
f(12, 1, 10)
|
||||||
|
f(25, 1, 30)
|
||||||
|
f(2.5, 1, 3)
|
||||||
|
f(-0.56, 1, -0.6)
|
||||||
|
f(1234567, 3, 1230000)
|
||||||
|
f(-1.234567, 4, -1.235)
|
||||||
|
}
|
||||||
|
|
||||||
func TestPositiveFloatToDecimal(t *testing.T) {
|
func TestPositiveFloatToDecimal(t *testing.T) {
|
||||||
f := func(f float64, decimalExpected int64, exponentExpected int16) {
|
f := func(f float64, decimalExpected int64, exponentExpected int16) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
Loading…
Reference in New Issue
Block a user