From d9581cf3f3dde1f46db97c9108aaff6e8d140f01 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 27 Feb 2023 11:03:49 -0800 Subject: [PATCH] app/vmagent: add -remoteWrite.vmProtoCompressLevel command-line flag for tuning the compression level for VictoriaMetrics remote write protocol --- app/vmagent/README.md | 4 ++++ app/vmagent/remotewrite/pendingseries.go | 5 ++++- docs/vmagent.md | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index e2307df656..09493e903e 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -198,6 +198,8 @@ VictoriaMetrics remote write protocol provides the following benefits comparing or `vminsert` at [cluster version](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html). It is possible to force switch to VictoriaMetrics remote write protocol by specifying `-remoteWrite.forceVMProto` command-line flag for the corresponding `-remoteWrite.url`. +It is possible to tune the compression level for VictoriaMetrics remote write protocol with `-remoteWrite.vmProtoCompressLevel` command-line flag. +Bigger values reduce network usage at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of higher network usage. `vmagent` automatically switches to Prometheus remote write protocol when it sends data to old versions of VictoriaMetrics components or to other Prometheus-compatible remote storage systems. It is possible to force switch to Prometheus remote write protocol @@ -1546,6 +1548,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . -remoteWrite.urlRelabelConfig array Optional path to relabel configs for the corresponding -remoteWrite.url. See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. See https://docs.victoriametrics.com/vmagent.html#relabeling Supports an array of values separated by comma or specified via multiple flags. + -remoteWrite.vmProtoCompressLevel int + The compression level for VictoriaMetrics remote write protocol. Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol -sortLabels Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit -tls diff --git a/app/vmagent/remotewrite/pendingseries.go b/app/vmagent/remotewrite/pendingseries.go index 75a5b1549d..f436e53a8f 100644 --- a/app/vmagent/remotewrite/pendingseries.go +++ b/app/vmagent/remotewrite/pendingseries.go @@ -24,6 +24,9 @@ var ( "This option takes effect only when less than 10K data points per second are pushed to -remoteWrite.url") maxUnpackedBlockSize = flagutil.NewBytes("remoteWrite.maxBlockSize", 8*1024*1024, "The maximum block size to send to remote storage. Bigger blocks may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxRowsPerBlock") maxRowsPerBlock = flag.Int("remoteWrite.maxRowsPerBlock", 10000, "The maximum number of samples to send in each block to remote storage. Higher number may improve performance at the cost of the increased memory usage. See also -remoteWrite.maxBlockSize") + vmProtoCompressLevel = flag.Int("remoteWrite.vmProtoCompressLevel", 0, "The compression level for VictoriaMetrics remote write protocol. "+ + "Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. "+ + "See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol") ) type pendingSeries struct { @@ -203,7 +206,7 @@ func pushWriteRequest(wr *prompbmarshal.WriteRequest, pushBlock func(block []byt if len(bb.B) <= maxUnpackedBlockSize.IntN() { zb := snappyBufPool.Get() if isVMRemoteWrite { - zb.B = zstd.CompressLevel(zb.B[:0], bb.B, 0) + zb.B = zstd.CompressLevel(zb.B[:0], bb.B, *vmProtoCompressLevel) } else { zb.B = snappy.Encode(zb.B[:cap(zb.B)], bb.B) } diff --git a/docs/vmagent.md b/docs/vmagent.md index 8fb77eb11d..2550137f2f 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -202,6 +202,8 @@ VictoriaMetrics remote write protocol provides the following benefits comparing or `vminsert` at [cluster version](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html). It is possible to force switch to VictoriaMetrics remote write protocol by specifying `-remoteWrite.forceVMProto` command-line flag for the corresponding `-remoteWrite.url`. +It is possible to tune the compression level for VictoriaMetrics remote write protocol with `-remoteWrite.vmProtoCompressLevel` command-line flag. +Bigger values reduce network usage at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of higher network usage. `vmagent` automatically switches to Prometheus remote write protocol when it sends data to old versions of VictoriaMetrics components or to other Prometheus-compatible remote storage systems. It is possible to force switch to Prometheus remote write protocol @@ -1550,6 +1552,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . -remoteWrite.urlRelabelConfig array Optional path to relabel configs for the corresponding -remoteWrite.url. See also -remoteWrite.relabelConfig. The path can point either to local file or to http url. See https://docs.victoriametrics.com/vmagent.html#relabeling Supports an array of values separated by comma or specified via multiple flags. + -remoteWrite.vmProtoCompressLevel int + The compression level for VictoriaMetrics remote write protocol. Higher values reduce network traffic at the cost of higher CPU usage. Negative values reduce CPU usage at the cost of increased network traffic. See https://docs.victoriametrics.com/vmagent.html#victoriametrics-remote-write-protocol -sortLabels Whether to sort labels for incoming samples before writing them to all the configured remote storage systems. This may be needed for reducing memory usage at remote storage when the order of labels in incoming samples is random. For example, if m{k1="v1",k2="v2"} may be sent as m{k2="v2",k1="v1"}Enabled sorting for labels can slow down ingestion performance a bit -tls