From af263fe88181d02bcd9f57cf462e187d9275c85d Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Sat, 7 Jan 2023 00:11:44 -0800 Subject: [PATCH] all: small improvements in error messages and command-line flag descriptions related to concurrency limiters --- README.md | 2 +- app/vmagent/README.md | 2 +- app/vmselect/main.go | 2 +- docs/Cluster-VictoriaMetrics.md | 10 +++++----- docs/README.md | 2 +- docs/Single-server-VictoriaMetrics.md | 2 +- docs/vmagent.md | 2 +- lib/writeconcurrencylimiter/concurrencylimiter.go | 7 ++++--- 8 files changed, 15 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9403e4d2a..248a539b0 100644 --- a/README.md +++ b/README.md @@ -2235,7 +2235,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -loggerWarnsPerSecondLimit int Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit -maxConcurrentInserts int - The maximum number of concurrent inserts. Default value should work for most cases, since it minimizes the overhead for concurrent inserts. This option is tigthly coupled with -insert.maxQueueDuration (default 16) + The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8) -maxInsertRequestSize size The maximum size in bytes of a single Prometheus remote_write API request Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 291ff15ce..da5c85755 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -1247,7 +1247,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . -loggerWarnsPerSecondLimit int Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit -maxConcurrentInserts int - The maximum number of concurrent inserts. Default value should work for most cases, since it minimizes the overhead for concurrent inserts. This option is tigthly coupled with -insert.maxQueueDuration (default 16) + The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8) -maxInsertRequestSize size The maximum size in bytes of a single Prometheus remote_write API request Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432) diff --git a/app/vmselect/main.go b/app/vmselect/main.go index 402106a02..f17f2e438 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -119,7 +119,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { concurrencyLimitTimeout.Inc() err := &httpserver.ErrorWithStatusCode{ Err: fmt.Errorf("couldn't start executing the request in %.3f seconds, since -search.maxConcurrentRequests=%d concurrent requests "+ - "are already executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+ + "are executed. Possible solutions: to reduce query load; to add more compute resources to the server; "+ "to increase -search.maxQueueDuration; to increase -search.maxQueryDuration; to increase -search.maxConcurrentRequests", d.Seconds(), *maxConcurrentRequests), StatusCode: http.StatusServiceUnavailable, diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index c5ac6814c..1976119fc 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -547,7 +547,7 @@ Some workloads may need fine-grained resource usage limits. In these cases the f `vmselect` and `vmstorage` provides `-search.maxQueueDuration` command-line flag for limiting the maximum wait time for suspended queries. See also `-search.maxMemoryPerQuery` command-line flag at `vmselect`. - `-search.maxQueueDuration` at `vmselect` and `vmstorage` limits the maximum duration queries may wait for execution when `-search.maxConcurrentRequests` - concurrent queries are already executed. + concurrent queries are executed. - `-search.maxSamplesPerSeries` at `vmselect` limits the number of raw samples the query can process per each time series. `vmselect` processes raw samples sequentially per each found time series during the query. It unpacks raw samples on the selected time range per each time series into memory and then applies the given [rollup function](https://docs.victoriametrics.com/MetricsQL.html#rollup-functions). @@ -860,7 +860,7 @@ Below is the output for `/path/to/vminsert -help`: -influxTrimTimestamp duration Trim timestamps for InfluxDB line protocol data to this duration. Minimum practical duration is 1ms. Higher duration (i.e. 1s) may be used for reducing disk space usage for timestamp data (default 1ms) -insert.maxQueueDuration duration - The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are already executed (default 1m0s) + The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s) -loggerDisableTimestamps Whether to disable writing timestamps in logs -loggerErrorsPerSecondLimit int @@ -878,7 +878,7 @@ Below is the output for `/path/to/vminsert -help`: -loggerWarnsPerSecondLimit int Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit -maxConcurrentInserts int - The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the overhead. See also -insert.maxQueueDuration (default 8) + The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8) -maxInsertRequestSize size The maximum size in bytes of a single Prometheus remote_write API request Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432) @@ -1241,7 +1241,7 @@ Below is the output for `/path/to/vmstorage -help`: -httpListenAddr string Address to listen for http connections (default ":8482") -insert.maxQueueDuration duration - The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are already executed (default 1m0s) + The maximum duration to wait in the queue when -maxConcurrentInserts concurrent insert requests are executed (default 1m0s) -logNewSeries Whether to log new series. This option is for debug purposes only. It can lead to performance issues when big number of new series are ingested into VictoriaMetrics -loggerDisableTimestamps @@ -1261,7 +1261,7 @@ Below is the output for `/path/to/vmstorage -help`: -loggerWarnsPerSecondLimit int Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit -maxConcurrentInserts int - The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the overhead. See also -insert.maxQueueDuration (default 8) + The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8) -memory.allowedBytes size Allowed size of system memory VictoriaMetrics caches may occupy. This option overrides -memory.allowedPercent if set to a non-zero value. Too low a value may increase the cache miss rate usually resulting in higher CPU and disk IO usage. Too high a value may evict too much data from OS page cache resulting in higher disk IO usage Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 0) diff --git a/docs/README.md b/docs/README.md index 24151b7d6..b86483327 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2236,7 +2236,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -loggerWarnsPerSecondLimit int Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit -maxConcurrentInserts int - The maximum number of concurrent inserts. Default value should work for most cases, since it minimizes the overhead for concurrent inserts. This option is tigthly coupled with -insert.maxQueueDuration (default 16) + The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8) -maxInsertRequestSize size The maximum size in bytes of a single Prometheus remote_write API request Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432) diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 5006c924e..b8f6d47d6 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -2239,7 +2239,7 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -loggerWarnsPerSecondLimit int Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit -maxConcurrentInserts int - The maximum number of concurrent inserts. Default value should work for most cases, since it minimizes the overhead for concurrent inserts. This option is tigthly coupled with -insert.maxQueueDuration (default 16) + The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8) -maxInsertRequestSize size The maximum size in bytes of a single Prometheus remote_write API request Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432) diff --git a/docs/vmagent.md b/docs/vmagent.md index 34b1f64fd..fbd61c471 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -1251,7 +1251,7 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . -loggerWarnsPerSecondLimit int Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit -maxConcurrentInserts int - The maximum number of concurrent inserts. Default value should work for most cases, since it minimizes the overhead for concurrent inserts. This option is tigthly coupled with -insert.maxQueueDuration (default 16) + The maximum number of concurrent insert requests. Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. See also -insert.maxQueueDuration (default 8) -maxInsertRequestSize size The maximum size in bytes of a single Prometheus remote_write API request Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 33554432) diff --git a/lib/writeconcurrencylimiter/concurrencylimiter.go b/lib/writeconcurrencylimiter/concurrencylimiter.go index a280ca058..eaf6fbc18 100644 --- a/lib/writeconcurrencylimiter/concurrencylimiter.go +++ b/lib/writeconcurrencylimiter/concurrencylimiter.go @@ -16,9 +16,10 @@ import ( var ( maxConcurrentInserts = flag.Int("maxConcurrentInserts", 2*cgroup.AvailableCPUs(), "The maximum number of concurrent insert requests. "+ - "Default value should work for most cases, since it minimizes the overhead. See also -insert.maxQueueDuration") + "Default value should work for most cases, since it minimizes the memory usage. The default value can be increased when clients send data over slow networks. "+ + "See also -insert.maxQueueDuration") maxQueueDuration = flag.Duration("insert.maxQueueDuration", time.Minute, "The maximum duration to wait in the queue when -maxConcurrentInserts "+ - "concurrent insert requests are already executed") + "concurrent insert requests are executed") ) // Reader is a reader, which increases the concurrency after the first Read() call @@ -64,7 +65,7 @@ func (r *Reader) Read(p []byte) (int, error) { if !r.increasedConcurrency { if !incConcurrency() { err = &httpserver.ErrorWithStatusCode{ - Err: fmt.Errorf("cannot process insert request for %.3f seconds because %d concurrent insert requests are already executed. "+ + Err: fmt.Errorf("cannot process insert request for %.3f seconds because %d concurrent insert requests are executed. "+ "Possible solutions: to reduce workload; to increase compute resources at the server; "+ "to increase -insert.maxQueueDuration; to increase -maxConcurrentInserts", maxQueueDuration.Seconds(), *maxConcurrentInserts),