From 6c6b9c3d93f7c0db64a7f190fa2e5d10e875e7d4 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 10 Feb 2023 21:57:49 -0800 Subject: [PATCH] app/vmauth: add `-maxConcurrentPerUserRequests` command-line option for limiting the number of concurrent requests on a per-user basis Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3346 --- app/vmauth/README.md | 8 +++++--- app/vmauth/auth_config.go | 34 ++++++++++++++++++---------------- app/vmauth/example_config.yml | 2 +- app/vmauth/main.go | 5 ++++- docs/CHANGELOG.md | 2 +- 5 files changed, 29 insertions(+), 22 deletions(-) diff --git a/app/vmauth/README.md b/app/vmauth/README.md index 9df57113c5..19e0e0406c 100644 --- a/app/vmauth/README.md +++ b/app/vmauth/README.md @@ -61,7 +61,7 @@ users: # # The given user can send maximum 10 concurrent requests according to the provided max_concurrent_requests. # Excess concurrent requests are rejected with 429 HTTP status code. - # See also -maxConcurrentRequests command-line flag for limiting the global number of concurrent requests. + # See also -maxConcurrentPerUserRequests and -maxConcurrentRequests command-line flags. - username: "local-single-node" password: "***" url_prefix: "http://localhost:8428" @@ -264,7 +264,7 @@ See the docs at https://docs.victoriametrics.com/vmauth.html . -httpListenAddr.useProxyProtocol Whether to use proxy protocol for connections accepted at -httpListenAddr . See https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt -internStringMaxLen int - The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 300) + The maximum length for strings to intern. Lower limit may save memory at the cost of higher CPU usage. See https://en.wikipedia.org/wiki/String_interning (default 500) -logInvalidAuthTokens Whether to log requests with invalid auth tokens. Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page -loggerDisableTimestamps @@ -283,8 +283,10 @@ See the docs at https://docs.victoriametrics.com/vmauth.html . Timezone to use for timestamps in logs. Timezone must be a valid IANA Time Zone. For example: America/New_York, Europe/Berlin, Etc/GMT+3 or Local (default "UTC") -loggerWarnsPerSecondLimit int Per-second limit on the number of WARN messages. If more than the given number of warns are emitted per second, then the remaining warns are suppressed. Zero values disable the rate limit + -maxConcurrentPerUserRequests int + The maximum number of concurrent requests vmauth can process per each configured user. Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option in per-user config (default 300) -maxConcurrentRequests int - The maximum number of concurrent requests vmauth can process. Other requests are rejected with '429 Too Many Requests' http status code. See also -maxIdleConnsPerBackend and max_concurrent_requests option per each user config (default 1000) + The maximum number of concurrent requests vmauth can process. Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options (default 1000) -maxIdleConnsPerBackend int The maximum number of idle connections vmauth can open per each backend host. See also -maxConcurrentRequests (default 100) -memory.allowedBytes size diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go index 99e87b5689..fbc24bbc6e 100644 --- a/app/vmauth/auth_config.go +++ b/app/vmauth/auth_config.go @@ -48,22 +48,25 @@ type UserInfo struct { } func (ui *UserInfo) beginConcurrencyLimit() error { - if ui.concurrencyLimitCh == nil { - return nil - } select { case ui.concurrencyLimitCh <- struct{}{}: return nil default: ui.concurrencyLimitReached.Inc() - return fmt.Errorf("cannot handle more than max_concurrent_requests=%d concurrent requests from user %s", ui.MaxConcurrentRequests, ui.name()) + return fmt.Errorf("cannot handle more than max_concurrent_requests=%d concurrent requests from user %s", ui.getMaxConcurrentRequests(), ui.name()) } } func (ui *UserInfo) endConcurrencyLimit() { - if ui.concurrencyLimitCh != nil { - <-ui.concurrencyLimitCh + <-ui.concurrencyLimitCh +} + +func (ui *UserInfo) getMaxConcurrentRequests() int { + mcr := ui.MaxConcurrentRequests + if mcr > *maxConcurrentPerUserRequests { + mcr = *maxConcurrentPerUserRequests } + return mcr } // Header is `Name: Value` http header, which must be added to the proxied request. @@ -331,16 +334,15 @@ func parseAuthConfig(data []byte) (map[string]*UserInfo, error) { if ui.Username != "" { ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name)) } - if ui.MaxConcurrentRequests > 0 { - ui.concurrencyLimitCh = make(chan struct{}, ui.MaxConcurrentRequests) - ui.concurrencyLimitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_concurrent_requests_limit_reached_total{username=%q}`, name)) - _ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_capacity{username=%q}`, name), func() float64 { - return float64(cap(ui.concurrencyLimitCh)) - }) - _ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_current{username=%q}`, name), func() float64 { - return float64(len(ui.concurrencyLimitCh)) - }) - } + mcr := ui.getMaxConcurrentRequests() + ui.concurrencyLimitCh = make(chan struct{}, mcr) + ui.concurrencyLimitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_concurrent_requests_limit_reached_total{username=%q}`, name)) + _ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_capacity{username=%q}`, name), func() float64 { + return float64(cap(ui.concurrencyLimitCh)) + }) + _ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_current{username=%q}`, name), func() float64 { + return float64(len(ui.concurrencyLimitCh)) + }) byAuthToken[at1] = ui byAuthToken[at2] = ui } diff --git a/app/vmauth/example_config.yml b/app/vmauth/example_config.yml index 1b6cd53fd8..461b42a865 100644 --- a/app/vmauth/example_config.yml +++ b/app/vmauth/example_config.yml @@ -24,7 +24,7 @@ users: # # The given user can send maximum 10 concurrent requests according to the provided max_concurrent_requests. # Excess concurrent requests are rejected with 429 HTTP status code. - # See also -maxConcurrentRequests command-line flag for limiting the global number of concurrent requests. + # See also -maxConcurrentPerUserRequests and -maxConcurrentRequests command-line flags. - username: "local-single-node" password: "***" url_prefix: "http://localhost:8428" diff --git a/app/vmauth/main.go b/app/vmauth/main.go index e9997a859c..e0a0ac44f4 100644 --- a/app/vmauth/main.go +++ b/app/vmauth/main.go @@ -33,7 +33,10 @@ var ( "See also -maxConcurrentRequests") responseTimeout = flag.Duration("responseTimeout", 5*time.Minute, "The timeout for receiving a response from backend") maxConcurrentRequests = flag.Int("maxConcurrentRequests", 1000, "The maximum number of concurrent requests vmauth can process. Other requests are rejected with "+ - "'429 Too Many Requests' http status code. See also -maxIdleConnsPerBackend and max_concurrent_requests option per each user config") + "'429 Too Many Requests' http status code. See also -maxConcurrentPerUserRequests and -maxIdleConnsPerBackend command-line options") + maxConcurrentPerUserRequests = flag.Int("maxConcurrentPerUserRequests", 300, "The maximum number of concurrent requests vmauth can process per each configured user. "+ + "Other requests are rejected with '429 Too Many Requests' http status code. See also -maxConcurrentRequests command-line option and max_concurrent_requests option "+ + "in per-user config") reloadAuthKey = flag.String("reloadAuthKey", "", "Auth key for /-/reload http endpoint. It must be passed as authKey=...") logInvalidAuthTokens = flag.Bool("logInvalidAuthTokens", false, "Whether to log requests with invalid auth tokens. "+ `Such requests are always counted at vmauth_http_request_errors_total{reason="invalid_auth_token"} metric, which is exposed at /metrics page`) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 773e5813f5..7fb43b4960 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,7 +15,7 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip -* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add the ability to limit the number of concurrent requests on a per-user basis via `max_concurrent_requests` option. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3346) and [these docs](https://docs.victoriametrics.com/vmauth.html#auth-config). +* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add the ability to limit the number of concurrent requests on a per-user basis via `-maxConcurrentPerUserRequests` command-line option and via `max_concurrent_requests` config option. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3346) and [these docs](https://docs.victoriametrics.com/vmauth.html#auth-config). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): automatically retry failing GET requests on all the configured backends. * FEATURE: [vmalert enterprise](https://docs.victoriametrics.com/vmalert.html): add ability to read alerting and recording rules from S3, GCS or S3-compatible object storage. See [these docs](https://docs.victoriametrics.com/vmalert.html#reading-rules-from-object-storage).