app/vmauth: adds metric_labels and backend_errors counter (#5585)

* app/vmauth: adds metric_labels and backend_errors counter
it must improve observability for user requests with new metric - per user backend errors counter.
it's needed to calculate requests fail rate to the configured backends.
metric_labels configuration allows to perform additional aggregations on top of multiple users from configuration section.
It could be multiple clients or clients with separate read/write tokens
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5565

* wip

---------

Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>

app/vmauth/auth_config.go

@@ -9,6 +9,7 @@ import (
 	"net/url"
 	"os"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -16,6 +17,7 @@ import (
 	"time"
 
 	"github.com/VictoriaMetrics/metrics"
+	"github.com/cespare/xxhash/v2"
 	"gopkg.in/yaml.v2"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
@@ -60,12 +62,15 @@ type UserInfo struct {
 	TLSInsecureSkipVerify  *bool       `yaml:"tls_insecure_skip_verify,omitempty"`
 	TLSCAFile              string      `yaml:"tls_ca_file,omitempty"`
 
+	MetricLabels map[string]string `yaml:"metric_labels,omitempty"`
+
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
 
 	httpTransport *http.Transport
 
 	requests         *metrics.Counter
+	backendErrors    *metrics.Counter
 	requestsDuration *metrics.Summary
 }
 
@@ -462,10 +467,12 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
 // authConfigData needs to be updated each time authConfig is updated.
 var authConfigData atomic.Pointer[[]byte]
 
-var authConfig atomic.Pointer[AuthConfig]
+var (
-var authUsers atomic.Pointer[map[string]*UserInfo]
+	authConfig   atomic.Pointer[AuthConfig]
-var authConfigWG sync.WaitGroup
+	authUsers    atomic.Pointer[map[string]*UserInfo]
-var stopCh chan struct{}
+	authConfigWG sync.WaitGroup
+	stopCh       chan struct{}
+)
 
 // loadAuthConfig loads and applies the config from *authConfigPath.
 // It returns bool value to identify if new config was applied.
@@ -527,16 +534,23 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		if err := ui.initURLs(); err != nil {
 			return nil, err
 		}
-		ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total`)
+
-		ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds`)
+		metricLabels, err := ui.getMetricLabels()
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse metric_labels for unauthorized_user: %w", err)
+		}
+		ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total` + metricLabels)
+		ui.backendErrors = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_request_backend_errors_total` + metricLabels)
+		ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds` + metricLabels)
 		ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
-		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total`)
+		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total` + metricLabels)
-		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`, func() float64 {
+		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`+metricLabels, func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_current`, func() float64 {
+		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_current`+metricLabels, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
+
 		tr, err := getTransport(ui.TLSInsecureSkipVerify, ui.TLSCAFile)
 		if err != nil {
 			return nil, fmt.Errorf("cannot initialize HTTP transport: %w", err)
@@ -572,25 +586,24 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 			return nil, err
 		}
 
-		name := ui.name()
+		if ui.BearerToken != "" && ui.Password != "" {
-		if ui.BearerToken != "" {
+			return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
-			if ui.Password != "" {
-				return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
-			}
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
-			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
 		}
-		if ui.Username != "" {
+
-			ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
+		metricLabels, err := ui.getMetricLabels()
-			ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
 		}
+		ui.requests = metrics.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
+		ui.backendErrors = metrics.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
+		ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
 		mcr := ui.getMaxConcurrentRequests()
 		ui.concurrencyLimitCh = make(chan struct{}, mcr)
-		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_concurrent_requests_limit_reached_total{username=%q}`, name))
+		ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_user_concurrent_requests_limit_reached_total` + metricLabels)
-		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_capacity{username=%q}`, name), func() float64 {
+		_ = metrics.GetOrCreateGauge(`vmauth_user_concurrent_requests_capacity`+metricLabels, func() float64 {
 			return float64(cap(ui.concurrencyLimitCh))
 		})
-		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_current{username=%q}`, name), func() float64 {
+		_ = metrics.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
 
@@ -606,6 +619,29 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 	return byAuthToken, nil
 }
 
+var labelNameRegexp = regexp.MustCompile("^[a-zA-Z_:.][a-zA-Z0-9_:.]*$")
+
+func (ui *UserInfo) getMetricLabels() (string, error) {
+	name := ui.name()
+	if len(name) == 0 && len(ui.MetricLabels) == 0 {
+		// fast path
+		return "", nil
+	}
+	labels := make([]string, 0, len(ui.MetricLabels)+1)
+	if len(name) > 0 {
+		labels = append(labels, fmt.Sprintf(`username=%q`, name))
+	}
+	for k, v := range ui.MetricLabels {
+		if !labelNameRegexp.MatchString(k) {
+			return "", fmt.Errorf("incorrect label name=%q, it must match regex=%q for user=%q", k, labelNameRegexp, name)
+		}
+		labels = append(labels, fmt.Sprintf(`%s=%q`, k, v))
+	}
+	sort.Strings(labels)
+	labelsStr := "{" + strings.Join(labels, ",") + "}"
+	return labelsStr, nil
+}
+
 func (ui *UserInfo) initURLs() error {
 	retryStatusCodes := defaultRetryStatusCodes.Values()
 	loadBalancingPolicy := *defaultLoadBalancingPolicy
@@ -676,7 +712,8 @@ func (ui *UserInfo) name() string {
 		return ui.Username
 	}
 	if ui.BearerToken != "" {
-		return "bearer_token"
+		h := xxhash.Sum64([]byte(ui.BearerToken))
+		return fmt.Sprintf("bearer_token:hash:%016X", h)
 	}
 	return ""
 }

app/vmauth/auth_config_test.go

@@ -229,6 +229,14 @@ users:
     url_prefix: http://foobar
     headers:
       aaa: bbb
+`)
+	// Invalid metric label name
+	f(`
+users:
+- username: foo
+  url_prefix: http://foo.bar
+  metric_labels:
+    not-prometheus-compatible: value
 `)
 }
 
@@ -489,7 +497,41 @@ users:
 			}),
 		},
 	})
-
+	// With metric_labels
+	f(`
+users:
+- username: foo-same
+  password: baz
+  url_prefix: http://foo
+  metric_labels:
+    dc: eu
+    team: dev
+- username: foo-same
+  password: bar
+  url_prefix: https://bar/x///
+  metric_labels:
+    backend_env: test
+    team: accounting
+`, map[string]*UserInfo{
+		getAuthToken("", "foo-same", "baz"): {
+			Username:  "foo-same",
+			Password:  "baz",
+			URLPrefix: mustParseURL("http://foo"),
+			MetricLabels: map[string]string{
+				"dc":   "eu",
+				"team": "dev",
+			},
+		},
+		getAuthToken("", "foo-same", "bar"): {
+			Username:  "foo-same",
+			Password:  "bar",
+			URLPrefix: mustParseURL("https://bar/x"),
+			MetricLabels: map[string]string{
+				"backend_env": "test",
+				"team":        "accounting",
+			},
+		},
+	})
 }
 
 func TestParseAuthConfigPassesTLSVerificationConfig(t *testing.T) {
@@ -526,6 +568,86 @@ unauthorized_user:
 	}
 }
 
+func TestUserInfoGetMetricLabels(t *testing.T) {
+	t.Run("empty-labels", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "user1",
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-username", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "user1",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-name", func(t *testing.T) {
+		ui := &UserInfo{
+			Name:        "user1",
+			BearerToken: "abc",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("non-empty-bearer-token", func(t *testing.T) {
+		ui := &UserInfo{
+			BearerToken: "abc",
+			MetricLabels: map[string]string{
+				"env":        "prod",
+				"datacenter": "dc1",
+			},
+		}
+		labels, err := ui.getMetricLabels()
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		labelsExpected := `{datacenter="dc1",env="prod",username="bearer_token:hash:44BC2CF5AD770999"}`
+		if labels != labelsExpected {
+			t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
+		}
+	})
+	t.Run("invalid-label", func(t *testing.T) {
+		ui := &UserInfo{
+			Username: "foo",
+			MetricLabels: map[string]string{
+				",{": "aaaa",
+			},
+		}
+		_, err := ui.getMetricLabels()
+		if err == nil {
+			t.Fatalf("expecting non-nil error")
+		}
+	})
+}
+
 func isSetBool(boolP *bool, expectedValue bool) bool {
 	if boolP == nil {
 		return false

app/vmauth/example_config.yml

@@ -10,6 +10,11 @@ users:
 - bearer_token: "XXXX"
   url_prefix: "http://localhost:8428"
 
+  # Adds labels to the exported metrics for given user section
+  # label name must be prometheus compatible and match regex: `^[a-zA-Z_:.][a-zA-Z0-9_:.]*$`
+  metric_labels:
+    backend_dc: eu
+    access_team: dev
   # Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
   # The `X-Scope-OrgID: foobar` http header is added to every proxied request.
   # The `X-Server-Hostname:` http header is removed from the proxied response.

app/vmauth/main.go

@@ -150,12 +150,20 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		if err := ui.beginConcurrencyLimit(); err != nil {
 			handleConcurrencyLimitError(w, r, err)
 			<-concurrencyLimitCh
+
+			// Requests failed because of concurrency limit must be counted as errors,
+			// since this usually means the backend cannot keep up with the current load.
+			ui.backendErrors.Inc()
 			return
 		}
 	default:
 		concurrentRequestsLimitReached.Inc()
 		err := fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh))
 		handleConcurrencyLimitError(w, r, err)
+
+		// Requests failed because of concurrency limit must be counted as errors,
+		// since this usually means the backend cannot keep up with the current load.
+		ui.backendErrors.Inc()
 		return
 	}
 	processRequest(w, r, ui)
@@ -201,7 +209,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		} else { // Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
 		}
-		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui.httpTransport)
+		ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
 		bu.put()
 		if ok {
 			return
@@ -213,15 +221,16 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		StatusCode: http.StatusServiceUnavailable,
 	}
 	httpserver.Errorf(w, r, "%s", err)
+	ui.backendErrors.Inc()
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, transport *http.Transport) bool {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) bool {
 	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := sanitizeRequestHeaders(r)
 	req.URL = targetURL
 	req.Host = targetURL.Host
 	updateHeadersByConfig(req.Header, hc.RequestHeaders)
-	res, err := transport.RoundTrip(req)
+	res, err := ui.httpTransport.RoundTrip(req)
 	rtb, rtbOK := req.Body.(*readTrackingBody)
 	if err != nil {
 		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
@@ -229,6 +238,10 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 			remoteAddr := httpserver.GetQuotedRemoteAddr(r)
 			requestURI := httpserver.GetRequestURI(r)
 			logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
+			if errors.Is(err, context.DeadlineExceeded) {
+				// Timed out request must be counted as errors, since this usually means that the backend is slow.
+				ui.backendErrors.Inc()
+			}
 			return true
 		}
 		if !rtbOK || !rtb.canRetry() {
@@ -238,6 +251,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
 				StatusCode: http.StatusServiceUnavailable,
 			}
 			httpserver.Errorf(w, r, "%s", err)
+			ui.backendErrors.Inc()
 			return true
 		}
 		// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
@@ -393,8 +407,10 @@ func getTransport(insecureSkipVerifyP *bool, caFile string) (*http.Transport, er
 	return tr, nil
 }
 
-var transportMap = make(map[string]*http.Transport)
+var (
-var transportMapLock sync.Mutex
+	transportMap     = make(map[string]*http.Transport)
+	transportMapLock sync.Mutex
+)
 
 func appendTransportKey(dst []byte, insecureSkipVerify bool, caFile string) []byte {
 	dst = encoding.MarshalBool(dst, insecureSkipVerify)

docs/CHANGELOG.md

@@ -41,6 +41,8 @@ The sandbox cluster installation is running under the constant load generated by
   - `-remoteRead.oauth2.endpointParams` for `-remoteRead.url`
   - `-remoteWrite.oauth2.endpointParams` for `-remoteWrite.url`
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy incoming requests to different backends based on the requested host via `src_hosts` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth.html#generic-http-proxy-for-different-backends).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): expose `vmauth_user_request_backend_errors_total` and `vmauth_unauthorized_user_request_backend_errors_total` [metrics](https://docs.victoriametrics.com/vmauth.html#monitoring), which track the number of failed requests because of backend errors. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5565).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add an ability to specify additional labels for [per-user metrics](https://docs.victoriametrics.com/vmauth.html#monitoring) via `metric_labels` section. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5565).
 * FEATURE: all VictoriaMetrics components: break HTTP client connection if an error occurs after the server at `-httpListenAddr` already sent response status code. Previously such an error couldn't be detected at client side. Now the client will get an error about invalid chunked response. The error message is simultaneously written to the server log and in the last line of the response. This should help detecting errors when migrating data between VictoriaMetrics instances by [vmctl](https://docs.victoriametrics.com/vmctl.html). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5645).
 * FEATURE: all VictoriaMetrics components: add ability to specify arbitrary HTTP headers to send with every request to `-pushmetrics.url`. See [`push metrics` docs](https://docs.victoriametrics.com/#push-metrics).
 * FEATURE: all VictoriaMetrics components: add `-metrics.exposeMetadata` command-line flag, which allows displaying `TYPE` and `HELP` metadata at `/metrics` page exposed at `-httpListenAddr`. This may be needed when the `/metrics` page is scraped by collector, which requires the `TYPE` and `HELP` metadata such as [Google Cloud Managed Prometheus](https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type).

docs/vmauth.md

@@ -455,7 +455,7 @@ The following [metrics](#monitoring) related to concurrency limits are exposed b
   because of the global concurrency limit has been reached.
 - `vmauth_user_concurrent_requests_capacity{username="..."}` - the limit on the number of concurrent requests for the given `username`.
 - `vmauth_user_concurrent_requests_current{username="..."}` - the current number of concurrent requests for the given `username`.
-- `vmauth_user_concurrent_requests_limit_reached_total{username="foo"}` - the number of requests rejected with `429 Too Many Requests` error
+- `vmauth_user_concurrent_requests_limit_reached_total{username="..."}` - the number of requests rejected with `429 Too Many Requests` error
   because of the concurrency limit has been reached for the given `username`.
 - `vmauth_unauthorized_user_concurrent_requests_capacity` - the limit on the number of concurrent requests for unauthorized users (if `unauthorized_user` section is used).
 - `vmauth_unauthorized_user_concurrent_requests_current` - the current number of concurrent requests for unauthorized users (if `unauthorized_user` section is used).
@@ -465,10 +465,10 @@ The following [metrics](#monitoring) related to concurrency limits are exposed b
 ## Backend TLS setup
 
 By default `vmauth` uses system settings when performing requests to HTTPS backends specified via `url_prefix` option
-in the [`-auth.config`](https://docs.victoriametrics.com/vmauth.html#auth-config). These settings can be overridden with the following command-line flags:
+in the [`-auth.config`](#auth-config). These settings can be overridden with the following command-line flags:
 
 - `-backend.tlsInsecureSkipVerify` allows skipping TLS verification when connecting to HTTPS backends.
-  This global setting can be overridden at per-user level inside [`-auth.config`](https://docs.victoriametrics.com/vmauth.html#auth-config)
+  This global setting can be overridden at per-user level inside [`-auth.config`](#auth-config)
   via `tls_insecure_skip_verify` option. For example:
 
   ```yml
@@ -479,7 +479,7 @@ in the [`-auth.config`](https://docs.victoriametrics.com/vmauth.html#auth-config
 
 - `-backend.tlsCAFile` allows specifying the path to TLS Root CA, which will be used for TLS verification when connecting to HTTPS backends.
   The `-backend.tlsCAFile` may point either to local file or to `http` / `https` url.
-  This global setting can be overridden at per-user level inside [`-auth.config`](https://docs.victoriametrics.com/vmauth.html#auth-config)
+  This global setting can be overridden at per-user level inside [`-auth.config`](#auth-config)
   via `tls_ca_file` option. For example:
 
   ```yml
@@ -695,12 +695,21 @@ It is recommended protecting the following endpoints with authKeys:
 `vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page
 either via [vmagent](https://docs.victoriametrics.com/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later.
 
-`vmauth` exports `vmauth_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric 
+`vmauth` exports the following metrics per each defined user in [`-auth.config`](#auth-config):
-and `vmauth_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) metric 
+
-with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file.
+* `vmauth_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of requests served for the given `username`
-It is possible to override or hide the value in the label by specifying `name` field. 
+* `vmauth_user_request_backend_errors_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of request errors for the given `username`
-For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` 
+* `vmauth_user_request_duration_seconds` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) - the duration of requests for the given `username`
-instead of `vmauth_user_requests_total{username="secret_user"}`:
+* `vmauth_user_concurrent_requests_limit_reached_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of failed requests
+  for the given `username` because of exceeded [concurrency limits](#concurrency-limiting)
+* `vmauth_user_concurrent_requests_capacity` [gauge](https://docs.victoriametrics.com/keyConcepts.html#gauge) - the maximum number of [concurrent requests](#concurrency-limiting)
+  for the given `username`
+* `vmauth_user_concurrent_requests_current` [gauge](https://docs.victoriametrics.com/keyConcepts.html#gauge) - the current number of [concurrent requests](#concurrency-limiting)
+  for the given `username`
+
+By default, per-user metrics contain only `username` label. This label is set to `username` field value at the corresponding user section in the [`-auth.config`](#auth-config) file.
+It is possible to override the `username` label value by specifying `name` field additionally to `username` field.
+For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` instead of `vmauth_user_requests_total{username="secret_user"}`:
 
 ```yml
 users:
@@ -709,10 +718,28 @@ users:
   # other config options here
 ```
 
-For unauthorized users `vmauth` exports `vmauth_unauthorized_user_requests_total` 
+Additional labels for per-user metrics can be specified via `metric_labels` section. For example, the following config
-[counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric and 
+defines `{dc="eu",team="dev"}` labels additionally to `username="foobar"` label:
-`vmauth_unauthorized_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary)
+
-metric without label (if `unauthorized_user` section of config is used).
+```yml
+users:
+- username: "foobar"
+  metric_labels:
+   dc: eu
+   team: dev
+  # other config options here
+```
+
+`vmauth` exports the following metrics if `unauthorized_user` section is defined in [`-auth.config`](#auth-config):
+
+* `vmauth_unauthorized_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of unauthorized requests served
+* `vmauth_unauthorized_user_request_backend_errors_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of unauthorized request errors
+* `vmauth_unauthorized_user_request_duration_seconds` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) - the duration of unauthorized requests
+* `vmauth_unauthorized_user_concurrent_requests_limit_reached_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of failed unauthorized requests
+  because of exceeded [concurrency limits](#concurrency-limiting)
+* `vmauth_unauthorized_user_concurrent_requests_capacity` [gauge](https://docs.victoriametrics.com/keyConcepts.html#gauge) - the maximum number
+  of [concurrent unauthorized requests](#concurrency-limiting)
+* `vmauth_user_concurrent_requests_current` [gauge](https://docs.victoriametrics.com/keyConcepts.html#gauge) - the current number of [concurrent unauthorized requests](#concurrency-limiting)
 
 ## How to build from sources