Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-23 12:31:07 +01:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

app/vmauth: adds metric_labels and backend_errors counter (#5585)

Browse Source 
* app/vmauth: adds metric_labels and backend_errors counter
it must improve observability for user requests with new metric - per user backend errors counter.
it's needed to calculate requests fail rate to the configured backends.
metric_labels configuration allows to perform additional aggregations on top of multiple users from configuration section.
It could be multiple clients or clients with separate read/write tokens
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5565

* wip

---------

Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>
This commit is contained in:
Nikolay 2024-01-21 03:40:52 +01:00 committed by GitHub
parent 3ea1294ad2
commit b3598ba2c1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 252 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
app/vmauth/auth_config.go
View File

@ -9,6 +9,7 @@ import (
"net/url"
"os"
"regexp"
"sort"
"strconv"
"strings"
"sync"
@ -16,6 +17,7 @@ import (
"time"
"github.com/VictoriaMetrics/metrics"
"github.com/cespare/xxhash/v2"
"gopkg.in/yaml.v2"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
@ -60,12 +62,15 @@ type UserInfo struct {
TLSInsecureSkipVerify *bool `yaml:"tls_insecure_skip_verify,omitempty"`
TLSCAFile string `yaml:"tls_ca_file,omitempty"`
MetricLabels map[string]string `yaml:"metric_labels,omitempty"`
concurrencyLimitCh chan struct{}
concurrencyLimitReached *metrics.Counter
httpTransport *http.Transport
requests *metrics.Counter
backendErrors *metrics.Counter
requestsDuration *metrics.Summary
}
@ -462,10 +467,12 @@ func authConfigReloader(sighupCh <-chan os.Signal) {
// authConfigData needs to be updated each time authConfig is updated.
var authConfigData atomic.Pointer[[]byte]
var authConfig atomic.Pointer[AuthConfig]
var authUsers atomic.Pointer[map[string]*UserInfo]
var authConfigWG sync.WaitGroup
var stopCh chan struct{}
var (
authConfig atomic.Pointer[AuthConfig]
authUsers atomic.Pointer[map[string]*UserInfo]
authConfigWG sync.WaitGroup
stopCh chan struct{}
)
// loadAuthConfig loads and applies the config from *authConfigPath.
// It returns bool value to identify if new config was applied.
@ -527,16 +534,23 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
if err := ui.initURLs(); err != nil {
return nil, err
}
ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total`)
ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds`)
metricLabels, err := ui.getMetricLabels()
if err != nil {
return nil, fmt.Errorf("cannot parse metric_labels for unauthorized_user: %w", err)
}
ui.requests = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_requests_total` + metricLabels)
ui.backendErrors = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_request_backend_errors_total` + metricLabels)
ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_unauthorized_user_request_duration_seconds` + metricLabels)
ui.concurrencyLimitCh = make(chan struct{}, ui.getMaxConcurrentRequests())
ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total`)
_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`, func() float64 {
ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_unauthorized_user_concurrent_requests_limit_reached_total` + metricLabels)
_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_capacity`+metricLabels, func() float64 {
return float64(cap(ui.concurrencyLimitCh))
})
_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_current`, func() float64 {
_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_current`+metricLabels, func() float64 {
return float64(len(ui.concurrencyLimitCh))
})
tr, err := getTransport(ui.TLSInsecureSkipVerify, ui.TLSCAFile)
if err != nil {
return nil, fmt.Errorf("cannot initialize HTTP transport: %w", err)
@ -572,25 +586,24 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
return nil, err
}
name := ui.name()
if ui.BearerToken != "" {
if ui.Password != "" {
return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
}
ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
if ui.BearerToken != "" && ui.Password != "" {
return nil, fmt.Errorf("password shouldn't be set for bearer_token %q", ui.BearerToken)
}
if ui.Username != "" {
ui.requests = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_requests_total{username=%q}`, name))
ui.requestsDuration = metrics.GetOrCreateSummary(fmt.Sprintf(`vmauth_user_request_duration_seconds{username=%q}`, name))
metricLabels, err := ui.getMetricLabels()
if err != nil {
return nil, fmt.Errorf("cannot parse metric_labels: %w", err)
}
ui.requests = metrics.GetOrCreateCounter(`vmauth_user_requests_total` + metricLabels)
ui.backendErrors = metrics.GetOrCreateCounter(`vmauth_user_request_backend_errors_total` + metricLabels)
ui.requestsDuration = metrics.GetOrCreateSummary(`vmauth_user_request_duration_seconds` + metricLabels)
mcr := ui.getMaxConcurrentRequests()
ui.concurrencyLimitCh = make(chan struct{}, mcr)
ui.concurrencyLimitReached = metrics.GetOrCreateCounter(fmt.Sprintf(`vmauth_user_concurrent_requests_limit_reached_total{username=%q}`, name))
_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_capacity{username=%q}`, name), func() float64 {
ui.concurrencyLimitReached = metrics.GetOrCreateCounter(`vmauth_user_concurrent_requests_limit_reached_total` + metricLabels)
_ = metrics.GetOrCreateGauge(`vmauth_user_concurrent_requests_capacity`+metricLabels, func() float64 {
return float64(cap(ui.concurrencyLimitCh))
})
_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_current{username=%q}`, name), func() float64 {
_ = metrics.GetOrCreateGauge(`vmauth_user_concurrent_requests_current`+metricLabels, func() float64 {
return float64(len(ui.concurrencyLimitCh))
})
@ -606,6 +619,29 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
return byAuthToken, nil
}
var labelNameRegexp = regexp.MustCompile("^[a-zA-Z_:.][a-zA-Z0-9_:.]*$")
func (ui *UserInfo) getMetricLabels() (string, error) {
name := ui.name()
if len(name) == 0 && len(ui.MetricLabels) == 0 {
// fast path
return "", nil
}
labels := make([]string, 0, len(ui.MetricLabels)+1)
if len(name) > 0 {
labels = append(labels, fmt.Sprintf(`username=%q`, name))
}
for k, v := range ui.MetricLabels {
if !labelNameRegexp.MatchString(k) {
return "", fmt.Errorf("incorrect label name=%q, it must match regex=%q for user=%q", k, labelNameRegexp, name)
}
labels = append(labels, fmt.Sprintf(`%s=%q`, k, v))
}
sort.Strings(labels)
labelsStr := "{" + strings.Join(labels, ",") + "}"
return labelsStr, nil
}
func (ui *UserInfo) initURLs() error {
retryStatusCodes := defaultRetryStatusCodes.Values()
loadBalancingPolicy := *defaultLoadBalancingPolicy
@ -676,7 +712,8 @@ func (ui *UserInfo) name() string {
return ui.Username
}
if ui.BearerToken != "" {
return "bearer_token"
h := xxhash.Sum64([]byte(ui.BearerToken))
return fmt.Sprintf("bearer_token:hash:%016X", h)
}
return ""
}

124
app/vmauth/auth_config_test.go
View File

@ -229,6 +229,14 @@ users:
url_prefix: http://foobar
headers:
aaa: bbb
`)
// Invalid metric label name
f(`
users:
- username: foo
url_prefix: http://foo.bar
metric_labels:
not-prometheus-compatible: value
`)
}
@ -489,7 +497,41 @@ users:
}),
},
})
// With metric_labels
f(`
users:
- username: foo-same
password: baz
url_prefix: http://foo
metric_labels:
dc: eu
team: dev
- username: foo-same
password: bar
url_prefix: https://bar/x///
metric_labels:
backend_env: test
team: accounting
`, map[string]*UserInfo{
getAuthToken("", "foo-same", "baz"): {
Username: "foo-same",
Password: "baz",
URLPrefix: mustParseURL("http://foo"),
MetricLabels: map[string]string{
"dc": "eu",
"team": "dev",
},
},
getAuthToken("", "foo-same", "bar"): {
Username: "foo-same",
Password: "bar",
URLPrefix: mustParseURL("https://bar/x"),
MetricLabels: map[string]string{
"backend_env": "test",
"team": "accounting",
},
},
})
}
func TestParseAuthConfigPassesTLSVerificationConfig(t *testing.T) {
@ -526,6 +568,86 @@ unauthorized_user:
}
}
func TestUserInfoGetMetricLabels(t *testing.T) {
t.Run("empty-labels", func(t *testing.T) {
ui := &UserInfo{
Username: "user1",
}
labels, err := ui.getMetricLabels()
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
labelsExpected := `{username="user1"}`
if labels != labelsExpected {
t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
}
})
t.Run("non-empty-username", func(t *testing.T) {
ui := &UserInfo{
Username: "user1",
MetricLabels: map[string]string{
"env": "prod",
"datacenter": "dc1",
},
}
labels, err := ui.getMetricLabels()
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
if labels != labelsExpected {
t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
}
})
t.Run("non-empty-name", func(t *testing.T) {
ui := &UserInfo{
Name: "user1",
BearerToken: "abc",
MetricLabels: map[string]string{
"env": "prod",
"datacenter": "dc1",
},
}
labels, err := ui.getMetricLabels()
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
labelsExpected := `{datacenter="dc1",env="prod",username="user1"}`
if labels != labelsExpected {
t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
}
})
t.Run("non-empty-bearer-token", func(t *testing.T) {
ui := &UserInfo{
BearerToken: "abc",
MetricLabels: map[string]string{
"env": "prod",
"datacenter": "dc1",
},
}
labels, err := ui.getMetricLabels()
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
labelsExpected := `{datacenter="dc1",env="prod",username="bearer_token:hash:44BC2CF5AD770999"}`
if labels != labelsExpected {
t.Fatalf("unexpected labels; got %s; want %s", labels, labelsExpected)
}
})
t.Run("invalid-label", func(t *testing.T) {
ui := &UserInfo{
Username: "foo",
MetricLabels: map[string]string{
",{": "aaaa",
},
}
_, err := ui.getMetricLabels()
if err == nil {
t.Fatalf("expecting non-nil error")
}
})
}
func isSetBool(boolP *bool, expectedValue bool) bool {
if boolP == nil {
return false

5
app/vmauth/example_config.yml
View File

@ -10,6 +10,11 @@ users:
- bearer_token: "XXXX"
url_prefix: "http://localhost:8428"
# Adds labels to the exported metrics for given user section
# label name must be prometheus compatible and match regex: `^[a-zA-Z_:.][a-zA-Z0-9_:.]*$`
metric_labels:
backend_dc: eu
access_team: dev
# Requests with the 'Authorization: Bearer YYY' header are proxied to http://localhost:8428 ,
# The `X-Scope-OrgID: foobar` http header is added to every proxied request.
# The `X-Server-Hostname:` http header is removed from the proxied response.

26
app/vmauth/main.go
View File

@ -150,12 +150,20 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
if err := ui.beginConcurrencyLimit(); err != nil {
handleConcurrencyLimitError(w, r, err)
<-concurrencyLimitCh
// Requests failed because of concurrency limit must be counted as errors,
// since this usually means the backend cannot keep up with the current load.
ui.backendErrors.Inc()
return
}
default:
concurrentRequestsLimitReached.Inc()
err := fmt.Errorf("cannot serve more than -maxConcurrentRequests=%d concurrent requests", cap(concurrencyLimitCh))
handleConcurrencyLimitError(w, r, err)
// Requests failed because of concurrency limit must be counted as errors,
// since this usually means the backend cannot keep up with the current load.
ui.backendErrors.Inc()
return
}
processRequest(w, r, ui)
@ -201,7 +209,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
} else { // Update path for regular routes.
targetURL = mergeURLs(targetURL, u, up.dropSrcPathPrefixParts)
}
ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui.httpTransport)
ok := tryProcessingRequest(w, r, targetURL, hc, up.retryStatusCodes, ui)
bu.put()
if ok {
return
@ -213,15 +221,16 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
StatusCode: http.StatusServiceUnavailable,
}
httpserver.Errorf(w, r, "%s", err)
ui.backendErrors.Inc()
}
func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, transport *http.Transport) bool {
func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, ui *UserInfo) bool {
// This code has been copied from net/http/httputil/reverseproxy.go
req := sanitizeRequestHeaders(r)
req.URL = targetURL
req.Host = targetURL.Host
updateHeadersByConfig(req.Header, hc.RequestHeaders)
res, err := transport.RoundTrip(req)
res, err := ui.httpTransport.RoundTrip(req)
rtb, rtbOK := req.Body.(*readTrackingBody)
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
@ -229,6 +238,10 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
remoteAddr := httpserver.GetQuotedRemoteAddr(r)
requestURI := httpserver.GetRequestURI(r)
logger.Warnf("remoteAddr: %s; requestURI: %s; error when proxying response body from %s: %s", remoteAddr, requestURI, targetURL, err)
if errors.Is(err, context.DeadlineExceeded) {
// Timed out request must be counted as errors, since this usually means that the backend is slow.
ui.backendErrors.Inc()
}
return true
}
if !rtbOK || !rtb.canRetry() {
@ -238,6 +251,7 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
StatusCode: http.StatusServiceUnavailable,
}
httpserver.Errorf(w, r, "%s", err)
ui.backendErrors.Inc()
return true
}
// Retry the request if its body wasn't read yet. This usually means that the backend isn't reachable.
@ -393,8 +407,10 @@ func getTransport(insecureSkipVerifyP *bool, caFile string) (*http.Transport, er
return tr, nil
}
var transportMap = make(map[string]*http.Transport)
var transportMapLock sync.Mutex
var (
transportMap = make(map[string]*http.Transport)
transportMapLock sync.Mutex
)
func appendTransportKey(dst []byte, insecureSkipVerify bool, caFile string) []byte {
dst = encoding.MarshalBool(dst, insecureSkipVerify)

2
docs/CHANGELOG.md
View File

@ -41,6 +41,8 @@ The sandbox cluster installation is running under the constant load generated by
- `-remoteRead.oauth2.endpointParams` for `-remoteRead.url`
- `-remoteWrite.oauth2.endpointParams` for `-remoteWrite.url`
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy incoming requests to different backends based on the requested host via `src_hosts` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth.html#generic-http-proxy-for-different-backends).
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): expose `vmauth_user_request_backend_errors_total` and `vmauth_unauthorized_user_request_backend_errors_total` [metrics](https://docs.victoriametrics.com/vmauth.html#monitoring), which track the number of failed requests because of backend errors. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5565).
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add an ability to specify additional labels for [per-user metrics](https://docs.victoriametrics.com/vmauth.html#monitoring) via `metric_labels` section. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5565).
* FEATURE: all VictoriaMetrics components: break HTTP client connection if an error occurs after the server at `-httpListenAddr` already sent response status code. Previously such an error couldn't be detected at client side. Now the client will get an error about invalid chunked response. The error message is simultaneously written to the server log and in the last line of the response. This should help detecting errors when migrating data between VictoriaMetrics instances by [vmctl](https://docs.victoriametrics.com/vmctl.html). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5645).
* FEATURE: all VictoriaMetrics components: add ability to specify arbitrary HTTP headers to send with every request to `-pushmetrics.url`. See [`push metrics` docs](https://docs.victoriametrics.com/#push-metrics).
* FEATURE: all VictoriaMetrics components: add `-metrics.exposeMetadata` command-line flag, which allows displaying `TYPE` and `HELP` metadata at `/metrics` page exposed at `-httpListenAddr`. This may be needed when the `/metrics` page is scraped by collector, which requires the `TYPE` and `HELP` metadata such as [Google Cloud Managed Prometheus](https://cloud.google.com/stackdriver/docs/managed-prometheus/troubleshooting#missing-metric-type).

55
docs/vmauth.md
View File

@ -455,7 +455,7 @@ The following [metrics](#monitoring) related to concurrency limits are exposed b
because of the global concurrency limit has been reached.
- `vmauth_user_concurrent_requests_capacity{username="..."}` - the limit on the number of concurrent requests for the given `username`.
- `vmauth_user_concurrent_requests_current{username="..."}` - the current number of concurrent requests for the given `username`.
- `vmauth_user_concurrent_requests_limit_reached_total{username="foo"}` - the number of requests rejected with `429 Too Many Requests` error
- `vmauth_user_concurrent_requests_limit_reached_total{username="..."}` - the number of requests rejected with `429 Too Many Requests` error
because of the concurrency limit has been reached for the given `username`.
- `vmauth_unauthorized_user_concurrent_requests_capacity` - the limit on the number of concurrent requests for unauthorized users (if `unauthorized_user` section is used).
- `vmauth_unauthorized_user_concurrent_requests_current` - the current number of concurrent requests for unauthorized users (if `unauthorized_user` section is used).
@ -465,10 +465,10 @@ The following [metrics](#monitoring) related to concurrency limits are exposed b
## Backend TLS setup
By default `vmauth` uses system settings when performing requests to HTTPS backends specified via `url_prefix` option
in the [`-auth.config`](https://docs.victoriametrics.com/vmauth.html#auth-config). These settings can be overridden with the following command-line flags:
in the [`-auth.config`](#auth-config). These settings can be overridden with the following command-line flags:
- `-backend.tlsInsecureSkipVerify` allows skipping TLS verification when connecting to HTTPS backends.
This global setting can be overridden at per-user level inside [`-auth.config`](https://docs.victoriametrics.com/vmauth.html#auth-config)
This global setting can be overridden at per-user level inside [`-auth.config`](#auth-config)
via `tls_insecure_skip_verify` option. For example:
```yml
@ -479,7 +479,7 @@ in the [`-auth.config`](https://docs.victoriametrics.com/vmauth.html#auth-config
- `-backend.tlsCAFile` allows specifying the path to TLS Root CA, which will be used for TLS verification when connecting to HTTPS backends.
The `-backend.tlsCAFile` may point either to local file or to `http` / `https` url.
This global setting can be overridden at per-user level inside [`-auth.config`](https://docs.victoriametrics.com/vmauth.html#auth-config)
This global setting can be overridden at per-user level inside [`-auth.config`](#auth-config)
via `tls_ca_file` option. For example:
```yml
@ -695,12 +695,21 @@ It is recommended protecting the following endpoints with authKeys:
`vmauth` exports various metrics in Prometheus exposition format at `http://vmauth-host:8427/metrics` page. It is recommended setting up regular scraping of this page
either via [vmagent](https://docs.victoriametrics.com/vmagent.html) or via Prometheus, so the exported metrics could be analyzed later.
`vmauth` exports `vmauth_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric
and `vmauth_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) metric
with `username` label. The `username` label value equals to `username` field value set in the `-auth.config` file.
It is possible to override or hide the value in the label by specifying `name` field.
For example, the following config will result in `vmauth_user_requests_total{username="foobar"}`
instead of `vmauth_user_requests_total{username="secret_user"}`:
`vmauth` exports the following metrics per each defined user in [`-auth.config`](#auth-config):
* `vmauth_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of requests served for the given `username`
* `vmauth_user_request_backend_errors_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of request errors for the given `username`
* `vmauth_user_request_duration_seconds` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) - the duration of requests for the given `username`
* `vmauth_user_concurrent_requests_limit_reached_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of failed requests
for the given `username` because of exceeded [concurrency limits](#concurrency-limiting)
* `vmauth_user_concurrent_requests_capacity` [gauge](https://docs.victoriametrics.com/keyConcepts.html#gauge) - the maximum number of [concurrent requests](#concurrency-limiting)
for the given `username`
* `vmauth_user_concurrent_requests_current` [gauge](https://docs.victoriametrics.com/keyConcepts.html#gauge) - the current number of [concurrent requests](#concurrency-limiting)
for the given `username`
By default, per-user metrics contain only `username` label. This label is set to `username` field value at the corresponding user section in the [`-auth.config`](#auth-config) file.
It is possible to override the `username` label value by specifying `name` field additionally to `username` field.
For example, the following config will result in `vmauth_user_requests_total{username="foobar"}` instead of `vmauth_user_requests_total{username="secret_user"}`:
```yml
users:
@ -709,10 +718,28 @@ users:
# other config options here
```
For unauthorized users `vmauth` exports `vmauth_unauthorized_user_requests_total`
[counter](https://docs.victoriametrics.com/keyConcepts.html#counter) metric and
`vmauth_unauthorized_user_request_duration_seconds_*` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary)
metric without label (if `unauthorized_user` section of config is used).
Additional labels for per-user metrics can be specified via `metric_labels` section. For example, the following config
defines `{dc="eu",team="dev"}` labels additionally to `username="foobar"` label:
```yml
users:
- username: "foobar"
metric_labels:
dc: eu
team: dev
# other config options here
```
`vmauth` exports the following metrics if `unauthorized_user` section is defined in [`-auth.config`](#auth-config):
* `vmauth_unauthorized_user_requests_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of unauthorized requests served
* `vmauth_unauthorized_user_request_backend_errors_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of unauthorized request errors
* `vmauth_unauthorized_user_request_duration_seconds` [summary](https://docs.victoriametrics.com/keyConcepts.html#summary) - the duration of unauthorized requests
* `vmauth_unauthorized_user_concurrent_requests_limit_reached_total` [counter](https://docs.victoriametrics.com/keyConcepts.html#counter) - the number of failed unauthorized requests
because of exceeded [concurrency limits](#concurrency-limiting)
* `vmauth_unauthorized_user_concurrent_requests_capacity` [gauge](https://docs.victoriametrics.com/keyConcepts.html#gauge) - the maximum number
of [concurrent unauthorized requests](#concurrency-limiting)
* `vmauth_user_concurrent_requests_current` [gauge](https://docs.victoriametrics.com/keyConcepts.html#gauge) - the current number of [concurrent unauthorized requests](#concurrency-limiting)
## How to build from sources