added default_url field in vmauth users config (#4084) (#4156)

* added default url field in vmauth users config (#4084)

---------

Signed-off-by: Alexander Marshalov <_@marshalov.org>

app/vmauth/README.md

@@ -185,6 +185,34 @@ users:
     headers:
     - "X-Scope-OrgID: abc"
 
+  # A single user for querying and inserting data:
+  # - Requests to http://vmauth:8427/api/v1/query, http://vmauth:8427/api/v1/query_range
+  #   and http://vmauth:8427/api/v1/label/<label_name>/values are proxied to the following urls in a round-robin manner:
+  #     - http://vmselect1:8481/select/42/prometheus
+  #     - http://vmselect2:8481/select/42/prometheus
+  #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect1:8480/select/42/prometheus/api/v1/query
+  #   or to http://vmselect2:8480/select/42/prometheus/api/v1/query .
+  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
+  # The requests which do not match `src_paths` from the `url_map` will be proxied to the urls rom `default_url` 
+  # in a round-robin manner (with request path in `request_path` query param).
+  # For example, request to http://vmauth:8427/non/existing/path will be proxied:
+  #  - to http://default1:8888/process?request_path=/non/existing/path
+  #  - or http://default2:8888/process?request_path=/non/existing/path
+- username: "foobar"
+  url_map:
+    - src_paths:
+        - "/api/v1/query"
+        - "/api/v1/query_range"
+        - "/api/v1/label/[^/]+/values"
+      url_prefix:
+        - "http://vmselect1:8481/select/42/prometheus"
+        - "http://vmselect2:8481/select/42/prometheus"
+    - src_paths: ["/api/v1/write"]
+      url_prefix: "http://vminsert:8480/insert/42/prometheus"
+  default_url: 
+    - "http://default1:8888/process"
+    - "http://default2:8888/process"
+
 # This requests will be executed for requests without Authorization header.
 # For instance, http://vmauth:8427/api/v1/query will be proxied to http://vmselect1:8481/select/0/prometheus/api/v1/query
 unauthorized_user:

app/vmauth/auth_config.go

@@ -45,6 +45,7 @@ type UserInfo struct {
 	URLMaps               []URLMap   `yaml:"url_map,omitempty"`
 	Headers               []Header   `yaml:"headers,omitempty"`
 	MaxConcurrentRequests int        `yaml:"max_concurrent_requests,omitempty"`
+	DefaultURL            *URLPrefix `yaml:"default_url,omitempty"`
 
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
@@ -416,6 +417,11 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 				return nil, err
 			}
 		}
+		if ui.DefaultURL != nil {
+			if err := ui.DefaultURL.sanitize(); err != nil {
+				return nil, err
+			}
+		}
 		for _, e := range ui.URLMaps {
 			if len(e.SrcPaths) == 0 {
 				return nil, fmt.Errorf("missing `src_paths` in `url_map`")

app/vmauth/auth_config_test.go

@@ -360,6 +360,83 @@ users:
 			URLPrefix: mustParseURL("https://bar/x"),
 		},
 	})
+	// with default url
+	f(`
+users:
+- bearer_token: foo
+  url_map:
+  - src_paths: ["/api/v1/query","/api/v1/query_range","/api/v1/label/[^./]+/.+"]
+    url_prefix: http://vmselect/select/0/prometheus
+  - src_paths: ["/api/v1/write"]
+    url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
+    headers:
+    - "foo: bar"
+    - "xxx: y"
+  default_url: 
+  - http://default1/select/0/prometheus
+  - http://default2/select/0/prometheus
+`, map[string]*UserInfo{
+		getAuthToken("foo", "", ""): {
+			BearerToken: "foo",
+			URLMaps: []URLMap{
+				{
+					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
+				},
+				{
+					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
+					URLPrefix: mustParseURLs([]string{
+						"http://vminsert1/insert/0/prometheus",
+						"http://vminsert2/insert/0/prometheus",
+					}),
+					Headers: []Header{
+						{
+							Name:  "foo",
+							Value: "bar",
+						},
+						{
+							Name:  "xxx",
+							Value: "y",
+						},
+					},
+				},
+			},
+			DefaultURL: mustParseURLs([]string{
+				"http://default1/select/0/prometheus",
+				"http://default2/select/0/prometheus",
+			}),
+		},
+		getAuthToken("", "foo", ""): {
+			BearerToken: "foo",
+			URLMaps: []URLMap{
+				{
+					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
+				},
+				{
+					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
+					URLPrefix: mustParseURLs([]string{
+						"http://vminsert1/insert/0/prometheus",
+						"http://vminsert2/insert/0/prometheus",
+					}),
+					Headers: []Header{
+						{
+							Name:  "foo",
+							Value: "bar",
+						},
+						{
+							Name:  "xxx",
+							Value: "y",
+						},
+					},
+				},
+			},
+			DefaultURL: mustParseURLs([]string{
+				"http://default1/select/0/prometheus",
+				"http://default2/select/0/prometheus",
+			}),
+		},
+	})
 
 }
 

app/vmauth/main.go

@@ -147,15 +147,30 @@ func processUserRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 
 func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	u := normalizeURL(r.URL)
-	up, headers, err := ui.getURLPrefixAndHeaders(u)
+	up, headers := ui.getURLPrefixAndHeaders(u)
-	if err != nil {
+	isDefault := false
-		httpserver.Errorf(w, r, "cannot determine targetURL: %s", err)
+	if up == nil {
+		missingRouteRequests.Inc()
+		if ui.DefaultURL == nil {
+			httpserver.Errorf(w, r, "missing route for %q", u.String())
 			return
 		}
+		up, headers = ui.DefaultURL, ui.Headers
+		isDefault = true
+	}
+
 	maxAttempts := up.getBackendsCount()
 	for i := 0; i < maxAttempts; i++ {
 		bu := up.getLeastLoadedBackendURL()
-		targetURL := mergeURLs(bu.url, u)
+		targetURL := bu.url
+		// Don't change path and add request_path query param for default route.
+		if isDefault {
+			query := targetURL.Query()
+			query.Set("request_path", u.Path)
+			targetURL.RawQuery = query.Encode()
+		} else { // Update path for regular routes.
+			targetURL = mergeURLs(targetURL, u)
+		}
 		ok := tryProcessingRequest(w, r, targetURL, headers)
 		bu.put()
 		if ok {
@@ -163,7 +178,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		}
 		bu.setBroken()
 	}
-	err = &httpserver.ErrorWithStatusCode{
+	err := &httpserver.ErrorWithStatusCode{
 		Err:        fmt.Errorf("all the backends for the user %q are unavailable", ui.name()),
 		StatusCode: http.StatusServiceUnavailable,
 	}

app/vmauth/target_url.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"fmt"
 	"net/url"
 	"path"
 	"strings"
@@ -30,19 +29,18 @@ func mergeURLs(uiURL, requestURI *url.URL) *url.URL {
 	return &targetURL
 }
 
-func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, []Header, error) {
+func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, []Header) {
 	for _, e := range ui.URLMaps {
 		for _, sp := range e.SrcPaths {
 			if sp.match(u.Path) {
-				return e.URLPrefix, e.Headers, nil
+				return e.URLPrefix, e.Headers
 			}
 		}
 	}
 	if ui.URLPrefix != nil {
-		return ui.URLPrefix, ui.Headers, nil
+		return ui.URLPrefix, ui.Headers
 	}
-	missingRouteRequests.Inc()
+	return nil, nil
-	return nil, nil, fmt.Errorf("missing route for %q", u.String())
 }
 
 func normalizeURL(uOrig *url.URL) *url.URL {

app/vmauth/target_url_test.go

@@ -14,9 +14,9 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
 		u = normalizeURL(u)
-		up, headers, err := ui.getURLPrefixAndHeaders(u)
+		up, headers := ui.getURLPrefixAndHeaders(u)
-		if err != nil {
+		if up == nil {
-			t.Fatalf("unexpected error: %s", err)
+			t.Fatalf("cannot determie backend: %s", err)
 		}
 		bu := up.getLeastLoadedBackendURL()
 		target := mergeURLs(bu.url, u)
@@ -124,10 +124,7 @@ func TestCreateTargetURLFailure(t *testing.T) {
 			t.Fatalf("cannot parse %q: %s", requestURI, err)
 		}
 		u = normalizeURL(u)
-		up, headers, err := ui.getURLPrefixAndHeaders(u)
+		up, headers := ui.getURLPrefixAndHeaders(u)
-		if err == nil {
-			t.Fatalf("expecting non-nil error")
-		}
 		if up != nil {
 			t.Fatalf("unexpected non-empty up=%#v", up)
 		}

docs/CHANGELOG.md

@@ -29,6 +29,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): display histogram metrics as a heatmap in the `explore metrics` tab. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4111).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add the metric relabel playground feature to the vmui. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3807).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy requests for unauthorized users. See [this doc](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4083).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to specify default route (`default_url`) for processing non-matched requests. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4084).
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to filter incoming requests by IP. See [these docs](https://docs.victoriametrics.com/vmauth.html#ip-filters) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3491).
 
 * BUGFIX: reduce the probability of sudden increase in the number of small parts on systems with small number of CPU cores.

docs/vmauth.md

@@ -189,6 +189,34 @@ users:
     headers:
     - "X-Scope-OrgID: abc"
 
+  # A single user for querying and inserting data:
+  # - Requests to http://vmauth:8427/api/v1/query, http://vmauth:8427/api/v1/query_range
+  #   and http://vmauth:8427/api/v1/label/<label_name>/values are proxied to the following urls in a round-robin manner:
+  #     - http://vmselect1:8481/select/42/prometheus
+  #     - http://vmselect2:8481/select/42/prometheus
+  #   For example, http://vmauth:8427/api/v1/query is proxied to http://vmselect1:8480/select/42/prometheus/api/v1/query
+  #   or to http://vmselect2:8480/select/42/prometheus/api/v1/query .
+  # - Requests to http://vmauth:8427/api/v1/write are proxied to http://vminsert:8480/insert/42/prometheus/api/v1/write .
+  # The requests which do not match `src_paths` from the `url_map` will be proxied to the urls rom `default_url` 
+  # in a round-robin manner (with request path in `request_path` query param).
+  # For example, request to http://vmauth:8427/non/existing/path will be proxied:
+  #  - to http://default1:8888/process?request_path=/non/existing/path
+  #  - or http://default2:8888/process?request_path=/non/existing/path
+- username: "foobar"
+  url_map:
+    - src_paths:
+        - "/api/v1/query"
+        - "/api/v1/query_range"
+        - "/api/v1/label/[^/]+/values"
+      url_prefix:
+        - "http://vmselect1:8481/select/42/prometheus"
+        - "http://vmselect2:8481/select/42/prometheus"
+    - src_paths: ["/api/v1/write"]
+      url_prefix: "http://vminsert:8480/insert/42/prometheus"
+  default_url: 
+    - "http://default1:8888/process"
+    - "http://default2:8888/process"
+
 # This requests will be executed for requests without Authorization header.
 # For instance, http://vmauth:8427/api/v1/query will be proxied to http://vmselect1:8481/select/0/prometheus/api/v1/query
 unauthorized_user: