app/vmauth: add ability to route requests to different backends depending on the request host

2024-11-23 12:31:07 +01:00 · 2023-12-14 00:46:36 +02:00 · 2023-12-14 00:46:36 +02:00 · 68be182075
commit 68be182075
parent 6f15ca4a16
7 changed files with 132 additions and 50 deletions
--- a/app/vmauth/auth_config.go
+++ b/app/vmauth/auth_config.go
@ -126,16 +126,30 @@ func (h *Header) MarshalYAML() (interface{}, error) {

 // URLMap is a mapping from source paths to target urls.
 type URLMap struct {
-	SrcPaths               []*SrcPath  `yaml:"src_paths,omitempty"`
-	URLPrefix              *URLPrefix  `yaml:"url_prefix,omitempty"`
-	HeadersConf            HeadersConf `yaml:",inline"`
-	RetryStatusCodes       []int       `yaml:"retry_status_codes,omitempty"`
-	LoadBalancingPolicy    string      `yaml:"load_balancing_policy,omitempty"`
-	DropSrcPathPrefixParts int         `yaml:"drop_src_path_prefix_parts,omitempty"`
+	// SrcHosts is the list of regular expressions, which match the request hostname.
+	SrcHosts []*Regex `yaml:"src_hosts,omitempty"`
+
+	// SrcPaths is the list of regular expressions, which match the request path.
+	SrcPaths []*Regex `yaml:"src_paths,omitempty"`
+
+	// UrlPrefix contains backend url prefixes for the proxied request url.
+	URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
+
+	// HeadersConf is the config for augumenting request and response headers.
+	HeadersConf HeadersConf `yaml:",inline"`
+
+	// RetryStatusCodes is the list of response status codes used for retrying requests.
+	RetryStatusCodes []int `yaml:"retry_status_codes,omitempty"`
+
+	// LoadBalancingPolicy is load balancing policy among UrlPrefix backends.
+	LoadBalancingPolicy string `yaml:"load_balancing_policy,omitempty"`
+
+	// DropSrcPathPrefixParts is the number of `/`-delimited request path prefix parts to drop before proxying the request to backend.
+	DropSrcPathPrefixParts int `yaml:"drop_src_path_prefix_parts,omitempty"`
 }

-// SrcPath represents an src path
-type SrcPath struct {
+// Regex represents a regex
+type Regex struct {
 	sOriginal string
 	re        *regexp.Regexp
 }
@ -333,8 +347,8 @@ func (up *URLPrefix) MarshalYAML() (interface{}, error) {
 	return string(b), nil
 }

-func (sp *SrcPath) match(s string) bool {
-	prefix, ok := sp.re.LiteralPrefix()
+func (r *Regex) match(s string) bool {
+	prefix, ok := r.re.LiteralPrefix()
 	if ok {
 		// Fast path - literal match
 		return s == prefix
@ -342,11 +356,11 @@ func (sp *SrcPath) match(s string) bool {
 	if !strings.HasPrefix(s, prefix) {
 		return false
 	}
-	return sp.re.MatchString(s)
+	return r.re.MatchString(s)
 }

 // UnmarshalYAML implements yaml.Unmarshaler
-func (sp *SrcPath) UnmarshalYAML(f func(interface{}) error) error {
+func (r *Regex) UnmarshalYAML(f func(interface{}) error) error {
 	var s string
 	if err := f(&s); err != nil {
 		return err
@ -356,14 +370,14 @@ func (sp *SrcPath) UnmarshalYAML(f func(interface{}) error) error {
 	if err != nil {
 		return fmt.Errorf("cannot build regexp from %q: %w", s, err)
 	}
-	sp.sOriginal = s
-	sp.re = re
+	r.sOriginal = s
+	r.re = re
 	return nil
 }

 // MarshalYAML implements yaml.Marshaler.
-func (sp *SrcPath) MarshalYAML() (interface{}, error) {
-	return sp.sOriginal, nil
+func (r *Regex) MarshalYAML() (interface{}, error) {
+	return r.sOriginal, nil
 }

 var (
@ -613,8 +627,8 @@ func (ui *UserInfo) initURLs() error {
 		}
 	}
 	for _, e := range ui.URLMaps {
-		if len(e.SrcPaths) == 0 {
-			return fmt.Errorf("missing `src_paths` in `url_map`")
+		if len(e.SrcPaths) == 0 && len(e.SrcHosts) == 0 {
+			return fmt.Errorf("missing `src_paths` and `src_hosts` in `url_map`")
 		}
 		if e.URLPrefix == nil {
 			return fmt.Errorf("missing `url_prefix` in `url_map`")
--- a/app/vmauth/auth_config_test.go
+++ b/app/vmauth/auth_config_test.go
@ -145,6 +145,12 @@ users:
  url_map:
  - src_paths: ["/foo/bar"]
 `)
+	f(`
+users:
+- username: a
+  url_map:
+  - src_hosts: ["foobar"]
+`)

 	// Invalid url_prefix in url_map
 	f(`
@ -154,6 +160,13 @@ users:
  - src_paths: ["/foo/bar"]
    url_prefix: foo.bar
 `)
+	f(`
+users:
+- username: a
+  url_map:
+  - src_hosts: ["foobar"]
+    url_prefix: foo.bar
+`)

 	// empty url_prefix in url_map
 	f(`
@ -163,8 +176,15 @@ users:
  - src_paths: ['/foo/bar']
    url_prefix: []
 `)
+	f(`
+users:
+- username: a
+  url_map:
+  - src_phosts: ['foobar']
+    url_prefix: []
+`)

-	// Missing src_paths in url_map
+	// Missing src_paths and src_hosts in url_map
 	f(`
 users:
 - username: a
@ -181,6 +201,15 @@ users:
    url_prefix: http://foobar
 `)

+	// Invalid regexp in src_hosts
+	f(`
+users:
+- username: a
+  url_map:
+  - src_hosts: ['fo[obar']
+    url_prefix: http://foobar
+`)
+
 	// Invalid headers in url_map (missing ':')
 	f(`
 users:
@ -293,6 +322,7 @@ users:
  - src_paths: ["/api/v1/query","/api/v1/query_range","/api/v1/label/[^./]+/.+"]
    url_prefix: http://vmselect/select/0/prometheus
  - src_paths: ["/api/v1/write"]
+    src_hosts: ["foo\\.bar", "baz:1234"]
    url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"]
    headers:
    - "foo: bar"
@ -302,11 +332,12 @@ users:
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
-					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					SrcPaths:  getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
 					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
 				},
 				{
-					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
+					SrcHosts: getRegexs([]string{"foo\\.bar", "baz:1234"}),
+					SrcPaths: getRegexs([]string{"/api/v1/write"}),
 					URLPrefix: mustParseURLs([]string{
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
@ -330,11 +361,12 @@ users:
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
-					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					SrcPaths:  getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
 					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
 				},
 				{
-					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
+					SrcHosts: getRegexs([]string{"foo\\.bar", "baz:1234"}),
+					SrcPaths: getRegexs([]string{"/api/v1/write"}),
 					URLPrefix: mustParseURLs([]string{
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
@ -396,11 +428,11 @@ users:
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
-					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					SrcPaths:  getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
 					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
 				},
 				{
-					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
+					SrcPaths: getRegexs([]string{"/api/v1/write"}),
 					URLPrefix: mustParseURLs([]string{
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
@ -428,11 +460,11 @@ users:
 			BearerToken: "foo",
 			URLMaps: []URLMap{
 				{
-					SrcPaths:  getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
+					SrcPaths:  getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}),
 					URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"),
 				},
 				{
-					SrcPaths: getSrcPaths([]string{"/api/v1/write"}),
+					SrcPaths: getRegexs([]string{"/api/v1/write"}),
 					URLPrefix: mustParseURLs([]string{
 						"http://vminsert1/insert/0/prometheus",
 						"http://vminsert2/insert/0/prometheus",
@ -501,10 +533,10 @@ func isSetBool(boolP *bool, expectedValue bool) bool {
 	return *boolP == expectedValue
 }

-func getSrcPaths(paths []string) []*SrcPath {
-	var sps []*SrcPath
+func getRegexs(paths []string) []*Regex {
+	var sps []*Regex
 	for _, path := range paths {
-		sps = append(sps, &SrcPath{
+		sps = append(sps, &Regex{
 			sOriginal: path,
 			re:        regexp.MustCompile("^(?:" + path + ")$"),
 		})
--- a/app/vmauth/example_config.yml
+++ b/app/vmauth/example_config.yml
@ -92,7 +92,7 @@ users:
  #  - to http://default1:8888/unsupported_url_handler?request_path=/non/existing/path
  #  - or http://default2:8888/unsupported_url_handler?request_path=/non/existing/path
  #
-  # Regular expressions are allowed in `src_paths` entries.
+  # Regular expressions are allowed in `src_paths` and `src_hosts` entries.
 - username: "foobar"
  url_map:
  - src_paths:
--- a/app/vmauth/target_url.go
+++ b/app/vmauth/target_url.go
@ -51,10 +51,8 @@ func dropPrefixParts(path string, parts int) string {

 func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, HeadersConf, int) {
 	for _, e := range ui.URLMaps {
-		for _, sp := range e.SrcPaths {
-			if sp.match(u.Path) {
-				return e.URLPrefix, e.HeadersConf, e.DropSrcPathPrefixParts
-			}
+		if matchAnyRegex(e.SrcHosts, u.Host) && matchAnyRegex(e.SrcPaths, u.Path) {
+			return e.URLPrefix, e.HeadersConf, e.DropSrcPathPrefixParts
 		}
 	}
 	if ui.URLPrefix != nil {
@ -63,6 +61,18 @@ func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, HeadersConf,
 	return nil, HeadersConf{}, 0
 }

+func matchAnyRegex(rs []*Regex, s string) bool {
+	if len(rs) == 0 {
+		return true
+	}
+	for _, r := range rs {
+		if r.match(s) {
+			return true
+		}
+	}
+	return false
+}
+
 func normalizeURL(uOrig *url.URL) *url.URL {
 	u := *uOrig
 	// Prevent from attacks with using `..` in r.URL.Path
--- a/app/vmauth/target_url_test.go
+++ b/app/vmauth/target_url_test.go
@ -149,7 +149,8 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 	ui := &UserInfo{
 		URLMaps: []URLMap{
 			{
-				SrcPaths:  getSrcPaths([]string{"/vmsingle/api/v1/query"}),
+				SrcHosts:  getRegexs([]string{"host42"}),
+				SrcPaths:  getRegexs([]string{"/vmsingle/api/v1/query"}),
 				URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
 				HeadersConf: HeadersConf{
 					RequestHeaders: []Header{
@ -174,7 +175,7 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 				DropSrcPathPrefixParts: 1,
 			},
 			{
-				SrcPaths:  getSrcPaths([]string{"/api/v1/write"}),
+				SrcPaths:  getRegexs([]string{"/api/v1/write"}),
 				URLPrefix: mustParseURL("http://vminsert/0/prometheus"),
 			},
 		},
@ -192,21 +193,28 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 		RetryStatusCodes:       []int{502},
 		DropSrcPathPrefixParts: 2,
 	}
-	f(ui, "/vmsingle/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", `[{"xx" "aa"} {"yy" "asdf"}]`, `[{"qwe" "rty"}]`, []int{503, 500, 501}, "first_available", 1)
-	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", []int{502}, "least_loaded", 0)
-	f(ui, "/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", `[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2)
+	f(ui, "http://host42/vmsingle/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up",
+		`[{"xx" "aa"} {"yy" "asdf"}]`, `[{"qwe" "rty"}]`, []int{503, 500, 501}, "first_available", 1)
+	f(ui, "http://host123/vmsingle/api/v1/query?query=up", "http://default-server/v1/query?query=up",
+		`[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2)
+	f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", []int{502}, "least_loaded", 0)
+	f(ui, "https://foo-host/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", `[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2)

 	// Complex routing regexp paths in `url_map`
 	ui = &UserInfo{
 		URLMaps: []URLMap{
 			{
-				SrcPaths:  getSrcPaths([]string{"/api/v1/query(_range)?", "/api/v1/label/[^/]+/values"}),
+				SrcPaths:  getRegexs([]string{"/api/v1/query(_range)?", "/api/v1/label/[^/]+/values"}),
 				URLPrefix: mustParseURL("http://vmselect/0/prometheus"),
 			},
 			{
-				SrcPaths:  getSrcPaths([]string{"/api/v1/write"}),
+				SrcPaths:  getRegexs([]string{"/api/v1/write"}),
 				URLPrefix: mustParseURL("http://vminsert/0/prometheus"),
 			},
+			{
+				SrcHosts:  getRegexs([]string{"vmui\\..+"}),
+				URLPrefix: mustParseURL("http://vmui.host:1234/vmui/"),
+			},
 		},
 		URLPrefix: mustParseURL("http://default-server"),
 	}
@ -215,6 +223,8 @@ func TestCreateTargetURLSuccess(t *testing.T) {
 	f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values", "[]", "[]", nil, "least_loaded", 0)
 	f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", nil, "least_loaded", 0)
 	f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar", "[]", "[]", nil, "least_loaded", 0)
+	f(ui, "https://vmui.foobar.com/a/b?c=d", "http://vmui.host:1234/vmui/a/b?c=d", "[]", "[]", nil, "least_loaded", 0)
+
 	f(&UserInfo{
 		URLPrefix: mustParseURL("http://foo.bar?extra_label=team=dev"),
 	}, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "[]", "[]", nil, "least_loaded", 0)
@ -249,7 +259,7 @@ func TestCreateTargetURLFailure(t *testing.T) {
 	f(&UserInfo{
 		URLMaps: []URLMap{
 			{
-				SrcPaths:  getSrcPaths([]string{"/api/v1/query"}),
+				SrcPaths:  getRegexs([]string{"/api/v1/query"}),
 				URLPrefix: mustParseURL("http://foobar/baz"),
 			},
 		},
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@ -28,6 +28,7 @@ The sandbox cluster installation is running under the constant load generated by

 ## tip

+* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy incoming requests to different backends based on the requested host via `src_hosts` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth.html#generic-http-proxy-for-different-backends).

 ## [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0)

--- a/docs/vmauth.md
+++ b/docs/vmauth.md
@ -74,13 +74,13 @@ unauthorized_user:

 ### Generic HTTP proxy for different backends

-`vmauth` can proxy requests to different backends depending on the requested path.
+`vmauth` can proxy requests to different backends depending on the requested host and/or path.
 For example, the following [`-auth.config`](#auth-config) instructs `vmauth` to make the following:

- Requests starting with `/app1/` are proxied to `http://app1-backend/`. For example, the request to `http://vmauth:8427/app1/foo/bar?baz=qwe`
-  is proxied to `http://app1-backend/foo/bar?baz=qwe`.
- Requests starting with `/app2/` are proxied to `http://app2-backend/`. For example, the request to `http://vmauth:8427/app2/index.html`
-  is proxied to `http://app2-backend/index.html`.
+- Requests starting with `/app1/` are proxied to `http://app1-backend/`, while the `/app1/` path prefix is dropped according to [`drop_src_path_prefix_parts`](#dropping-request-path-prefix).
+  For example, the request to `http://vmauth:8427/app1/foo/bar?baz=qwe` is proxied to `http://app1-backend/foo/bar?baz=qwe`.
+- Requests starting with `/app2/` are proxied to `http://app2-backend/`, while the `/app2/` path prefix is dropped according to [`drop_src_path_prefix_parts`](#dropping-request-path-prefix).
+  For example, the request to `http://vmauth:8427/app2/index.html` is proxied to `http://app2-backend/index.html`.
 - Other requests are proxied to `http://some-backend/404-page.html`, while the requested path is passed via `request_path` query arg.
  For example, the request to `http://vmauth:8427/foo/bar?baz=qwe` is proxied to `http://some-backend/404-page.html?request_path=%2Ffoo%2Fbar%3Fbaz%3Dqwe`.

@ -98,8 +98,23 @@ unauthorized_user:
  default_url: http://some-backend/404-page.html
 ```

-See [these docs](#dropping-request-path-prefix) for more details.
+The following config routes requests to host `app1.my-host.com` to `http://app1-backend`, while routing requests to `app2.my-host.com` to `http://app2-backend`:

+```yml
+unauthorized_user:
+  url_map:
+  - src_hosts:
+    - "app1\\.my-host\\.com"
+    url_prefix: "http://app1-backend/"
+  - src_paths:
+    - "app2\\.my-host\\.com"
+    url_prefix: "http://app2-backend/"
+```
+
+`src_paths` and `src_hosts` accept a list of [regular expressions](https://github.com/google/re2/wiki/Syntax). The incoming request is routed to the given `url_prefix`
+if the whole request path matches at least one `src_paths` entry. The incoming request is routed to the given `url_prefix` if the whole request host matches at least one `src_hosts` entry.
+If both `src_paths` and `src_hosts` lists are specified, then the request is routed to the given `url_prefix` when both request path and request host match at least one entry
+in the corresponding lists.

 ### Generic HTTP load balancer

@ -603,7 +618,7 @@ users:
  #  - to http://default1:8888/unsupported_url_handler?request_path=/non/existing/path
  #  - or http://default2:8888/unsupported_url_handler?request_path=/non/existing/path
  #
-  # Regular expressions are allowed in `src_paths` entries.
+  # Regular expressions are allowed in `src_paths` and `src_hosts` entries.
 - username: "foobar"
  url_map:
  - src_paths: