diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go index c70682825..89d86c0fa 100644 --- a/app/vmauth/auth_config.go +++ b/app/vmauth/auth_config.go @@ -126,16 +126,30 @@ func (h *Header) MarshalYAML() (interface{}, error) { // URLMap is a mapping from source paths to target urls. type URLMap struct { - SrcPaths []*SrcPath `yaml:"src_paths,omitempty"` - URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"` - HeadersConf HeadersConf `yaml:",inline"` - RetryStatusCodes []int `yaml:"retry_status_codes,omitempty"` - LoadBalancingPolicy string `yaml:"load_balancing_policy,omitempty"` - DropSrcPathPrefixParts int `yaml:"drop_src_path_prefix_parts,omitempty"` + // SrcHosts is the list of regular expressions, which match the request hostname. + SrcHosts []*Regex `yaml:"src_hosts,omitempty"` + + // SrcPaths is the list of regular expressions, which match the request path. + SrcPaths []*Regex `yaml:"src_paths,omitempty"` + + // UrlPrefix contains backend url prefixes for the proxied request url. + URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"` + + // HeadersConf is the config for augumenting request and response headers. + HeadersConf HeadersConf `yaml:",inline"` + + // RetryStatusCodes is the list of response status codes used for retrying requests. + RetryStatusCodes []int `yaml:"retry_status_codes,omitempty"` + + // LoadBalancingPolicy is load balancing policy among UrlPrefix backends. + LoadBalancingPolicy string `yaml:"load_balancing_policy,omitempty"` + + // DropSrcPathPrefixParts is the number of `/`-delimited request path prefix parts to drop before proxying the request to backend. + DropSrcPathPrefixParts int `yaml:"drop_src_path_prefix_parts,omitempty"` } -// SrcPath represents an src path -type SrcPath struct { +// Regex represents a regex +type Regex struct { sOriginal string re *regexp.Regexp } @@ -333,8 +347,8 @@ func (up *URLPrefix) MarshalYAML() (interface{}, error) { return string(b), nil } -func (sp *SrcPath) match(s string) bool { - prefix, ok := sp.re.LiteralPrefix() +func (r *Regex) match(s string) bool { + prefix, ok := r.re.LiteralPrefix() if ok { // Fast path - literal match return s == prefix @@ -342,11 +356,11 @@ func (sp *SrcPath) match(s string) bool { if !strings.HasPrefix(s, prefix) { return false } - return sp.re.MatchString(s) + return r.re.MatchString(s) } // UnmarshalYAML implements yaml.Unmarshaler -func (sp *SrcPath) UnmarshalYAML(f func(interface{}) error) error { +func (r *Regex) UnmarshalYAML(f func(interface{}) error) error { var s string if err := f(&s); err != nil { return err @@ -356,14 +370,14 @@ func (sp *SrcPath) UnmarshalYAML(f func(interface{}) error) error { if err != nil { return fmt.Errorf("cannot build regexp from %q: %w", s, err) } - sp.sOriginal = s - sp.re = re + r.sOriginal = s + r.re = re return nil } // MarshalYAML implements yaml.Marshaler. -func (sp *SrcPath) MarshalYAML() (interface{}, error) { - return sp.sOriginal, nil +func (r *Regex) MarshalYAML() (interface{}, error) { + return r.sOriginal, nil } var ( @@ -613,8 +627,8 @@ func (ui *UserInfo) initURLs() error { } } for _, e := range ui.URLMaps { - if len(e.SrcPaths) == 0 { - return fmt.Errorf("missing `src_paths` in `url_map`") + if len(e.SrcPaths) == 0 && len(e.SrcHosts) == 0 { + return fmt.Errorf("missing `src_paths` and `src_hosts` in `url_map`") } if e.URLPrefix == nil { return fmt.Errorf("missing `url_prefix` in `url_map`") diff --git a/app/vmauth/auth_config_test.go b/app/vmauth/auth_config_test.go index 73d30c720..fee8e9ac3 100644 --- a/app/vmauth/auth_config_test.go +++ b/app/vmauth/auth_config_test.go @@ -145,6 +145,12 @@ users: url_map: - src_paths: ["/foo/bar"] `) + f(` +users: +- username: a + url_map: + - src_hosts: ["foobar"] +`) // Invalid url_prefix in url_map f(` @@ -154,6 +160,13 @@ users: - src_paths: ["/foo/bar"] url_prefix: foo.bar `) + f(` +users: +- username: a + url_map: + - src_hosts: ["foobar"] + url_prefix: foo.bar +`) // empty url_prefix in url_map f(` @@ -163,8 +176,15 @@ users: - src_paths: ['/foo/bar'] url_prefix: [] `) + f(` +users: +- username: a + url_map: + - src_phosts: ['foobar'] + url_prefix: [] +`) - // Missing src_paths in url_map + // Missing src_paths and src_hosts in url_map f(` users: - username: a @@ -181,6 +201,15 @@ users: url_prefix: http://foobar `) + // Invalid regexp in src_hosts + f(` +users: +- username: a + url_map: + - src_hosts: ['fo[obar'] + url_prefix: http://foobar +`) + // Invalid headers in url_map (missing ':') f(` users: @@ -293,6 +322,7 @@ users: - src_paths: ["/api/v1/query","/api/v1/query_range","/api/v1/label/[^./]+/.+"] url_prefix: http://vmselect/select/0/prometheus - src_paths: ["/api/v1/write"] + src_hosts: ["foo\\.bar", "baz:1234"] url_prefix: ["http://vminsert1/insert/0/prometheus","http://vminsert2/insert/0/prometheus"] headers: - "foo: bar" @@ -302,11 +332,12 @@ users: BearerToken: "foo", URLMaps: []URLMap{ { - SrcPaths: getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}), + SrcPaths: getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}), URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"), }, { - SrcPaths: getSrcPaths([]string{"/api/v1/write"}), + SrcHosts: getRegexs([]string{"foo\\.bar", "baz:1234"}), + SrcPaths: getRegexs([]string{"/api/v1/write"}), URLPrefix: mustParseURLs([]string{ "http://vminsert1/insert/0/prometheus", "http://vminsert2/insert/0/prometheus", @@ -330,11 +361,12 @@ users: BearerToken: "foo", URLMaps: []URLMap{ { - SrcPaths: getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}), + SrcPaths: getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}), URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"), }, { - SrcPaths: getSrcPaths([]string{"/api/v1/write"}), + SrcHosts: getRegexs([]string{"foo\\.bar", "baz:1234"}), + SrcPaths: getRegexs([]string{"/api/v1/write"}), URLPrefix: mustParseURLs([]string{ "http://vminsert1/insert/0/prometheus", "http://vminsert2/insert/0/prometheus", @@ -396,11 +428,11 @@ users: BearerToken: "foo", URLMaps: []URLMap{ { - SrcPaths: getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}), + SrcPaths: getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}), URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"), }, { - SrcPaths: getSrcPaths([]string{"/api/v1/write"}), + SrcPaths: getRegexs([]string{"/api/v1/write"}), URLPrefix: mustParseURLs([]string{ "http://vminsert1/insert/0/prometheus", "http://vminsert2/insert/0/prometheus", @@ -428,11 +460,11 @@ users: BearerToken: "foo", URLMaps: []URLMap{ { - SrcPaths: getSrcPaths([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}), + SrcPaths: getRegexs([]string{"/api/v1/query", "/api/v1/query_range", "/api/v1/label/[^./]+/.+"}), URLPrefix: mustParseURL("http://vmselect/select/0/prometheus"), }, { - SrcPaths: getSrcPaths([]string{"/api/v1/write"}), + SrcPaths: getRegexs([]string{"/api/v1/write"}), URLPrefix: mustParseURLs([]string{ "http://vminsert1/insert/0/prometheus", "http://vminsert2/insert/0/prometheus", @@ -501,10 +533,10 @@ func isSetBool(boolP *bool, expectedValue bool) bool { return *boolP == expectedValue } -func getSrcPaths(paths []string) []*SrcPath { - var sps []*SrcPath +func getRegexs(paths []string) []*Regex { + var sps []*Regex for _, path := range paths { - sps = append(sps, &SrcPath{ + sps = append(sps, &Regex{ sOriginal: path, re: regexp.MustCompile("^(?:" + path + ")$"), }) diff --git a/app/vmauth/example_config.yml b/app/vmauth/example_config.yml index e4e426f1f..e1c814e12 100644 --- a/app/vmauth/example_config.yml +++ b/app/vmauth/example_config.yml @@ -92,7 +92,7 @@ users: # - to http://default1:8888/unsupported_url_handler?request_path=/non/existing/path # - or http://default2:8888/unsupported_url_handler?request_path=/non/existing/path # - # Regular expressions are allowed in `src_paths` entries. + # Regular expressions are allowed in `src_paths` and `src_hosts` entries. - username: "foobar" url_map: - src_paths: diff --git a/app/vmauth/target_url.go b/app/vmauth/target_url.go index e1ce736d1..1178318fd 100644 --- a/app/vmauth/target_url.go +++ b/app/vmauth/target_url.go @@ -51,10 +51,8 @@ func dropPrefixParts(path string, parts int) string { func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, HeadersConf, int) { for _, e := range ui.URLMaps { - for _, sp := range e.SrcPaths { - if sp.match(u.Path) { - return e.URLPrefix, e.HeadersConf, e.DropSrcPathPrefixParts - } + if matchAnyRegex(e.SrcHosts, u.Host) && matchAnyRegex(e.SrcPaths, u.Path) { + return e.URLPrefix, e.HeadersConf, e.DropSrcPathPrefixParts } } if ui.URLPrefix != nil { @@ -63,6 +61,18 @@ func (ui *UserInfo) getURLPrefixAndHeaders(u *url.URL) (*URLPrefix, HeadersConf, return nil, HeadersConf{}, 0 } +func matchAnyRegex(rs []*Regex, s string) bool { + if len(rs) == 0 { + return true + } + for _, r := range rs { + if r.match(s) { + return true + } + } + return false +} + func normalizeURL(uOrig *url.URL) *url.URL { u := *uOrig // Prevent from attacks with using `..` in r.URL.Path diff --git a/app/vmauth/target_url_test.go b/app/vmauth/target_url_test.go index 5b6b0ec38..2f9a27313 100644 --- a/app/vmauth/target_url_test.go +++ b/app/vmauth/target_url_test.go @@ -149,7 +149,8 @@ func TestCreateTargetURLSuccess(t *testing.T) { ui := &UserInfo{ URLMaps: []URLMap{ { - SrcPaths: getSrcPaths([]string{"/vmsingle/api/v1/query"}), + SrcHosts: getRegexs([]string{"host42"}), + SrcPaths: getRegexs([]string{"/vmsingle/api/v1/query"}), URLPrefix: mustParseURL("http://vmselect/0/prometheus"), HeadersConf: HeadersConf{ RequestHeaders: []Header{ @@ -174,7 +175,7 @@ func TestCreateTargetURLSuccess(t *testing.T) { DropSrcPathPrefixParts: 1, }, { - SrcPaths: getSrcPaths([]string{"/api/v1/write"}), + SrcPaths: getRegexs([]string{"/api/v1/write"}), URLPrefix: mustParseURL("http://vminsert/0/prometheus"), }, }, @@ -192,21 +193,28 @@ func TestCreateTargetURLSuccess(t *testing.T) { RetryStatusCodes: []int{502}, DropSrcPathPrefixParts: 2, } - f(ui, "/vmsingle/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", `[{"xx" "aa"} {"yy" "asdf"}]`, `[{"qwe" "rty"}]`, []int{503, 500, 501}, "first_available", 1) - f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", []int{502}, "least_loaded", 0) - f(ui, "/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", `[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2) + f(ui, "http://host42/vmsingle/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", + `[{"xx" "aa"} {"yy" "asdf"}]`, `[{"qwe" "rty"}]`, []int{503, 500, 501}, "first_available", 1) + f(ui, "http://host123/vmsingle/api/v1/query?query=up", "http://default-server/v1/query?query=up", + `[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2) + f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", []int{502}, "least_loaded", 0) + f(ui, "https://foo-host/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", `[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2) // Complex routing regexp paths in `url_map` ui = &UserInfo{ URLMaps: []URLMap{ { - SrcPaths: getSrcPaths([]string{"/api/v1/query(_range)?", "/api/v1/label/[^/]+/values"}), + SrcPaths: getRegexs([]string{"/api/v1/query(_range)?", "/api/v1/label/[^/]+/values"}), URLPrefix: mustParseURL("http://vmselect/0/prometheus"), }, { - SrcPaths: getSrcPaths([]string{"/api/v1/write"}), + SrcPaths: getRegexs([]string{"/api/v1/write"}), URLPrefix: mustParseURL("http://vminsert/0/prometheus"), }, + { + SrcHosts: getRegexs([]string{"vmui\\..+"}), + URLPrefix: mustParseURL("http://vmui.host:1234/vmui/"), + }, }, URLPrefix: mustParseURL("http://default-server"), } @@ -215,6 +223,8 @@ func TestCreateTargetURLSuccess(t *testing.T) { f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values", "[]", "[]", nil, "least_loaded", 0) f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", nil, "least_loaded", 0) f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar", "[]", "[]", nil, "least_loaded", 0) + f(ui, "https://vmui.foobar.com/a/b?c=d", "http://vmui.host:1234/vmui/a/b?c=d", "[]", "[]", nil, "least_loaded", 0) + f(&UserInfo{ URLPrefix: mustParseURL("http://foo.bar?extra_label=team=dev"), }, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "[]", "[]", nil, "least_loaded", 0) @@ -249,7 +259,7 @@ func TestCreateTargetURLFailure(t *testing.T) { f(&UserInfo{ URLMaps: []URLMap{ { - SrcPaths: getSrcPaths([]string{"/api/v1/query"}), + SrcPaths: getRegexs([]string{"/api/v1/query"}), URLPrefix: mustParseURL("http://foobar/baz"), }, }, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f447fbff5..c40f47a7f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -28,6 +28,7 @@ The sandbox cluster installation is running under the constant load generated by ## tip +* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy incoming requests to different backends based on the requested host via `src_hosts` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth.html#generic-http-proxy-for-different-backends). ## [v1.96.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.96.0) diff --git a/docs/vmauth.md b/docs/vmauth.md index 3f131eb5e..e20be290f 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -74,13 +74,13 @@ unauthorized_user: ### Generic HTTP proxy for different backends -`vmauth` can proxy requests to different backends depending on the requested path. +`vmauth` can proxy requests to different backends depending on the requested host and/or path. For example, the following [`-auth.config`](#auth-config) instructs `vmauth` to make the following: -- Requests starting with `/app1/` are proxied to `http://app1-backend/`. For example, the request to `http://vmauth:8427/app1/foo/bar?baz=qwe` - is proxied to `http://app1-backend/foo/bar?baz=qwe`. -- Requests starting with `/app2/` are proxied to `http://app2-backend/`. For example, the request to `http://vmauth:8427/app2/index.html` - is proxied to `http://app2-backend/index.html`. +- Requests starting with `/app1/` are proxied to `http://app1-backend/`, while the `/app1/` path prefix is dropped according to [`drop_src_path_prefix_parts`](#dropping-request-path-prefix). + For example, the request to `http://vmauth:8427/app1/foo/bar?baz=qwe` is proxied to `http://app1-backend/foo/bar?baz=qwe`. +- Requests starting with `/app2/` are proxied to `http://app2-backend/`, while the `/app2/` path prefix is dropped according to [`drop_src_path_prefix_parts`](#dropping-request-path-prefix). + For example, the request to `http://vmauth:8427/app2/index.html` is proxied to `http://app2-backend/index.html`. - Other requests are proxied to `http://some-backend/404-page.html`, while the requested path is passed via `request_path` query arg. For example, the request to `http://vmauth:8427/foo/bar?baz=qwe` is proxied to `http://some-backend/404-page.html?request_path=%2Ffoo%2Fbar%3Fbaz%3Dqwe`. @@ -98,8 +98,23 @@ unauthorized_user: default_url: http://some-backend/404-page.html ``` -See [these docs](#dropping-request-path-prefix) for more details. +The following config routes requests to host `app1.my-host.com` to `http://app1-backend`, while routing requests to `app2.my-host.com` to `http://app2-backend`: +```yml +unauthorized_user: + url_map: + - src_hosts: + - "app1\\.my-host\\.com" + url_prefix: "http://app1-backend/" + - src_paths: + - "app2\\.my-host\\.com" + url_prefix: "http://app2-backend/" +``` + +`src_paths` and `src_hosts` accept a list of [regular expressions](https://github.com/google/re2/wiki/Syntax). The incoming request is routed to the given `url_prefix` +if the whole request path matches at least one `src_paths` entry. The incoming request is routed to the given `url_prefix` if the whole request host matches at least one `src_hosts` entry. +If both `src_paths` and `src_hosts` lists are specified, then the request is routed to the given `url_prefix` when both request path and request host match at least one entry +in the corresponding lists. ### Generic HTTP load balancer @@ -603,7 +618,7 @@ users: # - to http://default1:8888/unsupported_url_handler?request_path=/non/existing/path # - or http://default2:8888/unsupported_url_handler?request_path=/non/existing/path # - # Regular expressions are allowed in `src_paths` entries. + # Regular expressions are allowed in `src_paths` and `src_hosts` entries. - username: "foobar" url_map: - src_paths: