diff --git a/app/vmauth/auth_config.go b/app/vmauth/auth_config.go index 9fa355929..84d7f5225 100644 --- a/app/vmauth/auth_config.go +++ b/app/vmauth/auth_config.go @@ -2,15 +2,17 @@ package main import ( "bytes" + "context" "encoding/base64" "flag" "fmt" + "math" + "net" "net/http" "net/url" "os" "regexp" "sort" - "strconv" "strings" "sync" "sync/atomic" @@ -36,7 +38,11 @@ var ( defaultRetryStatusCodes = flagutil.NewArrayInt("retryStatusCodes", 0, "Comma-separated list of default HTTP response status codes when vmauth re-tries the request on other backends. "+ "See https://docs.victoriametrics.com/vmauth.html#load-balancing for details") defaultLoadBalancingPolicy = flag.String("loadBalancingPolicy", "least_loaded", "The default load balancing policy to use for backend urls specified inside url_prefix section. "+ - "Supported policies: least_loaded, first_available. See https://docs.victoriametrics.com/vmauth.html#load-balancing for more details") + "Supported policies: least_loaded, first_available. See https://docs.victoriametrics.com/vmauth.html#load-balancing") + discoverBackendIPsGlobal = flag.Bool("discoverBackendIPs", false, "Whether to discover backend IPs via periodic DNS queries to hostnames specified in url_prefix. "+ + "This may be useful when url_prefix points to a hostname with dynamically scaled instances behind it. See https://docs.victoriametrics.com/vmauth.html#discovering-backend-ips") + discoverBackendIPsInterval = flag.Duration("discoverBackendIPsInterval", 10*time.Second, "The interval for re-discovering backend IPs if -discoverBackendIPs command-line flag is set. "+ + "Too low value may lead to DNS errors") ) // AuthConfig represents auth config. @@ -57,6 +63,7 @@ type UserInfo struct { Password string `yaml:"password,omitempty"` URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"` + DiscoverBackendIPs *bool `yaml:"discover_backend_ips,omitempty"` URLMaps []URLMap `yaml:"url_map,omitempty"` HeadersConf HeadersConf `yaml:",inline"` MaxConcurrentRequests int `yaml:"max_concurrent_requests,omitempty"` @@ -111,6 +118,8 @@ func (ui *UserInfo) getMaxConcurrentRequests() int { type Header struct { Name string Value string + + sOriginal string } // UnmarshalYAML unmarshals h from f. @@ -119,6 +128,8 @@ func (h *Header) UnmarshalYAML(f func(interface{}) error) error { if err := f(&s); err != nil { return err } + h.sOriginal = s + n := strings.IndexByte(s, ':') if n < 0 { return fmt.Errorf("missing speparator char ':' between Name and Value in the header %q; expected format - 'Name: Value'", s) @@ -130,8 +141,7 @@ func (h *Header) UnmarshalYAML(f func(interface{}) error) error { // MarshalYAML marshals h to yaml. func (h *Header) MarshalYAML() (interface{}, error) { - s := fmt.Sprintf("%s: %s", h.Name, h.Value) - return s, nil + return h.sOriginal, nil } // URLMap is a mapping from source paths to target urls. @@ -151,6 +161,9 @@ type URLMap struct { // UrlPrefix contains backend url prefixes for the proxied request url. URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"` + // DiscoverBackendIPs instructs discovering URLPrefix backend IPs via DNS. + DiscoverBackendIPs *bool `yaml:"discover_backend_ips,omitempty"` + // HeadersConf is the config for augumenting request and response headers. HeadersConf HeadersConf `yaml:",inline"` @@ -181,14 +194,16 @@ type QueryArg struct { // UnmarshalYAML unmarshals up from yaml. func (qa *QueryArg) UnmarshalYAML(f func(interface{}) error) error { - if err := f(&qa.sOriginal); err != nil { + var s string + if err := f(&s); err != nil { return err } + qa.sOriginal = s - n := strings.IndexByte(qa.sOriginal, '=') + n := strings.IndexByte(s, '=') if n >= 0 { - qa.Name = qa.sOriginal[:n] - qa.Value = qa.sOriginal[n+1:] + qa.Name = s[:n] + qa.Value = s[n+1:] } return nil } @@ -200,19 +215,34 @@ func (qa *QueryArg) MarshalYAML() (interface{}, error) { // URLPrefix represents passed `url_prefix` type URLPrefix struct { - n atomic.Uint32 - - // the list of backend urls - bus []*backendURL - // requests are re-tried on other backend urls for these http response status codes retryStatusCodes []int // load balancing policy used loadBalancingPolicy string - // how many request path prefix parts to drop before routing the request to backendURL. + // how many request path prefix parts to drop before routing the request to backendURL dropSrcPathPrefixParts int + + // busOriginal contains the original list of backends specified in yaml config. + busOriginal []*url.URL + + // n is an atomic counter, which is used for balancing load among available backends. + n atomic.Uint32 + + // the list of backend urls + // + // the list can be dynamically updated if `discover_backend_ips` option is set. + bus atomic.Pointer[[]*backendURL] + + // if this option is set, then backend ips for busOriginal are periodically re-discovered and put to bus. + discoverBackendIPs bool + + // The next deadline for DNS-based discovery of backend IPs + nextDiscoveryDeadline atomic.Uint64 + + // vOriginal contains the original yaml value for URLPrefix. + vOriginal interface{} } func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error { @@ -253,25 +283,121 @@ func (bu *backendURL) put() { } func (up *URLPrefix) getBackendsCount() int { - return len(up.bus) + pbus := up.bus.Load() + return len(*pbus) } // getBackendURL returns the backendURL depending on the load balance policy. // // backendURL.put() must be called on the returned backendURL after the request is complete. func (up *URLPrefix) getBackendURL() *backendURL { + up.discoverBackendIPsIfNeeded() + + pbus := up.bus.Load() + bus := *pbus if up.loadBalancingPolicy == "first_available" { - return up.getFirstAvailableBackendURL() + return getFirstAvailableBackendURL(bus) } - return up.getLeastLoadedBackendURL() + return getLeastLoadedBackendURL(bus, &up.n) +} + +func (up *URLPrefix) discoverBackendIPsIfNeeded() { + if !up.discoverBackendIPs { + // The discovery is disabled. + return + } + + ct := fasttime.UnixTimestamp() + deadline := up.nextDiscoveryDeadline.Load() + if ct < deadline { + // There is no need in discovering backends. + return + } + + intervalSec := math.Ceil(discoverBackendIPsInterval.Seconds()) + if intervalSec <= 0 { + intervalSec = 1 + } + nextDeadline := ct + uint64(intervalSec) + if !up.nextDiscoveryDeadline.CompareAndSwap(deadline, nextDeadline) { + // Concurrent goroutine already started the discovery. + return + } + + // Discover ips for all the backendURLs + ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(intervalSec)) + hostToIPs := make(map[string][]string) + for _, bu := range up.busOriginal { + host := bu.Hostname() + if hostToIPs[host] != nil { + // ips for the given host have been already discovered + continue + } + addrs, err := resolver.LookupIPAddr(ctx, host) + var ips []string + if err != nil { + logger.Warnf("cannot discover backend IPs for %s: %s; use it literally", bu, err) + ips = []string{host} + } else { + ips = make([]string, len(addrs)) + for i, addr := range addrs { + ips[i] = addr.String() + } + // sort ips, so they could be compared below in areEqualBackendURLs() + sort.Strings(ips) + } + hostToIPs[host] = ips + } + cancel() + + // generate new backendURLs for the resolved IPs + var busNew []*backendURL + for _, bu := range up.busOriginal { + host := bu.Hostname() + port := bu.Port() + for _, ip := range hostToIPs[host] { + buCopy := *bu + buCopy.Host = ip + if port != "" { + buCopy.Host += ":" + port + } + busNew = append(busNew, &backendURL{ + url: &buCopy, + }) + } + } + + pbus := up.bus.Load() + if areEqualBackendURLs(*pbus, busNew) { + return + } + + // Store new backend urls + up.bus.Store(&busNew) +} + +func areEqualBackendURLs(a, b []*backendURL) bool { + if len(a) != len(b) { + return false + } + for i, aURL := range a { + bURL := b[i] + if aURL.url.String() != bURL.url.String() { + return false + } + } + return true +} + +var resolver = &net.Resolver{ + PreferGo: true, + StrictErrors: true, } // getFirstAvailableBackendURL returns the first available backendURL, which isn't broken. // // backendURL.put() must be called on the returned backendURL after the request is complete. -func (up *URLPrefix) getFirstAvailableBackendURL() *backendURL { - bus := up.bus - +func getFirstAvailableBackendURL(bus []*backendURL) *backendURL { bu := bus[0] if !bu.isBroken() { // Fast path - send the request to the first url. @@ -293,8 +419,7 @@ func (up *URLPrefix) getFirstAvailableBackendURL() *backendURL { // getLeastLoadedBackendURL returns the backendURL with the minimum number of concurrent requests. // // backendURL.put() must be called on the returned backendURL after the request is complete. -func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL { - bus := up.bus +func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *backendURL { if len(bus) == 1 { // Fast path - return the only backend url. bu := bus[0] @@ -303,7 +428,7 @@ func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL { } // Slow path - select other backend urls. - n := up.n.Add(1) + n := atomicCounter.Add(1) for i := uint32(0); i < uint32(len(bus)); i++ { idx := (n + i) % uint32(len(bus)) @@ -341,6 +466,7 @@ func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error { if err := f(&v); err != nil { return err } + up.vOriginal = v var urls []string switch x := v.(type) { @@ -363,38 +489,21 @@ func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error { return fmt.Errorf("unexpected type for `url_prefix`: %T; want string or []string", v) } - bus := make([]*backendURL, len(urls)) + bus := make([]*url.URL, len(urls)) for i, u := range urls { pu, err := url.Parse(u) if err != nil { return fmt.Errorf("cannot unmarshal %q into url: %w", u, err) } - bus[i] = &backendURL{ - url: pu, - } + bus[i] = pu } - up.bus = bus + up.busOriginal = bus return nil } // MarshalYAML marshals up to yaml. func (up *URLPrefix) MarshalYAML() (interface{}, error) { - var b []byte - if len(up.bus) == 1 { - u := up.bus[0].url.String() - b = strconv.AppendQuote(b, u) - return string(b), nil - } - b = append(b, '[') - for i, bu := range up.bus { - u := bu.url.String() - b = strconv.AppendQuote(b, u) - if i+1 < len(up.bus) { - b = append(b, ',') - } - } - b = append(b, ']') - return string(b), nil + return up.vOriginal, nil } func (r *Regex) match(s string) bool { @@ -415,12 +524,13 @@ func (r *Regex) UnmarshalYAML(f func(interface{}) error) error { if err := f(&s); err != nil { return err } + r.sOriginal = s + sAnchored := "^(?:" + s + ")$" re, err := regexp.Compile(sAnchored) if err != nil { return fmt.Errorf("cannot build regexp from %q: %w", s, err) } - r.sOriginal = s r.re = re return nil } @@ -689,8 +799,9 @@ func (ui *UserInfo) initURLs() error { retryStatusCodes := defaultRetryStatusCodes.Values() loadBalancingPolicy := *defaultLoadBalancingPolicy dropSrcPathPrefixParts := 0 + discoverBackendIPs := *discoverBackendIPsGlobal if ui.URLPrefix != nil { - if err := ui.URLPrefix.sanitize(); err != nil { + if err := ui.URLPrefix.sanitizeAndInitialize(); err != nil { return err } if ui.RetryStatusCodes != nil { @@ -702,14 +813,18 @@ func (ui *UserInfo) initURLs() error { if ui.DropSrcPathPrefixParts != nil { dropSrcPathPrefixParts = *ui.DropSrcPathPrefixParts } + if ui.DiscoverBackendIPs != nil { + discoverBackendIPs = *ui.DiscoverBackendIPs + } ui.URLPrefix.retryStatusCodes = retryStatusCodes ui.URLPrefix.dropSrcPathPrefixParts = dropSrcPathPrefixParts + ui.URLPrefix.discoverBackendIPs = discoverBackendIPs if err := ui.URLPrefix.setLoadBalancingPolicy(loadBalancingPolicy); err != nil { return err } } if ui.DefaultURL != nil { - if err := ui.DefaultURL.sanitize(); err != nil { + if err := ui.DefaultURL.sanitizeAndInitialize(); err != nil { return err } } @@ -720,12 +835,13 @@ func (ui *UserInfo) initURLs() error { if e.URLPrefix == nil { return fmt.Errorf("missing `url_prefix` in `url_map`") } - if err := e.URLPrefix.sanitize(); err != nil { + if err := e.URLPrefix.sanitizeAndInitialize(); err != nil { return err } rscs := retryStatusCodes lbp := loadBalancingPolicy dsp := dropSrcPathPrefixParts + dbd := discoverBackendIPs if e.RetryStatusCodes != nil { rscs = e.RetryStatusCodes } @@ -735,11 +851,15 @@ func (ui *UserInfo) initURLs() error { if e.DropSrcPathPrefixParts != nil { dsp = *e.DropSrcPathPrefixParts } + if e.DiscoverBackendIPs != nil { + dbd = *e.DiscoverBackendIPs + } e.URLPrefix.retryStatusCodes = rscs if err := e.URLPrefix.setLoadBalancingPolicy(lbp); err != nil { return err } e.URLPrefix.dropSrcPathPrefixParts = dsp + e.URLPrefix.discoverBackendIPs = dbd } if len(ui.URLMaps) == 0 && ui.URLPrefix == nil { return fmt.Errorf("missing `url_prefix` or `url_map`") @@ -805,14 +925,24 @@ func getAuthTokensFromRequest(r *http.Request) []string { return ats } -func (up *URLPrefix) sanitize() error { - for _, bu := range up.bus { - puNew, err := sanitizeURLPrefix(bu.url) +func (up *URLPrefix) sanitizeAndInitialize() error { + for i, bu := range up.busOriginal { + puNew, err := sanitizeURLPrefix(bu) if err != nil { return err } - bu.url = puNew + up.busOriginal[i] = puNew } + + // Initialize up.bus + bus := make([]*backendURL, len(up.busOriginal)) + for i, bu := range up.busOriginal { + bus[i] = &backendURL{ + url: bu, + } + } + up.bus.Store(&bus) + return nil } diff --git a/app/vmauth/auth_config_test.go b/app/vmauth/auth_config_test.go index e3d1d52bf..00965ddab 100644 --- a/app/vmauth/auth_config_test.go +++ b/app/vmauth/auth_config_test.go @@ -328,7 +328,7 @@ users: - username: foo url_prefix: http://foo - username: bar - url_prefix: https://bar/x/// + url_prefix: https://bar/x/ `, map[string]*UserInfo{ getHTTPAuthBasicToken("foo", ""): { Username: "foo", @@ -336,7 +336,7 @@ users: }, getHTTPAuthBasicToken("bar", ""): { Username: "bar", - URLPrefix: mustParseURL("https://bar/x"), + URLPrefix: mustParseURL("https://bar/x/"), }, }) @@ -409,7 +409,7 @@ users: url_prefix: http://foo - username: foo-same password: bar - url_prefix: https://bar/x/// + url_prefix: https://bar/x `, map[string]*UserInfo{ getHTTPAuthBasicToken("foo-same", "baz"): { Username: "foo-same", @@ -516,7 +516,7 @@ users: team: dev - username: foo-same password: bar - url_prefix: https://bar/x/// + url_prefix: https://bar/x metric_labels: backend_env: test team: accounting @@ -710,9 +710,14 @@ func mustParseURLs(us []string) *URLPrefix { url: pu, } } - return &URLPrefix{ - bus: bus, + up := &URLPrefix{} + if len(us) == 1 { + up.vOriginal = us[0] + } else { + up.vOriginal = us } + up.bus.Store(&bus) + return up } func intp(n int) *int { diff --git a/app/vmauth/main.go b/app/vmauth/main.go index 05a8234ad..bcfc275f1 100644 --- a/app/vmauth/main.go +++ b/app/vmauth/main.go @@ -239,7 +239,14 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url // This code has been copied from net/http/httputil/reverseproxy.go req := sanitizeRequestHeaders(r) req.URL = targetURL - req.Host = targetURL.Host + + if req.URL.Scheme == "https" { + // Override req.Host only for https requests, since https server verifies hostnames during TLS handshake, + // so it expects the targetURL.Host in the request. + // There is no need in overriding the req.Host for http requests, since it is expected that backend server + // may properly process queries with the original req.Host. + req.Host = targetURL.Host + } updateHeadersByConfig(req.Header, hc.RequestHeaders) res, err := ui.httpTransport.RoundTrip(req) rtb, rtbOK := req.Body.(*readTrackingBody) diff --git a/app/vmauth/target_url_test.go b/app/vmauth/target_url_test.go index 9b8a1da63..17e298308 100644 --- a/app/vmauth/target_url_test.go +++ b/app/vmauth/target_url_test.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "reflect" + "strings" "testing" ) @@ -93,15 +94,17 @@ func TestCreateTargetURLSuccess(t *testing.T) { if up == nil { t.Fatalf("cannot determie backend: %s", err) } - bu := up.getLeastLoadedBackendURL() + bu := up.getBackendURL() target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts) bu.put() if target.String() != expectedTarget { t.Fatalf("unexpected target; got %q; want %q", target, expectedTarget) } - headersStr := fmt.Sprintf("%q", hc.RequestHeaders) - if headersStr != expectedRequestHeaders { - t.Fatalf("unexpected request headers; got %s; want %s", headersStr, expectedRequestHeaders) + if s := headersToString(hc.RequestHeaders); s != expectedRequestHeaders { + t.Fatalf("unexpected request headers; got %q; want %q", s, expectedRequestHeaders) + } + if s := headersToString(hc.ResponseHeaders); s != expectedResponseHeaders { + t.Fatalf("unexpected response headers; got %q; want %q", s, expectedResponseHeaders) } if !reflect.DeepEqual(up.retryStatusCodes, expectedRetryStatusCodes) { t.Fatalf("unexpected retryStatusCodes; got %d; want %d", up.retryStatusCodes, expectedRetryStatusCodes) @@ -116,34 +119,42 @@ func TestCreateTargetURLSuccess(t *testing.T) { // Simple routing with `url_prefix` f(&UserInfo{ URLPrefix: mustParseURL("http://foo.bar"), - }, "", "http://foo.bar/.", "[]", "[]", nil, "least_loaded", 0) + }, "", "http://foo.bar/.", "", "", nil, "least_loaded", 0) f(&UserInfo{ URLPrefix: mustParseURL("http://foo.bar"), HeadersConf: HeadersConf{ - RequestHeaders: []Header{{ - Name: "bb", - Value: "aaa", - }}, + RequestHeaders: []Header{ + { + Name: "bb", + Value: "aaa", + }, + }, + ResponseHeaders: []Header{ + { + Name: "x", + Value: "y", + }, + }, }, RetryStatusCodes: []int{503, 501}, LoadBalancingPolicy: "first_available", DropSrcPathPrefixParts: intp(2), - }, "/a/b/c", "http://foo.bar/c", `[{"bb" "aaa"}]`, `[]`, []int{503, 501}, "first_available", 2) + }, "/a/b/c", "http://foo.bar/c", `bb: aaa`, `x: y`, []int{503, 501}, "first_available", 2) f(&UserInfo{ URLPrefix: mustParseURL("http://foo.bar/federate"), - }, "/", "http://foo.bar/federate", "[]", "[]", nil, "least_loaded", 0) + }, "/", "http://foo.bar/federate", "", "", nil, "least_loaded", 0) f(&UserInfo{ URLPrefix: mustParseURL("http://foo.bar"), - }, "a/b?c=d", "http://foo.bar/a/b?c=d", "[]", "[]", nil, "least_loaded", 0) + }, "a/b?c=d", "http://foo.bar/a/b?c=d", "", "", nil, "least_loaded", 0) f(&UserInfo{ URLPrefix: mustParseURL("https://sss:3894/x/y"), - }, "/z", "https://sss:3894/x/y/z", "[]", "[]", nil, "least_loaded", 0) + }, "/z", "https://sss:3894/x/y/z", "", "", nil, "least_loaded", 0) f(&UserInfo{ URLPrefix: mustParseURL("https://sss:3894/x/y"), - }, "/../../aaa", "https://sss:3894/x/y/aaa", "[]", "[]", nil, "least_loaded", 0) + }, "/../../aaa", "https://sss:3894/x/y/aaa", "", "", nil, "least_loaded", 0) f(&UserInfo{ URLPrefix: mustParseURL("https://sss:3894/x/y"), - }, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s%2F..%2Fd", "[]", "[]", nil, "least_loaded", 0) + }, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s%2F..%2Fd", "", "", nil, "least_loaded", 0) // Complex routing with `url_map` ui := &UserInfo{ @@ -202,11 +213,11 @@ func TestCreateTargetURLSuccess(t *testing.T) { DropSrcPathPrefixParts: intp(2), } f(ui, "http://host42/vmsingle/api/v1/query?query=up&db=foo", "http://vmselect/0/prometheus/api/v1/query?db=foo&query=up", - `[{"xx" "aa"} {"yy" "asdf"}]`, `[{"qwe" "rty"}]`, []int{503, 500, 501}, "first_available", 1) + "xx: aa\nyy: asdf", "qwe: rty", []int{503, 500, 501}, "first_available", 1) f(ui, "http://host123/vmsingle/api/v1/query?query=up", "http://default-server/v1/query?query=up", - `[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2) - f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", []int{}, "least_loaded", 0) - f(ui, "https://foo-host/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", `[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2) + "bb: aaa", "x: y", []int{502}, "least_loaded", 2) + f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "", "", []int{}, "least_loaded", 0) + f(ui, "https://foo-host/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", "bb: aaa", "x: y", []int{502}, "least_loaded", 2) // Complex routing regexp paths in `url_map` ui = &UserInfo{ @@ -226,19 +237,19 @@ func TestCreateTargetURLSuccess(t *testing.T) { }, URLPrefix: mustParseURL("http://default-server"), } - f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", "[]", "[]", nil, "least_loaded", 0) - f(ui, "/api/v1/query_range?query=up", "http://vmselect/0/prometheus/api/v1/query_range?query=up", "[]", "[]", nil, "least_loaded", 0) - f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values", "[]", "[]", nil, "least_loaded", 0) - f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", nil, "least_loaded", 0) - f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar", "[]", "[]", nil, "least_loaded", 0) - f(ui, "https://vmui.foobar.com/a/b?c=d", "http://vmui.host:1234/vmui/a/b?c=d", "[]", "[]", nil, "least_loaded", 0) + f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", "", "", nil, "least_loaded", 0) + f(ui, "/api/v1/query_range?query=up", "http://vmselect/0/prometheus/api/v1/query_range?query=up", "", "", nil, "least_loaded", 0) + f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values", "", "", nil, "least_loaded", 0) + f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "", "", nil, "least_loaded", 0) + f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar", "", "", nil, "least_loaded", 0) + f(ui, "https://vmui.foobar.com/a/b?c=d", "http://vmui.host:1234/vmui/a/b?c=d", "", "", nil, "least_loaded", 0) f(&UserInfo{ URLPrefix: mustParseURL("http://foo.bar?extra_label=team=dev"), - }, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "[]", "[]", nil, "least_loaded", 0) + }, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "", "", nil, "least_loaded", 0) f(&UserInfo{ URLPrefix: mustParseURL("http://foo.bar?extra_label=team=mobile"), - }, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team%3Dmobile", "[]", "[]", nil, "least_loaded", 0) + }, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team%3Dmobile", "", "", nil, "least_loaded", 0) } func TestCreateTargetURLFailure(t *testing.T) { @@ -270,3 +281,11 @@ func TestCreateTargetURLFailure(t *testing.T) { }, }, "/api/v1/write") } + +func headersToString(hs []Header) string { + a := make([]string, len(hs)) + for i, h := range hs { + a[i] = fmt.Sprintf("%s: %s", h.Name, h.Value) + } + return strings.Join(a, "\n") +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b270c272d..75848f576 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -32,6 +32,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * SECURITY: upgrade Go builder from Go1.21.7 to Go1.22.1. See [the list of issues addressed in Go1.22.1](https://github.com/golang/go/issues?q=milestone%3AGo1.22.1+label%3ACherryPickApproved). +* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): allow discovering ip addresses for backend instances hidden behind a shared hostname, via `discover_backend_ips: true` option. This allows evenly spreading load among backend instances. See [these docs](https://docs.victoriametrics.com/vmauth/#discovering-backend-ips) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5707). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): allow routing incoming requests bassed on HTTP [query args](https://en.wikipedia.org/wiki/Query_string) via `src_query_args` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth/#generic-http-proxy-for-different-backends) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5878). * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): allow routing incoming requests bassed on HTTP request headers via `src_headers` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth/#generic-http-proxy-for-different-backends). * FEATURE: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): reduce memory usage by up to 5x when aggregating over big number of unique [time series](https://docs.victoriametrics.com/keyconcepts/#time-series). The memory usage reduction is most visible when [stream deduplication](https://docs.victoriametrics.com/stream-aggregation/#deduplication) is enabled. The downside is increased CPU usage by up to 30%. diff --git a/docs/vmauth.md b/docs/vmauth.md index 819134db9..4cc4d8cfb 100644 --- a/docs/vmauth.md +++ b/docs/vmauth.md @@ -401,6 +401,7 @@ Each `url_prefix` in the [-auth.config](#auth-config) can be specified in the fo In this case `vmauth` spreads requests among the specified urls using least-loaded round-robin policy. This guarantees that incoming load is shared uniformly among the specified backends. + See also [discovering backend IPs](#discovering-backend-ips). `vmauth` automatically detects temporarily unavailable backends and spreads incoming queries among the remaining available backends. This allows restarting the backends and peforming mantenance tasks on the backends without the need to remove them from the `url_prefix` list. @@ -466,6 +467,49 @@ Load balancing feature can be used in the following cases: Load balancig can be configured independently per each `user` entry and per each `url_map` entry. See [auth config docs](#auth-config) for more details. +See also [discovering backend IPs](#discovering-backend-ips). + +## Discovering backend IPs + +By default `vmauth` spreads load among the listed backends at `url_prefix` as described in [load balancing docs](#load-balancing). +Sometimes multiple backend instances can be hidden behind a single hostname. For example, `vmselect-service` hostname +may point to a cluster of `vmselect` instances in [VictoriaMetrics cluster setup](https://docs.victoriametrics.com/cluster-victoriametrics/#architecture-overview). +So the following config may fail spreading load among available `vmselect` instances, since `vmauth` will send all the requests to the same url, which may end up +to a single backend instance: + +```yaml +unauthorized_user: + url_prefix: http://vmselect-service/select/0/prometheus/ +``` + +There are the following solutions for this issue: + +- To enumerate every `vmselect` hosname or IP in the `url_prefix` list: + + ```yaml + unauthorized_user: + url_prefix: + - http://vmselect-1:8481/select/0/prometheus/ + - http://vmselect-2:8481/select/0/prometheus/ + - http://vmselect-3:8481/select/0/prometheus/ + ``` + + This scheme works great, but it needs manual updating of the [`-auth.config`](#auth-config) every time `vmselect` services are restarted, + downsaled or upscaled. + +- To set `discover_backend_ips: true` option, so `vmagent` automatically discovers IPs behind the given hostname and then spreads load among the discovered IPs: + + ```yaml + unauthorized_user: + url_prefix: http://vmselect-service/select/0/prometheus/ + discover_backend_ips: true + ``` + + The `discover_backend_ips` can be specified at `user` and `url_map` level in the [`-auth.config](#auth-config). It can also be enabled globally + via `-discoverBackendIPs` command-line flag. + +See also [load balancing docs](#load-balancing). + ## Modifying HTTP headers `vmauth` supports the ability to set and remove HTTP request headers before sending the requests to backends.