Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-23 12:31:07 +01:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

app/vmauth: allow discovering backend ips behind shared hostname and spreading load among the discovered ips

Browse Source 
This is done with the `discover_backend_ips` option at `user` and `url_map` level.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5707
This commit is contained in:
Aliaksandr Valialkin 2024-03-07 01:02:13 +02:00
parent 76ef84fcae
commit 7b2b980181
No known key found for this signature in database
GPG Key ID: 52C003EE2BCDB9EB
6 changed files with 292 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

234
app/vmauth/auth_config.go
View File

@ -2,15 +2,17 @@ package main
import (
"bytes"
"context"
"encoding/base64"
"flag"
"fmt"
"math"
"net"
"net/http"
"net/url"
"os"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
@ -36,7 +38,11 @@ var (
defaultRetryStatusCodes = flagutil.NewArrayInt("retryStatusCodes", 0, "Comma-separated list of default HTTP response status codes when vmauth re-tries the request on other backends. "+
"See https://docs.victoriametrics.com/vmauth.html#load-balancing for details")
defaultLoadBalancingPolicy = flag.String("loadBalancingPolicy", "least_loaded", "The default load balancing policy to use for backend urls specified inside url_prefix section. "+
"Supported policies: least_loaded, first_available. See https://docs.victoriametrics.com/vmauth.html#load-balancing for more details")
"Supported policies: least_loaded, first_available. See https://docs.victoriametrics.com/vmauth.html#load-balancing")
discoverBackendIPsGlobal = flag.Bool("discoverBackendIPs", false, "Whether to discover backend IPs via periodic DNS queries to hostnames specified in url_prefix. "+
"This may be useful when url_prefix points to a hostname with dynamically scaled instances behind it. See https://docs.victoriametrics.com/vmauth.html#discovering-backend-ips")
discoverBackendIPsInterval = flag.Duration("discoverBackendIPsInterval", 10*time.Second, "The interval for re-discovering backend IPs if -discoverBackendIPs command-line flag is set. "+
"Too low value may lead to DNS errors")
)
// AuthConfig represents auth config.
@ -57,6 +63,7 @@ type UserInfo struct {
Password string `yaml:"password,omitempty"`
URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
DiscoverBackendIPs *bool `yaml:"discover_backend_ips,omitempty"`
URLMaps []URLMap `yaml:"url_map,omitempty"`
HeadersConf HeadersConf `yaml:",inline"`
MaxConcurrentRequests int `yaml:"max_concurrent_requests,omitempty"`
@ -111,6 +118,8 @@ func (ui *UserInfo) getMaxConcurrentRequests() int {
type Header struct {
Name string
Value string
sOriginal string
}
// UnmarshalYAML unmarshals h from f.
@ -119,6 +128,8 @@ func (h *Header) UnmarshalYAML(f func(interface{}) error) error {
if err := f(&s); err != nil {
return err
}
h.sOriginal = s
n := strings.IndexByte(s, ':')
if n < 0 {
return fmt.Errorf("missing speparator char ':' between Name and Value in the header %q; expected format - 'Name: Value'", s)
@ -130,8 +141,7 @@ func (h *Header) UnmarshalYAML(f func(interface{}) error) error {
// MarshalYAML marshals h to yaml.
func (h *Header) MarshalYAML() (interface{}, error) {
s := fmt.Sprintf("%s: %s", h.Name, h.Value)
return s, nil
return h.sOriginal, nil
}
// URLMap is a mapping from source paths to target urls.
@ -151,6 +161,9 @@ type URLMap struct {
// UrlPrefix contains backend url prefixes for the proxied request url.
URLPrefix *URLPrefix `yaml:"url_prefix,omitempty"`
// DiscoverBackendIPs instructs discovering URLPrefix backend IPs via DNS.
DiscoverBackendIPs *bool `yaml:"discover_backend_ips,omitempty"`
// HeadersConf is the config for augumenting request and response headers.
HeadersConf HeadersConf `yaml:",inline"`
@ -181,14 +194,16 @@ type QueryArg struct {
// UnmarshalYAML unmarshals up from yaml.
func (qa *QueryArg) UnmarshalYAML(f func(interface{}) error) error {
if err := f(&qa.sOriginal); err != nil {
var s string
if err := f(&s); err != nil {
return err
}
qa.sOriginal = s
n := strings.IndexByte(qa.sOriginal, '=')
n := strings.IndexByte(s, '=')
if n >= 0 {
qa.Name = qa.sOriginal[:n]
qa.Value = qa.sOriginal[n+1:]
qa.Name = s[:n]
qa.Value = s[n+1:]
}
return nil
}
@ -200,19 +215,34 @@ func (qa *QueryArg) MarshalYAML() (interface{}, error) {
// URLPrefix represents passed `url_prefix`
type URLPrefix struct {
n atomic.Uint32
// the list of backend urls
bus []*backendURL
// requests are re-tried on other backend urls for these http response status codes
retryStatusCodes []int
// load balancing policy used
loadBalancingPolicy string
// how many request path prefix parts to drop before routing the request to backendURL.
// how many request path prefix parts to drop before routing the request to backendURL
dropSrcPathPrefixParts int
// busOriginal contains the original list of backends specified in yaml config.
busOriginal []*url.URL
// n is an atomic counter, which is used for balancing load among available backends.
n atomic.Uint32
// the list of backend urls
//
// the list can be dynamically updated if `discover_backend_ips` option is set.
bus atomic.Pointer[[]*backendURL]
// if this option is set, then backend ips for busOriginal are periodically re-discovered and put to bus.
discoverBackendIPs bool
// The next deadline for DNS-based discovery of backend IPs
nextDiscoveryDeadline atomic.Uint64
// vOriginal contains the original yaml value for URLPrefix.
vOriginal interface{}
}
func (up *URLPrefix) setLoadBalancingPolicy(loadBalancingPolicy string) error {
@ -253,25 +283,121 @@ func (bu *backendURL) put() {
}
func (up *URLPrefix) getBackendsCount() int {
return len(up.bus)
pbus := up.bus.Load()
return len(*pbus)
}
// getBackendURL returns the backendURL depending on the load balance policy.
//
// backendURL.put() must be called on the returned backendURL after the request is complete.
func (up *URLPrefix) getBackendURL() *backendURL {
up.discoverBackendIPsIfNeeded()
pbus := up.bus.Load()
bus := *pbus
if up.loadBalancingPolicy == "first_available" {
return up.getFirstAvailableBackendURL()
return getFirstAvailableBackendURL(bus)
}
return up.getLeastLoadedBackendURL()
return getLeastLoadedBackendURL(bus, &up.n)
}
func (up *URLPrefix) discoverBackendIPsIfNeeded() {
if !up.discoverBackendIPs {
// The discovery is disabled.
return
}
ct := fasttime.UnixTimestamp()
deadline := up.nextDiscoveryDeadline.Load()
if ct < deadline {
// There is no need in discovering backends.
return
}
intervalSec := math.Ceil(discoverBackendIPsInterval.Seconds())
if intervalSec <= 0 {
intervalSec = 1
}
nextDeadline := ct + uint64(intervalSec)
if !up.nextDiscoveryDeadline.CompareAndSwap(deadline, nextDeadline) {
// Concurrent goroutine already started the discovery.
return
}
// Discover ips for all the backendURLs
ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(intervalSec))
hostToIPs := make(map[string][]string)
for _, bu := range up.busOriginal {
host := bu.Hostname()
if hostToIPs[host] != nil {
// ips for the given host have been already discovered
continue
}
addrs, err := resolver.LookupIPAddr(ctx, host)
var ips []string
if err != nil {
logger.Warnf("cannot discover backend IPs for %s: %s; use it literally", bu, err)
ips = []string{host}
} else {
ips = make([]string, len(addrs))
for i, addr := range addrs {
ips[i] = addr.String()
}
// sort ips, so they could be compared below in areEqualBackendURLs()
sort.Strings(ips)
}
hostToIPs[host] = ips
}
cancel()
// generate new backendURLs for the resolved IPs
var busNew []*backendURL
for _, bu := range up.busOriginal {
host := bu.Hostname()
port := bu.Port()
for _, ip := range hostToIPs[host] {
buCopy := *bu
buCopy.Host = ip
if port != "" {
buCopy.Host += ":" + port
}
busNew = append(busNew, &backendURL{
url: &buCopy,
})
}
}
pbus := up.bus.Load()
if areEqualBackendURLs(*pbus, busNew) {
return
}
// Store new backend urls
up.bus.Store(&busNew)
}
func areEqualBackendURLs(a, b []*backendURL) bool {
if len(a) != len(b) {
return false
}
for i, aURL := range a {
bURL := b[i]
if aURL.url.String() != bURL.url.String() {
return false
}
}
return true
}
var resolver = &net.Resolver{
PreferGo: true,
StrictErrors: true,
}
// getFirstAvailableBackendURL returns the first available backendURL, which isn't broken.
//
// backendURL.put() must be called on the returned backendURL after the request is complete.
func (up *URLPrefix) getFirstAvailableBackendURL() *backendURL {
bus := up.bus
func getFirstAvailableBackendURL(bus []*backendURL) *backendURL {
bu := bus[0]
if !bu.isBroken() {
// Fast path - send the request to the first url.
@ -293,8 +419,7 @@ func (up *URLPrefix) getFirstAvailableBackendURL() *backendURL {
// getLeastLoadedBackendURL returns the backendURL with the minimum number of concurrent requests.
//
// backendURL.put() must be called on the returned backendURL after the request is complete.
func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL {
bus := up.bus
func getLeastLoadedBackendURL(bus []*backendURL, atomicCounter *atomic.Uint32) *backendURL {
if len(bus) == 1 {
// Fast path - return the only backend url.
bu := bus[0]
@ -303,7 +428,7 @@ func (up *URLPrefix) getLeastLoadedBackendURL() *backendURL {
}
// Slow path - select other backend urls.
n := up.n.Add(1)
n := atomicCounter.Add(1)
for i := uint32(0); i < uint32(len(bus)); i++ {
idx := (n + i) % uint32(len(bus))
@ -341,6 +466,7 @@ func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error {
if err := f(&v); err != nil {
return err
}
up.vOriginal = v
var urls []string
switch x := v.(type) {
@ -363,38 +489,21 @@ func (up *URLPrefix) UnmarshalYAML(f func(interface{}) error) error {
return fmt.Errorf("unexpected type for `url_prefix`: %T; want string or []string", v)
}
bus := make([]*backendURL, len(urls))
bus := make([]*url.URL, len(urls))
for i, u := range urls {
pu, err := url.Parse(u)
if err != nil {
return fmt.Errorf("cannot unmarshal %q into url: %w", u, err)
}
bus[i] = &backendURL{
url: pu,
}
bus[i] = pu
}
up.bus = bus
up.busOriginal = bus
return nil
}
// MarshalYAML marshals up to yaml.
func (up *URLPrefix) MarshalYAML() (interface{}, error) {
var b []byte
if len(up.bus) == 1 {
u := up.bus[0].url.String()
b = strconv.AppendQuote(b, u)
return string(b), nil
}
b = append(b, '[')
for i, bu := range up.bus {
u := bu.url.String()
b = strconv.AppendQuote(b, u)
if i+1 < len(up.bus) {
b = append(b, ',')
}
}
b = append(b, ']')
return string(b), nil
return up.vOriginal, nil
}
func (r *Regex) match(s string) bool {
@ -415,12 +524,13 @@ func (r *Regex) UnmarshalYAML(f func(interface{}) error) error {
if err := f(&s); err != nil {
return err
}
r.sOriginal = s
sAnchored := "^(?:" + s + ")$"
re, err := regexp.Compile(sAnchored)
if err != nil {
return fmt.Errorf("cannot build regexp from %q: %w", s, err)
}
r.sOriginal = s
r.re = re
return nil
}
@ -689,8 +799,9 @@ func (ui *UserInfo) initURLs() error {
retryStatusCodes := defaultRetryStatusCodes.Values()
loadBalancingPolicy := *defaultLoadBalancingPolicy
dropSrcPathPrefixParts := 0
discoverBackendIPs := *discoverBackendIPsGlobal
if ui.URLPrefix != nil {
if err := ui.URLPrefix.sanitize(); err != nil {
if err := ui.URLPrefix.sanitizeAndInitialize(); err != nil {
return err
}
if ui.RetryStatusCodes != nil {
@ -702,14 +813,18 @@ func (ui *UserInfo) initURLs() error {
if ui.DropSrcPathPrefixParts != nil {
dropSrcPathPrefixParts = *ui.DropSrcPathPrefixParts
}
if ui.DiscoverBackendIPs != nil {
discoverBackendIPs = *ui.DiscoverBackendIPs
}
ui.URLPrefix.retryStatusCodes = retryStatusCodes
ui.URLPrefix.dropSrcPathPrefixParts = dropSrcPathPrefixParts
ui.URLPrefix.discoverBackendIPs = discoverBackendIPs
if err := ui.URLPrefix.setLoadBalancingPolicy(loadBalancingPolicy); err != nil {
return err
}
}
if ui.DefaultURL != nil {
if err := ui.DefaultURL.sanitize(); err != nil {
if err := ui.DefaultURL.sanitizeAndInitialize(); err != nil {
return err
}
}
@ -720,12 +835,13 @@ func (ui *UserInfo) initURLs() error {
if e.URLPrefix == nil {
return fmt.Errorf("missing `url_prefix` in `url_map`")
}
if err := e.URLPrefix.sanitize(); err != nil {
if err := e.URLPrefix.sanitizeAndInitialize(); err != nil {
return err
}
rscs := retryStatusCodes
lbp := loadBalancingPolicy
dsp := dropSrcPathPrefixParts
dbd := discoverBackendIPs
if e.RetryStatusCodes != nil {
rscs = e.RetryStatusCodes
}
@ -735,11 +851,15 @@ func (ui *UserInfo) initURLs() error {
if e.DropSrcPathPrefixParts != nil {
dsp = *e.DropSrcPathPrefixParts
}
if e.DiscoverBackendIPs != nil {
dbd = *e.DiscoverBackendIPs
}
e.URLPrefix.retryStatusCodes = rscs
if err := e.URLPrefix.setLoadBalancingPolicy(lbp); err != nil {
return err
}
e.URLPrefix.dropSrcPathPrefixParts = dsp
e.URLPrefix.discoverBackendIPs = dbd
}
if len(ui.URLMaps) == 0 && ui.URLPrefix == nil {
return fmt.Errorf("missing `url_prefix` or `url_map`")
@ -805,14 +925,24 @@ func getAuthTokensFromRequest(r *http.Request) []string {
return ats
}
func (up *URLPrefix) sanitize() error {
for _, bu := range up.bus {
puNew, err := sanitizeURLPrefix(bu.url)
func (up *URLPrefix) sanitizeAndInitialize() error {
for i, bu := range up.busOriginal {
puNew, err := sanitizeURLPrefix(bu)
if err != nil {
return err
}
bu.url = puNew
up.busOriginal[i] = puNew
}
// Initialize up.bus
bus := make([]*backendURL, len(up.busOriginal))
for i, bu := range up.busOriginal {
bus[i] = &backendURL{
url: bu,
}
}
up.bus.Store(&bus)
return nil
}

17
app/vmauth/auth_config_test.go
View File

@ -328,7 +328,7 @@ users:
- username: foo
url_prefix: http://foo
- username: bar
url_prefix: https://bar/x///
url_prefix: https://bar/x/
`, map[string]*UserInfo{
getHTTPAuthBasicToken("foo", ""): {
Username: "foo",
@ -336,7 +336,7 @@ users:
},
getHTTPAuthBasicToken("bar", ""): {
Username: "bar",
URLPrefix: mustParseURL("https://bar/x"),
URLPrefix: mustParseURL("https://bar/x/"),
},
})
@ -409,7 +409,7 @@ users:
url_prefix: http://foo
- username: foo-same
password: bar
url_prefix: https://bar/x///
url_prefix: https://bar/x
`, map[string]*UserInfo{
getHTTPAuthBasicToken("foo-same", "baz"): {
Username: "foo-same",
@ -516,7 +516,7 @@ users:
team: dev
- username: foo-same
password: bar
url_prefix: https://bar/x///
url_prefix: https://bar/x
metric_labels:
backend_env: test
team: accounting
@ -710,9 +710,14 @@ func mustParseURLs(us []string) *URLPrefix {
url: pu,
}
}
return &URLPrefix{
bus: bus,
up := &URLPrefix{}
if len(us) == 1 {
up.vOriginal = us[0]
} else {
up.vOriginal = us
}
up.bus.Store(&bus)
return up
}
func intp(n int) *int {

9
app/vmauth/main.go
View File

@ -239,7 +239,14 @@ func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url
// This code has been copied from net/http/httputil/reverseproxy.go
req := sanitizeRequestHeaders(r)
req.URL = targetURL
req.Host = targetURL.Host
if req.URL.Scheme == "https" {
// Override req.Host only for https requests, since https server verifies hostnames during TLS handshake,
// so it expects the targetURL.Host in the request.
// There is no need in overriding the req.Host for http requests, since it is expected that backend server
// may properly process queries with the original req.Host.
req.Host = targetURL.Host
}
updateHeadersByConfig(req.Header, hc.RequestHeaders)
res, err := ui.httpTransport.RoundTrip(req)
rtb, rtbOK := req.Body.(*readTrackingBody)

73
app/vmauth/target_url_test.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"net/url"
"reflect"
"strings"
"testing"
)
@ -93,15 +94,17 @@ func TestCreateTargetURLSuccess(t *testing.T) {
if up == nil {
t.Fatalf("cannot determie backend: %s", err)
}
bu := up.getLeastLoadedBackendURL()
bu := up.getBackendURL()
target := mergeURLs(bu.url, u, up.dropSrcPathPrefixParts)
bu.put()
if target.String() != expectedTarget {
t.Fatalf("unexpected target; got %q; want %q", target, expectedTarget)
}
headersStr := fmt.Sprintf("%q", hc.RequestHeaders)
if headersStr != expectedRequestHeaders {
t.Fatalf("unexpected request headers; got %s; want %s", headersStr, expectedRequestHeaders)
if s := headersToString(hc.RequestHeaders); s != expectedRequestHeaders {
t.Fatalf("unexpected request headers; got %q; want %q", s, expectedRequestHeaders)
}
if s := headersToString(hc.ResponseHeaders); s != expectedResponseHeaders {
t.Fatalf("unexpected response headers; got %q; want %q", s, expectedResponseHeaders)
}
if !reflect.DeepEqual(up.retryStatusCodes, expectedRetryStatusCodes) {
t.Fatalf("unexpected retryStatusCodes; got %d; want %d", up.retryStatusCodes, expectedRetryStatusCodes)
@ -116,34 +119,42 @@ func TestCreateTargetURLSuccess(t *testing.T) {
// Simple routing with `url_prefix`
f(&UserInfo{
URLPrefix: mustParseURL("http://foo.bar"),
}, "", "http://foo.bar/.", "[]", "[]", nil, "least_loaded", 0)
}, "", "http://foo.bar/.", "", "", nil, "least_loaded", 0)
f(&UserInfo{
URLPrefix: mustParseURL("http://foo.bar"),
HeadersConf: HeadersConf{
RequestHeaders: []Header{{
Name: "bb",
Value: "aaa",
}},
RequestHeaders: []Header{
{
Name: "bb",
Value: "aaa",
},
},
ResponseHeaders: []Header{
{
Name: "x",
Value: "y",
},
},
},
RetryStatusCodes: []int{503, 501},
LoadBalancingPolicy: "first_available",
DropSrcPathPrefixParts: intp(2),
}, "/a/b/c", "http://foo.bar/c", `[{"bb" "aaa"}]`, `[]`, []int{503, 501}, "first_available", 2)
}, "/a/b/c", "http://foo.bar/c", `bb: aaa`, `x: y`, []int{503, 501}, "first_available", 2)
f(&UserInfo{
URLPrefix: mustParseURL("http://foo.bar/federate"),
}, "/", "http://foo.bar/federate", "[]", "[]", nil, "least_loaded", 0)
}, "/", "http://foo.bar/federate", "", "", nil, "least_loaded", 0)
f(&UserInfo{
URLPrefix: mustParseURL("http://foo.bar"),
}, "a/b?c=d", "http://foo.bar/a/b?c=d", "[]", "[]", nil, "least_loaded", 0)
}, "a/b?c=d", "http://foo.bar/a/b?c=d", "", "", nil, "least_loaded", 0)
f(&UserInfo{
URLPrefix: mustParseURL("https://sss:3894/x/y"),
}, "/z", "https://sss:3894/x/y/z", "[]", "[]", nil, "least_loaded", 0)
}, "/z", "https://sss:3894/x/y/z", "", "", nil, "least_loaded", 0)
f(&UserInfo{
URLPrefix: mustParseURL("https://sss:3894/x/y"),
}, "/../../aaa", "https://sss:3894/x/y/aaa", "[]", "[]", nil, "least_loaded", 0)
}, "/../../aaa", "https://sss:3894/x/y/aaa", "", "", nil, "least_loaded", 0)
f(&UserInfo{
URLPrefix: mustParseURL("https://sss:3894/x/y"),
}, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s%2F..%2Fd", "[]", "[]", nil, "least_loaded", 0)
}, "/./asd/../../aaa?a=d&s=s/../d", "https://sss:3894/x/y/aaa?a=d&s=s%2F..%2Fd", "", "", nil, "least_loaded", 0)
// Complex routing with `url_map`
ui := &UserInfo{
@ -202,11 +213,11 @@ func TestCreateTargetURLSuccess(t *testing.T) {
DropSrcPathPrefixParts: intp(2),
}
f(ui, "http://host42/vmsingle/api/v1/query?query=up&db=foo", "http://vmselect/0/prometheus/api/v1/query?db=foo&query=up",
`[{"xx" "aa"} {"yy" "asdf"}]`, `[{"qwe" "rty"}]`, []int{503, 500, 501}, "first_available", 1)
"xx: aa\nyy: asdf", "qwe: rty", []int{503, 500, 501}, "first_available", 1)
f(ui, "http://host123/vmsingle/api/v1/query?query=up", "http://default-server/v1/query?query=up",
`[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2)
f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", []int{}, "least_loaded", 0)
f(ui, "https://foo-host/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", `[{"bb" "aaa"}]`, `[{"x" "y"}]`, []int{502}, "least_loaded", 2)
"bb: aaa", "x: y", []int{502}, "least_loaded", 2)
f(ui, "https://foo-host/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "", "", []int{}, "least_loaded", 0)
f(ui, "https://foo-host/foo/bar/api/v1/query_range", "http://default-server/api/v1/query_range", "bb: aaa", "x: y", []int{502}, "least_loaded", 2)
// Complex routing regexp paths in `url_map`
ui = &UserInfo{
@ -226,19 +237,19 @@ func TestCreateTargetURLSuccess(t *testing.T) {
},
URLPrefix: mustParseURL("http://default-server"),
}
f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", "[]", "[]", nil, "least_loaded", 0)
f(ui, "/api/v1/query_range?query=up", "http://vmselect/0/prometheus/api/v1/query_range?query=up", "[]", "[]", nil, "least_loaded", 0)
f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values", "[]", "[]", nil, "least_loaded", 0)
f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "[]", "[]", nil, "least_loaded", 0)
f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar", "[]", "[]", nil, "least_loaded", 0)
f(ui, "https://vmui.foobar.com/a/b?c=d", "http://vmui.host:1234/vmui/a/b?c=d", "[]", "[]", nil, "least_loaded", 0)
f(ui, "/api/v1/query?query=up", "http://vmselect/0/prometheus/api/v1/query?query=up", "", "", nil, "least_loaded", 0)
f(ui, "/api/v1/query_range?query=up", "http://vmselect/0/prometheus/api/v1/query_range?query=up", "", "", nil, "least_loaded", 0)
f(ui, "/api/v1/label/foo/values", "http://vmselect/0/prometheus/api/v1/label/foo/values", "", "", nil, "least_loaded", 0)
f(ui, "/api/v1/write", "http://vminsert/0/prometheus/api/v1/write", "", "", nil, "least_loaded", 0)
f(ui, "/api/v1/foo/bar", "http://default-server/api/v1/foo/bar", "", "", nil, "least_loaded", 0)
f(ui, "https://vmui.foobar.com/a/b?c=d", "http://vmui.host:1234/vmui/a/b?c=d", "", "", nil, "least_loaded", 0)
f(&UserInfo{
URLPrefix: mustParseURL("http://foo.bar?extra_label=team=dev"),
}, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "[]", "[]", nil, "least_loaded", 0)
}, "/api/v1/query", "http://foo.bar/api/v1/query?extra_label=team=dev", "", "", nil, "least_loaded", 0)
f(&UserInfo{
URLPrefix: mustParseURL("http://foo.bar?extra_label=team=mobile"),
}, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team%3Dmobile", "[]", "[]", nil, "least_loaded", 0)
}, "/api/v1/query?extra_label=team=dev", "http://foo.bar/api/v1/query?extra_label=team%3Dmobile", "", "", nil, "least_loaded", 0)
}
func TestCreateTargetURLFailure(t *testing.T) {
@ -270,3 +281,11 @@ func TestCreateTargetURLFailure(t *testing.T) {
},
}, "/api/v1/write")
}
func headersToString(hs []Header) string {
a := make([]string, len(hs))
for i, h := range hs {
a[i] = fmt.Sprintf("%s: %s", h.Name, h.Value)
}
return strings.Join(a, "\n")
}

1
docs/CHANGELOG.md
View File

@ -32,6 +32,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
* SECURITY: upgrade Go builder from Go1.21.7 to Go1.22.1. See [the list of issues addressed in Go1.22.1](https://github.com/golang/go/issues?q=milestone%3AGo1.22.1+label%3ACherryPickApproved).
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): allow discovering ip addresses for backend instances hidden behind a shared hostname, via `discover_backend_ips: true` option. This allows evenly spreading load among backend instances. See [these docs](https://docs.victoriametrics.com/vmauth/#discovering-backend-ips) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5707).
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): allow routing incoming requests bassed on HTTP [query args](https://en.wikipedia.org/wiki/Query_string) via `src_query_args` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth/#generic-http-proxy-for-different-backends) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5878).
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth/): allow routing incoming requests bassed on HTTP request headers via `src_headers` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth/#generic-http-proxy-for-different-backends).
* FEATURE: [stream aggregation](https://docs.victoriametrics.com/stream-aggregation/): reduce memory usage by up to 5x when aggregating over big number of unique [time series](https://docs.victoriametrics.com/keyconcepts/#time-series). The memory usage reduction is most visible when [stream deduplication](https://docs.victoriametrics.com/stream-aggregation/#deduplication) is enabled. The downside is increased CPU usage by up to 30%.

44
docs/vmauth.md
View File

@ -401,6 +401,7 @@ Each `url_prefix` in the [-auth.config](#auth-config) can be specified in the fo
In this case `vmauth` spreads requests among the specified urls using least-loaded round-robin policy.
This guarantees that incoming load is shared uniformly among the specified backends.
See also [discovering backend IPs](#discovering-backend-ips).
`vmauth` automatically detects temporarily unavailable backends and spreads incoming queries among the remaining available backends.
This allows restarting the backends and peforming mantenance tasks on the backends without the need to remove them from the `url_prefix` list.
@ -466,6 +467,49 @@ Load balancing feature can be used in the following cases:
Load balancig can be configured independently per each `user` entry and per each `url_map` entry. See [auth config docs](#auth-config) for more details.
See also [discovering backend IPs](#discovering-backend-ips).
## Discovering backend IPs
By default `vmauth` spreads load among the listed backends at `url_prefix` as described in [load balancing docs](#load-balancing).
Sometimes multiple backend instances can be hidden behind a single hostname. For example, `vmselect-service` hostname
may point to a cluster of `vmselect` instances in [VictoriaMetrics cluster setup](https://docs.victoriametrics.com/cluster-victoriametrics/#architecture-overview).
So the following config may fail spreading load among available `vmselect` instances, since `vmauth` will send all the requests to the same url, which may end up
to a single backend instance:
```yaml
unauthorized_user:
url_prefix: http://vmselect-service/select/0/prometheus/
```
There are the following solutions for this issue:
- To enumerate every `vmselect` hosname or IP in the `url_prefix` list:
```yaml
unauthorized_user:
url_prefix:
- http://vmselect-1:8481/select/0/prometheus/
- http://vmselect-2:8481/select/0/prometheus/
- http://vmselect-3:8481/select/0/prometheus/
```
This scheme works great, but it needs manual updating of the [`-auth.config`](#auth-config) every time `vmselect` services are restarted,
downsaled or upscaled.
- To set `discover_backend_ips: true` option, so `vmagent` automatically discovers IPs behind the given hostname and then spreads load among the discovered IPs:
```yaml
unauthorized_user:
url_prefix: http://vmselect-service/select/0/prometheus/
discover_backend_ips: true
```
The `discover_backend_ips` can be specified at `user` and `url_map` level in the [`-auth.config](#auth-config). It can also be enabled globally
via `-discoverBackendIPs` command-line flag.
See also [load balancing docs](#load-balancing).
## Modifying HTTP headers
`vmauth` supports the ability to set and remove HTTP request headers before sending the requests to backends.