From 3ae6300497870c9234eff03d122472d4f68c7b33 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 22 Jun 2022 20:38:43 +0300 Subject: [PATCH] lib/promauth: add ability to send additional http headers in requests to scrape targets This solves https://stackoverflow.com/questions/66032498/prometheus-scrape-metric-with-custom-header --- app/vmagent/README.md | 18 +++- app/vmagent/remotewrite/client.go | 6 +- app/vmalert/datasource/vm.go | 4 +- app/vmalert/datasource/vm_test.go | 6 +- app/vmalert/notifier/alertmanager.go | 4 +- app/vmalert/remotewrite/remotewrite.go | 4 +- docs/CHANGELOG.md | 10 ++ docs/README.md | 5 +- docs/Single-server-VictoriaMetrics.md | 5 +- docs/vmagent.md | 18 +++- lib/promauth/config.go | 92 ++++++++++++++++-- lib/promauth/config_test.go | 93 ++++++++++++++++++- lib/promscrape/client.go | 38 ++++---- lib/promscrape/config_test.go | 24 ++++- lib/promscrape/discovery/kubernetes/api.go | 7 +- .../discovery/kubernetes/api_watcher.go | 14 ++- lib/promscrape/discovery/openstack/api.go | 2 +- lib/promscrape/discoveryutils/client.go | 40 ++++---- lib/proxy/proxy.go | 26 +++++- 19 files changed, 326 insertions(+), 90 deletions(-) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index da787f5cc6..6ce4966598 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -183,6 +183,16 @@ Please file feature requests to [our issue tracker](https://github.com/VictoriaM `vmagent` also support the following additional options in `scrape_configs` section: +* `headers` - a list of HTTP headers to send to scrape target with each scrape request. This can be used when the scrape target needs custom authorization and authentication. For example: + +```yaml +scrape_configs: +- job_name: custom_headers + headers: + - "TenantID: abc" + - "My-Auth: TopSecret" +``` + * `disable_compression: true` - to disable response compression on a per-job basis. By default `vmagent` requests compressed responses from scrape targets to save network bandwidth. * `disable_keepalive: true` - to disable [HTTP keep-alive connections](https://en.wikipedia.org/wiki/HTTP_persistent_connection) on a per-job basis. @@ -297,6 +307,8 @@ The relabeling can be defined in the following places: * At the `-remoteWrite.relabelConfig` file. This relabeling is applied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage. * At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`. +All the files with relabeling configs can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values. + You can read more about relabeling in the following articles: * [How to use Relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2) @@ -424,9 +436,11 @@ scrape_configs: Proxy can be configured with the following optional settings: * `proxy_authorization` for generic token authorization. See [Prometheus docs for details on authorization section](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) -* `proxy_bearer_token` and `proxy_bearer_token_file` for Bearer token authorization * `proxy_basic_auth` for Basic authorization. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config). +* `proxy_bearer_token` and `proxy_bearer_token_file` for Bearer token authorization +* `proxy_oauth2` for OAuth2 config. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2). * `proxy_tls_config` for TLS config. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config). +* `proxy_headers` for passing additional HTTP headers in requests to proxy. For example: @@ -443,6 +457,8 @@ scrape_configs: key_file: /path/to/key ca_file: /path/to/ca server_name: real-server-name + proxy_headers: + - "Proxy-Auth: top-secret" ``` ## Cardinality limiter diff --git a/app/vmagent/remotewrite/client.go b/app/vmagent/remotewrite/client.go index aa5f98c792..da6a8960b5 100644 --- a/app/vmagent/remotewrite/client.go +++ b/app/vmagent/remotewrite/client.go @@ -219,7 +219,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) { InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx), } - authCfg, err := promauth.NewConfig(".", nil, basicAuthCfg, token, tokenFile, oauth2Cfg, tlsCfg) + authCfg, err := promauth.NewConfig(".", nil, basicAuthCfg, token, tokenFile, oauth2Cfg, tlsCfg, nil) if err != nil { return nil, fmt.Errorf("cannot populate OAuth2 config for remoteWrite idx: %d, err: %w", argIdx, err) } @@ -306,9 +306,7 @@ again: h.Set("Content-Type", "application/x-protobuf") h.Set("Content-Encoding", "snappy") h.Set("X-Prometheus-Remote-Write-Version", "0.1.0") - if ah := c.authCfg.GetAuthHeader(); ah != "" { - req.Header.Set("Authorization", ah) - } + c.authCfg.SetHeaders(req, true) if c.awsCfg != nil { if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil { // there is no need in retry, request will be rejected by client.Do and retried by code below diff --git a/app/vmalert/datasource/vm.go b/app/vmalert/datasource/vm.go index 0b60bbcde9..df415dd66c 100644 --- a/app/vmalert/datasource/vm.go +++ b/app/vmalert/datasource/vm.go @@ -146,9 +146,7 @@ func (s *VMStorage) newRequestPOST() (*http.Request, error) { } req.Header.Set("Content-Type", "application/json") if s.authCfg != nil { - if auth := s.authCfg.GetAuthHeader(); auth != "" { - req.Header.Set("Authorization", auth) - } + s.authCfg.SetHeaders(req, true) } return req, nil } diff --git a/app/vmalert/datasource/vm_test.go b/app/vmalert/datasource/vm_test.go index dba6895500..1cd433ceea 100644 --- a/app/vmalert/datasource/vm_test.go +++ b/app/vmalert/datasource/vm_test.go @@ -83,7 +83,7 @@ func TestVMInstantQuery(t *testing.T) { srv := httptest.NewServer(mux) defer srv.Close() - authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil) + authCfg, err := baCfg.NewConfig(".") if err != nil { t.Fatalf("unexpected: %s", err) } @@ -206,7 +206,7 @@ func TestVMRangeQuery(t *testing.T) { srv := httptest.NewServer(mux) defer srv.Close() - authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil) + authCfg, err := baCfg.NewConfig(".") if err != nil { t.Fatalf("unexpected: %s", err) } @@ -247,7 +247,7 @@ func TestVMRangeQuery(t *testing.T) { } func TestRequestParams(t *testing.T) { - authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil) + authCfg, err := baCfg.NewConfig(".") if err != nil { t.Fatalf("unexpected: %s", err) } diff --git a/app/vmalert/notifier/alertmanager.go b/app/vmalert/notifier/alertmanager.go index c643878775..de6d5c226c 100644 --- a/app/vmalert/notifier/alertmanager.go +++ b/app/vmalert/notifier/alertmanager.go @@ -79,9 +79,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert) error { req = req.WithContext(ctx) if am.authCfg != nil { - if auth := am.authCfg.GetAuthHeader(); auth != "" { - req.Header.Set("Authorization", auth) - } + am.authCfg.SetHeaders(req, true) } resp, err := am.client.Do(req) if err != nil { diff --git a/app/vmalert/remotewrite/remotewrite.go b/app/vmalert/remotewrite/remotewrite.go index 08e60a5a51..ec5ba98e6c 100644 --- a/app/vmalert/remotewrite/remotewrite.go +++ b/app/vmalert/remotewrite/remotewrite.go @@ -245,9 +245,7 @@ func (c *Client) send(ctx context.Context, data []byte) error { req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0") if c.authCfg != nil { - if auth := c.authCfg.GetAuthHeader(); auth != "" { - req.Header.Set("Authorization", auth) - } + c.authCfg.SetHeaders(req, true) } if !*disablePathAppend { req.URL.Path = path.Join(req.URL.Path, "/api/v1/write") diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f0ff8d37e0..b6c612c595 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -16,7 +16,17 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip * FEATURE: add `-search.setLookbackToStep` command-line flag, which enables InfluxDB-like gap filling during querying. See [these docs](https://docs.victoriametrics.com/guides/migrate-from-influx.html) for details. +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add ability to specify additional HTTP headers to send to scrape targets via `headers` section in `scrape_configs`. This can be used when the scrape target requires custom authorization and authentication like in [this stackoverflow question](https://stackoverflow.com/questions/66032498/prometheus-scrape-metric-with-custom-header). For example, the following config instructs sending `My-Auth: top-secret` and `TenantID: FooBar` headers with each request to `http://host123:8080/metrics`: +```yaml +scrape_configs: +- job_name: foo + headers: + - "My-Auth: top-secret" + - "TenantID: FooBar" + static_configs: + - targets: ["host123:8080"] +``` ## [v1.78.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.0) diff --git a/docs/README.md b/docs/README.md index 1d507f4771..d8be3e812c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1134,6 +1134,8 @@ to a file containing a list of [relabel_config](https://prometheus.io/docs/prome The `-relabelConfig` also can point to http or https url. For example, `-relabelConfig=https://config-server/relabel_config.yml`. See [this article with relabeling tips and tricks](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2). +The `-relabelConfig` files can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values. + Example contents for `-relabelConfig` file: ```yml @@ -1147,8 +1149,7 @@ Example contents for `-relabelConfig` file: regex: true ``` -VictoriaMetrics components provide additional relabeling features such as Graphite-style relabeling. -See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details. +VictoriaMetrics provides additional relabeling features such as Graphite-style relabeling. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details. ## Federation diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index eb302a089d..45147f2e46 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -1138,6 +1138,8 @@ to a file containing a list of [relabel_config](https://prometheus.io/docs/prome The `-relabelConfig` also can point to http or https url. For example, `-relabelConfig=https://config-server/relabel_config.yml`. See [this article with relabeling tips and tricks](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2). +The `-relabelConfig` files can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values. + Example contents for `-relabelConfig` file: ```yml @@ -1151,8 +1153,7 @@ Example contents for `-relabelConfig` file: regex: true ``` -VictoriaMetrics components provide additional relabeling features such as Graphite-style relabeling. -See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details. +VictoriaMetrics provides additional relabeling features such as Graphite-style relabeling. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details. ## Federation diff --git a/docs/vmagent.md b/docs/vmagent.md index 9ad84df6e5..e3e97534c5 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -187,6 +187,16 @@ Please file feature requests to [our issue tracker](https://github.com/VictoriaM `vmagent` also support the following additional options in `scrape_configs` section: +* `headers` - a list of HTTP headers to send to scrape target with each scrape request. This can be used when the scrape target needs custom authorization and authentication. For example: + +```yaml +scrape_configs: +- job_name: custom_headers + headers: + - "TenantID: abc" + - "My-Auth: TopSecret" +``` + * `disable_compression: true` - to disable response compression on a per-job basis. By default `vmagent` requests compressed responses from scrape targets to save network bandwidth. * `disable_keepalive: true` - to disable [HTTP keep-alive connections](https://en.wikipedia.org/wiki/HTTP_persistent_connection) on a per-job basis. @@ -301,6 +311,8 @@ The relabeling can be defined in the following places: * At the `-remoteWrite.relabelConfig` file. This relabeling is applied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage. * At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`. +All the files with relabeling configs can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values. + You can read more about relabeling in the following articles: * [How to use Relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2) @@ -428,9 +440,11 @@ scrape_configs: Proxy can be configured with the following optional settings: * `proxy_authorization` for generic token authorization. See [Prometheus docs for details on authorization section](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) -* `proxy_bearer_token` and `proxy_bearer_token_file` for Bearer token authorization * `proxy_basic_auth` for Basic authorization. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config). +* `proxy_bearer_token` and `proxy_bearer_token_file` for Bearer token authorization +* `proxy_oauth2` for OAuth2 config. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2). * `proxy_tls_config` for TLS config. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config). +* `proxy_headers` for passing additional HTTP headers in requests to proxy. For example: @@ -447,6 +461,8 @@ scrape_configs: key_file: /path/to/key ca_file: /path/to/ca server_name: real-server-name + proxy_headers: + - "Proxy-Auth: top-secret" ``` ## Cardinality limiter diff --git a/lib/promauth/config.go b/lib/promauth/config.go index 3f7ee11787..020f6af252 100644 --- a/lib/promauth/config.go +++ b/lib/promauth/config.go @@ -15,6 +15,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime" "github.com/VictoriaMetrics/VictoriaMetrics/lib/fs" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" + "github.com/VictoriaMetrics/fasthttp" "github.com/cespare/xxhash/v2" "golang.org/x/oauth2" "golang.org/x/oauth2/clientcredentials" @@ -116,6 +117,9 @@ type HTTPClientConfig struct { BearerTokenFile string `yaml:"bearer_token_file,omitempty"` OAuth2 *OAuth2Config `yaml:"oauth2,omitempty"` TLSConfig *TLSConfig `yaml:"tls_config,omitempty"` + + // Headers contains optional HTTP headers, which must be sent in the request to the server + Headers []string `yaml:"headers,omitempty"` } // ProxyClientConfig represents proxy client config. @@ -124,7 +128,11 @@ type ProxyClientConfig struct { BasicAuth *BasicAuthConfig `yaml:"proxy_basic_auth,omitempty"` BearerToken *Secret `yaml:"proxy_bearer_token,omitempty"` BearerTokenFile string `yaml:"proxy_bearer_token_file,omitempty"` + OAuth2 *OAuth2Config `yaml:"proxy_oauth2,omitempty"` TLSConfig *TLSConfig `yaml:"proxy_tls_config,omitempty"` + + // Headers contains optional HTTP headers, which must be sent in the request to the proxy + Headers []string `yaml:"proxy_headers,omitempty"` } // OAuth2Config represent OAuth2 configuration @@ -257,9 +265,71 @@ type Config struct { authHeader string authHeaderDeadline uint64 + headers []keyValue + authDigest string } +type keyValue struct { + key string + value string +} + +func parseHeaders(headers []string) ([]keyValue, error) { + if len(headers) == 0 { + return nil, nil + } + kvs := make([]keyValue, len(headers)) + for i, h := range headers { + n := strings.IndexByte(h, ':') + if n < 0 { + return nil, fmt.Errorf(`missing ':' in header %q; expecting "key: value" format`, h) + } + kv := &kvs[i] + kv.key = strings.TrimSpace(h[:n]) + kv.value = strings.TrimSpace(h[n+1:]) + } + return kvs, nil +} + +// HeadersNoAuthString returns string representation of ac headers +func (ac *Config) HeadersNoAuthString() string { + if len(ac.headers) == 0 { + return "" + } + a := make([]string, len(ac.headers)) + for i, h := range ac.headers { + a[i] = h.key + ": " + h.value + "\r\n" + } + return strings.Join(a, "") +} + +// SetHeaders sets the configuted ac headers to req. +func (ac *Config) SetHeaders(req *http.Request, setAuthHeader bool) { + reqHeaders := req.Header + for _, h := range ac.headers { + reqHeaders.Set(h.key, h.value) + } + if setAuthHeader { + if ah := ac.GetAuthHeader(); ah != "" { + reqHeaders.Set("Authorization", ah) + } + } +} + +// SetFasthttpHeaders sets the configured ac headers to req. +func (ac *Config) SetFasthttpHeaders(req *fasthttp.Request, setAuthHeader bool) { + reqHeaders := &req.Header + for _, h := range ac.headers { + reqHeaders.Set(h.key, h.value) + } + if setAuthHeader { + if ah := ac.GetAuthHeader(); ah != "" { + reqHeaders.Set("Authorization", ah) + } + } +} + // GetAuthHeader returns optional `Authorization: ...` http header. func (ac *Config) GetAuthHeader() string { f := ac.getAuthHeader @@ -281,8 +351,8 @@ func (ac *Config) GetAuthHeader() string { // It is also used for comparing Config objects for equality. If two Config // objects have the same string representation, then they are considered equal. func (ac *Config) String() string { - return fmt.Sprintf("AuthDigest=%s, TLSRootCA=%s, TLSCertificate=%s, TLSServerName=%s, TLSInsecureSkipVerify=%v, TLSMinVersion=%d", - ac.authDigest, ac.tlsRootCAString(), ac.tlsCertDigest, ac.TLSServerName, ac.TLSInsecureSkipVerify, ac.TLSMinVersion) + return fmt.Sprintf("AuthDigest=%s, Headers=%s, TLSRootCA=%s, TLSCertificate=%s, TLSServerName=%s, TLSInsecureSkipVerify=%v, TLSMinVersion=%d", + ac.authDigest, ac.headers, ac.tlsRootCAString(), ac.tlsCertDigest, ac.TLSServerName, ac.TLSInsecureSkipVerify, ac.TLSMinVersion) } func (ac *Config) tlsRootCAString() string { @@ -330,21 +400,26 @@ func (ac *Config) NewTLSConfig() *tls.Config { // NewConfig creates auth config for the given hcc. func (hcc *HTTPClientConfig) NewConfig(baseDir string) (*Config, error) { - return NewConfig(baseDir, hcc.Authorization, hcc.BasicAuth, hcc.BearerToken.String(), hcc.BearerTokenFile, hcc.OAuth2, hcc.TLSConfig) + return NewConfig(baseDir, hcc.Authorization, hcc.BasicAuth, hcc.BearerToken.String(), hcc.BearerTokenFile, hcc.OAuth2, hcc.TLSConfig, hcc.Headers) } // NewConfig creates auth config for the given pcc. func (pcc *ProxyClientConfig) NewConfig(baseDir string) (*Config, error) { - return NewConfig(baseDir, pcc.Authorization, pcc.BasicAuth, pcc.BearerToken.String(), pcc.BearerTokenFile, nil, pcc.TLSConfig) + return NewConfig(baseDir, pcc.Authorization, pcc.BasicAuth, pcc.BearerToken.String(), pcc.BearerTokenFile, pcc.OAuth2, pcc.TLSConfig, pcc.Headers) } // NewConfig creates auth config for the given o. func (o *OAuth2Config) NewConfig(baseDir string) (*Config, error) { - return NewConfig(baseDir, nil, nil, "", "", nil, o.TLSConfig) + return NewConfig(baseDir, nil, nil, "", "", nil, o.TLSConfig, nil) +} + +// NewConfig creates auth config for the given ba. +func (ba *BasicAuthConfig) NewConfig(baseDir string) (*Config, error) { + return NewConfig(baseDir, nil, ba, "", "", nil, nil, nil) } // NewConfig creates auth config from the given args. -func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, bearerToken, bearerTokenFile string, o *OAuth2Config, tlsConfig *TLSConfig) (*Config, error) { +func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, bearerToken, bearerTokenFile string, o *OAuth2Config, tlsConfig *TLSConfig, headers []string) (*Config, error) { var getAuthHeader func() string authDigest := "" if az != nil { @@ -517,6 +592,10 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be tlsMinVersion = v } } + parsedHeaders, err := parseHeaders(headers) + if err != nil { + return nil, err + } ac := &Config{ TLSRootCA: tlsRootCA, TLSServerName: tlsServerName, @@ -527,6 +606,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be tlsCertDigest: tlsCertDigest, getAuthHeader: getAuthHeader, + headers: parsedHeaders, authDigest: authDigest, } return ac, nil diff --git a/lib/promauth/config_test.go b/lib/promauth/config_test.go index 04795710ae..dcbbd2a304 100644 --- a/lib/promauth/config_test.go +++ b/lib/promauth/config_test.go @@ -4,6 +4,8 @@ import ( "net/http" "net/http/httptest" "testing" + + "github.com/VictoriaMetrics/fasthttp" ) func TestNewConfig(t *testing.T) { @@ -116,18 +118,103 @@ func TestNewConfig(t *testing.T) { mock := httptest.NewServer(r) tt.args.oauth.TokenURL = mock.URL } - got, err := NewConfig(tt.args.baseDir, tt.args.az, tt.args.basicAuth, tt.args.bearerToken, tt.args.bearerTokenFile, tt.args.oauth, tt.args.tlsConfig) + got, err := NewConfig(tt.args.baseDir, tt.args.az, tt.args.basicAuth, tt.args.bearerToken, tt.args.bearerTokenFile, tt.args.oauth, tt.args.tlsConfig, nil) if (err != nil) != tt.wantErr { t.Errorf("NewConfig() error = %v, wantErr %v", err, tt.wantErr) return } if got != nil { - ah := got.GetAuthHeader() + req, err := http.NewRequest("GET", "http://foo", nil) + if err != nil { + t.Fatalf("unexpected error in http.NewRequest: %s", err) + } + got.SetHeaders(req, true) + ah := req.Header.Get("Authorization") if ah != tt.expectHeader { - t.Fatalf("unexpected auth header; got %q; want %q", ah, tt.expectHeader) + t.Fatalf("unexpected auth header from net/http request; got %q; want %q", ah, tt.expectHeader) + } + var fhreq fasthttp.Request + got.SetFasthttpHeaders(&fhreq, true) + ahb := fhreq.Header.Peek("Authorization") + if string(ahb) != tt.expectHeader { + t.Fatalf("unexpected auth header from fasthttp request; got %q; want %q", ahb, tt.expectHeader) } } }) } } + +func TestParseHeadersSuccess(t *testing.T) { + f := func(headers []string) { + t.Helper() + headersParsed, err := parseHeaders(headers) + if err != nil { + t.Fatalf("unexpected error when parsing %s: %s", headers, err) + } + for i, h := range headersParsed { + s := h.key + ": " + h.value + if s != headers[i] { + t.Fatalf("unexpected header parsed; got %q; want %q", s, headers[i]) + } + } + } + f(nil) + f([]string{"foo: bar"}) + f([]string{"Foo: bar", "A-b-c: d-e-f"}) +} + +func TestParseHeadersFailure(t *testing.T) { + f := func(headers []string) { + t.Helper() + headersParsed, err := parseHeaders(headers) + if err == nil { + t.Fatalf("expecting non-nil error from parseHeaders(%s)", headers) + } + if headersParsed != nil { + t.Fatalf("expecting nil result from parseHeaders(%s)", headers) + } + } + f([]string{"foo"}) + f([]string{"foo bar baz"}) +} + +func TestConfigHeaders(t *testing.T) { + f := func(headers []string, resultExpected string) { + t.Helper() + headersParsed, err := parseHeaders(headers) + if err != nil { + t.Fatalf("cannot parse headers: %s", err) + } + c, err := NewConfig("", nil, nil, "", "", nil, nil, headers) + if err != nil { + t.Fatalf("cannot create config: %s", err) + } + req, err := http.NewRequest("GET", "http://foo", nil) + if err != nil { + t.Fatalf("unexpected error in http.NewRequest: %s", err) + } + result := c.HeadersNoAuthString() + if result != resultExpected { + t.Fatalf("unexpected result from HeadersNoAuthString; got\n%s\nwant\n%s", result, resultExpected) + } + c.SetHeaders(req, false) + for _, h := range headersParsed { + v := req.Header.Get(h.key) + if v != h.value { + t.Fatalf("unexpected value for net/http header %q; got %q; want %q", h.key, v, h.value) + } + } + var fhreq fasthttp.Request + c.SetFasthttpHeaders(&fhreq, false) + for _, h := range headersParsed { + v := fhreq.Header.Peek(h.key) + if string(v) != h.value { + t.Fatalf("unexpected value for fasthttp header %q; got %q; want %q", h.key, v, h.value) + } + } + } + f(nil, "") + f([]string{"foo: bar"}, "foo: bar\r\n") + f([]string{"Foo-Bar: Baz s:sdf", "A:b", "X-Forwarded-For: A-B:c"}, "Foo-Bar: Baz s:sdf\r\nA: b\r\nX-Forwarded-For: A-B:c\r\n") +} diff --git a/lib/promscrape/client.go b/lib/promscrape/client.go index 1ad31af1dc..651001bba2 100644 --- a/lib/promscrape/client.go +++ b/lib/promscrape/client.go @@ -48,8 +48,10 @@ type client struct { scrapeTimeoutSecondsStr string host string requestURI string - getAuthHeader func() string - getProxyAuthHeader func() string + setHeaders func(req *http.Request) + setProxyHeaders func(req *http.Request) + setFasthttpHeaders func(req *fasthttp.Request) + setFasthttpProxyHeaders func(req *fasthttp.Request) denyRedirects bool disableCompression bool disableKeepAlive bool @@ -65,7 +67,8 @@ func newClient(sw *ScrapeWork) *client { if isTLS { tlsCfg = sw.AuthConfig.NewTLSConfig() } - getProxyAuthHeader := func() string { return "" } + setProxyHeaders := func(req *http.Request) {} + setFasthttpProxyHeaders := func(req *fasthttp.Request) {} proxyURL := sw.ProxyURL if !isTLS && proxyURL.IsHTTPOrHTTPS() { // Send full sw.ScrapeURL in requests to a proxy host for non-TLS scrape targets @@ -79,8 +82,11 @@ func newClient(sw *ScrapeWork) *client { tlsCfg = sw.ProxyAuthConfig.NewTLSConfig() } proxyURLOrig := proxyURL - getProxyAuthHeader = func() string { - return proxyURLOrig.GetAuthHeader(sw.ProxyAuthConfig) + setProxyHeaders = func(req *http.Request) { + proxyURLOrig.SetHeaders(sw.ProxyAuthConfig, req) + } + setFasthttpProxyHeaders = func(req *fasthttp.Request) { + proxyURLOrig.SetFasthttpHeaders(sw.ProxyAuthConfig, req) } proxyURL = &proxy.URL{} } @@ -148,8 +154,10 @@ func newClient(sw *ScrapeWork) *client { scrapeTimeoutSecondsStr: fmt.Sprintf("%.3f", sw.ScrapeTimeout.Seconds()), host: host, requestURI: requestURI, - getAuthHeader: sw.AuthConfig.GetAuthHeader, - getProxyAuthHeader: getProxyAuthHeader, + setHeaders: func(req *http.Request) { sw.AuthConfig.SetHeaders(req, true) }, + setProxyHeaders: setProxyHeaders, + setFasthttpHeaders: func(req *fasthttp.Request) { sw.AuthConfig.SetFasthttpHeaders(req, true) }, + setFasthttpProxyHeaders: setFasthttpProxyHeaders, denyRedirects: sw.DenyRedirects, disableCompression: sw.DisableCompression, disableKeepAlive: sw.DisableKeepAlive, @@ -173,12 +181,8 @@ func (c *client) GetStreamReader() (*streamReader, error) { // Set X-Prometheus-Scrape-Timeout-Seconds like Prometheus does, since it is used by some exporters such as PushProx. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1179#issuecomment-813117162 req.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", c.scrapeTimeoutSecondsStr) - if ah := c.getAuthHeader(); ah != "" { - req.Header.Set("Authorization", ah) - } - if ah := c.getProxyAuthHeader(); ah != "" { - req.Header.Set("Proxy-Authorization", ah) - } + c.setHeaders(req) + c.setProxyHeaders(req) resp, err := c.sc.Do(req) if err != nil { cancel() @@ -224,12 +228,8 @@ func (c *client) ReadData(dst []byte) ([]byte, error) { // Set X-Prometheus-Scrape-Timeout-Seconds like Prometheus does, since it is used by some exporters such as PushProx. // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1179#issuecomment-813117162 req.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", c.scrapeTimeoutSecondsStr) - if ah := c.getAuthHeader(); ah != "" { - req.Header.Set("Authorization", ah) - } - if ah := c.getProxyAuthHeader(); ah != "" { - req.Header.Set("Proxy-Authorization", ah) - } + c.setFasthttpHeaders(req) + c.setFasthttpProxyHeaders(req) if !*disableCompression && !c.disableCompression { req.Header.Set("Accept-Encoding", "gzip") } diff --git a/lib/promscrape/config_test.go b/lib/promscrape/config_test.go index 6659e5376e..bd6ba5277a 100644 --- a/lib/promscrape/config_test.go +++ b/lib/promscrape/config_test.go @@ -141,6 +141,9 @@ scrape_configs: - x authorization: type: foobar + headers: + - 'TenantID: fooBar' + - 'X: y:z' relabel_configs: - source_labels: [abc] static_configs: @@ -149,6 +152,8 @@ scrape_configs: relabel_debug: true scrape_align_interval: 1h30m0s proxy_bearer_token_file: file.txt + proxy_headers: + - 'My-Auth-Header: top-secret' `) } @@ -332,7 +337,7 @@ scrape_configs: jobNameOriginal: "blackbox", }} if !reflect.DeepEqual(sws, swsExpected) { - t.Fatalf("unexpected scrapeWork;\ngot\n%+v\nwant\n%+v", sws, swsExpected) + t.Fatalf("unexpected scrapeWork;\ngot\n%#v\nwant\n%#v", sws, swsExpected) } } @@ -1650,12 +1655,25 @@ scrape_configs: jobNameOriginal: "aaa", }, }) + + ac, err := promauth.NewConfig(".", nil, nil, "", "", nil, nil, []string{"My-Auth: foo-Bar"}) + if err != nil { + t.Fatalf("unexpected error when creating promauth.Config: %s", err) + } + proxyAC, err := promauth.NewConfig(".", nil, nil, "", "", nil, nil, []string{"Foo:bar"}) + if err != nil { + t.Fatalf("unexpected error when creating promauth.Config for proxy: %s", err) + } f(` scrape_configs: - job_name: 'snmp' sample_limit: 100 disable_keepalive: true disable_compression: true + headers: + - "My-Auth: foo-Bar" + proxy_headers: + - "Foo: bar" scrape_align_interval: 1s scrape_offset: 0.5s static_configs: @@ -1727,8 +1745,8 @@ scrape_configs: Value: "snmp", }, }, - AuthConfig: &promauth.Config{}, - ProxyAuthConfig: &promauth.Config{}, + AuthConfig: ac, + ProxyAuthConfig: proxyAC, SampleLimit: 100, DisableKeepAlive: true, DisableCompression: true, diff --git a/lib/promscrape/discovery/kubernetes/api.go b/lib/promscrape/discovery/kubernetes/api.go index 8950e623d8..45cb7a06fa 100644 --- a/lib/promscrape/discovery/kubernetes/api.go +++ b/lib/promscrape/discovery/kubernetes/api.go @@ -16,7 +16,8 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu default: return nil, fmt.Errorf("unexpected `role`: %q; must be one of `node`, `pod`, `service`, `endpoints`, `endpointslice` or `ingress`", role) } - ac, err := sdc.HTTPClientConfig.NewConfig(baseDir) + cc := &sdc.HTTPClientConfig + ac, err := cc.NewConfig(baseDir) if err != nil { return nil, fmt.Errorf("cannot parse auth config: %w", err) } @@ -30,7 +31,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu if err != nil { return nil, fmt.Errorf("cannot build kube config from the specified `kubeconfig_file` config option: %w", err) } - acNew, err := promauth.NewConfig(".", nil, kc.basicAuth, kc.token, kc.tokenFile, nil, kc.tlsConfig) + acNew, err := promauth.NewConfig(".", nil, kc.basicAuth, kc.token, kc.tokenFile, cc.OAuth2, kc.tlsConfig, cc.Headers) if err != nil { return nil, fmt.Errorf("cannot initialize auth config from `kubeconfig_file: %q`: %w", sdc.KubeConfigFile, err) } @@ -57,7 +58,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu tlsConfig := promauth.TLSConfig{ CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", } - acNew, err := promauth.NewConfig(".", nil, nil, "", "/var/run/secrets/kubernetes.io/serviceaccount/token", nil, &tlsConfig) + acNew, err := promauth.NewConfig(".", nil, nil, "", "/var/run/secrets/kubernetes.io/serviceaccount/token", cc.OAuth2, &tlsConfig, cc.Headers) if err != nil { return nil, fmt.Errorf("cannot initialize service account auth: %w; probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?", err) } diff --git a/lib/promscrape/discovery/kubernetes/api_watcher.go b/lib/promscrape/discovery/kubernetes/api_watcher.go index 6f76d5d0e2..63a266afb7 100644 --- a/lib/promscrape/discovery/kubernetes/api_watcher.go +++ b/lib/promscrape/discovery/kubernetes/api_watcher.go @@ -207,8 +207,8 @@ type groupWatcher struct { selectors []Selector attachNodeMetadata bool - getAuthHeader func() string - client *http.Client + setHeaders func(req *http.Request) + client *http.Client mu sync.Mutex m map[string]*urlWatcher @@ -235,9 +235,9 @@ func newGroupWatcher(apiServer string, ac *promauth.Config, namespaces []string, selectors: selectors, attachNodeMetadata: attachNodeMetadata, - getAuthHeader: ac.GetAuthHeader, - client: client, - m: make(map[string]*urlWatcher), + setHeaders: func(req *http.Request) { ac.SetHeaders(req, true) }, + client: client, + m: make(map[string]*urlWatcher), } } @@ -407,9 +407,7 @@ func (gw *groupWatcher) doRequest(requestURL string) (*http.Response, error) { if err != nil { logger.Fatalf("cannot create a request for %q: %s", requestURL, err) } - if ah := gw.getAuthHeader(); ah != "" { - req.Header.Set("Authorization", ah) - } + gw.setHeaders(req) resp, err := gw.client.Do(req) if err != nil { return nil, err diff --git a/lib/promscrape/discovery/openstack/api.go b/lib/promscrape/discovery/openstack/api.go index 40aa06ab92..010973c675 100644 --- a/lib/promscrape/discovery/openstack/api.go +++ b/lib/promscrape/discovery/openstack/api.go @@ -81,7 +81,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { port: sdc.Port, } if sdc.TLSConfig != nil { - ac, err := promauth.NewConfig(baseDir, nil, nil, "", "", nil, sdc.TLSConfig) + ac, err := promauth.NewConfig(baseDir, nil, nil, "", "", nil, sdc.TLSConfig, nil) if err != nil { return nil, err } diff --git a/lib/promscrape/discoveryutils/client.go b/lib/promscrape/discoveryutils/client.go index 6050b60bb9..d306f73a25 100644 --- a/lib/promscrape/discoveryutils/client.go +++ b/lib/promscrape/discoveryutils/client.go @@ -42,10 +42,10 @@ type Client struct { apiServer string - hostPort string - getAuthHeader func() string - getProxyAuthHeader func() string - sendFullURL bool + hostPort string + setFasthttpHeaders func(req *fasthttp.Request) + setFasthttpProxyHeaders func(req *fasthttp.Request) + sendFullURL bool } // NewClient returns new Client for the given args. @@ -70,7 +70,7 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy tlsCfg = ac.NewTLSConfig() } sendFullURL := !isTLS && proxyURL.IsHTTPOrHTTPS() - getProxyAuthHeader := func() string { return "" } + setFasthttpProxyHeaders := func(req *fasthttp.Request) {} if sendFullURL { // Send full urls in requests to a proxy host for non-TLS apiServer // like net/http package from Go does. @@ -82,8 +82,8 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy tlsCfg = proxyAC.NewTLSConfig() } proxyURLOrig := proxyURL - getProxyAuthHeader = func() string { - return proxyURLOrig.GetAuthHeader(proxyAC) + setFasthttpProxyHeaders = func(req *fasthttp.Request) { + proxyURLOrig.SetFasthttpHeaders(proxyAC, req) } proxyURL = &proxy.URL{} } @@ -123,18 +123,18 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy MaxConns: 64 * 1024, Dial: dialFunc, } - getAuthHeader := func() string { return "" } + setFasthttpHeaders := func(req *fasthttp.Request) {} if ac != nil { - getAuthHeader = ac.GetAuthHeader + setFasthttpHeaders = func(req *fasthttp.Request) { ac.SetFasthttpHeaders(req, true) } } return &Client{ - hc: hc, - blockingClient: blockingClient, - apiServer: apiServer, - hostPort: hostPort, - getAuthHeader: getAuthHeader, - getProxyAuthHeader: getProxyAuthHeader, - sendFullURL: sendFullURL, + hc: hc, + blockingClient: blockingClient, + apiServer: apiServer, + hostPort: hostPort, + setFasthttpHeaders: setFasthttpHeaders, + setFasthttpProxyHeaders: setFasthttpProxyHeaders, + sendFullURL: sendFullURL, }, nil } @@ -202,12 +202,8 @@ func (c *Client) getAPIResponseWithParamsAndClient(client *fasthttp.HostClient, } req.Header.SetHost(c.hostPort) req.Header.Set("Accept-Encoding", "gzip") - if ah := c.getAuthHeader(); ah != "" { - req.Header.Set("Authorization", ah) - } - if ah := c.getProxyAuthHeader(); ah != "" { - req.Header.Set("Proxy-Authorization", ah) - } + c.setFasthttpHeaders(&req) + c.setFasthttpProxyHeaders(&req) if modifyRequest != nil { modifyRequest(&req) } diff --git a/lib/proxy/proxy.go b/lib/proxy/proxy.go index 46c29bfbd8..3756f02208 100644 --- a/lib/proxy/proxy.go +++ b/lib/proxy/proxy.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "fmt" "net" + "net/http" "net/url" "strings" "time" @@ -60,8 +61,26 @@ func (u *URL) String() string { return pu.String() } -// GetAuthHeader returns Proxy-Authorization auth header for the given u and ac. -func (u *URL) GetAuthHeader(ac *promauth.Config) string { +// SetHeaders sets headers to req according to u and ac configs. +func (u *URL) SetHeaders(ac *promauth.Config, req *http.Request) { + ah := u.getAuthHeader(ac) + if ah != "" { + req.Header.Set("Proxy-Authorization", ah) + } + ac.SetHeaders(req, false) +} + +// SetFasthttpHeaders sets headers to req according to u and ac configs. +func (u *URL) SetFasthttpHeaders(ac *promauth.Config, req *fasthttp.Request) { + ah := u.getAuthHeader(ac) + if ah != "" { + req.Header.Set("Proxy-Authorization", ah) + } + ac.SetFasthttpHeaders(req, false) +} + +// getAuthHeader returns Proxy-Authorization auth header for the given u and ac. +func (u *URL) getAuthHeader(ac *promauth.Config) string { authHeader := "" if ac != nil { authHeader = ac.GetAuthHeader() @@ -130,9 +149,10 @@ func (u *URL) NewDialFunc(ac *promauth.Config) (fasthttp.DialFunc, error) { if isTLS { proxyConn = tls.Client(proxyConn, tlsCfg) } - authHeader := u.GetAuthHeader(ac) + authHeader := u.getAuthHeader(ac) if authHeader != "" { authHeader = "Proxy-Authorization: " + authHeader + "\r\n" + authHeader += ac.HeadersNoAuthString() } conn, err := sendConnectRequest(proxyConn, proxyAddr, addr, authHeader) if err != nil {