lib/promauth: add ability to send additional http headers in requests to scrape targets

This solves https://stackoverflow.com/questions/66032498/prometheus-scrape-metric-with-custom-header

app/vmagent/README.md

@@ -183,6 +183,16 @@ Please file feature requests to [our issue tracker](https://github.com/VictoriaM
 
 `vmagent` also support the following additional options in `scrape_configs` section:
 
+* `headers` - a list of HTTP headers to send to scrape target with each scrape request. This can be used when the scrape target needs custom authorization and authentication. For example:
+
+```yaml
+scrape_configs:
+- job_name: custom_headers
+  headers:
+  - "TenantID: abc"
+  - "My-Auth: TopSecret"
+```
+
 * `disable_compression: true` - to disable response compression on a per-job basis. By default `vmagent` requests compressed responses from scrape targets
   to save network bandwidth.
 * `disable_keepalive: true` - to disable [HTTP keep-alive connections](https://en.wikipedia.org/wiki/HTTP_persistent_connection) on a per-job basis.
@@ -297,6 +307,8 @@ The relabeling can be defined in the following places:
 * At the `-remoteWrite.relabelConfig` file. This relabeling is applied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage.
 * At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`.
 
+All the files with relabeling configs can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values.
+
 You can read more about relabeling in the following articles:
 
 * [How to use Relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
@@ -424,9 +436,11 @@ scrape_configs:
 Proxy can be configured with the following optional settings:
 
 * `proxy_authorization` for generic token authorization. See [Prometheus docs for details on authorization section](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config)
-* `proxy_bearer_token` and `proxy_bearer_token_file` for Bearer token authorization
 * `proxy_basic_auth` for Basic authorization. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config).
+* `proxy_bearer_token` and `proxy_bearer_token_file` for Bearer token authorization
+* `proxy_oauth2` for OAuth2 config. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2).
 * `proxy_tls_config` for TLS config. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config).
+* `proxy_headers` for passing additional HTTP headers in requests to proxy.
 
 For example:
 
@@ -443,6 +457,8 @@ scrape_configs:
     key_file: /path/to/key
     ca_file: /path/to/ca
     server_name: real-server-name
+  proxy_headers:
+  - "Proxy-Auth: top-secret"
 ```
 
 ## Cardinality limiter

app/vmagent/remotewrite/client.go

@@ -219,7 +219,7 @@ func getAuthConfig(argIdx int) (*promauth.Config, error) {
 		InsecureSkipVerify: tlsInsecureSkipVerify.GetOptionalArg(argIdx),
 	}
 
-	authCfg, err := promauth.NewConfig(".", nil, basicAuthCfg, token, tokenFile, oauth2Cfg, tlsCfg)
+	authCfg, err := promauth.NewConfig(".", nil, basicAuthCfg, token, tokenFile, oauth2Cfg, tlsCfg, nil)
 	if err != nil {
 		return nil, fmt.Errorf("cannot populate OAuth2 config for remoteWrite idx: %d, err: %w", argIdx, err)
 	}
@@ -306,9 +306,7 @@ again:
 	h.Set("Content-Type", "application/x-protobuf")
 	h.Set("Content-Encoding", "snappy")
 	h.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
-	if ah := c.authCfg.GetAuthHeader(); ah != "" {
+	c.authCfg.SetHeaders(req, true)
-		req.Header.Set("Authorization", ah)
-	}
 	if c.awsCfg != nil {
 		if err := c.awsCfg.SignRequest(req, sigv4Hash); err != nil {
 			// there is no need in retry, request will be rejected by client.Do and retried by code below

app/vmalert/datasource/vm.go

@@ -146,9 +146,7 @@ func (s *VMStorage) newRequestPOST() (*http.Request, error) {
 	}
 	req.Header.Set("Content-Type", "application/json")
 	if s.authCfg != nil {
-		if auth := s.authCfg.GetAuthHeader(); auth != "" {
+		s.authCfg.SetHeaders(req, true)
-			req.Header.Set("Authorization", auth)
-		}
 	}
 	return req, nil
 }

app/vmalert/datasource/vm_test.go

@@ -83,7 +83,7 @@ func TestVMInstantQuery(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
@@ -206,7 +206,7 @@ func TestVMRangeQuery(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}
@@ -247,7 +247,7 @@ func TestVMRangeQuery(t *testing.T) {
 }
 
 func TestRequestParams(t *testing.T) {
-	authCfg, err := promauth.NewConfig(".", nil, baCfg, "", "", nil, nil)
+	authCfg, err := baCfg.NewConfig(".")
 	if err != nil {
 		t.Fatalf("unexpected: %s", err)
 	}

app/vmalert/notifier/alertmanager.go

@@ -79,9 +79,7 @@ func (am *AlertManager) send(ctx context.Context, alerts []Alert) error {
 	req = req.WithContext(ctx)
 
 	if am.authCfg != nil {
-		if auth := am.authCfg.GetAuthHeader(); auth != "" {
+		am.authCfg.SetHeaders(req, true)
-			req.Header.Set("Authorization", auth)
-		}
 	}
 	resp, err := am.client.Do(req)
 	if err != nil {

app/vmalert/remotewrite/remotewrite.go

@@ -245,9 +245,7 @@ func (c *Client) send(ctx context.Context, data []byte) error {
 	req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0")
 
 	if c.authCfg != nil {
-		if auth := c.authCfg.GetAuthHeader(); auth != "" {
+		c.authCfg.SetHeaders(req, true)
-			req.Header.Set("Authorization", auth)
-		}
 	}
 	if !*disablePathAppend {
 		req.URL.Path = path.Join(req.URL.Path, "/api/v1/write")

docs/CHANGELOG.md

@@ -16,7 +16,17 @@ The following tip changes can be tested by building VictoriaMetrics components f
 ## tip
 
 * FEATURE: add `-search.setLookbackToStep` command-line flag, which enables InfluxDB-like gap filling during querying. See [these docs](https://docs.victoriametrics.com/guides/migrate-from-influx.html) for details.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add ability to specify additional HTTP headers to send to scrape targets via `headers` section in `scrape_configs`. This can be used when the scrape target requires custom authorization and authentication like in [this stackoverflow question](https://stackoverflow.com/questions/66032498/prometheus-scrape-metric-with-custom-header). For example, the following config instructs sending `My-Auth: top-secret` and `TenantID: FooBar` headers with each request to `http://host123:8080/metrics`:
 
+```yaml
+scrape_configs:
+- job_name: foo
+  headers:
+  - "My-Auth: top-secret"
+  - "TenantID: FooBar"
+  static_configs:
+  - targets: ["host123:8080"]
+```
 
 ## [v1.78.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.78.0)
 

docs/README.md

@@ -1134,6 +1134,8 @@ to a file containing a list of [relabel_config](https://prometheus.io/docs/prome
 The `-relabelConfig` also can point to http or https url. For example, `-relabelConfig=https://config-server/relabel_config.yml`.
 See [this article with relabeling tips and tricks](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2).
 
+The `-relabelConfig` files can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values.
+
 Example contents for `-relabelConfig` file:
 
 ```yml
@@ -1147,8 +1149,7 @@ Example contents for `-relabelConfig` file:
   regex: true
 ```
 
-VictoriaMetrics components provide additional relabeling features such as Graphite-style relabeling.
+VictoriaMetrics provides additional relabeling features such as Graphite-style relabeling. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
-See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
 
 
 ## Federation

docs/Single-server-VictoriaMetrics.md

@@ -1138,6 +1138,8 @@ to a file containing a list of [relabel_config](https://prometheus.io/docs/prome
 The `-relabelConfig` also can point to http or https url. For example, `-relabelConfig=https://config-server/relabel_config.yml`.
 See [this article with relabeling tips and tricks](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2).
 
+The `-relabelConfig` files can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values.
+
 Example contents for `-relabelConfig` file:
 
 ```yml
@@ -1151,8 +1153,7 @@ Example contents for `-relabelConfig` file:
   regex: true
 ```
 
-VictoriaMetrics components provide additional relabeling features such as Graphite-style relabeling.
+VictoriaMetrics provides additional relabeling features such as Graphite-style relabeling. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
-See [these docs](https://docs.victoriametrics.com/vmagent.html#relabeling) for more details.
 
 
 ## Federation

docs/vmagent.md

@@ -187,6 +187,16 @@ Please file feature requests to [our issue tracker](https://github.com/VictoriaM
 
 `vmagent` also support the following additional options in `scrape_configs` section:
 
+* `headers` - a list of HTTP headers to send to scrape target with each scrape request. This can be used when the scrape target needs custom authorization and authentication. For example:
+
+```yaml
+scrape_configs:
+- job_name: custom_headers
+  headers:
+  - "TenantID: abc"
+  - "My-Auth: TopSecret"
+```
+
 * `disable_compression: true` - to disable response compression on a per-job basis. By default `vmagent` requests compressed responses from scrape targets
   to save network bandwidth.
 * `disable_keepalive: true` - to disable [HTTP keep-alive connections](https://en.wikipedia.org/wiki/HTTP_persistent_connection) on a per-job basis.
@@ -301,6 +311,8 @@ The relabeling can be defined in the following places:
 * At the `-remoteWrite.relabelConfig` file. This relabeling is applied to all the collected metrics before sending them to remote storage. This relabeling can be debugged by passing `-remoteWrite.relabelDebug` command-line option to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to remote storage.
 * At the `-remoteWrite.urlRelabelConfig` files. This relabeling is applied to metrics before sending them to the corresponding `-remoteWrite.url`. This relabeling can be debugged by passing `-remoteWrite.urlRelabelDebug` command-line options to `vmagent`. In this case `vmagent` logs metrics before and after the relabeling and then drops all the logged metrics instead of sending them to the corresponding `-remoteWrite.url`.
 
+All the files with relabeling configs can contain special placeholders in the form `%{ENV_VAR}`, which are replaced by the corresponding environment variable values.
+
 You can read more about relabeling in the following articles:
 
 * [How to use Relabeling in Prometheus and VictoriaMetrics](https://valyala.medium.com/how-to-use-relabeling-in-prometheus-and-victoriametrics-8b90fc22c4b2)
@@ -428,9 +440,11 @@ scrape_configs:
 Proxy can be configured with the following optional settings:
 
 * `proxy_authorization` for generic token authorization. See [Prometheus docs for details on authorization section](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config)
-* `proxy_bearer_token` and `proxy_bearer_token_file` for Bearer token authorization
 * `proxy_basic_auth` for Basic authorization. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config).
+* `proxy_bearer_token` and `proxy_bearer_token_file` for Bearer token authorization
+* `proxy_oauth2` for OAuth2 config. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#oauth2).
 * `proxy_tls_config` for TLS config. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config).
+* `proxy_headers` for passing additional HTTP headers in requests to proxy.
 
 For example:
 
@@ -447,6 +461,8 @@ scrape_configs:
     key_file: /path/to/key
     ca_file: /path/to/ca
     server_name: real-server-name
+  proxy_headers:
+  - "Proxy-Auth: top-secret"
 ```
 
 ## Cardinality limiter

lib/promauth/config.go

@@ -15,6 +15,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
+	"github.com/VictoriaMetrics/fasthttp"
 	"github.com/cespare/xxhash/v2"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/clientcredentials"
@@ -116,6 +117,9 @@ type HTTPClientConfig struct {
 	BearerTokenFile string           `yaml:"bearer_token_file,omitempty"`
 	OAuth2          *OAuth2Config    `yaml:"oauth2,omitempty"`
 	TLSConfig       *TLSConfig       `yaml:"tls_config,omitempty"`
+
+	// Headers contains optional HTTP headers, which must be sent in the request to the server
+	Headers []string `yaml:"headers,omitempty"`
 }
 
 // ProxyClientConfig represents proxy client config.
@@ -124,7 +128,11 @@ type ProxyClientConfig struct {
 	BasicAuth       *BasicAuthConfig `yaml:"proxy_basic_auth,omitempty"`
 	BearerToken     *Secret          `yaml:"proxy_bearer_token,omitempty"`
 	BearerTokenFile string           `yaml:"proxy_bearer_token_file,omitempty"`
+	OAuth2          *OAuth2Config    `yaml:"proxy_oauth2,omitempty"`
 	TLSConfig       *TLSConfig       `yaml:"proxy_tls_config,omitempty"`
+
+	// Headers contains optional HTTP headers, which must be sent in the request to the proxy
+	Headers []string `yaml:"proxy_headers,omitempty"`
 }
 
 // OAuth2Config represent OAuth2 configuration
@@ -257,9 +265,71 @@ type Config struct {
 	authHeader         string
 	authHeaderDeadline uint64
 
+	headers []keyValue
+
 	authDigest string
 }
 
+type keyValue struct {
+	key   string
+	value string
+}
+
+func parseHeaders(headers []string) ([]keyValue, error) {
+	if len(headers) == 0 {
+		return nil, nil
+	}
+	kvs := make([]keyValue, len(headers))
+	for i, h := range headers {
+		n := strings.IndexByte(h, ':')
+		if n < 0 {
+			return nil, fmt.Errorf(`missing ':' in header %q; expecting "key: value" format`, h)
+		}
+		kv := &kvs[i]
+		kv.key = strings.TrimSpace(h[:n])
+		kv.value = strings.TrimSpace(h[n+1:])
+	}
+	return kvs, nil
+}
+
+// HeadersNoAuthString returns string representation of ac headers
+func (ac *Config) HeadersNoAuthString() string {
+	if len(ac.headers) == 0 {
+		return ""
+	}
+	a := make([]string, len(ac.headers))
+	for i, h := range ac.headers {
+		a[i] = h.key + ": " + h.value + "\r\n"
+	}
+	return strings.Join(a, "")
+}
+
+// SetHeaders sets the configuted ac headers to req.
+func (ac *Config) SetHeaders(req *http.Request, setAuthHeader bool) {
+	reqHeaders := req.Header
+	for _, h := range ac.headers {
+		reqHeaders.Set(h.key, h.value)
+	}
+	if setAuthHeader {
+		if ah := ac.GetAuthHeader(); ah != "" {
+			reqHeaders.Set("Authorization", ah)
+		}
+	}
+}
+
+// SetFasthttpHeaders sets the configured ac headers to req.
+func (ac *Config) SetFasthttpHeaders(req *fasthttp.Request, setAuthHeader bool) {
+	reqHeaders := &req.Header
+	for _, h := range ac.headers {
+		reqHeaders.Set(h.key, h.value)
+	}
+	if setAuthHeader {
+		if ah := ac.GetAuthHeader(); ah != "" {
+			reqHeaders.Set("Authorization", ah)
+		}
+	}
+}
+
 // GetAuthHeader returns optional `Authorization: ...` http header.
 func (ac *Config) GetAuthHeader() string {
 	f := ac.getAuthHeader
@@ -281,8 +351,8 @@ func (ac *Config) GetAuthHeader() string {
 // It is also used for comparing Config objects for equality. If two Config
 // objects have the same string representation, then they are considered equal.
 func (ac *Config) String() string {
-	return fmt.Sprintf("AuthDigest=%s, TLSRootCA=%s, TLSCertificate=%s, TLSServerName=%s, TLSInsecureSkipVerify=%v, TLSMinVersion=%d",
+	return fmt.Sprintf("AuthDigest=%s, Headers=%s, TLSRootCA=%s, TLSCertificate=%s, TLSServerName=%s, TLSInsecureSkipVerify=%v, TLSMinVersion=%d",
-		ac.authDigest, ac.tlsRootCAString(), ac.tlsCertDigest, ac.TLSServerName, ac.TLSInsecureSkipVerify, ac.TLSMinVersion)
+		ac.authDigest, ac.headers, ac.tlsRootCAString(), ac.tlsCertDigest, ac.TLSServerName, ac.TLSInsecureSkipVerify, ac.TLSMinVersion)
 }
 
 func (ac *Config) tlsRootCAString() string {
@@ -330,21 +400,26 @@ func (ac *Config) NewTLSConfig() *tls.Config {
 
 // NewConfig creates auth config for the given hcc.
 func (hcc *HTTPClientConfig) NewConfig(baseDir string) (*Config, error) {
-	return NewConfig(baseDir, hcc.Authorization, hcc.BasicAuth, hcc.BearerToken.String(), hcc.BearerTokenFile, hcc.OAuth2, hcc.TLSConfig)
+	return NewConfig(baseDir, hcc.Authorization, hcc.BasicAuth, hcc.BearerToken.String(), hcc.BearerTokenFile, hcc.OAuth2, hcc.TLSConfig, hcc.Headers)
 }
 
 // NewConfig creates auth config for the given pcc.
 func (pcc *ProxyClientConfig) NewConfig(baseDir string) (*Config, error) {
-	return NewConfig(baseDir, pcc.Authorization, pcc.BasicAuth, pcc.BearerToken.String(), pcc.BearerTokenFile, nil, pcc.TLSConfig)
+	return NewConfig(baseDir, pcc.Authorization, pcc.BasicAuth, pcc.BearerToken.String(), pcc.BearerTokenFile, pcc.OAuth2, pcc.TLSConfig, pcc.Headers)
 }
 
 // NewConfig creates auth config for the given o.
 func (o *OAuth2Config) NewConfig(baseDir string) (*Config, error) {
-	return NewConfig(baseDir, nil, nil, "", "", nil, o.TLSConfig)
+	return NewConfig(baseDir, nil, nil, "", "", nil, o.TLSConfig, nil)
+}
+
+// NewConfig creates auth config for the given ba.
+func (ba *BasicAuthConfig) NewConfig(baseDir string) (*Config, error) {
+	return NewConfig(baseDir, nil, ba, "", "", nil, nil, nil)
 }
 
 // NewConfig creates auth config from the given args.
-func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, bearerToken, bearerTokenFile string, o *OAuth2Config, tlsConfig *TLSConfig) (*Config, error) {
+func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, bearerToken, bearerTokenFile string, o *OAuth2Config, tlsConfig *TLSConfig, headers []string) (*Config, error) {
 	var getAuthHeader func() string
 	authDigest := ""
 	if az != nil {
@@ -517,6 +592,10 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
 			tlsMinVersion = v
 		}
 	}
+	parsedHeaders, err := parseHeaders(headers)
+	if err != nil {
+		return nil, err
+	}
 	ac := &Config{
 		TLSRootCA:             tlsRootCA,
 		TLSServerName:         tlsServerName,
@@ -527,6 +606,7 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
 		tlsCertDigest: tlsCertDigest,
 
 		getAuthHeader: getAuthHeader,
+		headers:       parsedHeaders,
 		authDigest:    authDigest,
 	}
 	return ac, nil

lib/promauth/config_test.go

@@ -4,6 +4,8 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"testing"
+
+	"github.com/VictoriaMetrics/fasthttp"
 )
 
 func TestNewConfig(t *testing.T) {
@@ -116,18 +118,103 @@ func TestNewConfig(t *testing.T) {
 				mock := httptest.NewServer(r)
 				tt.args.oauth.TokenURL = mock.URL
 			}
-			got, err := NewConfig(tt.args.baseDir, tt.args.az, tt.args.basicAuth, tt.args.bearerToken, tt.args.bearerTokenFile, tt.args.oauth, tt.args.tlsConfig)
+			got, err := NewConfig(tt.args.baseDir, tt.args.az, tt.args.basicAuth, tt.args.bearerToken, tt.args.bearerTokenFile, tt.args.oauth, tt.args.tlsConfig, nil)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("NewConfig() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			if got != nil {
-				ah := got.GetAuthHeader()
+				req, err := http.NewRequest("GET", "http://foo", nil)
+				if err != nil {
+					t.Fatalf("unexpected error in http.NewRequest: %s", err)
+				}
+				got.SetHeaders(req, true)
+				ah := req.Header.Get("Authorization")
 				if ah != tt.expectHeader {
-					t.Fatalf("unexpected auth header; got %q; want %q", ah, tt.expectHeader)
+					t.Fatalf("unexpected auth header from net/http request; got %q; want %q", ah, tt.expectHeader)
+				}
+				var fhreq fasthttp.Request
+				got.SetFasthttpHeaders(&fhreq, true)
+				ahb := fhreq.Header.Peek("Authorization")
+				if string(ahb) != tt.expectHeader {
+					t.Fatalf("unexpected auth header from fasthttp request; got %q; want %q", ahb, tt.expectHeader)
 				}
 			}
 
 		})
 	}
 }
+
+func TestParseHeadersSuccess(t *testing.T) {
+	f := func(headers []string) {
+		t.Helper()
+		headersParsed, err := parseHeaders(headers)
+		if err != nil {
+			t.Fatalf("unexpected error when parsing %s: %s", headers, err)
+		}
+		for i, h := range headersParsed {
+			s := h.key + ": " + h.value
+			if s != headers[i] {
+				t.Fatalf("unexpected header parsed; got %q; want %q", s, headers[i])
+			}
+		}
+	}
+	f(nil)
+	f([]string{"foo: bar"})
+	f([]string{"Foo: bar", "A-b-c: d-e-f"})
+}
+
+func TestParseHeadersFailure(t *testing.T) {
+	f := func(headers []string) {
+		t.Helper()
+		headersParsed, err := parseHeaders(headers)
+		if err == nil {
+			t.Fatalf("expecting non-nil error from parseHeaders(%s)", headers)
+		}
+		if headersParsed != nil {
+			t.Fatalf("expecting nil result from parseHeaders(%s)", headers)
+		}
+	}
+	f([]string{"foo"})
+	f([]string{"foo bar baz"})
+}
+
+func TestConfigHeaders(t *testing.T) {
+	f := func(headers []string, resultExpected string) {
+		t.Helper()
+		headersParsed, err := parseHeaders(headers)
+		if err != nil {
+			t.Fatalf("cannot parse headers: %s", err)
+		}
+		c, err := NewConfig("", nil, nil, "", "", nil, nil, headers)
+		if err != nil {
+			t.Fatalf("cannot create config: %s", err)
+		}
+		req, err := http.NewRequest("GET", "http://foo", nil)
+		if err != nil {
+			t.Fatalf("unexpected error in http.NewRequest: %s", err)
+		}
+		result := c.HeadersNoAuthString()
+		if result != resultExpected {
+			t.Fatalf("unexpected result from HeadersNoAuthString; got\n%s\nwant\n%s", result, resultExpected)
+		}
+		c.SetHeaders(req, false)
+		for _, h := range headersParsed {
+			v := req.Header.Get(h.key)
+			if v != h.value {
+				t.Fatalf("unexpected value for net/http header %q; got %q; want %q", h.key, v, h.value)
+			}
+		}
+		var fhreq fasthttp.Request
+		c.SetFasthttpHeaders(&fhreq, false)
+		for _, h := range headersParsed {
+			v := fhreq.Header.Peek(h.key)
+			if string(v) != h.value {
+				t.Fatalf("unexpected value for fasthttp header %q; got %q; want %q", h.key, v, h.value)
+			}
+		}
+	}
+	f(nil, "")
+	f([]string{"foo: bar"}, "foo: bar\r\n")
+	f([]string{"Foo-Bar: Baz s:sdf", "A:b", "X-Forwarded-For: A-B:c"}, "Foo-Bar: Baz s:sdf\r\nA: b\r\nX-Forwarded-For: A-B:c\r\n")
+}

lib/promscrape/client.go

@@ -48,8 +48,10 @@ type client struct {
 	scrapeTimeoutSecondsStr string
 	host                    string
 	requestURI              string
-	getAuthHeader           func() string
+	setHeaders              func(req *http.Request)
-	getProxyAuthHeader      func() string
+	setProxyHeaders         func(req *http.Request)
+	setFasthttpHeaders      func(req *fasthttp.Request)
+	setFasthttpProxyHeaders func(req *fasthttp.Request)
 	denyRedirects           bool
 	disableCompression      bool
 	disableKeepAlive        bool
@@ -65,7 +67,8 @@ func newClient(sw *ScrapeWork) *client {
 	if isTLS {
 		tlsCfg = sw.AuthConfig.NewTLSConfig()
 	}
-	getProxyAuthHeader := func() string { return "" }
+	setProxyHeaders := func(req *http.Request) {}
+	setFasthttpProxyHeaders := func(req *fasthttp.Request) {}
 	proxyURL := sw.ProxyURL
 	if !isTLS && proxyURL.IsHTTPOrHTTPS() {
 		// Send full sw.ScrapeURL in requests to a proxy host for non-TLS scrape targets
@@ -79,8 +82,11 @@ func newClient(sw *ScrapeWork) *client {
 			tlsCfg = sw.ProxyAuthConfig.NewTLSConfig()
 		}
 		proxyURLOrig := proxyURL
-		getProxyAuthHeader = func() string {
+		setProxyHeaders = func(req *http.Request) {
-			return proxyURLOrig.GetAuthHeader(sw.ProxyAuthConfig)
+			proxyURLOrig.SetHeaders(sw.ProxyAuthConfig, req)
+		}
+		setFasthttpProxyHeaders = func(req *fasthttp.Request) {
+			proxyURLOrig.SetFasthttpHeaders(sw.ProxyAuthConfig, req)
 		}
 		proxyURL = &proxy.URL{}
 	}
@@ -148,8 +154,10 @@ func newClient(sw *ScrapeWork) *client {
 		scrapeTimeoutSecondsStr: fmt.Sprintf("%.3f", sw.ScrapeTimeout.Seconds()),
 		host:                    host,
 		requestURI:              requestURI,
-		getAuthHeader:           sw.AuthConfig.GetAuthHeader,
+		setHeaders:              func(req *http.Request) { sw.AuthConfig.SetHeaders(req, true) },
-		getProxyAuthHeader:      getProxyAuthHeader,
+		setProxyHeaders:         setProxyHeaders,
+		setFasthttpHeaders:      func(req *fasthttp.Request) { sw.AuthConfig.SetFasthttpHeaders(req, true) },
+		setFasthttpProxyHeaders: setFasthttpProxyHeaders,
 		denyRedirects:           sw.DenyRedirects,
 		disableCompression:      sw.DisableCompression,
 		disableKeepAlive:        sw.DisableKeepAlive,
@@ -173,12 +181,8 @@ func (c *client) GetStreamReader() (*streamReader, error) {
 	// Set X-Prometheus-Scrape-Timeout-Seconds like Prometheus does, since it is used by some exporters such as PushProx.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1179#issuecomment-813117162
 	req.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", c.scrapeTimeoutSecondsStr)
-	if ah := c.getAuthHeader(); ah != "" {
+	c.setHeaders(req)
-		req.Header.Set("Authorization", ah)
+	c.setProxyHeaders(req)
-	}
-	if ah := c.getProxyAuthHeader(); ah != "" {
-		req.Header.Set("Proxy-Authorization", ah)
-	}
 	resp, err := c.sc.Do(req)
 	if err != nil {
 		cancel()
@@ -224,12 +228,8 @@ func (c *client) ReadData(dst []byte) ([]byte, error) {
 	// Set X-Prometheus-Scrape-Timeout-Seconds like Prometheus does, since it is used by some exporters such as PushProx.
 	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1179#issuecomment-813117162
 	req.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", c.scrapeTimeoutSecondsStr)
-	if ah := c.getAuthHeader(); ah != "" {
+	c.setFasthttpHeaders(req)
-		req.Header.Set("Authorization", ah)
+	c.setFasthttpProxyHeaders(req)
-	}
-	if ah := c.getProxyAuthHeader(); ah != "" {
-		req.Header.Set("Proxy-Authorization", ah)
-	}
 	if !*disableCompression && !c.disableCompression {
 		req.Header.Set("Accept-Encoding", "gzip")
 	}

lib/promscrape/config_test.go

@@ -141,6 +141,9 @@ scrape_configs:
     - x
   authorization:
     type: foobar
+  headers:
+  - 'TenantID: fooBar'
+  - 'X: y:z'
   relabel_configs:
   - source_labels: [abc]
   static_configs:
@@ -149,6 +152,8 @@ scrape_configs:
   relabel_debug: true
   scrape_align_interval: 1h30m0s
   proxy_bearer_token_file: file.txt
+  proxy_headers:
+  - 'My-Auth-Header: top-secret'
 `)
 
 }
@@ -332,7 +337,7 @@ scrape_configs:
 		jobNameOriginal: "blackbox",
 	}}
 	if !reflect.DeepEqual(sws, swsExpected) {
-		t.Fatalf("unexpected scrapeWork;\ngot\n%+v\nwant\n%+v", sws, swsExpected)
+		t.Fatalf("unexpected scrapeWork;\ngot\n%#v\nwant\n%#v", sws, swsExpected)
 	}
 }
 
@@ -1650,12 +1655,25 @@ scrape_configs:
 			jobNameOriginal: "aaa",
 		},
 	})
+
+	ac, err := promauth.NewConfig(".", nil, nil, "", "", nil, nil, []string{"My-Auth: foo-Bar"})
+	if err != nil {
+		t.Fatalf("unexpected error when creating promauth.Config: %s", err)
+	}
+	proxyAC, err := promauth.NewConfig(".", nil, nil, "", "", nil, nil, []string{"Foo:bar"})
+	if err != nil {
+		t.Fatalf("unexpected error when creating promauth.Config for proxy: %s", err)
+	}
 	f(`
 scrape_configs:
   - job_name: 'snmp'
     sample_limit: 100
     disable_keepalive: true
     disable_compression: true
+    headers:
+    - "My-Auth: foo-Bar"
+    proxy_headers:
+    - "Foo: bar"
     scrape_align_interval: 1s
     scrape_offset: 0.5s
     static_configs:
@@ -1727,8 +1745,8 @@ scrape_configs:
 					Value: "snmp",
 				},
 			},
-			AuthConfig:          &promauth.Config{},
+			AuthConfig:          ac,
-			ProxyAuthConfig:     &promauth.Config{},
+			ProxyAuthConfig:     proxyAC,
 			SampleLimit:         100,
 			DisableKeepAlive:    true,
 			DisableCompression:  true,

lib/promscrape/discovery/kubernetes/api.go

@@ -16,7 +16,8 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu
 	default:
 		return nil, fmt.Errorf("unexpected `role`: %q; must be one of `node`, `pod`, `service`, `endpoints`, `endpointslice` or `ingress`", role)
 	}
-	ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
+	cc := &sdc.HTTPClientConfig
+	ac, err := cc.NewConfig(baseDir)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse auth config: %w", err)
 	}
@@ -30,7 +31,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu
 		if err != nil {
 			return nil, fmt.Errorf("cannot build kube config from the specified `kubeconfig_file` config option: %w", err)
 		}
-		acNew, err := promauth.NewConfig(".", nil, kc.basicAuth, kc.token, kc.tokenFile, nil, kc.tlsConfig)
+		acNew, err := promauth.NewConfig(".", nil, kc.basicAuth, kc.token, kc.tokenFile, cc.OAuth2, kc.tlsConfig, cc.Headers)
 		if err != nil {
 			return nil, fmt.Errorf("cannot initialize auth config from `kubeconfig_file: %q`: %w", sdc.KubeConfigFile, err)
 		}
@@ -57,7 +58,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu
 		tlsConfig := promauth.TLSConfig{
 			CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
 		}
-		acNew, err := promauth.NewConfig(".", nil, nil, "", "/var/run/secrets/kubernetes.io/serviceaccount/token", nil, &tlsConfig)
+		acNew, err := promauth.NewConfig(".", nil, nil, "", "/var/run/secrets/kubernetes.io/serviceaccount/token", cc.OAuth2, &tlsConfig, cc.Headers)
 		if err != nil {
 			return nil, fmt.Errorf("cannot initialize service account auth: %w; probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?", err)
 		}

lib/promscrape/discovery/kubernetes/api_watcher.go

@@ -207,8 +207,8 @@ type groupWatcher struct {
 	selectors          []Selector
 	attachNodeMetadata bool
 
-	getAuthHeader func() string
+	setHeaders func(req *http.Request)
-	client        *http.Client
+	client     *http.Client
 
 	mu sync.Mutex
 	m  map[string]*urlWatcher
@@ -235,9 +235,9 @@ func newGroupWatcher(apiServer string, ac *promauth.Config, namespaces []string,
 		selectors:          selectors,
 		attachNodeMetadata: attachNodeMetadata,
 
-		getAuthHeader: ac.GetAuthHeader,
+		setHeaders: func(req *http.Request) { ac.SetHeaders(req, true) },
-		client:        client,
+		client:     client,
-		m:             make(map[string]*urlWatcher),
+		m:          make(map[string]*urlWatcher),
 	}
 }
 
@@ -407,9 +407,7 @@ func (gw *groupWatcher) doRequest(requestURL string) (*http.Response, error) {
 	if err != nil {
 		logger.Fatalf("cannot create a request for %q: %s", requestURL, err)
 	}
-	if ah := gw.getAuthHeader(); ah != "" {
+	gw.setHeaders(req)
-		req.Header.Set("Authorization", ah)
-	}
 	resp, err := gw.client.Do(req)
 	if err != nil {
 		return nil, err

lib/promscrape/discovery/openstack/api.go

@@ -81,7 +81,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 		port:         sdc.Port,
 	}
 	if sdc.TLSConfig != nil {
-		ac, err := promauth.NewConfig(baseDir, nil, nil, "", "", nil, sdc.TLSConfig)
+		ac, err := promauth.NewConfig(baseDir, nil, nil, "", "", nil, sdc.TLSConfig, nil)
 		if err != nil {
 			return nil, err
 		}

lib/promscrape/discoveryutils/client.go

@@ -42,10 +42,10 @@ type Client struct {
 
 	apiServer string
 
-	hostPort           string
+	hostPort                string
-	getAuthHeader      func() string
+	setFasthttpHeaders      func(req *fasthttp.Request)
-	getProxyAuthHeader func() string
+	setFasthttpProxyHeaders func(req *fasthttp.Request)
-	sendFullURL        bool
+	sendFullURL             bool
 }
 
 // NewClient returns new Client for the given args.
@@ -70,7 +70,7 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy
 		tlsCfg = ac.NewTLSConfig()
 	}
 	sendFullURL := !isTLS && proxyURL.IsHTTPOrHTTPS()
-	getProxyAuthHeader := func() string { return "" }
+	setFasthttpProxyHeaders := func(req *fasthttp.Request) {}
 	if sendFullURL {
 		// Send full urls in requests to a proxy host for non-TLS apiServer
 		// like net/http package from Go does.
@@ -82,8 +82,8 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy
 			tlsCfg = proxyAC.NewTLSConfig()
 		}
 		proxyURLOrig := proxyURL
-		getProxyAuthHeader = func() string {
+		setFasthttpProxyHeaders = func(req *fasthttp.Request) {
-			return proxyURLOrig.GetAuthHeader(proxyAC)
+			proxyURLOrig.SetFasthttpHeaders(proxyAC, req)
 		}
 		proxyURL = &proxy.URL{}
 	}
@@ -123,18 +123,18 @@ func NewClient(apiServer string, ac *promauth.Config, proxyURL *proxy.URL, proxy
 		MaxConns:            64 * 1024,
 		Dial:                dialFunc,
 	}
-	getAuthHeader := func() string { return "" }
+	setFasthttpHeaders := func(req *fasthttp.Request) {}
 	if ac != nil {
-		getAuthHeader = ac.GetAuthHeader
+		setFasthttpHeaders = func(req *fasthttp.Request) { ac.SetFasthttpHeaders(req, true) }
 	}
 	return &Client{
-		hc:                 hc,
+		hc:                      hc,
-		blockingClient:     blockingClient,
+		blockingClient:          blockingClient,
-		apiServer:          apiServer,
+		apiServer:               apiServer,
-		hostPort:           hostPort,
+		hostPort:                hostPort,
-		getAuthHeader:      getAuthHeader,
+		setFasthttpHeaders:      setFasthttpHeaders,
-		getProxyAuthHeader: getProxyAuthHeader,
+		setFasthttpProxyHeaders: setFasthttpProxyHeaders,
-		sendFullURL:        sendFullURL,
+		sendFullURL:             sendFullURL,
 	}, nil
 }
 
@@ -202,12 +202,8 @@ func (c *Client) getAPIResponseWithParamsAndClient(client *fasthttp.HostClient,
 	}
 	req.Header.SetHost(c.hostPort)
 	req.Header.Set("Accept-Encoding", "gzip")
-	if ah := c.getAuthHeader(); ah != "" {
+	c.setFasthttpHeaders(&req)
-		req.Header.Set("Authorization", ah)
+	c.setFasthttpProxyHeaders(&req)
-	}
-	if ah := c.getProxyAuthHeader(); ah != "" {
-		req.Header.Set("Proxy-Authorization", ah)
-	}
 	if modifyRequest != nil {
 		modifyRequest(&req)
 	}

lib/proxy/proxy.go

@@ -6,6 +6,7 @@ import (
 	"encoding/base64"
 	"fmt"
 	"net"
+	"net/http"
 	"net/url"
 	"strings"
 	"time"
@@ -60,8 +61,26 @@ func (u *URL) String() string {
 	return pu.String()
 }
 
-// GetAuthHeader returns Proxy-Authorization auth header for the given u and ac.
+// SetHeaders sets headers to req according to u and ac configs.
-func (u *URL) GetAuthHeader(ac *promauth.Config) string {
+func (u *URL) SetHeaders(ac *promauth.Config, req *http.Request) {
+	ah := u.getAuthHeader(ac)
+	if ah != "" {
+		req.Header.Set("Proxy-Authorization", ah)
+	}
+	ac.SetHeaders(req, false)
+}
+
+// SetFasthttpHeaders sets headers to req according to u and ac configs.
+func (u *URL) SetFasthttpHeaders(ac *promauth.Config, req *fasthttp.Request) {
+	ah := u.getAuthHeader(ac)
+	if ah != "" {
+		req.Header.Set("Proxy-Authorization", ah)
+	}
+	ac.SetFasthttpHeaders(req, false)
+}
+
+// getAuthHeader returns Proxy-Authorization auth header for the given u and ac.
+func (u *URL) getAuthHeader(ac *promauth.Config) string {
 	authHeader := ""
 	if ac != nil {
 		authHeader = ac.GetAuthHeader()
@@ -130,9 +149,10 @@ func (u *URL) NewDialFunc(ac *promauth.Config) (fasthttp.DialFunc, error) {
 		if isTLS {
 			proxyConn = tls.Client(proxyConn, tlsCfg)
 		}
-		authHeader := u.GetAuthHeader(ac)
+		authHeader := u.getAuthHeader(ac)
 		if authHeader != "" {
 			authHeader = "Proxy-Authorization: " + authHeader + "\r\n"
+			authHeader += ac.HeadersNoAuthString()
 		}
 		conn, err := sendConnectRequest(proxyConn, proxyAddr, addr, authHeader)
 		if err != nil {