lib/promscrape: add support for authorization config in -promscrape.config as Prometheus 2.26 does

See https://github.com/prometheus/prometheus/pull/8512

app/vmagent/remotewrite/client.go

@@ -160,7 +160,7 @@ func getTLSConfig(argIdx int) (*tls.Config, error) {
 	if c.CAFile == "" && c.CertFile == "" && c.KeyFile == "" && c.ServerName == "" && !c.InsecureSkipVerify {
 		return nil, nil
 	}
-	cfg, err := promauth.NewConfig(".", nil, "", "", c)
+	cfg, err := promauth.NewConfig(".", nil, nil, "", "", c)
 	if err != nil {
 		return nil, fmt.Errorf("cannot populate TLS config: %w", err)
 	}

docs/CHANGELOG.md

@@ -4,7 +4,8 @@
 
 * FEATURE: vminsert and vmagent: add `-sortLabels` command-line flag for sorting metric labels before pushing them to `vmstorage`. This should reduce the size of `MetricName -> internal_series_id` cache (aka `vm_cache_size_bytes{type="storage/tsid"}`) when ingesting samples for the same time series with distinct order of labels. For example, `foo{k1="v1",k2="v2"}` and `foo{k2="v2",k1="v1"}` represent a single time series.
 * FEATURE: update Go builder from `v1.16.2` to `v1.16.3`. This should fix [these issues](https://github.com/golang/go/issues?q=milestone%3AGo1.16.3+label%3ACherryPickApproved).
-* FEATURE: vmagent: add support for `follow_redirects` option to `scrape_configs` section in the same way as [Prometheus does](https://github.com/prometheus/prometheus/pull/8546).
+* FEATURE: vmagent: add support for `follow_redirects` option to `scrape_configs` section in the same way as [Prometheus 2.26 does](https://github.com/prometheus/prometheus/pull/8546).
+* FEATURE: vmagent: add support for `authorization` section in `-promscrape.config` in the same way as [Prometheus 2.26 does](https://github.com/prometheus/prometheus/pull/8512).
 * FEATURE: vmagent: reduce memory usage when `-remoteWrite.queues` is set to a big value. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1167).
 * FEATURE: vmagent: add AWS IAM roles for tasks support for EC2 service discovery according to [these docs](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html).
 

lib/promauth/config.go

@@ -20,6 +20,15 @@ type TLSConfig struct {
 	InsecureSkipVerify bool   `yaml:"insecure_skip_verify,omitempty"`
 }
 
+// Authorization represents generic authorization config.
+//
+// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/
+type Authorization struct {
+	Type            string `yaml:"type,omitempty"`
+	Credentials     string `yaml:"credentials,omitempty"`
+	CredentialsFile string `yaml:"credentials_file,omitempty"`
+}
+
 // BasicAuthConfig represents basic auth config.
 type BasicAuthConfig struct {
 	Username     string `yaml:"username"`
@@ -27,6 +36,15 @@ type BasicAuthConfig struct {
 	PasswordFile string `yaml:"password_file,omitempty"`
 }
 
+// HTTPClientConfig represents http client config.
+type HTTPClientConfig struct {
+	Authorization   *Authorization   `yaml:"authorization,omitempty"`
+	BasicAuth       *BasicAuthConfig `yaml:"basic_auth,omitempty"`
+	BearerToken     string           `yaml:"bearer_token,omitempty"`
+	BearerTokenFile string           `yaml:"bearer_token_file,omitempty"`
+	TLSConfig       *TLSConfig       `yaml:"tls_config,omitempty"`
+}
+
 // Config is auth config.
 type Config struct {
 	// Optional `Authorization` header.
@@ -80,10 +98,37 @@ func (ac *Config) NewTLSConfig() *tls.Config {
 	return tlsCfg
 }
 
+// NewConfig creates auth config for the given hcc.
+func (hcc *HTTPClientConfig) NewConfig(baseDir string) (*Config, error) {
+	return NewConfig(baseDir, hcc.Authorization, hcc.BasicAuth, hcc.BearerToken, hcc.BearerTokenFile, hcc.TLSConfig)
+}
+
 // NewConfig creates auth config from the given args.
-func NewConfig(baseDir string, basicAuth *BasicAuthConfig, bearerToken, bearerTokenFile string, tlsConfig *TLSConfig) (*Config, error) {
+func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, bearerToken, bearerTokenFile string, tlsConfig *TLSConfig) (*Config, error) {
 	var authorization string
+	if az != nil {
+		azType := "Bearer"
+		if az.Type != "" {
+			azType = az.Type
+		}
+		azToken := az.Credentials
+		if az.CredentialsFile != "" {
+			if az.Credentials != "" {
+				return nil, fmt.Errorf("both `credentials`=%q and `credentials_file`=%q are set", az.Credentials, az.CredentialsFile)
+			}
+			path := getFilepath(baseDir, az.CredentialsFile)
+			token, err := readPasswordFromFile(path)
+			if err != nil {
+				return nil, fmt.Errorf("cannot read credentials from `credentials_file`=%q: %w", az.CredentialsFile, err)
+			}
+			azToken = token
+		}
+		authorization = azType + " " + azToken
+	}
 	if basicAuth != nil {
+		if authorization != "" {
+			return nil, fmt.Errorf("cannot use both `authorization` and `basic_auth`")
+		}
 		if basicAuth.Username == "" {
 			return nil, fmt.Errorf("missing `username` in `basic_auth` section")
 		}
@@ -106,6 +151,9 @@ func NewConfig(baseDir string, basicAuth *BasicAuthConfig, bearerToken, bearerTo
 		authorization = "Basic " + token64
 	}
 	if bearerTokenFile != "" {
+		if authorization != "" {
+			return nil, fmt.Errorf("cannot simultaneously use `authorization`, `basic_auth` and `bearer_token_file`")
+		}
 		if bearerToken != "" {
 			return nil, fmt.Errorf("both `bearer_token`=%q and `bearer_token_file`=%q are set", bearerToken, bearerTokenFile)
 		}
@@ -114,11 +162,11 @@ func NewConfig(baseDir string, basicAuth *BasicAuthConfig, bearerToken, bearerTo
 		if err != nil {
 			return nil, fmt.Errorf("cannot read bearer token from `bearer_token_file`=%q: %w", bearerTokenFile, err)
 		}
-		bearerToken = token
+		authorization = "Bearer " + token
 	}
 	if bearerToken != "" {
 		if authorization != "" {
-			return nil, fmt.Errorf("cannot use both `basic_auth` and `bearer_token`")
+			return nil, fmt.Errorf("cannot simultaneously use `authorization`, `basic_auth` and `bearer_token`")
 		}
 		authorization = "Bearer " + bearerToken
 	}

lib/promscrape/config.go

@@ -91,35 +91,34 @@ type ScrapeConfig struct {
 	FollowRedirects      *bool                       `yaml:"follow_redirects"` // omitempty isn't set, since the default value for this flag is true.
 	Scheme               string                      `yaml:"scheme,omitempty"`
 	Params               map[string][]string         `yaml:"params,omitempty"`
-	BasicAuth            *promauth.BasicAuthConfig   `yaml:"basic_auth,omitempty"`
+	HTTPClientConfig     promauth.HTTPClientConfig   `yaml:",inline"`
-	BearerToken          string                      `yaml:"bearer_token,omitempty"`
-	BearerTokenFile      string                      `yaml:"bearer_token_file,omitempty"`
 	ProxyURL             proxy.URL                   `yaml:"proxy_url,omitempty"`
-	TLSConfig            *promauth.TLSConfig         `yaml:"tls_config,omitempty"`
-	StaticConfigs        []StaticConfig              `yaml:"static_configs,omitempty"`
-	FileSDConfigs        []FileSDConfig              `yaml:"file_sd_configs,omitempty"`
-	KubernetesSDConfigs  []kubernetes.SDConfig       `yaml:"kubernetes_sd_configs,omitempty"`
-	OpenStackSDConfigs   []openstack.SDConfig        `yaml:"openstack_sd_configs,omitempty"`
-	ConsulSDConfigs      []consul.SDConfig           `yaml:"consul_sd_configs,omitempty"`
-	EurekaSDConfigs      []eureka.SDConfig           `yaml:"eureka_sd_configs,omitempty"`
-	DockerSwarmSDConfigs []dockerswarm.SDConfig      `yaml:"dockerswarm_sd_configs,omitempty"`
-	DNSSDConfigs         []dns.SDConfig              `yaml:"dns_sd_configs,omitempty"`
-	EC2SDConfigs         []ec2.SDConfig              `yaml:"ec2_sd_configs,omitempty"`
-	GCESDConfigs         []gce.SDConfig              `yaml:"gce_sd_configs,omitempty"`
 	RelabelConfigs       []promrelabel.RelabelConfig `yaml:"relabel_configs,omitempty"`
 	MetricRelabelConfigs []promrelabel.RelabelConfig `yaml:"metric_relabel_configs,omitempty"`
 	SampleLimit          int                         `yaml:"sample_limit,omitempty"`
 
+	StaticConfigs        []StaticConfig         `yaml:"static_configs,omitempty"`
+	FileSDConfigs        []FileSDConfig         `yaml:"file_sd_configs,omitempty"`
+	KubernetesSDConfigs  []kubernetes.SDConfig  `yaml:"kubernetes_sd_configs,omitempty"`
+	OpenStackSDConfigs   []openstack.SDConfig   `yaml:"openstack_sd_configs,omitempty"`
+	ConsulSDConfigs      []consul.SDConfig      `yaml:"consul_sd_configs,omitempty"`
+	EurekaSDConfigs      []eureka.SDConfig      `yaml:"eureka_sd_configs,omitempty"`
+	DockerSwarmSDConfigs []dockerswarm.SDConfig `yaml:"dockerswarm_sd_configs,omitempty"`
+	DNSSDConfigs         []dns.SDConfig         `yaml:"dns_sd_configs,omitempty"`
+	EC2SDConfigs         []ec2.SDConfig         `yaml:"ec2_sd_configs,omitempty"`
+	GCESDConfigs         []gce.SDConfig         `yaml:"gce_sd_configs,omitempty"`
+
 	// These options are supported only by lib/promscrape.
 	DisableCompression   bool                      `yaml:"disable_compression,omitempty"`
 	DisableKeepAlive     bool                      `yaml:"disable_keepalive,omitempty"`
 	StreamParse          bool                      `yaml:"stream_parse,omitempty"`
 	ScrapeAlignInterval  time.Duration             `yaml:"scrape_align_interval,omitempty"`
 	ScrapeOffset         time.Duration             `yaml:"scrape_offset,omitempty"`
-	ProxyTLSConfig       *promauth.TLSConfig       `yaml:"proxy_tls_config,omitempty"`
+	ProxyAuthorization   *promauth.Authorization   `yaml:"proxy_authorization,omitempty"`
 	ProxyBasicAuth       *promauth.BasicAuthConfig `yaml:"proxy_basic_auth,omitempty"`
 	ProxyBearerToken     string                    `yaml:"proxy_bearer_token,omitempty"`
 	ProxyBearerTokenFile string                    `yaml:"proxy_bearer_token_file,omitempty"`
+	ProxyTLSConfig       *promauth.TLSConfig       `yaml:"proxy_tls_config,omitempty"`
 
 	// This is set in loadConfig
 	swc *scrapeWorkConfig
@@ -548,11 +547,11 @@ func getScrapeWorkConfig(sc *ScrapeConfig, baseDir string, globalCfg *GlobalConf
 		return nil, fmt.Errorf("unexpected `scheme` for `job_name` %q: %q; supported values: http or https", jobName, scheme)
 	}
 	params := sc.Params
-	ac, err := promauth.NewConfig(baseDir, sc.BasicAuth, sc.BearerToken, sc.BearerTokenFile, sc.TLSConfig)
+	ac, err := sc.HTTPClientConfig.NewConfig(baseDir)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse auth config for `job_name` %q: %w", jobName, err)
 	}
-	proxyAC, err := promauth.NewConfig(baseDir, sc.ProxyBasicAuth, sc.ProxyBearerToken, sc.ProxyBearerTokenFile, sc.ProxyTLSConfig)
+	proxyAC, err := promauth.NewConfig(baseDir, sc.ProxyAuthorization, sc.ProxyBasicAuth, sc.ProxyBearerToken, sc.ProxyBearerTokenFile, sc.ProxyTLSConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse proxy auth config for `job_name` %q: %w", jobName, err)
 	}

lib/promscrape/discovery/consul/api.go

@@ -50,7 +50,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 		}
 		token = ""
 	}
-	ac, err := promauth.NewConfig(baseDir, ba, token, "", sdc.TLSConfig)
+	ac, err := promauth.NewConfig(baseDir, nil, ba, token, "", sdc.TLSConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse auth config: %w", err)
 	}

lib/promscrape/discovery/dockerswarm/api.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
@@ -34,8 +33,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 		port:            sdc.Port,
 		filtersQueryArg: getFiltersQueryArg(sdc.Filters),
 	}
-
+	ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
-	ac, err := promauth.NewConfig(baseDir, sdc.BasicAuth, sdc.BearerToken, sdc.BearerTokenFile, sdc.TLSConfig)
 	if err != nil {
 		return nil, err
 	}

lib/promscrape/discovery/dockerswarm/dockerswarm.go

@@ -16,12 +16,9 @@ type SDConfig struct {
 	Port    int      `yaml:"port,omitempty"`
 	Filters []Filter `yaml:"filters,omitempty"`
 
-	ProxyURL  proxy.URL           `yaml:"proxy_url,omitempty"`
+	ProxyURL         proxy.URL                 `yaml:"proxy_url,omitempty"`
-	TLSConfig *promauth.TLSConfig `yaml:"tls_config,omitempty"`
+	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
 	// refresh_interval is obtained from `-promscrape.dockerswarmSDCheckInterval` command-line option
-	BasicAuth       *promauth.BasicAuthConfig `yaml:"basic_auth,omitempty"`
-	BearerToken     string                    `yaml:"bearer_token,omitempty"`
-	BearerTokenFile string                    `yaml:"bearer_token_file,omitempty"`
 }
 
 // Filter is a filter, which can be passed to SDConfig.

lib/promscrape/discovery/eureka/api.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
 )
 
@@ -16,19 +15,7 @@ type apiConfig struct {
 }
 
 func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
-	token := ""
+	ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
-	if sdc.Token != nil {
-		token = *sdc.Token
-	}
-	var ba *promauth.BasicAuthConfig
-	if len(sdc.Username) > 0 {
-		ba = &promauth.BasicAuthConfig{
-			Username: sdc.Username,
-			Password: sdc.Password,
-		}
-		token = ""
-	}
-	ac, err := promauth.NewConfig(baseDir, ba, token, "", sdc.TLSConfig)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse auth config: %w", err)
 	}
@@ -37,9 +24,9 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 		apiServer = "localhost:8080/eureka/v2"
 	}
 	if !strings.Contains(apiServer, "://") {
-		scheme := sdc.Scheme
+		scheme := "http"
-		if scheme == "" {
+		if sdc.HTTPClientConfig.TLSConfig != nil {
-			scheme = "http"
+			scheme = "https"
 		}
 		apiServer = scheme + "://" + apiServer
 	}

lib/promscrape/discovery/eureka/eureka.go

@@ -16,17 +16,11 @@ const appsAPIPath = "/apps"
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka
 type SDConfig struct {
-	Server     string              `yaml:"server,omitempty"`
+	Server           string                    `yaml:"server,omitempty"`
-	Token      *string             `yaml:"token"`
+	ProxyURL         proxy.URL                 `yaml:"proxy_url,omitempty"`
-	Datacenter string              `yaml:"datacenter"`
+	HTTPClientConfig promauth.HTTPClientConfig `ymal:",inline"`
-	Scheme     string              `yaml:"scheme,omitempty"`
-	Username   string              `yaml:"username"`
-	Password   string              `yaml:"password"`
-	ProxyURL   proxy.URL           `yaml:"proxy_url,omitempty"`
-	TLSConfig  *promauth.TLSConfig `yaml:"tls_config,omitempty"`
 	// RefreshInterval time.Duration `yaml:"refresh_interval"`
 	// refresh_interval is obtained from `-promscrape.ec2SDCheckInterval` command-line option.
-	Port *int `yaml:"port,omitempty"`
 }
 
 type applications struct {
@@ -95,11 +89,7 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	port := 80
+	return addInstanceLabels(apps), nil
-	if sdc.Port != nil {
-		port = *sdc.Port
-	}
-	return addInstanceLabels(apps, port), nil
 }
 
 // MustStop stops further usage for sdc.
@@ -107,11 +97,11 @@ func (sdc *SDConfig) MustStop() {
 	configMap.Delete(sdc)
 }
 
-func addInstanceLabels(apps *applications, port int) []map[string]string {
+func addInstanceLabels(apps *applications) []map[string]string {
 	var ms []map[string]string
 	for _, app := range apps.Applications {
 		for _, instance := range app.Instances {
-			instancePort := port
+			instancePort := 80
 			if instance.Port.Port != 0 {
 				instancePort = instance.Port.Port
 			}

lib/promscrape/discovery/eureka/eureka_test.go

@@ -11,7 +11,6 @@ import (
 func Test_addInstanceLabels(t *testing.T) {
 	type args struct {
 		applications *applications
-		port         int
 	}
 	tests := []struct {
 		name string
@@ -21,7 +20,6 @@ func Test_addInstanceLabels(t *testing.T) {
 		{
 			name: "1 application",
 			args: args{
-				port: 9100,
 				applications: &applications{
 					Applications: []Application{
 						{
@@ -43,6 +41,9 @@ func Test_addInstanceLabels(t *testing.T) {
 											XMLName: struct{ Space, Local string }{Local: "key-1"},
 										},
 									}},
+									Port: Port{
+										Port: 9100,
+									},
 								},
 							},
 						},
@@ -64,6 +65,8 @@ func Test_addInstanceLabels(t *testing.T) {
 					"__meta_eureka_app_instance_statuspage_url":     "some-status-url",
 					"__meta_eureka_app_instance_id":                 "some-id",
 					"__meta_eureka_app_instance_metadata_key_1":     "value-1",
+					"__meta_eureka_app_instance_port":               "9100",
+					"__meta_eureka_app_instance_port_enabled":       "false",
 					"__meta_eureka_app_instance_status":             "Ok",
 				}),
 			},
@@ -71,7 +74,7 @@ func Test_addInstanceLabels(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got := addInstanceLabels(tt.args.applications, tt.args.port)
+			got := addInstanceLabels(tt.args.applications)
 			var sortedLabelss [][]prompbmarshal.Label
 			for _, labels := range got {
 				sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))

lib/promscrape/discovery/kubernetes/api.go

@@ -35,7 +35,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu
 	default:
 		return nil, fmt.Errorf("unexpected `role`: %q; must be one of `node`, `pod`, `service`, `endpoints`, `endpointslices` or `ingress`", sdc.Role)
 	}
-	ac, err := promauth.NewConfig(baseDir, sdc.BasicAuth, sdc.BearerToken, sdc.BearerTokenFile, sdc.TLSConfig)
+	ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
 	if err != nil {
 		return nil, fmt.Errorf("cannot parse auth config: %w", err)
 	}
@@ -58,7 +58,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu
 		tlsConfig := promauth.TLSConfig{
 			CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
 		}
-		acNew, err := promauth.NewConfig(".", nil, "", "/var/run/secrets/kubernetes.io/serviceaccount/token", &tlsConfig)
+		acNew, err := promauth.NewConfig(".", nil, nil, "", "/var/run/secrets/kubernetes.io/serviceaccount/token", &tlsConfig)
 		if err != nil {
 			return nil, fmt.Errorf("cannot initialize service account auth: %w; probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?", err)
 		}
@@ -66,7 +66,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFu
 	}
 	if !strings.Contains(apiServer, "://") {
 		proto := "http"
-		if sdc.TLSConfig != nil {
+		if sdc.HTTPClientConfig.TLSConfig != nil {
 			proto = "https"
 		}
 		apiServer = proto + "://" + apiServer

lib/promscrape/discovery/kubernetes/kubernetes.go

@@ -11,15 +11,12 @@ import (
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config
 type SDConfig struct {
-	APIServer       string                    `yaml:"api_server,omitempty"`
+	APIServer        string                    `yaml:"api_server,omitempty"`
-	Role            string                    `yaml:"role"`
+	Role             string                    `yaml:"role"`
-	BasicAuth       *promauth.BasicAuthConfig `yaml:"basic_auth,omitempty"`
+	HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"`
-	BearerToken     string                    `yaml:"bearer_token,omitempty"`
+	ProxyURL         proxy.URL                 `yaml:"proxy_url,omitempty"`
-	BearerTokenFile string                    `yaml:"bearer_token_file,omitempty"`
+	Namespaces       Namespaces                `yaml:"namespaces,omitempty"`
-	ProxyURL        proxy.URL                 `yaml:"proxy_url,omitempty"`
+	Selectors        []Selector                `yaml:"selectors,omitempty"`
-	TLSConfig       *promauth.TLSConfig       `yaml:"tls_config,omitempty"`
-	Namespaces      Namespaces                `yaml:"namespaces,omitempty"`
-	Selectors       []Selector                `yaml:"selectors,omitempty"`
 }
 
 // Namespaces represents namespaces for SDConfig

lib/promscrape/discovery/openstack/api.go

@@ -77,7 +77,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
 		port:         sdc.Port,
 	}
 	if sdc.TLSConfig != nil {
-		ac, err := promauth.NewConfig(baseDir, nil, "", "", sdc.TLSConfig)
+		ac, err := promauth.NewConfig(baseDir, nil, nil, "", "", sdc.TLSConfig)
 		if err != nil {
 			return nil, err
 		}