Support for username_file in scrape config (basic_auth) similar to Prometheus for having config compatibility (#5720)

* adding support for username_file in basic_auth of scrape config

Signed-off-by: Syed Nihal <syed.nihal@nokia.com>

* adding support for username_file in basic_auth of scrape config. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5511

Signed-off-by: Syed Nihal <syed.nihal@nokia.com>

* adding support for username_file in basic_auth of scrape config

Signed-off-by: Syed Nihal <syed.nihal@nokia.com>

* adding support for username_file in basic_auth of scrape config

Signed-off-by: Syed Nihal <syed.nihal@nokia.com>

* adding support for username_file in basic_auth of scrape config

Signed-off-by: Syed Nihal <syed.nihal@nokia.com>

---------

Signed-off-by: Syed Nihal <syed.nihal@nokia.com>

docs/CHANGELOG.md

@@ -39,6 +39,7 @@ Released at 2024-01-30
 * SECURITY: upgrade base docker image (Alpine) from 3.19.0 to 3.19.1. See [alpine 3.19.1 release notes](https://www.alpinelinux.org/posts/Alpine-3.19.1-released.html).
 * SECURITY: upgrade Go builder from Go1.21.5 to Go1.21.6. See [the list of issues addressed in Go1.21.6](https://github.com/golang/go/issues?q=milestone%3AGo1.21.6+label%3ACherryPickApproved).
 
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html) added support of `username_file` for `basic_auth` in scrape_config to have config compatibility with Prometheus. See [these docs](https://docs.victoriametrics.com/sd_configs/#http-api-client-options) and [issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5511). Thanks to @wasim-nihal for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5720).
 * FEATURE: improve new [time series](https://docs.victoriametrics.com/keyConcepts.html#time-series) registration speed on systems with high number of CPU cores. Thanks to @misutoth for the initial idea and [implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5212).
 * FEATURE: make [background merge](https://docs.victoriametrics.com/#storage) more responsive and scalable. This should help the following issues: [5190](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5190), [3425](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3425), [648](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/648).
 * FEATURE: [graphite](https://docs.victoriametrics.com/#graphite-render-api-usage): add support for negative index in `groupByNode` and `aliasByNode` functions. Thanks to @rbizos for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5581).

docs/sd_configs.md

@@ -821,6 +821,7 @@ scrape_configs:
     #
     # basic_auth:
     #  username: "..."
+    #  username_file: "..."  # is mutually-exclusive with username
     #  password: "..."
     #  password_file: "..."  # is mutually-exclusive with password
 
@@ -1625,8 +1626,9 @@ and in the majority of [supported service discovery configs](#supported-service-
     # basic_auth is an optional HTTP basic authentication configuration.
     # basic_auth:
     #   username: "..."
+    #   username_file: "..."  # is mutually-exclusive with username
     #   password: "..."
-    #   password_file: "..."
+    #   password_file: "..."  # is mutually-exclusive with password
 
     # bearer_token is an optional Bearer token to send in every HTTP API request during service discovery.
     # bearer_token: "..."
@@ -1663,8 +1665,9 @@ and in the majority of [supported service discovery configs](#supported-service-
     # proxy_basic_auth is an optional HTTP basic authentication configuration for the proxy_url.
     # proxy_basic_auth:
     #   username: "..."
+    #   username_file: "..."  # is mutually-exclusive with username
     #   password: "..."
-    #   password_file: "..."
+    #   password_file: "..."  # is mutually-exclusive with password
 
     # proxy_bearer_token is an optional Bearer token to send to proxy_url.
     # proxy_bearer_token: "..."

lib/promauth/config.go

@@ -93,6 +93,7 @@ type Authorization struct {
 // BasicAuthConfig represents basic auth config.
 type BasicAuthConfig struct {
 	Username     string  `yaml:"username"`
+	UsernameFile string  `yaml:"username_file"`
 	Password     *Secret `yaml:"password,omitempty"`
 	PasswordFile string  `yaml:"password_file,omitempty"`
 }
@@ -643,36 +644,76 @@ func (actx *authContext) initFromAuthorization(baseDir string, az *Authorization
 }
 
 func (actx *authContext) initFromBasicAuthConfig(baseDir string, ba *BasicAuthConfig) error {
-	if ba.Username == "" {
+	if ba.Username == "" && ba.UsernameFile == "" {
-		return fmt.Errorf("missing `username` in `basic_auth` section")
+		return fmt.Errorf("missing `username` and `username_file` in `basic_auth` section. either one should be configured")
 	}
-	if ba.PasswordFile == "" {
+	if ba.Username != "" && ba.UsernameFile != "" {
-		// See https://en.wikipedia.org/wiki/Basic_access_authentication
+		return fmt.Errorf("both `username`=%q and `username_file`=%q are set in `basic_auth` section", ba.Username, ba.UsernameFile)
-		token := ba.Username + ":" + ba.Password.String()
-		token64 := base64.StdEncoding.EncodeToString([]byte(token))
-		ah := "Basic " + token64
-		actx.getAuthHeader = func() (string, error) {
-			return ah, nil
-		}
-		actx.authHeaderDigest = fmt.Sprintf("basic(username=%q, password=%q)", ba.Username, ba.Password)
-		return nil
 	}
-	if ba.Password != nil {
+	if ba.Password != nil && ba.PasswordFile != "" {
 		return fmt.Errorf("both `password`=%q and `password_file`=%q are set in `basic_auth` section", ba.Password, ba.PasswordFile)
 	}
-	filePath := fscore.GetFilepath(baseDir, ba.PasswordFile)
+	actx.getAuthHeader, actx.authHeaderDigest = fetchBasicAuthHeaderAndDigest(baseDir, ba)
-	actx.getAuthHeader = func() (string, error) {
+	return nil
-		password, err := fscore.ReadPasswordFromFileOrHTTP(filePath)
+}
-		if err != nil {
+
-			return "", fmt.Errorf("cannot read password from `password_file`=%q set in `basic_auth` section: %w", ba.PasswordFile, err)
+func fetchBasicAuthHeaderAndDigest(baseDir string, ba *BasicAuthConfig) (func() (string, error), string) {
+	var getUsername, getPassword func() (string, error)
+	var usernameDigest, passwordDigest string
+
+	if ba.Username != "" {
+		usernameDigest = fmt.Sprintf("username=%q", ba.Username)
+		getUsername = func() (string, error) {
+			return ba.Username, nil
 		}
-		// See https://en.wikipedia.org/wiki/Basic_access_authentication
+	}
-		token := ba.Username + ":" + password
+	if ba.UsernameFile != "" {
-		token64 := base64.StdEncoding.EncodeToString([]byte(token))
+		usernameDigest = fmt.Sprintf("usernameFile=%q", ba.UsernameFile)
+		getUsername = func() (string, error) {
+			username, err := fscore.ReadPasswordFromFileOrHTTP(fscore.GetFilepath(baseDir, ba.UsernameFile))
+			if err != nil {
+				return "", fmt.Errorf("cannot read username from `username_file`=%q set in `basic_auth` section: %w", ba.UsernameFile, err)
+			}
+			return username, nil
+		}
+	}
+	if ba.Password != nil {
+		passwordDigest = fmt.Sprintf("password=%q", ba.Password.String())
+		getPassword = func() (string, error) {
+			return ba.Password.String(), nil
+		}
+	}
+	if ba.PasswordFile != "" {
+		passwordDigest = fmt.Sprintf("passwordFile=%q", ba.PasswordFile)
+		getPassword = func() (string, error) {
+			password, err := fscore.ReadPasswordFromFileOrHTTP(fscore.GetFilepath(baseDir, ba.PasswordFile))
+			if err != nil {
+				return "", fmt.Errorf("cannot read password from `password_file`=%q set in `basic_auth` section: %w", ba.PasswordFile, err)
+			}
+			return password, nil
+		}
+	}
+	authHeader := func() (string, error) {
+		username, err := getUsername()
+		if err != nil {
+			return "", err
+		}
+		password, err := getPassword()
+		if err != nil {
+			return "", err
+		}
+		token64 := generateBasicAuthEncodedToken(username, password)
 		return "Basic " + token64, nil
 	}
-	actx.authHeaderDigest = fmt.Sprintf("basic(username=%q, passwordFile=%q)", ba.Username, filePath)
+	authHeaderDigest := fmt.Sprintf("basic(%q, %q)", usernameDigest, passwordDigest)
-	return nil
+	return authHeader, authHeaderDigest
+}
+
+func generateBasicAuthEncodedToken(username string, password string) string {
+	// See https://en.wikipedia.org/wiki/Basic_access_authentication
+	token := username + ":" + password
+	token64 := base64.StdEncoding.EncodeToString([]byte(token))
+	return token64
 }
 
 func (actx *authContext) mustInitFromBearerTokenFile(baseDir string, bearerTokenFile string) {

lib/promauth/config_test.go

@@ -55,6 +55,14 @@ basic_auth:
   password_file: testdata/test_secretfile.txt
 `)
 
+	// basic_auth: username and username_file are set
+	f(`
+basic_auth:
+  username: user
+  username_file: testdata/test_secretfile.txt
+  password: pass
+`)
+
 	// bearer_token: both authorization and bearer_token are set
 	f(`
 authorization:
@@ -345,13 +353,20 @@ oauth2:
     ca_file: non-existing-file
 `)
 
-	// basic auth via non-existing file
+	// basic auth via non-existing password_file
 	f(`
 basic_auth:
   username: user
   password_file: non-existing-file
 `)
 
+	// basic auth via non-existing username_file
+	f(`
+basic_auth:
+  username_file: non-existing-file
+  password: pass
+`)
+
 	// bearer token via non-existing file
 	f(`
 bearer_token_file: non-existing-file
@@ -471,6 +486,20 @@ basic_auth:
   password_file: testdata/test_secretfile.txt
 `, "Basic dXNlcjpzZWNyZXQtY29udGVudA==")
 
+	// basic auth username via file
+	f(`
+   basic_auth:
+     username_file: testdata/test_secretfile.txt
+     password: password
+   `, "Basic c2VjcmV0LWNvbnRlbnQ6cGFzc3dvcmQ=")
+
+	// basic auth username and password via file
+	f(`
+   basic_auth:
+     username_file: testdata/test_secretfile.txt
+     password_file: testdata/test_secretfile.txt
+`, "Basic c2VjcmV0LWNvbnRlbnQ6c2VjcmV0LWNvbnRlbnQ=")
+
 	// inline authorization config
 	f(`
 authorization: