lib/promauth: reload TLS certificates from disk on every mTLS connection as Prometheus does

This allows updating client certificates without the need to restart vmagent and/or single-node VictoriaMetrics.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/470

docs/CHANGELOG.md

@@ -6,6 +6,7 @@ sort: 15
 
 ## tip
 
+* FEATURE: vmagent: dynamically reload client TLS certificates from disk on every [mTLS connection](https://developers.cloudflare.com/cloudflare-one/identity/devices/mutual-tls-authentication). This should allow using `vmagent` with [Istio service mesh](https://istio.io/latest/about/service-mesh/). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420).
 * FEATURE: reduce memory usage when performing heavy queries over high number of time series.
 
 * BUGFIX: vmagent: remove `{ %space %}` typo in `/targets` output. The typo has been introduced in v1.62.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1408).

lib/promauth/config.go

@@ -158,10 +158,12 @@ func (oi *oauth2ConfigInternal) getTokenSource() (oauth2.TokenSource, error) {
 type Config struct {
 	// Optional TLS config
 	TLSRootCA             *x509.CertPool
-	TLSCertificate        *tls.Certificate
 	TLSServerName         string
 	TLSInsecureSkipVerify bool
 
+	getTLSCert    func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+	tlsCertDigest string
+
 	getAuthHeader      func() string
 	authHeaderLock     sync.Mutex
 	authHeader         string
@@ -189,7 +191,7 @@ func (ac *Config) GetAuthHeader() string {
 // String returns human-readable representation for ac.
 func (ac *Config) String() string {
 	return fmt.Sprintf("AuthDigest=%s, TLSRootCA=%s, TLSCertificate=%s, TLSServerName=%s, TLSInsecureSkipVerify=%v",
-		ac.authDigest, ac.tlsRootCAString(), ac.tlsCertificateString(), ac.TLSServerName, ac.TLSInsecureSkipVerify)
+		ac.authDigest, ac.tlsRootCAString(), ac.tlsCertDigest, ac.TLSServerName, ac.TLSInsecureSkipVerify)
 }
 
 func (ac *Config) tlsRootCAString() string {
@@ -200,13 +202,6 @@ func (ac *Config) tlsRootCAString() string {
 	return string(bytes.Join(data, []byte("\n")))
 }
 
-func (ac *Config) tlsCertificateString() string {
-	if ac.TLSCertificate == nil {
-		return ""
-	}
-	return string(bytes.Join(ac.TLSCertificate.Certificate, []byte("\n")))
-}
-
 // NewTLSConfig returns new TLS config for the given ac.
 func (ac *Config) NewTLSConfig() *tls.Config {
 	tlsCfg := &tls.Config{
@@ -215,10 +210,7 @@ func (ac *Config) NewTLSConfig() *tls.Config {
 	if ac == nil {
 		return tlsCfg
 	}
-	if ac.TLSCertificate != nil {
-		// Do not set tlsCfg.GetClientCertificate, since tlsCfg.Certificates should work OK.
-		tlsCfg.Certificates = []tls.Certificate{*ac.TLSCertificate}
-	}
+	tlsCfg.GetClientCertificate = ac.getTLSCert
 	tlsCfg.RootCAs = ac.TLSRootCA
 	tlsCfg.ServerName = ac.TLSServerName
 	tlsCfg.InsecureSkipVerify = ac.TLSInsecureSkipVerify
@@ -350,20 +342,29 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
 		authDigest = fmt.Sprintf("oauth2(%s)", o.String())
 	}
 	var tlsRootCA *x509.CertPool
-	var tlsCertificate *tls.Certificate
+	var getTLSCert func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+	tlsCertDigest := ""
 	tlsServerName := ""
 	tlsInsecureSkipVerify := false
 	if tlsConfig != nil {
 		tlsServerName = tlsConfig.ServerName
 		tlsInsecureSkipVerify = tlsConfig.InsecureSkipVerify
 		if tlsConfig.CertFile != "" || tlsConfig.KeyFile != "" {
-			certPath := getFilepath(baseDir, tlsConfig.CertFile)
-			keyPath := getFilepath(baseDir, tlsConfig.KeyFile)
-			cert, err := tls.LoadX509KeyPair(certPath, keyPath)
-			if err != nil {
-				return nil, fmt.Errorf("cannot load TLS certificate from `cert_file`=%q, `key_file`=%q: %w", tlsConfig.CertFile, tlsConfig.KeyFile, err)
+			getTLSCert = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+				// Re-read TLS certificate from disk. This is needed for https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1420
+				certPath := getFilepath(baseDir, tlsConfig.CertFile)
+				keyPath := getFilepath(baseDir, tlsConfig.KeyFile)
+				cert, err := tls.LoadX509KeyPair(certPath, keyPath)
+				if err != nil {
+					return nil, fmt.Errorf("cannot load TLS certificate from `cert_file`=%q, `key_file`=%q: %w", tlsConfig.CertFile, tlsConfig.KeyFile, err)
+				}
+				return &cert, nil
 			}
-			tlsCertificate = &cert
+			// Check whether the configured TLS cert can be loaded.
+			if _, err := getTLSCert(nil); err != nil {
+				return nil, err
+			}
+			tlsCertDigest = fmt.Sprintf("certFile=%q, keyFile=%q", tlsConfig.CertFile, tlsConfig.KeyFile)
 		}
 		if tlsConfig.CAFile != "" {
 			path := getFilepath(baseDir, tlsConfig.CAFile)
@@ -379,10 +380,12 @@ func NewConfig(baseDir string, az *Authorization, basicAuth *BasicAuthConfig, be
 	}
 	ac := &Config{
 		TLSRootCA:             tlsRootCA,
-		TLSCertificate:        tlsCertificate,
 		TLSServerName:         tlsServerName,
 		TLSInsecureSkipVerify: tlsInsecureSkipVerify,
 
+		getTLSCert:    getTLSCert,
+		tlsCertDigest: tlsCertDigest,
+
 		getAuthHeader: getAuthHeader,
 		authDigest:    authDigest,
 	}

lib/promscrape/config_test.go

@@ -1,7 +1,6 @@
 package promscrape
 
 import (
-	"crypto/tls"
 	"fmt"
 	"reflect"
 	"strconv"
@@ -1246,52 +1245,6 @@ scrape_configs:
 			jobNameOriginal: "foo",
 		},
 	})
-	snakeoilCert, err := tls.LoadX509KeyPair("testdata/ssl-cert-snakeoil.pem", "testdata/ssl-cert-snakeoil.key")
-	if err != nil {
-		t.Fatalf("cannot load snakeoil cert: %s", err)
-	}
-	f(`
-scrape_configs:
-- job_name: foo
-  tls_config:
-    cert_file: testdata/ssl-cert-snakeoil.pem
-    key_file: testdata/ssl-cert-snakeoil.key
-  static_configs:
-  - targets: ["foo.bar:1234"]
-`, []*ScrapeWork{
-		{
-			ScrapeURL:      "http://foo.bar:1234/metrics",
-			ScrapeInterval: defaultScrapeInterval,
-			ScrapeTimeout:  defaultScrapeTimeout,
-			Labels: []prompbmarshal.Label{
-				{
-					Name:  "__address__",
-					Value: "foo.bar:1234",
-				},
-				{
-					Name:  "__metrics_path__",
-					Value: "/metrics",
-				},
-				{
-					Name:  "__scheme__",
-					Value: "http",
-				},
-				{
-					Name:  "instance",
-					Value: "foo.bar:1234",
-				},
-				{
-					Name:  "job",
-					Value: "foo",
-				},
-			},
-			AuthConfig: &promauth.Config{
-				TLSCertificate: &snakeoilCert,
-			},
-			ProxyAuthConfig: &promauth.Config{},
-			jobNameOriginal: "foo",
-		},
-	})
 	f(`
 global:
   external_labels:

lib/promscrape/testdata/ssl-cert-snakeoil.key

@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQD4IQusAs8PJdnG
-3mURt/AXtgC+ceqLOatJ49JJE1VPTkMAy+oE1f1XvkMrYsHqmDf6GWVzgVXryL4U
-wq2/nJSm56ddhN55nI8oSN3dtywUB8/ShelEN73nlN77PeD9tl6NksPwWaKrqxq0
-FlabRPZSQCfmgZbhDV8Sa8mfCkFU0G0lit6kLGceCKMvmW+9Bz7ebsYmVdmVMxmf
-IJStFD44lWFTdUc65WISKEdW2ELcUefb0zOLw+0PCbXFGJH5x5ktksW8+BBk2Hkg
-GeQRL/qPCccthbScO0VgNj3zJ3ZZL0ObSDAbvNDG85joeNjDNq5DT/BAZ0bOSbEF
-sh+f9BAzAgMBAAECggEBAJWv2cq7Jw6MVwSRxYca38xuD6TUNBopgBvjREixURW2
-sNUaLuMb9Omp7fuOaE2N5rcJ+xnjPGIxh/oeN5MQctz9gwn3zf6vY+15h97pUb4D
-uGvYPRDaT8YVGS+X9NMZ4ZCmqW2lpWzKnCFoGHcy8yZLbcaxBsRdvKzwOYGoPiFb
-K2QuhXZ/1UPmqK9i2DFKtj40X6vBszTNboFxOVpXrPu0FJwLVSDf2hSZ4fMM0DH3
-YqwKcYf5te+hxGKgrqRA3tn0NCWii0in6QIwXMC+kMw1ebg/tZKqyDLMNptAK8J+
-DVw9m5X1seUHS5ehU/g2jrQrtK5WYn7MrFK4lBzlRwECgYEA/d1TeANYECDWRRDk
-B0aaRZs87Rwl/J9PsvbsKvtU/bX+OfSOUjOa9iQBqn0LmU8GqusEET/QVUfocVwV
-Bggf/5qDLxz100Rj0ags/yE/kNr0Bb31kkkKHFMnCT06YasR7qKllwrAlPJvQv9x
-IzBKq+T/Dx08Wep9bCRSFhzRCnsCgYEA+jdeZXTDr/Vz+D2B3nAw1frqYFfGnEVY
-wqmoK3VXMDkGuxsloO2rN+SyiUo3JNiQNPDub/t7175GH5pmKtZOlftePANsUjBj
-wZ1D0rI5Bxu/71ibIUYIRVmXsTEQkh/ozoh3jXCZ9+bLgYiYx7789IUZZSokFQ3D
-FICUT9KJ36kCgYAGoq9Y1rWJjmIrYfqj2guUQC+CfxbbGIrrwZqAsRsSmpwvhZ3m
-tiSZxG0quKQB+NfSxdvQW5ulbwC7Xc3K35F+i9pb8+TVBdeaFkw+yu6vaZmxQLrX
-fQM/pEjD7A7HmMIaO7QaU5SfEAsqdCTP56Y8AftMuNXn/8IRfo2KuGwaWwKBgFpU
-ILzJoVdlad9E/Rw7LjYhZfkv1uBVXIyxyKcfrkEXZSmozDXDdxsvcZCEfVHM6Ipk
-K/+7LuMcqp4AFEAEq8wTOdq6daFaHLkpt/FZK6M4TlruhtpFOPkoNc3e45eM83OT
-6mziKINJC1CQ6m65sQHpBtjxlKMRG8rL/D6wx9s5AoGBAMRlqNPMwglT3hvDmsAt
-9Lf9pdmhERUlHhD8bj8mDaBj2Aqv7f6VRJaYZqP403pKKQexuqcn80mtjkSAPFkN
-Cj7BVt/RXm5uoxDTnfi26RF9F6yNDEJ7UU9+peBr99aazF/fTgW/1GcMkQnum8uV
-c257YgaWmjK9uB0Y2r2VxS0G
------END PRIVATE KEY-----

lib/promscrape/testdata/ssl-cert-snakeoil.pem

@@ -1,17 +0,0 @@
------BEGIN CERTIFICATE-----
-MIICujCCAaKgAwIBAgIJAMbXnKZ/cikUMA0GCSqGSIb3DQEBCwUAMBUxEzARBgNV
-BAMTCnVidW50dS5uYW4wHhcNMTUwMjA0MDgwMTM5WhcNMjUwMjAxMDgwMTM5WjAV
-MRMwEQYDVQQDEwp1YnVudHUubmFuMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
-CgKCAQEA+CELrALPDyXZxt5lEbfwF7YAvnHqizmrSePSSRNVT05DAMvqBNX9V75D
-K2LB6pg3+hllc4FV68i+FMKtv5yUpuenXYTeeZyPKEjd3bcsFAfP0oXpRDe955Te
-+z3g/bZejZLD8Fmiq6satBZWm0T2UkAn5oGW4Q1fEmvJnwpBVNBtJYrepCxnHgij
-L5lvvQc+3m7GJlXZlTMZnyCUrRQ+OJVhU3VHOuViEihHVthC3FHn29Mzi8PtDwm1
-xRiR+ceZLZLFvPgQZNh5IBnkES/6jwnHLYW0nDtFYDY98yd2WS9Dm0gwG7zQxvOY
-6HjYwzauQ0/wQGdGzkmxBbIfn/QQMwIDAQABow0wCzAJBgNVHRMEAjAAMA0GCSqG
-SIb3DQEBCwUAA4IBAQBQjKm/4KN/iTgXbLTL3i7zaxYXFLXsnT1tF+ay4VA8aj98
-L3JwRTciZ3A5iy/W4VSCt3eASwOaPWHKqDBB5RTtL73LoAqsWmO3APOGQAbixcQ2
-45GXi05OKeyiYRi1Nvq7Unv9jUkRDHUYVPZVSAjCpsXzPhFkmZoTRxmx5l0ZF7Li
-K91lI5h+eFq0dwZwrmlPambyh1vQUi70VHv8DNToVU29kel7YLbxGbuqETfhrcy6
-X+Mha6RYITkAn5FqsZcKMsc9eYGEF4l3XV+oS7q6xfTxktYJMFTI18J0lQ2Lv/CI
-whdMnYGntDQBE/iFCrJEGNsKGc38796GBOb5j+zd
------END CERTIFICATE-----