app/vmauth: add option to skip TLS verification (#5256)

Add `tls_insecure_skip_verify` option on per-user basis which allows to disable TLS verification for all requests to backend on behalf of this user.

See: https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5240

Signed-off-by: Zakhar Bessarab <z.bessarab@victoriametrics.com>
(cherry picked from commit 323f3720ed)

app/vmauth/README.md

@@ -181,6 +181,15 @@ users:
   password: "***"
   url_prefix: "http://localhost:8428?extra_label=team=dev"
 
+  # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # are proxied to http://localhost:8428 with extra_label=team=dev query arg.
+  # For example, http://vmauth:8427/api/v1/query is routed to https://localhost/api/v1/query?extra_label=team=dev
+  # TLS verification is skipped for https://localhost.
+- username: "local-single-node-with-tls"
+  password: "***"
+  url_prefix: "https://localhost?extra_label=team=test"
+  tls_insecure_skip_verify: true
+
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # are load-balanced among http://vmselect1:8481/select/123/prometheus and http://vmselect2:8481/select/123/prometheus
   # For example, http://vmauth:8427/api/v1/query is proxied to the following urls in a round-robin manner:
@@ -257,6 +266,7 @@ unauthorized_user:
   - http://vmselect-az1/?deny_partial_response=1
   - http://vmselect-az2/?deny_partial_response=1
   retry_status_codes: [503, 500]
+  tls_insecure_skip_verify: true
 
 ip_filters:
   allow_list: ["1.2.3.0/24", "127.0.0.1"]

app/vmauth/auth_config.go

@@ -5,6 +5,7 @@ import (
 	"encoding/base64"
 	"flag"
 	"fmt"
+	"net/http"
 	"net/url"
 	"os"
 	"regexp"
@@ -14,13 +15,14 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+	"gopkg.in/yaml.v2"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
-	"github.com/VictoriaMetrics/metrics"
-	"gopkg.in/yaml.v2"
 )
 
 var (
@@ -48,10 +50,13 @@ type UserInfo struct {
 	MaxConcurrentRequests int         `yaml:"max_concurrent_requests,omitempty"`
 	DefaultURL            *URLPrefix  `yaml:"default_url,omitempty"`
 	RetryStatusCodes      []int       `yaml:"retry_status_codes,omitempty"`
+	TLSInsecureSkipVerify bool        `yaml:"tls_insecure_skip_verify,omitempty"`
 
 	concurrencyLimitCh      chan struct{}
 	concurrencyLimitReached *metrics.Counter
 
+	httpTransport *http.Transport
+
 	requests         *metrics.Counter
 	requestsDuration *metrics.Summary
 }
@@ -442,6 +447,7 @@ func parseAuthConfig(data []byte) (*AuthConfig, error) {
 		_ = metrics.GetOrCreateGauge(`vmauth_unauthorized_user_concurrent_requests_current`, func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
+		ui.httpTransport = getTransport(ui.TLSInsecureSkipVerify)
 	}
 	return &ac, nil
 }
@@ -512,6 +518,8 @@ func parseAuthConfigUsers(ac *AuthConfig) (map[string]*UserInfo, error) {
 		_ = metrics.GetOrCreateGauge(fmt.Sprintf(`vmauth_user_concurrent_requests_current{username=%q}`, name), func() float64 {
 			return float64(len(ui.concurrencyLimitCh))
 		})
+		ui.httpTransport = getTransport(ui.TLSInsecureSkipVerify)
+
 		byAuthToken[at1] = ui
 		byAuthToken[at2] = ui
 	}

app/vmauth/auth_config_test.go

@@ -227,12 +227,14 @@ users:
   password: bar
   url_prefix: http://aaa:343/bbb
   max_concurrent_requests: 5
+  tls_insecure_skip_verify: true
 `, map[string]*UserInfo{
 		getAuthToken("", "foo", "bar"): {
 			Username:              "foo",
 			Password:              "bar",
 			URLPrefix:             mustParseURL("http://aaa:343/bbb"),
 			MaxConcurrentRequests: 5,
+			TLSInsecureSkipVerify: true,
 		},
 	})
 
@@ -448,6 +450,42 @@ users:
 
 }
 
+func TestParseAuthConfigPassesTLSVerificationConfig(t *testing.T) {
+	c := `
+users:
+- username: foo
+  password: bar
+  url_prefix: https://aaa/bbb
+  max_concurrent_requests: 5
+  tls_insecure_skip_verify: true
+
+unauthorized_user:
+  url_prefix: http://aaa:343/bbb
+  max_concurrent_requests: 5
+  tls_insecure_skip_verify: false
+`
+
+	ac, err := parseAuthConfig([]byte(c))
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	m, err := parseAuthConfigUsers(ac)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	ui := m[getAuthToken("", "foo", "bar")]
+	if ui.TLSInsecureSkipVerify != true || ui.httpTransport.TLSClientConfig.InsecureSkipVerify != true {
+		t.Fatalf("unexpected TLSInsecureSkipVerify value for user foo")
+	}
+
+	if ac.UnauthorizedUser.TLSInsecureSkipVerify != false ||
+		ac.UnauthorizedUser.httpTransport.TLSClientConfig.InsecureSkipVerify != false {
+		t.Fatalf("unexpected TLSInsecureSkipVerify value for unauthorized_user")
+	}
+
+}
+
 func getSrcPaths(paths []string) []*SrcPath {
 	var sps []*SrcPath
 	for _, path := range paths {

app/vmauth/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"flag"
 	"fmt"
@@ -15,6 +16,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/metrics"
+
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/bytesutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
@@ -24,7 +27,6 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/netutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/pushmetrics"
-	"github.com/VictoriaMetrics/metrics"
 )
 
 var (
@@ -191,7 +193,7 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 		} else { // Update path for regular routes.
 			targetURL = mergeURLs(targetURL, u)
 		}
-		ok := tryProcessingRequest(w, r, targetURL, hc, retryStatusCodes)
+		ok := tryProcessingRequest(w, r, targetURL, hc, retryStatusCodes, ui.httpTransport)
 		bu.put()
 		if ok {
 			return
@@ -205,12 +207,11 @@ func processRequest(w http.ResponseWriter, r *http.Request, ui *UserInfo) {
 	httpserver.Errorf(w, r, "%s", err)
 }
 
-func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int) bool {
+func tryProcessingRequest(w http.ResponseWriter, r *http.Request, targetURL *url.URL, hc HeadersConf, retryStatusCodes []int, transport *http.Transport) bool {
 	// This code has been copied from net/http/httputil/reverseproxy.go
 	req := sanitizeRequestHeaders(r)
 	req.URL = targetURL
 	updateHeadersByConfig(req.Header, hc.RequestHeaders)
-	transportOnce.Do(transportInit)
 	res, err := transport.RoundTrip(req)
 	rtb, rtbOK := req.Body.(*readTrackingBody)
 	if err != nil {
@@ -353,12 +354,7 @@ var (
 	missingRouteRequests     = metrics.NewCounter(`vmauth_http_request_errors_total{reason="missing_route"}`)
 )
 
-var (
+func getTransport(skipTLSVerification bool) *http.Transport {
-	transport     *http.Transport
-	transportOnce sync.Once
-)
-
-func transportInit() {
 	tr := http.DefaultTransport.(*http.Transport).Clone()
 	tr.ResponseHeaderTimeout = *responseTimeout
 	// Automatic compression must be disabled in order to fix https://github.com/VictoriaMetrics/VictoriaMetrics/issues/535
@@ -369,7 +365,11 @@ func transportInit() {
 	if tr.MaxIdleConns != 0 && tr.MaxIdleConns < tr.MaxIdleConnsPerHost {
 		tr.MaxIdleConns = tr.MaxIdleConnsPerHost
 	}
-	transport = tr
+	if tr.TLSClientConfig == nil {
+		tr.TLSClientConfig = &tls.Config{}
+	}
+	tr.TLSClientConfig.InsecureSkipVerify = skipTLSVerification
+	return tr
 }
 
 var (

docs/CHANGELOG.md

@@ -76,6 +76,7 @@ The sandbox cluster installation is running under the constant load generated by
 * FEATURE: [vmalert-tool](https://docs.victoriametrics.com/#vmalert-tool): add `unittest` command to run unittest for alerting and recording rules. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4789) for details.
 * FEATURE: dashboards/vmalert: add new panel `Missed evaluations` for indicating alerting groups that miss their evaluations.
 * FEATURE: all: track requests with wrong auth key and wrong basic auth at `vm_http_request_errors_total` [metric](https://docs.victoriametrics.com/#monitoring) with `reason="wrong_auth_key"` and `reason="wrong_basic_auth"`. See [this issue](https://github.com/victoriaMetrics/victoriaMetrics/issues/4590). Thanks to @venkatbvc for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5166).
+* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add `tls_insecure_skip_verify` parameter which allows to disable TLS verification for backend connection. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5240).
 
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): strip sensitive information such as auth headers or passwords from datasource, remote-read, remote-write or notifier URLs in log messages or UI. This behavior is by default and is controlled via `-datasource.showURL`, `-remoteRead.showURL`, `remoteWrite.showURL` or `-notifier.showURL` cmd-line flags. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5044).
 * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix vmalert web UI when running on 32-bit architectures machine.

docs/vmauth.md

@@ -192,6 +192,15 @@ users:
   password: "***"
   url_prefix: "http://localhost:8428?extra_label=team=dev"
 
+  # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
+  # are proxied to http://localhost:8428 with extra_label=team=dev query arg.
+  # For example, http://vmauth:8427/api/v1/query is routed to https://localhost/api/v1/query?extra_label=team=dev
+  # TLS verification is ignored for https://localhost.
+- username: "local-single-node-with-tls"
+  password: "***"
+  url_prefix: "https://localhost?extra_label=team=dev"
+  tls_insecure_skip_verify: false
+
   # All the requests to http://vmauth:8427 with the given Basic Auth (username:password)
   # are load-balanced among http://vmselect1:8481/select/123/prometheus and http://vmselect2:8481/select/123/prometheus
   # For example, http://vmauth:8427/api/v1/query is proxied to the following urls in a round-robin manner:
@@ -268,6 +277,7 @@ unauthorized_user:
   - http://vmselect-az1/?deny_partial_response=1
   - http://vmselect-az2/?deny_partial_response=1
   retry_status_codes: [503, 500]
+  tls_insecure_skip_verify: true
 
 ip_filters:
   allow_list: ["1.2.3.0/24", "127.0.0.1"]