Use httpAuth.* flags as a fallback for endpoints protected by *AuthKey flags (#3582)

* {lib/server, app/}: use `httpAuth.*` flag as fallback for `*AuthKey` if it is not set

* lib/ingestserver/opentsdbhttp: fix opentdb HTTP handler not respecting `httpAuth.*` flags

* Apply suggestions from code review

Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>

app/vmagent/main.go

@@ -347,12 +347,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey, "configAuthKey") {
-			err := &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
-				StatusCode: http.StatusUnauthorized,
-			}
-			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -361,12 +356,7 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey, "configAuthKey") {
-			err := &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
-				StatusCode: http.StatusUnauthorized,
-			}
-			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()

app/vmauth/main.go

@@ -60,9 +60,7 @@ func main() {
 func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 	switch r.URL.Path {
 	case "/-/reload":
-		authKey := r.FormValue("authKey")
+		if !httpserver.CheckAuthFlag(w, r, reloadAuthKey, "reloadAuthKey") {
-		if authKey != *reloadAuthKey {
-			httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -reloadAuthKey command line flag", authKey)
 			return true
 		}
 		configReloadRequests.Inc()

app/vminsert/main.go

@@ -246,12 +246,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/prometheus/config", "/config":
-		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey, "configAuthKey") {
-			err := &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
-				StatusCode: http.StatusUnauthorized,
-			}
-			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		promscrapeConfigRequests.Inc()
@@ -260,12 +255,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	case "/prometheus/api/v1/status/config", "/api/v1/status/config":
 		// See https://prometheus.io/docs/prometheus/latest/querying/api/#config
-		if *configAuthKey != "" && r.FormValue("authKey") != *configAuthKey {
+		if !httpserver.CheckAuthFlag(w, r, configAuthKey, "configAuthKey") {
-			err := &httpserver.ErrorWithStatusCode{
-				Err:        fmt.Errorf("The provided authKey doesn't match -configAuthKey"),
-				StatusCode: http.StatusUnauthorized,
-			}
-			httpserver.Errorf(w, r, "%s", err)
 			return true
 		}
 		promscrapeStatusConfigRequests.Inc()

app/vmselect/main.go

@@ -145,8 +145,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 
 	path := strings.Replace(r.URL.Path, "//", "/", -1)
 	if path == "/internal/resetRollupResultCache" {
-		if len(*resetCacheAuthKey) > 0 && r.FormValue("authKey") != *resetCacheAuthKey {
+		if !httpserver.CheckAuthFlag(w, r, resetCacheAuthKey, "resetCacheAuthKey") {
-			sendPrometheusError(w, r, fmt.Errorf("invalid authKey=%q for %q", r.FormValue("authKey"), path))
 			return true
 		}
 		promql.ResetRollupResultCache()
@@ -412,12 +411,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		}
 		return true
 	case "/tags/delSeries":
-		graphiteTagsDelSeriesRequests.Inc()
+		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey, "deleteAuthKey") {
-		authKey := r.FormValue("authKey")
-		if authKey != *deleteAuthKey {
-			httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -deleteAuthKey command line flag", authKey)
 			return true
 		}
+		graphiteTagsDelSeriesRequests.Inc()
 		if err := graphite.TagsDelSeriesHandler(startTime, w, r); err != nil {
 			graphiteTagsDelSeriesErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)
@@ -474,12 +471,10 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		fmt.Fprintf(w, "%s", `{"status":"success","data":[]}`)
 		return true
 	case "/api/v1/admin/tsdb/delete_series":
-		deleteRequests.Inc()
+		if !httpserver.CheckAuthFlag(w, r, deleteAuthKey, "deleteAuthKey") {
-		authKey := r.FormValue("authKey")
-		if authKey != *deleteAuthKey {
-			httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -deleteAuthKey command line flag", authKey)
 			return true
 		}
+		deleteRequests.Inc()
 		if err := prometheus.DeleteHandler(startTime, r); err != nil {
 			deleteErrors.Inc()
 			httpserver.Errorf(w, r, "%s", err)

app/vmstorage/main.go

@@ -255,9 +255,7 @@ func Stop() {
 func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	path := r.URL.Path
 	if path == "/internal/force_merge" {
-		authKey := r.FormValue("authKey")
+		if !httpserver.CheckAuthFlag(w, r, forceMergeAuthKey, "forceMergeAuthKey") {
-		if authKey != *forceMergeAuthKey {
-			httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -forceMergeAuthKey command line flag", authKey)
 			return true
 		}
 		// Run force merge in background
@@ -275,9 +273,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		return true
 	}
 	if path == "/internal/force_flush" {
-		authKey := r.FormValue("authKey")
+		if !httpserver.CheckAuthFlag(w, r, forceFlushAuthKey, "forceFlushAuthKey") {
-		if authKey != *forceFlushAuthKey {
-			httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -forceFlushAuthKey command line flag", authKey)
 			return true
 		}
 		logger.Infof("flushing storage to make pending data available for reading")
@@ -293,9 +289,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 	if !strings.HasPrefix(path, "/snapshot") {
 		return false
 	}
-	authKey := r.FormValue("authKey")
+	if !httpserver.CheckAuthFlag(w, r, snapshotAuthKey, "snapshotAuthKey") {
-	if authKey != *snapshotAuthKey {
-		httpserver.Errorf(w, r, "invalid authKey %q. It must match the value from -snapshotAuthKey command line flag", authKey)
 		return true
 	}
 	path = path[len("/snapshot"):]

docs/CHANGELOG.md

@@ -30,13 +30,13 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * FEATURE: [csvimport](https://docs.victoriametrics.com/#how-to-import-csv-data): support empty values for imported metrics. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3540).
 * FEATURE: [vmalert](httpпоs://docs.victoriametrics.com/vmalert.html): allow configuring the default number of stored rule's update states in memory via global `-rule.updateEntriesLimit` command-line flag or per-rule via rule's `update_entries_limit` configuration param. See [these docs](https://docs.victoriametrics.com/vmalert.html#rules) and [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3556).
 * FEATURE: improve the logic benhind `-maxConcurrentInserts` command-line flag. Previously this flag was limiting the number of concurrent connections from clients, which write data to VictoriaMetrics or [vmagent](https://docs.victoriametrics.com/vmagent.html). Some of these connections could be idle for some time. These connections do not need significant amounts of CPU and memory, so there is no sense in limiting their count. The updated logic behind `-maxConcurrentInserts` limits the number of **active** insert requests, not counting idle connections.
+* FEATURE: protect all the http endpoints with `-httpAuth.*` command-line flag. Previously endpoints protected by `-*AuthKey` command-line flags weren't protected by `-httpAuth.*`. This could complicate the proper security setup. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3060).
 * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): add `-maxConcurrentInserts` and `-insert.maxQueueDuration` command-line flags to `vmstorage`, so they could be tuned if needed in the same way as at `vminsert` nodes.
 * FEATURE: [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html): limit the number of concurrently executed requests at `vmstorage` proportionally to the number of available CPU cores, since every request can saturate a single CPU core at `vmstorage`. Previously a single `vmstorage` could accept and start processing arbitrary number of concurrent requests received from big number of `vmselect` nodes. This could result in increased RAM, CPU and disk IO usage or event to out of memory crash at `vmstorage` side under high load. The limit can be fine-tuned if needed via `-search.maxConcurrentRequests` command-line flag at `vmstorage` according to [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#resource-usage-limits). `vmstorage` now [exposes](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#monitoring) the following additional metrics at `http://vmstorage:8482/metrics` page:
   - `vm_vmselect_concurrent_requests_capacity` - the maximum number of requests allowed to execute concurrently
   - `vm_vmselect_concurrent_requests_current` - the current number of concurrently executed requests
   - `vm_vmselect_concurrent_requests_limit_reached_total` - the total number of requests, which were put in the wait queue when `-search.maxConcurrentRequests` concurrent requests are being executed
   - `vm_vmselect_concurrent_requests_limit_timeout_total` - the total number of canceled requests because they were sitting in the wait queue for more than `-search.maxQueueDuration`
-* FEATURE [vmctl](https://docs.victoriametrics.com/vmctl.html): add `-remote-read-insecure-skip-verify` command-line flag for remote read protocol. It can be used for skipping TLS certificate verification when connecting to the remote read address.
 
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly update the `step` value in url after the `step` input field has been manually changed. This allows preserving the proper `step` when copy-n-pasting the url to another instance of web browser. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3513).
 * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): properly update tooltip when quickly hovering multiple lines on the graph. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3530).
@@ -47,6 +47,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): fix possible resource leak after hot reload of the updated [consul_sd_configs](https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3468).
 * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): fix a panic in [gce_sd_configs](https://docs.victoriametrics.com/sd_configs.html#gce_sd_configs) when the discovered instance has zero labels. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3624). The issue has been introduced in [v1.85.0](https://docs.victoriametrics.com/CHANGELOG.html#v1850).
 * BUGFIX: properly return label names starting from uppercase such as `CamelCaseLabel` from [/api/v1/labels](https://docs.victoriametrics.com/url-examples.html#apiv1labels). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3566).
+* BUGFIX: fix `opentsdb` HTTP endpoint not respecting `-httpAuth.*` flags. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3060)
 * BUGFIX: consistently select the sample with the biggest value out of samples with identical timestamps during querying when the [deduplication](https://docs.victoriametrics.com/#deduplication) is enabled according to [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3333). Previously random samples could be selected during querying.
 
 

lib/httpserver/httpserver.go

@@ -292,8 +292,7 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 		return
 	case "/metrics":
 		metricsRequests.Inc()
-		if len(*metricsAuthKey) > 0 && r.FormValue("authKey") != *metricsAuthKey {
+		if !CheckAuthFlag(w, r, metricsAuthKey, "metricsAuthKey") {
-			http.Error(w, "The provided authKey doesn't match -metricsAuthKey", http.StatusUnauthorized)
 			return
 		}
 		startTime := time.Now()
@@ -302,8 +301,7 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 		metricsHandlerDuration.UpdateDuration(startTime)
 		return
 	case "/flags":
-		if len(*flagsAuthKey) > 0 && r.FormValue("authKey") != *flagsAuthKey {
+		if !CheckAuthFlag(w, r, flagsAuthKey, "flagsAuthKey") {
-			http.Error(w, "The provided authKey doesn't match -flagsAuthKey", http.StatusUnauthorized)
 			return
 		}
 		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
@@ -322,8 +320,7 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 	default:
 		if strings.HasPrefix(r.URL.Path, "/debug/pprof/") {
 			pprofRequests.Inc()
-			if len(*pprofAuthKey) > 0 && r.FormValue("authKey") != *pprofAuthKey {
+			if !CheckAuthFlag(w, r, pprofAuthKey, "pprofAuthKey") {
-				http.Error(w, "The provided authKey doesn't match -pprofAuthKey", http.StatusUnauthorized)
 				return
 			}
 			DisableResponseCompression(w)
@@ -331,7 +328,7 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 			return
 		}
 
-		if !checkBasicAuth(w, r) {
+		if !CheckBasicAuth(w, r) {
 			return
 		}
 		if rh(w, r) {
@@ -344,7 +341,25 @@ func handlerWrapper(s *server, w http.ResponseWriter, r *http.Request, rh Reques
 	}
 }
 
-func checkBasicAuth(w http.ResponseWriter, r *http.Request) bool {
+// CheckAuthFlag checks whether the given authKey is set and valid
+//
+// Falls back to checkBasicAuth if authKey is not set
+func CheckAuthFlag(w http.ResponseWriter, r *http.Request, flagValue *string, flagName string) bool {
+	if len(*flagValue) == 0 {
+		return CheckBasicAuth(w, r)
+	}
+
+	if r.FormValue("authKey") != *flagValue {
+		http.Error(w, fmt.Sprintf("The provided authKey doesn't match -%s", flagName), http.StatusUnauthorized)
+		return false
+	}
+
+	return true
+}
+
+// CheckBasicAuth validates credentials provided in request if httpAuth.* flags are set
+// returns true if credentials are valid or httpAuth.* flags are not set
+func CheckBasicAuth(w http.ResponseWriter, r *http.Request) bool {
 	if len(*httpAuthUsername) == 0 {
 		// HTTP Basic Auth is disabled.
 		return true

lib/ingestserver/opentsdbhttp/server.go

@@ -86,6 +86,9 @@ func (s *Server) MustStop() {
 
 func newRequestHandler(insertHandler func(r *http.Request) error) http.Handler {
 	rh := func(w http.ResponseWriter, r *http.Request) {
+		if !httpserver.CheckBasicAuth(w, r) {
+			return
+		}
 		writeRequests.Inc()
 		if err := insertHandler(r); err != nil {
 			writeErrors.Inc()