lib/promscrape/discovery/consul: reduce load on Consul API server by increasing timeout for blocking requests from 50 seconds to 9 minutes

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/574

lib/promscrape/discovery/consul/api.go

@@ -111,13 +111,20 @@ func getDatacenter(client *discoveryutils.Client, dc string) (string, error) {
 	return a.Config.Datacenter, nil
 }
 
-// maxWaitTime is duration for consul blocking request, maximum wait time is 10 min.
+// maxWaitTime is duration for consul blocking request.
-// But fasthttp client has readTimeout for 1 min, so we use 50s timeout.
+var maxWaitTime = func() time.Duration {
-// also consul adds random delay up to wait/16, so there is no need in jitter.
+	d := discoveryutils.BlockingClientReadTimeout
-// https://www.consul.io/api-docs/features/blocking
+	// Consul adds random delay up to wait/16, so reduce the timeout in order to keep it below BlockingClientReadTimeout.
-const maxWaitTime = 50 * time.Second
+	// See https://www.consul.io/api-docs/features/blocking
+	d -= d / 8
+	// The timeout cannot exceed 10 minuntes. See https://www.consul.io/api-docs/features/blocking
+	if d > 10*time.Minute {
+		d = 10 * time.Minute
+	}
+	return d
+}()
 
-var maxWaitTimeStr = maxWaitTime.String()
+var maxWaitTimeStr = fmt.Sprintf("%ds", int(maxWaitTime.Seconds()))
 
 // getBlockingAPIResponse perfoms blocking request to Consul via client and returns response.
 //

lib/promscrape/discoveryutils/client.go

@@ -91,7 +91,7 @@ func NewClient(apiServer string, ac *promauth.Config) (*Client, error) {
 		DialDualStack:       netutil.TCP6Enabled(),
 		IsTLS:               isTLS,
 		TLSConfig:           tlsCfg,
-		ReadTimeout:         time.Minute * 3,
+		ReadTimeout:         BlockingClientReadTimeout,
 		WriteTimeout:        10 * time.Second,
 		MaxResponseBodySize: 300 * 1024 * 1024,
 		MaxConns:            64 * 1024,
@@ -106,6 +106,9 @@ func NewClient(apiServer string, ac *promauth.Config) (*Client, error) {
 	}, nil
 }
 
+// BlockingClientReadTimeout is the maximum duration for waiting the response from GetBlockingAPI*
+const BlockingClientReadTimeout = 10 * time.Minute
+
 var (
 	concurrencyLimitCh     chan struct{}
 	concurrencyLimitChOnce sync.Once