From 5f9d88a3cbff581bd331957e274ca9c565f4f1ad Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Fri, 11 Dec 2020 17:22:37 +0200
Subject: [PATCH] lib/promscrape/discovery/consul: reduce load on Consul API
 server by increasing timeout for blocking requests from 50 seconds to 9
 minutes

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/574
---
 lib/promscrape/discovery/consul/api.go  | 19 +++++++++++++------
 lib/promscrape/discoveryutils/client.go |  5 ++++-
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/lib/promscrape/discovery/consul/api.go b/lib/promscrape/discovery/consul/api.go
index 266d68437..6a888eabb 100644
--- a/lib/promscrape/discovery/consul/api.go
+++ b/lib/promscrape/discovery/consul/api.go
@@ -111,13 +111,20 @@ func getDatacenter(client *discoveryutils.Client, dc string) (string, error) {
 	return a.Config.Datacenter, nil
 }
 
-// maxWaitTime is duration for consul blocking request, maximum wait time is 10 min.
-// But fasthttp client has readTimeout for 1 min, so we use 50s timeout.
-// also consul adds random delay up to wait/16, so there is no need in jitter.
-// https://www.consul.io/api-docs/features/blocking
-const maxWaitTime = 50 * time.Second
+// maxWaitTime is duration for consul blocking request.
+var maxWaitTime = func() time.Duration {
+	d := discoveryutils.BlockingClientReadTimeout
+	// Consul adds random delay up to wait/16, so reduce the timeout in order to keep it below BlockingClientReadTimeout.
+	// See https://www.consul.io/api-docs/features/blocking
+	d -= d / 8
+	// The timeout cannot exceed 10 minuntes. See https://www.consul.io/api-docs/features/blocking
+	if d > 10*time.Minute {
+		d = 10 * time.Minute
+	}
+	return d
+}()
 
-var maxWaitTimeStr = maxWaitTime.String()
+var maxWaitTimeStr = fmt.Sprintf("%ds", int(maxWaitTime.Seconds()))
 
 // getBlockingAPIResponse perfoms blocking request to Consul via client and returns response.
 //
diff --git a/lib/promscrape/discoveryutils/client.go b/lib/promscrape/discoveryutils/client.go
index bbf12d595..9469fe06d 100644
--- a/lib/promscrape/discoveryutils/client.go
+++ b/lib/promscrape/discoveryutils/client.go
@@ -91,7 +91,7 @@ func NewClient(apiServer string, ac *promauth.Config) (*Client, error) {
 		DialDualStack:       netutil.TCP6Enabled(),
 		IsTLS:               isTLS,
 		TLSConfig:           tlsCfg,
-		ReadTimeout:         time.Minute * 3,
+		ReadTimeout:         BlockingClientReadTimeout,
 		WriteTimeout:        10 * time.Second,
 		MaxResponseBodySize: 300 * 1024 * 1024,
 		MaxConns:            64 * 1024,
@@ -106,6 +106,9 @@ func NewClient(apiServer string, ac *promauth.Config) (*Client, error) {
 	}, nil
 }
 
+// BlockingClientReadTimeout is the maximum duration for waiting the response from GetBlockingAPI*
+const BlockingClientReadTimeout = 10 * time.Minute
+
 var (
 	concurrencyLimitCh     chan struct{}
 	concurrencyLimitChOnce sync.Once