diff --git a/lib/promscrape/discovery/consul/api.go b/lib/promscrape/discovery/consul/api.go index 266d68437d..6a888eabba 100644 --- a/lib/promscrape/discovery/consul/api.go +++ b/lib/promscrape/discovery/consul/api.go @@ -111,13 +111,20 @@ func getDatacenter(client *discoveryutils.Client, dc string) (string, error) { return a.Config.Datacenter, nil } -// maxWaitTime is duration for consul blocking request, maximum wait time is 10 min. -// But fasthttp client has readTimeout for 1 min, so we use 50s timeout. -// also consul adds random delay up to wait/16, so there is no need in jitter. -// https://www.consul.io/api-docs/features/blocking -const maxWaitTime = 50 * time.Second +// maxWaitTime is duration for consul blocking request. +var maxWaitTime = func() time.Duration { + d := discoveryutils.BlockingClientReadTimeout + // Consul adds random delay up to wait/16, so reduce the timeout in order to keep it below BlockingClientReadTimeout. + // See https://www.consul.io/api-docs/features/blocking + d -= d / 8 + // The timeout cannot exceed 10 minuntes. See https://www.consul.io/api-docs/features/blocking + if d > 10*time.Minute { + d = 10 * time.Minute + } + return d +}() -var maxWaitTimeStr = maxWaitTime.String() +var maxWaitTimeStr = fmt.Sprintf("%ds", int(maxWaitTime.Seconds())) // getBlockingAPIResponse perfoms blocking request to Consul via client and returns response. // diff --git a/lib/promscrape/discoveryutils/client.go b/lib/promscrape/discoveryutils/client.go index bbf12d5958..9469fe06d0 100644 --- a/lib/promscrape/discoveryutils/client.go +++ b/lib/promscrape/discoveryutils/client.go @@ -91,7 +91,7 @@ func NewClient(apiServer string, ac *promauth.Config) (*Client, error) { DialDualStack: netutil.TCP6Enabled(), IsTLS: isTLS, TLSConfig: tlsCfg, - ReadTimeout: time.Minute * 3, + ReadTimeout: BlockingClientReadTimeout, WriteTimeout: 10 * time.Second, MaxResponseBodySize: 300 * 1024 * 1024, MaxConns: 64 * 1024, @@ -106,6 +106,9 @@ func NewClient(apiServer string, ac *promauth.Config) (*Client, error) { }, nil } +// BlockingClientReadTimeout is the maximum duration for waiting the response from GetBlockingAPI* +const BlockingClientReadTimeout = 10 * time.Minute + var ( concurrencyLimitCh chan struct{} concurrencyLimitChOnce sync.Once