Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-15 00:13:30 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

lib/promscrape/discovery/consul: reduce load on Consul API server by increasing timeout for blocking requests from 50 seconds to 9 minutes

Browse Source 
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/574
This commit is contained in:
Aliaksandr Valialkin 2020-12-11 17:22:37 +02:00
parent d6f9bf2d19
commit c80d38f00c
2 changed files with 17 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
lib/promscrape/discovery/consul/api.go
View File

@ -111,13 +111,20 @@ func getDatacenter(client *discoveryutils.Client, dc string) (string, error) {
return a.Config.Datacenter, nil
}
// maxWaitTime is duration for consul blocking request, maximum wait time is 10 min.
// But fasthttp client has readTimeout for 1 min, so we use 50s timeout.
// also consul adds random delay up to wait/16, so there is no need in jitter.
// https://www.consul.io/api-docs/features/blocking
const maxWaitTime = 50 * time.Second
// maxWaitTime is duration for consul blocking request.
var maxWaitTime = func() time.Duration {
d := discoveryutils.BlockingClientReadTimeout
// Consul adds random delay up to wait/16, so reduce the timeout in order to keep it below BlockingClientReadTimeout.
// See https://www.consul.io/api-docs/features/blocking
d -= d / 8
// The timeout cannot exceed 10 minuntes. See https://www.consul.io/api-docs/features/blocking
if d > 10*time.Minute {
d = 10 * time.Minute
}
return d
}()
var maxWaitTimeStr = maxWaitTime.String()
var maxWaitTimeStr = fmt.Sprintf("%ds", int(maxWaitTime.Seconds()))
// getBlockingAPIResponse perfoms blocking request to Consul via client and returns response.
//

5
lib/promscrape/discoveryutils/client.go
View File

@ -91,7 +91,7 @@ func NewClient(apiServer string, ac *promauth.Config) (*Client, error) {
DialDualStack: netutil.TCP6Enabled(),
IsTLS: isTLS,
TLSConfig: tlsCfg,
ReadTimeout: time.Minute * 3,
ReadTimeout: BlockingClientReadTimeout,
WriteTimeout: 10 * time.Second,
MaxResponseBodySize: 300 * 1024 * 1024,
MaxConns: 64 * 1024,
@ -106,6 +106,9 @@ func NewClient(apiServer string, ac *promauth.Config) (*Client, error) {
}, nil
}
// BlockingClientReadTimeout is the maximum duration for waiting the response from GetBlockingAPI*
const BlockingClientReadTimeout = 10 * time.Minute
var (
concurrencyLimitCh chan struct{}
concurrencyLimitChOnce sync.Once