Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-15 08:23:34 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

lib/promscrape: always initialize http client for stream parsing mode

Browse Source 
Stream parsing mode can be automatically enabled when scraping targets with big response bodies
exceeding the -promscrape.minResponseSizeForStreamParse , so it must be always initialized.
This commit is contained in:
Aliaksandr Valialkin 2021-10-16 13:18:20 +03:00
parent 0f4fda1bda
commit 99011c6b63
No known key found for this signature in database
GPG Key ID: A72BEC6CD3D0DED1
1 changed files with 27 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
lib/promscrape/client.go
View File

@ -106,37 +106,35 @@ func newClient(sw *ScrapeWork) *client {
MaxIdempotentRequestAttempts: 1, MaxIdempotentRequestAttempts: 1,
} }
var sc *http.Client var sc *http.Client
if *streamParse || sw.StreamParse { var proxyURLFunc func(*http.Request) (*url.URL, error)
var proxyURLFunc func(*http.Request) (*url.URL, error) if proxyURL := sw.ProxyURL.URL(); proxyURL != nil {
if proxyURL := sw.ProxyURL.URL(); proxyURL != nil { proxyURLFunc = http.ProxyURL(proxyURL)
proxyURLFunc = http.ProxyURL(proxyURL) }
} sc = &http.Client{
sc = &http.Client{ Transport: &http.Transport{
Transport: &http.Transport{ TLSClientConfig: tlsCfg,
TLSClientConfig: tlsCfg, Proxy: proxyURLFunc,
Proxy: proxyURLFunc, TLSHandshakeTimeout: 10 * time.Second,
TLSHandshakeTimeout: 10 * time.Second, IdleConnTimeout: 2 * sw.ScrapeInterval,
IdleConnTimeout: 2 * sw.ScrapeInterval, DisableCompression: *disableCompression || sw.DisableCompression,
DisableCompression: *disableCompression || sw.DisableCompression, DisableKeepAlives: *disableKeepAlive || sw.DisableKeepAlive,
DisableKeepAlives: *disableKeepAlive || sw.DisableKeepAlive, DialContext: statStdDial,
DialContext: statStdDial, MaxIdleConnsPerHost: 100,
MaxIdleConnsPerHost: 100,
// Set timeout for receiving the first response byte, // Set timeout for receiving the first response byte,
// since the duration for reading the full response can be much bigger because of stream parsing. // since the duration for reading the full response can be much bigger because of stream parsing.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1017#issuecomment-767235047
ResponseHeaderTimeout: sw.ScrapeTimeout,
},
// Set 30x bigger timeout than the sw.ScrapeTimeout, since the duration for reading the full response
// can be much bigger because of stream parsing.
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1017#issuecomment-767235047 // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1017#issuecomment-767235047
Timeout: 30 * sw.ScrapeTimeout, ResponseHeaderTimeout: sw.ScrapeTimeout,
} },
if sw.DenyRedirects {
sc.CheckRedirect = func(req *http.Request, via []*http.Request) error { // Set 30x bigger timeout than the sw.ScrapeTimeout, since the duration for reading the full response
return http.ErrUseLastResponse // can be much bigger because of stream parsing.
} // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1017#issuecomment-767235047
Timeout: 30 * sw.ScrapeTimeout,
}
if sw.DenyRedirects {
sc.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
} }
} }
return &client{ return &client{