From 908e35affd31975bf808effa7072f59197d8d507 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Fri, 23 Apr 2021 21:53:32 +0300 Subject: [PATCH] lib/promscrape: apply `scrape_timeout` on receiving the first response byte for `stream_parse: true` scrape targets Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1017#issuecomment-767235047 --- docs/CHANGELOG.md | 1 + lib/promscrape/client.go | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 612a58fb2..ac359e98f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -11,6 +11,7 @@ sort: 14 Thanks to @johnseekins! * BUGFIX: vmagent: properly update `role: endpoints` and `role: endpointslices` scrape targets if the underlying service changes in `kubernetes_sd_config`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1240). +* BUGFIX: vmagent: apply `scrape_timeout` on receiving the first response byte from `stream_parse: true` scrape targets. Previously it was applied to receiving and *processing* the full response stream. This could result in false timeout errors when scrape target exposes millions of metrics as described [here](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1017#issuecomment-767235047). * BUGFIX: vmstorage: remove empty directories on startup. Such directories can be left after unclean shutdown on NFS storage. Previously such directories could lead to crashloop until manually removed. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1142). diff --git a/lib/promscrape/client.go b/lib/promscrape/client.go index e023a95d3..7cc9ce158 100644 --- a/lib/promscrape/client.go +++ b/lib/promscrape/client.go @@ -118,8 +118,17 @@ func newClient(sw *ScrapeWork) *client { DisableCompression: *disableCompression || sw.DisableCompression, DisableKeepAlives: *disableKeepAlive || sw.DisableKeepAlive, DialContext: statStdDial, + + // Set timeout for receiving the first response byte, + // since the duration for reading the full response can be much bigger because of stream parsing. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1017#issuecomment-767235047 + ResponseHeaderTimeout: sw.ScrapeTimeout, }, - Timeout: sw.ScrapeTimeout, + + // Set 10x bigger timeout than the sw.ScrapeTimeout, since the duration for reading the full response + // can be much bigger because of stream parsing. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1017#issuecomment-767235047 + Timeout: 10 * sw.ScrapeTimeout, } if sw.DenyRedirects { sc.CheckRedirect = func(req *http.Request, via []*http.Request) error {