From 2c05066f193accab78f8e4de939c04d412df9743 Mon Sep 17 00:00:00 2001 From: Zakhar Bessarab Date: Wed, 22 Feb 2023 20:05:16 +0400 Subject: [PATCH] lib/promscrape: fix cancelling in-flight scrape requests during configuration reload (#3853) * lib/promscrape: fix cancelling in-flight scrape requests during configuration reload (see #3747) Signed-off-by: Zakhar Bessarab * lib/promscrape: fix order of params for `doRequestWithPossibleRetry` to follow codestyle Signed-off-by: Zakhar Bessarab * lib/promscrape: accept deadline explicitly and extend passed context for local use Signed-off-by: Zakhar Bessarab --------- Signed-off-by: Zakhar Bessarab --- lib/promscrape/client.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/promscrape/client.go b/lib/promscrape/client.go index 52fa157b5c..bf30e0369b 100644 --- a/lib/promscrape/client.go +++ b/lib/promscrape/client.go @@ -262,7 +262,8 @@ func (c *client) ReadData(dst []byte) ([]byte, error) { // This should reduce memory uage when scraping big targets. dst = resp.SwapBody(dst) } - err := doRequestWithPossibleRetry(c.hc, req, resp, deadline) + + err := doRequestWithPossibleRetry(c.ctx, c.hc, req, resp, deadline) statusCode := resp.StatusCode() redirectsCount := 0 for err == nil && isStatusRedirect(statusCode) { @@ -282,7 +283,7 @@ func (c *client) ReadData(dst []byte) ([]byte, error) { break } req.URI().UpdateBytes(location) - err = doRequestWithPossibleRetry(c.hc, req, resp, deadline) + err = doRequestWithPossibleRetry(c.ctx, c.hc, req, resp, deadline) statusCode = resp.StatusCode() redirectsCount++ } @@ -349,13 +350,14 @@ var ( scrapeRetries = metrics.NewCounter(`vm_promscrape_scrape_retries_total`) ) -func doRequestWithPossibleRetry(hc *fasthttp.HostClient, req *fasthttp.Request, resp *fasthttp.Response, deadline time.Time) error { +func doRequestWithPossibleRetry(ctx context.Context, hc *fasthttp.HostClient, req *fasthttp.Request, resp *fasthttp.Response, deadline time.Time) error { sleepTime := time.Second scrapeRequests.Inc() + reqCtx, cancel := context.WithDeadline(ctx, deadline) + defer cancel() for { - // Use DoDeadline instead of Do even if hc.ReadTimeout is already set in order to guarantee the given deadline - // across multiple retries. - err := hc.DoDeadline(req, resp, deadline) + // Use DoCtx instead of Do in order to support context cancellation + err := hc.DoCtx(reqCtx, req, resp) if err == nil { statusCode := resp.StatusCode() if statusCode != fasthttp.StatusTooManyRequests { @@ -364,6 +366,7 @@ func doRequestWithPossibleRetry(hc *fasthttp.HostClient, req *fasthttp.Request, } else if err != fasthttp.ErrConnectionClosed && !strings.Contains(err.Error(), "broken pipe") { return err } + // Retry request after exponentially increased sleep. maxSleepTime := time.Until(deadline) if sleepTime > maxSleepTime {