From d8de26bbfd549937c839010f14d0060f93da1a8f Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 23 Sep 2021 14:47:20 +0300 Subject: [PATCH] lib/promscrape: add `vm_promscrape_max_scrape_size_exceeded_errors_total` metric for counting of the failed scrapes due to the exceeded response size Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1639 --- docs/CHANGELOG.md | 2 ++ lib/promscrape/client.go | 13 ++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fcfcd832ed..6373cbe806 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -6,6 +6,8 @@ sort: 15 ## tip +* FEATURE: vmagent: add `vm_promscrape_max_scrape_size_exceeded_errors_total` metric for counting of the failed scrapes due to the exceeded response size (the response size limit can be configured via `-promscrape.maxScrapeSize` command-line flag). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1639). + ## [v1.66.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.66.1) diff --git a/lib/promscrape/client.go b/lib/promscrape/client.go index 3d62f1d094..a2ad7e4d6f 100644 --- a/lib/promscrape/client.go +++ b/lib/promscrape/client.go @@ -259,6 +259,7 @@ func (c *client) ReadData(dst []byte) ([]byte, error) { return dst, fmt.Errorf("error when scraping %q with timeout %s: %w", c.scrapeURL, c.hc.ReadTimeout, err) } if err == fasthttp.ErrBodyTooLarge { + maxScrapeSizeExceeded.Inc() return dst, fmt.Errorf("the response from %q exceeds -promscrape.maxScrapeSize=%d; "+ "either reduce the response size for the target or increase -promscrape.maxScrapeSize", c.scrapeURL, maxScrapeSize.N) } @@ -296,11 +297,12 @@ func (c *client) ReadData(dst []byte) ([]byte, error) { var gunzipBufPool bytesutil.ByteBufferPool var ( - scrapesTimedout = metrics.NewCounter(`vm_promscrape_scrapes_timed_out_total`) - scrapesOK = metrics.NewCounter(`vm_promscrape_scrapes_total{status_code="200"}`) - scrapesGunzipped = metrics.NewCounter(`vm_promscrape_scrapes_gunziped_total`) - scrapesGunzipFailed = metrics.NewCounter(`vm_promscrape_scrapes_gunzip_failed_total`) - scrapeRetries = metrics.NewCounter(`vm_promscrape_scrape_retries_total`) + maxScrapeSizeExceeded = metrics.NewCounter(`vm_promscrape_max_scrape_size_exceeded_errors_total`) + scrapesTimedout = metrics.NewCounter(`vm_promscrape_scrapes_timed_out_total`) + scrapesOK = metrics.NewCounter(`vm_promscrape_scrapes_total{status_code="200"}`) + scrapesGunzipped = metrics.NewCounter(`vm_promscrape_scrapes_gunziped_total`) + scrapesGunzipFailed = metrics.NewCounter(`vm_promscrape_scrapes_gunzip_failed_total`) + scrapeRetries = metrics.NewCounter(`vm_promscrape_scrape_retries_total`) ) func doRequestWithPossibleRetry(hc *fasthttp.HostClient, req *fasthttp.Request, resp *fasthttp.Response, deadline time.Time) error { @@ -341,6 +343,7 @@ func (sr *streamReader) Read(p []byte) (int, error) { n, err := sr.r.Read(p) sr.bytesRead += int64(n) if err == nil && sr.bytesRead > sr.maxBodySize { + maxScrapeSizeExceeded.Inc() err = fmt.Errorf("the response from %q exceeds -promscrape.maxScrapeSize=%d; "+ "either reduce the response size for the target or increase -promscrape.maxScrapeSize", sr.scrapeURL, sr.maxBodySize) }