diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7a4fb6f238..958d0e344a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -11,9 +11,9 @@ * FATURE: vmagent: accept `scrape_offset` option at `scrape_config`. This option may be useful when scrapes must start at the specified offset of every scrape interval. See [these docs](https://victoriametrics.github.io/vmagent.html#troubleshooting) for details. * FEATURE: vmauth: allow using regexp paths in `url_map`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1112) for details. - * BUGFIX: vmagent: prevent from high CPU usage bug during failing scrapes with small `scrape_timeout` (less than a few seconds). -* BUGFIX: vmagent: reduce memory usage when Kubernetes service discovery is used in big number of distinct jobs by sharing the cache. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1113 +* BUGFIX: vmagent: reduce memory usage when Kubernetes service discovery is used in big number of distinct scrape config jobs by sharing Kubernetes object cache. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1113 +* BUGFIX: vmagent: apply `sample_limit` only after `metric_relabel_configs` are applied as Prometheus does. Previously the `sample_limit` was applied before metrics relabeling. * BUGFUX: avoid `duplicate time series` error if `prometheus_buckets()` covers a time range with distinct set of buckets. * BUGFIX: prevent exponent overflow when processing extremely small values close to zero such as `2.964393875E-314`. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1114 diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go index e5e28ff42d..003b0348a9 100644 --- a/lib/promscrape/scrapework.go +++ b/lib/promscrape/scrapework.go @@ -290,27 +290,15 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error srcRows := wc.rows.Rows samplesScraped := len(srcRows) scrapedSamples.Update(float64(samplesScraped)) - if sw.Config.SampleLimit > 0 && samplesScraped > sw.Config.SampleLimit { - srcRows = srcRows[:0] + for i := range srcRows { + sw.addRowToTimeseries(wc, &srcRows[i], scrapeTimestamp, true) + } + samplesPostRelabeling := len(wc.writeRequest.Timeseries) + if sw.Config.SampleLimit > 0 && samplesPostRelabeling > sw.Config.SampleLimit { + wc.resetNoRows() up = 0 scrapesSkippedBySampleLimit.Inc() } - samplesPostRelabeling := 0 - for i := range srcRows { - sw.addRowToTimeseries(wc, &srcRows[i], scrapeTimestamp, true) - if len(wc.labels) > 40000 { - // Limit the maximum size of wc.writeRequest. - // This should reduce memory usage when scraping targets with millions of metrics and/or labels. - // For example, when scraping /federate handler from Prometheus - see https://prometheus.io/docs/prometheus/latest/federation/ - samplesPostRelabeling += len(wc.writeRequest.Timeseries) - sw.updateSeriesAdded(wc) - startTime := time.Now() - sw.PushData(&wc.writeRequest) - pushDataDuration.UpdateDuration(startTime) - wc.resetNoRows() - } - } - samplesPostRelabeling += len(wc.writeRequest.Timeseries) sw.updateSeriesAdded(wc) seriesAdded := sw.finalizeSeriesAdded(samplesPostRelabeling) sw.addAutoTimeseries(wc, "up", float64(up), scrapeTimestamp) diff --git a/lib/promscrape/scrapework_test.go b/lib/promscrape/scrapework_test.go index 74b6f67e8e..79e6cb6cea 100644 --- a/lib/promscrape/scrapework_test.go +++ b/lib/promscrape/scrapework_test.go @@ -332,7 +332,7 @@ func TestScrapeWorkScrapeInternalSuccess(t *testing.T) { up 0 123 scrape_samples_scraped 2 123 scrape_duration_seconds 0 123 - scrape_samples_post_metric_relabeling 0 123 + scrape_samples_post_metric_relabeling 2 123 scrape_series_added 0 123 `) }