From c7884f8686dc25a08d1bc6145f3c84ea6d30b10f Mon Sep 17 00:00:00 2001 From: Dmytro Kozlov Date: Fri, 2 Jun 2023 15:34:04 +0300 Subject: [PATCH] app/{graphite,netstorage,prometheus}: fix graphite search tags api limits, remove redudant limit from SeriesHandler handler (#4352) * app/{graphite,netstorage,prometheus}: fix graphite search tags api limits, remove unused limit from SeriesHandler handler, * app/{graphite,netstorage,prometheus}: use search.maxTagValues for Graphite * app/{graphite,netstorage,prometheus}: update CHANGELOG.md * app/{graphite,netstorage,prometheus}: use own flags for Graphite API * app/{graphite,netstorage,prometheus}: cleanup * app/{graphite,netstorage,prometheus}: cleanup * app/{graphite,netstorage,prometheus}: update docs --------- Co-authored-by: Nikolay --- README.md | 4 ++++ app/vmselect/graphite/tags_api.go | 26 +++++++++++++++++++------- app/vmselect/prometheus/prometheus.go | 6 +----- docs/CHANGELOG.md | 4 ++++ docs/Cluster-VictoriaMetrics.md | 4 ++++ docs/README.md | 4 ++++ docs/Single-server-VictoriaMetrics.md | 4 ++++ 7 files changed, 40 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index ebe1f79b0..91d64636b 100644 --- a/README.md +++ b/README.md @@ -2498,6 +2498,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000) -search.maxGraphiteSeries int The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000) + -search.maxGraphiteTagKeys int + The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API (default 100000) + -search.maxGraphiteTagValues int + The maximum number of tag values returned Graphite /tags/ API (default 100000) -search.maxLookback duration Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons -search.maxMemoryPerQuery size diff --git a/app/vmselect/graphite/tags_api.go b/app/vmselect/graphite/tags_api.go index 6236aff86..adbe3a886 100644 --- a/app/vmselect/graphite/tags_api.go +++ b/app/vmselect/graphite/tags_api.go @@ -1,6 +1,7 @@ package graphite import ( + "flag" "fmt" "net/http" "regexp" @@ -18,6 +19,11 @@ import ( "github.com/VictoriaMetrics/metrics" ) +var ( + maxGraphiteTagKeysPerSearch = flag.Int("search.maxGraphiteTagKeys", 100e3, "The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API") + maxGraphiteTagValuesPerSearch = flag.Int("search.maxGraphiteTagValues", 100e3, "The maximum number of tag values returned Graphite /tags/ API") +) + // TagsDelSeriesHandler implements /tags/delSeries handler. // // See https://graphite.readthedocs.io/en/stable/tags.html#removing-series-from-the-tagdb @@ -182,13 +188,13 @@ func TagsAutoCompleteValuesHandler(startTime time.Time, w http.ResponseWriter, r // Escape special chars in tagPrefix as Graphite does. // See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L228 filter := regexp.QuoteMeta(valuePrefix) - tagValues, err = netstorage.GraphiteTagValues(nil, tag, filter, limit, deadline) + tagValues, err = netstorage.GraphiteTagValues(nil, tag, filter, *maxGraphiteTagKeysPerSearch, deadline) if err != nil { return err } } else { // Slow path: use netstorage.SearchMetricNames for applying `expr` filters. - sq, err := getSearchQueryForExprs(startTime, etfs, exprs, limit*10) + sq, err := getSearchQueryForExprs(startTime, etfs, exprs, *maxGraphiteTagKeysPerSearch) if err != nil { return err } @@ -269,13 +275,13 @@ func TagsAutoCompleteTagsHandler(startTime time.Time, w http.ResponseWriter, r * // Escape special chars in tagPrefix as Graphite does. // See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L181 filter := regexp.QuoteMeta(tagPrefix) - labels, err = netstorage.GraphiteTags(nil, filter, limit, deadline) + labels, err = netstorage.GraphiteTags(nil, filter, *maxGraphiteTagKeysPerSearch, deadline) if err != nil { return err } } else { // Slow path: use netstorage.SearchMetricNames for applying `expr` filters. - sq, err := getSearchQueryForExprs(startTime, etfs, exprs, limit*10) + sq, err := getSearchQueryForExprs(startTime, etfs, exprs, *maxGraphiteTagKeysPerSearch) if err != nil { return err } @@ -343,7 +349,7 @@ func TagsFindSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.R if err != nil { return fmt.Errorf("cannot setup tag filters: %w", err) } - sq, err := getSearchQueryForExprs(startTime, etfs, exprs, limit*10) + sq, err := getSearchQueryForExprs(startTime, etfs, exprs, *maxGraphiteTagKeysPerSearch) if err != nil { return err } @@ -411,11 +417,14 @@ func TagValuesHandler(startTime time.Time, tagName string, w http.ResponseWriter return err } filter := r.FormValue("filter") - tagValues, err := netstorage.GraphiteTagValues(nil, tagName, filter, limit, deadline) + tagValues, err := netstorage.GraphiteTagValues(nil, tagName, filter, *maxGraphiteTagValuesPerSearch, deadline) if err != nil { return err } + if limit > 0 && limit < len(tagValues) { + tagValues = tagValues[:limit] + } w.Header().Set("Content-Type", "application/json") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) @@ -439,11 +448,14 @@ func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) er return err } filter := r.FormValue("filter") - labels, err := netstorage.GraphiteTags(nil, filter, limit, deadline) + labels, err := netstorage.GraphiteTags(nil, filter, *maxGraphiteTagKeysPerSearch, deadline) if err != nil { return err } + if limit > 0 && limit < len(labels) { + labels = labels[:limit] + } w.Header().Set("Content-Type", "application/json") bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) diff --git a/app/vmselect/prometheus/prometheus.go b/app/vmselect/prometheus/prometheus.go index 9dd67ede2..dc9749805 100644 --- a/app/vmselect/prometheus/prometheus.go +++ b/app/vmselect/prometheus/prometheus.go @@ -633,11 +633,7 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW return err } - minLimit := *maxSeriesLimit - if limit > 0 && limit < *maxSeriesLimit { - minLimit = limit - } - sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, minLimit) + sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxSeriesLimit) metricNames, err := netstorage.SearchMetricNames(qt, sq, cp.deadline) if err != nil { return fmt.Errorf("cannot fetch time series for %q: %w", sq, err) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 797e1cc52..7ff9628a1 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -25,6 +25,10 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip * BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix nil map assignment panic in runtime introduced in this [change](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4341). +* BUGFIX: add the following command-line flags, which can be used for limiting Graphite API calls: + `--search.maxGraphiteTagKeys` for limiting the number of tag keys returned from Graphite `/tags`, `/tags/autoComplete/*`, `/tags/findSeries` API. + `--search.maxGraphiteTagValues` for limiting the number of tag values returned Graphite `/tags/` API. + Remove redundant limit from [Prometheus api/v1/series](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#prometheus-querying-api-usage). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4339). ## [v1.91.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.91.1) diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index 05f4cb0a3..dc7d61965 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -1156,6 +1156,10 @@ Below is the output for `/path/to/vmselect -help`: The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000) -search.maxGraphiteSeries int The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000) + -search.maxGraphiteTagKeys int + The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API (default 100000) + -search.maxGraphiteTagValues int + The maximum number of tag values returned Graphite /tags/ API (default 100000) -search.maxLookback duration Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaining due to historical reasons -search.maxMemoryPerQuery size diff --git a/docs/README.md b/docs/README.md index f246c8520..601bcbbe6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2501,6 +2501,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000) -search.maxGraphiteSeries int The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000) + -search.maxGraphiteTagKeys int + The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API (default 100000) + -search.maxGraphiteTagValues int + The maximum number of tag values returned Graphite /tags/ API (default 100000) -search.maxLookback duration Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons -search.maxMemoryPerQuery size diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 22572602e..bd0615035 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -2509,6 +2509,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000) -search.maxGraphiteSeries int The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000) + -search.maxGraphiteTagKeys int + The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API (default 100000) + -search.maxGraphiteTagValues int + The maximum number of tag values returned Graphite /tags/ API (default 100000) -search.maxLookback duration Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons -search.maxMemoryPerQuery size