Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-23 12:31:07 +01:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

app/{graphite,netstorage,prometheus}: fix graphite search tags api limits, remove redudant limit from SeriesHandler handler (#4352)

Browse Source 
* app/{graphite,netstorage,prometheus}: fix graphite search tags api limits, remove unused limit from SeriesHandler handler,

* app/{graphite,netstorage,prometheus}: use search.maxTagValues for Graphite

* app/{graphite,netstorage,prometheus}: update CHANGELOG.md

* app/{graphite,netstorage,prometheus}: use own flags for Graphite API

* app/{graphite,netstorage,prometheus}: cleanup

* app/{graphite,netstorage,prometheus}: cleanup

* app/{graphite,netstorage,prometheus}: update docs

---------

Co-authored-by: Nikolay <nik@victoriametrics.com>
This commit is contained in:
Dmytro Kozlov 2023-06-02 15:34:04 +03:00 committed by GitHub
parent de94812088
commit c7884f8686
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 40 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -2498,6 +2498,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000)
-search.maxGraphiteSeries int
The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000)
-search.maxGraphiteTagKeys int
The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API (default 100000)
-search.maxGraphiteTagValues int
The maximum number of tag values returned Graphite /tags/<tag_name> API (default 100000)
-search.maxLookback duration
Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons
-search.maxMemoryPerQuery size

26
app/vmselect/graphite/tags_api.go
View File

@ -1,6 +1,7 @@
package graphite
import (
"flag"
"fmt"
"net/http"
"regexp"
@ -18,6 +19,11 @@ import (
"github.com/VictoriaMetrics/metrics"
)
var (
maxGraphiteTagKeysPerSearch = flag.Int("search.maxGraphiteTagKeys", 100e3, "The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API")
maxGraphiteTagValuesPerSearch = flag.Int("search.maxGraphiteTagValues", 100e3, "The maximum number of tag values returned Graphite /tags/<tag_name> API")
)
// TagsDelSeriesHandler implements /tags/delSeries handler.
//
// See https://graphite.readthedocs.io/en/stable/tags.html#removing-series-from-the-tagdb
@ -182,13 +188,13 @@ func TagsAutoCompleteValuesHandler(startTime time.Time, w http.ResponseWriter, r
// Escape special chars in tagPrefix as Graphite does.
// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L228
filter := regexp.QuoteMeta(valuePrefix)
tagValues, err = netstorage.GraphiteTagValues(nil, tag, filter, limit, deadline)
tagValues, err = netstorage.GraphiteTagValues(nil, tag, filter, *maxGraphiteTagKeysPerSearch, deadline)
if err != nil {
return err
}
} else {
// Slow path: use netstorage.SearchMetricNames for applying `expr` filters.
sq, err := getSearchQueryForExprs(startTime, etfs, exprs, limit*10)
sq, err := getSearchQueryForExprs(startTime, etfs, exprs, *maxGraphiteTagKeysPerSearch)
if err != nil {
return err
}
@ -269,13 +275,13 @@ func TagsAutoCompleteTagsHandler(startTime time.Time, w http.ResponseWriter, r *
// Escape special chars in tagPrefix as Graphite does.
// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L181
filter := regexp.QuoteMeta(tagPrefix)
labels, err = netstorage.GraphiteTags(nil, filter, limit, deadline)
labels, err = netstorage.GraphiteTags(nil, filter, *maxGraphiteTagKeysPerSearch, deadline)
if err != nil {
return err
}
} else {
// Slow path: use netstorage.SearchMetricNames for applying `expr` filters.
sq, err := getSearchQueryForExprs(startTime, etfs, exprs, limit*10)
sq, err := getSearchQueryForExprs(startTime, etfs, exprs, *maxGraphiteTagKeysPerSearch)
if err != nil {
return err
}
@ -343,7 +349,7 @@ func TagsFindSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.R
if err != nil {
return fmt.Errorf("cannot setup tag filters: %w", err)
}
sq, err := getSearchQueryForExprs(startTime, etfs, exprs, limit*10)
sq, err := getSearchQueryForExprs(startTime, etfs, exprs, *maxGraphiteTagKeysPerSearch)
if err != nil {
return err
}
@ -411,11 +417,14 @@ func TagValuesHandler(startTime time.Time, tagName string, w http.ResponseWriter
return err
}
filter := r.FormValue("filter")
tagValues, err := netstorage.GraphiteTagValues(nil, tagName, filter, limit, deadline)
tagValues, err := netstorage.GraphiteTagValues(nil, tagName, filter, *maxGraphiteTagValuesPerSearch, deadline)
if err != nil {
return err
}
if limit > 0 && limit < len(tagValues) {
tagValues = tagValues[:limit]
}
w.Header().Set("Content-Type", "application/json")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)
@ -439,11 +448,14 @@ func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) er
return err
}
filter := r.FormValue("filter")
labels, err := netstorage.GraphiteTags(nil, filter, limit, deadline)
labels, err := netstorage.GraphiteTags(nil, filter, *maxGraphiteTagKeysPerSearch, deadline)
if err != nil {
return err
}
if limit > 0 && limit < len(labels) {
labels = labels[:limit]
}
w.Header().Set("Content-Type", "application/json")
bw := bufferedwriter.Get(w)
defer bufferedwriter.Put(bw)

6
app/vmselect/prometheus/prometheus.go
View File

@ -633,11 +633,7 @@ func SeriesHandler(qt *querytracer.Tracer, startTime time.Time, w http.ResponseW
return err
}
minLimit := *maxSeriesLimit
if limit > 0 && limit < *maxSeriesLimit {
minLimit = limit
}
sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, minLimit)
sq := storage.NewSearchQuery(cp.start, cp.end, cp.filterss, *maxSeriesLimit)
metricNames, err := netstorage.SearchMetricNames(qt, sq, cp.deadline)
if err != nil {
return fmt.Errorf("cannot fetch time series for %q: %w", sq, err)

4
docs/CHANGELOG.md
View File

@ -25,6 +25,10 @@ The following tip changes can be tested by building VictoriaMetrics components f
## tip
* BUGFIX: [vmalert](https://docs.victoriametrics.com/vmalert.html): fix nil map assignment panic in runtime introduced in this [change](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4341).
* BUGFIX: add the following command-line flags, which can be used for limiting Graphite API calls:
`--search.maxGraphiteTagKeys` for limiting the number of tag keys returned from Graphite `/tags`, `/tags/autoComplete/*`, `/tags/findSeries` API.
`--search.maxGraphiteTagValues` for limiting the number of tag values returned Graphite `/tags/<tag_name>` API.
Remove redundant limit from [Prometheus api/v1/series](https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#prometheus-querying-api-usage). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4339).
## [v1.91.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.91.1)

4
docs/Cluster-VictoriaMetrics.md
View File

@ -1156,6 +1156,10 @@ Below is the output for `/path/to/vmselect -help`:
The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000)
-search.maxGraphiteSeries int
The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000)
-search.maxGraphiteTagKeys int
The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API (default 100000)
-search.maxGraphiteTagValues int
The maximum number of tag values returned Graphite /tags/<tag_name> API (default 100000)
-search.maxLookback duration
Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaining due to historical reasons
-search.maxMemoryPerQuery size

4
docs/README.md
View File

@ -2501,6 +2501,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000)
-search.maxGraphiteSeries int
The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000)
-search.maxGraphiteTagKeys int
The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API (default 100000)
-search.maxGraphiteTagValues int
The maximum number of tag values returned Graphite /tags/<tag_name> API (default 100000)
-search.maxLookback duration
Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons
-search.maxMemoryPerQuery size

4
docs/Single-server-VictoriaMetrics.md
View File

@ -2509,6 +2509,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
The maximum number of time series, which can be returned from /federate. This option allows limiting memory usage (default 1000000)
-search.maxGraphiteSeries int
The maximum number of time series, which can be scanned during queries to Graphite Render API. See https://docs.victoriametrics.com/#graphite-render-api-usage (default 300000)
-search.maxGraphiteTagKeys int
The maximum number of tag keys returned from Graphite /tags, /tags/autoComplete/*, /tags/findSeries API (default 100000)
-search.maxGraphiteTagValues int
The maximum number of tag values returned Graphite /tags/<tag_name> API (default 100000)
-search.maxLookback duration
Synonym to -search.lookback-delta from Prometheus. The value is dynamically detected from interval between time series datapoints if not set. It can be overridden on per-query basis via max_lookback arg. See also '-search.maxStalenessInterval' flag, which has the same meaning due to historical reasons
-search.maxMemoryPerQuery size