From eb763bcb9d392b969ab287ca2ebb3bdce789c328 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 16 Nov 2020 15:22:36 +0200 Subject: [PATCH] app/vmselect/graphite: add `/tags/autoComplete/values` handler from Graphite Tags API --- README.md | 1 + app/vmselect/graphite/tags_api.go | 122 +++++++++++++++--- .../graphite/tags_autocomplete_response.qtpl | 16 +++ .../tags_autocomplete_response.qtpl.go | 81 ++++++++++++ .../tags_autocomplete_tags_response.qtpl | 16 --- .../tags_autocomplete_tags_response.qtpl.go | 81 ------------ app/vmselect/main.go | 12 ++ docs/Cluster-VictoriaMetrics.md | 1 + docs/Single-server-VictoriaMetrics.md | 1 + 9 files changed, 218 insertions(+), 113 deletions(-) create mode 100644 app/vmselect/graphite/tags_autocomplete_response.qtpl create mode 100644 app/vmselect/graphite/tags_autocomplete_response.qtpl.go delete mode 100644 app/vmselect/graphite/tags_autocomplete_tags_response.qtpl delete mode 100644 app/vmselect/graphite/tags_autocomplete_tags_response.qtpl.go diff --git a/README.md b/README.md index b89963ce9b..fd98c59c99 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,7 @@ or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/gr - `tags/` - returns tag values for the given ``. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags). - `tags/findSeries` - returns series matching the given `expr`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags). - `tags/autoComplete/tags` - returns tags matching the given `tagPrefix` and/or `expr`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support). + - `tags/autoComplete/values` - returns tag values matching the given `valuePrefix` and/or `expr`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support). * URL for time series deletion: `http://:8481/delete//prometheus/api/v1/admin/tsdb/delete_series?match[]=`. Note that the `delete_series` handler should be used only in exceptional cases such as deletion of accidentally ingested incorrect time series. It shouldn't diff --git a/app/vmselect/graphite/tags_api.go b/app/vmselect/graphite/tags_api.go index 96a5ea7a8c..b0e6bb0035 100644 --- a/app/vmselect/graphite/tags_api.go +++ b/app/vmselect/graphite/tags_api.go @@ -17,6 +17,94 @@ import ( "github.com/VictoriaMetrics/metrics" ) +// TagsAutoCompleteValuesHandler implements /tags/autoComplete/values endpoint from Graphite Tags API. +// +// See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support +func TagsAutoCompleteValuesHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *http.Request) error { + deadline := searchutils.GetDeadlineForQuery(r, startTime) + if err := r.ParseForm(); err != nil { + return fmt.Errorf("cannot parse form values: %w", err) + } + limit, err := getInt(r, "limit") + if err != nil { + return err + } + if limit <= 0 { + // Use limit=100 by default. See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support + limit = 100 + } + tag := r.FormValue("tag") + if len(tag) == 0 { + return fmt.Errorf("missing `tag` query arg") + } + valuePrefix := r.FormValue("valuePrefix") + exprs := r.Form["expr"] + var tagValues []string + denyPartialResponse := searchutils.GetDenyPartialResponse(r) + isPartial := false + if len(exprs) == 0 { + // Fast path: there are no `expr` filters, so use netstorage.GetGraphiteTagValues. + // Escape special chars in tagPrefix as Graphite does. + // See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L228 + filter := regexp.QuoteMeta(valuePrefix) + tagValues, isPartial, err = netstorage.GetGraphiteTagValues(at, denyPartialResponse, tag, filter, limit, deadline) + if err != nil { + return err + } + } else { + // Slow path: use netstorage.SearchMetricNames for applying `expr` filters. + sq, err := getSearchQueryForExprs(exprs) + if err != nil { + return err + } + mns, isPartialResponse, err := netstorage.SearchMetricNames(at, denyPartialResponse, sq, deadline) + if err != nil { + return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err) + } + isPartial = isPartialResponse + m := make(map[string]struct{}) + if tag == "name" { + tag = "__name__" + } + for _, mn := range mns { + tagValue := mn.GetTagValue(tag) + if len(tagValue) == 0 { + continue + } + m[string(tagValue)] = struct{}{} + } + if len(valuePrefix) > 0 { + for tagValue := range m { + if !strings.HasPrefix(tagValue, valuePrefix) { + delete(m, tagValue) + } + } + } + tagValues = make([]string, 0, len(m)) + for tagValue := range m { + tagValues = append(tagValues, tagValue) + } + sort.Strings(tagValues) + if limit > 0 && limit < len(tagValues) { + tagValues = tagValues[:limit] + } + } + + jsonp := r.FormValue("jsonp") + contentType := getContentType(jsonp) + w.Header().Set("Content-Type", contentType) + bw := bufferedwriter.Get(w) + defer bufferedwriter.Put(bw) + WriteTagsAutoCompleteResponse(bw, isPartial, tagValues, jsonp) + if err := bw.Flush(); err != nil { + return err + } + tagsAutoCompleteValuesDuration.UpdateDuration(startTime) + return nil +} + +var tagsAutoCompleteValuesDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/autoComplete/values"}`) + // TagsAutoCompleteTagsHandler implements /tags/autoComplete/tags endpoint from Graphite Tags API. // // See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support @@ -39,7 +127,7 @@ func TagsAutoCompleteTagsHandler(startTime time.Time, at *auth.Token, w http.Res var labels []string isPartial := false if len(exprs) == 0 { - // Fast path: there are no `expr` filters. + // Fast path: there are no `expr` filters, so use netstorage.GetGraphiteTags. // Escape special chars in tagPrefix as Graphite does. // See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L181 @@ -50,16 +138,10 @@ func TagsAutoCompleteTagsHandler(startTime time.Time, at *auth.Token, w http.Res } } else { // Slow path: use netstorage.SearchMetricNames for applying `expr` filters. - tfs, err := exprsToTagFilters(exprs) + sq, err := getSearchQueryForExprs(exprs) if err != nil { return err } - ct := time.Now().UnixNano() / 1e6 - sq := &storage.SearchQuery{ - MinTimestamp: 0, - MaxTimestamp: ct, - TagFilterss: [][]storage.TagFilter{tfs}, - } mns, isPartialResponse, err := netstorage.SearchMetricNames(at, denyPartialResponse, sq, deadline) if err != nil { return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err) @@ -94,7 +176,7 @@ func TagsAutoCompleteTagsHandler(startTime time.Time, at *auth.Token, w http.Res w.Header().Set("Content-Type", contentType) bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) - WriteTagsAutoCompleteTagsResponse(bw, isPartial, labels, jsonp) + WriteTagsAutoCompleteResponse(bw, isPartial, labels, jsonp) if err := bw.Flush(); err != nil { return err } @@ -120,16 +202,10 @@ func TagsFindSeriesHandler(startTime time.Time, at *auth.Token, w http.ResponseW if len(exprs) == 0 { return fmt.Errorf("expecting at least one `expr` query arg") } - tfs, err := exprsToTagFilters(exprs) + sq, err := getSearchQueryForExprs(exprs) if err != nil { return err } - ct := time.Now().UnixNano() / 1e6 - sq := &storage.SearchQuery{ - MinTimestamp: 0, - MaxTimestamp: ct, - TagFilterss: [][]storage.TagFilter{tfs}, - } denyPartialResponse := searchutils.GetDenyPartialResponse(r) mns, isPartial, err := netstorage.SearchMetricNames(at, denyPartialResponse, sq, deadline) if err != nil { @@ -251,6 +327,20 @@ func getInt(r *http.Request, argName string) (int, error) { return n, nil } +func getSearchQueryForExprs(exprs []string) (*storage.SearchQuery, error) { + tfs, err := exprsToTagFilters(exprs) + if err != nil { + return nil, err + } + ct := time.Now().UnixNano() / 1e6 + sq := &storage.SearchQuery{ + MinTimestamp: 0, + MaxTimestamp: ct, + TagFilterss: [][]storage.TagFilter{tfs}, + } + return sq, nil +} + func exprsToTagFilters(exprs []string) ([]storage.TagFilter, error) { tfs := make([]storage.TagFilter, 0, len(exprs)) for _, expr := range exprs { diff --git a/app/vmselect/graphite/tags_autocomplete_response.qtpl b/app/vmselect/graphite/tags_autocomplete_response.qtpl new file mode 100644 index 0000000000..900d438cba --- /dev/null +++ b/app/vmselect/graphite/tags_autocomplete_response.qtpl @@ -0,0 +1,16 @@ +{% stripspace %} + +TagsAutoCompleteResponse generates responses for /tags/autoComplete/{tags,values} handlers in Graphite Tags API. +See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support +{% func TagsAutoCompleteResponse(isPartial bool, ss []string, jsonp string) %} + {% if jsonp != "" %}{%s= jsonp %}({% endif %} + [ + {% for i, s := range ss %} + {%q= s %} + {% if i+1 < len(ss) %},{% endif %} + {% endfor %} + ] + {% if jsonp != "" %}){% endif %} +{% endfunc %} + +{% endstripspace %} diff --git a/app/vmselect/graphite/tags_autocomplete_response.qtpl.go b/app/vmselect/graphite/tags_autocomplete_response.qtpl.go new file mode 100644 index 0000000000..f9bec61217 --- /dev/null +++ b/app/vmselect/graphite/tags_autocomplete_response.qtpl.go @@ -0,0 +1,81 @@ +// Code generated by qtc from "tags_autocomplete_response.qtpl". DO NOT EDIT. +// See https://github.com/valyala/quicktemplate for details. + +// TagsAutoCompleteResponse generates responses for /tags/autoComplete/{tags,values} handlers in Graphite Tags API.See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support + +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:5 +package graphite + +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:5 +import ( + qtio422016 "io" + + qt422016 "github.com/valyala/quicktemplate" +) + +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:5 +var ( + _ = qtio422016.Copy + _ = qt422016.AcquireByteBuffer +) + +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:5 +func StreamTagsAutoCompleteResponse(qw422016 *qt422016.Writer, isPartial bool, ss []string, jsonp string) { +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6 + if jsonp != "" { +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6 + qw422016.N().S(jsonp) +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6 + qw422016.N().S(`(`) +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6 + } +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:6 + qw422016.N().S(`[`) +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:8 + for i, s := range ss { +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:9 + qw422016.N().Q(s) +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:10 + if i+1 < len(ss) { +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:10 + qw422016.N().S(`,`) +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:10 + } +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:11 + } +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:11 + qw422016.N().S(`]`) +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:13 + if jsonp != "" { +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:13 + qw422016.N().S(`)`) +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:13 + } +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14 +} + +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14 +func WriteTagsAutoCompleteResponse(qq422016 qtio422016.Writer, isPartial bool, ss []string, jsonp string) { +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14 + StreamTagsAutoCompleteResponse(qw422016, isPartial, ss, jsonp) +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14 + qt422016.ReleaseWriter(qw422016) +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14 +} + +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14 +func TagsAutoCompleteResponse(isPartial bool, ss []string, jsonp string) string { +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14 + WriteTagsAutoCompleteResponse(qb422016, isPartial, ss, jsonp) +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14 + qs422016 := string(qb422016.B) +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14 + return qs422016 +//line app/vmselect/graphite/tags_autocomplete_response.qtpl:14 +} diff --git a/app/vmselect/graphite/tags_autocomplete_tags_response.qtpl b/app/vmselect/graphite/tags_autocomplete_tags_response.qtpl deleted file mode 100644 index f4f77de2a5..0000000000 --- a/app/vmselect/graphite/tags_autocomplete_tags_response.qtpl +++ /dev/null @@ -1,16 +0,0 @@ -{% stripspace %} - -TagsAutoCompleteTagsResponse generates response for /tags/autoComplete/tags handler in Graphite Tags API. -See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support -{% func TagsAutoCompleteTagsResponse(isPartial bool, labels []string, jsonp string) %} - {% if jsonp != "" %}{%s= jsonp %}({% endif %} - [ - {% for i, label := range labels %} - {%q= label %} - {% if i+1 < len(labels) %},{% endif %} - {% endfor %} - ] - {% if jsonp != "" %}){% endif %} -{% endfunc %} - -{% endstripspace %} diff --git a/app/vmselect/graphite/tags_autocomplete_tags_response.qtpl.go b/app/vmselect/graphite/tags_autocomplete_tags_response.qtpl.go deleted file mode 100644 index 5fc152c153..0000000000 --- a/app/vmselect/graphite/tags_autocomplete_tags_response.qtpl.go +++ /dev/null @@ -1,81 +0,0 @@ -// Code generated by qtc from "tags_autocomplete_tags_response.qtpl". DO NOT EDIT. -// See https://github.com/valyala/quicktemplate for details. - -// TagsAutoCompleteTagsResponse generates response for /tags/autoComplete/tags handler in Graphite Tags API.See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support - -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:5 -package graphite - -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:5 -import ( - qtio422016 "io" - - qt422016 "github.com/valyala/quicktemplate" -) - -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:5 -var ( - _ = qtio422016.Copy - _ = qt422016.AcquireByteBuffer -) - -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:5 -func StreamTagsAutoCompleteTagsResponse(qw422016 *qt422016.Writer, isPartial bool, labels []string, jsonp string) { -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:6 - if jsonp != "" { -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:6 - qw422016.N().S(jsonp) -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:6 - qw422016.N().S(`(`) -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:6 - } -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:6 - qw422016.N().S(`[`) -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:8 - for i, label := range labels { -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:9 - qw422016.N().Q(label) -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:10 - if i+1 < len(labels) { -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:10 - qw422016.N().S(`,`) -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:10 - } -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:11 - } -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:11 - qw422016.N().S(`]`) -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:13 - if jsonp != "" { -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:13 - qw422016.N().S(`)`) -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:13 - } -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 -} - -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 -func WriteTagsAutoCompleteTagsResponse(qq422016 qtio422016.Writer, isPartial bool, labels []string, jsonp string) { -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 - qw422016 := qt422016.AcquireWriter(qq422016) -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 - StreamTagsAutoCompleteTagsResponse(qw422016, isPartial, labels, jsonp) -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 - qt422016.ReleaseWriter(qw422016) -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 -} - -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 -func TagsAutoCompleteTagsResponse(isPartial bool, labels []string, jsonp string) string { -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 - qb422016 := qt422016.AcquireByteBuffer() -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 - WriteTagsAutoCompleteTagsResponse(qb422016, isPartial, labels, jsonp) -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 - qs422016 := string(qb422016.B) -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 - qt422016.ReleaseByteBuffer(qb422016) -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 - return qs422016 -//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 -} diff --git a/app/vmselect/main.go b/app/vmselect/main.go index 67bec36f26..76e2a3c531 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -371,6 +371,15 @@ func selectHandler(startTime time.Time, w http.ResponseWriter, r *http.Request, return true } return true + case "/tags/autoComplete/values": + graphiteTagsAutoCompleteValuesRequests.Inc() + httpserver.EnableCORS(w, r) + if err := graphite.TagsAutoCompleteValuesHandler(startTime, at, w, r); err != nil { + graphiteTagsAutoCompleteValuesErrors.Inc() + httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err) + return true + } + return true case "prometheus/api/v1/rules": // Return dumb placeholder rulesRequests.Inc() @@ -498,6 +507,9 @@ var ( graphiteTagsAutoCompleteTagsRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/graphite/tags/autoComplete/tags"}`) graphiteTagsAutoCompleteTagsErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/graphite/tags/autoComplete/tags"}`) + graphiteTagsAutoCompleteValuesRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/graphite/tags/autoComplete/values"}`) + graphiteTagsAutoCompleteValuesErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/select/{}/graphite/tags/autoComplete/values"}`) + rulesRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/rules"}`) alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/alerts"}`) metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/select/{}/prometheus/api/v1/metadata"}`) diff --git a/docs/Cluster-VictoriaMetrics.md b/docs/Cluster-VictoriaMetrics.md index b89963ce9b..fd98c59c99 100644 --- a/docs/Cluster-VictoriaMetrics.md +++ b/docs/Cluster-VictoriaMetrics.md @@ -209,6 +209,7 @@ or [an alternative dashboard for VictoriaMetrics cluster](https://grafana.com/gr - `tags/` - returns tag values for the given ``. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags). - `tags/findSeries` - returns series matching the given `expr`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags). - `tags/autoComplete/tags` - returns tags matching the given `tagPrefix` and/or `expr`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support). + - `tags/autoComplete/values` - returns tag values matching the given `valuePrefix` and/or `expr`. See [these docs](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support). * URL for time series deletion: `http://:8481/delete//prometheus/api/v1/admin/tsdb/delete_series?match[]=`. Note that the `delete_series` handler should be used only in exceptional cases such as deletion of accidentally ingested incorrect time series. It shouldn't diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 458915ae15..22812cf5e5 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -552,6 +552,7 @@ VictoriaMetrics supports the following handlers from [Graphite Tags API](https:/ * [/tags/tag_name](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags) * [/tags/findSeries](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags) * [/tags/autoComplete/tags](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support) +* [/tags/autoComplete/values](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support) ### How to build from sources