From 86f99c6b551b9e29efe3e9dce2863a4e9a7b8a00 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 16 Nov 2020 14:49:46 +0200 Subject: [PATCH] app/vmselect/graphite: add `/tags/autoComplete/tags` handler from Graphite Tags API See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support --- README.md | 1 + app/vmselect/graphite/metrics_api.go | 22 ++-- app/vmselect/graphite/tags_api.go | 111 ++++++++++++++++-- .../tags_autocomplete_tags_response.qtpl | 16 +++ .../tags_autocomplete_tags_response.qtpl.go | 81 +++++++++++++ app/vmselect/main.go | 26 +++- app/vmselect/netstorage/netstorage.go | 1 + docs/Single-server-VictoriaMetrics.md | 1 + 8 files changed, 235 insertions(+), 24 deletions(-) create mode 100644 app/vmselect/graphite/tags_autocomplete_tags_response.qtpl create mode 100644 app/vmselect/graphite/tags_autocomplete_tags_response.qtpl.go diff --git a/README.md b/README.md index 0c722fc4f..458915ae1 100644 --- a/README.md +++ b/README.md @@ -551,6 +551,7 @@ VictoriaMetrics supports the following handlers from [Graphite Tags API](https:/ * [/tags](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags) * [/tags/tag_name](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags) * [/tags/findSeries](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags) +* [/tags/autoComplete/tags](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support) ### How to build from sources diff --git a/app/vmselect/graphite/metrics_api.go b/app/vmselect/graphite/metrics_api.go index 6578b7938..3eb3e7f11 100644 --- a/app/vmselect/graphite/metrics_api.go +++ b/app/vmselect/graphite/metrics_api.go @@ -84,10 +84,7 @@ func MetricsFindHandler(startTime time.Time, w http.ResponseWriter, r *http.Requ } paths = deduplicatePaths(paths, delimiter) sortPaths(paths, delimiter) - contentType := "application/json; charset=utf-8" - if jsonp != "" { - contentType = "text/javascript; charset=utf-8" - } + contentType := getContentType(jsonp) w.Header().Set("Content-Type", contentType) bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) @@ -166,10 +163,7 @@ func MetricsExpandHandler(startTime time.Time, w http.ResponseWriter, r *http.Re } m[query] = paths } - contentType := "application/json; charset=utf-8" - if jsonp != "" { - contentType = "text/javascript; charset=utf-8" - } + contentType := getContentType(jsonp) w.Header().Set("Content-Type", contentType) if groupByExpr { for _, paths := range m { @@ -215,10 +209,7 @@ func MetricsIndexHandler(startTime time.Time, w http.ResponseWriter, r *http.Req if err != nil { return fmt.Errorf(`cannot obtain metric names: %w`, err) } - contentType := "application/json; charset=utf-8" - if jsonp != "" { - contentType = "text/javascript; charset=utf-8" - } + contentType := getContentType(jsonp) w.Header().Set("Content-Type", contentType) bw := bufferedwriter.Get(w) defer bufferedwriter.Put(bw) @@ -417,3 +408,10 @@ var regexpCache = make(map[regexpCacheKey]*regexpCacheEntry) var regexpCacheLock sync.Mutex const maxRegexpCacheSize = 10000 + +func getContentType(jsonp string) string { + if jsonp == "" { + return "application/json; charset=utf-8" + } + return "text/javascript; charset=utf-8" +} diff --git a/app/vmselect/graphite/tags_api.go b/app/vmselect/graphite/tags_api.go index 8c76b6031..90da70197 100644 --- a/app/vmselect/graphite/tags_api.go +++ b/app/vmselect/graphite/tags_api.go @@ -3,6 +3,7 @@ package graphite import ( "fmt" "net/http" + "regexp" "sort" "strconv" "strings" @@ -15,6 +16,90 @@ import ( "github.com/VictoriaMetrics/metrics" ) +// TagsAutoCompleteTagsHandler implements /tags/autoComplete/tags endpoint from Graphite Tags API. +// +// See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support +func TagsAutoCompleteTagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error { + deadline := searchutils.GetDeadlineForQuery(r, startTime) + if err := r.ParseForm(); err != nil { + return fmt.Errorf("cannot parse form values: %w", err) + } + limit, err := getInt(r, "limit") + if err != nil { + return err + } + if limit <= 0 { + // Use limit=100 by default. See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support + limit = 100 + } + tagPrefix := r.FormValue("tagPrefix") + exprs := r.Form["expr"] + var labels []string + if len(exprs) == 0 { + // Fast path: there are no `expr` filters. + + // Escape special chars in tagPrefix as Graphite does. + // See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/base.py#L181 + filter := regexp.QuoteMeta(tagPrefix) + labels, err = netstorage.GetGraphiteTags(filter, limit, deadline) + if err != nil { + return err + } + } else { + // Slow path: use netstorage.SearchMetricNames for applying `expr` filters. + tfs, err := exprsToTagFilters(exprs) + if err != nil { + return err + } + ct := time.Now().UnixNano() / 1e6 + sq := &storage.SearchQuery{ + MinTimestamp: 0, + MaxTimestamp: ct, + TagFilterss: [][]storage.TagFilter{tfs}, + } + mns, err := netstorage.SearchMetricNames(sq, deadline) + if err != nil { + return fmt.Errorf("cannot fetch metric names for %q: %w", sq, err) + } + m := make(map[string]struct{}) + for _, mn := range mns { + m["name"] = struct{}{} + for _, tag := range mn.Tags { + m[string(tag.Key)] = struct{}{} + } + } + if len(tagPrefix) > 0 { + for label := range m { + if !strings.HasPrefix(label, tagPrefix) { + delete(m, label) + } + } + } + labels = make([]string, 0, len(m)) + for label := range m { + labels = append(labels, label) + } + sort.Strings(labels) + if limit > 0 && limit < len(labels) { + labels = labels[:limit] + } + } + + jsonp := r.FormValue("jsonp") + contentType := getContentType(jsonp) + w.Header().Set("Content-Type", contentType) + bw := bufferedwriter.Get(w) + defer bufferedwriter.Put(bw) + WriteTagsAutoCompleteTagsResponse(bw, labels, jsonp) + if err := bw.Flush(); err != nil { + return err + } + tagsAutoCompleteTagsDuration.UpdateDuration(startTime) + return nil +} + +var tagsAutoCompleteTagsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags/autoComplete/tags"}`) + // TagsFindSeriesHandler implements /tags/findSeries endpoint from Graphite Tags API. // // See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags @@ -31,18 +116,10 @@ func TagsFindSeriesHandler(startTime time.Time, w http.ResponseWriter, r *http.R if len(exprs) == 0 { return fmt.Errorf("expecting at least one `expr` query arg") } - - // Convert exprs to []storage.TagFilter - tfs := make([]storage.TagFilter, 0, len(exprs)) - for _, expr := range exprs { - tf, err := parseFilterExpr(expr) - if err != nil { - return fmt.Errorf("cannot parse `expr` query arg: %w", err) - } - tfs = append(tfs, *tf) + tfs, err := exprsToTagFilters(exprs) + if err != nil { + return err } - - // Send the request to storage ct := time.Now().UnixNano() / 1e6 sq := &storage.SearchQuery{ MinTimestamp: 0, @@ -167,6 +244,18 @@ func getInt(r *http.Request, argName string) (int, error) { return n, nil } +func exprsToTagFilters(exprs []string) ([]storage.TagFilter, error) { + tfs := make([]storage.TagFilter, 0, len(exprs)) + for _, expr := range exprs { + tf, err := parseFilterExpr(expr) + if err != nil { + return nil, fmt.Errorf("cannot parse `expr` query arg: %w", err) + } + tfs = append(tfs, *tf) + } + return tfs, nil +} + func parseFilterExpr(s string) (*storage.TagFilter, error) { n := strings.Index(s, "=") if n < 0 { diff --git a/app/vmselect/graphite/tags_autocomplete_tags_response.qtpl b/app/vmselect/graphite/tags_autocomplete_tags_response.qtpl new file mode 100644 index 000000000..42e1ebfce --- /dev/null +++ b/app/vmselect/graphite/tags_autocomplete_tags_response.qtpl @@ -0,0 +1,16 @@ +{% stripspace %} + +TagsAutoCompleteTagsResponse generates response for /tags/autoComplete/tags handler in Graphite Tags API. +See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support +{% func TagsAutoCompleteTagsResponse(labels []string, jsonp string) %} + {% if jsonp != "" %}{%s= jsonp %}({% endif %} + [ + {% for i, label := range labels %} + {%q= label %} + {% if i+1 < len(labels) %},{% endif %} + {% endfor %} + ] + {% if jsonp != "" %}){% endif %} +{% endfunc %} + +{% endstripspace %} diff --git a/app/vmselect/graphite/tags_autocomplete_tags_response.qtpl.go b/app/vmselect/graphite/tags_autocomplete_tags_response.qtpl.go new file mode 100644 index 000000000..8102e62a0 --- /dev/null +++ b/app/vmselect/graphite/tags_autocomplete_tags_response.qtpl.go @@ -0,0 +1,81 @@ +// Code generated by qtc from "tags_autocomplete_tags_response.qtpl". DO NOT EDIT. +// See https://github.com/valyala/quicktemplate for details. + +// TagsAutoCompleteTagsResponse generates response for /tags/autoComplete/tags handler in Graphite Tags API.See https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support + +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:5 +package graphite + +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:5 +import ( + qtio422016 "io" + + qt422016 "github.com/valyala/quicktemplate" +) + +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:5 +var ( + _ = qtio422016.Copy + _ = qt422016.AcquireByteBuffer +) + +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:5 +func StreamTagsAutoCompleteTagsResponse(qw422016 *qt422016.Writer, labels []string, jsonp string) { +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:6 + if jsonp != "" { +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:6 + qw422016.N().S(jsonp) +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:6 + qw422016.N().S(`(`) +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:6 + } +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:6 + qw422016.N().S(`[`) +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:8 + for i, label := range labels { +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:9 + qw422016.N().Q(label) +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:10 + if i+1 < len(labels) { +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:10 + qw422016.N().S(`,`) +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:10 + } +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:11 + } +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:11 + qw422016.N().S(`]`) +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:13 + if jsonp != "" { +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:13 + qw422016.N().S(`)`) +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:13 + } +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 +} + +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 +func WriteTagsAutoCompleteTagsResponse(qq422016 qtio422016.Writer, labels []string, jsonp string) { +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 + StreamTagsAutoCompleteTagsResponse(qw422016, labels, jsonp) +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 + qt422016.ReleaseWriter(qw422016) +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 +} + +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 +func TagsAutoCompleteTagsResponse(labels []string, jsonp string) string { +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 + WriteTagsAutoCompleteTagsResponse(qb422016, labels, jsonp) +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 + qs422016 := string(qb422016.B) +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 + return qs422016 +//line app/vmselect/graphite/tags_autocomplete_tags_response.qtpl:14 +} diff --git a/app/vmselect/main.go b/app/vmselect/main.go index 1f3f92d40..73cc995e0 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -132,7 +132,7 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { return true } } - if strings.HasPrefix(path, "/tags/") && path != "/tags/findSeries" { + if strings.HasPrefix(path, "/tags/") && !isGraphiteTagsPath(path) { tagName := r.URL.Path[len("/tags/"):] graphiteTagValuesRequests.Inc() if err := graphite.TagValuesHandler(startTime, tagName, w, r); err != nil { @@ -285,6 +285,15 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { return true } return true + case "/tags/autoComplete/tags": + graphiteTagsAutoCompleteTagsRequests.Inc() + httpserver.EnableCORS(w, r) + if err := graphite.TagsAutoCompleteTagsHandler(startTime, w, r); err != nil { + graphiteTagsAutoCompleteTagsErrors.Inc() + httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err) + return true + } + return true case "/api/v1/rules": // Return dumb placeholder rulesRequests.Inc() @@ -322,6 +331,18 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { } } +func isGraphiteTagsPath(path string) bool { + switch path { + // See https://graphite.readthedocs.io/en/stable/tags.html for a list of Graphite Tags API paths. + // Do not include `/tags/` here, since this will fool the caller. + case "/tags/tagSeries", "/tags/tagMultiSeries", "/tags/findSeries", + "/tags/autoComplete/tags", "/tags/autoComplete/values", "/tags/delSeries": + return true + default: + return false + } +} + func sendPrometheusError(w http.ResponseWriter, r *http.Request, err error) { logger.Warnf("error in %q: %s", r.RequestURI, err) @@ -395,6 +416,9 @@ var ( graphiteTagsFindSeriesRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/findSeries"}`) graphiteTagsFindSeriesErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/findSeries"}`) + graphiteTagsAutoCompleteTagsRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags/autoComplete/tags"}`) + graphiteTagsAutoCompleteTagsErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/tags/autoComplete/tags"}`) + rulesRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`) alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`) metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`) diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go index a672b6d14..97dc45375 100644 --- a/app/vmselect/netstorage/netstorage.go +++ b/app/vmselect/netstorage/netstorage.go @@ -493,6 +493,7 @@ func GetGraphiteTags(filter string, limit int, deadline searchutils.Deadline) ([ for i := range labels { if labels[i] == "__name__" { labels[i] = "name" + sort.Strings(labels) break } } diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 0c722fc4f..458915ae1 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -551,6 +551,7 @@ VictoriaMetrics supports the following handlers from [Graphite Tags API](https:/ * [/tags](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags) * [/tags/tag_name](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags) * [/tags/findSeries](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags) +* [/tags/autoComplete/tags](https://graphite.readthedocs.io/en/stable/tags.html#auto-complete-support) ### How to build from sources