From 6251762787f615d4c3857a25725e602ca4e46cfc Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 16 Nov 2020 01:25:38 +0200 Subject: [PATCH] app/vmselect/graphite: add `/tags` handler from Graphite Tags API See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags --- README.md | 1 + .../graphite/{graphite.go => metrics_api.go} | 0 .../{graphite_test.go => metrics_api_test.go} | 0 app/vmselect/graphite/tags_api.go | 65 +++++++++++++++++ app/vmselect/graphite/tags_response.qtpl | 16 +++++ app/vmselect/graphite/tags_response.qtpl.go | 71 +++++++++++++++++++ app/vmselect/main.go | 11 +++ app/vmselect/netstorage/netstorage.go | 27 +++++++ docs/Single-server-VictoriaMetrics.md | 1 + 9 files changed, 192 insertions(+) rename app/vmselect/graphite/{graphite.go => metrics_api.go} (100%) rename app/vmselect/graphite/{graphite_test.go => metrics_api_test.go} (100%) create mode 100644 app/vmselect/graphite/tags_api.go create mode 100644 app/vmselect/graphite/tags_response.qtpl create mode 100644 app/vmselect/graphite/tags_response.qtpl.go diff --git a/README.md b/README.md index e316e46ad..2f0385969 100644 --- a/README.md +++ b/README.md @@ -548,6 +548,7 @@ VictoriaMetrics supports the following handlers from [Graphite Tags API](https:/ * [/tags/tagSeries](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb) * [/tags/tagMultiSeries](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb) +* [/tags](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags) ### How to build from sources diff --git a/app/vmselect/graphite/graphite.go b/app/vmselect/graphite/metrics_api.go similarity index 100% rename from app/vmselect/graphite/graphite.go rename to app/vmselect/graphite/metrics_api.go diff --git a/app/vmselect/graphite/graphite_test.go b/app/vmselect/graphite/metrics_api_test.go similarity index 100% rename from app/vmselect/graphite/graphite_test.go rename to app/vmselect/graphite/metrics_api_test.go diff --git a/app/vmselect/graphite/tags_api.go b/app/vmselect/graphite/tags_api.go new file mode 100644 index 000000000..f21acf135 --- /dev/null +++ b/app/vmselect/graphite/tags_api.go @@ -0,0 +1,65 @@ +package graphite + +import ( + "fmt" + "net/http" + "regexp" + "strconv" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/bufferedwriter" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/netstorage" + "github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils" + "github.com/VictoriaMetrics/metrics" +) + +// TagsHandler implements handler for /tags endpoint. +// +// See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags +func TagsHandler(startTime time.Time, w http.ResponseWriter, r *http.Request) error { + deadline := searchutils.GetDeadlineForQuery(r, startTime) + if err := r.ParseForm(); err != nil { + return fmt.Errorf("cannot parse form values: %w", err) + } + limit := 0 + if limitStr := r.FormValue("limit"); len(limitStr) > 0 { + var err error + limit, err = strconv.Atoi(limitStr) + if err != nil { + return fmt.Errorf("cannot parse limit=%q: %w", limit, err) + } + } + labels, err := netstorage.GetGraphiteTags(limit, deadline) + if err != nil { + return err + } + filter := r.FormValue("filter") + if len(filter) > 0 { + // Anchor filter regexp to the beginning of the string as Graphite does. + // See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/localdatabase.py#L157 + filter = "^(?:" + filter + ")" + re, err := regexp.Compile(filter) + if err != nil { + return fmt.Errorf("cannot parse regexp filter=%q: %w", filter, err) + } + dst := labels[:0] + for _, label := range labels { + if re.MatchString(label) { + dst = append(dst, label) + } + } + labels = dst + } + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + bw := bufferedwriter.Get(w) + defer bufferedwriter.Put(bw) + WriteTagsResponse(bw, labels) + if err := bw.Flush(); err != nil { + return err + } + tagsDuration.UpdateDuration(startTime) + return nil +} + +var tagsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags"}`) diff --git a/app/vmselect/graphite/tags_response.qtpl b/app/vmselect/graphite/tags_response.qtpl new file mode 100644 index 000000000..c562d409e --- /dev/null +++ b/app/vmselect/graphite/tags_response.qtpl @@ -0,0 +1,16 @@ +{% stripspace %} + +Tags generates response for /tags handler +See https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags +{% func TagsResponse(tags []string) %} +[ + {% for i, tag := range tags %} + { + "tag":{%q= tag %} + } + {% if i+1 < len(tags) %},{% endif %} + {% endfor %} +] +{% endfunc %} + +{% endstripspace %} diff --git a/app/vmselect/graphite/tags_response.qtpl.go b/app/vmselect/graphite/tags_response.qtpl.go new file mode 100644 index 000000000..f95b63d4a --- /dev/null +++ b/app/vmselect/graphite/tags_response.qtpl.go @@ -0,0 +1,71 @@ +// Code generated by qtc from "tags_response.qtpl". DO NOT EDIT. +// See https://github.com/valyala/quicktemplate for details. + +// Tags generates response for /tags handlerSee https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags + +//line app/vmselect/graphite/tags_response.qtpl:5 +package graphite + +//line app/vmselect/graphite/tags_response.qtpl:5 +import ( + qtio422016 "io" + + qt422016 "github.com/valyala/quicktemplate" +) + +//line app/vmselect/graphite/tags_response.qtpl:5 +var ( + _ = qtio422016.Copy + _ = qt422016.AcquireByteBuffer +) + +//line app/vmselect/graphite/tags_response.qtpl:5 +func StreamTagsResponse(qw422016 *qt422016.Writer, tags []string) { +//line app/vmselect/graphite/tags_response.qtpl:5 + qw422016.N().S(`[`) +//line app/vmselect/graphite/tags_response.qtpl:7 + for i, tag := range tags { +//line app/vmselect/graphite/tags_response.qtpl:7 + qw422016.N().S(`{"tag":`) +//line app/vmselect/graphite/tags_response.qtpl:9 + qw422016.N().Q(tag) +//line app/vmselect/graphite/tags_response.qtpl:9 + qw422016.N().S(`}`) +//line app/vmselect/graphite/tags_response.qtpl:11 + if i+1 < len(tags) { +//line app/vmselect/graphite/tags_response.qtpl:11 + qw422016.N().S(`,`) +//line app/vmselect/graphite/tags_response.qtpl:11 + } +//line app/vmselect/graphite/tags_response.qtpl:12 + } +//line app/vmselect/graphite/tags_response.qtpl:12 + qw422016.N().S(`]`) +//line app/vmselect/graphite/tags_response.qtpl:14 +} + +//line app/vmselect/graphite/tags_response.qtpl:14 +func WriteTagsResponse(qq422016 qtio422016.Writer, tags []string) { +//line app/vmselect/graphite/tags_response.qtpl:14 + qw422016 := qt422016.AcquireWriter(qq422016) +//line app/vmselect/graphite/tags_response.qtpl:14 + StreamTagsResponse(qw422016, tags) +//line app/vmselect/graphite/tags_response.qtpl:14 + qt422016.ReleaseWriter(qw422016) +//line app/vmselect/graphite/tags_response.qtpl:14 +} + +//line app/vmselect/graphite/tags_response.qtpl:14 +func TagsResponse(tags []string) string { +//line app/vmselect/graphite/tags_response.qtpl:14 + qb422016 := qt422016.AcquireByteBuffer() +//line app/vmselect/graphite/tags_response.qtpl:14 + WriteTagsResponse(qb422016, tags) +//line app/vmselect/graphite/tags_response.qtpl:14 + qs422016 := string(qb422016.B) +//line app/vmselect/graphite/tags_response.qtpl:14 + qt422016.ReleaseByteBuffer(qb422016) +//line app/vmselect/graphite/tags_response.qtpl:14 + return qs422016 +//line app/vmselect/graphite/tags_response.qtpl:14 +} diff --git a/app/vmselect/main.go b/app/vmselect/main.go index 1b2fb9a05..2b652470c 100644 --- a/app/vmselect/main.go +++ b/app/vmselect/main.go @@ -259,6 +259,14 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { return true } return true + case "/tags": + graphiteTagsRequests.Inc() + if err := graphite.TagsHandler(startTime, w, r); err != nil { + graphiteTagsErrors.Inc() + httpserver.Errorf(w, r, "error in %q: %s", r.URL.Path, err) + return true + } + return true case "/api/v1/rules": // Return dumb placeholder rulesRequests.Inc() @@ -360,6 +368,9 @@ var ( graphiteMetricsIndexRequests = metrics.NewCounter(`vm_http_requests_total{path="/metrics/index.json"}`) graphiteMetricsIndexErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/metrics/index.json"}`) + graphiteTagsRequests = metrics.NewCounter(`vm_http_requests_total{path="/tags"}`) + graphiteTagsErrors = metrics.NewCounter(`vm_http_request_errors_total{path="/tags"}`) + rulesRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/rules"}`) alertsRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/alerts"}`) metadataRequests = metrics.NewCounter(`vm_http_requests_total{path="/api/v1/metadata"}`) diff --git a/app/vmselect/netstorage/netstorage.go b/app/vmselect/netstorage/netstorage.go index e572be55e..8c51d31d9 100644 --- a/app/vmselect/netstorage/netstorage.go +++ b/app/vmselect/netstorage/netstorage.go @@ -473,6 +473,33 @@ func GetLabelsOnTimeRange(tr storage.TimeRange, deadline searchutils.Deadline) ( return labels, nil } +// GetGraphiteTags returns Graphite tags until the given deadline. +func GetGraphiteTags(limit int, deadline searchutils.Deadline) ([]string, error) { + if deadline.Exceeded() { + return nil, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String()) + } + if limit <= 0 { + limit = *maxTagKeysPerSearch + } + if limit > *maxTagKeysPerSearch { + return nil, fmt.Errorf("limit=%d exceeds -search.maxTagKeys=%d; either decrease limit or increase -search.maxTagKeys command-line flag value", + limit, *maxTagKeysPerSearch) + } + labels, err := vmstorage.SearchTagKeys(limit, deadline.Deadline()) + if err != nil { + return nil, fmt.Errorf("error during tags search: %w", err) + } + // Substitute "" with "name" for Graphite compatibility + for i := range labels { + if labels[i] == "" { + labels[i] = "name" + } + } + // Sort labels like Graphite does + sort.Strings(labels) + return labels, nil +} + // GetLabels returns labels until the given deadline. func GetLabels(deadline searchutils.Deadline) ([]string, error) { if deadline.Exceeded() { diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index e316e46ad..2f0385969 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -548,6 +548,7 @@ VictoriaMetrics supports the following handlers from [Graphite Tags API](https:/ * [/tags/tagSeries](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb) * [/tags/tagMultiSeries](https://graphite.readthedocs.io/en/stable/tags.html#adding-series-to-the-tagdb) +* [/tags](https://graphite.readthedocs.io/en/stable/tags.html#exploring-tags) ### How to build from sources