Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-15 00:13:30 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

app/vmselect/graphite: apply filter then limit

Browse Source
This commit is contained in:
Aliaksandr Valialkin 2020-11-16 03:58:12 +02:00
parent 5889273920
commit 97100b1d42
2 changed files with 59 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
app/vmselect/graphite/tags_api.go
View File

@ -3,7 +3,6 @@ package graphite
import (
"fmt"
"net/http"
"regexp"
"strconv"
"time"
@ -22,26 +21,16 @@ func TagValuesHandler(startTime time.Time, at *auth.Token, tagName string, w htt
if err := r.ParseForm(); err != nil {
return fmt.Errorf("cannot parse form values: %w", err)
}
limit := 0
if limitStr := r.FormValue("limit"); len(limitStr) > 0 {
var err error
limit, err = strconv.Atoi(limitStr)
if err != nil {
return fmt.Errorf("cannot parse limit=%q: %w", limit, err)
}
}
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
tagValues, isPartial, err := netstorage.GetGraphiteTagValues(at, denyPartialResponse, tagName, limit, deadline)
limit, err := getInt(r, "limit")
if err != nil {
return err
}
filter := r.FormValue("filter")
if len(filter) > 0 {
tagValues, err = applyRegexpFilter(filter, tagValues)
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
tagValues, isPartial, err := netstorage.GetGraphiteTagValues(at, denyPartialResponse, tagName, filter, limit, deadline)
if err != nil {
return err
}
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
bw := bufferedwriter.Get(w)
@ -64,26 +53,16 @@ func TagsHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *
if err := r.ParseForm(); err != nil {
return fmt.Errorf("cannot parse form values: %w", err)
}
limit := 0
if limitStr := r.FormValue("limit"); len(limitStr) > 0 {
var err error
limit, err = strconv.Atoi(limitStr)
if err != nil {
return fmt.Errorf("cannot parse limit=%q: %w", limit, err)
}
}
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
labels, isPartial, err := netstorage.GetGraphiteTags(at, denyPartialResponse, limit, deadline)
limit, err := getInt(r, "limit")
if err != nil {
return err
}
filter := r.FormValue("filter")
if len(filter) > 0 {
labels, err = applyRegexpFilter(filter, labels)
denyPartialResponse := searchutils.GetDenyPartialResponse(r)
labels, isPartial, err := netstorage.GetGraphiteTags(at, denyPartialResponse, filter, limit, deadline)
if err != nil {
return err
}
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
bw := bufferedwriter.Get(w)
@ -98,19 +77,14 @@ func TagsHandler(startTime time.Time, at *auth.Token, w http.ResponseWriter, r *
var tagsDuration = metrics.NewSummary(`vm_request_duration_seconds{path="/tags"}`)
func applyRegexpFilter(filter string, ss []string) ([]string, error) {
// Anchor filter regexp to the beginning of the string as Graphite does.
// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/localdatabase.py#L157
filter = "^(?:" + filter + ")"
re, err := regexp.Compile(filter)
func getInt(r *http.Request, argName string) (int, error) {
argValue := r.FormValue(argName)
if len(argValue) == 0 {
return 0, nil
}
n, err := strconv.Atoi(argValue)
if err != nil {
return nil, fmt.Errorf("cannot parse regexp filter=%q: %w", filter, err)
return 0, fmt.Errorf("cannot parse %q=%q: %w", argName, argValue, err)
}
dst := ss[:0]
for _, s := range ss {
if re.MatchString(s) {
dst = append(dst, s)
}
}
return dst, nil
return n, nil
}

50
app/vmselect/netstorage/netstorage.go
View File

@ -6,6 +6,7 @@ import (
"fmt"
"io"
"net/http"
"regexp"
"runtime"
"sort"
"strings"
@ -558,22 +559,30 @@ func GetLabelsOnTimeRange(at *auth.Token, denyPartialResponse bool, tr storage.T
}
// GetGraphiteTags returns Graphite tags until the given deadline.
func GetGraphiteTags(at *auth.Token, denyPartialResponse bool, limit int, deadline searchutils.Deadline) ([]string, bool, error) {
func GetGraphiteTags(at *auth.Token, denyPartialResponse bool, filter string, limit int, deadline searchutils.Deadline) ([]string, bool, error) {
if deadline.Exceeded() {
return nil, false, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
}
labels, isPartial, err := GetLabels(at, denyPartialResponse, deadline)
if err != nil {
return nil, false, err
}
if limit < len(labels) {
labels = labels[:limit]
if len(filter) > 0 {
labels, err = applyGraphiteRegexpFilter(filter, labels)
if err != nil {
return nil, false, err
}
// Convert __name__ to name in labels according to Graphite tags specs.
// See https://graphite.readthedocs.io/en/stable/tags.html#querying
for i, label := range labels {
if label == "__name__" {
}
// Substitute "__name__" with "name" for Graphite compatibility
for i := range labels {
if labels[i] == "__name__" {
labels[i] = "name"
break
}
}
if limit > 0 && limit < len(labels) {
labels = labels[:limit]
}
return labels, isPartial, nil
}
@ -714,7 +723,7 @@ func GetLabelValuesOnTimeRange(at *auth.Token, denyPartialResponse bool, labelNa
}
// GetGraphiteTagValues returns tag values for the given tagName until the given deadline.
func GetGraphiteTagValues(at *auth.Token, denyPartialResponse bool, tagName string, limit int, deadline searchutils.Deadline) ([]string, bool, error) {
func GetGraphiteTagValues(at *auth.Token, denyPartialResponse bool, tagName, filter string, limit int, deadline searchutils.Deadline) ([]string, bool, error) {
if deadline.Exceeded() {
return nil, false, fmt.Errorf("timeout exceeded before starting the query processing: %s", deadline.String())
}
@ -725,7 +734,13 @@ func GetGraphiteTagValues(at *auth.Token, denyPartialResponse bool, tagName stri
if err != nil {
return nil, false, err
}
if limit < len(tagValues) {
if len(filter) > 0 {
tagValues, err = applyGraphiteRegexpFilter(filter, tagValues)
if err != nil {
return nil, false, err
}
}
if limit > 0 && limit < len(tagValues) {
tagValues = tagValues[:limit]
}
return tagValues, isPartial, nil
@ -2263,3 +2278,20 @@ var (
// The maximum number of concurrent queries per storageNode.
const maxConcurrentQueriesPerStorageNode = 100
func applyGraphiteRegexpFilter(filter string, ss []string) ([]string, error) {
// Anchor filter regexp to the beginning of the string as Graphite does.
// See https://github.com/graphite-project/graphite-web/blob/3ad279df5cb90b211953e39161df416e54a84948/webapp/graphite/tags/localdatabase.py#L157
filter = "^(?:" + filter + ")"
re, err := regexp.Compile(filter)
if err != nil {
return nil, fmt.Errorf("cannot parse regexp filter=%q: %w", filter, err)
}
dst := ss[:0]
for _, s := range ss {
if re.MatchString(s) {
dst = append(dst, s)
}
}
return dst, nil
}