Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-12 12:46:23 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

lib/storage: fix Graphite wildcard matching, which has been broken in v1.36.0

Browse Source
This commit is contained in:
Aliaksandr Valialkin 2020-05-28 11:58:47 +03:00
parent b0131c79b6
commit eca1afdc20
3 changed files with 61 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
lib/storage/index_db_test.go
View File

@ -7,6 +7,7 @@ import (
"math/rand"
"os"
"reflect"
"regexp"
"sync/atomic"
"testing"
"time"
@ -907,6 +908,27 @@ func testIndexDBCheckTSIDByName(db *indexDB, mns []MetricName, tsids []TSID, isC
return fmt.Errorf("unexpected tsid found for exact negative filter\ntsid=%+v\ntsidsFound=%+v\ntfs=%s\nmn=%s", tsid, tsidsFound, tfs, mn)
}
// Search for Graphite wildcard
tfs.Reset(mn.AccountID, mn.ProjectID)
n := bytes.IndexByte(mn.MetricGroup, '.')
if n < 0 {
return fmt.Errorf("cannot find dot in MetricGroup %q", mn.MetricGroup)
}
re := "[^.]*" + regexp.QuoteMeta(string(mn.MetricGroup[n:]))
if err := tfs.Add(nil, []byte(re), false, true); err != nil {
return fmt.Errorf("cannot create regexp tag filter for Graphite wildcard")
}
if tfsNew := tfs.Finalize(); len(tfsNew) > 0 {
return fmt.Errorf("unexpected non-empty tag filters returned by TagFilters.Finalize: %v", tfsNew)
}
tsidsFound, err = db.searchTSIDs([]*TagFilters{tfs}, TimeRange{}, 1e5)
if err != nil {
return fmt.Errorf("cannot search by regexp tag filter for Graphite wildcard: %s", err)
}
if !testHasTSID(tsidsFound, tsid) {
return fmt.Errorf("tsids is missing in regexp for Graphite wildcard tsidsFound\ntsid=%+v\ntsidsFound=%+v\ntfs=%s\nmn=%s", tsid, tsidsFound, tfs, mn)
}
// Search with regexps.
tfs.Reset(mn.AccountID, mn.ProjectID)
if err := tfs.Add(nil, mn.MetricGroup, false, true); err != nil {

29
lib/storage/tag_filters.go
View File

@ -66,8 +66,9 @@ func (tfs *TagFilters) Add(key, value []byte, isNegative, isRegexp bool) error {
return fmt.Errorf("cannot initialize tagFilter: %s", err)
}
if len(tf.graphiteReverseSuffix) > 0 {
tf = tfs.addTagFilter()
if err := tf.Init(tfs.commonPrefix, graphiteReverseTagKey, tf.graphiteReverseSuffix, false, false); err != nil {
re := regexp.QuoteMeta(string(tf.graphiteReverseSuffix)) + ".*"
tfNew := tfs.addTagFilter()
if err := tfNew.Init(tfs.commonPrefix, graphiteReverseTagKey, []byte(re), false, true); err != nil {
return fmt.Errorf("cannot initialize reverse tag filter for Graphite wildcard: %s", err)
}
}
@ -414,7 +415,8 @@ func getOptimizedReMatchFunc(reMatch func(b []byte) bool, expr string) (func(b [
}
if matchFunc, literalSuffix := getOptimizedReMatchFuncExt(reMatch, sre); matchFunc != nil {
// Found optimized function for matching the expr.
return matchFunc, literalSuffix
suffixUnescaped := tagCharsReverseRegexpEscaper.Replace(literalSuffix)
return matchFunc, suffixUnescaped
}
// Fall back to un-optimized reMatch.
return reMatch, ""
@ -685,12 +687,21 @@ func getOrValuesExt(sre *syntax.Regexp) []string {
const maxOrValues = 20
var tagCharsRegexpEscaper = strings.NewReplacer(
"\\x00", "(?:\\x000)", // escapeChar
"\x00", "(?:\\x000)", // escapeChar
"\\x01", "(?:\\x001)", // tagSeparatorChar
"\x01", "(?:\\x001)", // tagSeparatorChar
"\\x02", "(?:\\x002)", // kvSeparatorChar
"\x02", "(?:\\x002)", // kvSeparatorChar
"\\x00", "\\x000", // escapeChar
"\x00", "\\x000", // escapeChar
"\\x01", "\\x001", // tagSeparatorChar
"\x01", "\\x001", // tagSeparatorChar
"\\x02", "\\x002", // kvSeparatorChar
"\x02", "\\x002", // kvSeparatorChar
)
var tagCharsReverseRegexpEscaper = strings.NewReplacer(
"\\x000", "\x00", // escapeChar
"\x000", "\x00", // escapeChar
"\\x001", "\x01", // tagSeparatorChar
"\x001", "\x01", // tagSeparatorChar
"\\x002", "\x02", // kvSeparatorChar
"\x002", "\x02", // kvSeparatorChar
)
func getMaxRegexpCacheSize() int {

19
lib/storage/tag_filters_test.go
View File

@ -579,6 +579,25 @@ func TestGetRegexpPrefix(t *testing.T) {
f(t, "(foo|bar$)x*", "", "(?:foo|bar(?-m:$))x*")
}
func TestTagFiltersString(t *testing.T) {
tfs := NewTagFilters(12, 34)
mustAdd := func(key, value string, isNegative, isRegexp bool) {
t.Helper()
if err := tfs.Add([]byte(key), []byte(value), isNegative, isRegexp); err != nil {
t.Fatalf("unexpected error: %s", err)
}
}
mustAdd("", "metric_name", false, false)
mustAdd("tag_re", "re.value", false, true)
mustAdd("tag_nre", "nre.value", true, true)
mustAdd("tag_n", "n_value", true, false)
s := tfs.String()
sExpected := `AccountID=12, ProjectID=34 {__name__="metric_name", tag_re=~"re.value", tag_nre!~"nre.value", tag_n!="n_value"}`
if s != sExpected {
t.Fatalf("unexpected TagFilters.String(); got %q; want %q", s, sExpected)
}
}
func TestTagFiltersAddEmpty(t *testing.T) {
tfs := NewTagFilters(0, 0)