From d15d036a5a143244244b8e9275d4629d8c54e148 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 23 Sep 2021 21:48:48 +0300 Subject: [PATCH] lib/storage: properly handle `{__name__=~"prefix(suffix1|suffix2)",other_label="..."}` queries They were broken in the commit 00cbb099b6444c2e285ef32ffce7b794186bc0be Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1644 --- docs/CHANGELOG.md | 1 + lib/storage/tag_filters.go | 21 ++++++++-- lib/storage/tag_filters_test.go | 71 +++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 81c3b0ffd8..0cfd6bfc22 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,6 +9,7 @@ sort: 15 * FEATURE: vmagent: add `vm_promscrape_max_scrape_size_exceeded_errors_total` metric for counting of the failed scrapes due to the exceeded response size (the response size limit can be configured via `-promscrape.maxScrapeSize` command-line flag). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1639). * BUGFIX: vmalert: properly reload rule groups if only the `interval` config option is changed. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1641). +* BUGFIX: properly handle `{__name__=~"prefix(suffix1|suffix2)",other_label="..."}` queries. They may return unexpected empty responses since v1.66.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1644). ## [v1.66.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.66.1) diff --git a/lib/storage/tag_filters.go b/lib/storage/tag_filters.go index af3a869255..72df2c049a 100644 --- a/lib/storage/tag_filters.go +++ b/lib/storage/tag_filters.go @@ -30,6 +30,7 @@ func convertToCompositeTagFilters(tfs *TagFilters) []*TagFilters { var tfssCompiled []*TagFilters // Search for filters on metric name, which will be used for creating composite filters. var names [][]byte + namePrefix := "" hasPositiveFilter := false for _, tf := range tfs.tfs { if len(tf.key) == 0 { @@ -43,6 +44,7 @@ func convertToCompositeTagFilters(tfs *TagFilters) []*TagFilters { for _, orSuffix := range tf.orSuffixes { names = append(names, []byte(orSuffix)) } + namePrefix = tf.regexpPrefix } } else if !tf.isNegative && !tf.isEmptyMatch { hasPositiveFilter = true @@ -54,7 +56,7 @@ func convertToCompositeTagFilters(tfs *TagFilters) []*TagFilters { } // Create composite filters for the found names. - var compositeKey []byte + var compositeKey, nameWithPrefix []byte for _, name := range names { compositeFilters := 0 tfsNew := make([]tagFilter, 0, len(tfs.tfs)) @@ -95,7 +97,9 @@ func convertToCompositeTagFilters(tfs *TagFilters) []*TagFilters { continue } // Create composite filter on (name, tf) - compositeKey = marshalCompositeTagKey(compositeKey[:0], name, tf.key) + nameWithPrefix = append(nameWithPrefix[:0], namePrefix...) + nameWithPrefix = append(nameWithPrefix, name...) + compositeKey = marshalCompositeTagKey(compositeKey[:0], nameWithPrefix, tf.key) var tfNew tagFilter if err := tfNew.Init(tfs.commonPrefix, compositeKey, tf.value, tf.isNegative, tf.isRegexp); err != nil { logger.Panicf("BUG: unexpected error when creating composite tag filter for name=%q and key=%q: %s", name, tf.key, err) @@ -239,13 +243,18 @@ type tagFilter struct { // matchCost is a cost for matching a filter against a single string. matchCost uint64 + // contains the prefix for regexp filter if isRegexp==true. + regexpPrefix string + // Prefix always contains {nsPrefixTagToMetricIDs, AccountID, ProjectID, key}. // Additionally it contains: // - value if !isRegexp. - // - non-regexp prefix if isRegexp. + // - regexpPrefix if isRegexp. prefix []byte - // or values obtained from regexp suffix if it equals to "foo|bar|..." + // `or` values obtained from regexp suffix if it equals to "foo|bar|..." + // + // the regexp prefix is stored in regexpPrefix. // // This array is also populated with matching Graphite metrics if key="__graphite__" orSuffixes []string @@ -350,6 +359,7 @@ func (tf *tagFilter) InitFromGraphiteQuery(commonPrefix, query []byte, paths []s tf.value = append(tf.value[:0], query...) tf.isNegative = isNegative tf.isRegexp = true // this is needed for tagFilter.matchSuffix + tf.regexpPrefix = prefix tf.prefix = append(tf.prefix[:0], commonPrefix...) tf.prefix = marshalTagValue(tf.prefix, nil) tf.prefix = marshalTagValueNoTrailingTagSeparator(tf.prefix, []byte(prefix)) @@ -395,6 +405,7 @@ func (tf *tagFilter) Init(commonPrefix, key, value []byte, isNegative, isRegexp tf.isRegexp = isRegexp tf.matchCost = 0 + tf.regexpPrefix = "" tf.prefix = tf.prefix[:0] tf.orSuffixes = tf.orSuffixes[:0] @@ -412,6 +423,8 @@ func (tf *tagFilter) Init(commonPrefix, key, value []byte, isNegative, isRegexp if len(expr) == 0 { tf.value = append(tf.value[:0], prefix...) tf.isRegexp = false + } else { + tf.regexpPrefix = string(prefix) } } tf.prefix = marshalTagValueNoTrailingTagSeparator(tf.prefix, prefix) diff --git a/lib/storage/tag_filters_test.go b/lib/storage/tag_filters_test.go index ca24c4245a..dc128e2c70 100644 --- a/lib/storage/tag_filters_test.go +++ b/lib/storage/tag_filters_test.go @@ -418,6 +418,44 @@ func TestConvertToCompositeTagFilters(t *testing.T) { }, }) + // Multiple values regexp filter, which can be converted to non-regexp. + f([]TagFilter{ + { + Key: nil, + Value: []byte("bar|foo"), + IsNegative: false, + IsRegexp: true, + }, + }, [][]TagFilter{ + { + { + Key: nil, + Value: []byte("bar|foo"), + IsNegative: false, + IsRegexp: true, + }, + }, + }) + + // Multiple values regexp filter with common prefix, which can be converted to non-regexp. + f([]TagFilter{ + { + Key: nil, + Value: []byte("xxx(bar|foo)"), + IsNegative: false, + IsRegexp: true, + }, + }, [][]TagFilter{ + { + { + Key: nil, + Value: []byte("xxx(bar|foo)"), + IsNegative: false, + IsRegexp: true, + }, + }, + }) + // Multiple values regexp filter, which can be converted to non-regexp, with non-name filter. f([]TagFilter{ { @@ -451,6 +489,39 @@ func TestConvertToCompositeTagFilters(t *testing.T) { }, }) + // Multiple values regexp filter with common prefix, which can be converted to non-regexp, with non-name filter. + f([]TagFilter{ + { + Key: nil, + Value: []byte("xxx(bar|foox)"), + IsNegative: false, + IsRegexp: true, + }, + { + Key: []byte("foo"), + Value: []byte("abc"), + IsNegative: false, + IsRegexp: false, + }, + }, [][]TagFilter{ + { + { + Key: []byte("\xfe\x06xxxbarfoo"), + Value: []byte("abc"), + IsNegative: false, + IsRegexp: false, + }, + }, + { + { + Key: []byte("\xfe\x07xxxfooxfoo"), + Value: []byte("abc"), + IsNegative: false, + IsRegexp: false, + }, + }, + }) + // Two multiple values regexp filter, which can be converted to non-regexp, with non-name filter. f([]TagFilter{ {