lib/logstorage: LogsQL: replace exact_prefix("...") with exact("..."*)

This makes LogsQL queries more consistent with i("...") and i("..."*) syntax
This commit is contained in:
Aliaksandr Valialkin 2023-07-17 17:19:41 -07:00
parent 5ace0701d3
commit 8fdfd13a29
No known key found for this signature in database
GPG Key ID: A72BEC6CD3D0DED1
4 changed files with 46 additions and 59 deletions

View File

@ -589,11 +589,11 @@ See also:
### Exact prefix filter
Sometimes it is needed to find log messages starting with some prefix. This can be done with the `exact_prefix(...)` filter.
Sometimes it is needed to find log messages starting with some prefix. This can be done with the `exact("prefix"*)` filter.
For example, the following query matches log messages, which start from `Processing request` prefix:
```logsql
exact_prefix("Processing request")
exact("Processing request"*)
```
This filter matches the following [log messages](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field):
@ -603,30 +603,30 @@ This filter matches the following [log messages](https://docs.victoriametrics.co
It doesn't match the following log messages:
- `processing request foobar`, since the log message starts with lowercase `p`. Use `exact_prefix("processing request") OR exact_prefix("Processing request")`
- `processing request foobar`, since the log message starts with lowercase `p`. Use `exact("processing request"*) OR exact("Processing request"*)`
query in this case. See [these docs](#logical-filter) for details.
- `start: Processing request`, since the log message doesn't start with `Processing request`. Use `"Processing request"` query in this case.
See [these docs](#phrase-filter) for details.
By default the `exact_prefix()` filter is applied to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
Specify the [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the `exact_prefix()` filter and put a colon after it
By default the `exact()` filter is applied to the [`_msg` field](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#message-field).
Specify the [field name](https://docs.victoriametrics.com/VictoriaLogs/keyConcepts.html#data-model) in front of the `exact()` filter and put a colon after it
if it must be searched in the given field. For example, the following query returns log entries with `log.level` field, which starts with `err` prefix:
```logsql
log.level:exact_prefix("err")
log.level:exact("err"*)
```
Both the field name and the phrase can contain arbitrary [utf-8](https://en.wikipedia.org/wiki/UTF-8)-encoded chars. For example:
```logsql
log.уровень:exact_prefix("ошиб")
log.уровень:exact("ошиб"*)
```
The field name can be put inside quotes if it contains special chars, which may clash with the query syntax.
For example, the following query matches `log:level` values starting with `err` prefix:
```logsql
"log:level":exact_prefix("err")
"log:level":exact("err"*)
```
See also:
@ -809,7 +809,7 @@ Performance tips:
Note that the `re("error|warning")` matches `errors` as well as `warnings` [words](#word), while `error OR warning` matches
only the specified [words](#word). See also [multi-exact filter](#multi-exact-filter).
- Prefer moving the regexp filter to the end of the [logical filter](#logical-filter), so lightweighter filters are executed first.
- Prefer using `exact_prefix("some prefix")` instead of `re("^some prefix")`, since the [exact_prefix()](#exact-prefix-filter) works much faster than the `re()` filter.
- Prefer using `exact("some prefix"*)` instead of `re("^some prefix")`, since the [exact()](#exact-prefix-filter) works much faster than the `re()` filter.
- See [other performance tips](#performance-tips).
See also:

View File

@ -481,7 +481,7 @@ func (sf *sequenceFilter) apply(bs *blockSearch, bm *filterBitmap) {
// exactPrefixFilter matches the exact prefix.
//
// Example LogsQL: `fieldName:exact_prefix("foo bar")
// Example LogsQL: `fieldName:exact("foo bar"*)
type exactPrefixFilter struct {
fieldName string
prefix string
@ -491,7 +491,7 @@ type exactPrefixFilter struct {
}
func (ef *exactPrefixFilter) String() string {
return fmt.Sprintf("%sexact_prefix(%s)", quoteFieldNameIfNeeded(ef.fieldName), quoteTokenIfNeeded(ef.prefix))
return fmt.Sprintf("%sexact(%s*)", quoteFieldNameIfNeeded(ef.fieldName), quoteTokenIfNeeded(ef.prefix))
}
func (ef *exactPrefixFilter) getTokens() []string {

View File

@ -320,8 +320,6 @@ func parseGenericFilter(lex *lexer, fieldName string) (filter, error) {
return parseNotFilter(lex, fieldName)
case lex.isKeyword("exact"):
return parseExactFilter(lex, fieldName)
case lex.isKeyword("exact_prefix"):
return parseExactPrefixFilter(lex, fieldName)
case lex.isKeyword("i"):
return parseAnyCaseFilter(lex, fieldName)
case lex.isKeyword("in"):
@ -474,28 +472,7 @@ func parseNotFilter(lex *lexer, fieldName string) (filter, error) {
}
func parseAnyCaseFilter(lex *lexer, fieldName string) (filter, error) {
phrase := lex.token
lex.nextToken()
if !lex.isKeyword("(") {
phrase += getCompoundSuffix(lex, fieldName)
return parseFilterForPhrase(lex, phrase, fieldName)
}
if !lex.mustNextToken() {
return nil, fmt.Errorf("missing arg for i()")
}
phrase = getCompoundFuncArg(lex)
isPrefixFilter := false
if lex.isKeyword("*") && !lex.isSkippedSpace {
isPrefixFilter = true
if !lex.mustNextToken() {
return nil, fmt.Errorf("missing ')' after i()")
}
}
if !lex.isKeyword(")") {
return nil, fmt.Errorf("unexpected token %q instead of ')' in i()", lex.token)
}
lex.nextToken()
return parseFuncArgMaybePrefix(lex, "i", fieldName, func(phrase string, isPrefixFilter bool) (filter, error) {
if isPrefixFilter {
f := &anyCasePrefixFilter{
fieldName: fieldName,
@ -508,6 +485,32 @@ func parseAnyCaseFilter(lex *lexer, fieldName string) (filter, error) {
phrase: phrase,
}
return f, nil
})
}
func parseFuncArgMaybePrefix(lex *lexer, funcName, fieldName string, f func(arg string, isPrefiFilter bool) (filter, error)) (filter, error) {
phrase := lex.token
lex.nextToken()
if !lex.isKeyword("(") {
phrase += getCompoundSuffix(lex, fieldName)
return parseFilterForPhrase(lex, phrase, fieldName)
}
if !lex.mustNextToken() {
return nil, fmt.Errorf("missing arg for %s()", funcName)
}
phrase = getCompoundFuncArg(lex)
isPrefixFilter := false
if lex.isKeyword("*") && !lex.isSkippedSpace {
isPrefixFilter = true
if !lex.mustNextToken() {
return nil, fmt.Errorf("missing ')' after %s()", funcName)
}
}
if !lex.isKeyword(")") {
return nil, fmt.Errorf("unexpected token %q instead of ')' in %s()", lex.token, funcName)
}
lex.nextToken()
return f(phrase, isPrefixFilter)
}
func parseLenRangeFilter(lex *lexer, fieldName string) (filter, error) {
@ -624,22 +627,19 @@ func parseSequenceFilter(lex *lexer, fieldName string) (filter, error) {
}
func parseExactFilter(lex *lexer, fieldName string) (filter, error) {
return parseFuncArg(lex, fieldName, func(arg string) (filter, error) {
ef := &exactFilter{
return parseFuncArgMaybePrefix(lex, "exact", fieldName, func(phrase string, isPrefixFilter bool) (filter, error) {
if isPrefixFilter {
f := &exactPrefixFilter{
fieldName: fieldName,
value: arg,
prefix: phrase,
}
return ef, nil
})
}
func parseExactPrefixFilter(lex *lexer, fieldName string) (filter, error) {
return parseFuncArg(lex, fieldName, func(arg string) (filter, error) {
ef := &exactPrefixFilter{
return f, nil
}
f := &exactFilter{
fieldName: fieldName,
prefix: arg,
value: phrase,
}
return ef, nil
return f, nil
})
}
@ -1082,7 +1082,6 @@ var reservedKeywords = func() map[string]struct{} {
// functions
"exact",
"exact_prefix",
"i",
"in",
"ipv4_range",

View File

@ -627,12 +627,6 @@ func TestParseQuerySuccess(t *testing.T) {
f("a:exact", `a:"exact"`)
f("a:exact-foo", `a:exact-foo`)
f("exact-foo:b", `exact-foo:b`)
f("exact_prefix", `"exact_prefix"`)
f("exact_prefix:a", `"exact_prefix":a`)
f("exact_prefix-foo", `exact_prefix-foo`)
f("a:exact_prefix", `a:"exact_prefix"`)
f("a:exact_prefix-foo", `a:exact_prefix-foo`)
f("exact_prefix-foo:b", `exact_prefix-foo:b`)
f("i", `"i"`)
f("i-foo", `i-foo`)
f("a:i-foo", `a:i-foo`)
@ -676,17 +670,11 @@ func TestParseQuerySuccess(t *testing.T) {
// exact filter
f("exact(foo)", `exact(foo)`)
f("exact(foo*)", `exact(foo*)`)
f("exact('foo bar),|baz')", `exact("foo bar),|baz")`)
f(`exact(foo-bar,)`, `exact(foo-bar)`)
f("exact('foo bar),|baz'*)", `exact("foo bar),|baz"*)`)
f(`exact(foo|b:ar)`, `exact("foo|b:ar")`)
f(`foo:exact(f,)`, `foo:exact(f)`)
// exact_prefix filter
f("exact_prefix(foo)", `exact_prefix(foo)`)
f(`exact_prefix("foo bar")`, `exact_prefix("foo bar")`)
f(`exact_prefix(foo-bar,)`, `exact_prefix(foo-bar)`)
f(`exact_prefix(foo|b:ar)`, `exact_prefix("foo|b:ar")`)
f(`foo:exact_prefix(f,)`, `foo:exact_prefix(f)`)
f(`foo:exact(foo|b:ar*)`, `foo:exact("foo|b:ar"*)`)
// i filter
f("i(foo)", `i(foo)`)
@ -877,9 +865,9 @@ func TestParseQueryFailure(t *testing.T) {
f(`exact(f, b)`)
f(`exact(foo`)
f(`exact(foo,`)
f(`exact(foo*)`)
f(`exact(foo bar)`)
f(`exact(foo, bar`)
f(`exact(foo,)`)
// invalid i
f(`i(`)