From acf1a2c72b05e94668523c84eb7ae70fa03e4297 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 13 Jan 2021 10:57:34 +0200 Subject: [PATCH] app/vmselect/promql: properly parse escaped multibyte utf8 code sequences in metric names and labels names See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/990 --- app/vmselect/promql/parser_test.go | 1 + docs/CHANGELOG.md | 1 + go.mod | 2 +- go.sum | 4 +-- .../VictoriaMetrics/metricsql/lexer.go | 31 ++++++++++++------- vendor/modules.txt | 2 +- 6 files changed, 26 insertions(+), 15 deletions(-) diff --git a/app/vmselect/promql/parser_test.go b/app/vmselect/promql/parser_test.go index 6d80e45b4..cd079c03b 100644 --- a/app/vmselect/promql/parser_test.go +++ b/app/vmselect/promql/parser_test.go @@ -24,6 +24,7 @@ func TestParseMetricSelectorSuccess(t *testing.T) { f(`foo {bar != "baz"}`) f(` foo { bar !~ "^ddd(x+)$", a="ss", __name__="sffd"} `) f(`(foo)`) + f(`\п\р\и\в\е\т{\ы="111"}`) } func TestParseMetricSelectorError(t *testing.T) { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4ca2aa7a1..30b414d25 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,6 +8,7 @@ * FEATURE: enforce at least TLS v1.2 when accepting HTTPS requests if `-tls`, `-tlsCertFile` and `-tlsKeyFile` command-line flags are set, because older TLS protocols such as v1.0 and v1.1 have been deprecated due to security vulnerabilities. * FEATURE: support `extra_label` query arg for all HTTP-based [data ingestion protocols](https://victoriametrics.github.io/#how-to-import-time-series-data). This query arg can be used for specifying extra labels which should be added for the ingested data. +* BUGFIX: properly parse escaped unicode chars in MetricsQL metric names, label names and function names. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/990 * BUGFIX: override user-provided labels with labels set in `extra_label` query args during data ingestion over HTTP-based protocols. * BUGFIX: vmagent: prevent from `dialing to the given TCP address time out` error when scraping big number of unavailable targets. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/987 * BUGFIX: vmagent: properly show scrape duration on `/targets` page. Previously it was incorrectly shown as 0.000s. diff --git a/go.mod b/go.mod index 52c92b999..aabe762ba 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( // like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b github.com/VictoriaMetrics/fasthttp v1.0.11 github.com/VictoriaMetrics/metrics v1.12.3 - github.com/VictoriaMetrics/metricsql v0.9.2 + github.com/VictoriaMetrics/metricsql v0.10.0 github.com/aws/aws-sdk-go v1.36.23 github.com/cespare/xxhash/v2 v2.1.1 github.com/golang/snappy v0.0.2 diff --git a/go.sum b/go.sum index 0c736ca10..ef87f3eeb 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.11/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrt github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= github.com/VictoriaMetrics/metrics v1.12.3 h1:Fe6JHC6MSEKa+BtLhPN8WIvS+HKPzMc2evEpNeCGy7I= github.com/VictoriaMetrics/metrics v1.12.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE= -github.com/VictoriaMetrics/metricsql v0.9.2 h1:16emP9IXVUrY6aai3P+AFakGJ92rCDomD7uCy1fToo0= -github.com/VictoriaMetrics/metricsql v0.9.2/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= +github.com/VictoriaMetrics/metricsql v0.10.0 h1:45BARAP2shaL/5p67Hvz+YrWUbr0X0VCy9t+gvdIm8o= +github.com/VictoriaMetrics/metricsql v0.10.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= diff --git a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go index c429a2fca..5a95d133a 100644 --- a/vendor/github.com/VictoriaMetrics/metricsql/lexer.go +++ b/vendor/github.com/VictoriaMetrics/metricsql/lexer.go @@ -4,6 +4,8 @@ import ( "fmt" "strconv" "strings" + "unicode" + "unicode/utf8" ) type lexer struct { @@ -220,13 +222,12 @@ func scanIdent(s string) string { if s[i] != '\\' { break } + i++ // Do not verify the next char, since it is escaped. - i += 2 - if i > len(s) { - i-- - break - } + // The next char may be encoded as multi-byte UTF8 sequence. See https://en.wikipedia.org/wiki/UTF-8#Encoding + _, size := utf8.DecodeRuneInString(s[i:]) + i += size } if i == 0 { panic("BUG: scanIdent couldn't find a single ident char; make sure isIdentPrefix called before scanIdent") @@ -257,8 +258,10 @@ func unescapeIdent(s string) string { s = s[1:] } } else { - dst = append(dst, s[0]) - s = s[1:] + // UTF8 char. See https://en.wikipedia.org/wiki/UTF-8#Encoding + _, size := utf8.DecodeRuneInString(s) + dst = append(dst, s[:size]...) + s = s[size:] } n = strings.IndexByte(s, '\\') if n < 0 { @@ -298,12 +301,18 @@ func appendEscapedIdent(dst []byte, s string) []byte { } else { dst = append(dst, ch) } - } else if ch >= 0x20 && ch < 0x7f { - // Leave ASCII printable chars as is - dst = append(dst, '\\', ch) + continue + } + + // escape ch + dst = append(dst, '\\') + r, size := utf8.DecodeRuneInString(s[i:]) + if r != utf8.RuneError && unicode.IsPrint(r) { + dst = append(dst, s[i:i+size]...) + i += size - 1 } else { // hex-encode non-printable chars - dst = append(dst, '\\', 'x', toHex(ch>>4), toHex(ch&0xf)) + dst = append(dst, 'x', toHex(ch>>4), toHex(ch&0xf)) } } return dst diff --git a/vendor/modules.txt b/vendor/modules.txt index af5f5b173..446ddd216 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -16,7 +16,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil github.com/VictoriaMetrics/fasthttp/stackless # github.com/VictoriaMetrics/metrics v1.12.3 github.com/VictoriaMetrics/metrics -# github.com/VictoriaMetrics/metricsql v0.9.2 +# github.com/VictoriaMetrics/metricsql v0.10.0 github.com/VictoriaMetrics/metricsql github.com/VictoriaMetrics/metricsql/binaryop # github.com/aws/aws-sdk-go v1.36.23