app/vmselect/promql: properly parse escaped multibyte utf8 code sequences in metric names and labels names

See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/990

app/vmselect/promql/parser_test.go

@@ -24,6 +24,7 @@ func TestParseMetricSelectorSuccess(t *testing.T) {
 	f(`foo {bar != "baz"}`)
 	f(` foo { bar !~ "^ddd(x+)$", a="ss", __name__="sffd"}  `)
 	f(`(foo)`)
+	f(`\п\р\и\в\е\т{\ы="111"}`)
 }
 
 func TestParseMetricSelectorError(t *testing.T) {

docs/CHANGELOG.md

@@ -8,6 +8,7 @@
 * FEATURE: enforce at least TLS v1.2 when accepting HTTPS requests if `-tls`, `-tlsCertFile` and `-tlsKeyFile` command-line flags are set, because older TLS protocols such as v1.0 and v1.1 have been deprecated due to security vulnerabilities.
 * FEATURE: support `extra_label` query arg for all HTTP-based [data ingestion protocols](https://victoriametrics.github.io/#how-to-import-time-series-data). This query arg can be used for specifying extra labels which should be added for the ingested data.
 
+* BUGFIX: properly parse escaped unicode chars in MetricsQL metric names, label names and function names. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/990
 * BUGFIX: override user-provided labels with labels set in `extra_label` query args during data ingestion over HTTP-based protocols.
 * BUGFIX: vmagent: prevent from `dialing to the given TCP address time out` error when scraping big number of unavailable targets. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/987
 * BUGFIX: vmagent: properly show scrape duration on `/targets` page. Previously it was incorrectly shown as 0.000s.

go.mod

@@ -9,7 +9,7 @@ require (
 	// like https://github.com/valyala/fasthttp/commit/996610f021ff45fdc98c2ce7884d5fa4e7f9199b
 	github.com/VictoriaMetrics/fasthttp v1.0.11
 	github.com/VictoriaMetrics/metrics v1.12.3
-	github.com/VictoriaMetrics/metricsql v0.9.2
+	github.com/VictoriaMetrics/metricsql v0.10.0
 	github.com/aws/aws-sdk-go v1.36.23
 	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/golang/snappy v0.0.2

go.sum

@@ -46,8 +46,8 @@ github.com/VictoriaMetrics/fasthttp v1.0.11/go.mod h1:3SeUL4zwB/p/a9aEeRc6gdlbrt
 github.com/VictoriaMetrics/metrics v1.12.2/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
 github.com/VictoriaMetrics/metrics v1.12.3 h1:Fe6JHC6MSEKa+BtLhPN8WIvS+HKPzMc2evEpNeCGy7I=
 github.com/VictoriaMetrics/metrics v1.12.3/go.mod h1:Z1tSfPfngDn12bTfZSCqArT3OPY3u88J12hSoOhuiRE=
-github.com/VictoriaMetrics/metricsql v0.9.2 h1:16emP9IXVUrY6aai3P+AFakGJ92rCDomD7uCy1fToo0=
-github.com/VictoriaMetrics/metricsql v0.9.2/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
+github.com/VictoriaMetrics/metricsql v0.10.0 h1:45BARAP2shaL/5p67Hvz+YrWUbr0X0VCy9t+gvdIm8o=
+github.com/VictoriaMetrics/metricsql v0.10.0/go.mod h1:ylO7YITho/Iw6P71oEaGyHbO94bGoGtzWfLGqFhMIg8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=

vendor/github.com/VictoriaMetrics/metricsql/lexer.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"unicode"
+	"unicode/utf8"
 )
 
 type lexer struct {
@@ -220,13 +222,12 @@ func scanIdent(s string) string {
 		if s[i] != '\\' {
 			break
 		}
+		i++
 
 		// Do not verify the next char, since it is escaped.
-		i += 2
-		if i > len(s) {
-			i--
-			break
-		}
+		// The next char may be encoded as multi-byte UTF8 sequence. See https://en.wikipedia.org/wiki/UTF-8#Encoding
+		_, size := utf8.DecodeRuneInString(s[i:])
+		i += size
 	}
 	if i == 0 {
 		panic("BUG: scanIdent couldn't find a single ident char; make sure isIdentPrefix called before scanIdent")
@@ -257,8 +258,10 @@ func unescapeIdent(s string) string {
 				s = s[1:]
 			}
 		} else {
-			dst = append(dst, s[0])
-			s = s[1:]
+			// UTF8 char. See https://en.wikipedia.org/wiki/UTF-8#Encoding
+			_, size := utf8.DecodeRuneInString(s)
+			dst = append(dst, s[:size]...)
+			s = s[size:]
 		}
 		n = strings.IndexByte(s, '\\')
 		if n < 0 {
@@ -298,12 +301,18 @@ func appendEscapedIdent(dst []byte, s string) []byte {
 			} else {
 				dst = append(dst, ch)
 			}
-		} else if ch >= 0x20 && ch < 0x7f {
-			// Leave ASCII printable chars as is
-			dst = append(dst, '\\', ch)
+			continue
+		}
+
+		// escape ch
+		dst = append(dst, '\\')
+		r, size := utf8.DecodeRuneInString(s[i:])
+		if r != utf8.RuneError && unicode.IsPrint(r) {
+			dst = append(dst, s[i:i+size]...)
+			i += size - 1
 		} else {
 			// hex-encode non-printable chars
-			dst = append(dst, '\\', 'x', toHex(ch>>4), toHex(ch&0xf))
+			dst = append(dst, 'x', toHex(ch>>4), toHex(ch&0xf))
 		}
 	}
 	return dst

vendor/modules.txt

@@ -16,7 +16,7 @@ github.com/VictoriaMetrics/fasthttp/fasthttputil
 github.com/VictoriaMetrics/fasthttp/stackless
 # github.com/VictoriaMetrics/metrics v1.12.3
 github.com/VictoriaMetrics/metrics
-# github.com/VictoriaMetrics/metricsql v0.9.2
+# github.com/VictoriaMetrics/metricsql v0.10.0
 github.com/VictoriaMetrics/metricsql
 github.com/VictoriaMetrics/metricsql/binaryop
 # github.com/aws/aws-sdk-go v1.36.23