VictoriaMetrics/lib/logstorage/values_encoder_test.go
Aliaksandr Valialkin 037652d5ae
app/vlinsert: support _time field without timezone information during data ingestion
Use local timezone of the host server in this case. The timezone can be overridden
with TZ environment variable if needed.

While at it, allow using whitespace instead of T as a delimiter between data and time
in the ingested _time field. For example, '2024-09-20 10:20:30' is now accepted
during data ingestion. This is valid ISO8601 format, which is used by some log shippers,
so it should be supported. This format is also known as SQL datetime format.

Also assume local time zone when time without timezone information is passed to querying APIs.
Previously such a time was parsed in UTC timezone. Add `Z` to the end of the time string
if the old behaviour is preferred.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6721
2024-09-26 12:49:35 +02:00

754 lines
15 KiB
Go

package logstorage
import (
"fmt"
"math"
"reflect"
"strconv"
"testing"
)
func TestValuesEncoder(t *testing.T) {
f := func(values []string, expectedValueType valueType, expectedMinValue, expectedMaxValue uint64) {
t.Helper()
ve := getValuesEncoder()
var dict valuesDict
vt, minValue, maxValue := ve.encode(values, &dict)
if vt != expectedValueType {
t.Fatalf("unexpected value type; got %d; want %d", vt, expectedValueType)
}
if minValue != expectedMinValue {
t.Fatalf("unexpected minValue; got %d; want %d", minValue, expectedMinValue)
}
if maxValue != expectedMaxValue {
t.Fatalf("unexpected maxValue; got %d; want %d", maxValue, expectedMaxValue)
}
encodedValues := append([]string{}, ve.values...)
putValuesEncoder(ve)
vd := getValuesDecoder()
if err := vd.decodeInplace(encodedValues, vt, dict.values); err != nil {
t.Fatalf("unexpected error in decodeInplace(): %s", err)
}
if len(values) == 0 {
values = []string{}
}
if !reflect.DeepEqual(values, encodedValues) {
t.Fatalf("unexpected values decoded\ngot\n%q\nwant\n%q", encodedValues, values)
}
putValuesDecoder(vd)
}
// An empty values list
f(nil, valueTypeString, 0, 0)
// string values
values := make([]string, maxDictLen+1)
for i := range values {
values[i] = fmt.Sprintf("value_%d", i)
}
f(values, valueTypeString, 0, 0)
// dict values
f([]string{"foobar"}, valueTypeDict, 0, 0)
f([]string{"foo", "bar"}, valueTypeDict, 0, 0)
f([]string{"1", "2foo"}, valueTypeDict, 0, 0)
// uint8 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1))
}
f(values, valueTypeUint8, 1, uint64(len(values)))
// uint16 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1)<<8)
}
f(values, valueTypeUint16, 1<<8, uint64(len(values))<<8)
// uint32 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1)<<16)
}
f(values, valueTypeUint32, 1<<16, uint64(len(values))<<16)
// uint64 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1)<<32)
}
f(values, valueTypeUint64, 1<<32, uint64(len(values))<<32)
// float64 values
for i := range values {
values[i] = fmt.Sprintf("%g", math.Sqrt(float64(i+1)))
}
f(values, valueTypeFloat64, 4607182418800017408, 4613937818241073152)
// ipv4 values
for i := range values {
values[i] = fmt.Sprintf("1.2.3.%d", i)
}
f(values, valueTypeIPv4, 16909056, 16909064)
// iso8601 timestamps
for i := range values {
values[i] = fmt.Sprintf("2011-04-19T03:44:01.%03dZ", i)
}
f(values, valueTypeTimestampISO8601, 1303184641000000000, 1303184641008000000)
}
func TestTryParseIPv4String_Success(t *testing.T) {
f := func(s string) {
t.Helper()
n, ok := tryParseIPv4(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
data := marshalIPv4String(nil, n)
if string(data) != s {
t.Fatalf("unexpected ip; got %q; want %q", data, s)
}
}
f("0.0.0.0")
f("1.2.3.4")
f("255.255.255.255")
f("127.0.0.1")
}
func TestTryParseIPv4_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseIPv4(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
f("")
f("foo")
f("a.b.c.d")
f("127.0.0.x")
f("127.0.x.0")
f("127.x.0.0")
f("x.0.0.0")
// Too big octets
f("127.127.127.256")
f("127.127.256.127")
f("127.256.127.127")
f("256.127.127.127")
// Negative octets
f("-1.127.127.127")
f("127.-1.127.127")
f("127.127.-1.127")
f("127.127.127.-1")
}
func TestTryParseTimestampRFC3339NanoString_Success(t *testing.T) {
f := func(s, timestampExpected string) {
t.Helper()
nsecs, ok := TryParseTimestampRFC3339Nano(s)
if !ok {
t.Fatalf("cannot parse timestamp %q", s)
}
timestamp := marshalTimestampRFC3339NanoString(nil, nsecs)
if string(timestamp) != timestampExpected {
t.Fatalf("unexpected timestamp; got %q; want %q", timestamp, timestampExpected)
}
}
// No fractional seconds
f("2023-01-15T23:45:51Z", "2023-01-15T23:45:51Z")
// Different number of fractional seconds
f("2023-01-15T23:45:51.1Z", "2023-01-15T23:45:51.1Z")
f("2023-01-15T23:45:51.12Z", "2023-01-15T23:45:51.12Z")
f("2023-01-15T23:45:51.123Z", "2023-01-15T23:45:51.123Z")
f("2023-01-15T23:45:51.1234Z", "2023-01-15T23:45:51.1234Z")
f("2023-01-15T23:45:51.12345Z", "2023-01-15T23:45:51.12345Z")
f("2023-01-15T23:45:51.123456Z", "2023-01-15T23:45:51.123456Z")
f("2023-01-15T23:45:51.1234567Z", "2023-01-15T23:45:51.1234567Z")
f("2023-01-15T23:45:51.12345678Z", "2023-01-15T23:45:51.12345678Z")
f("2023-01-15T23:45:51.123456789Z", "2023-01-15T23:45:51.123456789Z")
// The minimum possible timestamp
f("1677-09-21T00:12:44Z", "1677-09-21T00:12:44Z")
// The maximum possible timestamp
f("2262-04-11T23:47:15.999999999Z", "2262-04-11T23:47:15.999999999Z")
// timestamp with timezone
f("2023-01-16T00:45:51+01:00", "2023-01-15T23:45:51Z")
f("2023-01-16T00:45:51.123-01:00", "2023-01-16T01:45:51.123Z")
// SQL datetime format
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6721
f("2023-01-16 00:45:51+01:00", "2023-01-15T23:45:51Z")
f("2023-01-16 00:45:51.123-01:00", "2023-01-16T01:45:51.123Z")
}
func TestTryParseTimestampRFC3339Nano_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := TryParseTimestampRFC3339Nano(s)
if ok {
t.Fatalf("expecting faulure when parsing %q", s)
}
}
// invalid length
f("")
f("foobar")
// missing fractional part after dot
f("2023-01-15T22:15:51.Z")
// too small year
f("1676-09-21T00:12:43Z")
// too big year
f("2263-04-11T23:47:17Z")
// too small timestamp
f("1677-09-21T00:12:43.999999999Z")
// too big timestamp
f("2262-04-11T23:47:16Z")
// invalid year
f("YYYY-04-11T23:47:17Z")
// invalid moth
f("2023-MM-11T23:47:17Z")
// invalid day
f("2023-01-DDT23:47:17Z")
// invalid hour
f("2023-01-23Thh:47:17Z")
// invalid minute
f("2023-01-23T23:mm:17Z")
// invalid second
f("2023-01-23T23:33:ssZ")
}
func TestTryParseTimestampISO8601String_Success(t *testing.T) {
f := func(s string) {
t.Helper()
nsecs, ok := tryParseTimestampISO8601(s)
if !ok {
t.Fatalf("cannot parse timestamp %q", s)
}
data := marshalTimestampISO8601String(nil, nsecs)
if string(data) != s {
t.Fatalf("unexpected timestamp; got %q; want %q", data, s)
}
}
// regular timestamp
f("2023-01-15T23:45:51.123Z")
// The minimum possible timestamp
f("1677-09-21T00:12:44.000Z")
// The maximum possible timestamp
f("2262-04-11T23:47:15.999Z")
}
func TestTryParseTimestampISO8601_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseTimestampISO8601(s)
if ok {
t.Fatalf("expecting faulure when parsing %q", s)
}
}
// invalid length
f("")
f("foobar")
// Missing Z at the end
f("2023-01-15T22:15:51.123")
f("2023-01-15T22:15:51.1234")
// timestamp with timezone
f("2023-01-16T00:45:51.123+01:00")
// too small year
f("1676-09-21T00:12:43.434Z")
// too big year
f("2263-04-11T23:47:17.434Z")
// too small timestamp
f("1677-09-21T00:12:43.999Z")
// too big timestamp
f("2262-04-11T23:47:16.000Z")
// invalid year
f("YYYY-04-11T23:47:17.123Z")
// invalid moth
f("2023-MM-11T23:47:17.123Z")
// invalid day
f("2023-01-DDT23:47:17.123Z")
// invalid hour
f("2023-01-23Thh:47:17.123Z")
// invalid minute
f("2023-01-23T23:mm:17.123Z")
// invalid second
f("2023-01-23T23:33:ss.123Z")
}
func TestTryParseDuration_Success(t *testing.T) {
f := func(s string, nsecsExpected int64) {
t.Helper()
nsecs, ok := tryParseDuration(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if nsecs != nsecsExpected {
t.Fatalf("unexpected value; got %d; want %d", nsecs, nsecsExpected)
}
}
// zero duration
f("0s", 0)
f("0.0w0d0h0s0.0ms", 0)
f("-0w", 0)
// positive duration
f("1s", nsecsPerSecond)
f("1.5ms", 1.5*nsecsPerMillisecond)
f("1µs", nsecsPerMicrosecond)
f("1ns", 1)
f("1h", nsecsPerHour)
f("1.5d", 1.5*nsecsPerDay)
f("1.5w", 1.5*nsecsPerWeek)
f("2.5y", 2.5*nsecsPerYear)
f("1m5.123456789s", nsecsPerMinute+5.123456789*nsecsPerSecond)
// composite duration
f("1h5m", nsecsPerHour+5*nsecsPerMinute)
f("1.1h5m2.5s3_456ns", 1.1*nsecsPerHour+5*nsecsPerMinute+2.5*nsecsPerSecond+3456)
// nedgative duration
f("-1h5m3s", -(nsecsPerHour + 5*nsecsPerMinute + 3*nsecsPerSecond))
}
func TestTryParseDuration_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseDuration(s)
if ok {
t.Fatalf("expecting error for parsing %q", s)
}
}
// empty string
f("")
// missing suffix
f("2")
f("2.5")
// invalid string
f("foobar")
f("1foo")
f("1soo")
f("3.43e")
f("3.43es")
// superflouous space
f(" 2s")
f("2s ")
f("2s 3ms")
}
func TestMarshalDurationString(t *testing.T) {
f := func(nsecs int64, resultExpected string) {
t.Helper()
result := marshalDurationString(nil, nsecs)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1, "1ns")
f(-1, "-1ns")
f(12345, "12µs345ns")
f(123456789, "123ms456µs789ns")
f(12345678901, "12.345678901s")
f(1234567890143, "20m34.567890143s")
f(1234567890123457, "2w6h56m7.890123457s")
}
func TestTryParseBytes_Success(t *testing.T) {
f := func(s string, resultExpected int64) {
t.Helper()
result, ok := tryParseBytes(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if result != resultExpected {
t.Fatalf("unexpected result; got %d; want %d", result, resultExpected)
}
}
f("1_500", 1_500)
f("2.5B", 2)
f("1.5K", 1_500)
f("1.5M", 1_500_000)
f("1.5G", 1_500_000_000)
f("1.5T", 1_500_000_000_000)
f("1.5KB", 1_500)
f("1.5MB", 1_500_000)
f("1.5GB", 1_500_000_000)
f("1.5TB", 1_500_000_000_000)
f("1.5Ki", 1.5*(1<<10))
f("1.5Mi", 1.5*(1<<20))
f("1.5Gi", 1.5*(1<<30))
f("1.5Ti", 1.5*(1<<40))
f("1.5KiB", 1.5*(1<<10))
f("1.5MiB", 1.5*(1<<20))
f("1.5GiB", 1.5*(1<<30))
f("1.5TiB", 1.5*(1<<40))
f("1MiB500KiB200B", (1<<20)+500*(1<<10)+200)
}
func TestTryParseBytes_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseBytes(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// empty string
f("")
// invalid number
f("foobar")
// invalid suffix
f("123q")
f("123qs")
f("123qsb")
f("123sqsb")
f("123s5qsb")
// invalid case for the suffix
f("1b")
f("1k")
f("1m")
f("1g")
f("1t")
f("1kb")
f("1mb")
f("1gb")
f("1tb")
f("1ki")
f("1mi")
f("1gi")
f("1ti")
f("1kib")
f("1mib")
f("1gib")
f("1tib")
f("1KIB")
f("1MIB")
f("1GIB")
f("1TIB")
// fractional number without suffix
f("123.456")
}
func TestTryParseFloat64_Success(t *testing.T) {
f := func(s string, resultExpected float64) {
t.Helper()
result, ok := tryParseFloat64(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if !float64Equal(result, resultExpected) {
t.Fatalf("unexpected value; got %f; want %f", result, resultExpected)
}
}
f("0", 0)
f("1", 1)
f("-1", -1)
f("1234567890", 1234567890)
f("1_234_567_890", 1234567890)
f("-1.234_567", -1.234567)
f("0.345", 0.345)
f("-0.345", -0.345)
}
func float64Equal(a, b float64) bool {
return math.Abs(a-b)*math.Abs(max(a, b)) < 1e-15
}
func TestTryParseFloat64_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseFloat64(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// Empty value
f("")
// Plus in the value isn't allowed, since it cannot be convered back to the same string representation
f("+123")
// Dot at the beginning and the end of value isn't allowed, since it cannot converted back to the same string representation
f(".123")
f("123.")
// Multiple dots aren't allowed
f("123.434.55")
// Invalid dots
f("-.123")
f(".")
// Scientific notation isn't allowed, since it cannot be converted back to the same string representation
f("12e5")
// Minus in the middle of string isn't allowed
f("12-5")
}
func TestMarshalFloat64String(t *testing.T) {
f := func(f float64, resultExpected string) {
t.Helper()
result := marshalFloat64String(nil, f)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1234, "1234")
f(-12345678, "-12345678")
f(1.234, "1.234")
f(-1.234567, "-1.234567")
}
func TestTryParseUint64_Success(t *testing.T) {
f := func(s string, resultExpected uint64) {
t.Helper()
result, ok := tryParseUint64(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if result != resultExpected {
t.Fatalf("unexpected value; got %d; want %d", result, resultExpected)
}
}
f("0", 0)
f("123", 123)
f("123456", 123456)
f("123456789", 123456789)
f("123456789012", 123456789012)
f("123456789012345", 123456789012345)
f("123456789012345678", 123456789012345678)
f("12345678901234567890", 12345678901234567890)
f("12_345_678_901_234_567_890", 12345678901234567890)
// the maximum possible value
f("18446744073709551615", 18446744073709551615)
}
func TestTryParseUint64_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseUint64(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// empty value
f("")
// too big value
f("18446744073709551616")
// invalid value
f("foo")
}
func TestMarshalUint8String(t *testing.T) {
f := func(n uint8, resultExpected string) {
t.Helper()
result := marshalUint8String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
for i := 0; i < 256; i++ {
resultExpected := strconv.Itoa(i)
f(uint8(i), resultExpected)
}
// the maximum possible value
f(math.MaxUint8, "255")
}
func TestMarshalUint16String(t *testing.T) {
f := func(n uint16, resultExpected string) {
t.Helper()
result := marshalUint16String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1, "1")
f(10, "10")
f(12, "12")
f(120, "120")
f(1203, "1203")
f(12345, "12345")
// the maximum possible value
f(math.MaxUint16, "65535")
}
func TestMarshalUint32String(t *testing.T) {
f := func(n uint32, resultExpected string) {
t.Helper()
result := marshalUint32String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1, "1")
f(10, "10")
f(12, "12")
f(120, "120")
f(1203, "1203")
f(12034, "12034")
f(123456, "123456")
f(1234567, "1234567")
f(12345678, "12345678")
f(123456789, "123456789")
f(1234567890, "1234567890")
// the maximum possible value
f(math.MaxUint32, "4294967295")
}
func TestMarshalUint64String(t *testing.T) {
f := func(n uint64, resultExpected string) {
t.Helper()
result := marshalUint64String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(123456, "123456")
// the maximum possible value
f(math.MaxUint64, "18446744073709551615")
}
func TestTryParseIPv4Mask_Success(t *testing.T) {
f := func(s string, resultExpected uint64) {
t.Helper()
result, ok := tryParseIPv4Mask(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if result != resultExpected {
t.Fatalf("unexpected result; got %d; want %d", result, resultExpected)
}
}
f("/0", 1<<32)
f("/1", 1<<31)
f("/8", 1<<24)
f("/24", 1<<8)
f("/32", 1)
}
func TestTryParseIPv4Mask_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseIPv4Mask(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// Empty mask
f("")
// Invalid prefix
f("foo")
// Non-numeric mask
f("/foo")
// Too big mask
f("/33")
// Negative mask
f("/-1")
}