VictoriaMetrics/lib/logstorage/values_encoder_test.go
Aliaksandr Valialkin 9e1c037249
lib/logstorage: properly parse timezone offset at TryParseTimestampRFC3339Nano()
The TryParseTimestampRFC3339Nano() must properly parse RFC3339 timestamps with timezone offsets.

While at it, make tryParseTimestampISO8601 function private in order to prevent
from improper usage of this function from outside the lib/logstorage package.

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/6508
2024-06-25 14:53:38 +02:00

752 lines
15 KiB
Go

package logstorage
import (
"fmt"
"math"
"reflect"
"strconv"
"testing"
)
func TestValuesEncoder(t *testing.T) {
f := func(values []string, expectedValueType valueType, expectedMinValue, expectedMaxValue uint64) {
t.Helper()
ve := getValuesEncoder()
var dict valuesDict
vt, minValue, maxValue := ve.encode(values, &dict)
if vt != expectedValueType {
t.Fatalf("unexpected value type; got %d; want %d", vt, expectedValueType)
}
if minValue != expectedMinValue {
t.Fatalf("unexpected minValue; got %d; want %d", minValue, expectedMinValue)
}
if maxValue != expectedMaxValue {
t.Fatalf("unexpected maxValue; got %d; want %d", maxValue, expectedMaxValue)
}
encodedValues := append([]string{}, ve.values...)
putValuesEncoder(ve)
vd := getValuesDecoder()
if err := vd.decodeInplace(encodedValues, vt, dict.values); err != nil {
t.Fatalf("unexpected error in decodeInplace(): %s", err)
}
if len(values) == 0 {
values = []string{}
}
if !reflect.DeepEqual(values, encodedValues) {
t.Fatalf("unexpected values decoded\ngot\n%q\nwant\n%q", encodedValues, values)
}
putValuesDecoder(vd)
}
// An empty values list
f(nil, valueTypeString, 0, 0)
// string values
values := make([]string, maxDictLen+1)
for i := range values {
values[i] = fmt.Sprintf("value_%d", i)
}
f(values, valueTypeString, 0, 0)
// dict values
f([]string{"foobar"}, valueTypeDict, 0, 0)
f([]string{"foo", "bar"}, valueTypeDict, 0, 0)
f([]string{"1", "2foo"}, valueTypeDict, 0, 0)
// uint8 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1))
}
f(values, valueTypeUint8, 1, uint64(len(values)))
// uint16 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1)<<8)
}
f(values, valueTypeUint16, 1<<8, uint64(len(values))<<8)
// uint32 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1)<<16)
}
f(values, valueTypeUint32, 1<<16, uint64(len(values))<<16)
// uint64 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1)<<32)
}
f(values, valueTypeUint64, 1<<32, uint64(len(values))<<32)
// float64 values
for i := range values {
values[i] = fmt.Sprintf("%g", math.Sqrt(float64(i+1)))
}
f(values, valueTypeFloat64, 4607182418800017408, 4613937818241073152)
// ipv4 values
for i := range values {
values[i] = fmt.Sprintf("1.2.3.%d", i)
}
f(values, valueTypeIPv4, 16909056, 16909064)
// iso8601 timestamps
for i := range values {
values[i] = fmt.Sprintf("2011-04-19T03:44:01.%03dZ", i)
}
f(values, valueTypeTimestampISO8601, 1303184641000000000, 1303184641008000000)
}
func TestTryParseIPv4String_Success(t *testing.T) {
f := func(s string) {
t.Helper()
n, ok := tryParseIPv4(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
data := marshalIPv4String(nil, n)
if string(data) != s {
t.Fatalf("unexpected ip; got %q; want %q", data, s)
}
}
f("0.0.0.0")
f("1.2.3.4")
f("255.255.255.255")
f("127.0.0.1")
}
func TestTryParseIPv4_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseIPv4(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
f("")
f("foo")
f("a.b.c.d")
f("127.0.0.x")
f("127.0.x.0")
f("127.x.0.0")
f("x.0.0.0")
// Too big octets
f("127.127.127.256")
f("127.127.256.127")
f("127.256.127.127")
f("256.127.127.127")
// Negative octets
f("-1.127.127.127")
f("127.-1.127.127")
f("127.127.-1.127")
f("127.127.127.-1")
}
func TestTryParseTimestampRFC3339NanoString_Success(t *testing.T) {
f := func(s, timestampExpected string) {
t.Helper()
nsecs, ok := TryParseTimestampRFC3339Nano(s)
if !ok {
t.Fatalf("cannot parse timestamp %q", s)
}
timestamp := marshalTimestampRFC3339NanoString(nil, nsecs)
if string(timestamp) != timestampExpected {
t.Fatalf("unexpected timestamp; got %q; want %q", timestamp, timestampExpected)
}
}
// No fractional seconds
f("2023-01-15T23:45:51Z", "2023-01-15T23:45:51Z")
// Different number of fractional seconds
f("2023-01-15T23:45:51.1Z", "2023-01-15T23:45:51.1Z")
f("2023-01-15T23:45:51.12Z", "2023-01-15T23:45:51.12Z")
f("2023-01-15T23:45:51.123Z", "2023-01-15T23:45:51.123Z")
f("2023-01-15T23:45:51.1234Z", "2023-01-15T23:45:51.1234Z")
f("2023-01-15T23:45:51.12345Z", "2023-01-15T23:45:51.12345Z")
f("2023-01-15T23:45:51.123456Z", "2023-01-15T23:45:51.123456Z")
f("2023-01-15T23:45:51.1234567Z", "2023-01-15T23:45:51.1234567Z")
f("2023-01-15T23:45:51.12345678Z", "2023-01-15T23:45:51.12345678Z")
f("2023-01-15T23:45:51.123456789Z", "2023-01-15T23:45:51.123456789Z")
// The minimum possible timestamp
f("1677-09-21T00:12:44Z", "1677-09-21T00:12:44Z")
// The maximum possible timestamp
f("2262-04-11T23:47:15.999999999Z", "2262-04-11T23:47:15.999999999Z")
// timestamp with timezone
f("2023-01-16T00:45:51+01:00", "2023-01-15T23:45:51Z")
f("2023-01-16T00:45:51.123-01:00", "2023-01-16T01:45:51.123Z")
}
func TestTryParseTimestampRFC3339Nano_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := TryParseTimestampRFC3339Nano(s)
if ok {
t.Fatalf("expecting faulure when parsing %q", s)
}
}
// invalid length
f("")
f("foobar")
// Missing Z at the end
f("2023-01-15T22:15:51")
f("2023-01-15T22:15:51.123")
// missing fractional part after dot
f("2023-01-15T22:15:51.Z")
// too small year
f("1676-09-21T00:12:43Z")
// too big year
f("2263-04-11T23:47:17Z")
// too small timestamp
f("1677-09-21T00:12:43.999999999Z")
// too big timestamp
f("2262-04-11T23:47:16Z")
// invalid year
f("YYYY-04-11T23:47:17Z")
// invalid moth
f("2023-MM-11T23:47:17Z")
// invalid day
f("2023-01-DDT23:47:17Z")
// invalid hour
f("2023-01-23Thh:47:17Z")
// invalid minute
f("2023-01-23T23:mm:17Z")
// invalid second
f("2023-01-23T23:33:ssZ")
}
func TestTryParseTimestampISO8601String_Success(t *testing.T) {
f := func(s string) {
t.Helper()
nsecs, ok := tryParseTimestampISO8601(s)
if !ok {
t.Fatalf("cannot parse timestamp %q", s)
}
data := marshalTimestampISO8601String(nil, nsecs)
if string(data) != s {
t.Fatalf("unexpected timestamp; got %q; want %q", data, s)
}
}
// regular timestamp
f("2023-01-15T23:45:51.123Z")
// The minimum possible timestamp
f("1677-09-21T00:12:44.000Z")
// The maximum possible timestamp
f("2262-04-11T23:47:15.999Z")
}
func TestTryParseTimestampISO8601_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseTimestampISO8601(s)
if ok {
t.Fatalf("expecting faulure when parsing %q", s)
}
}
// invalid length
f("")
f("foobar")
// Missing Z at the end
f("2023-01-15T22:15:51.123")
f("2023-01-15T22:15:51.1234")
// timestamp with timezone
f("2023-01-16T00:45:51.123+01:00")
// too small year
f("1676-09-21T00:12:43.434Z")
// too big year
f("2263-04-11T23:47:17.434Z")
// too small timestamp
f("1677-09-21T00:12:43.999Z")
// too big timestamp
f("2262-04-11T23:47:16.000Z")
// invalid year
f("YYYY-04-11T23:47:17.123Z")
// invalid moth
f("2023-MM-11T23:47:17.123Z")
// invalid day
f("2023-01-DDT23:47:17.123Z")
// invalid hour
f("2023-01-23Thh:47:17.123Z")
// invalid minute
f("2023-01-23T23:mm:17.123Z")
// invalid second
f("2023-01-23T23:33:ss.123Z")
}
func TestTryParseDuration_Success(t *testing.T) {
f := func(s string, nsecsExpected int64) {
t.Helper()
nsecs, ok := tryParseDuration(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if nsecs != nsecsExpected {
t.Fatalf("unexpected value; got %d; want %d", nsecs, nsecsExpected)
}
}
// zero duration
f("0s", 0)
f("0.0w0d0h0s0.0ms", 0)
f("-0w", 0)
// positive duration
f("1s", nsecsPerSecond)
f("1.5ms", 1.5*nsecsPerMillisecond)
f("1µs", nsecsPerMicrosecond)
f("1ns", 1)
f("1h", nsecsPerHour)
f("1.5d", 1.5*nsecsPerDay)
f("1.5w", 1.5*nsecsPerWeek)
f("2.5y", 2.5*nsecsPerYear)
f("1m5.123456789s", nsecsPerMinute+5.123456789*nsecsPerSecond)
// composite duration
f("1h5m", nsecsPerHour+5*nsecsPerMinute)
f("1.1h5m2.5s3_456ns", 1.1*nsecsPerHour+5*nsecsPerMinute+2.5*nsecsPerSecond+3456)
// nedgative duration
f("-1h5m3s", -(nsecsPerHour + 5*nsecsPerMinute + 3*nsecsPerSecond))
}
func TestTryParseDuration_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseDuration(s)
if ok {
t.Fatalf("expecting error for parsing %q", s)
}
}
// empty string
f("")
// missing suffix
f("2")
f("2.5")
// invalid string
f("foobar")
f("1foo")
f("1soo")
f("3.43e")
f("3.43es")
// superflouous space
f(" 2s")
f("2s ")
f("2s 3ms")
}
func TestMarshalDurationString(t *testing.T) {
f := func(nsecs int64, resultExpected string) {
t.Helper()
result := marshalDurationString(nil, nsecs)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1, "1ns")
f(-1, "-1ns")
f(12345, "12µs345ns")
f(123456789, "123ms456µs789ns")
f(12345678901, "12.345678901s")
f(1234567890143, "20m34.567890143s")
f(1234567890123457, "2w6h56m7.890123457s")
}
func TestTryParseBytes_Success(t *testing.T) {
f := func(s string, resultExpected int64) {
t.Helper()
result, ok := tryParseBytes(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if result != resultExpected {
t.Fatalf("unexpected result; got %d; want %d", result, resultExpected)
}
}
f("1_500", 1_500)
f("2.5B", 2)
f("1.5K", 1_500)
f("1.5M", 1_500_000)
f("1.5G", 1_500_000_000)
f("1.5T", 1_500_000_000_000)
f("1.5KB", 1_500)
f("1.5MB", 1_500_000)
f("1.5GB", 1_500_000_000)
f("1.5TB", 1_500_000_000_000)
f("1.5Ki", 1.5*(1<<10))
f("1.5Mi", 1.5*(1<<20))
f("1.5Gi", 1.5*(1<<30))
f("1.5Ti", 1.5*(1<<40))
f("1.5KiB", 1.5*(1<<10))
f("1.5MiB", 1.5*(1<<20))
f("1.5GiB", 1.5*(1<<30))
f("1.5TiB", 1.5*(1<<40))
f("1MiB500KiB200B", (1<<20)+500*(1<<10)+200)
}
func TestTryParseBytes_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseBytes(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// empty string
f("")
// invalid number
f("foobar")
// invalid suffix
f("123q")
f("123qs")
f("123qsb")
f("123sqsb")
f("123s5qsb")
// invalid case for the suffix
f("1b")
f("1k")
f("1m")
f("1g")
f("1t")
f("1kb")
f("1mb")
f("1gb")
f("1tb")
f("1ki")
f("1mi")
f("1gi")
f("1ti")
f("1kib")
f("1mib")
f("1gib")
f("1tib")
f("1KIB")
f("1MIB")
f("1GIB")
f("1TIB")
// fractional number without suffix
f("123.456")
}
func TestTryParseFloat64_Success(t *testing.T) {
f := func(s string, resultExpected float64) {
t.Helper()
result, ok := tryParseFloat64(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if !float64Equal(result, resultExpected) {
t.Fatalf("unexpected value; got %f; want %f", result, resultExpected)
}
}
f("0", 0)
f("1", 1)
f("-1", -1)
f("1234567890", 1234567890)
f("1_234_567_890", 1234567890)
f("-1.234_567", -1.234567)
f("0.345", 0.345)
f("-0.345", -0.345)
}
func float64Equal(a, b float64) bool {
return math.Abs(a-b)*math.Abs(max(a, b)) < 1e-15
}
func TestTryParseFloat64_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseFloat64(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// Empty value
f("")
// Plus in the value isn't allowed, since it cannot be convered back to the same string representation
f("+123")
// Dot at the beginning and the end of value isn't allowed, since it cannot converted back to the same string representation
f(".123")
f("123.")
// Multiple dots aren't allowed
f("123.434.55")
// Invalid dots
f("-.123")
f(".")
// Scientific notation isn't allowed, since it cannot be converted back to the same string representation
f("12e5")
// Minus in the middle of string isn't allowed
f("12-5")
}
func TestMarshalFloat64String(t *testing.T) {
f := func(f float64, resultExpected string) {
t.Helper()
result := marshalFloat64String(nil, f)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1234, "1234")
f(-12345678, "-12345678")
f(1.234, "1.234")
f(-1.234567, "-1.234567")
}
func TestTryParseUint64_Success(t *testing.T) {
f := func(s string, resultExpected uint64) {
t.Helper()
result, ok := tryParseUint64(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if result != resultExpected {
t.Fatalf("unexpected value; got %d; want %d", result, resultExpected)
}
}
f("0", 0)
f("123", 123)
f("123456", 123456)
f("123456789", 123456789)
f("123456789012", 123456789012)
f("123456789012345", 123456789012345)
f("123456789012345678", 123456789012345678)
f("12345678901234567890", 12345678901234567890)
f("12_345_678_901_234_567_890", 12345678901234567890)
// the maximum possible value
f("18446744073709551615", 18446744073709551615)
}
func TestTryParseUint64_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseUint64(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// empty value
f("")
// too big value
f("18446744073709551616")
// invalid value
f("foo")
}
func TestMarshalUint8String(t *testing.T) {
f := func(n uint8, resultExpected string) {
t.Helper()
result := marshalUint8String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
for i := 0; i < 256; i++ {
resultExpected := strconv.Itoa(i)
f(uint8(i), resultExpected)
}
// the maximum possible value
f(math.MaxUint8, "255")
}
func TestMarshalUint16String(t *testing.T) {
f := func(n uint16, resultExpected string) {
t.Helper()
result := marshalUint16String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1, "1")
f(10, "10")
f(12, "12")
f(120, "120")
f(1203, "1203")
f(12345, "12345")
// the maximum possible value
f(math.MaxUint16, "65535")
}
func TestMarshalUint32String(t *testing.T) {
f := func(n uint32, resultExpected string) {
t.Helper()
result := marshalUint32String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1, "1")
f(10, "10")
f(12, "12")
f(120, "120")
f(1203, "1203")
f(12034, "12034")
f(123456, "123456")
f(1234567, "1234567")
f(12345678, "12345678")
f(123456789, "123456789")
f(1234567890, "1234567890")
// the maximum possible value
f(math.MaxUint32, "4294967295")
}
func TestMarshalUint64String(t *testing.T) {
f := func(n uint64, resultExpected string) {
t.Helper()
result := marshalUint64String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(123456, "123456")
// the maximum possible value
f(math.MaxUint64, "18446744073709551615")
}
func TestTryParseIPv4Mask_Success(t *testing.T) {
f := func(s string, resultExpected uint64) {
t.Helper()
result, ok := tryParseIPv4Mask(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if result != resultExpected {
t.Fatalf("unexpected result; got %d; want %d", result, resultExpected)
}
}
f("/0", 1<<32)
f("/1", 1<<31)
f("/8", 1<<24)
f("/24", 1<<8)
f("/32", 1)
}
func TestTryParseIPv4Mask_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseIPv4Mask(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// Empty mask
f("")
// Invalid prefix
f("foo")
// Non-numeric mask
f("/foo")
// Too big mask
f("/33")
// Negative mask
f("/-1")
}