VictoriaMetrics/lib/logstorage/values_encoder_test.go
2024-05-20 04:09:15 +02:00

752 lines
15 KiB
Go

package logstorage
import (
"fmt"
"math"
"reflect"
"strconv"
"testing"
)
func TestValuesEncoder(t *testing.T) {
f := func(values []string, expectedValueType valueType, expectedMinValue, expectedMaxValue uint64) {
t.Helper()
ve := getValuesEncoder()
var dict valuesDict
vt, minValue, maxValue := ve.encode(values, &dict)
if vt != expectedValueType {
t.Fatalf("unexpected value type; got %d; want %d", vt, expectedValueType)
}
if minValue != expectedMinValue {
t.Fatalf("unexpected minValue; got %d; want %d", minValue, expectedMinValue)
}
if maxValue != expectedMaxValue {
t.Fatalf("unexpected maxValue; got %d; want %d", maxValue, expectedMaxValue)
}
encodedValues := append([]string{}, ve.values...)
putValuesEncoder(ve)
vd := getValuesDecoder()
if err := vd.decodeInplace(encodedValues, vt, dict.values); err != nil {
t.Fatalf("unexpected error in decodeInplace(): %s", err)
}
if len(values) == 0 {
values = []string{}
}
if !reflect.DeepEqual(values, encodedValues) {
t.Fatalf("unexpected values decoded\ngot\n%q\nwant\n%q", encodedValues, values)
}
putValuesDecoder(vd)
}
// An empty values list
f(nil, valueTypeString, 0, 0)
// string values
values := make([]string, maxDictLen+1)
for i := range values {
values[i] = fmt.Sprintf("value_%d", i)
}
f(values, valueTypeString, 0, 0)
// dict values
f([]string{"foobar"}, valueTypeDict, 0, 0)
f([]string{"foo", "bar"}, valueTypeDict, 0, 0)
f([]string{"1", "2foo"}, valueTypeDict, 0, 0)
// uint8 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1))
}
f(values, valueTypeUint8, 1, uint64(len(values)))
// uint16 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1)<<8)
}
f(values, valueTypeUint16, 1<<8, uint64(len(values))<<8)
// uint32 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1)<<16)
}
f(values, valueTypeUint32, 1<<16, uint64(len(values))<<16)
// uint64 values
for i := range values {
values[i] = fmt.Sprintf("%d", uint64(i+1)<<32)
}
f(values, valueTypeUint64, 1<<32, uint64(len(values))<<32)
// float64 values
for i := range values {
values[i] = fmt.Sprintf("%g", math.Sqrt(float64(i+1)))
}
f(values, valueTypeFloat64, 4607182418800017408, 4613937818241073152)
// ipv4 values
for i := range values {
values[i] = fmt.Sprintf("1.2.3.%d", i)
}
f(values, valueTypeIPv4, 16909056, 16909064)
// iso8601 timestamps
for i := range values {
values[i] = fmt.Sprintf("2011-04-19T03:44:01.%03dZ", i)
}
f(values, valueTypeTimestampISO8601, 1303184641000000000, 1303184641008000000)
}
func TestTryParseIPv4String_Success(t *testing.T) {
f := func(s string) {
t.Helper()
n, ok := tryParseIPv4(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
data := marshalIPv4String(nil, n)
if string(data) != s {
t.Fatalf("unexpected ip; got %q; want %q", data, s)
}
}
f("0.0.0.0")
f("1.2.3.4")
f("255.255.255.255")
f("127.0.0.1")
}
func TestTryParseIPv4_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseIPv4(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
f("")
f("foo")
f("a.b.c.d")
f("127.0.0.x")
f("127.0.x.0")
f("127.x.0.0")
f("x.0.0.0")
// Too big octets
f("127.127.127.256")
f("127.127.256.127")
f("127.256.127.127")
f("256.127.127.127")
// Negative octets
f("-1.127.127.127")
f("127.-1.127.127")
f("127.127.-1.127")
f("127.127.127.-1")
}
func TestTryParseTimestampRFC3339NanoString_Success(t *testing.T) {
f := func(s string) {
t.Helper()
nsecs, ok := tryParseTimestampRFC3339Nano(s)
if !ok {
t.Fatalf("cannot parse timestamp %q", s)
}
data := marshalTimestampRFC3339NanoString(nil, nsecs)
if string(data) != s {
t.Fatalf("unexpected timestamp; got %q; want %q", data, s)
}
}
// No fractional seconds
f("2023-01-15T23:45:51Z")
// Different number of fractional seconds
f("2023-01-15T23:45:51.1Z")
f("2023-01-15T23:45:51.12Z")
f("2023-01-15T23:45:51.123Z")
f("2023-01-15T23:45:51.1234Z")
f("2023-01-15T23:45:51.12345Z")
f("2023-01-15T23:45:51.123456Z")
f("2023-01-15T23:45:51.1234567Z")
f("2023-01-15T23:45:51.12345678Z")
f("2023-01-15T23:45:51.123456789Z")
// The minimum possible timestamp
f("1677-09-21T00:12:44Z")
// The maximum possible timestamp
f("2262-04-11T23:47:15.999999999Z")
}
func TestTryParseTimestampRFC3339Nano_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseTimestampRFC3339Nano(s)
if ok {
t.Fatalf("expecting faulure when parsing %q", s)
}
}
// invalid length
f("")
f("foobar")
// Missing Z at the end
f("2023-01-15T22:15:51")
f("2023-01-15T22:15:51.123")
// missing fractional part after dot
f("2023-01-15T22:15:51.Z")
// timestamp with timezone
f("2023-01-16T00:45:51+01:00")
f("2023-01-16T00:45:51.123+01:00")
// too small year
f("1676-09-21T00:12:43Z")
// too big year
f("2263-04-11T23:47:17Z")
// too small timestamp
f("1677-09-21T00:12:43.999999999Z")
// too big timestamp
f("2262-04-11T23:47:16Z")
// invalid year
f("YYYY-04-11T23:47:17Z")
// invalid moth
f("2023-MM-11T23:47:17Z")
// invalid day
f("2023-01-DDT23:47:17Z")
// invalid hour
f("2023-01-23Thh:47:17Z")
// invalid minute
f("2023-01-23T23:mm:17Z")
// invalid second
f("2023-01-23T23:33:ssZ")
}
func TestTryParseTimestampISO8601String_Success(t *testing.T) {
f := func(s string) {
t.Helper()
nsecs, ok := tryParseTimestampISO8601(s)
if !ok {
t.Fatalf("cannot parse timestamp %q", s)
}
data := marshalTimestampISO8601String(nil, nsecs)
if string(data) != s {
t.Fatalf("unexpected timestamp; got %q; want %q", data, s)
}
}
// regular timestamp
f("2023-01-15T23:45:51.123Z")
// The minimum possible timestamp
f("1677-09-21T00:12:44.000Z")
// The maximum possible timestamp
f("2262-04-11T23:47:15.999Z")
}
func TestTryParseTimestampISO8601_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseTimestampISO8601(s)
if ok {
t.Fatalf("expecting faulure when parsing %q", s)
}
}
// invalid length
f("")
f("foobar")
// Missing Z at the end
f("2023-01-15T22:15:51.123")
f("2023-01-15T22:15:51.1234")
// timestamp with timezone
f("2023-01-16T00:45:51.123+01:00")
// too small year
f("1676-09-21T00:12:43.434Z")
// too big year
f("2263-04-11T23:47:17.434Z")
// too small timestamp
f("1677-09-21T00:12:43.999Z")
// too big timestamp
f("2262-04-11T23:47:16.000Z")
// invalid year
f("YYYY-04-11T23:47:17.123Z")
// invalid moth
f("2023-MM-11T23:47:17.123Z")
// invalid day
f("2023-01-DDT23:47:17.123Z")
// invalid hour
f("2023-01-23Thh:47:17.123Z")
// invalid minute
f("2023-01-23T23:mm:17.123Z")
// invalid second
f("2023-01-23T23:33:ss.123Z")
}
func TestTryParseDuration_Success(t *testing.T) {
f := func(s string, nsecsExpected int64) {
t.Helper()
nsecs, ok := tryParseDuration(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if nsecs != nsecsExpected {
t.Fatalf("unexpected value; got %d; want %d", nsecs, nsecsExpected)
}
}
// zero duration
f("0s", 0)
f("0.0w0d0h0s0.0ms", 0)
f("-0w", 0)
// positive duration
f("1s", nsecsPerSecond)
f("1.5ms", 1.5*nsecsPerMillisecond)
f("1µs", nsecsPerMicrosecond)
f("1ns", 1)
f("1h", nsecsPerHour)
f("1.5d", 1.5*nsecsPerDay)
f("1.5w", 1.5*nsecsPerWeek)
f("2.5y", 2.5*nsecsPerYear)
f("1m5.123456789s", nsecsPerMinute+5.123456789*nsecsPerSecond)
// composite duration
f("1h5m", nsecsPerHour+5*nsecsPerMinute)
f("1.1h5m2.5s3_456ns", 1.1*nsecsPerHour+5*nsecsPerMinute+2.5*nsecsPerSecond+3456)
// nedgative duration
f("-1h5m3s", -(nsecsPerHour + 5*nsecsPerMinute + 3*nsecsPerSecond))
}
func TestTryParseDuration_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseDuration(s)
if ok {
t.Fatalf("expecting error for parsing %q", s)
}
}
// empty string
f("")
// missing suffix
f("2")
f("2.5")
// invalid string
f("foobar")
f("1foo")
f("1soo")
f("3.43e")
f("3.43es")
// superflouous space
f(" 2s")
f("2s ")
f("2s 3ms")
}
func TestMarshalDuration(t *testing.T) {
f := func(nsecs int64, resultExpected string) {
t.Helper()
result := marshalDuration(nil, nsecs)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1, "1ns")
f(-1, "-1ns")
f(12345, "12µs345ns")
f(123456789, "123ms456µs789ns")
f(12345678901, "12.345678901s")
f(1234567890143, "20m34.567890143s")
f(1234567890123457, "2w6h56m7.890123457s")
}
func TestTryParseBytes_Success(t *testing.T) {
f := func(s string, resultExpected int64) {
t.Helper()
result, ok := tryParseBytes(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if result != resultExpected {
t.Fatalf("unexpected result; got %d; want %d", result, resultExpected)
}
}
f("1_500", 1_500)
f("2.5B", 2)
f("1.5K", 1_500)
f("1.5M", 1_500_000)
f("1.5G", 1_500_000_000)
f("1.5T", 1_500_000_000_000)
f("1.5KB", 1_500)
f("1.5MB", 1_500_000)
f("1.5GB", 1_500_000_000)
f("1.5TB", 1_500_000_000_000)
f("1.5Ki", 1.5*(1<<10))
f("1.5Mi", 1.5*(1<<20))
f("1.5Gi", 1.5*(1<<30))
f("1.5Ti", 1.5*(1<<40))
f("1.5KiB", 1.5*(1<<10))
f("1.5MiB", 1.5*(1<<20))
f("1.5GiB", 1.5*(1<<30))
f("1.5TiB", 1.5*(1<<40))
f("1MiB500KiB200B", (1<<20)+500*(1<<10)+200)
}
func TestTryParseBytes_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseBytes(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// empty string
f("")
// invalid number
f("foobar")
// invalid suffix
f("123q")
f("123qs")
f("123qsb")
f("123sqsb")
f("123s5qsb")
// invalid case for the suffix
f("1b")
f("1k")
f("1m")
f("1g")
f("1t")
f("1kb")
f("1mb")
f("1gb")
f("1tb")
f("1ki")
f("1mi")
f("1gi")
f("1ti")
f("1kib")
f("1mib")
f("1gib")
f("1tib")
f("1KIB")
f("1MIB")
f("1GIB")
f("1TIB")
// fractional number without suffix
f("123.456")
}
func TestTryParseFloat64_Success(t *testing.T) {
f := func(s string, resultExpected float64) {
t.Helper()
result, ok := tryParseFloat64(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if !float64Equal(result, resultExpected) {
t.Fatalf("unexpected value; got %f; want %f", result, resultExpected)
}
}
f("0", 0)
f("1", 1)
f("-1", -1)
f("1234567890", 1234567890)
f("1_234_567_890", 1234567890)
f("-1.234_567", -1.234567)
f("0.345", 0.345)
f("-0.345", -0.345)
}
func float64Equal(a, b float64) bool {
return math.Abs(a-b)*math.Abs(max(a, b)) < 1e-15
}
func TestTryParseFloat64_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseFloat64(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// Empty value
f("")
// Plus in the value isn't allowed, since it cannot be convered back to the same string representation
f("+123")
// Dot at the beginning and the end of value isn't allowed, since it cannot converted back to the same string representation
f(".123")
f("123.")
// Multiple dots aren't allowed
f("123.434.55")
// Invalid dots
f("-.123")
f(".")
// Scientific notation isn't allowed, since it cannot be converted back to the same string representation
f("12e5")
// Minus in the middle of string isn't allowed
f("12-5")
}
func TestMarshalFloat64String(t *testing.T) {
f := func(f float64, resultExpected string) {
t.Helper()
result := marshalFloat64String(nil, f)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1234, "1234")
f(-12345678, "-12345678")
f(1.234, "1.234")
f(-1.234567, "-1.234567")
}
func TestTryParseUint64_Success(t *testing.T) {
f := func(s string, resultExpected uint64) {
t.Helper()
result, ok := tryParseUint64(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if result != resultExpected {
t.Fatalf("unexpected value; got %d; want %d", result, resultExpected)
}
}
f("0", 0)
f("123", 123)
f("123456", 123456)
f("123456789", 123456789)
f("123456789012", 123456789012)
f("123456789012345", 123456789012345)
f("123456789012345678", 123456789012345678)
f("12345678901234567890", 12345678901234567890)
f("12_345_678_901_234_567_890", 12345678901234567890)
// the maximum possible value
f("18446744073709551615", 18446744073709551615)
}
func TestTryParseUint64_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseUint64(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// empty value
f("")
// too big value
f("18446744073709551616")
// invalid value
f("foo")
}
func TestMarshalUint8String(t *testing.T) {
f := func(n uint8, resultExpected string) {
t.Helper()
result := marshalUint8String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
for i := 0; i < 256; i++ {
resultExpected := strconv.Itoa(i)
f(uint8(i), resultExpected)
}
// the maximum possible value
f(math.MaxUint8, "255")
}
func TestMarshalUint16String(t *testing.T) {
f := func(n uint16, resultExpected string) {
t.Helper()
result := marshalUint16String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1, "1")
f(10, "10")
f(12, "12")
f(120, "120")
f(1203, "1203")
f(12345, "12345")
// the maximum possible value
f(math.MaxUint16, "65535")
}
func TestMarshalUint32String(t *testing.T) {
f := func(n uint32, resultExpected string) {
t.Helper()
result := marshalUint32String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(1, "1")
f(10, "10")
f(12, "12")
f(120, "120")
f(1203, "1203")
f(12034, "12034")
f(123456, "123456")
f(1234567, "1234567")
f(12345678, "12345678")
f(123456789, "123456789")
f(1234567890, "1234567890")
// the maximum possible value
f(math.MaxUint32, "4294967295")
}
func TestMarshalUint64String(t *testing.T) {
f := func(n uint64, resultExpected string) {
t.Helper()
result := marshalUint64String(nil, n)
if string(result) != resultExpected {
t.Fatalf("unexpected result; got %q; want %q", result, resultExpected)
}
}
f(0, "0")
f(123456, "123456")
// the maximum possible value
f(math.MaxUint64, "18446744073709551615")
}
func TestTryParseIPv4Mask_Success(t *testing.T) {
f := func(s string, resultExpected uint64) {
t.Helper()
result, ok := tryParseIPv4Mask(s)
if !ok {
t.Fatalf("cannot parse %q", s)
}
if result != resultExpected {
t.Fatalf("unexpected result; got %d; want %d", result, resultExpected)
}
}
f("/0", 1<<32)
f("/1", 1<<31)
f("/8", 1<<24)
f("/24", 1<<8)
f("/32", 1)
}
func TestTryParseIPv4Mask_Failure(t *testing.T) {
f := func(s string) {
t.Helper()
_, ok := tryParseIPv4Mask(s)
if ok {
t.Fatalf("expecting error when parsing %q", s)
}
}
// Empty mask
f("")
// Invalid prefix
f("foo")
// Non-numeric mask
f("/foo")
// Too big mask
f("/33")
// Negative mask
f("/-1")
}