VictoriaMetrics/lib/protoparser/csvimport/parser_test.go
Nikolay 22762d7a69
app/vmselect: change export/csv timestamp format for rfc3339 to respect milliseconds ()
* app/vmselect: adds milliseconds to the csv export response for rfc3339
* milliseconds is a standard prescion for VictoriaMetrics query request responses
https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5837

* app/victoria-metrics: adds tests for csv export/import
follow-up after 3541a8d0cf96dd4f8563624c4aab6816615d0756

---------

Signed-off-by: hagen1778 <roman@victoriametrics.com>
Co-authored-by: hagen1778 <roman@victoriametrics.com>
2024-02-23 01:16:08 +02:00

321 lines
6.2 KiB
Go

package csvimport
import (
"reflect"
"testing"
)
func TestRowsUnmarshalFailure(t *testing.T) {
f := func(format, s string) {
t.Helper()
cds, err := ParseColumnDescriptors(format)
if err != nil {
t.Fatalf("unexpected error when parsing %q: %s", format, err)
}
var rs Rows
rs.Unmarshal(s, cds)
if len(rs.Rows) != 0 {
t.Fatalf("unexpected rows unmarshaled: %#v", rs.Rows)
}
}
// Invalid timestamp
f("1:metric:foo,2:time:rfc3339", "234,foobar")
f("1:metric:foo,2:time:unix_s", "234,foobar")
f("1:metric:foo,2:time:unix_ms", "234,foobar")
f("1:metric:foo,2:time:unix_ns", "234,foobar")
f("1:metric:foo,2:time:custom:foobar", "234,234")
// Too big timestamp in seconds.
f("1:metric:foo,2:time:unix_s", "1,12345678901234567")
// Missing columns
f("3:metric:aaa", "123,456")
f("1:metric:foo,2:label:bar", "123")
f("1:label:foo,2:metric:bar", "aaa")
// Invalid value
f("1:metric:foo", "12foobar")
}
func TestRowsUnmarshalSuccess(t *testing.T) {
f := func(format, s string, rowsExpected []Row) {
t.Helper()
cds, err := ParseColumnDescriptors(format)
if err != nil {
t.Fatalf("unexpected error when parsing %q: %s", format, err)
}
var rs Rows
rs.Unmarshal(s, cds)
if !reflect.DeepEqual(rs.Rows, rowsExpected) {
t.Fatalf("unexpected rows;\ngot\n%v\nwant\n%v", rs.Rows, rowsExpected)
}
rs.Reset()
// Unmarshal rows the second time
rs.Unmarshal(s, cds)
if !reflect.DeepEqual(rs.Rows, rowsExpected) {
t.Fatalf("unexpected rows on the second unmarshal;\ngot\n%v\nwant\n%v", rs.Rows, rowsExpected)
}
}
f("1:metric:foo", "", nil)
f("1:metric:foo", `123`, []Row{
{
Metric: "foo",
Value: 123,
},
})
f("1:metric:foo,2:time:unix_s,3:label:foo,4:label:bar", `123,456,xxx,yy`, []Row{
{
Metric: "foo",
Tags: []Tag{
{
Key: "foo",
Value: "xxx",
},
{
Key: "bar",
Value: "yy",
},
},
Value: 123,
Timestamp: 456000,
},
})
// Multiple metrics
f("2:metric:bar,1:metric:foo,3:label:foo,4:label:bar,5:time:custom:2006-01-02 15:04:05.999Z",
`"2.34",5.6,"foo"",bar","aa",2015-08-10 20:04:40.123Z`, []Row{
{
Metric: "foo",
Tags: []Tag{
{
Key: "foo",
Value: "foo\",bar",
},
{
Key: "bar",
Value: "aa",
},
},
Value: 2.34,
Timestamp: 1439237080123,
},
{
Metric: "bar",
Tags: []Tag{
{
Key: "foo",
Value: "foo\",bar",
},
{
Key: "bar",
Value: "aa",
},
},
Value: 5.6,
Timestamp: 1439237080123,
},
})
f("2:label:symbol,3:time:custom:2006-01-02 15:04:05.999Z,4:metric:bid,5:metric:ask",
`
"aaa","AUDCAD","2015-08-10 00:00:01.000Z",0.9725,0.97273
"aaa","AUDCAD","2015-08-10 00:00:02.000Z",0.97253,0.97276
`, []Row{
{
Metric: "bid",
Tags: []Tag{
{
Key: "symbol",
Value: "AUDCAD",
},
},
Value: 0.9725,
Timestamp: 1439164801000,
},
{
Metric: "ask",
Tags: []Tag{
{
Key: "symbol",
Value: "AUDCAD",
},
},
Value: 0.97273,
Timestamp: 1439164801000,
},
{
Metric: "bid",
Tags: []Tag{
{
Key: "symbol",
Value: "AUDCAD",
},
},
Value: 0.97253,
Timestamp: 1439164802000,
},
{
Metric: "ask",
Tags: []Tag{
{
Key: "symbol",
Value: "AUDCAD",
},
},
Value: 0.97276,
Timestamp: 1439164802000,
},
})
// Superfluous columns
f("1:metric:foo", `123,456,foo,bar`, []Row{
{
Metric: "foo",
Value: 123,
},
})
f("2:metric:foo", `123,-45.6,foo,bar`, []Row{
{
Metric: "foo",
Value: -45.6,
},
})
// skip metrics with empty values
f("1:metric:foo,2:metric:bar,3:metric:baz,4:metric:quux", `1,,,2`, []Row{
{
Metric: "foo",
Value: 1,
},
{
Metric: "quux",
Value: 2,
},
})
// last metric with empty value
// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4048
f("1:metric:foo,2:metric:bar", `123,`, []Row{
{
Metric: "foo",
Value: 123,
},
})
// all the metrics with empty values
f(`1:metric:foo,2:metric:bar,3:label:xx`, `,,abc`, nil)
// labels with empty value
f("1:metric:foo,2:label:bar,3:label:baz,4:label:xxx", "123,x,,", []Row{
{
Metric: "foo",
Tags: []Tag{
{
Key: "bar",
Value: "x",
},
},
Value: 123,
},
})
f("1:metric:foo,2:label:bar,3:label:baz,4:label:xxx", "123,,,", []Row{
{
Metric: "foo",
Value: 123,
},
})
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3540
f("1:label:mytest,2:time:rfc3339,3:metric:M10,4:metric:M20,5:metric:M30,6:metric:M40,7:metric:M50,8:metric:M60",
`test,2022-12-25T16:57:12+01:00,10,20,30,,,60,70,80`, []Row{
{
Metric: "M10",
Tags: []Tag{
{
Key: "mytest",
Value: "test",
},
},
Timestamp: 1671983832000,
Value: 10,
},
{
Metric: "M20",
Tags: []Tag{
{
Key: "mytest",
Value: "test",
},
},
Timestamp: 1671983832000,
Value: 20,
},
{
Metric: "M30",
Tags: []Tag{
{
Key: "mytest",
Value: "test",
},
},
Timestamp: 1671983832000,
Value: 30,
},
{
Metric: "M60",
Tags: []Tag{
{
Key: "mytest",
Value: "test",
},
},
Timestamp: 1671983832000,
Value: 60,
},
})
// rfc3339 with millisecond precision
// see https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5837
f("1:label:mytest,2:time:rfc3339,3:metric:M10,4:metric:M20,5:metric:M30,6:metric:M40,7:metric:M50,8:metric:M60",
`test,2022-12-25T16:57:12.000+01:00,10,20,30,,,60,70,80`, []Row{
{
Metric: "M10",
Tags: []Tag{
{
Key: "mytest",
Value: "test",
},
},
Timestamp: 1671983832000,
Value: 10,
},
{
Metric: "M20",
Tags: []Tag{
{
Key: "mytest",
Value: "test",
},
},
Timestamp: 1671983832000,
Value: 20,
},
{
Metric: "M30",
Tags: []Tag{
{
Key: "mytest",
Value: "test",
},
},
Timestamp: 1671983832000,
Value: 30,
},
{
Metric: "M60",
Tags: []Tag{
{
Key: "mytest",
Value: "test",
},
},
Timestamp: 1671983832000,
Value: 60,
},
})
}