mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-18 14:40:26 +01:00
276 lines
8.7 KiB
Go
276 lines
8.7 KiB
Go
package logstorage
|
|
|
|
import (
|
|
"testing"
|
|
)
|
|
|
|
func TestParsePipeUnpackSyslogSuccess(t *testing.T) {
|
|
f := func(pipeStr string) {
|
|
t.Helper()
|
|
expectParsePipeSuccess(t, pipeStr)
|
|
}
|
|
|
|
f(`unpack_syslog`)
|
|
f(`unpack_syslog offset 6h30m`)
|
|
f(`unpack_syslog offset -6h30m`)
|
|
f(`unpack_syslog keep_original_fields`)
|
|
f(`unpack_syslog offset -6h30m keep_original_fields`)
|
|
f(`unpack_syslog if (a:x)`)
|
|
f(`unpack_syslog if (a:x) keep_original_fields`)
|
|
f(`unpack_syslog if (a:x) offset 2h keep_original_fields`)
|
|
f(`unpack_syslog from x`)
|
|
f(`unpack_syslog from x keep_original_fields`)
|
|
f(`unpack_syslog if (a:x) from x`)
|
|
f(`unpack_syslog from x result_prefix abc`)
|
|
f(`unpack_syslog from x offset 2h30m result_prefix abc`)
|
|
f(`unpack_syslog if (a:x) from x result_prefix abc`)
|
|
f(`unpack_syslog result_prefix abc`)
|
|
f(`unpack_syslog if (a:x) result_prefix abc`)
|
|
f(`unpack_syslog if (a:x) offset -1h result_prefix abc`)
|
|
}
|
|
|
|
func TestParsePipeUnpackSyslogFailure(t *testing.T) {
|
|
f := func(pipeStr string) {
|
|
t.Helper()
|
|
expectParsePipeFailure(t, pipeStr)
|
|
}
|
|
|
|
f(`unpack_syslog foo`)
|
|
f(`unpack_syslog if`)
|
|
f(`unpack_syslog offset`)
|
|
f(`unpack_syslog if (x:y) foobar`)
|
|
f(`unpack_syslog from`)
|
|
f(`unpack_syslog from x y`)
|
|
f(`unpack_syslog from x if`)
|
|
f(`unpack_syslog from x result_prefix`)
|
|
f(`unpack_syslog from x result_prefix a b`)
|
|
f(`unpack_syslog from x result_prefix a if`)
|
|
f(`unpack_syslog result_prefix`)
|
|
f(`unpack_syslog result_prefix a b`)
|
|
f(`unpack_syslog result_prefix a if`)
|
|
}
|
|
|
|
func TestPipeUnpackSyslog(t *testing.T) {
|
|
f := func(pipeStr string, rows, rowsExpected [][]Field) {
|
|
t.Helper()
|
|
expectPipeResults(t, pipeStr, rows, rowsExpected)
|
|
}
|
|
|
|
// no skip empty results
|
|
f("unpack_syslog", [][]Field{
|
|
{
|
|
{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
{"foo", "321"},
|
|
},
|
|
}, [][]Field{
|
|
{
|
|
{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
{"foo", "321"},
|
|
{"priority", "165"},
|
|
{"facility", "20"},
|
|
{"severity", "5"},
|
|
{"format", "rfc5424"},
|
|
{"timestamp", "2023-06-03T17:42:32.123456789Z"},
|
|
{"hostname", "mymachine.example.com"},
|
|
{"app_name", "appname"},
|
|
{"proc_id", "12345"},
|
|
{"msg_id", "ID47"},
|
|
{"message", "This is a test message with structured data"},
|
|
},
|
|
})
|
|
|
|
// keep original fields
|
|
f("unpack_syslog keep_original_fields", [][]Field{
|
|
{
|
|
{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
{"foo", "321"},
|
|
{"app_name", "foobar"},
|
|
{"msg_id", "baz"},
|
|
},
|
|
}, [][]Field{
|
|
{
|
|
{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
{"foo", "321"},
|
|
{"priority", "165"},
|
|
{"facility", "20"},
|
|
{"severity", "5"},
|
|
{"format", "rfc5424"},
|
|
{"timestamp", "2023-06-03T17:42:32.123456789Z"},
|
|
{"hostname", "mymachine.example.com"},
|
|
{"app_name", "foobar"},
|
|
{"proc_id", "12345"},
|
|
{"msg_id", "baz"},
|
|
{"message", "This is a test message with structured data"},
|
|
},
|
|
})
|
|
|
|
// unpack from other field
|
|
f("unpack_syslog from x", [][]Field{
|
|
{
|
|
{"x", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
},
|
|
}, [][]Field{
|
|
{
|
|
{"x", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
{"priority", "165"},
|
|
{"facility", "20"},
|
|
{"severity", "5"},
|
|
{"format", "rfc5424"},
|
|
{"timestamp", "2023-06-03T17:42:32.123456789Z"},
|
|
{"hostname", "mymachine.example.com"},
|
|
{"app_name", "appname"},
|
|
{"proc_id", "12345"},
|
|
{"msg_id", "ID47"},
|
|
{"message", "This is a test message with structured data"},
|
|
},
|
|
})
|
|
|
|
// offset should be ignored when parsing non-rfc3164 messages
|
|
f("unpack_syslog from x offset 2h30m", [][]Field{
|
|
{
|
|
{"x", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
},
|
|
}, [][]Field{
|
|
{
|
|
{"x", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
{"priority", "165"},
|
|
{"facility", "20"},
|
|
{"severity", "5"},
|
|
{"format", "rfc5424"},
|
|
{"timestamp", "2023-06-03T17:42:32.123456789Z"},
|
|
{"hostname", "mymachine.example.com"},
|
|
{"app_name", "appname"},
|
|
{"proc_id", "12345"},
|
|
{"msg_id", "ID47"},
|
|
{"message", "This is a test message with structured data"},
|
|
},
|
|
})
|
|
|
|
// failed if condition
|
|
f("unpack_syslog if (foo:bar)", [][]Field{
|
|
{
|
|
{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
},
|
|
}, [][]Field{
|
|
{
|
|
{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
},
|
|
})
|
|
|
|
// matched if condition
|
|
f("unpack_syslog if (appname)", [][]Field{
|
|
{
|
|
{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
},
|
|
}, [][]Field{
|
|
{
|
|
{"_msg", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
{"priority", "165"},
|
|
{"facility", "20"},
|
|
{"severity", "5"},
|
|
{"format", "rfc5424"},
|
|
{"timestamp", "2023-06-03T17:42:32.123456789Z"},
|
|
{"hostname", "mymachine.example.com"},
|
|
{"app_name", "appname"},
|
|
{"proc_id", "12345"},
|
|
{"msg_id", "ID47"},
|
|
{"message", "This is a test message with structured data"},
|
|
},
|
|
})
|
|
|
|
// single row, unpack from missing field
|
|
f("unpack_syslog from x", [][]Field{
|
|
{
|
|
{"_msg", `foo=bar`},
|
|
},
|
|
}, [][]Field{
|
|
{
|
|
{"_msg", `foo=bar`},
|
|
},
|
|
})
|
|
|
|
// single row, unpack from non-syslog field
|
|
f("unpack_syslog from x", [][]Field{
|
|
{
|
|
{"x", `foobar`},
|
|
},
|
|
}, [][]Field{
|
|
{
|
|
{"x", `foobar`},
|
|
{"format", "rfc3164"},
|
|
{"message", "foobar"},
|
|
},
|
|
})
|
|
|
|
// multiple rows with distinct number of fields
|
|
f("unpack_syslog from x result_prefix qwe_", [][]Field{
|
|
{
|
|
{"x", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
},
|
|
{
|
|
{"x", `<163>1 2024-12-13T18:21:43Z mymachine.example.com appname2 345 ID7 - foobar`},
|
|
{"y", `z=bar`},
|
|
},
|
|
}, [][]Field{
|
|
{
|
|
{"x", `<165>1 2023-06-03T17:42:32.123456789Z mymachine.example.com appname 12345 ID47 - This is a test message with structured data`},
|
|
{"qwe_priority", "165"},
|
|
{"qwe_facility", "20"},
|
|
{"qwe_severity", "5"},
|
|
{"qwe_format", "rfc5424"},
|
|
{"qwe_timestamp", "2023-06-03T17:42:32.123456789Z"},
|
|
{"qwe_hostname", "mymachine.example.com"},
|
|
{"qwe_app_name", "appname"},
|
|
{"qwe_proc_id", "12345"},
|
|
{"qwe_msg_id", "ID47"},
|
|
{"qwe_message", "This is a test message with structured data"},
|
|
},
|
|
{
|
|
{"x", `<163>1 2024-12-13T18:21:43Z mymachine.example.com appname2 345 ID7 - foobar`},
|
|
{"y", `z=bar`},
|
|
{"qwe_priority", "163"},
|
|
{"qwe_facility", "20"},
|
|
{"qwe_severity", "3"},
|
|
{"qwe_format", "rfc5424"},
|
|
{"qwe_timestamp", "2024-12-13T18:21:43Z"},
|
|
{"qwe_hostname", "mymachine.example.com"},
|
|
{"qwe_app_name", "appname2"},
|
|
{"qwe_proc_id", "345"},
|
|
{"qwe_msg_id", "ID7"},
|
|
{"qwe_message", "foobar"},
|
|
},
|
|
})
|
|
}
|
|
|
|
func TestPipeUnpackSyslogUpdateNeededFields(t *testing.T) {
|
|
f := func(s string, neededFields, unneededFields, neededFieldsExpected, unneededFieldsExpected string) {
|
|
t.Helper()
|
|
expectPipeNeededFields(t, s, neededFields, unneededFields, neededFieldsExpected, unneededFieldsExpected)
|
|
}
|
|
|
|
// all the needed fields
|
|
f("unpack_syslog", "*", "", "*", "")
|
|
f("unpack_syslog keep_original_fields", "*", "", "*", "")
|
|
f("unpack_syslog if (y:z) from x", "*", "", "*", "")
|
|
|
|
// all the needed fields, unneeded fields do not intersect with src
|
|
f("unpack_syslog from x", "*", "f1,f2", "*", "f1,f2")
|
|
f("unpack_syslog if (y:z) from x", "*", "f1,f2", "*", "f1,f2")
|
|
f("unpack_syslog if (f1:z) from x", "*", "f1,f2", "*", "f2")
|
|
|
|
// all the needed fields, unneeded fields intersect with src
|
|
f("unpack_syslog from x", "*", "f2,x", "*", "f2")
|
|
f("unpack_syslog if (y:z) from x", "*", "f2,x", "*", "f2")
|
|
f("unpack_syslog if (f2:z) from x", "*", "f1,f2,x", "*", "f1")
|
|
|
|
// needed fields do not intersect with src
|
|
f("unpack_syslog from x", "f1,f2", "", "f1,f2,x", "")
|
|
f("unpack_syslog if (y:z) from x", "f1,f2", "", "f1,f2,x,y", "")
|
|
f("unpack_syslog if (f1:z) from x", "f1,f2", "", "f1,f2,x", "")
|
|
|
|
// needed fields intersect with src
|
|
f("unpack_syslog from x", "f2,x", "", "f2,x", "")
|
|
f("unpack_syslog if (y:z) from x", "f2,x", "", "f2,x,y", "")
|
|
f("unpack_syslog if (f2:z y:qwe) from x", "f2,x", "", "f2,x,y", "")
|
|
}
|