diff --git a/docs/VictoriaLogs/LogsQL.md b/docs/VictoriaLogs/LogsQL.md index da9257cf13..e15007fbb4 100644 --- a/docs/VictoriaLogs/LogsQL.md +++ b/docs/VictoriaLogs/LogsQL.md @@ -259,16 +259,16 @@ _time:1h AND error The following formats are supported for `_time` filter: -- `_time:duration`, the equivalent to `_time:(now-duration, now]`. It matches logs with timestamps on the time range `(now-duration, now]`. Examples: +- `_time:duration` matches logs with timestamps on the time range `(now-duration, now]`. Examples: - `_time:5m` - returns logs for the last 5 minutes - `_time:2.5d15m42.345s` - returns logs for the last 2.5 days, 15 minutes and 42.345 seconds - `_time:1y` - returns logs for the last year -- `_time:YYYY-MM-DD` - matches all the logs for the particular day by UTC. For example, `_time:2023-04-25`. -- `_time:YYYY-MM` - matches all the logs for the particular month by UTC. For example, `_time:2023-02`. -- `_time:YYYY` - matches all the logs for the particular year by UTC. For example, `_time:2023`. -- `_time:YYYY-MM-DDTHH` - matches all the logs for the particular hour by UTC. For example, `_time:2023-04-25T22`. -- `_time:YYYY-MM-DDTHH:MM` - matches all the logs for the particular minute by UTC. For example, `_time:2023-04-25T22:45`. -- `_time:YYYY-MM-DDTHH:MM:SS` - matches all the logs for the particular second by UTC. For example, `_time:2023-04-25T22:45:59`. +- `_time:YYYY-MM-DD` - matches all the logs for the particular day by UTC. For example, `_time:2023-04-25` matches logs on April 25, 2023 by UTC. +- `_time:YYYY-MM` - matches all the logs for the particular month by UTC. For example, `_time:2023-02` matches logs on February, 2023 by UTC. +- `_time:YYYY` - matches all the logs for the particular year by UTC. For example, `_time:2023` matches logs on 2023 by UTC. +- `_time:YYYY-MM-DDTHH` - matches all the logs for the particular hour by UTC. For example, `_time:2023-04-25T22` matches logs on April 25, 2023 at 22 hour by UTC. +- `_time:YYYY-MM-DDTHH:MM` - matches all the logs for the particular minute by UTC. For example, `_time:2023-04-25T22:45` matches logs on April 25, 2023 at 22:45 by UTC. +- `_time:YYYY-MM-DDTHH:MM:SS` - matches all the logs for the particular second by UTC. For example, `_time:2023-04-25T22:45:59` matches logs on April 25, 2023 at 22:45:59 by UTC. - `_time:[min_time, max_time]` - matches logs on the time range `[min_time, max_time]`, including both `min_time` and `max_time`. The `min_time` and `max_time` can contain any format specified [here](https://docs.victoriametrics.com/#timestamp-formats). For example, `_time:[2023-04-01, 2023-04-30]` matches logs for the whole April, 2023 by UTC, e.g. it is equivalent to `_time:2023-04`. @@ -277,8 +277,14 @@ The following formats are supported for `_time` filter: For example, `_time:[2023-02-01, 2023-03-01)` matches logs for the whole February, 2023 by UTC, e.g. it is equivalent to `_time:2023-02`. It is possible to specify time zone offset for all the absolute time formats by appending `+hh:mm` or `-hh:mm` suffix. -For example, `_time:2023-04-25+05:30` matches all the log messages on April 25, 2023 by India time zone, -while `_time:2023-02-07:00` matches all the log messages from February, 2023 by California time zone. +For example, `_time:2023-04-25+05:30` matches all the logs on April 25, 2023 by India time zone, +while `_time:2023-02-07:00` matches all the logs on February, 2023 by California time zone. + +It is possible to specify generic offset for the selected time range by appending `offset` after the `_time` filter. Examples: + +- `_time:5m offset 1h` matches logs on the time range `(now-1h5m, now-1h]`. +- `_time:2023-07 offset 5h30m` matches logs on July, 2023 by UTC with offset 5h30m. +- `_time:[2023-02-01, 2023-03-01) offset 1w` matches logs the week before the time range `[2023-02-01, 2023-03-01)` by UTC. Performance tips: diff --git a/lib/logstorage/parser.go b/lib/logstorage/parser.go index 0605ca82bb..0200289262 100644 --- a/lib/logstorage/parser.go +++ b/lib/logstorage/parser.go @@ -429,7 +429,7 @@ func parseFilterForPhrase(lex *lexer, phrase, fieldName string) (filter, error) } switch fieldName { case "_time": - return parseTimeFilter(lex) + return parseTimeFilterWithOffset(lex) case "_stream": return parseStreamFilter(lex) default: @@ -800,6 +800,29 @@ func startsWithYear(s string) bool { return c == '-' || c == '+' || c == 'Z' || c == 'z' } +func parseTimeFilterWithOffset(lex *lexer) (*timeFilter, error) { + tf, err := parseTimeFilter(lex) + if err != nil { + return nil, err + } + if !lex.isKeyword("offset") { + return tf, nil + } + if !lex.mustNextToken() { + return nil, fmt.Errorf("missing offset for _time filter %s", tf) + } + s := getCompoundToken(lex) + d, err := promutils.ParseDuration(s) + if err != nil { + return nil, fmt.Errorf("cannot parse offset for _time filter %s: %w", tf, err) + } + offset := int64(d) + tf.minTimestamp -= offset + tf.maxTimestamp -= offset + tf.stringRepr += " offset " + s + return tf, nil +} + func parseTimeFilter(lex *lexer) (*timeFilter, error) { startTimeInclude := false switch { @@ -809,7 +832,8 @@ func parseTimeFilter(lex *lexer) (*timeFilter, error) { startTimeInclude = false default: s := getCompoundToken(lex) - if strings.ToLower(s) == "now" || startsWithYear(s) { + sLower := strings.ToLower(s) + if sLower == "now" || startsWithYear(s) { // Parse '_time:YYYY-MM-DD', which transforms to '_time:[YYYY-MM-DD, YYYY-MM-DD+1)' t, err := promutils.ParseTimeAt(s, float64(lex.currentTimestamp)/1e9) if err != nil { diff --git a/lib/logstorage/parser_test.go b/lib/logstorage/parser_test.go index edd7d9a043..22360f6fed 100644 --- a/lib/logstorage/parser_test.go +++ b/lib/logstorage/parser_test.go @@ -99,8 +99,12 @@ func TestParseTimeDuration(t *testing.T) { } } f("5m", 5*time.Minute) + f("5m offset 1h", 5*time.Minute) + f("5m offset -3.5h5m45s", 5*time.Minute) f("-5.5m", 5*time.Minute+30*time.Second) + f("-5.5m offset 1d5m", 5*time.Minute+30*time.Second) f("3d2h12m34s45ms", 3*24*time.Hour+2*time.Hour+12*time.Minute+34*time.Second+45*time.Millisecond) + f("3d2h12m34s45ms offset 10ms", 3*24*time.Hour+2*time.Hour+12*time.Minute+34*time.Second+45*time.Millisecond) } func TestParseTimeRange(t *testing.T) { @@ -258,10 +262,16 @@ func TestParseTimeRange(t *testing.T) { maxTimestamp = time.Date(2023, time.April, 7, 0, 0, 0, 0, time.UTC).UnixNano() - 1 f(`(2023-03-01T21:20,2023-04-06]`, minTimestamp, maxTimestamp) - // _time:[start, end] + // _time:[start, end] with timezone minTimestamp = time.Date(2023, time.February, 28, 21, 40, 0, 0, time.UTC).UnixNano() maxTimestamp = time.Date(2023, time.April, 7, 0, 0, 0, 0, time.UTC).UnixNano() - 1 f(`[2023-03-01+02:20,2023-04-06T23]`, minTimestamp, maxTimestamp) + + // _time:[start, end] with timezone and offset + offset := int64(30*time.Minute + 5*time.Second) + minTimestamp = time.Date(2023, time.February, 28, 21, 40, 0, 0, time.UTC).UnixNano() - offset + maxTimestamp = time.Date(2023, time.April, 7, 0, 0, 0, 0, time.UTC).UnixNano() - 1 - offset + f(`[2023-03-01+02:20,2023-04-06T23] offset 30m5s`, minTimestamp, maxTimestamp) } func TestParseSequenceFilter(t *testing.T) { @@ -863,6 +873,8 @@ func TestParseQueryFailure(t *testing.T) { f("_time:[2023-01-02T04:05:06-12,2023]") f("_time:2023-01-02T04:05:06.789") f("_time:234foo") + f("_time:5m offset") + f("_time:10m offset foobar") // long query with error f(`very long query with error aaa ffdfd fdfdfd fdfd:( ffdfdfdfdfd`)