From d21a244641135ebbea4cf16dfaef12a0424e0424 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 11 May 2023 13:23:32 -0700 Subject: [PATCH] lib/promutils: properly parse time strings with timezones at ParseTime() --- docs/CHANGELOG.md | 1 + docs/README.md | 8 ++-- docs/Single-server-VictoriaMetrics.md | 8 ++-- lib/promutils/time.go | 51 ++++++++++++++++++++------ lib/promutils/time_test.go | 53 +++++++++++++++++++++++++-- 5 files changed, 101 insertions(+), 20 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c2403cecd5..a21ae762c0 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -29,6 +29,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: deprecate `-bigMergeConcurrency` command-line flag, since improper configuration for this flag frequently led to uncontrolled growth of unmerged parts, which, in turn, could lead to queries slowdown and increased CPU usage. The concurrency for [background merges](https://docs.victoriametrics.com/#storage) can be controlled via `-smallMergeConcurrency` command-line flag, though it isn't recommended to change this flag in general case. * FEATURE: do not execute the incoming request if it has been canceled by the client before the execution start. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4223). +* FEATURE: support time formats with timezones. For example, `2024-01-02+02:00` means `January 2, 2024` at `+02:00` time zone. See [these docs](https://docs.victoriametrics.com/#timestamp-formats). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): support the ability to filter [consul_sd_configs](https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs) targets in more optimal way via new `filter` option. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4183). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [consulagent_sd_configs](https://docs.victoriametrics.com/sd_configs.html#consulagent_sd_configs). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3953). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): emit a warning if too small value is passed to `-remoteWrite.maxDiskUsagePerURL` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4195). diff --git a/docs/README.md b/docs/README.md index 2958424e25..50d81bed48 100644 --- a/docs/README.md +++ b/docs/README.md @@ -818,9 +818,11 @@ in [query APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) in [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). - Unix timestamps in seconds with optional milliseconds after the point. For example, `1562529662.678`. -- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, '2022-03-29T01:02:03Z`. -- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`. -- Relative duration comparing to the current time. For example, `1h5m` means `one hour and five minutes ago`. +- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, `2022-03-29T01:02:03Z` or `2022-03-29T01:02:03+02:30`. +- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`, `2022-03-29T01:02:03`. + The partial RFC3339 time is in UTC timezone by default. It is possible to specify timezone there by adding `+hh:mm` or `-hh:mm` suffix to partial time. + For example, `2022-03-01+06:30` is `2022-03-01` at `06:30` timezone. +- Relative duration comparing to the current time. For example, `1h5m`, `-1h5m` or `now-1h5m` means `one hour and five minutes ago`, while `now` means `now`. ## Graphite API usage diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index ec3372200b..dea0e6b037 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -821,9 +821,11 @@ in [query APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) in [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). - Unix timestamps in seconds with optional milliseconds after the point. For example, `1562529662.678`. -- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, '2022-03-29T01:02:03Z`. -- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`. -- Relative duration comparing to the current time. For example, `1h5m` means `one hour and five minutes ago`. +- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, `2022-03-29T01:02:03Z` or `2022-03-29T01:02:03+02:30`. +- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`, `2022-03-29T01:02:03`. + The partial RFC3339 time is in UTC timezone by default. It is possible to specify timezone there by adding `+hh:mm` or `-hh:mm` suffix to partial time. + For example, `2022-03-01+06:30` is `2022-03-01` at `06:30` timezone. +- Relative duration comparing to the current time. For example, `1h5m`, `-1h5m` or `now-1h5m` means `one hour and five minutes ago`, while `now` means `now`. ## Graphite API usage diff --git a/lib/promutils/time.go b/lib/promutils/time.go index e0490301b7..432c2dbc07 100644 --- a/lib/promutils/time.go +++ b/lib/promutils/time.go @@ -1,6 +1,7 @@ package promutils import ( + "fmt" "strconv" "strings" "time" @@ -12,8 +13,35 @@ import ( // // It returns unix timestamp in seconds. func ParseTime(s string) (float64, error) { - if len(s) > 0 && (s[len(s)-1] != 'Z' && s[len(s)-1] > '9' || s[0] == '-') { + if s == "now" { + return float64(time.Now().UnixNano()) / 1e9, nil + } + sOrig := s + tzOffset := float64(0) + if len(sOrig) > 6 { + // Try parsing timezone offset + tz := sOrig[len(sOrig)-6:] + if (tz[0] == '-' || tz[0] == '+') && tz[3] == ':' { + isPlus := tz[0] == '+' + hour, err := strconv.ParseUint(tz[1:3], 10, 64) + if err != nil { + return 0, fmt.Errorf("cannot parse hour from timezone offset %q: %w", tz, err) + } + minute, err := strconv.ParseUint(tz[4:], 10, 64) + if err != nil { + return 0, fmt.Errorf("cannot parse minute from timezone offset %q: %w", tz, err) + } + tzOffset = float64(hour*3600 + minute*60) + if isPlus { + tzOffset = -tzOffset + } + s = sOrig[:len(sOrig)-6] + } + } + s = strings.TrimSuffix(s, "Z") + if len(s) > 0 && (s[len(s)-1] > '9' || s[0] == '-') || strings.HasPrefix(s, "now") { // Parse duration relative to the current time + s = strings.TrimPrefix(s, "now") d, err := ParseDuration(s) if err != nil { return 0, err @@ -30,11 +58,11 @@ func ParseTime(s string) (float64, error) { if err != nil { return 0, err } - return float64(t.UnixNano()) / 1e9, nil + return tzOffset + float64(t.UnixNano())/1e9, nil } - if !strings.Contains(s, "-") { - // Parse the timestamp in milliseconds - return strconv.ParseFloat(s, 64) + if !strings.Contains(sOrig, "-") { + // Parse the timestamp in seconds + return strconv.ParseFloat(sOrig, 64) } if len(s) == 7 { // Parse YYYY-MM @@ -42,7 +70,7 @@ func ParseTime(s string) (float64, error) { if err != nil { return 0, err } - return float64(t.UnixNano()) / 1e9, nil + return tzOffset + float64(t.UnixNano())/1e9, nil } if len(s) == 10 { // Parse YYYY-MM-DD @@ -50,7 +78,7 @@ func ParseTime(s string) (float64, error) { if err != nil { return 0, err } - return float64(t.UnixNano()) / 1e9, nil + return tzOffset + float64(t.UnixNano())/1e9, nil } if len(s) == 13 { // Parse YYYY-MM-DDTHH @@ -58,7 +86,7 @@ func ParseTime(s string) (float64, error) { if err != nil { return 0, err } - return float64(t.UnixNano()) / 1e9, nil + return tzOffset + float64(t.UnixNano())/1e9, nil } if len(s) == 16 { // Parse YYYY-MM-DDTHH:MM @@ -66,7 +94,7 @@ func ParseTime(s string) (float64, error) { if err != nil { return 0, err } - return float64(t.UnixNano()) / 1e9, nil + return tzOffset + float64(t.UnixNano())/1e9, nil } if len(s) == 19 { // Parse YYYY-MM-DDTHH:MM:SS @@ -74,9 +102,10 @@ func ParseTime(s string) (float64, error) { if err != nil { return 0, err } - return float64(t.UnixNano()) / 1e9, nil + return tzOffset + float64(t.UnixNano())/1e9, nil } - t, err := time.Parse(time.RFC3339, s) + // Parse RFC3339 + t, err := time.Parse(time.RFC3339, sOrig) if err != nil { return 0, err } diff --git a/lib/promutils/time_test.go b/lib/promutils/time_test.go index e085fdc8e1..740805227f 100644 --- a/lib/promutils/time_test.go +++ b/lib/promutils/time_test.go @@ -19,34 +19,81 @@ func TestParseTimeSuccess(t *testing.T) { } now := float64(time.Now().UnixNano()) / 1e9 + // duration relative to the current time + f("now", now) f("1h5s", now-3605) // negative duration relative to the current time f("-5m", now-5*60) + f("-123", now-123) + f("-123.456", now-123.456) + f("now-1h5m", now-(3600+5*60)) // Year f("2023", 1.6725312e+09) + f("2023Z", 1.6725312e+09) + f("2023+02:00", 1.672524e+09) + f("2023-02:00", 1.6725384e+09) // Year and month f("2023-05", 1.6828992e+09) + f("2023-05Z", 1.6828992e+09) + f("2023-05+02:00", 1.682892e+09) + f("2023-05-02:00", 1.6829064e+09) // Year, month and day f("2023-05-20", 1.6845408e+09) + f("2023-05-20Z", 1.6845408e+09) + f("2023-05-20+02:30", 1.6845318e+09) + f("2023-05-20-02:30", 1.6845498e+09) // Year, month, day and hour f("2023-05-20T04", 1.6845552e+09) + f("2023-05-20T04Z", 1.6845552e+09) + f("2023-05-20T04+02:30", 1.6845462e+09) + f("2023-05-20T04-02:30", 1.6845642e+09) // Year, month, day, hour and minute f("2023-05-20T04:57", 1.68455862e+09) + f("2023-05-20T04:57Z", 1.68455862e+09) + f("2023-05-20T04:57+02:30", 1.68454962e+09) + f("2023-05-20T04:57-02:30", 1.68456762e+09) // Year, month, day, hour, minute and second f("2023-05-20T04:57:43", 1.684558663e+09) - - // RFC3339 f("2023-05-20T04:57:43Z", 1.684558663e+09) f("2023-05-20T04:57:43+02:30", 1.684549663e+09) f("2023-05-20T04:57:43-02:30", 1.684567663e+09) + + // milliseconds f("2023-05-20T04:57:43.123Z", 1.6845586631230001e+09) - f("2023-05-20T04:57:43.123456789Z", 1.6845586631230001e+09) + f("2023-05-20T04:57:43.123456789+02:30", 1.6845496631234567e+09) + f("2023-05-20T04:57:43.123456789-02:30", 1.6845676631234567e+09) +} + +func TestParseTimeFailure(t *testing.T) { + f := func(s string) { + t.Helper() + ts, err := ParseTime(s) + if ts != 0 { + t.Fatalf("unexpected time parsed: %f; want 0", ts) + } + if err == nil { + t.Fatalf("expecting non-nil error") + } + } + + f("") + f("23-45:50") + f("1223-fo:ba") + f("1223-12:ba") + f("23-45") + f("-123foobar") + f("2oo5") + f("2oob-a5") + f("2oob-ar-a5") + f("2oob-ar-azTx5") + f("2oob-ar-azTxx:y5") + f("2oob-ar-azTxx:yy:z5") }