From 73812c71a5a004d50dbe2c0072eded7f8dadb913 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Thu, 11 May 2023 13:23:32 -0700 Subject: [PATCH] lib/promutils: properly parse time strings with timezones at ParseTime() --- README.md | 10 +++-- docs/CHANGELOG.md | 1 + docs/README.md | 8 ++-- docs/Single-server-VictoriaMetrics.md | 8 ++-- lib/promutils/time.go | 51 ++++++++++++++++++++------ lib/promutils/time_test.go | 53 +++++++++++++++++++++++++-- 6 files changed, 107 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 9f0d812a8..2b95f2958 100644 --- a/README.md +++ b/README.md @@ -282,7 +282,7 @@ http://:8428 Substitute `` with the hostname or IP address of VictoriaMetrics. -Then build graphs and dashboards for the created datasource using [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/) +Then build graphs and dashboards for the created datasource using [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/) or [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html). Alternatively, use VictoriaMetrics [datasource plugin](https://github.com/VictoriaMetrics/grafana-datasource) with support of extra features. @@ -817,9 +817,11 @@ in [query APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) in [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). - Unix timestamps in seconds with optional milliseconds after the point. For example, `1562529662.678`. -- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, '2022-03-29T01:02:03Z`. -- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`. -- Relative duration comparing to the current time. For example, `1h5m` means `one hour and five minutes ago`. +- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, `2022-03-29T01:02:03Z` or `2022-03-29T01:02:03+02:30`. +- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`, `2022-03-29T01:02:03`. + The partial RFC3339 time is in UTC timezone by default. It is possible to specify timezone there by adding `+hh:mm` or `-hh:mm` suffix to partial time. + For example, `2022-03-01+06:30` is `2022-03-01` at `06:30` timezone. +- Relative duration comparing to the current time. For example, `1h5m`, `-1h5m` or `now-1h5m` means `one hour and five minutes ago`, while `now` means `now`. ## Graphite API usage diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c2403cecd..a21ae762c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -29,6 +29,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: deprecate `-bigMergeConcurrency` command-line flag, since improper configuration for this flag frequently led to uncontrolled growth of unmerged parts, which, in turn, could lead to queries slowdown and increased CPU usage. The concurrency for [background merges](https://docs.victoriametrics.com/#storage) can be controlled via `-smallMergeConcurrency` command-line flag, though it isn't recommended to change this flag in general case. * FEATURE: do not execute the incoming request if it has been canceled by the client before the execution start. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4223). +* FEATURE: support time formats with timezones. For example, `2024-01-02+02:00` means `January 2, 2024` at `+02:00` time zone. See [these docs](https://docs.victoriametrics.com/#timestamp-formats). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): support the ability to filter [consul_sd_configs](https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs) targets in more optimal way via new `filter` option. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4183). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for [consulagent_sd_configs](https://docs.victoriametrics.com/sd_configs.html#consulagent_sd_configs). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3953). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): emit a warning if too small value is passed to `-remoteWrite.maxDiskUsagePerURL` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4195). diff --git a/docs/README.md b/docs/README.md index 2958424e2..50d81bed4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -818,9 +818,11 @@ in [query APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) in [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). - Unix timestamps in seconds with optional milliseconds after the point. For example, `1562529662.678`. -- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, '2022-03-29T01:02:03Z`. -- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`. -- Relative duration comparing to the current time. For example, `1h5m` means `one hour and five minutes ago`. +- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, `2022-03-29T01:02:03Z` or `2022-03-29T01:02:03+02:30`. +- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`, `2022-03-29T01:02:03`. + The partial RFC3339 time is in UTC timezone by default. It is possible to specify timezone there by adding `+hh:mm` or `-hh:mm` suffix to partial time. + For example, `2022-03-01+06:30` is `2022-03-01` at `06:30` timezone. +- Relative duration comparing to the current time. For example, `1h5m`, `-1h5m` or `now-1h5m` means `one hour and five minutes ago`, while `now` means `now`. ## Graphite API usage diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index ec3372200..dea0e6b03 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -821,9 +821,11 @@ in [query APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) in [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). - Unix timestamps in seconds with optional milliseconds after the point. For example, `1562529662.678`. -- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, '2022-03-29T01:02:03Z`. -- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`. -- Relative duration comparing to the current time. For example, `1h5m` means `one hour and five minutes ago`. +- [RFC3339](https://www.ietf.org/rfc/rfc3339.txt). For example, `2022-03-29T01:02:03Z` or `2022-03-29T01:02:03+02:30`. +- Partial RFC3339. Examples: `2022`, `2022-03`, `2022-03-29`, `2022-03-29T01`, `2022-03-29T01:02`, `2022-03-29T01:02:03`. + The partial RFC3339 time is in UTC timezone by default. It is possible to specify timezone there by adding `+hh:mm` or `-hh:mm` suffix to partial time. + For example, `2022-03-01+06:30` is `2022-03-01` at `06:30` timezone. +- Relative duration comparing to the current time. For example, `1h5m`, `-1h5m` or `now-1h5m` means `one hour and five minutes ago`, while `now` means `now`. ## Graphite API usage diff --git a/lib/promutils/time.go b/lib/promutils/time.go index e0490301b..432c2dbc0 100644 --- a/lib/promutils/time.go +++ b/lib/promutils/time.go @@ -1,6 +1,7 @@ package promutils import ( + "fmt" "strconv" "strings" "time" @@ -12,8 +13,35 @@ import ( // // It returns unix timestamp in seconds. func ParseTime(s string) (float64, error) { - if len(s) > 0 && (s[len(s)-1] != 'Z' && s[len(s)-1] > '9' || s[0] == '-') { + if s == "now" { + return float64(time.Now().UnixNano()) / 1e9, nil + } + sOrig := s + tzOffset := float64(0) + if len(sOrig) > 6 { + // Try parsing timezone offset + tz := sOrig[len(sOrig)-6:] + if (tz[0] == '-' || tz[0] == '+') && tz[3] == ':' { + isPlus := tz[0] == '+' + hour, err := strconv.ParseUint(tz[1:3], 10, 64) + if err != nil { + return 0, fmt.Errorf("cannot parse hour from timezone offset %q: %w", tz, err) + } + minute, err := strconv.ParseUint(tz[4:], 10, 64) + if err != nil { + return 0, fmt.Errorf("cannot parse minute from timezone offset %q: %w", tz, err) + } + tzOffset = float64(hour*3600 + minute*60) + if isPlus { + tzOffset = -tzOffset + } + s = sOrig[:len(sOrig)-6] + } + } + s = strings.TrimSuffix(s, "Z") + if len(s) > 0 && (s[len(s)-1] > '9' || s[0] == '-') || strings.HasPrefix(s, "now") { // Parse duration relative to the current time + s = strings.TrimPrefix(s, "now") d, err := ParseDuration(s) if err != nil { return 0, err @@ -30,11 +58,11 @@ func ParseTime(s string) (float64, error) { if err != nil { return 0, err } - return float64(t.UnixNano()) / 1e9, nil + return tzOffset + float64(t.UnixNano())/1e9, nil } - if !strings.Contains(s, "-") { - // Parse the timestamp in milliseconds - return strconv.ParseFloat(s, 64) + if !strings.Contains(sOrig, "-") { + // Parse the timestamp in seconds + return strconv.ParseFloat(sOrig, 64) } if len(s) == 7 { // Parse YYYY-MM @@ -42,7 +70,7 @@ func ParseTime(s string) (float64, error) { if err != nil { return 0, err } - return float64(t.UnixNano()) / 1e9, nil + return tzOffset + float64(t.UnixNano())/1e9, nil } if len(s) == 10 { // Parse YYYY-MM-DD @@ -50,7 +78,7 @@ func ParseTime(s string) (float64, error) { if err != nil { return 0, err } - return float64(t.UnixNano()) / 1e9, nil + return tzOffset + float64(t.UnixNano())/1e9, nil } if len(s) == 13 { // Parse YYYY-MM-DDTHH @@ -58,7 +86,7 @@ func ParseTime(s string) (float64, error) { if err != nil { return 0, err } - return float64(t.UnixNano()) / 1e9, nil + return tzOffset + float64(t.UnixNano())/1e9, nil } if len(s) == 16 { // Parse YYYY-MM-DDTHH:MM @@ -66,7 +94,7 @@ func ParseTime(s string) (float64, error) { if err != nil { return 0, err } - return float64(t.UnixNano()) / 1e9, nil + return tzOffset + float64(t.UnixNano())/1e9, nil } if len(s) == 19 { // Parse YYYY-MM-DDTHH:MM:SS @@ -74,9 +102,10 @@ func ParseTime(s string) (float64, error) { if err != nil { return 0, err } - return float64(t.UnixNano()) / 1e9, nil + return tzOffset + float64(t.UnixNano())/1e9, nil } - t, err := time.Parse(time.RFC3339, s) + // Parse RFC3339 + t, err := time.Parse(time.RFC3339, sOrig) if err != nil { return 0, err } diff --git a/lib/promutils/time_test.go b/lib/promutils/time_test.go index e085fdc8e..740805227 100644 --- a/lib/promutils/time_test.go +++ b/lib/promutils/time_test.go @@ -19,34 +19,81 @@ func TestParseTimeSuccess(t *testing.T) { } now := float64(time.Now().UnixNano()) / 1e9 + // duration relative to the current time + f("now", now) f("1h5s", now-3605) // negative duration relative to the current time f("-5m", now-5*60) + f("-123", now-123) + f("-123.456", now-123.456) + f("now-1h5m", now-(3600+5*60)) // Year f("2023", 1.6725312e+09) + f("2023Z", 1.6725312e+09) + f("2023+02:00", 1.672524e+09) + f("2023-02:00", 1.6725384e+09) // Year and month f("2023-05", 1.6828992e+09) + f("2023-05Z", 1.6828992e+09) + f("2023-05+02:00", 1.682892e+09) + f("2023-05-02:00", 1.6829064e+09) // Year, month and day f("2023-05-20", 1.6845408e+09) + f("2023-05-20Z", 1.6845408e+09) + f("2023-05-20+02:30", 1.6845318e+09) + f("2023-05-20-02:30", 1.6845498e+09) // Year, month, day and hour f("2023-05-20T04", 1.6845552e+09) + f("2023-05-20T04Z", 1.6845552e+09) + f("2023-05-20T04+02:30", 1.6845462e+09) + f("2023-05-20T04-02:30", 1.6845642e+09) // Year, month, day, hour and minute f("2023-05-20T04:57", 1.68455862e+09) + f("2023-05-20T04:57Z", 1.68455862e+09) + f("2023-05-20T04:57+02:30", 1.68454962e+09) + f("2023-05-20T04:57-02:30", 1.68456762e+09) // Year, month, day, hour, minute and second f("2023-05-20T04:57:43", 1.684558663e+09) - - // RFC3339 f("2023-05-20T04:57:43Z", 1.684558663e+09) f("2023-05-20T04:57:43+02:30", 1.684549663e+09) f("2023-05-20T04:57:43-02:30", 1.684567663e+09) + + // milliseconds f("2023-05-20T04:57:43.123Z", 1.6845586631230001e+09) - f("2023-05-20T04:57:43.123456789Z", 1.6845586631230001e+09) + f("2023-05-20T04:57:43.123456789+02:30", 1.6845496631234567e+09) + f("2023-05-20T04:57:43.123456789-02:30", 1.6845676631234567e+09) +} + +func TestParseTimeFailure(t *testing.T) { + f := func(s string) { + t.Helper() + ts, err := ParseTime(s) + if ts != 0 { + t.Fatalf("unexpected time parsed: %f; want 0", ts) + } + if err == nil { + t.Fatalf("expecting non-nil error") + } + } + + f("") + f("23-45:50") + f("1223-fo:ba") + f("1223-12:ba") + f("23-45") + f("-123foobar") + f("2oo5") + f("2oob-a5") + f("2oob-ar-a5") + f("2oob-ar-azTx5") + f("2oob-ar-azTxx:y5") + f("2oob-ar-azTxx:yy:z5") }