Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-15 00:13:30 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

app/vmctl: add support for the different time format in the native binary protocol (#4189)

Browse Source 
* app/vmctl: add support for the different time format in the native binary protocol

* app/vmctl: update flag description, update CHANGELOG.md

* app/vmctl: add comment to exported function
This commit is contained in:
Dmytro Kozlov 2023-04-24 19:33:30 +03:00 committed by Aliaksandr Valialkin
parent 574a0559d5
commit 3da856af0c
No known key found for this signature in database
GPG Key ID: A72BEC6CD3D0DED1
5 changed files with 295 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/vmctl/flags.go
View File

@ -352,12 +352,12 @@ var (
},
&cli.StringFlag{
Name: vmNativeFilterTimeStart,
Usage: "The time filter may contain either unix timestamp in seconds or RFC3339 values. E.g. '2020-01-01T20:07:00Z'",
Usage: "The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#timestamp-formats",
Required: true,
},
&cli.StringFlag{
Name: vmNativeFilterTimeEnd,
Usage: "The time filter may contain either unix timestamp in seconds or RFC3339 values. E.g. '2020-01-01T20:07:00Z'",
Usage: "The time filter may contain different timestamp formats. See more details here https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#timestamp-formats",
},
&cli.StringFlag{
Name: vmNativeStepInterval,

105
app/vmctl/utils/time.go Normal file
View File

@ -0,0 +1,105 @@
package utils
import (
"fmt"
"strconv"
"strings"
"time"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
)
const (
// These values prevent from overflow when storing msec-precision time in int64.
minTimeMsecs = 0 // use 0 instead of `int64(-1<<63) / 1e6` because the storage engine doesn't actually support negative time
maxTimeMsecs = int64(1<<63-1) / 1e6
)
func parseTime(s string) (float64, error) {
if len(s) > 0 && (s[len(s)-1] != 'Z' && s[len(s)-1] > '9' || s[0] == '-') {
// Parse duration relative to the current time
d, err := promutils.ParseDuration(s)
if err != nil {
return 0, err
}
if d > 0 {
d = -d
}
t := time.Now().Add(d)
return float64(t.UnixNano()) / 1e9, nil
}
if len(s) == 4 {
// Parse YYYY
t, err := time.Parse("2006", s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
if !strings.Contains(s, "-") {
// Parse the timestamp in milliseconds
return strconv.ParseFloat(s, 64)
}
if len(s) == 7 {
// Parse YYYY-MM
t, err := time.Parse("2006-01", s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
if len(s) == 10 {
// Parse YYYY-MM-DD
t, err := time.Parse("2006-01-02", s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
if len(s) == 13 {
// Parse YYYY-MM-DDTHH
t, err := time.Parse("2006-01-02T15", s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
if len(s) == 16 {
// Parse YYYY-MM-DDTHH:MM
t, err := time.Parse("2006-01-02T15:04", s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
if len(s) == 19 {
// Parse YYYY-MM-DDTHH:MM:SS
t, err := time.Parse("2006-01-02T15:04:05", s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
t, err := time.Parse(time.RFC3339, s)
if err != nil {
return 0, err
}
return float64(t.UnixNano()) / 1e9, nil
}
// GetTime returns time from the given string.
func GetTime(s string) (time.Time, error) {
secs, err := parseTime(s)
if err != nil {
return time.Time{}, fmt.Errorf("cannot parse %s: %w", s, err)
}
msecs := int64(secs * 1e3)
if msecs < minTimeMsecs {
msecs = 0
}
if msecs > maxTimeMsecs {
msecs = maxTimeMsecs
}
return time.Unix(0, msecs*int64(time.Millisecond)), nil
}

182
app/vmctl/utils/time_test.go Normal file
View File

@ -0,0 +1,182 @@
package utils
import (
"testing"
"time"
)
func TestGetTime(t *testing.T) {
l, _ := time.LoadLocation("UTC")
tests := []struct {
name string
s string
want func() time.Time
wantErr bool
}{
{
name: "empty string",
s: "",
want: func() time.Time { return time.Time{} },
wantErr: true,
},
{
name: "only year",
s: "2019",
want: func() time.Time {
t := time.Date(2019, 1, 1, 0, 0, 0, 0, l)
return t
},
},
{
name: "year and month",
s: "2019-01",
want: func() time.Time {
t := time.Date(2019, 1, 1, 0, 0, 0, 0, l)
return t
},
},
{
name: "year and not first month",
s: "2019-02",
want: func() time.Time {
t := time.Date(2019, 2, 1, 0, 0, 0, 0, l)
return t
},
},
{
name: "year, month and day",
s: "2019-02-01",
want: func() time.Time {
t := time.Date(2019, 2, 1, 0, 0, 0, 0, l)
return t
},
},
{
name: "year, month and not first day",
s: "2019-02-10",
want: func() time.Time {
t := time.Date(2019, 2, 10, 0, 0, 0, 0, l)
return t
},
},
{
name: "year, month, day and time",
s: "2019-02-02T00",
want: func() time.Time {
t := time.Date(2019, 2, 2, 0, 0, 0, 0, l)
return t
},
},
{
name: "year, month, day and one hour time",
s: "2019-02-02T01",
want: func() time.Time {
t := time.Date(2019, 2, 2, 1, 0, 0, 0, l)
return t
},
},
{
name: "time with zero minutes",
s: "2019-02-02T01:00",
want: func() time.Time {
t := time.Date(2019, 2, 2, 1, 0, 0, 0, l)
return t
},
},
{
name: "time with one minute",
s: "2019-02-02T01:01",
want: func() time.Time {
t := time.Date(2019, 2, 2, 1, 1, 0, 0, l)
return t
},
},
{
name: "time with zero seconds",
s: "2019-02-02T01:01:00",
want: func() time.Time {
t := time.Date(2019, 2, 2, 1, 1, 0, 0, l)
return t
},
},
{
name: "timezone with one second",
s: "2019-02-02T01:01:01",
want: func() time.Time {
t := time.Date(2019, 2, 2, 1, 1, 1, 0, l)
return t
},
},
{
name: "time with two second and timezone",
s: "2019-07-07T20:01:02Z",
want: func() time.Time {
t := time.Date(2019, 7, 7, 20, 1, 02, 0, l)
return t
},
},
{
name: "time with seconds and timezone",
s: "2019-07-07T20:47:40+03:00",
want: func() time.Time {
l, _ = time.LoadLocation("Europe/Kiev")
t := time.Date(2019, 7, 7, 20, 47, 40, 0, l)
return t
},
},
{
name: "negative time",
s: "-292273086-05-16T16:47:06Z",
want: func() time.Time { return time.Time{} },
wantErr: true,
},
{
name: "float timestamp representation",
s: "1562529662.324",
want: func() time.Time {
t := time.Date(2019, 7, 7, 23, 01, 02, 324, l)
return t
},
},
{
name: "negative timestamp",
s: "-9223372036.855",
want: func() time.Time {
l, _ = time.LoadLocation("Europe/Kiev")
return time.Date(1970, 01, 01, 03, 00, 00, 00, l)
},
wantErr: false,
},
{
name: "big timestamp",
s: "9223372036.855",
want: func() time.Time {
l, _ = time.LoadLocation("Europe/Kiev")
t := time.Date(2262, 04, 12, 02, 47, 16, 855, l)
return t
},
wantErr: false,
},
{
name: "duration time",
s: "1h5m",
want: func() time.Time {
t := time.Now().Add(-1 * time.Hour).Add(-5 * time.Minute)
return t
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := GetTime(tt.s)
if (err != nil) != tt.wantErr {
t.Errorf("ParseTime() error = %v, wantErr %v", err, tt.wantErr)
return
}
w := tt.want()
if got.Unix() != w.Unix() {
t.Errorf("ParseTime() got = %v, want %v", got, w)
}
})
}
}

11
app/vmctl/vm_native.go
View File

@ -13,6 +13,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/limiter"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/native"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/stepper"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/utils"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmctl/vm"
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmselect/searchutils"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@ -48,18 +49,16 @@ func (p *vmNativeProcessor) run(ctx context.Context, silent bool) error {
startTime: time.Now(),
}
start, err := time.Parse(time.RFC3339, p.filter.TimeStart)
start, err := utils.GetTime(p.filter.TimeStart)
if err != nil {
return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %w",
vmNativeFilterTimeStart, p.filter.TimeStart, time.RFC3339, err)
return fmt.Errorf("failed to parse %s, provided: %s, error: %w", vmNativeFilterTimeStart, p.filter.TimeStart, err)
}
end := time.Now().In(start.Location())
if p.filter.TimeEnd != "" {
end, err = time.Parse(time.RFC3339, p.filter.TimeEnd)
end, err = utils.GetTime(p.filter.TimeEnd)
if err != nil {
return fmt.Errorf("failed to parse %s, provided: %s, expected format: %s, error: %w",
vmNativeFilterTimeEnd, p.filter.TimeEnd, time.RFC3339, err)
return fmt.Errorf("failed to parse %s, provided: %s, error: %w", vmNativeFilterTimeEnd, p.filter.TimeEnd, err)
}
}

1
docs/CHANGELOG.md
View File

@ -23,6 +23,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
* FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): add `created_at` field to the output of `/api/v1/backups` API and `vmbackupmanager backup list` command. See this [doc](https://docs.victoriametrics.com/vmbackupmanager.html#api-methods) for data format details.
* FEATURE: deprecate `-bigMergeConcurrency` command-line flag, since improper configuration for this flag frequently led to uncontrolled growth of unmerged parts, which, in turn, could lead to queries slowdown and increased CPU usage. The concurrency for [background merges](https://docs.victoriametrics.com/#storage) can be controlled via `-smallMergeConcurrency` command-line flag, though it isn't recommended to do in general case.
* FEATURE: introduce `-http.maxConcurrentRequests` command-line flag to protect VM components from resource exhaustion during unexpected spikes of HTTP requests. By default, the new flag's value is set to 0 which means no limits are applied.
* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support for the different time formats for `--vm-native-filter-time-start` and `--vm-native-filter-time-end` flags if the native binary protocol is used for migration. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4091).
* FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to filter incoming requests by IP. See [these docs](https://docs.victoriametrics.com/vmauth.html#ip-filters) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3491).
* BUGFIX: reduce the probability of sudden increase in the number of small parts on systems with small number of CPU cores.