From bcb2b8247cf9f6533931d4de84960b41bf0546bc Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Fri, 15 Dec 2023 12:36:28 +0100 Subject: [PATCH] vmctl: rename `vm-native-disable-retries` to `vm-native-disable-per-metric-migration` (#5476) The change supposed to better reflect the meaning of this flag. Signed-off-by: hagen1778 --- app/vmctl/flags.go | 10 +++++----- app/vmctl/main.go | 10 +++++----- app/vmctl/vm_native.go | 23 ++++++++++++----------- docs/CHANGELOG.md | 1 + docs/vmctl.md | 9 ++++++--- 5 files changed, 29 insertions(+), 24 deletions(-) diff --git a/app/vmctl/flags.go b/app/vmctl/flags.go index ef0e0c66aa..1b34f2d12c 100644 --- a/app/vmctl/flags.go +++ b/app/vmctl/flags.go @@ -326,9 +326,9 @@ const ( vmNativeFilterTimeReverse = "vm-native-filter-time-reverse" vmNativeStepInterval = "vm-native-step-interval" - vmNativeDisableBinaryProtocol = "vm-native-disable-binary-protocol" - vmNativeDisableHTTPKeepAlive = "vm-native-disable-http-keep-alive" - vmNativeDisableRetries = "vm-native-disable-retries" + vmNativeDisableBinaryProtocol = "vm-native-disable-binary-protocol" + vmNativeDisableHTTPKeepAlive = "vm-native-disable-http-keep-alive" + vmNativeDisablePerMetricMigration = "vm-native-disable-per-metric-migration" vmNativeSrcAddr = "vm-native-src-addr" vmNativeSrcUser = "vm-native-src-user" @@ -454,8 +454,8 @@ var ( Value: 2, }, &cli.BoolFlag{ - Name: vmNativeDisableRetries, - Usage: "Defines whether to disable retries with backoff policy for migration process", + Name: vmNativeDisablePerMetricMigration, + Usage: "Defines whether to disable per-metric migration and migrate all data via one connection. In this mode, vmctl makes less export/import requests, but can't provide a progress bar or retry failed requests.", Value: false, }, &cli.BoolFlag{ diff --git a/app/vmctl/main.go b/app/vmctl/main.go index 9ee47b058d..95743081fc 100644 --- a/app/vmctl/main.go +++ b/app/vmctl/main.go @@ -254,11 +254,11 @@ func main() { ExtraLabels: dstExtraLabels, HTTPClient: dstHTTPClient, }, - backoff: backoff.New(), - cc: c.Int(vmConcurrency), - disableRetries: c.Bool(vmNativeDisableRetries), - isSilent: c.Bool(globalSilent), - isNative: !c.Bool(vmNativeDisableBinaryProtocol), + backoff: backoff.New(), + cc: c.Int(vmConcurrency), + disablePerMetricRequests: c.Bool(vmNativeDisablePerMetricMigration), + isSilent: c.Bool(globalSilent), + isNative: !c.Bool(vmNativeDisableBinaryProtocol), } return p.run(ctx) }, diff --git a/app/vmctl/vm_native.go b/app/vmctl/vm_native.go index cb876dcb7a..3cbea821d7 100644 --- a/app/vmctl/vm_native.go +++ b/app/vmctl/vm_native.go @@ -29,13 +29,14 @@ type vmNativeProcessor struct { src *native.Client backoff *backoff.Backoff - s *stats - rateLimit int64 - interCluster bool - cc int - disableRetries bool - isSilent bool - isNative bool + s *stats + rateLimit int64 + interCluster bool + cc int + isSilent bool + isNative bool + + disablePerMetricRequests bool } const ( @@ -119,7 +120,7 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcU return fmt.Errorf("failed to init export pipe: %w", err) } - if p.disableRetries && bar != nil { + if p.disablePerMetricRequests && bar != nil { fmt.Printf("Continue import process with filter %s:\n", f.String()) reader = bar.NewProxyReader(reader) } @@ -189,7 +190,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, var foundSeriesMsg string metrics := []string{p.filter.Match} - if !p.disableRetries { + if !p.disablePerMetricRequests { log.Printf("Exploring metrics...") metrics, err = p.src.Explore(ctx, p.filter, tenantID) if err != nil { @@ -227,7 +228,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, var bar *pb.ProgressBar if !silent { bar = barpool.NewSingleProgress(fmt.Sprintf(nativeWithBackoffTpl, barPrefix), len(metrics)*len(ranges)) - if p.disableRetries { + if p.disablePerMetricRequests { bar = barpool.NewSingleProgress(nativeSingleProcessTpl, 0) } bar.Start() @@ -243,7 +244,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, go func() { defer wg.Done() for f := range filterCh { - if !p.disableRetries { + if !p.disablePerMetricRequests { if err := p.do(ctx, f, srcURL, dstURL, nil); err != nil { errCh <- err return diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fdf9976b76..fcf9a19dea 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -29,6 +29,7 @@ The sandbox cluster installation is running under the constant load generated by ## tip * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy incoming requests to different backends based on the requested host via `src_hosts` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth.html#generic-http-proxy-for-different-backends). +* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): rename cmd-line flag `vm-native-disable-retries` to `vm-native-disable-per-metric-migration` to better reflect its meaning. * BUGFIX: `vminsert`: properly accept samples via [OpenTelemetry data ingestion protocol](https://docs.victoriametrics.com/#sending-data-via-opentelemetry) when these samples have no [resource attributes](https://opentelemetry.io/docs/instrumentation/go/resources/). Previously such samples were silently skipped. * BUGFIX: `vmstorage`: added missing `-inmemoryDataFlushInterval` command-line flag, which was missing in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) after implementing [this feature](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3337) in [v1.85.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.0). diff --git a/docs/vmctl.md b/docs/vmctl.md index 57949f7f37..407b77459b 100644 --- a/docs/vmctl.md +++ b/docs/vmctl.md @@ -821,7 +821,7 @@ Requests to make: 9 / 9 [██████████████████ ``` _To disable explore phase and switch to the old way of data migration via single connection use -`--vm-native-disable-retries` cmd-line flag. Please note, in this mode vmctl won't be able to retry failed requests._ +`--vm-native-disable-per-metric-migration` cmd-line flag. Please note, in this mode vmctl won't be able to retry failed requests._ Importing tips: @@ -831,14 +831,17 @@ Importing tips: [here](https://docs.victoriametrics.com/#how-to-export-data-in-native-format). If hitting `the number of matching timeseries exceeds...` error, adjust filters to match less time series or update `-search.maxSeries` command-line flag on vmselect/vmsingle; +1. Using smaller intervals via `--vm-native-step-interval` cmd-line flag can reduce the number of matched series per-request + for sources with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate). + See more about [step interval here](#using-time-based-chunking-of-migration). 1. Migrating all the metrics from one VM to another may collide with existing application metrics (prefixed with `vm_`) at destination and lead to confusion when using [official Grafana dashboards](https://grafana.com/orgs/victoriametrics/dashboards). To avoid such situation try to filter out VM process metrics via `--vm-native-filter-match='{__name__!~"vm_.*"}'` flag. 1. Migrating data with overlapping time range or via unstable network can produce duplicates series at destination. To avoid duplicates set `-dedup.minScrapeInterval=1ms` for `vmselect`/`vmstorage` at the destination. - This will instruct `vmselect`/`vmstorage` to ignore duplicates with identical timestamps. -1. When migrating large volumes of data use `--vm-native-step-interval` flag to split migration [into steps](#using-time-based-chunking-of-migration). + This will instruct `vmselect`/`vmstorage` to ignore duplicates with identical timestamps. Ignore this recommendation + if you already have `-dedup.minScrapeInterval` set to 1ms or higher values at destination. 1. When migrating data from one VM cluster to another, consider using [cluster-to-cluster mode](#cluster-to-cluster-migration-mode). Or manually specify addresses according to [URL format](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format): ```console