vmctl: rename vm-native-disable-retries to vm-native-disable-per-metric-migration (#5476)

The change supposed to better reflect the meaning of this flag.

Signed-off-by: hagen1778 <roman@victoriametrics.com>

app/vmctl/flags.go

@@ -326,9 +326,9 @@ const (
 	vmNativeFilterTimeReverse = "vm-native-filter-time-reverse"
 	vmNativeStepInterval      = "vm-native-step-interval"
 
-	vmNativeDisableBinaryProtocol = "vm-native-disable-binary-protocol"
+	vmNativeDisableBinaryProtocol     = "vm-native-disable-binary-protocol"
-	vmNativeDisableHTTPKeepAlive  = "vm-native-disable-http-keep-alive"
+	vmNativeDisableHTTPKeepAlive      = "vm-native-disable-http-keep-alive"
-	vmNativeDisableRetries        = "vm-native-disable-retries"
+	vmNativeDisablePerMetricMigration = "vm-native-disable-per-metric-migration"
 
 	vmNativeSrcAddr        = "vm-native-src-addr"
 	vmNativeSrcUser        = "vm-native-src-user"
@@ -454,8 +454,8 @@ var (
 			Value: 2,
 		},
 		&cli.BoolFlag{
-			Name:  vmNativeDisableRetries,
+			Name:  vmNativeDisablePerMetricMigration,
-			Usage: "Defines whether to disable retries with backoff policy for migration process",
+			Usage: "Defines whether to disable per-metric migration and migrate all data via one connection. In this mode, vmctl makes less export/import requests, but can't provide a progress bar or retry failed requests.",
 			Value: false,
 		},
 		&cli.BoolFlag{

app/vmctl/main.go

@@ -254,11 +254,11 @@ func main() {
 							ExtraLabels: dstExtraLabels,
 							HTTPClient:  dstHTTPClient,
 						},
-						backoff:        backoff.New(),
+						backoff:                  backoff.New(),
-						cc:             c.Int(vmConcurrency),
+						cc:                       c.Int(vmConcurrency),
-						disableRetries: c.Bool(vmNativeDisableRetries),
+						disablePerMetricRequests: c.Bool(vmNativeDisablePerMetricMigration),
-						isSilent:       c.Bool(globalSilent),
+						isSilent:                 c.Bool(globalSilent),
-						isNative:       !c.Bool(vmNativeDisableBinaryProtocol),
+						isNative:                 !c.Bool(vmNativeDisableBinaryProtocol),
 					}
 					return p.run(ctx)
 				},

app/vmctl/vm_native.go

@@ -29,13 +29,14 @@ type vmNativeProcessor struct {
 	src     *native.Client
 	backoff *backoff.Backoff
 
-	s              *stats
+	s            *stats
-	rateLimit      int64
+	rateLimit    int64
-	interCluster   bool
+	interCluster bool
-	cc             int
+	cc           int
-	disableRetries bool
+	isSilent     bool
-	isSilent       bool
+	isNative     bool
-	isNative       bool
+
+	disablePerMetricRequests bool
 }
 
 const (
@@ -119,7 +120,7 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcU
 		return fmt.Errorf("failed to init export pipe: %w", err)
 	}
 
-	if p.disableRetries && bar != nil {
+	if p.disablePerMetricRequests && bar != nil {
 		fmt.Printf("Continue import process with filter %s:\n", f.String())
 		reader = bar.NewProxyReader(reader)
 	}
@@ -189,7 +190,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 	var foundSeriesMsg string
 
 	metrics := []string{p.filter.Match}
-	if !p.disableRetries {
+	if !p.disablePerMetricRequests {
 		log.Printf("Exploring metrics...")
 		metrics, err = p.src.Explore(ctx, p.filter, tenantID)
 		if err != nil {
@@ -227,7 +228,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 	var bar *pb.ProgressBar
 	if !silent {
 		bar = barpool.NewSingleProgress(fmt.Sprintf(nativeWithBackoffTpl, barPrefix), len(metrics)*len(ranges))
-		if p.disableRetries {
+		if p.disablePerMetricRequests {
 			bar = barpool.NewSingleProgress(nativeSingleProcessTpl, 0)
 		}
 		bar.Start()
@@ -243,7 +244,7 @@ func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string,
 		go func() {
 			defer wg.Done()
 			for f := range filterCh {
-				if !p.disableRetries {
+				if !p.disablePerMetricRequests {
 					if err := p.do(ctx, f, srcURL, dstURL, nil); err != nil {
 						errCh <- err
 						return

docs/CHANGELOG.md

@@ -29,6 +29,7 @@ The sandbox cluster installation is running under the constant load generated by
 ## tip
 
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): add ability to proxy incoming requests to different backends based on the requested host via `src_hosts` option at `url_map`. See [these docs](https://docs.victoriametrics.com/vmauth.html#generic-http-proxy-for-different-backends).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): rename cmd-line flag `vm-native-disable-retries` to `vm-native-disable-per-metric-migration` to better reflect its meaning.
 
 * BUGFIX: `vmstorage`: added missing `-inmemoryDataFlushInterval` command-line flag, which was missing in [VictoriaMetrics cluster](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html) after implementing [this feature](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3337) in [v1.85.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.0).
 

docs/vmctl.md

@@ -821,7 +821,7 @@ Requests to make: 9 / 9 [██████████████████
 ```
 
 _To disable explore phase and switch to the old way of data migration via single connection use 
-`--vm-native-disable-retries` cmd-line flag. Please note, in this mode vmctl won't be able to retry failed requests._
+`--vm-native-disable-per-metric-migration` cmd-line flag. Please note, in this mode vmctl won't be able to retry failed requests._
 
 Importing tips:
 
@@ -831,14 +831,17 @@ Importing tips:
    [here](https://docs.victoriametrics.com/#how-to-export-data-in-native-format).
    If hitting `the number of matching timeseries exceeds...` error, adjust filters to match less time series or 
    update `-search.maxSeries` command-line flag on vmselect/vmsingle;
+1. Using smaller intervals via `--vm-native-step-interval` cmd-line flag can reduce the number of matched series per-request
+   for sources with [high churn rate](https://docs.victoriametrics.com/FAQ.html#what-is-high-churn-rate).
+   See more about [step interval here](#using-time-based-chunking-of-migration).
 1. Migrating all the metrics from one VM to another may collide with existing application metrics
    (prefixed with `vm_`) at destination and lead to confusion when using
    [official Grafana dashboards](https://grafana.com/orgs/victoriametrics/dashboards).
    To avoid such situation try to filter out VM process metrics via `--vm-native-filter-match='{__name__!~"vm_.*"}'` flag.
 1. Migrating data with overlapping time range or via unstable network can produce duplicates series at destination.
    To avoid duplicates set `-dedup.minScrapeInterval=1ms` for `vmselect`/`vmstorage` at the destination.
-   This will instruct `vmselect`/`vmstorage` to ignore duplicates with identical timestamps.
+   This will instruct `vmselect`/`vmstorage` to ignore duplicates with identical timestamps. Ignore this recommendation
-1. When migrating large volumes of data use `--vm-native-step-interval` flag to split migration [into steps](#using-time-based-chunking-of-migration).
+   if you already have `-dedup.minScrapeInterval` set to 1ms or higher values at destination.
 1. When migrating data from one VM cluster to another, consider using [cluster-to-cluster mode](#cluster-to-cluster-migration-mode).
    Or manually specify addresses according to [URL format](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format):
    ```console