From 833ab331b1d7953b9be320e3514254f89fa1b61a Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Tue, 1 Aug 2023 09:45:50 +0200 Subject: [PATCH] vmctl: allow disabling binary export protocol (#4716) Binary export API protocol can be disabled via `-vm-native-disable-binary-protocol` cmd-line flag when migrating data from VictoriaMetrics. Disabling binary protocol can be useful for deduplication of the exported data before ingestion. For this, deduplication need to be configured at `-vm-native-src-addr` side and `-vm-native-disable-binary-protocol` should be set on vmctl side. Signed-off-by: hagen1778 --- app/vmctl/flags.go | 13 +++++++++++-- app/vmctl/main.go | 1 + app/vmctl/vm_native.go | 12 +++++++++--- app/vmctl/vm_native_test.go | 1 + docs/CHANGELOG.md | 7 ++++--- 5 files changed, 26 insertions(+), 8 deletions(-) diff --git a/app/vmctl/flags.go b/app/vmctl/flags.go index c41c2cef3..4d72e1867 100644 --- a/app/vmctl/flags.go +++ b/app/vmctl/flags.go @@ -325,8 +325,9 @@ const ( vmNativeFilterTimeEnd = "vm-native-filter-time-end" vmNativeStepInterval = "vm-native-step-interval" - vmNativeDisableHTTPKeepAlive = "vm-native-disable-http-keep-alive" - vmNativeDisableRetries = "vm-native-disable-retries" + vmNativeDisableBinaryProtocol = "vm-native-disable-binary-protocol" + vmNativeDisableHTTPKeepAlive = "vm-native-disable-http-keep-alive" + vmNativeDisableRetries = "vm-native-disable-retries" vmNativeSrcAddr = "vm-native-src-addr" vmNativeSrcUser = "vm-native-src-user" @@ -450,6 +451,14 @@ var ( Usage: "Defines whether to disable retries with backoff policy for migration process", Value: false, }, + &cli.BoolFlag{ + Name: vmNativeDisableBinaryProtocol, + Usage: "Whether to use https://docs.victoriametrics.com/#how-to-export-data-in-json-line-format" + + "instead of https://docs.victoriametrics.com/#how-to-export-data-in-native-format API." + + "Binary export/import API protocol implies less network and resource usage, as it transfers compressed binary data blocks." + + "Non-binary export/import API is less efficient, but supports deduplication if it is configured on vm-native-src-addr side.", + Value: false, + }, } ) diff --git a/app/vmctl/main.go b/app/vmctl/main.go index e81746a76..ba6c570b6 100644 --- a/app/vmctl/main.go +++ b/app/vmctl/main.go @@ -254,6 +254,7 @@ func main() { cc: c.Int(vmConcurrency), disableRetries: c.Bool(vmNativeDisableRetries), isSilent: c.Bool(globalSilent), + isNative: !c.Bool(vmNativeDisableBinaryProtocol), } return p.run(ctx) }, diff --git a/app/vmctl/vm_native.go b/app/vmctl/vm_native.go index 3c8392b9a..8d94d2bbe 100644 --- a/app/vmctl/vm_native.go +++ b/app/vmctl/vm_native.go @@ -34,11 +34,12 @@ type vmNativeProcessor struct { cc int disableRetries bool isSilent bool + isNative bool } const ( - nativeExportAddr = "api/v1/export/native" - nativeImportAddr = "api/v1/import/native" + nativeExportAddr = "api/v1/export" + nativeImportAddr = "api/v1/import" nativeWithBackoffTpl = `{{ blue "%s:" }} {{ counters . }} {{ bar . "[" "█" (cycle . "█") "▒" "]" }} {{ percent . }}` nativeSingleProcessTpl = `Total: {{counters . }} {{ cycle . "↖" "↗" "↘" "↙" }} Speed: {{speed . }} {{string . "suffix"}}` ) @@ -159,9 +160,14 @@ func (p *vmNativeProcessor) runSingle(ctx context.Context, f native.Filter, srcU func (p *vmNativeProcessor) runBackfilling(ctx context.Context, tenantID string, ranges [][]time.Time, silent bool) error { exportAddr := nativeExportAddr + importAddr := nativeImportAddr + if p.isNative { + exportAddr += "/native" + importAddr += "/native" + } srcURL := fmt.Sprintf("%s/%s", p.src.Addr, exportAddr) - importAddr, err := vm.AddExtraLabelsToImportPath(nativeImportAddr, p.dst.ExtraLabels) + importAddr, err := vm.AddExtraLabelsToImportPath(importAddr, p.dst.ExtraLabels) if err != nil { return fmt.Errorf("failed to add labels to import path: %s", err) } diff --git a/app/vmctl/vm_native_test.go b/app/vmctl/vm_native_test.go index 457437f75..217df70aa 100644 --- a/app/vmctl/vm_native_test.go +++ b/app/vmctl/vm_native_test.go @@ -228,6 +228,7 @@ func Test_vmNativeProcessor_run(t *testing.T) { interCluster: tt.fields.interCluster, cc: tt.fields.cc, isSilent: tt.args.silent, + isNative: true, } if err := p.run(tt.args.ctx); (err != nil) != tt.wantErr { diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 641e29d97..ec1db9cca 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -27,10 +27,11 @@ The following `tip` changes can be tested by building VictoriaMetrics components * FEATURE: [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html): add `share_eq_over_time(m[d], eq)` function for calculating the share (in the range `[0...1]`) of raw samples on the given lookbehind window `d`, which are equal to `eq`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4441). Thanks to @Damon07 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/4725). * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove deprecated in [v1.61.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.61.0) `-rule.configCheckInterval` command-line flag. Use `-configCheckInterval` command-line flag instead. * FEATURE: [vmalert](https://docs.victoriametrics.com/vmalert.html): remove support of deprecated web links of `/api/v1///status` form in favour of `/api/v1/alerts?group_id=<>&alert_id=<>` links. Links of `/api/v1///status` form were deprecated in v1.79.0. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2825) for details. - -* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). +* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): allow disabling binary export API protocol via `-vm-native-disable-binary-protocol` cmd-line flag when [migrating data from VictoriaMetrics](https://docs.victoriametrics.com/vmctl.html#migrating-data-from-victoriametrics). Disabling binary protocol can be useful for deduplication of the exported data before ingestion. For this, deduplication need [to be configured](https://docs.victoriametrics.com/#deduplication) at `-vm-native-src-addr` side and `-vm-native-disable-binary-protocol` should be set on vmctl side. * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support of `week` step for [time-based chunking migration](https://docs.victoriametrics.com/vmctl.html#using-time-based-chunking-of-migration). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4738). -* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). + +* BUGFIX: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): fix panic when creating a backup to a local filesystem on Windows. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4704). +* BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): use local scrape timestamps for the scraped metrics unless `honor_timestamps: true` option is explicitly set at [scrape_config](https://docs.victoriametrics.com/sd_configs.html#scrape_configs). This fixes gaps for metrics collected from [cadvisor](https://github.com/google/cadvisor) or similar exporters, which export metrics with invalid timestamps. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697) and [this comment](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4697#issuecomment-1654614799) for details. The issue has been introduced in [v1.68.0](#v1680). ## [v1.92.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.92.1)