From 0b7ce70df4d15da9a76831793e24510772278d57 Mon Sep 17 00:00:00 2001 From: hagen1778 Date: Fri, 8 Mar 2024 20:47:36 +0100 Subject: [PATCH] app/vmctl: support TLS configuration for VictoriaMetrics destination VictoriaMetrics destination is specified via `--vm-*` cmd-line flags and is used in opentsdb, influx, prometheus, remote-read modes. updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5426 Signed-off-by: hagen1778 --- app/vmctl/flags.go | 28 ++++++++++++++++++++++- app/vmctl/main.go | 42 ++++++++++++++++++++++++++++------- app/vmctl/remote_read_test.go | 4 +++- app/vmctl/vm/vm.go | 17 ++++++++++---- docs/CHANGELOG.md | 1 + 5 files changed, 78 insertions(+), 14 deletions(-) diff --git a/app/vmctl/flags.go b/app/vmctl/flags.go index 66e71643a..e451d83fc 100644 --- a/app/vmctl/flags.go +++ b/app/vmctl/flags.go @@ -40,6 +40,11 @@ const ( vmSignificantFigures = "vm-significant-figures" vmRoundDigits = "vm-round-digits" vmDisableProgressBar = "vm-disable-progress-bar" + vmCertFile = "vm-cert-file" + vmKeyFile = "vm-key-file" + vmCAFile = "vm-CA-file" + vmServerName = "vm-server-name" + vmInsecureSkipVerify = "vm-insecure-skip-verify" // also used in vm-native vmExtraLabel = "vm-extra-label" @@ -119,6 +124,27 @@ var ( Name: vmDisableProgressBar, Usage: "Whether to disable progress bar per each worker during the import.", }, + &cli.StringFlag{ + Name: vmCertFile, + Usage: "Optional path to client-side TLS certificate file to use when connecting to '--vmAddr'", + }, + &cli.StringFlag{ + Name: vmKeyFile, + Usage: "Optional path to client-side TLS key to use when connecting to '--vmAddr'", + }, + &cli.StringFlag{ + Name: vmCAFile, + Usage: "Optional path to TLS CA file to use for verifying connections to '--vmAddr'. By default, system CA is used", + }, + &cli.StringFlag{ + Name: vmServerName, + Usage: "Optional TLS server name to use for connections to '--vmAddr'. By default, the server name from '--vmAddr' is used", + }, + &cli.BoolFlag{ + Name: vmInsecureSkipVerify, + Usage: "Whether to skip tls verification when connecting to '--vmAddr'", + Value: false, + }, } ) @@ -210,7 +236,7 @@ var ( }, &cli.StringFlag{ Name: otsdbServerName, - Usage: "Optional TLS server name to use for connections to -otsdb-addr. By default, the server name from otsdbAddr is used", + Usage: "Optional TLS server name to use for connections to -otsdb-addr. By default, the server name from -otsdb-addr is used", }, &cli.BoolFlag{ Name: otsdbInsecureSkipVerify, diff --git a/app/vmctl/main.go b/app/vmctl/main.go index a32f3c9bd..84ec640e9 100644 --- a/app/vmctl/main.go +++ b/app/vmctl/main.go @@ -77,7 +77,10 @@ func main() { return fmt.Errorf("failed to create opentsdb client: %s", err) } - vmCfg := initConfigVM(c) + vmCfg, err := initConfigVM(c) + if err != nil { + return fmt.Errorf("failed to init VM configuration: %s", err) + } // disable progress bars since openTSDB implementation // does not use progress bar pool vmCfg.DisableProgressBar = true @@ -129,7 +132,10 @@ func main() { return fmt.Errorf("failed to create influx client: %s", err) } - vmCfg := initConfigVM(c) + vmCfg, err := initConfigVM(c) + if err != nil { + return fmt.Errorf("failed to init VM configuration: %s", err) + } importer, err = vm.NewImporter(ctx, vmCfg) if err != nil { return fmt.Errorf("failed to create VM importer: %s", err) @@ -172,8 +178,10 @@ func main() { return fmt.Errorf("error create remote read client: %s", err) } - vmCfg := initConfigVM(c) - + vmCfg, err := initConfigVM(c) + if err != nil { + return fmt.Errorf("failed to init VM configuration: %s", err) + } importer, err := vm.NewImporter(ctx, vmCfg) if err != nil { return fmt.Errorf("failed to create VM importer: %s", err) @@ -202,7 +210,10 @@ func main() { Action: func(c *cli.Context) error { fmt.Println("Prometheus import mode") - vmCfg := initConfigVM(c) + vmCfg, err := initConfigVM(c) + if err != nil { + return fmt.Errorf("failed to init VM configuration: %s", err) + } importer, err = vm.NewImporter(ctx, vmCfg) if err != nil { return fmt.Errorf("failed to create VM importer: %s", err) @@ -381,9 +392,24 @@ func main() { log.Printf("Total time: %v", time.Since(start)) } -func initConfigVM(c *cli.Context) vm.Config { +func initConfigVM(c *cli.Context) (vm.Config, error) { + addr := c.String(vmAddr) + + // create Transport with given TLS config + certFile := c.String(vmCertFile) + keyFile := c.String(vmKeyFile) + caFile := c.String(vmCAFile) + serverName := c.String(vmServerName) + insecureSkipVerify := c.Bool(vmInsecureSkipVerify) + + tr, err := httputils.Transport(addr, certFile, caFile, keyFile, serverName, insecureSkipVerify) + if err != nil { + return vm.Config{}, fmt.Errorf("failed to create Transport: %s", err) + } + return vm.Config{ - Addr: c.String(vmAddr), + Addr: addr, + Transport: tr, User: c.String(vmUser), Password: c.String(vmPassword), Concurrency: uint8(c.Int(vmConcurrency)), @@ -395,5 +421,5 @@ func initConfigVM(c *cli.Context) vm.Config { ExtraLabels: c.StringSlice(vmExtraLabel), RateLimit: c.Int64(vmRateLimit), DisableProgressBar: c.Bool(vmDisableProgressBar), - } + }, nil } diff --git a/app/vmctl/remote_read_test.go b/app/vmctl/remote_read_test.go index dcb6c6889..759331317 100644 --- a/app/vmctl/remote_read_test.go +++ b/app/vmctl/remote_read_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "net/http" "testing" "time" @@ -61,7 +62,8 @@ func TestRemoteRead(t *testing.T) { { name: "step month on month time range", remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"}, - vmCfg: vm.Config{Addr: "", Concurrency: 1, DisableProgressBar: true}, + vmCfg: vm.Config{Addr: "", Concurrency: 1, DisableProgressBar: true, + Transport: http.DefaultTransport.(*http.Transport)}, start: "2022-09-26T11:23:05+02:00", end: "2022-11-26T11:24:05+02:00", numOfSamples: 2, diff --git a/app/vmctl/vm/vm.go b/app/vmctl/vm/vm.go index a35afdb14..1936ee721 100644 --- a/app/vmctl/vm/vm.go +++ b/app/vmctl/vm/vm.go @@ -26,6 +26,8 @@ type Config struct { // --httpListenAddr value for single node version // --httpListenAddr value of vmselect component for cluster version Addr string + // Transport allows specifying custom http.Transport + Transport *http.Transport // Concurrency defines number of worker // performing the import requests concurrently Concurrency uint8 @@ -62,6 +64,7 @@ type Config struct { // see https://docs.victoriametrics.com/#how-to-import-time-series-data type Importer struct { addr string + client *http.Client importPath string compress bool user string @@ -128,8 +131,14 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) { return nil, err } + client := &http.Client{} + if cfg.Transport != nil { + client.Transport = cfg.Transport + } + im := &Importer{ addr: addr, + client: client, importPath: importPath, compress: cfg.Compress, user: cfg.User, @@ -291,7 +300,7 @@ func (im *Importer) Ping() error { if im.user != "" { req.SetBasicAuth(im.user, im.password) } - resp, err := http.DefaultClient.Do(req) + resp, err := im.client.Do(req) if err != nil { return err } @@ -321,7 +330,7 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error { errCh := make(chan error) go func() { - errCh <- do(req) + errCh <- im.do(req) close(errCh) }() @@ -375,8 +384,8 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error { // ErrBadRequest represents bad request error. var ErrBadRequest = errors.New("bad request") -func do(req *http.Request) error { - resp, err := http.DefaultClient.Do(req) +func (im *Importer) do(req *http.Request) error { + resp, err := im.client.Do(req) if err != nil { return fmt.Errorf("unexpected error when performing request: %s", err) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d21e9d0e6..c56c2da5c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -47,6 +47,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): improve trace display for better visual separation of branches. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5926). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): use the provided `-remoteWrite.tlsServerName` as `Host` header in requests to `-remoteWrite.url`. This allows sending data to https remote storage by IP address instead of hostname. Thanks to @minor-fixes for initial idea and [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5802). * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): support client-side TLS configuration for [native protocol](https://docs.victoriametrics.com/vmctl/#migrating-data-from-victoriametrics). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5748). Thanks to @khushijain21 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5824). +* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): support client-side TLS configuration for VictoriaMetrics destination specified via `--vm-*` cmd-line flags used in [InfluxDB](https://docs.victoriametrics.com/vmctl/#migrating-data-from-influxdb-1x), [Remote Read protocol](https://docs.victoriametrics.com/vmctl/#migrating-data-by-remote-read-protocol), [OpenTSDB](https://docs.victoriametrics.com/vmctl/#migrating-data-from-opentsdb), [Prometheus](https://docs.victoriametrics.com/vmctl/#migrating-data-from-prometheus) and [Promscale](https://docs.victoriametrics.com/vmctl/#migrating-data-from-promscale) migration modes. * BUGFIX: do not drop `match[]` filter at [`/api/v1/series`](https://docs.victoriametrics.com/url-examples/#apiv1series) if `-search.ignoreExtraFiltersAtLabelsAPI` command-line flag is set, since missing `match[]` filter breaks `/api/v1/series` requests.