app/vmctl: support TLS configuration for VictoriaMetrics destination

VictoriaMetrics destination is specified via `--vm-*` cmd-line flags
and is used in opentsdb, influx, prometheus, remote-read modes.

updates https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5426

Signed-off-by: hagen1778 <roman@victoriametrics.com>

app/vmctl/flags.go

@@ -40,6 +40,11 @@ const (
 	vmSignificantFigures = "vm-significant-figures"
 	vmRoundDigits        = "vm-round-digits"
 	vmDisableProgressBar = "vm-disable-progress-bar"
+	vmCertFile           = "vm-cert-file"
+	vmKeyFile            = "vm-key-file"
+	vmCAFile             = "vm-CA-file"
+	vmServerName         = "vm-server-name"
+	vmInsecureSkipVerify = "vm-insecure-skip-verify"
 
 	// also used in vm-native
 	vmExtraLabel = "vm-extra-label"
@@ -119,6 +124,27 @@ var (
 			Name:  vmDisableProgressBar,
 			Usage: "Whether to disable progress bar per each worker during the import.",
 		},
+		&cli.StringFlag{
+			Name:  vmCertFile,
+			Usage: "Optional path to client-side TLS certificate file to use when connecting to '--vmAddr'",
+		},
+		&cli.StringFlag{
+			Name:  vmKeyFile,
+			Usage: "Optional path to client-side TLS key to use when connecting to '--vmAddr'",
+		},
+		&cli.StringFlag{
+			Name:  vmCAFile,
+			Usage: "Optional path to TLS CA file to use for verifying connections to '--vmAddr'. By default, system CA is used",
+		},
+		&cli.StringFlag{
+			Name:  vmServerName,
+			Usage: "Optional TLS server name to use for connections to '--vmAddr'. By default, the server name from '--vmAddr' is used",
+		},
+		&cli.BoolFlag{
+			Name:  vmInsecureSkipVerify,
+			Usage: "Whether to skip tls verification when connecting to '--vmAddr'",
+			Value: false,
+		},
 	}
 )
 
@@ -210,7 +236,7 @@ var (
 		},
 		&cli.StringFlag{
 			Name:  otsdbServerName,
-			Usage: "Optional TLS server name to use for connections to -otsdb-addr. By default, the server name from otsdbAddr is used",
+			Usage: "Optional TLS server name to use for connections to -otsdb-addr. By default, the server name from -otsdb-addr is used",
 		},
 		&cli.BoolFlag{
 			Name:  otsdbInsecureSkipVerify,

app/vmctl/main.go

@@ -77,7 +77,10 @@ func main() {
 						return fmt.Errorf("failed to create opentsdb client: %s", err)
 					}
 
-					vmCfg := initConfigVM(c)
+					vmCfg, err := initConfigVM(c)
+					if err != nil {
+						return fmt.Errorf("failed to init VM configuration: %s", err)
+					}
 					// disable progress bars since openTSDB implementation
 					// does not use progress bar pool
 					vmCfg.DisableProgressBar = true
@@ -129,7 +132,10 @@ func main() {
 						return fmt.Errorf("failed to create influx client: %s", err)
 					}
 
-					vmCfg := initConfigVM(c)
+					vmCfg, err := initConfigVM(c)
+					if err != nil {
+						return fmt.Errorf("failed to init VM configuration: %s", err)
+					}
 					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -172,8 +178,10 @@ func main() {
 						return fmt.Errorf("error create remote read client: %s", err)
 					}
 
-					vmCfg := initConfigVM(c)
-
+					vmCfg, err := initConfigVM(c)
+					if err != nil {
+						return fmt.Errorf("failed to init VM configuration: %s", err)
+					}
 					importer, err := vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -202,7 +210,10 @@ func main() {
 				Action: func(c *cli.Context) error {
 					fmt.Println("Prometheus import mode")
 
-					vmCfg := initConfigVM(c)
+					vmCfg, err := initConfigVM(c)
+					if err != nil {
+						return fmt.Errorf("failed to init VM configuration: %s", err)
+					}
 					importer, err = vm.NewImporter(ctx, vmCfg)
 					if err != nil {
 						return fmt.Errorf("failed to create VM importer: %s", err)
@@ -381,9 +392,24 @@ func main() {
 	log.Printf("Total time: %v", time.Since(start))
 }
 
-func initConfigVM(c *cli.Context) vm.Config {
+func initConfigVM(c *cli.Context) (vm.Config, error) {
+	addr := c.String(vmAddr)
+
+	// create Transport with given TLS config
+	certFile := c.String(vmCertFile)
+	keyFile := c.String(vmKeyFile)
+	caFile := c.String(vmCAFile)
+	serverName := c.String(vmServerName)
+	insecureSkipVerify := c.Bool(vmInsecureSkipVerify)
+
+	tr, err := httputils.Transport(addr, certFile, caFile, keyFile, serverName, insecureSkipVerify)
+	if err != nil {
+		return vm.Config{}, fmt.Errorf("failed to create Transport: %s", err)
+	}
+
 	return vm.Config{
-		Addr:               c.String(vmAddr),
+		Addr:               addr,
+		Transport:          tr,
 		User:               c.String(vmUser),
 		Password:           c.String(vmPassword),
 		Concurrency:        uint8(c.Int(vmConcurrency)),
@@ -395,5 +421,5 @@ func initConfigVM(c *cli.Context) vm.Config {
 		ExtraLabels:        c.StringSlice(vmExtraLabel),
 		RateLimit:          c.Int64(vmRateLimit),
 		DisableProgressBar: c.Bool(vmDisableProgressBar),
-	}
+	}, nil
 }

app/vmctl/remote_read_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"net/http"
 	"testing"
 	"time"
 
@@ -61,7 +62,8 @@ func TestRemoteRead(t *testing.T) {
 		{
 			name:             "step month on month time range",
 			remoteReadConfig: remoteread.Config{Addr: "", LabelName: "__name__", LabelValue: ".*"},
-			vmCfg:            vm.Config{Addr: "", Concurrency: 1, DisableProgressBar: true},
+			vmCfg: vm.Config{Addr: "", Concurrency: 1, DisableProgressBar: true,
+				Transport: http.DefaultTransport.(*http.Transport)},
 			start:            "2022-09-26T11:23:05+02:00",
 			end:              "2022-11-26T11:24:05+02:00",
 			numOfSamples:     2,

app/vmctl/vm/vm.go

@@ -26,6 +26,8 @@ type Config struct {
 	//   --httpListenAddr value for single node version
 	//   --httpListenAddr value of vmselect  component for cluster version
 	Addr string
+	// Transport allows specifying custom http.Transport
+	Transport *http.Transport
 	// Concurrency defines number of worker
 	// performing the import requests concurrently
 	Concurrency uint8
@@ -62,6 +64,7 @@ type Config struct {
 // see https://docs.victoriametrics.com/#how-to-import-time-series-data
 type Importer struct {
 	addr       string
+	client     *http.Client
 	importPath string
 	compress   bool
 	user       string
@@ -128,8 +131,14 @@ func NewImporter(ctx context.Context, cfg Config) (*Importer, error) {
 		return nil, err
 	}
 
+	client := &http.Client{}
+	if cfg.Transport != nil {
+		client.Transport = cfg.Transport
+	}
+
 	im := &Importer{
 		addr:       addr,
+		client:     client,
 		importPath: importPath,
 		compress:   cfg.Compress,
 		user:       cfg.User,
@@ -291,7 +300,7 @@ func (im *Importer) Ping() error {
 	if im.user != "" {
 		req.SetBasicAuth(im.user, im.password)
 	}
-	resp, err := http.DefaultClient.Do(req)
+	resp, err := im.client.Do(req)
 	if err != nil {
 		return err
 	}
@@ -321,7 +330,7 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 
 	errCh := make(chan error)
 	go func() {
-		errCh <- do(req)
+		errCh <- im.do(req)
 		close(errCh)
 	}()
 
@@ -375,8 +384,8 @@ func (im *Importer) Import(tsBatch []*TimeSeries) error {
 // ErrBadRequest represents bad request error.
 var ErrBadRequest = errors.New("bad request")
 
-func do(req *http.Request) error {
-	resp, err := http.DefaultClient.Do(req)
+func (im *Importer) do(req *http.Request) error {
+	resp, err := im.client.Do(req)
 	if err != nil {
 		return fmt.Errorf("unexpected error when performing request: %s", err)
 	}

docs/CHANGELOG.md

@@ -47,6 +47,7 @@ See also [LTS releases](https://docs.victoriametrics.com/lts-releases/).
 * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): improve trace display for better visual separation of branches. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5926).
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): use the provided `-remoteWrite.tlsServerName` as `Host` header in requests to `-remoteWrite.url`. This allows sending data to https remote storage by IP address instead of hostname. Thanks to @minor-fixes for initial idea and [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5802).
 * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): support client-side TLS configuration for [native protocol](https://docs.victoriametrics.com/vmctl/#migrating-data-from-victoriametrics). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5748). Thanks to @khushijain21 for the [pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5824).
+* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): support client-side TLS configuration for VictoriaMetrics destination specified via `--vm-*` cmd-line flags used in [InfluxDB](https://docs.victoriametrics.com/vmctl/#migrating-data-from-influxdb-1x), [Remote Read protocol](https://docs.victoriametrics.com/vmctl/#migrating-data-by-remote-read-protocol), [OpenTSDB](https://docs.victoriametrics.com/vmctl/#migrating-data-from-opentsdb), [Prometheus](https://docs.victoriametrics.com/vmctl/#migrating-data-from-prometheus) and [Promscale](https://docs.victoriametrics.com/vmctl/#migrating-data-from-promscale) migration modes.
 
 * BUGFIX: do not drop `match[]` filter at [`/api/v1/series`](https://docs.victoriametrics.com/url-examples/#apiv1series) if `-search.ignoreExtraFiltersAtLabelsAPI` command-line flag is set, since missing `match[]` filter breaks `/api/v1/series` requests.