From bb1279bfc438693714d5a8181ad20400e5a8f7d1 Mon Sep 17 00:00:00 2001 From: Roman Khavronenko Date: Tue, 20 Feb 2024 16:22:58 +0100 Subject: [PATCH] vmctl : Provide TLS config options for Open TSDB datasource #5797 (#5832) Originally implemented here https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5797 --------- Signed-off-by: hagen1778 Co-authored-by: khushijain21 --- app/vmctl/flags.go | 44 +++++++++++++++++++++++++++------- app/vmctl/main.go | 15 +++++++++++- app/vmctl/opentsdb/opentsdb.go | 11 ++++++--- docs/CHANGELOG.md | 3 +-- docs/vmctl.md | 8 +++++++ 5 files changed, 66 insertions(+), 15 deletions(-) diff --git a/app/vmctl/flags.go b/app/vmctl/flags.go index 7aa227cd7..27c9186e6 100644 --- a/app/vmctl/flags.go +++ b/app/vmctl/flags.go @@ -123,15 +123,20 @@ var ( ) const ( - otsdbAddr = "otsdb-addr" - otsdbConcurrency = "otsdb-concurrency" - otsdbQueryLimit = "otsdb-query-limit" - otsdbOffsetDays = "otsdb-offset-days" - otsdbHardTSStart = "otsdb-hard-ts-start" - otsdbRetentions = "otsdb-retentions" - otsdbFilters = "otsdb-filters" - otsdbNormalize = "otsdb-normalize" - otsdbMsecsTime = "otsdb-msecstime" + otsdbAddr = "otsdb-addr" + otsdbConcurrency = "otsdb-concurrency" + otsdbQueryLimit = "otsdb-query-limit" + otsdbOffsetDays = "otsdb-offset-days" + otsdbHardTSStart = "otsdb-hard-ts-start" + otsdbRetentions = "otsdb-retentions" + otsdbFilters = "otsdb-filters" + otsdbNormalize = "otsdb-normalize" + otsdbMsecsTime = "otsdb-msecstime" + otsdbCertFile = "otsdb-cert-file" + otsdbKeyFile = "otsdb-key-file" + otsdbCAFile = "otsdb-CA-file" + otsdbServerName = "otsdb-server-name" + otsdbInsecureSkipVerify = "otsdb-insecure-skip-verify" ) var ( @@ -191,6 +196,27 @@ var ( Value: false, Usage: "Whether to normalize all data received to lower case before forwarding to VictoriaMetrics", }, + &cli.StringFlag{ + Name: otsdbCertFile, + Usage: "Optional path to client-side TLS certificate file to use when connecting to -otsdb-addr", + }, + &cli.StringFlag{ + Name: otsdbKeyFile, + Usage: "Optional path to client-side TLS key to use when connecting to -otsdb-addr", + }, + &cli.StringFlag{ + Name: otsdbCAFile, + Usage: "Optional path to TLS CA file to use for verifying connections to -otsdb-addr. By default, system CA is used", + }, + &cli.StringFlag{ + Name: otsdbServerName, + Usage: "Optional TLS server name to use for connections to -otsdb-addr. By default, the server name from otsdbAddr is used", + }, + &cli.BoolFlag{ + Name: otsdbInsecureSkipVerify, + Usage: "Whether to skip tls verification when connecting to -otsdb-addr", + Value: false, + }, } ) diff --git a/app/vmctl/main.go b/app/vmctl/main.go index 78913790b..1071389b2 100644 --- a/app/vmctl/main.go +++ b/app/vmctl/main.go @@ -50,8 +50,20 @@ func main() { Action: func(c *cli.Context) error { fmt.Println("OpenTSDB import mode") + // create Transport with given TLS config + certFile := c.String(otsdbCertFile) + keyFile := c.String(otsdbKeyFile) + caFile := c.String(otsdbCAFile) + serverName := c.String(otsdbServerName) + insecureSkipVerify := c.Bool(otsdbInsecureSkipVerify) + addr := c.String(otsdbAddr) + + tr, err := httputils.Transport(addr, certFile, caFile, keyFile, serverName, insecureSkipVerify) + if err != nil { + return fmt.Errorf("failed to create Transport: %s", err) + } oCfg := opentsdb.Config{ - Addr: c.String(otsdbAddr), + Addr: addr, Limit: c.Int(otsdbQueryLimit), Offset: c.Int64(otsdbOffsetDays), HardTS: c.Int64(otsdbHardTSStart), @@ -59,6 +71,7 @@ func main() { Filters: c.StringSlice(otsdbFilters), Normalize: c.Bool(otsdbNormalize), MsecsTime: c.Bool(otsdbMsecsTime), + Transport: tr, } otsdbClient, err := opentsdb.NewClient(oCfg) if err != nil { diff --git a/app/vmctl/opentsdb/opentsdb.go b/app/vmctl/opentsdb/opentsdb.go index bfa6bbcb6..4efc18c6c 100644 --- a/app/vmctl/opentsdb/opentsdb.go +++ b/app/vmctl/opentsdb/opentsdb.go @@ -47,6 +47,8 @@ type Client struct { Normalize bool HardTS int64 MsecsTime bool + + c *http.Client } // Config contains fields required @@ -60,6 +62,7 @@ type Config struct { Filters []string Normalize bool MsecsTime bool + Transport *http.Transport } // TimeRange contains data about time ranges to query @@ -107,7 +110,8 @@ type Metric struct { // FindMetrics discovers all metrics that OpenTSDB knows about (given a filter) // e.g. /api/suggest?type=metrics&q=system&max=100000 func (c Client) FindMetrics(q string) ([]string, error) { - resp, err := http.Get(q) + + resp, err := c.c.Get(q) if err != nil { return nil, fmt.Errorf("failed to send GET request to %q: %s", q, err) } @@ -131,7 +135,7 @@ func (c Client) FindMetrics(q string) ([]string, error) { // e.g. /api/search/lookup?m=system.load5&limit=1000000 func (c Client) FindSeries(metric string) ([]Meta, error) { q := fmt.Sprintf("%s/api/search/lookup?m=%s&limit=%d", c.Addr, metric, c.Limit) - resp, err := http.Get(q) + resp, err := c.c.Get(q) if err != nil { return nil, fmt.Errorf("failed to set GET request to %q: %s", q, err) } @@ -184,7 +188,7 @@ func (c Client) GetData(series Meta, rt RetentionMeta, start int64, end int64, m series.Metric, tagStr) q := fmt.Sprintf("%s/api/query?%s", c.Addr, queryStr) - resp, err := http.Get(q) + resp, err := c.c.Get(q) if err != nil { return Metric{}, fmt.Errorf("failed to send GET request to %q: %s", q, err) } @@ -325,6 +329,7 @@ func NewClient(cfg Config) (*Client, error) { Normalize: cfg.Normalize, HardTS: cfg.HardTS, MsecsTime: cfg.MsecsTime, + c: &http.Client{Transport: cfg.Transport}, } return client, nil } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 86b87aebd..26ca37cb3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -34,8 +34,7 @@ See also [LTS releases](https://docs.victoriametrics.com/LTS-releases.html). * FEATURE: [Single-node VictoriaMetrics](https://docs.victoriametrics.com/) and `vmstorage` in [VictoriaMetrics cluster](https://docs.victoriametrics.com/cluster-victoriametrics/): expose `vm_last_partition_parts` [metrics](https://docs.victoriametrics.com/#monitoring), which show the number of [parts in the latest partition](https://docs.victoriametrics.com/#storage). These metrics may help debugging query performance slowdown related to the increased number of parts in the last partition, since usually all the ingested data is written to the last partition and all the queries are performed over the recently ingested data, e.g. the last partition. * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for `client_id` option into [kuma_sd_configs](https://docs.victoriametrics.com/sd_configs/#kuma_sd_configs) in the same way as Prometheus does. See [this pull request](https://github.com/prometheus/prometheus/pull/13278). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for `enable_compression` option in [scrape_configs](https://docs.victoriametrics.com/sd_configs/#scrape_configs) in order to be compatible with Prometheus scrape configs. See [this pull request](https://github.com/prometheus/prometheus/pull/13166) and [this feature request](https://github.com/prometheus/prometheus/issues/12319). Note that `vmagent` was always supporting [`disable_compression` option](https://docs.victoriametrics.com/vmagent/#scrape_config-enhancements) before Prometheus added `enable_compression` option. -* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): support client-side TLS configuration for [InfluxDB](https://docs.victoriametrics.com/vmctl/#migrating-data-from-influxdb-1x). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5748). Thanks to @khushijain21 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5783). -* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): support client-side TLS configuration for [Remote Read protocol](https://docs.victoriametrics.com/vmctl/#migrating-data-by-remote-read-protocol). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5748). Thanks to @khushijain21 for [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5798). +* FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): support client-side TLS configuration for [InfluxDB](https://docs.victoriametrics.com/vmctl/#migrating-data-from-influxdb-1x), [Remote Read protocol](https://docs.victoriametrics.com/vmctl/#migrating-data-by-remote-read-protocol) and [OpenTSDB](https://docs.victoriametrics.com/vmctl/#migrating-data-from-opentsdb). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5748). Thanks to @khushijain21 for pull requests [1](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5783), [2](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5798), [3](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5797). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): preserve [`WITH` templates](https://play.victoriametrics.com/select/accounting/1/6a716b0f-38bc-4856-90ce-448fd713e3fe/expand-with-exprs) when clicking the `prettify query` button at the right side of query input field. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5383). * FEATURE: [vmalert](https://docs.victoriametrics.com/#vmalert): support filtering by group, rule or labels in [vmalert's UI](https://docs.victoriametrics.com/vmalert/#web) for `/groups` and `/alerts` pages. See [the pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/5791) by @victoramsantos. diff --git a/docs/vmctl.md b/docs/vmctl.md index 135939237..99c981239 100644 --- a/docs/vmctl.md +++ b/docs/vmctl.md @@ -207,6 +207,14 @@ http://opentsdb:4242/api/query?start=721h-ago&end=720h-ago&m=sum:1m-avg-none: