From 36b92f07f7dd26dead65083d38886d13ed5157e8 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Wed, 26 Oct 2022 01:52:05 +0300 Subject: [PATCH] lib/envflag: allow referring environment variables in command-line flags --- docs/CHANGELOG.md | 1 + docs/README.md | 7 ++++++- docs/Single-server-VictoriaMetrics.md | 7 ++++++- lib/envflag/envflag.go | 17 +++++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 8668ffddee..06b56f3f84 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -20,6 +20,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [VictoriaMetric enterprise](https://docs.victoriametrics.com/enterprise.html): allow configuring multiple retentions for distinct sets of time series. See [these docs](https://docs.victoriametrics.com/#retention-filters), [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/143) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/289) feature request. * FEATURE: [VictoriaMetric cluster enterprise](https://docs.victoriametrics.com/enterprise.html): add support for multiple retentions for distinct tenants - see [these docs](https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#retention-filters) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/143) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/289) feature request. * FEATURE: allow limiting memory usage on a per-query basis with `-search.maxMemoryPerQuery` command-line flag. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3203). +* FEATURE: allow using environment variables inside command-line flags passed to all VictoriaMetrics components. For example, if `AUTH_KEY=top-secret` environment variable is set, then `-metricsAuthKey=%{AUTH_KEY}` command-line flag is automatically expanded to `-storageDataPath=top-secret` at VictoriaMetrics startup. See [these docs](https://docs.victoriametrics.com/#environment-variables) for details. * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): drop all the labels with `__` prefix from discovered targets in the same way as Prometheus does according to [this article](https://www.robustperception.io/life-of-a-label/). Previously the following labels were available during [metric-level relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs): `__address__`, `__scheme__`, `__metrics_path__`, `__scrape_interval__`, `__scrape_timeout__`, `__param_*`. Now these labels are available only during [target-level relabeling](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config). This should reduce CPU usage and memory usage for `vmagent` setups, which scrape big number of targets. * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): improve the performance for metric-level [relabeling](https://docs.victoriametrics.com/vmagent.html#relabeling), which can be applied via `metric_relabel_configs` section at [scrape_configs](https://docs.victoriametrics.com/sd_configs.html#scrape_configs), via `-remoteWrite.relabelConfig` or via `-remoteWrite.urlRelabelConfig` command-line options. * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): allow specifying full url in scrape target addresses (aka `__address__` label). This makes valid the following `-promscrape.config`: diff --git a/docs/README.md b/docs/README.md index 21f04eebb8..9c9d3c299d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -133,7 +133,12 @@ VictoriaMetrics is developed at a fast pace, so it is recommended periodically c ### Environment variables -Each flag value can be set via environment variables according to these rules: +All the VictoriaMetrics components allow referring environment variables in command-line flags via `${ENV_VAR}` syntax. +For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret` +if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup. +This expansion doesn't need any special shell - it is performed by VictoriaMetrics itself. + +Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules: * The `-envflag.enable` flag must be set. * Each `.` char in flag name must be substituted with `_` (for example `-insert.maxQueueDuration ` will translate to `insert_maxQueueDuration=`). diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index b3bcaa4080..449ede768a 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -136,7 +136,12 @@ VictoriaMetrics is developed at a fast pace, so it is recommended periodically c ### Environment variables -Each flag value can be set via environment variables according to these rules: +All the VictoriaMetrics components allow referring environment variables in command-line flags via `${ENV_VAR}` syntax. +For example, `-metricsAuthKey=%{METRICS_AUTH_KEY}` is automatically expanded to `-metricsAuthKey=top-secret` +if `METRICS_AUTH_KEY=top-secret` environment variable exists at VictoriaMetrics startup. +This expansion doesn't need any special shell - it is performed by VictoriaMetrics itself. + +Additionally, all the VictoriaMetrics components allow setting flag values via environment variables according to these rules: * The `-envflag.enable` flag must be set. * Each `.` char in flag name must be substituted with `_` (for example `-insert.maxQueueDuration ` will translate to `insert_maxQueueDuration=`). diff --git a/lib/envflag/envflag.go b/lib/envflag/envflag.go index e4cb5b2e28..6b333c8c7e 100644 --- a/lib/envflag/envflag.go +++ b/lib/envflag/envflag.go @@ -5,6 +5,8 @@ import ( "log" "os" "strings" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/envtemplate" ) var ( @@ -20,6 +22,21 @@ var ( // // This function must be called instead of flag.Parse() before using any flags in the program. func Parse() { + // Substitute %{ENV_VAR} inside args with the corresponding environment variable values + args := os.Args[1:] + dstArgs := args[:0] + for _, arg := range args { + b, err := envtemplate.Replace([]byte(arg)) + if err != nil { + log.Fatalf("cannot process arg %q: %s", arg, err) + } + if len(b) > 0 { + dstArgs = append(dstArgs, string(b)) + } + } + os.Args = os.Args[:1+len(dstArgs)] + + // Parse flags flag.Parse() if !*enable { return