From 7c37e9aea938c9c4c9b0b1d3510165e71be1ce3b Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 15 Mar 2021 21:37:13 +0200 Subject: [PATCH] app/{vminsert,vmagent}: a follow-up for b1aa8c3d8f68e9cc8f6bbcfa56bf3f6462c328db Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1124 --- README.md | 4 ++++ app/vmagent/README.md | 8 ++++++-- app/vmagent/main.go | 14 ++----------- app/vminsert/main.go | 14 ++----------- docs/CHANGELOG.md | 1 + docs/Single-server-VictoriaMetrics.md | 4 ++++ docs/vmagent.md | 8 ++++++-- lib/influxutils/influxutils.go | 29 +++++++++++++++++++++++++++ 8 files changed, 54 insertions(+), 28 deletions(-) create mode 100644 lib/influxutils/influxutils.go diff --git a/README.md b/README.md index ef1c954be..07ff8889c 100644 --- a/README.md +++ b/README.md @@ -413,6 +413,10 @@ while VictoriaMetrics stores them with *milliseconds* precision. Extra labels may be added to all the written time series by passing `extra_label=name=value` query args. For example, `/write?extra_label=foo=bar` would add `{foo="bar"}` label to all the ingested metrics. +Some plugins for Telegraf such as [fluentd](https://github.com/fangli/fluent-plugin-influxdb), [Juniper/open-nti](https://github.com/Juniper/open-nti) +or [Juniper/jitmon](https://github.com/Juniper/jtimon) send `SHOW DATABASES` query to `/query` and expect a particular database name in the response. +Comma-separated list of expected databases can be passed to VictoriaMetrics via `-influx.databaseNames` command-line flag. + ## How to send data from Graphite-compatible agents such as [StatsD](https://github.com/etsy/statsd) Enable Graphite receiver in VictoriaMetrics by setting `-graphiteListenAddr` command line flag. For instance, diff --git a/app/vmagent/README.md b/app/vmagent/README.md index cf6a739b8..86fd70dd6 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -515,7 +515,9 @@ See the docs at https://victoriametrics.github.io/vmagent.html . -import.maxLineLen max_rows_per_line The maximum length in bytes of a single line accepted by /api/v1/import; the line length can be limited with max_rows_per_line query arg passed to /api/v1/export Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 104857600) - -influx.databasesNames comma separated names of influx database, that will be returned for /query and /influx/query request. + -influx.databaseNames array + Comma-separated list of database names to return from /query and /influx/query API. This can be needed for accepting data from Telegraf plugins such as https://github.com/fangli/fluent-plugin-influxdb + Supports array of values separated by comma or specified via multiple flags. -influx.maxLineSize value The maximum size in bytes for a single Influx line during parsing Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 262144) @@ -574,6 +576,8 @@ See the docs at https://victoriametrics.github.io/vmagent.html . The number of number in the cluster of scrapers. It must be an unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster -promscrape.cluster.membersCount int The number of members in a cluster of scrapers. Each member must have an unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default cluster scraping is disabled, i.e. a single scraper scrapes all the targets + -promscrape.cluster.replicationFactor int + The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 2, then the deduplication must be enabled at remote storage side. See https://victoriametrics.github.io/#deduplication (default 1) -promscrape.config string Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. See https://victoriametrics.github.io/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details -promscrape.config.dryRun @@ -607,7 +611,7 @@ See the docs at https://victoriametrics.github.io/vmagent.html . -promscrape.gceSDCheckInterval gce_sd_configs Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s) -promscrape.kubernetes.apiServerTimeout duration - How frequently to reload the full state from Kuberntes API server (default 10m0s) + How frequently to reload the full state from Kuberntes API server (default 30m0s) -promscrape.kubernetesSDCheckInterval kubernetes_sd_configs Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s) -promscrape.maxDroppedTargets droppedTargets diff --git a/app/vmagent/main.go b/app/vmagent/main.go index 02b65641c..2011ccffb 100644 --- a/app/vmagent/main.go +++ b/app/vmagent/main.go @@ -23,6 +23,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag" "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutils" graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite" influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx" opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb" @@ -49,7 +50,6 @@ var ( dryRun = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+ "-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+ "Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse") - influxDatabasesNames = flag.String("influx.databasesNames", "_internal", "Comma separated names of databases, that will be returned for /query and /influx/query api.") ) var ( @@ -205,18 +205,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool { w.WriteHeader(http.StatusNoContent) return true case "/query": - // Emulate fake response for influx query. - // This is required for TSBS benchmark and some telegraph plugins. influxQueryRequests.Inc() - var dbs string - influxDbs := strings.Split(*influxDatabasesNames, ",") - for i := range influxDbs { - dbs += fmt.Sprintf(`"[%s]"`, influxDbs[i]) - if i != len(influxDbs)-1 { - dbs += "," - } - } - fmt.Fprintf(w, `{"results":[{"name":"databases","columns":["name"],"series":[{"values":[%s]}]}]}`, dbs) + influxutils.WriteDatabaseNames(w) return true case "/targets": promscrapeTargetsRequests.Inc() diff --git a/app/vminsert/main.go b/app/vminsert/main.go index 55696ec68..0a3449ca7 100644 --- a/app/vminsert/main.go +++ b/app/vminsert/main.go @@ -19,6 +19,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel" "github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport" "github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutils" graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite" influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx" opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb" @@ -40,7 +41,6 @@ var ( "Usually :4242 must be set. Doesn't work if empty") opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty") maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped") - influxDatabasesNames = flag.String("influx.databasesNames", "_internal", "Comma separated names of databases, that will be returned for /query and /influx/query api.") ) var ( @@ -148,18 +148,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool { w.WriteHeader(http.StatusNoContent) return true case "/influx/query", "/query": - // Emulate fake response for influx query. - // This is required for TSBS benchmark and some telegraph plugins. influxQueryRequests.Inc() - var dbs string - influxDbs := strings.Split(*influxDatabasesNames, ",") - for i := range influxDbs { - dbs += fmt.Sprintf(`"[%s]"`, influxDbs[i]) - if i != len(influxDbs)-1 { - dbs += "," - } - } - fmt.Fprintf(w, `{"results":[{"name":"databases","columns":["name"],"series":[{"values":[%s]}]}]}`, dbs) + influxutils.WriteDatabaseNames(w) return true case "/prometheus/targets", "/targets": promscrapeTargetsRequests.Inc() diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7665d4aaf..26411e4e8 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -13,6 +13,7 @@ * FEATURE: vmagent: support `proxy_tls_config`, `proxy_basic_auth`, `proxy_bearer_token` and `proxy_bearer_token_file` options at `scrape_config` section for configuring proxies specified via `proxy_url`. See [these docs](https://victoriametrics.github.io/vmagent.html#scraping-targets-via-a-proxy). * FEATURE: vmauth: allow using regexp paths in `url_map`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1112) for details. * FEATURE: accept `round_digits` query arg at `/api/v1/query` and `/api/v1/query_range` handlers. This option can be set at Prometheus datasource in Grafana for limiting the number of digits after the decimal point in response values. +* FEATURE: add `-influx.databaseNames` command-line flag, which can be used for accepting data from some Telegraf plugins such as [fluentd plugin](https://github.com/fangli/fluent-plugin-influxdb). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1124). * BUGFIX: vmagent: prevent from high CPU usage bug during failing scrapes with small `scrape_timeout` (less than a few seconds). * BUGFIX: vmagent: reduce memory usage when Kubernetes service discovery is used in big number of distinct scrape config jobs by sharing Kubernetes object cache. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1113 diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index ef1c954be..07ff8889c 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -413,6 +413,10 @@ while VictoriaMetrics stores them with *milliseconds* precision. Extra labels may be added to all the written time series by passing `extra_label=name=value` query args. For example, `/write?extra_label=foo=bar` would add `{foo="bar"}` label to all the ingested metrics. +Some plugins for Telegraf such as [fluentd](https://github.com/fangli/fluent-plugin-influxdb), [Juniper/open-nti](https://github.com/Juniper/open-nti) +or [Juniper/jitmon](https://github.com/Juniper/jtimon) send `SHOW DATABASES` query to `/query` and expect a particular database name in the response. +Comma-separated list of expected databases can be passed to VictoriaMetrics via `-influx.databaseNames` command-line flag. + ## How to send data from Graphite-compatible agents such as [StatsD](https://github.com/etsy/statsd) Enable Graphite receiver in VictoriaMetrics by setting `-graphiteListenAddr` command line flag. For instance, diff --git a/docs/vmagent.md b/docs/vmagent.md index cf6a739b8..86fd70dd6 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -515,7 +515,9 @@ See the docs at https://victoriametrics.github.io/vmagent.html . -import.maxLineLen max_rows_per_line The maximum length in bytes of a single line accepted by /api/v1/import; the line length can be limited with max_rows_per_line query arg passed to /api/v1/export Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 104857600) - -influx.databasesNames comma separated names of influx database, that will be returned for /query and /influx/query request. + -influx.databaseNames array + Comma-separated list of database names to return from /query and /influx/query API. This can be needed for accepting data from Telegraf plugins such as https://github.com/fangli/fluent-plugin-influxdb + Supports array of values separated by comma or specified via multiple flags. -influx.maxLineSize value The maximum size in bytes for a single Influx line during parsing Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 262144) @@ -574,6 +576,8 @@ See the docs at https://victoriametrics.github.io/vmagent.html . The number of number in the cluster of scrapers. It must be an unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster -promscrape.cluster.membersCount int The number of members in a cluster of scrapers. Each member must have an unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default cluster scraping is disabled, i.e. a single scraper scrapes all the targets + -promscrape.cluster.replicationFactor int + The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 2, then the deduplication must be enabled at remote storage side. See https://victoriametrics.github.io/#deduplication (default 1) -promscrape.config string Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. See https://victoriametrics.github.io/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details -promscrape.config.dryRun @@ -607,7 +611,7 @@ See the docs at https://victoriametrics.github.io/vmagent.html . -promscrape.gceSDCheckInterval gce_sd_configs Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s) -promscrape.kubernetes.apiServerTimeout duration - How frequently to reload the full state from Kuberntes API server (default 10m0s) + How frequently to reload the full state from Kuberntes API server (default 30m0s) -promscrape.kubernetesSDCheckInterval kubernetes_sd_configs Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s) -promscrape.maxDroppedTargets droppedTargets diff --git a/lib/influxutils/influxutils.go b/lib/influxutils/influxutils.go new file mode 100644 index 000000000..5a648b9e5 --- /dev/null +++ b/lib/influxutils/influxutils.go @@ -0,0 +1,29 @@ +package influxutils + +import ( + "fmt" + "net/http" + "strings" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil" +) + +var influxDatabaseNames = flagutil.NewArray("influx.databaseNames", "Comma-separated list of database names to return from /query and /influx/query API. "+ + "This can be needed for accepting data from Telegraf plugins such as https://github.com/fangli/fluent-plugin-influxdb") + +// WriteDatabaseNames writes influxDatabaseNames to w. +func WriteDatabaseNames(w http.ResponseWriter) { + // Emulate fake response for influx query. + // This is required for TSBS benchmark and some Telegraf plugins. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1124 + w.Header().Set("Content-Type", "application/json; charset=utf-8") + dbNames := *influxDatabaseNames + if len(dbNames) == 0 { + dbNames = []string{"_internal"} + } + dbs := make([]string, len(dbNames)) + for i := range dbNames { + dbs[i] = fmt.Sprintf(`"[%s]"`, dbNames[i]) + } + fmt.Fprintf(w, `{"results":[{"name":"databases","columns":["name"],"series":[{"values":[%s]}]}]}`, strings.Join(dbs, ",")) +}