app/{vminsert,vmagent}: a follow-up for b1aa8c3d8f

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1124

README.md

@@ -413,6 +413,10 @@ while VictoriaMetrics stores them with *milliseconds* precision.
 Extra labels may be added to all the written time series by passing `extra_label=name=value` query args.
 For example, `/write?extra_label=foo=bar` would add `{foo="bar"}` label to all the ingested metrics.
 
+Some plugins for Telegraf such as [fluentd](https://github.com/fangli/fluent-plugin-influxdb), [Juniper/open-nti](https://github.com/Juniper/open-nti)
+or [Juniper/jitmon](https://github.com/Juniper/jtimon) send `SHOW DATABASES` query to `/query` and expect a particular database name in the response.
+Comma-separated list of expected databases can be passed to VictoriaMetrics via `-influx.databaseNames` command-line flag.
+
 ## How to send data from Graphite-compatible agents such as [StatsD](https://github.com/etsy/statsd)
 
 Enable Graphite receiver in VictoriaMetrics by setting `-graphiteListenAddr` command line flag. For instance,

app/vmagent/README.md

@@ -515,7 +515,9 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
   -import.maxLineLen max_rows_per_line
     	The maximum length in bytes of a single line accepted by /api/v1/import; the line length can be limited with max_rows_per_line query arg passed to /api/v1/export
     	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 104857600)
-  -influx.databasesNames comma separated names of influx database, that will be returned for /query and /influx/query request.
+  -influx.databaseNames array
+    	Comma-separated list of database names to return from /query and /influx/query API. This can be needed for accepting data from Telegraf plugins such as https://github.com/fangli/fluent-plugin-influxdb
+    	Supports array of values separated by comma or specified via multiple flags.
   -influx.maxLineSize value
     	The maximum size in bytes for a single Influx line during parsing
     	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 262144)
@@ -574,6 +576,8 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
     	The number of number in the cluster of scrapers. It must be an unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster
   -promscrape.cluster.membersCount int
     	The number of members in a cluster of scrapers. Each member must have an unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default cluster scraping is disabled, i.e. a single scraper scrapes all the targets
+  -promscrape.cluster.replicationFactor int
+    	The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 2, then the deduplication must be enabled at remote storage side. See https://victoriametrics.github.io/#deduplication (default 1)
   -promscrape.config string
     	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. See https://victoriametrics.github.io/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
   -promscrape.config.dryRun
@@ -607,7 +611,7 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
   -promscrape.gceSDCheckInterval gce_sd_configs
     	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
   -promscrape.kubernetes.apiServerTimeout duration
-    	How frequently to reload the full state from Kuberntes API server (default 10m0s)
+    	How frequently to reload the full state from Kuberntes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval kubernetes_sd_configs
     	Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s)
   -promscrape.maxDroppedTargets droppedTargets

app/vmagent/main.go

@@ -23,6 +23,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutils"
 	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
 	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
 	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
@@ -49,7 +50,6 @@ var (
 	dryRun                 = flag.Bool("dryRun", false, "Whether to check only config files without running vmagent. The following files are checked: "+
 		"-promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . "+
 		"Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse")
-	influxDatabasesNames = flag.String("influx.databasesNames", "_internal", "Comma separated names of databases, that will be returned for /query and /influx/query api.")
 )
 
 var (
@@ -205,18 +205,8 @@ func requestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(http.StatusNoContent)
 		return true
 	case "/query":
-		// Emulate fake response for influx query.
-		// This is required for TSBS benchmark and some telegraph plugins.
 		influxQueryRequests.Inc()
-		var dbs string
+		influxutils.WriteDatabaseNames(w)
-		influxDbs := strings.Split(*influxDatabasesNames, ",")
-		for i := range influxDbs {
-			dbs += fmt.Sprintf(`"[%s]"`, influxDbs[i])
-			if i != len(influxDbs)-1 {
-				dbs += ","
-			}
-		}
-		fmt.Fprintf(w, `{"results":[{"name":"databases","columns":["name"],"series":[{"values":[%s]}]}]}`, dbs)
 		return true
 	case "/targets":
 		promscrapeTargetsRequests.Inc()

app/vminsert/main.go

@@ -19,6 +19,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/relabel"
 	"github.com/VictoriaMetrics/VictoriaMetrics/app/vminsert/vmimport"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/influxutils"
 	graphiteserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/graphite"
 	influxserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/influx"
 	opentsdbserver "github.com/VictoriaMetrics/VictoriaMetrics/lib/ingestserver/opentsdb"
@@ -40,7 +41,6 @@ var (
 		"Usually :4242 must be set. Doesn't work if empty")
 	opentsdbHTTPListenAddr = flag.String("opentsdbHTTPListenAddr", "", "TCP address to listen for OpentTSDB HTTP put requests. Usually :4242 must be set. Doesn't work if empty")
 	maxLabelsPerTimeseries = flag.Int("maxLabelsPerTimeseries", 30, "The maximum number of labels accepted per time series. Superfluous labels are dropped")
-	influxDatabasesNames   = flag.String("influx.databasesNames", "_internal", "Comma separated names of databases, that will be returned for /query and /influx/query api.")
 )
 
 var (
@@ -148,18 +148,8 @@ func RequestHandler(w http.ResponseWriter, r *http.Request) bool {
 		w.WriteHeader(http.StatusNoContent)
 		return true
 	case "/influx/query", "/query":
-		// Emulate fake response for influx query.
-		// This is required for TSBS benchmark and some telegraph plugins.
 		influxQueryRequests.Inc()
-		var dbs string
+		influxutils.WriteDatabaseNames(w)
-		influxDbs := strings.Split(*influxDatabasesNames, ",")
-		for i := range influxDbs {
-			dbs += fmt.Sprintf(`"[%s]"`, influxDbs[i])
-			if i != len(influxDbs)-1 {
-				dbs += ","
-			}
-		}
-		fmt.Fprintf(w, `{"results":[{"name":"databases","columns":["name"],"series":[{"values":[%s]}]}]}`, dbs)
 		return true
 	case "/prometheus/targets", "/targets":
 		promscrapeTargetsRequests.Inc()

docs/CHANGELOG.md

@@ -13,6 +13,7 @@
 * FEATURE: vmagent: support `proxy_tls_config`, `proxy_basic_auth`, `proxy_bearer_token` and `proxy_bearer_token_file` options at `scrape_config` section for configuring proxies specified via `proxy_url`. See [these docs](https://victoriametrics.github.io/vmagent.html#scraping-targets-via-a-proxy).
 * FEATURE: vmauth: allow using regexp paths in `url_map`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1112) for details.
 * FEATURE: accept `round_digits` query arg at `/api/v1/query` and `/api/v1/query_range` handlers. This option can be set at Prometheus datasource in Grafana for limiting the number of digits after the decimal point in response values.
+* FEATURE: add `-influx.databaseNames` command-line flag, which can be used for accepting data from some Telegraf plugins such as [fluentd plugin](https://github.com/fangli/fluent-plugin-influxdb). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1124).
 
 * BUGFIX: vmagent: prevent from high CPU usage bug during failing scrapes with small `scrape_timeout` (less than a few seconds).
 * BUGFIX: vmagent: reduce memory usage when Kubernetes service discovery is used in big number of distinct scrape config jobs by sharing Kubernetes object cache. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1113

docs/Single-server-VictoriaMetrics.md

@@ -413,6 +413,10 @@ while VictoriaMetrics stores them with *milliseconds* precision.
 Extra labels may be added to all the written time series by passing `extra_label=name=value` query args.
 For example, `/write?extra_label=foo=bar` would add `{foo="bar"}` label to all the ingested metrics.
 
+Some plugins for Telegraf such as [fluentd](https://github.com/fangli/fluent-plugin-influxdb), [Juniper/open-nti](https://github.com/Juniper/open-nti)
+or [Juniper/jitmon](https://github.com/Juniper/jtimon) send `SHOW DATABASES` query to `/query` and expect a particular database name in the response.
+Comma-separated list of expected databases can be passed to VictoriaMetrics via `-influx.databaseNames` command-line flag.
+
 ## How to send data from Graphite-compatible agents such as [StatsD](https://github.com/etsy/statsd)
 
 Enable Graphite receiver in VictoriaMetrics by setting `-graphiteListenAddr` command line flag. For instance,

docs/vmagent.md

@@ -515,7 +515,9 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
   -import.maxLineLen max_rows_per_line
     	The maximum length in bytes of a single line accepted by /api/v1/import; the line length can be limited with max_rows_per_line query arg passed to /api/v1/export
     	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 104857600)
-  -influx.databasesNames comma separated names of influx database, that will be returned for /query and /influx/query request.
+  -influx.databaseNames array
+    	Comma-separated list of database names to return from /query and /influx/query API. This can be needed for accepting data from Telegraf plugins such as https://github.com/fangli/fluent-plugin-influxdb
+    	Supports array of values separated by comma or specified via multiple flags.
   -influx.maxLineSize value
     	The maximum size in bytes for a single Influx line during parsing
     	Supports the following optional suffixes for values: KB, MB, GB, KiB, MiB, GiB (default 262144)
@@ -574,6 +576,8 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
     	The number of number in the cluster of scrapers. It must be an unique value in the range 0 ... promscrape.cluster.membersCount-1 across scrapers in the cluster
   -promscrape.cluster.membersCount int
     	The number of members in a cluster of scrapers. Each member must have an unique -promscrape.cluster.memberNum in the range 0 ... promscrape.cluster.membersCount-1 . Each member then scrapes roughly 1/N of all the targets. By default cluster scraping is disabled, i.e. a single scraper scrapes all the targets
+  -promscrape.cluster.replicationFactor int
+    	The number of members in the cluster, which scrape the same targets. If the replication factor is greater than 2, then the deduplication must be enabled at remote storage side. See https://victoriametrics.github.io/#deduplication (default 1)
   -promscrape.config string
     	Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. See https://victoriametrics.github.io/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details
   -promscrape.config.dryRun
@@ -607,7 +611,7 @@ See the docs at https://victoriametrics.github.io/vmagent.html .
   -promscrape.gceSDCheckInterval gce_sd_configs
     	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
   -promscrape.kubernetes.apiServerTimeout duration
-    	How frequently to reload the full state from Kuberntes API server (default 10m0s)
+    	How frequently to reload the full state from Kuberntes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval kubernetes_sd_configs
     	Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s)
   -promscrape.maxDroppedTargets droppedTargets

lib/influxutils/influxutils.go

@@ -0,0 +1,29 @@
+package influxutils
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
+)
+
+var influxDatabaseNames = flagutil.NewArray("influx.databaseNames", "Comma-separated list of database names to return from /query and /influx/query API. "+
+	"This can be needed for accepting data from Telegraf plugins such as https://github.com/fangli/fluent-plugin-influxdb")
+
+// WriteDatabaseNames writes influxDatabaseNames to w.
+func WriteDatabaseNames(w http.ResponseWriter) {
+	// Emulate fake response for influx query.
+	// This is required for TSBS benchmark and some Telegraf plugins.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1124
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	dbNames := *influxDatabaseNames
+	if len(dbNames) == 0 {
+		dbNames = []string{"_internal"}
+	}
+	dbs := make([]string, len(dbNames))
+	for i := range dbNames {
+		dbs[i] = fmt.Sprintf(`"[%s]"`, dbNames[i])
+	}
+	fmt.Fprintf(w, `{"results":[{"name":"databases","columns":["name"],"series":[{"values":[%s]}]}]}`, strings.Join(dbs, ","))
+}