diff --git a/app/vmagent/README.md b/app/vmagent/README.md index f1625ada8b..b6c29c339a 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -483,7 +483,7 @@ See the docs at https://victoriametrics.github.io/vmagent.html . -dryRun Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse -enableTCP6 - Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP is used + Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used -envflag.enable Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set -envflag.prefix string @@ -586,6 +586,8 @@ See the docs at https://victoriametrics.github.io/vmagent.html . Whether to allow only supported fields in -promscrape.config . By default unsupported fields are silently skipped -promscrape.configCheckInterval duration Interval for checking for changes in '-promscrape.config' file. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes + -promscrape.consul.waitTime duration + Wait time used by Consul service discovery. Default value is used if not set -promscrape.consulSDCheckInterval duration Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s) -promscrape.disableCompression @@ -615,7 +617,7 @@ See the docs at https://victoriametrics.github.io/vmagent.html . -promscrape.kubernetesSDCheckInterval duration Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s) -promscrape.maxDroppedTargets int - The maximum number of droppedTargets shown at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000) + The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000) -promscrape.maxScrapeSize size The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216) @@ -684,7 +686,7 @@ See the docs at https://victoriametrics.github.io/vmagent.html . Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url Supports array of values separated by comma or specified via multiple flags. -remoteWrite.tmpDataPath string - Path to directory where temporary data for remote write component is stored (default "vmagent-remotewrite-data") + Path to directory where temporary data for remote write component is stored. See also -remoteWrite.maxDiskUsagePerURL (default "vmagent-remotewrite-data") -remoteWrite.url array Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems Supports array of values separated by comma or specified via multiple flags. diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 44539ad127..ace1d4f6a1 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -10,6 +10,7 @@ * `process_virtual_memory_peak_bytes` - peak virtual memory usage for the process. * FEATURE: accept and enforce `extra_label==` query arg at [Graphite APIs](https://victoriametrics.github.io/#graphite-api-usage). * FEATURE: use Influx field as metric name if measurement is empty and `-influxSkipSingleField` command-line is set. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1139 +* FEATURE: vmagent: add `-promscrape.consul.waitTime` command-line flag for tuning the maximum wait time for Consul service discovery. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1144). * BUGFIX: prevent from infinite loop on `{__graphite__="..."}` filters when a metric name contains `*`, `{` or `[` chars. * BUGFIX: prevent from infinite loop in `/metrics/find` and `/metrics/expand` [Graphite Metrics API handlers](https://victoriametrics.github.io/#graphite-metrics-api-usage) when they match metric names or labels with `*`, `{` or `[` chars. diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 0468a5a43b..1804fa2f71 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -1685,6 +1685,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Whether to allow only supported fields in -promscrape.config . By default unsupported fields are silently skipped -promscrape.configCheckInterval duration Interval for checking for changes in '-promscrape.config' file. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes + -promscrape.consul.waitTime duration + Wait time used by Consul service discovery. Default value is used if not set -promscrape.consulSDCheckInterval duration Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s) -promscrape.disableCompression diff --git a/docs/vmagent.md b/docs/vmagent.md index f1625ada8b..b6c29c339a 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -483,7 +483,7 @@ See the docs at https://victoriametrics.github.io/vmagent.html . -dryRun Whether to check only config files without running vmagent. The following files are checked: -promscrape.config, -remoteWrite.relabelConfig, -remoteWrite.urlRelabelConfig . Unknown config entries are allowed in -promscrape.config by default. This can be changed with -promscrape.config.strictParse -enableTCP6 - Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP is used + Whether to enable IPv6 for listening and dialing. By default only IPv4 TCP and UDP is used -envflag.enable Whether to enable reading flags from environment variables additionally to command line. Command line flag values have priority over values from environment vars. Flags are read only from command line if this flag isn't set -envflag.prefix string @@ -586,6 +586,8 @@ See the docs at https://victoriametrics.github.io/vmagent.html . Whether to allow only supported fields in -promscrape.config . By default unsupported fields are silently skipped -promscrape.configCheckInterval duration Interval for checking for changes in '-promscrape.config' file. By default the checking is disabled. Send SIGHUP signal in order to force config check for changes + -promscrape.consul.waitTime duration + Wait time used by Consul service discovery. Default value is used if not set -promscrape.consulSDCheckInterval duration Interval for checking for changes in Consul. This works only if consul_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details (default 30s) -promscrape.disableCompression @@ -615,7 +617,7 @@ See the docs at https://victoriametrics.github.io/vmagent.html . -promscrape.kubernetesSDCheckInterval duration Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s) -promscrape.maxDroppedTargets int - The maximum number of droppedTargets shown at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000) + The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000) -promscrape.maxScrapeSize size The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216) @@ -684,7 +686,7 @@ See the docs at https://victoriametrics.github.io/vmagent.html . Optional TLS server name to use for connections to -remoteWrite.url. By default the server name from -remoteWrite.url is used. If multiple args are set, then they are applied independently for the corresponding -remoteWrite.url Supports array of values separated by comma or specified via multiple flags. -remoteWrite.tmpDataPath string - Path to directory where temporary data for remote write component is stored (default "vmagent-remotewrite-data") + Path to directory where temporary data for remote write component is stored. See also -remoteWrite.maxDiskUsagePerURL (default "vmagent-remotewrite-data") -remoteWrite.url array Remote storage URL to write data to. It must support Prometheus remote_write API. It is recommended using VictoriaMetrics as remote storage. Example url: http://:8428/api/v1/write . Pass multiple -remoteWrite.url flags in order to write data concurrently to multiple remote storage systems Supports array of values separated by comma or specified via multiple flags. diff --git a/lib/promscrape/discovery/consul/api.go b/lib/promscrape/discovery/consul/api.go index 8927e64451..583af0e759 100644 --- a/lib/promscrape/discovery/consul/api.go +++ b/lib/promscrape/discovery/consul/api.go @@ -1,6 +1,7 @@ package consul import ( + "flag" "fmt" "io/ioutil" "os" @@ -14,6 +15,8 @@ import ( "github.com/VictoriaMetrics/fasthttp" ) +var waitTime = flag.Duration("promscrape.consul.waitTime", 0, "Wait time used by Consul service discovery. Default value is used if not set") + // apiConfig contains config for API server. type apiConfig struct { tagSeparator string @@ -116,7 +119,7 @@ func getDatacenter(client *discoveryutils.Client, dc string) (string, error) { } // maxWaitTime is duration for consul blocking request. -var maxWaitTime = func() time.Duration { +func maxWaitTime() time.Duration { d := discoveryutils.BlockingClientReadTimeout // Consul adds random delay up to wait/16, so reduce the timeout in order to keep it below BlockingClientReadTimeout. // See https://www.consul.io/api-docs/features/blocking @@ -125,17 +128,18 @@ var maxWaitTime = func() time.Duration { if d > 10*time.Minute { d = 10 * time.Minute } + if *waitTime > time.Second && *waitTime < d { + d = *waitTime + } return d -}() - -var maxWaitTimeStr = fmt.Sprintf("%ds", int(maxWaitTime.Seconds())) +} // getBlockingAPIResponse perfoms blocking request to Consul via client and returns response. // // See https://www.consul.io/api-docs/features/blocking . func getBlockingAPIResponse(client *discoveryutils.Client, path string, index int64) ([]byte, int64, error) { path += "&index=" + strconv.FormatInt(index, 10) - path += "&wait=" + maxWaitTimeStr + path += "&wait=" + fmt.Sprintf("%ds", int(maxWaitTime().Seconds())) getMeta := func(resp *fasthttp.Response) { ind := resp.Header.Peek("X-Consul-Index") if len(ind) == 0 {