diff --git a/README.md b/README.md index 14d679180..f31cca561 100644 --- a/README.md +++ b/README.md @@ -1719,6 +1719,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s) -promscrape.maxDroppedTargets int The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000) + -promscrape.maxResponseHeadersSize size + The maximum size of http response headers from Prometheus scrape targets + Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 4096) -promscrape.maxScrapeSize size The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216) diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 13b764081..d1c5134ac 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -858,6 +858,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s) -promscrape.maxDroppedTargets int The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000) + -promscrape.maxResponseHeadersSize size + The maximum size of http response headers from Prometheus scrape targets + Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 4096) -promscrape.maxScrapeSize size The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 44fe2878c..573df4a08 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -10,6 +10,7 @@ sort: 15 * FEATURE: vmagent: add `collapse` and `expand` buttons per each group of targets with the same `job_name` at `http://vmagent:8429/targets` page. * FEATURE: automatically detect timestamp precision (ns, us, ms or s) for the data ingested into VictoriaMetrics via [InfluxDB line protocol](https://docs.victoriametrics.com/#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf). * FEATURE: vmagent: add ability to protect `/config` page with auth key via `-configAuthKey` command-line flag. This page may contain sensitive information such as passwords, so it may be good to restrict access to this page. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1764). +* FEATURE: vmagent: add `-promscrape.maxResponseHeadersSize` command-line flag for tuning the maximum HTTP response headers size for Prometheus scrape targets. * FEATURE: add [label_graphite_group](https://docs.victoriametrics.com/MetricsQL.html#label_graphite_group) function for extracting the given groups from Graphite metric names. * FEATURE: add [limit_offset](https://docs.victoriametrics.com/MetricsQL.html#limit_offset) function, which can be used for implementing simple paging over big number of time series. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1778). diff --git a/docs/README.md b/docs/README.md index 14d679180..f31cca561 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1719,6 +1719,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s) -promscrape.maxDroppedTargets int The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000) + -promscrape.maxResponseHeadersSize size + The maximum size of http response headers from Prometheus scrape targets + Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 4096) -promscrape.maxScrapeSize size The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216) diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index bfb3e891b..190943e77 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -1723,6 +1723,9 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s) -promscrape.maxDroppedTargets int The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000) + -promscrape.maxResponseHeadersSize size + The maximum size of http response headers from Prometheus scrape targets + Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 4096) -promscrape.maxScrapeSize size The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216) diff --git a/docs/vmagent.md b/docs/vmagent.md index e2176f68d..a39b8d9e2 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -862,6 +862,9 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Interval for checking for changes in Kubernetes API server. This works only if kubernetes_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details (default 30s) -promscrape.maxDroppedTargets int The maximum number of droppedTargets to show at /api/v1/targets page. Increase this value if your setup drops more scrape targets during relabeling and you need investigating labels for all the dropped targets. Note that the increased number of tracked dropped targets may result in increased memory usage (default 1000) + -promscrape.maxResponseHeadersSize size + The maximum size of http response headers from Prometheus scrape targets + Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 4096) -promscrape.maxScrapeSize size The maximum size of scrape response in bytes to process from Prometheus targets. Bigger responses are rejected Supports the following optional suffixes for size values: KB, MB, GB, KiB, MiB, GiB (default 16777216) diff --git a/lib/promscrape/client.go b/lib/promscrape/client.go index 31c13456b..2447a92e5 100644 --- a/lib/promscrape/client.go +++ b/lib/promscrape/client.go @@ -23,7 +23,8 @@ import ( var ( maxScrapeSize = flagutil.NewBytes("promscrape.maxScrapeSize", 16*1024*1024, "The maximum size of scrape response in bytes to process from Prometheus targets. "+ "Bigger responses are rejected") - disableCompression = flag.Bool("promscrape.disableCompression", false, "Whether to disable sending 'Accept-Encoding: gzip' request headers to all the scrape targets. "+ + maxResponseHeadersSize = flagutil.NewBytes("promscrape.maxResponseHeadersSize", 4096, "The maximum size of http response headers from Prometheus scrape targets") + disableCompression = flag.Bool("promscrape.disableCompression", false, "Whether to disable sending 'Accept-Encoding: gzip' request headers to all the scrape targets. "+ "This may reduce CPU usage on scrape targets at the cost of higher network bandwidth utilization. "+ "It is possible to set 'disable_compression: true' individually per each 'scrape_config' section in '-promscrape.config' for fine grained control") disableKeepAlive = flag.Bool("promscrape.disableKeepAlive", false, "Whether to disable HTTP keep-alive connections when scraping all the targets. "+ @@ -105,6 +106,7 @@ func newClient(sw *ScrapeWork) *client { WriteTimeout: 10 * time.Second, MaxResponseBodySize: maxScrapeSize.N, MaxIdempotentRequestAttempts: 1, + ReadBufferSize: maxResponseHeadersSize.N, } var sc *http.Client var proxyURLFunc func(*http.Request) (*url.URL, error) @@ -113,14 +115,15 @@ func newClient(sw *ScrapeWork) *client { } sc = &http.Client{ Transport: &http.Transport{ - TLSClientConfig: tlsCfg, - Proxy: proxyURLFunc, - TLSHandshakeTimeout: 10 * time.Second, - IdleConnTimeout: 2 * sw.ScrapeInterval, - DisableCompression: *disableCompression || sw.DisableCompression, - DisableKeepAlives: *disableKeepAlive || sw.DisableKeepAlive, - DialContext: statStdDial, - MaxIdleConnsPerHost: 100, + TLSClientConfig: tlsCfg, + Proxy: proxyURLFunc, + TLSHandshakeTimeout: 10 * time.Second, + IdleConnTimeout: 2 * sw.ScrapeInterval, + DisableCompression: *disableCompression || sw.DisableCompression, + DisableKeepAlives: *disableKeepAlive || sw.DisableKeepAlive, + DialContext: statStdDial, + MaxIdleConnsPerHost: 100, + MaxResponseHeaderBytes: int64(maxResponseHeadersSize.N), // Set timeout for receiving the first response byte, // since the duration for reading the full response can be much bigger because of stream parsing.