From 50aa34bcbe254025db0a133e1b5034e66d7cf089 Mon Sep 17 00:00:00 2001
From: Aliaksandr Valialkin <valyala@gmail.com>
Date: Sat, 20 Jun 2020 18:19:43 +0300
Subject: [PATCH] lib/promscrape/discovery/consul: reduce load on Consul when
 discovering big number of targets by using background caching

Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/574
---
 lib/promscrape/discovery/consul/api.go          | 1 +
 lib/promscrape/discovery/consul/service_node.go | 9 ++++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/promscrape/discovery/consul/api.go b/lib/promscrape/discovery/consul/api.go
index 92eb3685cb..cbcd93ccf6 100644
--- a/lib/promscrape/discovery/consul/api.go
+++ b/lib/promscrape/discovery/consul/api.go
@@ -124,6 +124,7 @@ func getAPIResponse(cfg *apiConfig, path string) ([]byte, error) {
 	}
 	path += fmt.Sprintf("%sdc=%s", separator, url.QueryEscape(cfg.datacenter))
 	if cfg.allowStale {
+		// See https://www.consul.io/api/features/consistency
 		path += "&stale"
 	}
 	if len(cfg.nodeMeta) > 0 {
diff --git a/lib/promscrape/discovery/consul/service_node.go b/lib/promscrape/discovery/consul/service_node.go
index 83141b363e..950591ece7 100644
--- a/lib/promscrape/discovery/consul/service_node.go
+++ b/lib/promscrape/discovery/consul/service_node.go
@@ -109,12 +109,19 @@ func shouldCollectServiceByTags(filterTags, tags []string) bool {
 func getServiceNodes(cfg *apiConfig, serviceName string) ([]ServiceNode, error) {
 	// See https://www.consul.io/api/health.html#list-nodes-for-service
 	path := fmt.Sprintf("/v1/health/service/%s", serviceName)
+	// The /v1/health/service/:service endpoint supports background refresh caching,
+	// which guarantees fresh results obtained from local Consul agent.
+	// See https://www.consul.io/api-docs/health#list-nodes-for-service
+	// and https://www.consul.io/api/features/caching for details.
+	// Query cached results in order to reduce load on Consul cluster.
+	// See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/574 .
+	path += "?cached"
 	var tagsArgs []string
 	for _, tag := range cfg.tags {
 		tagsArgs = append(tagsArgs, fmt.Sprintf("tag=%s", url.QueryEscape(tag)))
 	}
 	if len(tagsArgs) > 0 {
-		path += "?" + strings.Join(tagsArgs, "&")
+		path += "&" + strings.Join(tagsArgs, "&")
 	}
 	data, err := getAPIResponse(cfg, path)
 	if err != nil {