adds http_sd (#1399)

* adds http_sd

* adds X-Prometheus-Refresh-Interval-Seconds header

* Update lib/promscrape/discovery/http/api.go

Co-authored-by: Aliaksandr Valialkin <valyala@gmail.com>

README.md

@@ -344,6 +344,7 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la
 * [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config)
 * [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config)
 * [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config)
+* [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config)
 
 
 Other `*_sd_config` types will be supported in the future.
@@ -1746,6 +1747,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
     	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s)
   -promscrape.gceSDCheckInterval duration
     	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
+  -promscrape.httpSDCheckInterval duration
+        Interval for checking for changes in http service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details (default 1m0s)    	
   -promscrape.kubernetes.apiServerTimeout duration
     	How frequently to reload the full state from Kuberntes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval duration

app/vmagent/README.md

@@ -179,6 +179,8 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details.
 * `digitalocean_sd_configs` is for scraping targerts registered in [DigitalOcean](https://www.digitalocean.com/)
   See [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) for details.
+* `http_sd_configs` is for scraping targerts registered in http service discovery.
+  See [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) for details.
 
 Please file feature requests to [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
 
@@ -653,6 +655,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s)
   -promscrape.gceSDCheckInterval duration
     	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
+  -promscrape.httpSDCheckInterval duration
+        Interval for checking for changes in http service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details (default 1m0s)    	   	
   -promscrape.kubernetes.apiServerTimeout duration
     	How frequently to reload the full state from Kuberntes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval duration

docs/Single-server-VictoriaMetrics.md

@@ -348,6 +348,7 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la
 * [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config)
 * [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config)
 * [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config)
+* [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config)
 
 
 Other `*_sd_config` types will be supported in the future.
@@ -1750,6 +1751,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li
     	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s)
   -promscrape.gceSDCheckInterval duration
     	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
+  -promscrape.httpSDCheckInterval duration
+        Interval for checking for changes in http service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details (default 1m0s)    	
   -promscrape.kubernetes.apiServerTimeout duration
     	How frequently to reload the full state from Kuberntes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval duration

docs/vmagent.md

@@ -183,6 +183,8 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details.
 * `digitalocean_sd_configs` is for scraping targerts registered in [DigitalOcean](https://www.digitalocean.com/)
   See [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) for details.
+* `http_sd_configs` is for scraping targerts registered in http service discovery.
+  See [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) for details.
 
 Please file feature requests to [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
 
@@ -657,6 +659,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html .
     	Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s)
   -promscrape.gceSDCheckInterval duration
     	Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s)
+  -promscrape.httpSDCheckInterval duration
+        Interval for checking for changes in http service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details (default 1m0s)    	   	
   -promscrape.kubernetes.apiServerTimeout duration
     	How frequently to reload the full state from Kuberntes API server (default 30m0s)
   -promscrape.kubernetesSDCheckInterval duration

lib/promscrape/config.go

@@ -25,6 +25,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ec2"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/eureka"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/http"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/openstack"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
@@ -129,6 +130,7 @@ type ScrapeConfig struct {
 	EC2SDConfigs          []ec2.SDConfig          `yaml:"ec2_sd_configs,omitempty"`
 	GCESDConfigs          []gce.SDConfig          `yaml:"gce_sd_configs,omitempty"`
 	DigitaloceanSDConfigs []digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"`
+	HTTPSDConfigs         []http.SDConfig         `yaml:"http_sd_configs,omitempty"`
 
 	// These options are supported only by lib/promscrape.
 	RelabelDebug        bool                       `yaml:"relabel_debug,omitempty"`
@@ -529,6 +531,34 @@ func (cfg *Config) getDigitalOceanDScrapeWork(prev []*ScrapeWork) []*ScrapeWork
 	return dst
 }
 
+// getHTTPDScrapeWork returns `http_sd_configs` ScrapeWork from cfg.
+func (cfg *Config) getHTTPDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
+	swsPrevByJob := getSWSByJob(prev)
+	dst := make([]*ScrapeWork, 0, len(prev))
+	for i := range cfg.ScrapeConfigs {
+		sc := &cfg.ScrapeConfigs[i]
+		dstLen := len(dst)
+		ok := true
+		for j := range sc.HTTPSDConfigs {
+			sdc := &sc.HTTPSDConfigs[j]
+			var okLocal bool
+			dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "http_sd_config")
+			if ok {
+				ok = okLocal
+			}
+		}
+		if ok {
+			continue
+		}
+		swsPrev := swsPrevByJob[sc.swc.jobName]
+		if len(swsPrev) > 0 {
+			logger.Errorf("there were errors when discovering http targets for job %q, so preserving the previous targets", sc.swc.jobName)
+			dst = append(dst[:dstLen], swsPrev...)
+		}
+	}
+	return dst
+}
+
 // getFileSDScrapeWork returns `file_sd_configs` ScrapeWork from cfg.
 func (cfg *Config) getFileSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork {
 	// Create a map for the previous scrape work.

lib/promscrape/discovery/http/api.go

@@ -0,0 +1,78 @@
+package http
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strconv"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+	"github.com/VictoriaMetrics/fasthttp"
+)
+
+var configMap = discoveryutils.NewConfigMap()
+
+type apiConfig struct {
+	client *discoveryutils.Client
+	path   string
+}
+
+// httpGroupTarget respresent prometheus GroupTarget
+// https://prometheus.io/docs/prometheus/latest/http_sd/
+type httpGroupTarget struct {
+	Targets []string          `json:"targets"`
+	Labels  map[string]string `json:"labels"`
+}
+
+func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
+	ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse auth config: %w", err)
+	}
+	parsedURL, err := url.Parse(sdc.URL)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse http_sd URL: %w", err)
+	}
+	apiServer := fmt.Sprintf("%s://%s", parsedURL.Scheme, parsedURL.Host)
+
+	proxyAC, err := sdc.ProxyClientConfig.NewConfig(baseDir)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse proxy auth config: %w", err)
+	}
+	client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create HTTP client for %q: %w", apiServer, err)
+	}
+	cfg := &apiConfig{
+		client: client,
+		path:   parsedURL.RequestURI(),
+	}
+	return cfg, nil
+}
+
+func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
+	v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) })
+	if err != nil {
+		return nil, err
+	}
+	return v.(*apiConfig), nil
+}
+
+func getHTTPTargets(cfg *apiConfig) ([]httpGroupTarget, error) {
+	data, err := cfg.client.GetAPIResponseWithReqParams(cfg.path, func(request *fasthttp.Request) {
+		request.Header.Set("X-Prometheus-Refresh-Interval-Seconds", strconv.FormatFloat(SDCheckInterval.Seconds(), 'f', 0, 64))
+		request.Header.Set("Accept", "application/json")
+	})
+	if err != nil {
+		return nil, fmt.Errorf("cannot read http_sd api response: %w", err)
+	}
+	return parseAPIResponse(data, cfg.path)
+}
+
+func parseAPIResponse(data []byte, path string) ([]httpGroupTarget, error) {
+	var r []httpGroupTarget
+	if err := json.Unmarshal(data, &r); err != nil {
+		return nil, fmt.Errorf("cannot parse http_sd api response path: %s, err:  %w", path, err)
+	}
+	return r, nil
+}

lib/promscrape/discovery/http/api_test.go

@@ -0,0 +1,49 @@
+package http
+
+import (
+	"reflect"
+	"testing"
+)
+
+func Test_parseAPIResponse(t *testing.T) {
+	type args struct {
+		data []byte
+		path string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []httpGroupTarget
+		wantErr bool
+	}{
+
+		{
+			name: "parse ok",
+			args: args{
+				path: "/ok",
+				data: []byte(`[
+                {"targets": ["http://target-1:9100","http://target-2:9150"],
+                "labels": {"label-1":"value-1"} }
+                ]`),
+			},
+			want: []httpGroupTarget{
+				{
+					Labels:  map[string]string{"label-1": "value-1"},
+					Targets: []string{"http://target-1:9100", "http://target-2:9150"},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseAPIResponse(tt.args.data, tt.args.path)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseAPIResponse() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("parseAPIResponse() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

lib/promscrape/discovery/http/http.go

@@ -0,0 +1,56 @@
+package http
+
+import (
+	"flag"
+	"fmt"
+	"time"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy"
+)
+
+// SDCheckInterval defines interval for targets refresh.
+var SDCheckInterval = flag.Duration("promscrape.httpSDCheckInterval", time.Minute, "Interval for checking for changes in http endpoint service discovery. "+
+	"This works only if http_sd_configs is configured in '-promscrape.config' file. "+
+	"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details")
+
+// SDConfig represents service discovery config for http.
+//
+// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config
+type SDConfig struct {
+	URL               string                     `yaml:"url"`
+	HTTPClientConfig  promauth.HTTPClientConfig  `yaml:",inline"`
+	ProxyURL          proxy.URL                  `yaml:"proxy_url,omitempty"`
+	ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"`
+}
+
+// GetLabels returns http service discovery labels according to sdc.
+func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) {
+	cfg, err := getAPIConfig(sdc, baseDir)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get API config: %w", err)
+	}
+
+	hts, err := getHTTPTargets(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return addHTTPTargetLabels(hts), nil
+}
+
+func addHTTPTargetLabels(src []httpGroupTarget) []map[string]string {
+	ms := make([]map[string]string, 0, len(src))
+	for _, targetGroup := range src {
+		labels := targetGroup.Labels
+		for _, target := range targetGroup.Targets {
+			m := make(map[string]string, len(labels))
+			for k, v := range labels {
+				m[k] = v
+			}
+			m["__address__"] = target
+			ms = append(ms, m)
+		}
+	}
+	return ms
+}

lib/promscrape/discovery/http/http_test.go

@@ -0,0 +1,56 @@
+package http
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
+)
+
+func Test_addHTTPTargetLabels(t *testing.T) {
+	type args struct {
+		src []httpGroupTarget
+	}
+	tests := []struct {
+		name string
+		args args
+		want [][]prompbmarshal.Label
+	}{
+		{
+			name: "add ok",
+			args: args{
+				src: []httpGroupTarget{
+					{
+						Targets: []string{"127.0.0.1:9100", "127.0.0.2:91001"},
+						Labels:  map[string]string{"__meta__kubernetes_pod": "pod-1", "__meta_consul_dc": "dc-2"},
+					},
+				},
+			},
+			want: [][]prompbmarshal.Label{
+				discoveryutils.GetSortedLabels(map[string]string{
+					"__address__":            "127.0.0.1:9100",
+					"__meta__kubernetes_pod": "pod-1",
+					"__meta_consul_dc":       "dc-2",
+				}),
+				discoveryutils.GetSortedLabels(map[string]string{
+					"__address__":            "127.0.0.2:91001",
+					"__meta__kubernetes_pod": "pod-1",
+					"__meta_consul_dc":       "dc-2",
+				}),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := addHTTPTargetLabels(tt.args.src)
+			var sortedLabelss [][]prompbmarshal.Label
+			for _, labels := range got {
+				sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
+			}
+			if !reflect.DeepEqual(sortedLabelss, tt.want) {
+				t.Errorf("addHTTPTargetLabels() \ngot  \n%v\n, \nwant \n%v\n", sortedLabelss, tt.want)
+			}
+		})
+	}
+}

lib/promscrape/discoveryutils/client.go

@@ -154,8 +154,19 @@ func (c *Client) Addr() string {
 	return c.hc.Addr
 }
 
+// GetAPIResponseWithReqParams returns response for given absolute path with optional callback for request.
+// modifyRequestParams should never reference data from request.
+func (c *Client) GetAPIResponseWithReqParams(path string, modifyRequestParams func(request *fasthttp.Request)) ([]byte, error) {
+	return c.getAPIResponse(path, modifyRequestParams)
+}
+
 // GetAPIResponse returns response for the given absolute path.
 func (c *Client) GetAPIResponse(path string) ([]byte, error) {
+	return c.getAPIResponse(path, nil)
+}
+
+// GetAPIResponse returns response for the given absolute path with optional callback for request.
+func (c *Client) getAPIResponse(path string, modifyRequest func(request *fasthttp.Request)) ([]byte, error) {
 	// Limit the number of concurrent API requests.
 	concurrencyLimitChOnce.Do(concurrencyLimitChInit)
 	t := timerpool.Get(*maxWaitTime)
@@ -168,17 +179,17 @@ func (c *Client) GetAPIResponse(path string) ([]byte, error) {
 			c.apiServer, *maxWaitTime, *maxConcurrency)
 	}
 	defer func() { <-concurrencyLimitCh }()
-	return c.getAPIResponseWithParamsAndClient(c.hc, path, nil)
+	return c.getAPIResponseWithParamsAndClient(c.hc, path, modifyRequest, nil)
 }
 
 // GetBlockingAPIResponse returns response for given absolute path with blocking client and optional callback for api response,
 // inspectResponse - should never reference data from response.
 func (c *Client) GetBlockingAPIResponse(path string, inspectResponse func(resp *fasthttp.Response)) ([]byte, error) {
-	return c.getAPIResponseWithParamsAndClient(c.blockingClient, path, inspectResponse)
+	return c.getAPIResponseWithParamsAndClient(c.blockingClient, path, nil, inspectResponse)
 }
 
-// getAPIResponseWithParamsAndClient returns response for the given absolute path with optional callback for response.
-func (c *Client) getAPIResponseWithParamsAndClient(client *fasthttp.HostClient, path string, inspectResponse func(resp *fasthttp.Response)) ([]byte, error) {
+// getAPIResponseWithParamsAndClient returns response for the given absolute path with optional callback for request and for response.
+func (c *Client) getAPIResponseWithParamsAndClient(client *fasthttp.HostClient, path string, modifyRequest func(req *fasthttp.Request), inspectResponse func(resp *fasthttp.Response)) ([]byte, error) {
 	requestURL := c.apiServer + path
 	var u fasthttp.URI
 	u.Update(requestURL)
@@ -196,6 +207,9 @@ func (c *Client) getAPIResponseWithParamsAndClient(client *fasthttp.HostClient,
 	if ah := c.getProxyAuthHeader(); ah != "" {
 		req.Header.Set("Proxy-Authorization", ah)
 	}
+	if modifyRequest != nil {
+		modifyRequest(&req)
+	}
 
 	var resp fasthttp.Response
 	deadline := time.Now().Add(client.ReadTimeout)

lib/promscrape/scraper.go

@@ -12,6 +12,7 @@ import (
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/http"
 	"github.com/VictoriaMetrics/metrics"
 )
 
@@ -115,6 +116,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
 	scs.add("gce_sd_configs", *gceSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getGCESDScrapeWork(swsPrev) })
 	scs.add("dockerswarm_sd_configs", *dockerswarmSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDockerSwarmSDScrapeWork(swsPrev) })
 	scs.add("digitalocean_sd_configs", *digitaloceanSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDigitalOceanDScrapeWork(swsPrev) })
+	scs.add("http_sd_configs", *http.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getHTTPDScrapeWork(swsPrev) })
 
 	var tickerCh <-chan time.Time
 	if *configCheckInterval > 0 {