diff --git a/README.md b/README.md index 806c5b605..fa2cb3691 100644 --- a/README.md +++ b/README.md @@ -344,6 +344,7 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la * [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) * [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) * [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) +* [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) Other `*_sd_config` types will be supported in the future. @@ -1746,6 +1747,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s) -promscrape.gceSDCheckInterval duration Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s) + -promscrape.httpSDCheckInterval duration + Interval for checking for changes in http service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details (default 1m0s) -promscrape.kubernetes.apiServerTimeout duration How frequently to reload the full state from Kuberntes API server (default 30m0s) -promscrape.kubernetesSDCheckInterval duration diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 641a7ab9a..2dc8b42b3 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -179,6 +179,8 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details. * `digitalocean_sd_configs` is for scraping targerts registered in [DigitalOcean](https://www.digitalocean.com/) See [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) for details. +* `http_sd_configs` is for scraping targerts registered in http service discovery. + See [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) for details. Please file feature requests to [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`. @@ -653,6 +655,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s) -promscrape.gceSDCheckInterval duration Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s) + -promscrape.httpSDCheckInterval duration + Interval for checking for changes in http service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details (default 1m0s) -promscrape.kubernetes.apiServerTimeout duration How frequently to reload the full state from Kuberntes API server (default 30m0s) -promscrape.kubernetesSDCheckInterval duration diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 98fbc296b..36e3b8861 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -348,6 +348,7 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la * [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) * [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) * [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) +* [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) Other `*_sd_config` types will be supported in the future. @@ -1750,6 +1751,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s) -promscrape.gceSDCheckInterval duration Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s) + -promscrape.httpSDCheckInterval duration + Interval for checking for changes in http service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details (default 1m0s) -promscrape.kubernetes.apiServerTimeout duration How frequently to reload the full state from Kuberntes API server (default 30m0s) -promscrape.kubernetesSDCheckInterval duration diff --git a/docs/vmagent.md b/docs/vmagent.md index 1f1a29463..5b9e949b5 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -183,6 +183,8 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details. * `digitalocean_sd_configs` is for scraping targerts registered in [DigitalOcean](https://www.digitalocean.com/) See [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) for details. +* `http_sd_configs` is for scraping targerts registered in http service discovery. + See [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) for details. Please file feature requests to [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`. @@ -657,6 +659,8 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . Interval for checking for changes in 'file_sd_config'. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details (default 30s) -promscrape.gceSDCheckInterval duration Interval for checking for changes in gce. This works only if gce_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details (default 1m0s) + -promscrape.httpSDCheckInterval duration + Interval for checking for changes in http service discovery. This works only if http_sd_configs is configured in '-promscrape.config' file. See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details (default 1m0s) -promscrape.kubernetes.apiServerTimeout duration How frequently to reload the full state from Kuberntes API server (default 30m0s) -promscrape.kubernetesSDCheckInterval duration diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 02af9abc8..21b663719 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -25,6 +25,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ec2" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/eureka" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/http" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/openstack" "github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy" @@ -129,6 +130,7 @@ type ScrapeConfig struct { EC2SDConfigs []ec2.SDConfig `yaml:"ec2_sd_configs,omitempty"` GCESDConfigs []gce.SDConfig `yaml:"gce_sd_configs,omitempty"` DigitaloceanSDConfigs []digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"` + HTTPSDConfigs []http.SDConfig `yaml:"http_sd_configs,omitempty"` // These options are supported only by lib/promscrape. RelabelDebug bool `yaml:"relabel_debug,omitempty"` @@ -529,6 +531,34 @@ func (cfg *Config) getDigitalOceanDScrapeWork(prev []*ScrapeWork) []*ScrapeWork return dst } +// getHTTPDScrapeWork returns `http_sd_configs` ScrapeWork from cfg. +func (cfg *Config) getHTTPDScrapeWork(prev []*ScrapeWork) []*ScrapeWork { + swsPrevByJob := getSWSByJob(prev) + dst := make([]*ScrapeWork, 0, len(prev)) + for i := range cfg.ScrapeConfigs { + sc := &cfg.ScrapeConfigs[i] + dstLen := len(dst) + ok := true + for j := range sc.HTTPSDConfigs { + sdc := &sc.HTTPSDConfigs[j] + var okLocal bool + dst, okLocal = appendSDScrapeWork(dst, sdc, cfg.baseDir, sc.swc, "http_sd_config") + if ok { + ok = okLocal + } + } + if ok { + continue + } + swsPrev := swsPrevByJob[sc.swc.jobName] + if len(swsPrev) > 0 { + logger.Errorf("there were errors when discovering http targets for job %q, so preserving the previous targets", sc.swc.jobName) + dst = append(dst[:dstLen], swsPrev...) + } + } + return dst +} + // getFileSDScrapeWork returns `file_sd_configs` ScrapeWork from cfg. func (cfg *Config) getFileSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork { // Create a map for the previous scrape work. diff --git a/lib/promscrape/discovery/http/api.go b/lib/promscrape/discovery/http/api.go new file mode 100644 index 000000000..e9bf7b08d --- /dev/null +++ b/lib/promscrape/discovery/http/api.go @@ -0,0 +1,78 @@ +package http + +import ( + "encoding/json" + "fmt" + "net/url" + "strconv" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" + "github.com/VictoriaMetrics/fasthttp" +) + +var configMap = discoveryutils.NewConfigMap() + +type apiConfig struct { + client *discoveryutils.Client + path string +} + +// httpGroupTarget respresent prometheus GroupTarget +// https://prometheus.io/docs/prometheus/latest/http_sd/ +type httpGroupTarget struct { + Targets []string `json:"targets"` + Labels map[string]string `json:"labels"` +} + +func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { + ac, err := sdc.HTTPClientConfig.NewConfig(baseDir) + if err != nil { + return nil, fmt.Errorf("cannot parse auth config: %w", err) + } + parsedURL, err := url.Parse(sdc.URL) + if err != nil { + return nil, fmt.Errorf("cannot parse http_sd URL: %w", err) + } + apiServer := fmt.Sprintf("%s://%s", parsedURL.Scheme, parsedURL.Host) + + proxyAC, err := sdc.ProxyClientConfig.NewConfig(baseDir) + if err != nil { + return nil, fmt.Errorf("cannot parse proxy auth config: %w", err) + } + client, err := discoveryutils.NewClient(apiServer, ac, sdc.ProxyURL, proxyAC) + if err != nil { + return nil, fmt.Errorf("cannot create HTTP client for %q: %w", apiServer, err) + } + cfg := &apiConfig{ + client: client, + path: parsedURL.RequestURI(), + } + return cfg, nil +} + +func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { + v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) }) + if err != nil { + return nil, err + } + return v.(*apiConfig), nil +} + +func getHTTPTargets(cfg *apiConfig) ([]httpGroupTarget, error) { + data, err := cfg.client.GetAPIResponseWithReqParams(cfg.path, func(request *fasthttp.Request) { + request.Header.Set("X-Prometheus-Refresh-Interval-Seconds", strconv.FormatFloat(SDCheckInterval.Seconds(), 'f', 0, 64)) + request.Header.Set("Accept", "application/json") + }) + if err != nil { + return nil, fmt.Errorf("cannot read http_sd api response: %w", err) + } + return parseAPIResponse(data, cfg.path) +} + +func parseAPIResponse(data []byte, path string) ([]httpGroupTarget, error) { + var r []httpGroupTarget + if err := json.Unmarshal(data, &r); err != nil { + return nil, fmt.Errorf("cannot parse http_sd api response path: %s, err: %w", path, err) + } + return r, nil +} diff --git a/lib/promscrape/discovery/http/api_test.go b/lib/promscrape/discovery/http/api_test.go new file mode 100644 index 000000000..22c66a717 --- /dev/null +++ b/lib/promscrape/discovery/http/api_test.go @@ -0,0 +1,49 @@ +package http + +import ( + "reflect" + "testing" +) + +func Test_parseAPIResponse(t *testing.T) { + type args struct { + data []byte + path string + } + tests := []struct { + name string + args args + want []httpGroupTarget + wantErr bool + }{ + + { + name: "parse ok", + args: args{ + path: "/ok", + data: []byte(`[ + {"targets": ["http://target-1:9100","http://target-2:9150"], + "labels": {"label-1":"value-1"} } + ]`), + }, + want: []httpGroupTarget{ + { + Labels: map[string]string{"label-1": "value-1"}, + Targets: []string{"http://target-1:9100", "http://target-2:9150"}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseAPIResponse(tt.args.data, tt.args.path) + if (err != nil) != tt.wantErr { + t.Errorf("parseAPIResponse() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseAPIResponse() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/lib/promscrape/discovery/http/http.go b/lib/promscrape/discovery/http/http.go new file mode 100644 index 000000000..0b5dff8cd --- /dev/null +++ b/lib/promscrape/discovery/http/http.go @@ -0,0 +1,56 @@ +package http + +import ( + "flag" + "fmt" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/proxy" +) + +// SDCheckInterval defines interval for targets refresh. +var SDCheckInterval = flag.Duration("promscrape.httpSDCheckInterval", time.Minute, "Interval for checking for changes in http endpoint service discovery. "+ + "This works only if http_sd_configs is configured in '-promscrape.config' file. "+ + "See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config for details") + +// SDConfig represents service discovery config for http. +// +// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config +type SDConfig struct { + URL string `yaml:"url"` + HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"` + ProxyURL proxy.URL `yaml:"proxy_url,omitempty"` + ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"` +} + +// GetLabels returns http service discovery labels according to sdc. +func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) { + cfg, err := getAPIConfig(sdc, baseDir) + if err != nil { + return nil, fmt.Errorf("cannot get API config: %w", err) + } + + hts, err := getHTTPTargets(cfg) + if err != nil { + return nil, err + } + + return addHTTPTargetLabels(hts), nil +} + +func addHTTPTargetLabels(src []httpGroupTarget) []map[string]string { + ms := make([]map[string]string, 0, len(src)) + for _, targetGroup := range src { + labels := targetGroup.Labels + for _, target := range targetGroup.Targets { + m := make(map[string]string, len(labels)) + for k, v := range labels { + m[k] = v + } + m["__address__"] = target + ms = append(ms, m) + } + } + return ms +} diff --git a/lib/promscrape/discovery/http/http_test.go b/lib/promscrape/discovery/http/http_test.go new file mode 100644 index 000000000..c287866cc --- /dev/null +++ b/lib/promscrape/discovery/http/http_test.go @@ -0,0 +1,56 @@ +package http + +import ( + "reflect" + "testing" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" +) + +func Test_addHTTPTargetLabels(t *testing.T) { + type args struct { + src []httpGroupTarget + } + tests := []struct { + name string + args args + want [][]prompbmarshal.Label + }{ + { + name: "add ok", + args: args{ + src: []httpGroupTarget{ + { + Targets: []string{"127.0.0.1:9100", "127.0.0.2:91001"}, + Labels: map[string]string{"__meta__kubernetes_pod": "pod-1", "__meta_consul_dc": "dc-2"}, + }, + }, + }, + want: [][]prompbmarshal.Label{ + discoveryutils.GetSortedLabels(map[string]string{ + "__address__": "127.0.0.1:9100", + "__meta__kubernetes_pod": "pod-1", + "__meta_consul_dc": "dc-2", + }), + discoveryutils.GetSortedLabels(map[string]string{ + "__address__": "127.0.0.2:91001", + "__meta__kubernetes_pod": "pod-1", + "__meta_consul_dc": "dc-2", + }), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := addHTTPTargetLabels(tt.args.src) + var sortedLabelss [][]prompbmarshal.Label + for _, labels := range got { + sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels)) + } + if !reflect.DeepEqual(sortedLabelss, tt.want) { + t.Errorf("addHTTPTargetLabels() \ngot \n%v\n, \nwant \n%v\n", sortedLabelss, tt.want) + } + }) + } +} diff --git a/lib/promscrape/discoveryutils/client.go b/lib/promscrape/discoveryutils/client.go index 3b417503e..92b2b4b9b 100644 --- a/lib/promscrape/discoveryutils/client.go +++ b/lib/promscrape/discoveryutils/client.go @@ -154,8 +154,19 @@ func (c *Client) Addr() string { return c.hc.Addr } +// GetAPIResponseWithReqParams returns response for given absolute path with optional callback for request. +// modifyRequestParams should never reference data from request. +func (c *Client) GetAPIResponseWithReqParams(path string, modifyRequestParams func(request *fasthttp.Request)) ([]byte, error) { + return c.getAPIResponse(path, modifyRequestParams) +} + // GetAPIResponse returns response for the given absolute path. func (c *Client) GetAPIResponse(path string) ([]byte, error) { + return c.getAPIResponse(path, nil) +} + +// GetAPIResponse returns response for the given absolute path with optional callback for request. +func (c *Client) getAPIResponse(path string, modifyRequest func(request *fasthttp.Request)) ([]byte, error) { // Limit the number of concurrent API requests. concurrencyLimitChOnce.Do(concurrencyLimitChInit) t := timerpool.Get(*maxWaitTime) @@ -168,17 +179,17 @@ func (c *Client) GetAPIResponse(path string) ([]byte, error) { c.apiServer, *maxWaitTime, *maxConcurrency) } defer func() { <-concurrencyLimitCh }() - return c.getAPIResponseWithParamsAndClient(c.hc, path, nil) + return c.getAPIResponseWithParamsAndClient(c.hc, path, modifyRequest, nil) } // GetBlockingAPIResponse returns response for given absolute path with blocking client and optional callback for api response, // inspectResponse - should never reference data from response. func (c *Client) GetBlockingAPIResponse(path string, inspectResponse func(resp *fasthttp.Response)) ([]byte, error) { - return c.getAPIResponseWithParamsAndClient(c.blockingClient, path, inspectResponse) + return c.getAPIResponseWithParamsAndClient(c.blockingClient, path, nil, inspectResponse) } -// getAPIResponseWithParamsAndClient returns response for the given absolute path with optional callback for response. -func (c *Client) getAPIResponseWithParamsAndClient(client *fasthttp.HostClient, path string, inspectResponse func(resp *fasthttp.Response)) ([]byte, error) { +// getAPIResponseWithParamsAndClient returns response for the given absolute path with optional callback for request and for response. +func (c *Client) getAPIResponseWithParamsAndClient(client *fasthttp.HostClient, path string, modifyRequest func(req *fasthttp.Request), inspectResponse func(resp *fasthttp.Response)) ([]byte, error) { requestURL := c.apiServer + path var u fasthttp.URI u.Update(requestURL) @@ -196,6 +207,9 @@ func (c *Client) getAPIResponseWithParamsAndClient(client *fasthttp.HostClient, if ah := c.getProxyAuthHeader(); ah != "" { req.Header.Set("Proxy-Authorization", ah) } + if modifyRequest != nil { + modifyRequest(&req) + } var resp fasthttp.Response deadline := time.Now().Add(client.ReadTimeout) diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go index cbb225ef0..ecd2ecf05 100644 --- a/lib/promscrape/scraper.go +++ b/lib/promscrape/scraper.go @@ -12,6 +12,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/http" "github.com/VictoriaMetrics/metrics" ) @@ -115,6 +116,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest) scs.add("gce_sd_configs", *gceSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getGCESDScrapeWork(swsPrev) }) scs.add("dockerswarm_sd_configs", *dockerswarmSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDockerSwarmSDScrapeWork(swsPrev) }) scs.add("digitalocean_sd_configs", *digitaloceanSDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getDigitalOceanDScrapeWork(swsPrev) }) + scs.add("http_sd_configs", *http.SDCheckInterval, func(cfg *Config, swsPrev []*ScrapeWork) []*ScrapeWork { return cfg.getHTTPDScrapeWork(swsPrev) }) var tickerCh <-chan time.Time if *configCheckInterval > 0 {