From c2186279b780767f4dffde2c3cba6707850e4d3d Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 23 Nov 2020 16:26:52 +0200 Subject: [PATCH] lib/promscrape: add `filters` option to `dockerswarm_sd_config` like Prometheus did in v2.23.0 --- docs/CHANGELOG.md | 1 + lib/promscrape/discovery/dockerswarm/api.go | 43 ++++++++++++++++++- .../discovery/dockerswarm/api_test.go | 26 +++++++++++ .../discovery/dockerswarm/dockerswarm.go | 14 ++++-- .../discovery/dockerswarm/network.go | 2 +- lib/promscrape/discovery/dockerswarm/nodes.go | 2 +- .../discovery/dockerswarm/services.go | 2 +- lib/promscrape/discovery/dockerswarm/tasks.go | 2 +- 8 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 lib/promscrape/discovery/dockerswarm/api_test.go diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 44de82a629..4dccfff4db 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -11,6 +11,7 @@ * FEATURE: vminsert: export `vm_rpc_vmstorage_is_reachable` metric, which can be used for monitoring reachability of vmstorage nodes from vminsert nodes. * FEATURE: vmagent: add Netflix Eureka service discovery (aka [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config)). See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/851 +* FEATURE: add `filters` option to `dockerswarm_sd_config` like Prometheus did in v2.23.0 - see https://github.com/prometheus/prometheus/pull/8074 * FEATURE: add `-loggerWarnsPerSecondLimit` command-line flag for rate limiting of WARN messages in logs. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/905 * FEATURE: apply `loggerErrorsPerSecondLimit` and `-loggerWarnsPerSecondLimit` rate limit per caller. I.e. log messages are suppressed if the same caller logs the same message at the rate exceeding the given limit. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/905#issuecomment-729395855 diff --git a/lib/promscrape/discovery/dockerswarm/api.go b/lib/promscrape/discovery/dockerswarm/api.go index 25dffe2074..3853600a41 100644 --- a/lib/promscrape/discovery/dockerswarm/api.go +++ b/lib/promscrape/discovery/dockerswarm/api.go @@ -1,8 +1,12 @@ package dockerswarm import ( + "encoding/json" "fmt" + "net/url" + "strings" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" ) @@ -12,6 +16,9 @@ var configMap = discoveryutils.NewConfigMap() type apiConfig struct { client *discoveryutils.Client port int + + // filtersQueryArg contains escaped `filters` query arg to add to each request to Docker Swarm API. + filtersQueryArg string } func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { @@ -24,7 +31,8 @@ func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { cfg := &apiConfig{ - port: sdc.Port, + port: sdc.Port, + filtersQueryArg: getFiltersQueryArg(sdc.Filters), } ac, err := promauth.NewConfig(baseDir, sdc.BasicAuth, sdc.BearerToken, sdc.BearerTokenFile, sdc.TLSConfig) if err != nil { @@ -37,3 +45,36 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { cfg.client = client return cfg, nil } + +func (cfg *apiConfig) getAPIResponse(path string) ([]byte, error) { + if len(cfg.filtersQueryArg) > 0 { + separator := "?" + if strings.Contains(path, "?") { + separator = "&" + } + path += separator + "filters=" + cfg.filtersQueryArg + } + return cfg.client.GetAPIResponse(path) +} + +func getFiltersQueryArg(filters []Filter) string { + if len(filters) == 0 { + return "" + } + m := make(map[string]map[string]bool) + for _, f := range filters { + x := m[f.Name] + if x == nil { + x = make(map[string]bool) + m[f.Name] = x + } + for _, value := range f.Values { + x[value] = true + } + } + buf, err := json.Marshal(m) + if err != nil { + logger.Panicf("BUG: unexpected error in json.Marshal: %s", err) + } + return url.QueryEscape(string(buf)) +} diff --git a/lib/promscrape/discovery/dockerswarm/api_test.go b/lib/promscrape/discovery/dockerswarm/api_test.go new file mode 100644 index 0000000000..fe36a75a2c --- /dev/null +++ b/lib/promscrape/discovery/dockerswarm/api_test.go @@ -0,0 +1,26 @@ +package dockerswarm + +import ( + "testing" +) + +func TestGetFiltersQueryArg(t *testing.T) { + f := func(filters []Filter, queryArgExpected string) { + t.Helper() + queryArg := getFiltersQueryArg(filters) + if queryArg != queryArgExpected { + t.Fatalf("unexpected query arg; got %s; want %s", queryArg, queryArgExpected) + } + } + f(nil, "") + f([]Filter{ + { + Name: "name", + Values: []string{"foo", "bar"}, + }, + { + Name: "xxx", + Values: []string{"aa"}, + }, + }, "%7B%22name%22%3A%7B%22bar%22%3Atrue%2C%22foo%22%3Atrue%7D%2C%22xxx%22%3A%7B%22aa%22%3Atrue%7D%7D") +} diff --git a/lib/promscrape/discovery/dockerswarm/dockerswarm.go b/lib/promscrape/discovery/dockerswarm/dockerswarm.go index 3d49365a36..61b4b02f67 100644 --- a/lib/promscrape/discovery/dockerswarm/dockerswarm.go +++ b/lib/promscrape/discovery/dockerswarm/dockerswarm.go @@ -10,17 +10,25 @@ import ( // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config type SDConfig struct { - Host string `yaml:"host"` + Host string `yaml:"host"` + Role string `yaml:"role"` + Port int `yaml:"port,omitempty"` + Filters []Filter `yaml:"filters,omitempty"` + // TODO: add support for proxy_url TLSConfig *promauth.TLSConfig `yaml:"tls_config,omitempty"` - Role string `yaml:"role"` - Port int `yaml:"port,omitempty"` // refresh_interval is obtained from `-promscrape.dockerswarmSDCheckInterval` command-line option BasicAuth *promauth.BasicAuthConfig `yaml:"basic_auth,omitempty"` BearerToken string `yaml:"bearer_token,omitempty"` BearerTokenFile string `yaml:"bearer_token_file,omitempty"` } +// Filter is a filter, which can be passed to SDConfig. +type Filter struct { + Name string `yaml:"name"` + Values []string `yaml:"values"` +} + // GetLabels returns dockerswarm labels according to sdc. func GetLabels(sdc *SDConfig, baseDir string) ([]map[string]string, error) { cfg, err := getAPIConfig(sdc, baseDir) diff --git a/lib/promscrape/discovery/dockerswarm/network.go b/lib/promscrape/discovery/dockerswarm/network.go index bcf21dd0a4..27bb748cbb 100644 --- a/lib/promscrape/discovery/dockerswarm/network.go +++ b/lib/promscrape/discovery/dockerswarm/network.go @@ -27,7 +27,7 @@ func getNetworksLabelsByNetworkID(cfg *apiConfig) (map[string]map[string]string, } func getNetworks(cfg *apiConfig) ([]network, error) { - resp, err := cfg.client.GetAPIResponse("/networks") + resp, err := cfg.getAPIResponse("/networks") if err != nil { return nil, fmt.Errorf("cannot query dockerswarm api for networks: %w", err) } diff --git a/lib/promscrape/discovery/dockerswarm/nodes.go b/lib/promscrape/discovery/dockerswarm/nodes.go index d5eec44cc2..c6db715f2b 100644 --- a/lib/promscrape/discovery/dockerswarm/nodes.go +++ b/lib/promscrape/discovery/dockerswarm/nodes.go @@ -46,7 +46,7 @@ func getNodesLabels(cfg *apiConfig) ([]map[string]string, error) { } func getNodes(cfg *apiConfig) ([]node, error) { - resp, err := cfg.client.GetAPIResponse("/nodes") + resp, err := cfg.getAPIResponse("/nodes") if err != nil { return nil, fmt.Errorf("cannot query dockerswarm api for nodes: %w", err) } diff --git a/lib/promscrape/discovery/dockerswarm/services.go b/lib/promscrape/discovery/dockerswarm/services.go index 147c610cce..29e4f0dcb0 100644 --- a/lib/promscrape/discovery/dockerswarm/services.go +++ b/lib/promscrape/discovery/dockerswarm/services.go @@ -59,7 +59,7 @@ func getServicesLabels(cfg *apiConfig) ([]map[string]string, error) { } func getServices(cfg *apiConfig) ([]service, error) { - data, err := cfg.client.GetAPIResponse("/services") + data, err := cfg.getAPIResponse("/services") if err != nil { return nil, fmt.Errorf("cannot query dockerswarm api for services: %w", err) } diff --git a/lib/promscrape/discovery/dockerswarm/tasks.go b/lib/promscrape/discovery/dockerswarm/tasks.go index e8724c2cba..ecf69e4a55 100644 --- a/lib/promscrape/discovery/dockerswarm/tasks.go +++ b/lib/promscrape/discovery/dockerswarm/tasks.go @@ -58,7 +58,7 @@ func getTasksLabels(cfg *apiConfig) ([]map[string]string, error) { } func getTasks(cfg *apiConfig) ([]task, error) { - resp, err := cfg.client.GetAPIResponse("/tasks") + resp, err := cfg.getAPIResponse("/tasks") if err != nil { return nil, fmt.Errorf("cannot query dockerswarm api for tasks: %w", err) }