From 5ee344824ffe6509a5caaa3ddcdb0ccbd7a1de0d Mon Sep 17 00:00:00 2001 From: Nikolay Date: Wed, 26 Apr 2023 19:16:27 +0200 Subject: [PATCH] lib/promscrape: adds filter for consul_sd_configs: (#4184) * lib/promscrape: adds filter for consul_sd_configs: it allows advanced filtering for consul service discovery requests https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4183 * typo fix * removes deprecation mentions since it's not relevant * Update docs/CHANGELOG.md Co-authored-by: Roman Khavronenko --------- Co-authored-by: Roman Khavronenko --- docs/CHANGELOG.md | 1 + docs/sd_configs.md | 9 +++++++++ lib/promscrape/discovery/consul/consul.go | 4 ++++ lib/promscrape/discovery/consul/watch.go | 8 ++++++++ 4 files changed, 22 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 9392054df4..e480a72de2 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -22,6 +22,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: [vmbackup](https://docs.victoriametrics.com/vmbackup.html): store backup creation and completion time in `backup_complete.ignore` file of backup contents. This is useful to determine point in time when backup was created and completed. * FEATURE: [vmbackupmanager](https://docs.victoriametrics.com/vmbackupmanager.html): add `created_at` field to the output of `/api/v1/backups` API and `vmbackupmanager backup list` command. See this [doc](https://docs.victoriametrics.com/vmbackupmanager.html#api-methods) for data format details. * FEATURE: deprecate `-bigMergeConcurrency` command-line flag, since improper configuration for this flag frequently led to uncontrolled growth of unmerged parts, which, in turn, could lead to queries slowdown and increased CPU usage. The concurrency for [background merges](https://docs.victoriametrics.com/#storage) can be controlled via `-smallMergeConcurrency` command-line flag, though it isn't recommended to do in general case. +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): support new filtering options `filter` and `node_filter` for [consul service discovery](https://docs.victoriametrics.com/sd_configs.html#consul_sd_configs). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4183) for details. * FEATURE: introduce `-http.maxConcurrentRequests` command-line flag to protect VM components from resource exhaustion during unexpected spikes of HTTP requests. By default, the new flag's value is set to 0 which means no limits are applied. * FEATURE: [vmctl](https://docs.victoriametrics.com/vmctl.html): add support for the different time formats for `--vm-native-filter-time-start` and `--vm-native-filter-time-end` flags if the native binary protocol is used for migration. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/4091). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): integrate WITH template playground. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3811). diff --git a/docs/sd_configs.md b/docs/sd_configs.md index b923bc9102..a535169d19 100644 --- a/docs/sd_configs.md +++ b/docs/sd_configs.md @@ -141,9 +141,11 @@ scrape_configs: # tags is an optional list of tags used to filter nodes for a given service. # Services must contain all tags in the list. + # Deprecated: use filter instead with ServiceTags selector. # tags: ["...", "..."] # node_meta is an optional node metadata key/value pairs to filter nodes for a given service. + # Deprecated: use filter instead with NodeMeta selector. # node_meta: # "...": "..." @@ -152,6 +154,13 @@ scrape_configs: # Individual tags are also available via __meta_consul_tag_ labels - see below. # tag_separator: "..." + # filter is optional filter for service nodes discovery request. + # Replaces tags and node_metadata options. + # consul supports it since 1.14 version + # list of supported filters https://developer.hashicorp.com/consul/api-docs/catalog#filtering-1 + # syntax examples https://developer.hashicorp.com/consul/api-docs/features/filtering + # filter: "..." + # allow_stale is an optional config, which allows stale Consul results. # See https://www.consul.io/api/features/consistency.html # Reduce load on Consul if set to true. By default is is set to true. diff --git a/lib/promscrape/discovery/consul/consul.go b/lib/promscrape/discovery/consul/consul.go index fb755c82df..77ac64f183 100644 --- a/lib/promscrape/discovery/consul/consul.go +++ b/lib/promscrape/discovery/consul/consul.go @@ -34,6 +34,10 @@ type SDConfig struct { NodeMeta map[string]string `yaml:"node_meta,omitempty"` TagSeparator *string `yaml:"tag_separator,omitempty"` AllowStale *bool `yaml:"allow_stale,omitempty"` + // See https://developer.hashicorp.com/consul/api-docs/features/filtering + // list of supported filters https://developer.hashicorp.com/consul/api-docs/catalog#filtering-1 + Filter string `yaml:"filter,omitempty"` + // RefreshInterval time.Duration `yaml:"refresh_interval"` // refresh_interval is obtained from `-promscrape.consulSDCheckInterval` command-line option. } diff --git a/lib/promscrape/discovery/consul/watch.go b/lib/promscrape/discovery/consul/watch.go index 6b0ae267cc..47b7aaa6ad 100644 --- a/lib/promscrape/discovery/consul/watch.go +++ b/lib/promscrape/discovery/consul/watch.go @@ -62,10 +62,18 @@ func newConsulWatcher(client *discoveryutils.Client, sdc *SDConfig, datacenter, for k, v := range sdc.NodeMeta { baseQueryArgs += "&node-meta=" + url.QueryEscape(k+":"+v) } + serviceNodesQueryArgs := baseQueryArgs for _, tag := range sdc.Tags { serviceNodesQueryArgs += "&tag=" + url.QueryEscape(tag) } + + // filter could be added only for baseQuery requests for /v1/catalog/services + // serviceNodesQueryArgs doesn't support it + if len(sdc.Filter) > 0 { + baseQueryArgs += "&filter=" + url.QueryEscape(sdc.Filter) + } + cw := &consulWatcher{ client: client, serviceNamesQueryArgs: baseQueryArgs,