diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 087c46e52e..fbb7e35149 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -16,6 +16,7 @@ The following tip changes can be tested by building VictoriaMetrics components f ## tip * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for aggregation of incoming [samples](https://docs.victoriametrics.com/keyConcepts.html#raw-samples) by time and by labels. See [these docs](https://docs.victoriametrics.com/stream-aggregation.html) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3460). +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for Prometheus-compatible target discovery for [HashiCorp Nomad](https://www.nomadproject.io/) services via [nomad_sd_configs](https://docs.victoriametrics.com/sd_configs.html#nomad_sd_configs). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3367). Thanks to @mr-karan for [the implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3549). * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): automatically pre-fetch `metric_relabel_configs` and the target labels when clicking on the `debug metrics relabeling` link at the `http://vmagent:8429/targets` page at the particular target. See [these docs](https://docs.victoriametrics.com/vmagent.html#relabel-debug). * FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add ability to explore metrics exported by a particular `job` / `instance`. See [these docs](https://docs.victoriametrics.com/#metrics-explorer) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3386). * FEATURE: allow passing partial `RFC3339` date/time to `time`, `start` and `end` query args at [querying APIs](https://docs.victoriametrics.com/#prometheus-querying-api-usage) and [export APIs](https://docs.victoriametrics.com/#how-to-export-time-series). For example, `2022` is equivalent to `2022-01-01T00:00:00Z`, while `2022-01-30T14` is equivalent to `2022-01-30T14:00:00Z`. See [these docs](https://docs.victoriametrics.com/#timestamp-formats). @@ -31,7 +32,6 @@ The following tip changes can be tested by building VictoriaMetrics components f * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): fix a panic during target discovery when `vmagent` runs with `-promscrape.dropOriginalLabels` command-line flag. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3580). The bug has been introduced in [v1.85.0](https://docs.victoriametrics.com/CHANGELOG.html#v1850). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): [dockerswarm_sd_configs](https://docs.victoriametrics.com/sd_configs.html#dockerswarm_sd_configs): properly encode `filters` field. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3579). -* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add support for Prometheus compatible service discovery for Nomad services. These targets are discovered via [nomad_sd_configs](https://docs.victoriametrics.com/sd_configs.html#nomad_sd_configs). ## [v1.85.3](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.85.3) diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index 62d344d82b..5006c924ee 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -2336,6 +2336,10 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li -promscrape.minResponseSizeForStreamParse size The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 1000000) + -promscrape.nomad.waitTime duration + Wait time used by Nomad service discovery. Default value is used if not set + -promscrape.nomadSDCheckInterval duration + Interval for checking for changes in Nomad. This works only if nomad_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#nomad_sd_configs for details (default 30s) -promscrape.noStaleMarkers Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series -promscrape.openstackSDCheckInterval duration diff --git a/docs/sd_configs.md b/docs/sd_configs.md index 07e307e501..2cfdc1bd1e 100644 --- a/docs/sd_configs.md +++ b/docs/sd_configs.md @@ -20,7 +20,7 @@ sort: 24 * `gce_sd_configs` is for discovering and scraping [Google Compute Engine](https://cloud.google.com/compute) targets. See [these docs](#gce_sd_configs). * `http_sd_configs` is for discovering and scraping targerts provided by external http-based service discovery. See [these docs](#http_sd_configs). * `kubernetes_sd_configs` is for discovering and scraping [Kubernetes](https://kubernetes.io/) targets. See [these docs](#kubernetes_sd_configs). -* `nomad_sd_configs` is for discovering and scraping targets registered in [Nomad](https://www.nomadproject.io/). See [these docs](#nomad_sd_configs). +* `nomad_sd_configs` is for discovering and scraping targets registered in [HashiCorp Nomad](https://www.nomadproject.io/). See [these docs](#nomad_sd_configs). * `openstack_sd_configs` is for discovering and scraping OpenStack targets. See [these docs](#openstack_sd_configs). * `static_configs` is for scraping statically defined targets. See [these docs](#static_configs). * `yandexcloud_sd_configs` is for discoverying and scraping [Yandex Cloud](https://cloud.yandex.com/en/) targets. See [these docs](#yandexcloud_sd_configs). @@ -139,7 +139,7 @@ scrape_configs: # node_meta: # "...": "..." - # tag_separate is an optional string by which Consul tags are joined into the __meta_consul_tags label. + # tag_separator is an optional string by which Consul tags are joined into the __meta_consul_tags label. # By default "," is used as a tag separator. # Individual tags are also available via __meta_consul_tag_ labels - see below. # tag_separator: "..." @@ -170,7 +170,7 @@ The following meta labels are available on discovered targets during [relabeling * `__meta_consul_tagged_address_`: each node tagged address key value of the target * `__meta_consul_tag_`: the value for the given tag of the target * `__meta_consul_tagpresent_`: "true" for every tag of the target -* `__meta_consul_tags`: the list of tags of the target joined by the tag_separator +* `__meta_consul_tags`: the list of tags of the target joined by the `tag_separator` ## digitalocean_sd_configs @@ -872,7 +872,7 @@ One of the following `role` types can be configured to discover targets: ## nomad_sd_configs -Nomad SD configuration allows retrieving scrape targets from [Nomad Services](https://www.hashicorp.com/blog/nomad-service-discovery). +Nomad SD configuration allows retrieving scrape targets from [HashiCorp Nomad Services](https://www.hashicorp.com/blog/nomad-service-discovery). Configuration example: @@ -910,14 +910,14 @@ scrape_configs: # Services must contain all tags in the list. # tags: ["...", "..."] - # tag_separate is an optional string by which Nomad tags are joined into the __meta_nomad_tags label. + # tag_separator is an optional string by which Nomad tags are joined into the __meta_nomad_tags label. # By default "," is used as a tag separator. # Individual tags are also available via __meta_nomad_tag_ labels - see below. # tag_separator: "..." # allow_stale is an optional config, which allows stale Nomad results. # See https://developer.hashicorp.com/nomad/api-docs#consistency-modes - # Reduce load on Nomad if set to true. By default is is set to true. + # Reduces load on Nomad if set to true. By default is is set to true. # allow_stale: ... # Additional HTTP API client options can be specified here. @@ -932,11 +932,13 @@ The following meta labels are available on discovered targets during [relabeling * `__meta_nomad_node_id`: the node ID defined for the target * `__meta_nomad_service`: the name of the service the target belongs to * `__meta_nomad_service_address`: the service address of the target -* `__meta_nomad_service_id`: the service ID of the target +* `__meta_nomad_service_alloc_id`: the AllocID of the target service +* `__meta_nomad_service_id`: the ID of the target service +* `__meta_nomad_service_job_id`: the JobID of the target service * `__meta_nomad_service_port`: the service port of the target * `__meta_nomad_tag_`: the value for the given tag of the target * `__meta_nomad_tagpresent_`: "true" for every tag of the target -* `__meta_nomad_tags`: the list of tags of the target joined by the tag_separator +* `__meta_nomad_tags`: the list of tags of the target joined by the `tag_separator` ## openstack_sd_configs diff --git a/docs/vmagent.md b/docs/vmagent.md index c25ba3125c..34b1f64fd8 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -1342,6 +1342,10 @@ See the docs at https://docs.victoriametrics.com/vmagent.html . -promscrape.minResponseSizeForStreamParse size The minimum target response size for automatic switching to stream parsing mode, which can reduce memory usage. See https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB (default 1000000) + -promscrape.nomad.waitTime duration + Wait time used by Nomad service discovery. Default value is used if not set + -promscrape.nomadSDCheckInterval duration + Interval for checking for changes in Nomad. This works only if nomad_sd_configs is configured in '-promscrape.config' file. See https://docs.victoriametrics.com/sd_configs.html#nomad_sd_configs for details (default 30s) -promscrape.noStaleMarkers Whether to disable sending Prometheus stale markers for metrics when scrape target disappears. This option may reduce memory usage if stale markers aren't needed for your setup. This option also disables populating the scrape_series_added metric. See https://prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series -promscrape.openstackSDCheckInterval duration diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 000f6e3716..b4d5317f7d 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -831,7 +831,7 @@ func (cfg *Config) getNomadSDScrapeWork(prev []*ScrapeWork) []*ScrapeWork { } swsPrev := swsPrevByJob[sc.swc.jobName] if len(swsPrev) > 0 { - logger.Errorf("there were errors when discovering nomad targets for job %q, so preserving the previous targets", sc.swc.jobName) + logger.Errorf("there were errors when discovering nomad_sd_config targets for job %q, so preserving the previous targets", sc.swc.jobName) dst = append(dst[:dstLen], swsPrev...) } } diff --git a/lib/promscrape/discovery/consul/api.go b/lib/promscrape/discovery/consul/api.go index 249a536ae6..9da5893ca0 100644 --- a/lib/promscrape/discovery/consul/api.go +++ b/lib/promscrape/discovery/consul/api.go @@ -86,7 +86,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { } dc, err := getDatacenter(client, sdc.Datacenter) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot obtain consul datacenter: %w", err) } namespace := sdc.Namespace diff --git a/lib/promscrape/discovery/nomad/api.go b/lib/promscrape/discovery/nomad/api.go index e09b63352e..b7a016512c 100644 --- a/lib/promscrape/discovery/nomad/api.go +++ b/lib/promscrape/discovery/nomad/api.go @@ -38,25 +38,13 @@ func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { hcc := sdc.HTTPClientConfig - token, err := getToken(sdc.Token) - if err != nil { - return nil, err - } + token := getToken(sdc.Token) if token != "" { if hcc.BearerToken != nil { return nil, fmt.Errorf("cannot set both token and bearer_token configs") } hcc.BearerToken = promauth.NewSecret(token) } - if len(sdc.Username) > 0 { - if hcc.BasicAuth != nil { - return nil, fmt.Errorf("cannot set both username and basic_auth configs") - } - hcc.BasicAuth = &promauth.BasicAuthConfig{ - Username: sdc.Username, - Password: sdc.Password, - } - } ac, err := hcc.NewConfig(baseDir) if err != nil { return nil, fmt.Errorf("cannot parse auth config: %w", err) @@ -86,7 +74,7 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { } dc, err := getDatacenter(client, sdc.Datacenter) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot obtain Nomad datacenter: %w", err) } namespace := sdc.Namespace @@ -103,13 +91,13 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { return cfg, nil } -func getToken(token *promauth.Secret) (string, error) { +func getToken(token *promauth.Secret) string { if token != nil { - return token.String(), nil + return token.String() } t := os.Getenv("NOMAD_TOKEN") // Allow empty token - it should work if ACL is disabled in Nomad. - return t, nil + return t } func getDatacenter(client *discoveryutils.Client, dc string) (string, error) { diff --git a/lib/promscrape/discovery/nomad/nomad.go b/lib/promscrape/discovery/nomad/nomad.go index 7b1c7e2271..799fc80ede 100644 --- a/lib/promscrape/discovery/nomad/nomad.go +++ b/lib/promscrape/discovery/nomad/nomad.go @@ -13,22 +13,20 @@ import ( // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#nomad_sd_config type SDConfig struct { Server string `yaml:"server,omitempty"` - Token *promauth.Secret `yaml:"token"` - Datacenter string `yaml:"datacenter"` + Token *promauth.Secret `yaml:"token,omitempty"` + Datacenter string `yaml:"datacenter,omitempty"` Namespace string `yaml:"namespace,omitempty"` // RefreshInterval time.Duration `yaml:"refresh_interval"` // refresh_interval is obtained from `-promscrape.nomadSDCheckInterval` command-line option. - Region string `yaml:"region,omitempty"` - Scheme string `yaml:"scheme,omitempty"` - Username string `yaml:"username"` - Password *promauth.Secret `yaml:"password"` + Scheme string `yaml:"scheme,omitempty"` + Services []string `yaml:"services,omitempty"` + Tags []string `yaml:"tags,omitempty"` + TagSeparator *string `yaml:"tag_separator,omitempty"` + AllowStale *bool `yaml:"allow_stale,omitempty"` + HTTPClientConfig promauth.HTTPClientConfig `yaml:",inline"` ProxyURL *proxy.URL `yaml:"proxy_url,omitempty"` ProxyClientConfig promauth.ProxyClientConfig `yaml:",inline"` - Services []string `yaml:"services,omitempty"` - Tags []string `yaml:"tags,omitempty"` - AllowStale *bool `yaml:"allow_stale,omitempty"` - TagSeparator *string `yaml:"tag_separator,omitempty"` } // GetLabels returns Nomad labels according to sdc. diff --git a/lib/promscrape/discovery/nomad/service.go b/lib/promscrape/discovery/nomad/service.go index f19e7ec36c..41ce4932ec 100644 --- a/lib/promscrape/discovery/nomad/service.go +++ b/lib/promscrape/discovery/nomad/service.go @@ -57,13 +57,15 @@ func (svc *Service) appendTargetLabels(ms []*promutils.Labels, tagSeparator stri addr := discoveryutils.JoinHostPort(svc.Address, svc.Port) m := promutils.NewLabels(16) m.Add("__address__", addr) - m.Add("__meta_nomad_dc", svc.Datacenter) m.Add("__meta_nomad_address", svc.Address) + m.Add("__meta_nomad_dc", svc.Datacenter) m.Add("__meta_nomad_namespace", svc.Namespace) m.Add("__meta_nomad_node_id", svc.NodeID) m.Add("__meta_nomad_service", svc.ServiceName) m.Add("__meta_nomad_service_address", svc.Address) + m.Add("__meta_nomad_service_alloc_id", svc.AllocID) m.Add("__meta_nomad_service_id", svc.ID) + m.Add("__meta_nomad_service_job_id", svc.JobID) m.Add("__meta_nomad_service_port", strconv.Itoa(svc.Port)) // We surround the separated list with the separator as well. This way regular expressions // in relabeling rules don't have to consider tag positions. diff --git a/lib/promscrape/discovery/nomad/service_test.go b/lib/promscrape/discovery/nomad/service_test.go index 3a52c43b97..52debd4797 100644 --- a/lib/promscrape/discovery/nomad/service_test.go +++ b/lib/promscrape/discovery/nomad/service_test.go @@ -66,7 +66,9 @@ func TestParseServiceNodesSuccess(t *testing.T) { "__meta_nomad_namespace": "default", "__meta_nomad_service": "doggo-web", "__meta_nomad_service_address": "192.168.29.76", + "__meta_nomad_service_alloc_id": "1a321d90-79b5-681f-e6fa-8a43c8ec6b69", "__meta_nomad_service_id": "_nomad-task-1a321d90-79b5-681f-e6fa-8a43c8ec6b69-web-doggo-web-http", + "__meta_nomad_service_job_id": "doggo", "__meta_nomad_service_port": "23761", "__meta_nomad_tag_doggo": "", "__meta_nomad_tag_web": "",