diff --git a/README.md b/README.md index 0cfef66c3..130c59825 100644 --- a/README.md +++ b/README.md @@ -310,27 +310,12 @@ Prometheus doesn't drop data during VictoriaMetrics restart. See [this article]( ## How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter) -VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets. Currently the following [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) types are supported: - -* [static_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) -* [file_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) -* [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) -* [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) -* [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) -* [azure_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#azure_sd_config) -* [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) -* [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) -* [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) -* [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config) -* [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) -* [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) -* [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) -* [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) - -File a [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need support for other `*_sd_config` types. +VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets. The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values. +See [the list of supported service discovery types for Prometheus scrape targets](https://docs.victoriametrics.com/sd_configs.html). + VictoriaMetrics also supports [importing data in Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format). See also [vmagent](https://docs.victoriametrics.com/vmagent.html), which can be used as drop-in replacement for Prometheus. diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 08901dbdb..c10117969 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -155,33 +155,8 @@ Use `-remoteWrite.*` command-line flag instead for configuring remote write sett The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders which are substituted by the corresponding `ENV_VAR` environment variable values. -The following scrape types in [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) section are supported: +See [the list of supported service discovery types for Prometheus scrape targets](https://docs.victoriametrics.com/sd_configs.html). -* `static_configs` is for scraping statically defined targets. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) for details. -* `file_sd_configs` is for scraping targets defined in external files (aka file-based service discovery). See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details. -* `kubernetes_sd_configs` is for discovering and scraping Kubernetes (K8S) targets. See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details. -* `ec2_sd_configs` is for discovering and scraping Amazon EC2 targets. See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details. `vmagent` doesn't support the `profile` config param yet. -* `gce_sd_configs` is for discovering and scraping Google Compute Engine (GCE) targets. See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details. `vmagent` provides the following additional functionality for `gce_sd_config`: - * if `project` arg is missing then `vmagent` uses the project for the instance where it runs; - * if `zone` arg is missing then `vmagent` uses the zone for the instance where it runs; - * if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project; - * `zone` may contain a list of zones, i.e. `zone: [us-east1-a, us-east1-b]`. -* `azure_sd_configs` - is for scraping the targets registered in Azure Cloud. - See [azure_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#azure_sd_config) for details. -* `consul_sd_configs` is for discovering and scraping targets registered in Consul. See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details. -* `dns_sd_configs` is for discovering and scraping targets from DNS records (SRV, A and AAAA). See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details. -* `openstack_sd_configs` is for discovering and scraping OpenStack targets. See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details. [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only. -* `docker_sd_configs` is for discovering and scraping Docker targets. See [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config) for details. -* `dockerswarm_sd_configs` is for discovering and scraping Docker Swarm targets. See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details. -* `eureka_sd_configs` is for discovering and scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka). See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details. -* `digitalocean_sd_configs` is for discovering and scraping targerts registered in [DigitalOcean](https://www.digitalocean.com/). See [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) for details. -* `http_sd_configs` is for discovering and scraping targerts provided by external http-based service discovery. See [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) for details. - -Note that `vmagent` doesn't support `refresh_interval` option for these scrape configs. Use the corresponding `-promscrape.*CheckInterval` -command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs` -entries to 60s. Run `vmagent -help` in order to see default values for the `-promscrape.*CheckInterval` flags. - -Please file feature requests to [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`. ## scrape_config enhancements diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d63010d51..a6875d3f0 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -21,6 +21,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * FEATURE: add ability to push internal metrics (e.g. metrics exposed at `/metrics` page) to the configured remote storage from all the VictoriaMetrics components. See [these docs](https://docs.victoriametrics.com/#push-metrics). * FEATURE: improve performance for heavy queries over big number of time series on systems with big number of CPU cores. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2896). Thanks to @zqyzyq for [the idea](https://github.com/VictoriaMetrics/VictoriaMetrics/commit/b596ac3745314fcc170a14e3ded062971cf7ced2). * FEATURE: improve performance for registering new time series in `indexdb` by up to 50%. Thanks to @ahfuzhang for [the issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2249). +* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add service discovery for [Yandex Cloud](https://cloud.yandex.com/en/). See [these docs](https://docs.victoriametrics.com/sd_configs.html#yandexcloud-sd-configs) and [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1386). * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): set `up` metric to `0` for partial scrapes in [stream parsing mode](https://docs.victoriametrics.com/vmagent.html#stream-parsing-mode). Previously the `up` metric was set to `1` when at least a single metric has been scraped before the error. This aligns the behaviour of `vmselect` with Prometheus. * BUGFIX: [vmagent](https://docs.victoriametrics.com/vmagent.html): restart all the scrape jobs during [config reload](https://docs.victoriametrics.com/vmagent.html#configuration-update) after `global` section is changed inside `-promscrape.config`. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2884). diff --git a/docs/README.md b/docs/README.md index 16edc70a3..130c59825 100644 --- a/docs/README.md +++ b/docs/README.md @@ -310,28 +310,12 @@ Prometheus doesn't drop data during VictoriaMetrics restart. See [this article]( ## How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter) -VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets. Currently the following [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) types are supported: - -* [static_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) -* [file_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) -* [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) -* [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) -* [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) -* [azure_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#azure_sd_config) -* [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) -* [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) -* [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) -* [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config) -* [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) -* [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) -* [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) -* [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) -* [yandexcloud_sd_config](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/sd_configs.md#yandex_cloud_service_discovery_config) - -File a [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need support for other `*_sd_config` types. +VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets. The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values. +See [the list of supported service discovery types for Prometheus scrape targets](https://docs.victoriametrics.com/sd_configs.html). + VictoriaMetrics also supports [importing data in Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format). See also [vmagent](https://docs.victoriametrics.com/vmagent.html), which can be used as drop-in replacement for Prometheus. @@ -2118,6 +2102,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed. See also -promscrape.suppressScrapeErrorsDelay -promscrape.suppressScrapeErrorsDelay duration The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors + -promscrape.yandexcloudSDCheckInterval duration + Interval for checking for changes in Yandex Cloud. This works only if yandexcloud_sd_configs is configured in '-promscrape.config' file. (default 30s) -pushmetrics.extraLabel array Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index d17694397..68b0d82d6 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -314,27 +314,12 @@ Prometheus doesn't drop data during VictoriaMetrics restart. See [this article]( ## How to scrape Prometheus exporters such as [node-exporter](https://github.com/prometheus/node_exporter) -VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets. Currently the following [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) types are supported: - -* [static_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) -* [file_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) -* [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) -* [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) -* [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) -* [azure_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#azure_sd_config) -* [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) -* [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) -* [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) -* [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config) -* [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) -* [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) -* [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) -* [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) - -File a [feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need support for other `*_sd_config` types. +VictoriaMetrics can be used as drop-in replacement for Prometheus for scraping targets configured in `prometheus.yml` config file according to [the specification](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file). Just set `-promscrape.config` command-line flag to the path to `prometheus.yml` config - and VictoriaMetrics should start scraping the configured targets. The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders, which are substituted by the corresponding `ENV_VAR` environment variable values. +See [the list of supported service discovery types for Prometheus scrape targets](https://docs.victoriametrics.com/sd_configs.html). + VictoriaMetrics also supports [importing data in Prometheus exposition format](#how-to-import-data-in-prometheus-exposition-format). See also [vmagent](https://docs.victoriametrics.com/vmagent.html), which can be used as drop-in replacement for Prometheus. @@ -2121,6 +2106,8 @@ Pass `-help` to VictoriaMetrics in order to see the list of supported command-li Whether to suppress scrape errors logging. The last error for each target is always available at '/targets' page even if scrape errors logging is suppressed. See also -promscrape.suppressScrapeErrorsDelay -promscrape.suppressScrapeErrorsDelay duration The delay for suppressing repeated scrape errors logging per each scrape targets. This may be used for reducing the number of log lines related to scrape errors. See also -promscrape.suppressScrapeErrors + -promscrape.yandexcloudSDCheckInterval duration + Interval for checking for changes in Yandex Cloud. This works only if yandexcloud_sd_configs is configured in '-promscrape.config' file. (default 30s) -pushmetrics.extraLabel array Optional labels to add to metrics pushed to -pushmetrics.url . For example, -pushmetrics.extraLabel='instance="foo"' adds instance="foo" label to all the metrics pushed to -pushmetrics.url Supports an array of values separated by comma or specified via multiple flags. diff --git a/docs/sd_configs.md b/docs/sd_configs.md index b420658a3..016cf8d70 100644 --- a/docs/sd_configs.md +++ b/docs/sd_configs.md @@ -1,4 +1,36 @@ -## Yandex Cloud Service Discovery Configs +# Prometheus service discovery + +[vmagent](https://docs.victoriametrics.com/vmagent.html) and [single-node VictoriaMetrics](https://docs.victoriametrics.com/#how-to-scrape-prometheus-exporters-such-as-node-exporter) supports the following Prometheus-compatible service discovery options for Prometheus-compatible scrape targets in the file pointed by `-promscrape.config` command-line flag. + +* `azure_sd_configs` - is for scraping the targets registered in Azure Cloud. + See [azure_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#azure_sd_config) for details. +* `consul_sd_configs` is for discovering and scraping targets registered in Consul. See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details. +* `digitalocean_sd_configs` is for discovering and scraping targerts registered in [DigitalOcean](https://www.digitalocean.com/). See [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) for details. +* `dns_sd_configs` is for discovering and scraping targets from DNS records (SRV, A and AAAA). See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details. +* `docker_sd_configs` is for discovering and scraping Docker targets. See [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config) for details. +* `dockerswarm_sd_configs` is for discovering and scraping Docker Swarm targets. See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details. +* `ec2_sd_configs` is for discovering and scraping Amazon EC2 targets. See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details. `vmagent` doesn't support the `profile` config param yet. +* `eureka_sd_configs` is for discovering and scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka). See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details. +* `file_sd_configs` is for scraping targets defined in external files (aka file-based service discovery). See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details. +* `gce_sd_configs` is for discovering and scraping Google Compute Engine (GCE) targets. See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details. `vmagent` provides the following additional functionality for `gce_sd_config`: + * if `project` arg is missing then `vmagent` uses the project for the instance where it runs; + * if `zone` arg is missing then `vmagent` uses the zone for the instance where it runs; + * if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project; + * `zone` may contain a list of zones, i.e. `zone: [us-east1-a, us-east1-b]`. +* `http_sd_configs` is for discovering and scraping targerts provided by external http-based service discovery. See [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) for details. +* `kubernetes_sd_configs` is for discovering and scraping Kubernetes (K8S) targets. See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details. +* `openstack_sd_configs` is for discovering and scraping OpenStack targets. See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details. [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only. +* `static_configs` is for scraping statically defined targets. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) for details. +* `yandexcloud_sd_configs` is for discoverying and scraping [Yandex Cloud](https://cloud.yandex.com/en/) targets. See [these docs](#yandexcloud-sd-configs) for details. + +Note that the `refresh_interval` option isn't supported for these scrape configs. Use the corresponding `-promscrape.*CheckInterval` +command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs` +entries to 60s. Run `vmagent -help` or `victoria-metrics -help` in order to see default values for the `-promscrape.*CheckInterval` flags. + +Please file feature requests to [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by VictoriaMetrics and `vmagent`. + + +## yandexcloud_sd_configs Yandex Cloud SD configurations allow retrieving scrape targets from accessible folders. @@ -43,4 +75,4 @@ scrape_configs: - source_labels: [__meta_yandexcloud_instance_private_ip_0] target_label: __address__ replacement: "$1:9100" -``` \ No newline at end of file +``` diff --git a/docs/vmagent.md b/docs/vmagent.md index 462b4f23d..8c04a466f 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -159,34 +159,8 @@ Use `-remoteWrite.*` command-line flag instead for configuring remote write sett The file pointed by `-promscrape.config` may contain `%{ENV_VAR}` placeholders which are substituted by the corresponding `ENV_VAR` environment variable values. -The following scrape types in [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) section are supported: +See [the list of supported service discovery types for Prometheus scrape targets](https://docs.victoriametrics.com/sd_configs.html). -* `static_configs` is for scraping statically defined targets. See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) for details. -* `file_sd_configs` is for scraping targets defined in external files (aka file-based service discovery). See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details. -* `kubernetes_sd_configs` is for discovering and scraping Kubernetes (K8S) targets. See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details. -* `ec2_sd_configs` is for discovering and scraping Amazon EC2 targets. See [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) for details. `vmagent` doesn't support the `profile` config param yet. -* `gce_sd_configs` is for discovering and scraping Google Compute Engine (GCE) targets. See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details. `vmagent` provides the following additional functionality for `gce_sd_config`: - * if `project` arg is missing then `vmagent` uses the project for the instance where it runs; - * if `zone` arg is missing then `vmagent` uses the zone for the instance where it runs; - * if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project; - * `zone` may contain a list of zones, i.e. `zone: [us-east1-a, us-east1-b]`. -* `azure_sd_configs` - is for scraping the targets registered in Azure Cloud. - See [azure_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#azure_sd_config) for details. -* `consul_sd_configs` is for discovering and scraping targets registered in Consul. See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details. -* `dns_sd_configs` is for discovering and scraping targets from DNS records (SRV, A and AAAA). See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details. -* `openstack_sd_configs` is for discovering and scraping OpenStack targets. See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details. [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only. -* `docker_sd_configs` is for discovering and scraping Docker targets. See [docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config) for details. -* `dockerswarm_sd_configs` is for discovering and scraping Docker Swarm targets. See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details. -* `eureka_sd_configs` is for discovering and scraping targets registered in [Netflix Eureka](https://github.com/Netflix/eureka). See [eureka_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#eureka_sd_config) for details. -* `digitalocean_sd_configs` is for discovering and scraping targerts registered in [DigitalOcean](https://www.digitalocean.com/). See [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config) for details. -* `http_sd_configs` is for discovering and scraping targerts provided by external http-based service discovery. See [http_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_sd_config) for details. -* `yandexcloud_sd_configs` is for discovering and scraping targets registered in [Yandex Cloud](https://cloud.yandex.ru/). See [yandexcloud_sd_configs](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/docs/sd_configs.md#yandex_cloud_service_discovery_configs) for details. - -Note that `vmagent` doesn't support `refresh_interval` option for these scrape configs. Use the corresponding `-promscrape.*CheckInterval` -command-line flag instead. For example, `-promscrape.consulSDCheckInterval=60s` sets `refresh_interval` for all the `consul_sd_configs` -entries to 60s. Run `vmagent -help` in order to see default values for the `-promscrape.*CheckInterval` flags. - -Please file feature requests to [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`. ## scrape_config enhancements diff --git a/lib/promscrape/discovery/yandexcloud/api.go b/lib/promscrape/discovery/yandexcloud/api.go index f84ec61fc..16776aa88 100644 --- a/lib/promscrape/discovery/yandexcloud/api.go +++ b/lib/promscrape/discovery/yandexcloud/api.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "net/http" "net/url" - "path" "sync" "time" @@ -16,12 +15,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" ) -const ( - defaultInstanceCredsEndpoint = "http://169.254.169.254/latest/meta-data/iam/security-credentials/default" - defaultAPIEndpoint = "https://api.cloud.yandex.net" - defaultAPIVersion = "v1" -) - var configMap = discoveryutils.NewConfigMap() type apiCredentials struct { @@ -35,19 +28,14 @@ type yandexPassportOAuth struct { YandexPassportOAuthToken string `json:"yandexPassportOauthToken"` } -// iamToken Yandex Cloud IAM token response -// https://cloud.yandex.com/en-ru/docs/iam/operations/iam-token/create -type iamToken struct { - IAMToken string `json:"iamToken"` - ExpiresAt time.Time `json:"expiresAt"` -} - type apiConfig struct { client *http.Client - tokenLock sync.Mutex - creds *apiCredentials yandexPassportOAuth *yandexPassportOAuth - serviceEndpoints map[string]*url.URL + serviceEndpoints map[string]string + + // credsLock protects the refresh of creds + credsLock sync.Mutex + creds *apiCredentials } func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { @@ -59,12 +47,8 @@ func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { } func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { - cfg := &apiConfig{ - client: &http.Client{ - Transport: &http.Transport{ - MaxIdleConnsPerHost: 100, - }, - }, + transport := &http.Transport{ + MaxIdleConnsPerHost: 100, } if sdc.TLSConfig != nil { opts := &promauth.Options{ @@ -73,47 +57,49 @@ func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) { } ac, err := opts.NewConfig() if err != nil { - return nil, err - } - cfg.client.Transport = &http.Transport{ - TLSClientConfig: ac.NewTLSConfig(), - MaxIdleConnsPerHost: 100, + return nil, fmt.Errorf("cannot initialize TLS config: %w", err) } + transport.TLSClientConfig = ac.NewTLSConfig() } - - if err := cfg.getEndpoints(sdc.APIEndpoint); err != nil { - return nil, err + cfg := &apiConfig{ + client: &http.Client{ + Transport: transport, + }, } - + apiEndpoint := sdc.APIEndpoint + if apiEndpoint == "" { + apiEndpoint = "https://api.cloud.yandex.net" + } + serviceEndpoints, err := cfg.getServiceEndpoints(apiEndpoint) + if err != nil { + return nil, fmt.Errorf("cannot obtain endpoints for yandex services: %w", err) + } + cfg.serviceEndpoints = serviceEndpoints if sdc.YandexPassportOAuthToken != nil { - logger.Infof("Using yandex passport OAuth token") - + logger.Infof("yandexcloud_sd: using yandex passport OAuth token") cfg.yandexPassportOAuth = &yandexPassportOAuth{ YandexPassportOAuthToken: sdc.YandexPassportOAuthToken.String(), } } - return cfg, nil } // getFreshAPICredentials checks token lifetime and update if needed func (cfg *apiConfig) getFreshAPICredentials() (*apiCredentials, error) { - cfg.tokenLock.Lock() - defer cfg.tokenLock.Unlock() + cfg.credsLock.Lock() + defer cfg.credsLock.Unlock() if cfg.creds != nil && time.Until(cfg.creds.Expiration) > 10*time.Second { // Credentials aren't expired yet. return cfg.creds, nil } - + // Refresh credentials. newCreds, err := getCreds(cfg) if err != nil { return nil, fmt.Errorf("cannot refresh service account api token: %w", err) } cfg.creds = newCreds - - logger.Infof("successfully refreshed service account api token; expiration: %.3f seconds", time.Until(newCreds.Expiration).Seconds()) - + logger.Infof("yandexcloud_sd: successfully refreshed service account api token; expiration: %.3f seconds", time.Until(newCreds.Expiration).Seconds()) return newCreds, nil } @@ -122,12 +108,10 @@ func getCreds(cfg *apiConfig) (*apiCredentials, error) { if cfg.yandexPassportOAuth == nil { return getInstanceCreds(cfg) } - it, err := getIAMToken(cfg) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot get IAM token: %w", err) } - return &apiCredentials{ Token: it.IAMToken, Expiration: it.ExpiresAt, @@ -135,104 +119,118 @@ func getCreds(cfg *apiConfig) (*apiCredentials, error) { } // getInstanceCreds gets Yandex Cloud IAM token using instance Service Account -// https://cloud.yandex.com/en-ru/docs/compute/operations/vm-connect/auth-inside-vm +// +// See https://cloud.yandex.com/en-ru/docs/compute/operations/vm-connect/auth-inside-vm func getInstanceCreds(cfg *apiConfig) (*apiCredentials, error) { - resp, err := cfg.client.Get(defaultInstanceCredsEndpoint) + endpoint := "http://169.254.169.254/latest/meta-data/iam/security-credentials/default" + resp, err := cfg.client.Get(endpoint) if err != nil { - return nil, fmt.Errorf("failed query security credentials api, url: %s, err: %w", defaultInstanceCredsEndpoint, err) + return nil, fmt.Errorf("cannot read instance creds from %s: %w", endpoint, err) } - r, err := ioutil.ReadAll(resp.Body) - _ = resp.Body.Close() + data, err := readResponseBody(resp, endpoint) if err != nil { - return nil, fmt.Errorf("cannot read response from %q: %w", defaultInstanceCredsEndpoint, err) + return nil, err } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("auth failed, bad status code: %d, want: 200", resp.StatusCode) - } - var ac apiCredentials - if err := json.Unmarshal(r, &ac); err != nil { - return nil, fmt.Errorf("cannot parse auth credentials response: %w", err) + if err := json.Unmarshal(data, &ac); err != nil { + return nil, fmt.Errorf("cannot parse auth credentials response from %s: %w", endpoint, err) } - return &ac, nil } -// getIAMToken gets Yandex Cloud IAM token using OAuth: -// https://cloud.yandex.com/en-ru/docs/iam/operations/iam-token/create +// getIAMToken gets Yandex Cloud IAM token using OAuth +// +// See https://cloud.yandex.com/en-ru/docs/iam/operations/iam-token/create func getIAMToken(cfg *apiConfig) (*iamToken, error) { - iamURL := *cfg.serviceEndpoints["iam"] - iamURL.Path = path.Join(iamURL.Path, "iam", defaultAPIVersion, "tokens") - + iamURL := cfg.serviceEndpoints["iam"] + "/iam/v1/tokens" passport, err := json.Marshal(cfg.yandexPassportOAuth) if err != nil { - return nil, fmt.Errorf("failed marshall yandex passport OAuth token, err: %w", err) + logger.Panicf("BUG: cannot marshal yandex passport OAuth token: %s", err) } - - resp, err := cfg.client.Post(iamURL.String(), "application/json", bytes.NewBuffer(passport)) + body := bytes.NewBuffer(passport) + resp, err := cfg.client.Post(iamURL, "application/json", body) if err != nil { - return nil, fmt.Errorf("failed query yandex cloud iam api, url: %s, err: %w", iamURL.String(), err) + logger.Panicf("BUG: cannot create request to yandex cloud iam api %q: %s", iamURL, err) } - - r, err := ioutil.ReadAll(resp.Body) - _ = resp.Body.Close() + data, err := readResponseBody(resp, iamURL) if err != nil { - return nil, fmt.Errorf("cannot read response from %q: %w", iamURL.String(), err) + return nil, err } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("auth failed, bad status code: %d, want: 200", resp.StatusCode) + var it iamToken + if err := json.Unmarshal(data, &it); err != nil { + return nil, fmt.Errorf("cannot parse iam token: %w; data: %s", err, data) } - - it := iamToken{} - if err := json.Unmarshal(r, &it); err != nil { - return nil, fmt.Errorf("cannot parse auth credentials response: %w", err) - } - return &it, nil } -// getEndpoints makes services endpoints map: -// https://cloud.yandex.com/en-ru/docs/api-design-guide/concepts/endpoints -func (cfg *apiConfig) getEndpoints(apiEndpoint string) error { - if apiEndpoint == "" { - apiEndpoint = defaultAPIEndpoint - } +// iamToken represents Yandex Cloud IAM token response +// +// See https://cloud.yandex.com/en-ru/docs/iam/operations/iam-token/create +type iamToken struct { + IAMToken string `json:"iamToken"` + ExpiresAt time.Time `json:"expiresAt"` +} +// getServiceEndpoints returns services endpoints map +// +// See https://cloud.yandex.com/en-ru/docs/api-design-guide/concepts/endpoints +func (cfg *apiConfig) getServiceEndpoints(apiEndpoint string) (map[string]string, error) { apiEndpointURL, err := url.Parse(apiEndpoint) if err != nil { - return fmt.Errorf("cannot parse api_endpoint: %s as url, err: %w", apiEndpoint, err) + return nil, fmt.Errorf("cannot parse api_endpoint %q: %w", apiEndpoint, err) } - - apiEndpointURL.Path = path.Join(apiEndpointURL.Path, "endpoints") - - resp, err := cfg.client.Get(apiEndpointURL.String()) + scheme := apiEndpointURL.Scheme + if scheme == "" { + return nil, fmt.Errorf("missing scheme in api_endpoint %q", apiEndpoint) + } + if apiEndpointURL.Host == "" { + return nil, fmt.Errorf("missing host in api_endpoint %q", apiEndpoint) + } + endpointsURL := apiEndpoint + "/endpoints" + resp, err := cfg.client.Get(endpointsURL) if err != nil { - return fmt.Errorf("failed query endpoints, url: %s, err: %w", apiEndpointURL.String(), err) + return nil, fmt.Errorf("cannot query %q: %w", endpointsURL, err) } - r, err := ioutil.ReadAll(resp.Body) - _ = resp.Body.Close() + data, err := readResponseBody(resp, endpointsURL) if err != nil { - return fmt.Errorf("cannot read response from %q: %w", apiEndpointURL.String(), err) + return nil, err } - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("auth failed, bad status code: %d, want: 200", resp.StatusCode) + var eps endpoints + if err := json.Unmarshal(data, &eps); err != nil { + return nil, fmt.Errorf("cannot parse API endpoints list: %w; data=%s", err, data) } + m := make(map[string]string, len(eps.Endpoints)) + for _, endpoint := range eps.Endpoints { + m[endpoint.ID] = scheme + "://" + endpoint.Address + } + return m, nil +} - endpoints, err := parseEndpoints(r) +type endpoint struct { + ID string `json:"id"` + Address string `json:"address"` +} + +type endpoints struct { + Endpoints []endpoint `json:"endpoints"` +} + +// getAPIResponse calls Yandex Cloud apiURL and returns response body. +func getAPIResponse(apiURL string, cfg *apiConfig) ([]byte, error) { + creds, err := cfg.getFreshAPICredentials() if err != nil { - return err + return nil, err } - - cfg.serviceEndpoints = make(map[string]*url.URL, len(endpoints.Endpoints)) - for _, endpoint := range endpoints.Endpoints { - cfg.serviceEndpoints[endpoint.ID] = &url.URL{ - Scheme: apiEndpointURL.Scheme, - Host: endpoint.Address, - } + req, err := http.NewRequest("GET", apiURL, nil) + if err != nil { + logger.Panicf("BUG: cannot create new request for yandex cloud api url %s: %s", apiURL, err) } - - return nil + req.Header.Set("Authorization", "Bearer "+creds.Token) + resp, err := cfg.client.Do(req) + if err != nil { + return nil, fmt.Errorf("cannot query yandex cloud api url %s: %w", apiURL, err) + } + return readResponseBody(resp, apiURL) } // readResponseBody reads body from http.Response. @@ -243,30 +241,8 @@ func readResponseBody(resp *http.Response, apiURL string) ([]byte, error) { return nil, fmt.Errorf("cannot read response from %q: %w", apiURL, err) } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code for %q; got %d; want %d; response body: %q", + return nil, fmt.Errorf("unexpected status code for %q; got %d; want %d; response body: %s", apiURL, resp.StatusCode, http.StatusOK, data) } - return data, nil } - -// getAPIResponse calls Yandex Cloud apiURL and returns response body. -func getAPIResponse(apiURL string, cfg *apiConfig) ([]byte, error) { - creds, err := cfg.getFreshAPICredentials() - if err != nil { - return nil, err - } - - req, err := http.NewRequest("GET", apiURL, nil) - if err != nil { - return nil, fmt.Errorf("cannot create new request for yandex cloud api url %s: %w", apiURL, err) - } - - req.Header.Set("Authorization", "Bearer "+creds.Token) - resp, err := cfg.client.Do(req) - if err != nil { - return nil, fmt.Errorf("cannot query yandex cloud api url %s: %w", apiURL, err) - } - - return readResponseBody(resp, apiURL) -} diff --git a/lib/promscrape/discovery/yandexcloud/instance.go b/lib/promscrape/discovery/yandexcloud/instance.go index c3ca4541c..a887745aa 100644 --- a/lib/promscrape/discovery/yandexcloud/instance.go +++ b/lib/promscrape/discovery/yandexcloud/instance.go @@ -27,11 +27,10 @@ func getInstancesLabels(cfg *apiConfig) ([]map[string]string, error) { if err != nil { return nil, err } - instances = append(instances, inst...) } - logger.Infof("Collected %d instances", len(instances)) + logger.Infof("yandexcloud_sd: collected %d instances", len(instances)) return addInstanceLabels(instances), nil } @@ -55,20 +54,20 @@ func addInstanceLabels(instances []instance) []map[string]string { m["__meta_yandexcloud_instance_label_"+discoveryutils.SanitizeLabelName(k)] = v } - for _, netInterface := range server.NetworkInterfaces { - privateIPLabel := fmt.Sprintf("__meta_yandexcloud_instance_private_ip_%s", netInterface.Index) - m[privateIPLabel] = netInterface.PrimaryV4Address.Address - if len(netInterface.PrimaryV4Address.OneToOneNat.Address) > 0 { - publicIPLabel := fmt.Sprintf("__meta_yandexcloud_instance_public_ip_%s", netInterface.Index) - m[publicIPLabel] = netInterface.PrimaryV4Address.OneToOneNat.Address + for _, ni := range server.NetworkInterfaces { + privateIPLabel := fmt.Sprintf("__meta_yandexcloud_instance_private_ip_%s", ni.Index) + m[privateIPLabel] = ni.PrimaryV4Address.Address + if len(ni.PrimaryV4Address.OneToOneNat.Address) > 0 { + publicIPLabel := fmt.Sprintf("__meta_yandexcloud_instance_public_ip_%s", ni.Index) + m[publicIPLabel] = ni.PrimaryV4Address.OneToOneNat.Address } - for j, dnsRecord := range netInterface.PrimaryV4Address.DNSRecords { + for j, dnsRecord := range ni.PrimaryV4Address.DNSRecords { dnsRecordLabel := fmt.Sprintf("__meta_yandexcloud_instance_private_dns_%d", j) m[dnsRecordLabel] = dnsRecord.FQDN } - for j, dnsRecord := range netInterface.PrimaryV4Address.OneToOneNat.DNSRecords { + for j, dnsRecord := range ni.PrimaryV4Address.OneToOneNat.DNSRecords { dnsRecordLabel := fmt.Sprintf("__meta_yandexcloud_instance_public_dns_%d", j) m[dnsRecordLabel] = dnsRecord.FQDN } diff --git a/lib/promscrape/discovery/yandexcloud/resources.go b/lib/promscrape/discovery/yandexcloud/resources.go index d65f663d2..f2e7e59ce 100644 --- a/lib/promscrape/discovery/yandexcloud/resources.go +++ b/lib/promscrape/discovery/yandexcloud/resources.go @@ -2,34 +2,10 @@ package yandexcloud import ( "encoding/json" - "errors" "fmt" "time" ) -type endpoint struct { - ID string `json:"id"` - Address string `json:"address"` -} - -type endpoints struct { - Endpoints []endpoint `json:"endpoints"` -} - -// See https://cloud.yandex.com/en-ru/docs/api-design-guide/concepts/endpoints -func parseEndpoints(data []byte) (*endpoints, error) { - var endpointsResponse endpoints - if err := json.Unmarshal(data, &endpointsResponse); err != nil { - return nil, fmt.Errorf("cannot parse endpoints list: %w", err) - } - - if endpointsResponse.Endpoints == nil { - return nil, errors.New("yandex cloud API endpoints list is empty") - } - - return &endpointsResponse, nil -} - type organization struct { Name string `json:"name"` ID string `json:"id"` diff --git a/lib/promscrape/discovery/yandexcloud/yandexcloud.go b/lib/promscrape/discovery/yandexcloud/yandexcloud.go index 6542cd24f..87cd381f5 100644 --- a/lib/promscrape/discovery/yandexcloud/yandexcloud.go +++ b/lib/promscrape/discovery/yandexcloud/yandexcloud.go @@ -3,7 +3,7 @@ package yandexcloud import ( "flag" "fmt" - "path" + "net/url" "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" @@ -36,142 +36,103 @@ func (sdc *SDConfig) GetLabels(baseDir string) ([]map[string]string, error) { } func (cfg *apiConfig) getInstances(folderID string) ([]instance, error) { - computeURL := *cfg.serviceEndpoints["compute"] - computeURL.Path = path.Join(computeURL.Path, "compute", defaultAPIVersion, "instances") - q := computeURL.Query() - q.Set("folderId", folderID) - computeURL.RawQuery = q.Encode() - nextLink := computeURL.String() + instancesURL := cfg.serviceEndpoints["compute"] + "/compute/v1/instances" + instancesURL += "?folderId=" + url.QueryEscape(folderID) - instances := make([]instance, 0) + var instances []instance + nextLink := instancesURL for { - resp, err := getAPIResponse(nextLink, cfg) + data, err := getAPIResponse(nextLink, cfg) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot get instances: %w", err) } - instancesPage, err := parseInstancesPage(resp) + ip, err := parseInstancesPage(data) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot parse instances response from %q: %w; response body: %s", nextLink, err, data) } - instances = append(instances, instancesPage.Instances...) - if len(instancesPage.NextPageToken) == 0 { + instances = append(instances, ip.Instances...) + if len(ip.NextPageToken) == 0 { return instances, nil } - - q.Set("pageToken", instancesPage.NextPageToken) - computeURL.RawQuery = q.Encode() - nextLink = computeURL.String() + nextLink = instancesURL + "&pageToken=" + url.QueryEscape(ip.NextPageToken) } } func (cfg *apiConfig) getFolders(clouds []cloud) ([]folder, error) { - rmURL := *cfg.serviceEndpoints["resource-manager"] - rmURL.Path = path.Join(rmURL.Path, "resource-manager", defaultAPIVersion, "folders") - q := rmURL.Query() - - folders := make([]folder, 0) + foldersURL := cfg.serviceEndpoints["resource-manager"] + "/resource-manager/v1/folders" + var folders []folder for _, cl := range clouds { - q.Set("cloudId", cl.ID) - rmURL.RawQuery = q.Encode() - - nextLink := rmURL.String() + cloudURL := foldersURL + "?cloudId=" + url.QueryEscape(cl.ID) + nextLink := cloudURL for { - resp, err := getAPIResponse(nextLink, cfg) + data, err := getAPIResponse(nextLink, cfg) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot get folders: %w", err) } - - foldersPage, err := parseFoldersPage(resp) + fp, err := parseFoldersPage(data) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot parse folders response from %q: %w; response body: %s", nextLink, err, data) } - - folders = append(folders, foldersPage.Folders...) - - if len(foldersPage.NextPageToken) == 0 { + folders = append(folders, fp.Folders...) + if len(fp.NextPageToken) == 0 { break } - - q.Set("pageToken", foldersPage.NextPageToken) - rmURL.RawQuery = q.Encode() - nextLink = rmURL.String() + nextLink = cloudURL + "&pageToken=" + url.QueryEscape(fp.NextPageToken) } } - return folders, nil } -func (cfg *apiConfig) getClouds(organizations []organization) ([]cloud, error) { - rmURL := *cfg.serviceEndpoints["resource-manager"] - rmURL.Path = path.Join(rmURL.Path, "resource-manager", defaultAPIVersion, "clouds") - q := rmURL.Query() - - if len(organizations) == 0 { - organizations = append(organizations, organization{ +func (cfg *apiConfig) getClouds(orgs []organization) ([]cloud, error) { + cloudsURL := cfg.serviceEndpoints["resource-manager"] + "/resource-manager/v1/clouds" + if len(orgs) == 0 { + orgs = append(orgs, organization{ ID: "", }) } - - clouds := make([]cloud, 0) - for _, org := range organizations { + var clouds []cloud + for _, org := range orgs { + orgURL := cloudsURL if org.ID != "" { - q.Set("organizationId", org.ID) - rmURL.RawQuery = q.Encode() + orgURL += "?organizationId=" + url.QueryEscape(org.ID) } - - nextLink := rmURL.String() + nextLink := orgURL for { - resp, err := getAPIResponse(nextLink, cfg) + data, err := getAPIResponse(nextLink, cfg) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot get clouds: %w", err) } - - cloudsPage, err := parseCloudsPage(resp) + cp, err := parseCloudsPage(data) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot parse clouds response from %q: %w; response body: %s", nextLink, err, data) } - - clouds = append(clouds, cloudsPage.Clouds...) - - if len(cloudsPage.NextPageToken) == 0 { + clouds = append(clouds, cp.Clouds...) + if len(cp.NextPageToken) == 0 { break } - - q.Set("pageToken", cloudsPage.NextPageToken) - rmURL.RawQuery = q.Encode() - nextLink = rmURL.String() + nextLink = orgURL + "&pageToken=" + url.QueryEscape(cp.NextPageToken) } } - return clouds, nil } func (cfg *apiConfig) getOrganizations() ([]organization, error) { - omURL := *cfg.serviceEndpoints["organization-manager"] - omURL.Path = path.Join(omURL.Path, "organization-manager", defaultAPIVersion, "organizations") - q := omURL.Query() - nextLink := omURL.String() - - organizations := make([]organization, 0) + orgsURL := cfg.serviceEndpoints["organization-manager"] + "/organization-manager/v1/organizations" + var orgs []organization + nextLink := orgsURL for { - resp, err := getAPIResponse(nextLink, cfg) + data, err := getAPIResponse(nextLink, cfg) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot get organizations: %w", err) } - - organizationsPage, err := parseOrganizationsPage(resp) + op, err := parseOrganizationsPage(data) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot parse organizations response from %q: %w; response body: %s", nextLink, err, data) } - - organizations = append(organizations, organizationsPage.Organizations...) - - if len(organizationsPage.NextPageToken) == 0 { - return organizations, nil + orgs = append(orgs, op.Organizations...) + if len(op.NextPageToken) == 0 { + return orgs, nil } - - q.Set("pageToken", organizationsPage.NextPageToken) - omURL.RawQuery = q.Encode() - nextLink = omURL.String() + nextLink = orgsURL + "&pageToken=" + url.QueryEscape(op.NextPageToken) } }