diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 0ed7a0721a..67dcc8c012 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -135,6 +135,7 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details. * `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE). See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details. + `vmagent` supports empty `zone` arg inside `gce_sd_config` - in this case it autodetects all the zones for the given project. The following service discovery mechanisms will be added to `vmagent` soon: diff --git a/docs/vmagent.md b/docs/vmagent.md index 0ed7a0721a..67dcc8c012 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -135,6 +135,7 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details. * `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE). See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details. + `vmagent` supports empty `zone` arg inside `gce_sd_config` - in this case it autodetects all the zones for the given project. The following service discovery mechanisms will be added to `vmagent` soon: diff --git a/lib/promscrape/discovery/gce/api.go b/lib/promscrape/discovery/gce/api.go index 96da0e0113..7fd03cf8fc 100644 --- a/lib/promscrape/discovery/gce/api.go +++ b/lib/promscrape/discovery/gce/api.go @@ -3,8 +3,10 @@ package gce import ( "context" "fmt" + "io/ioutil" "net/http" "net/url" + "strings" "sync" "time" @@ -13,8 +15,9 @@ import ( type apiConfig struct { client *http.Client - apiURL string + zones []string project string + filter string tagSeparator string port int } @@ -74,10 +77,16 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) { if err != nil { return nil, fmt.Errorf("cannot create oauth2 client for gce: %s", err) } - // See https://cloud.google.com/compute/docs/reference/rest/v1/instances/list - apiURL := fmt.Sprintf("https://compute.googleapis.com/compute/v1/projects/%s/zones/%s/instances", sdc.Project, sdc.Zone) - if len(sdc.Filter) > 0 { - apiURL += fmt.Sprintf("?filter=%s", url.QueryEscape(sdc.Filter)) + var zones []string + if len(sdc.Zone) == 0 { + // Autodetect zones for sdc.Project. + zs, err := getZonesForProject(client, sdc.Project, sdc.Filter) + if err != nil { + return nil, fmt.Errorf("cannot obtain zones for project %q: %s", sdc.Project, err) + } + zones = zs + } else { + zones = []string{sdc.Zone} } tagSeparator := "," if sdc.TagSeparator != nil { @@ -89,9 +98,40 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) { } return &apiConfig{ client: client, - apiURL: apiURL, + zones: zones, project: sdc.Project, + filter: sdc.Filter, tagSeparator: tagSeparator, port: port, }, nil } + +func getAPIResponse(client *http.Client, apiURL, filter, pageToken string) ([]byte, error) { + apiURL = appendNonEmptyQueryArg(apiURL, filter) + apiURL = appendNonEmptyQueryArg(apiURL, pageToken) + resp, err := client.Get(apiURL) + if err != nil { + return nil, fmt.Errorf("cannot query %q: %s", apiURL, err) + } + data, err := ioutil.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + return nil, fmt.Errorf("cannot read response from %q: %s", apiURL, err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code for %q; got %d; want %d; response body: %q", + apiURL, resp.StatusCode, http.StatusOK, data) + } + return data, nil +} + +func appendNonEmptyQueryArg(apiURL, arg string) string { + if len(arg) == 0 { + return apiURL + } + prefix := "?" + if strings.Contains(apiURL, "?") { + prefix = "&" + } + return apiURL + fmt.Sprintf("%spageToken=%s", prefix, url.QueryEscape(arg)) +} diff --git a/lib/promscrape/discovery/gce/instance.go b/lib/promscrape/discovery/gce/instance.go index 392646665f..89281aeff1 100644 --- a/lib/promscrape/discovery/gce/instance.go +++ b/lib/promscrape/discovery/gce/instance.go @@ -3,9 +3,7 @@ package gce import ( "encoding/json" "fmt" - "io/ioutil" "net/http" - "net/url" "strings" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" @@ -25,51 +23,37 @@ func getInstancesLabels(cfg *apiConfig) ([]map[string]string, error) { } func getInstances(cfg *apiConfig) ([]Instance, error) { - var result []Instance - pageToken := "" - for { - insts, nextPageToken, err := getInstancesPage(cfg, pageToken) + var insts []Instance + for _, zone := range cfg.zones { + zoneInsts, err := getInstancesForProjectAndZone(cfg.client, cfg.project, zone, cfg.filter) if err != nil { return nil, err } - result = append(result, insts...) - if len(nextPageToken) == 0 { - return result, nil - } - pageToken = nextPageToken + insts = append(insts, zoneInsts...) } + return insts, nil } -func getInstancesPage(cfg *apiConfig, pageToken string) ([]Instance, string, error) { - apiURL := cfg.apiURL - if len(pageToken) > 0 { - // See https://cloud.google.com/compute/docs/reference/rest/v1/instances/list about pageToken - prefix := "?" - if strings.Contains(apiURL, "?") { - prefix = "&" +func getInstancesForProjectAndZone(client *http.Client, project, zone, filter string) ([]Instance, error) { + // See https://cloud.google.com/compute/docs/reference/rest/v1/instances/list + instsURL := fmt.Sprintf("https://compute.googleapis.com/compute/v1/projects/%s/zones/%s/instances", project, zone) + var insts []Instance + pageToken := "" + for { + data, err := getAPIResponse(client, instsURL, filter, pageToken) + if err != nil { + return nil, fmt.Errorf("cannot obtain instances: %s", err) } - apiURL += fmt.Sprintf("%spageToken=%s", prefix, url.QueryEscape(pageToken)) + il, err := parseInstanceList(data) + if err != nil { + return nil, fmt.Errorf("cannot parse instance list from %q: %s", instsURL, err) + } + insts = append(insts, il.Items...) + if len(il.NextPageToken) == 0 { + return insts, nil + } + pageToken = il.NextPageToken } - resp, err := cfg.client.Get(apiURL) - if err != nil { - return nil, "", fmt.Errorf("cannot obtain instances data from API server: %s", err) - } - defer func() { - _ = resp.Body.Close() - }() - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, "", fmt.Errorf("cannot read instances data from API server: %s", err) - } - if resp.StatusCode != http.StatusOK { - return nil, "", fmt.Errorf("unexpected status code when reading instances data from API server; got %d; want %d; response body: %q", - resp.StatusCode, http.StatusOK, data) - } - il, err := parseInstanceList(data) - if err != nil { - return nil, "", fmt.Errorf("cannot parse instances response from API server: %s", err) - } - return il.Items, il.NextPageToken, nil } // InstanceList is response to https://cloud.google.com/compute/docs/reference/rest/v1/instances/list diff --git a/lib/promscrape/discovery/gce/zone.go b/lib/promscrape/discovery/gce/zone.go new file mode 100644 index 0000000000..98dbfcff8b --- /dev/null +++ b/lib/promscrape/discovery/gce/zone.go @@ -0,0 +1,51 @@ +package gce + +import ( + "encoding/json" + "fmt" + "net/http" +) + +func getZonesForProject(client *http.Client, project, filter string) ([]string, error) { + // See https://cloud.google.com/compute/docs/reference/rest/v1/zones + zonesURL := fmt.Sprintf("https://compute.googleapis.com/compute/v1/projects/%s/zones", project) + var zones []string + pageToken := "" + for { + data, err := getAPIResponse(client, zonesURL, filter, pageToken) + if err != nil { + return nil, fmt.Errorf("cannot obtain zones: %s", err) + } + zl, err := parseZoneList(data) + if err != nil { + return nil, fmt.Errorf("cannot parse zone list from %q: %s", zonesURL, err) + } + for _, z := range zl.Items { + zones = append(zones, z.Name) + } + if len(zl.NextPageToken) == 0 { + return zones, nil + } + pageToken = zl.NextPageToken + } +} + +// ZoneList is response to https://cloud.google.com/compute/docs/reference/rest/v1/zones/list +type ZoneList struct { + Items []Zone + NextPageToken string +} + +// Zone is zone from https://cloud.google.com/compute/docs/reference/rest/v1/zones/list +type Zone struct { + Name string +} + +// parseZoneList parses ZoneList from data. +func parseZoneList(data []byte) (*ZoneList, error) { + var zl ZoneList + if err := json.Unmarshal(data, &zl); err != nil { + return nil, fmt.Errorf("cannot unmarshal ZoneList from %q: %s", data, err) + } + return &zl, nil +}