lib/promscrape/discovery/gce: allow empty zone arg in gce_sd_config - in this case zones for the given project are automatically discovered

app/vmagent/README.md

@@ -135,6 +135,7 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
 * `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
   See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
+  `vmagent` supports empty `zone` arg inside `gce_sd_config` - in this case it autodetects all the zones for the given project.
 
 The following service discovery mechanisms will be added to `vmagent` soon:
 

docs/vmagent.md

@@ -135,6 +135,7 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
 * `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
   See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
+  `vmagent` supports empty `zone` arg inside `gce_sd_config` - in this case it autodetects all the zones for the given project.
 
 The following service discovery mechanisms will be added to `vmagent` soon:
 

lib/promscrape/discovery/gce/api.go

@@ -3,8 +3,10 @@ package gce
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
+	"strings"
 	"sync"
 	"time"
 
@@ -13,8 +15,9 @@ import (
 
 type apiConfig struct {
 	client       *http.Client
-	apiURL       string
+	zones        []string
 	project      string
+	filter       string
 	tagSeparator string
 	port         int
 }
@@ -74,10 +77,16 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot create oauth2 client for gce: %s", err)
 	}
-	// See https://cloud.google.com/compute/docs/reference/rest/v1/instances/list
+	var zones []string
-	apiURL := fmt.Sprintf("https://compute.googleapis.com/compute/v1/projects/%s/zones/%s/instances", sdc.Project, sdc.Zone)
+	if len(sdc.Zone) == 0 {
-	if len(sdc.Filter) > 0 {
+		// Autodetect zones for sdc.Project.
-		apiURL += fmt.Sprintf("?filter=%s", url.QueryEscape(sdc.Filter))
+		zs, err := getZonesForProject(client, sdc.Project, sdc.Filter)
+		if err != nil {
+			return nil, fmt.Errorf("cannot obtain zones for project %q: %s", sdc.Project, err)
+		}
+		zones = zs
+	} else {
+		zones = []string{sdc.Zone}
 	}
 	tagSeparator := ","
 	if sdc.TagSeparator != nil {
@@ -89,9 +98,40 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
 	}
 	return &apiConfig{
 		client:       client,
-		apiURL:       apiURL,
+		zones:        zones,
 		project:      sdc.Project,
+		filter:       sdc.Filter,
 		tagSeparator: tagSeparator,
 		port:         port,
 	}, nil
 }
+
+func getAPIResponse(client *http.Client, apiURL, filter, pageToken string) ([]byte, error) {
+	apiURL = appendNonEmptyQueryArg(apiURL, filter)
+	apiURL = appendNonEmptyQueryArg(apiURL, pageToken)
+	resp, err := client.Get(apiURL)
+	if err != nil {
+		return nil, fmt.Errorf("cannot query %q: %s", apiURL, err)
+	}
+	data, err := ioutil.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	if err != nil {
+		return nil, fmt.Errorf("cannot read response from %q: %s", apiURL, err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code for %q; got %d; want %d; response body: %q",
+			apiURL, resp.StatusCode, http.StatusOK, data)
+	}
+	return data, nil
+}
+
+func appendNonEmptyQueryArg(apiURL, arg string) string {
+	if len(arg) == 0 {
+		return apiURL
+	}
+	prefix := "?"
+	if strings.Contains(apiURL, "?") {
+		prefix = "&"
+	}
+	return apiURL + fmt.Sprintf("%spageToken=%s", prefix, url.QueryEscape(arg))
+}

lib/promscrape/discovery/gce/instance.go

@@ -3,9 +3,7 @@ package gce
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
-	"net/url"
 	"strings"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
@@ -25,51 +23,37 @@ func getInstancesLabels(cfg *apiConfig) ([]map[string]string, error) {
 }
 
 func getInstances(cfg *apiConfig) ([]Instance, error) {
-	var result []Instance
+	var insts []Instance
-	pageToken := ""
+	for _, zone := range cfg.zones {
-	for {
+		zoneInsts, err := getInstancesForProjectAndZone(cfg.client, cfg.project, zone, cfg.filter)
-		insts, nextPageToken, err := getInstancesPage(cfg, pageToken)
 		if err != nil {
 			return nil, err
 		}
-		result = append(result, insts...)
+		insts = append(insts, zoneInsts...)
-		if len(nextPageToken) == 0 {
-			return result, nil
-		}
-		pageToken = nextPageToken
 	}
+	return insts, nil
 }
 
-func getInstancesPage(cfg *apiConfig, pageToken string) ([]Instance, string, error) {
+func getInstancesForProjectAndZone(client *http.Client, project, zone, filter string) ([]Instance, error) {
-	apiURL := cfg.apiURL
+	// See https://cloud.google.com/compute/docs/reference/rest/v1/instances/list
-	if len(pageToken) > 0 {
+	instsURL := fmt.Sprintf("https://compute.googleapis.com/compute/v1/projects/%s/zones/%s/instances", project, zone)
-		// See https://cloud.google.com/compute/docs/reference/rest/v1/instances/list about pageToken
+	var insts []Instance
-		prefix := "?"
+	pageToken := ""
-		if strings.Contains(apiURL, "?") {
+	for {
-			prefix = "&"
+		data, err := getAPIResponse(client, instsURL, filter, pageToken)
-		}
-		apiURL += fmt.Sprintf("%spageToken=%s", prefix, url.QueryEscape(pageToken))
-	}
-	resp, err := cfg.client.Get(apiURL)
 		if err != nil {
-		return nil, "", fmt.Errorf("cannot obtain instances data from API server: %s", err)
+			return nil, fmt.Errorf("cannot obtain instances: %s", err)
-	}
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-	data, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, "", fmt.Errorf("cannot read instances data from API server: %s", err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		return nil, "", fmt.Errorf("unexpected status code when reading instances data from API server; got %d; want %d; response body: %q",
-			resp.StatusCode, http.StatusOK, data)
 		}
 		il, err := parseInstanceList(data)
 		if err != nil {
-		return nil, "", fmt.Errorf("cannot parse instances response from API server: %s", err)
+			return nil, fmt.Errorf("cannot parse instance list from %q: %s", instsURL, err)
+		}
+		insts = append(insts, il.Items...)
+		if len(il.NextPageToken) == 0 {
+			return insts, nil
+		}
+		pageToken = il.NextPageToken
 	}
-	return il.Items, il.NextPageToken, nil
 }
 
 // InstanceList is response to https://cloud.google.com/compute/docs/reference/rest/v1/instances/list

lib/promscrape/discovery/gce/zone.go

@@ -0,0 +1,51 @@
+package gce
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+func getZonesForProject(client *http.Client, project, filter string) ([]string, error) {
+	// See https://cloud.google.com/compute/docs/reference/rest/v1/zones
+	zonesURL := fmt.Sprintf("https://compute.googleapis.com/compute/v1/projects/%s/zones", project)
+	var zones []string
+	pageToken := ""
+	for {
+		data, err := getAPIResponse(client, zonesURL, filter, pageToken)
+		if err != nil {
+			return nil, fmt.Errorf("cannot obtain zones: %s", err)
+		}
+		zl, err := parseZoneList(data)
+		if err != nil {
+			return nil, fmt.Errorf("cannot parse zone list from %q: %s", zonesURL, err)
+		}
+		for _, z := range zl.Items {
+			zones = append(zones, z.Name)
+		}
+		if len(zl.NextPageToken) == 0 {
+			return zones, nil
+		}
+		pageToken = zl.NextPageToken
+	}
+}
+
+// ZoneList is response to https://cloud.google.com/compute/docs/reference/rest/v1/zones/list
+type ZoneList struct {
+	Items         []Zone
+	NextPageToken string
+}
+
+// Zone is zone from https://cloud.google.com/compute/docs/reference/rest/v1/zones/list
+type Zone struct {
+	Name string
+}
+
+// parseZoneList parses ZoneList from data.
+func parseZoneList(data []byte) (*ZoneList, error) {
+	var zl ZoneList
+	if err := json.Unmarshal(data, &zl); err != nil {
+		return nil, fmt.Errorf("cannot unmarshal ZoneList from %q: %s", data, err)
+	}
+	return &zl, nil
+}