lib/promscrape/discovery/gce: allow empty project and zone for gce_sd_config

app/vmagent/README.md

@@ -135,7 +135,11 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
 * `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
   See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
-  `vmagent` supports empty `zone` arg inside `gce_sd_config` - in this case it autodetects all the zones for the given project.
+  `vmagent` provides the following additional functionality `gce_sd_config`:
+  * if `project` arg is missing, then `vmagent` uses the project for the instance where it runs;
+  * if `zone` arg is missing, then `vmagent` uses the zone for the instance where it runs;
+  * if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project;
+  * `zone` may contain arbitrary number of zones, i.e. `zone: [us-east1-a, us-east1-b]`.
 
 The following service discovery mechanisms will be added to `vmagent` soon:
 

docs/vmagent.md

@@ -135,7 +135,11 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
   See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details.
 * `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE).
   See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details.
-  `vmagent` supports empty `zone` arg inside `gce_sd_config` - in this case it autodetects all the zones for the given project.
+  `vmagent` provides the following additional functionality `gce_sd_config`:
+  * if `project` arg is missing, then `vmagent` uses the project for the instance where it runs;
+  * if `zone` arg is missing, then `vmagent` uses the zone for the instance where it runs;
+  * if `zone` arg equals to `"*"`, then `vmagent` discovers all the zones for the given project;
+  * `zone` may contain arbitrary number of zones, i.e. `zone: [us-east1-a, us-east1-b]`.
 
 The following service discovery mechanisms will be added to `vmagent` soon:
 

lib/promscrape/discovery/gce/api.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
 	"golang.org/x/oauth2/google"
 )
 
@@ -77,16 +78,32 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot create oauth2 client for gce: %s", err)
 	}
-	var zones []string
-	if len(sdc.Zone) == 0 {
-		// Autodetect zones for sdc.Project.
-		zs, err := getZonesForProject(client, sdc.Project, sdc.Filter)
+	project := sdc.Project
+	if len(project) == 0 {
+		proj, err := getCurrentProject()
 		if err != nil {
-			return nil, fmt.Errorf("cannot obtain zones for project %q: %s", sdc.Project, err)
+			return nil, fmt.Errorf("cannot determine the current project; make sure `vmagent` runs inside GCE; error: %s", err)
+		}
+		project = proj
+		logger.Infof("autodetected the current GCE project: %q", project)
+	}
+	zones := sdc.Zone.zones
+	if len(zones) == 0 {
+		// Autodetect the current zone.
+		zone, err := getCurrentZone()
+		if err != nil {
+			return nil, fmt.Errorf("cannot determine the current zone; make sure `vmagent` runs inside GCE; error: %s", err)
+		}
+		zones = append(zones, zone)
+		logger.Infof("autodetected the current GCE zone: %q", zone)
+	} else if len(zones) == 1 && zones[0] == "*" {
+		// Autodetect zones for project.
+		zs, err := getZonesForProject(client, project, sdc.Filter)
+		if err != nil {
+			return nil, fmt.Errorf("cannot obtain zones for project %q: %s", project, err)
 		}
 		zones = zs
-	} else {
-		zones = []string{sdc.Zone}
+		logger.Infof("autodetected all the zones for the GCE project %q: %q", project, zones)
 	}
 	tagSeparator := ","
 	if sdc.TagSeparator != nil {
@@ -99,7 +116,7 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) {
 	return &apiConfig{
 		client:       client,
 		zones:        zones,
-		project:      sdc.Project,
+		project:      project,
 		filter:       sdc.Filter,
 		tagSeparator: tagSeparator,
 		port:         port,
@@ -113,6 +130,10 @@ func getAPIResponse(client *http.Client, apiURL, filter, pageToken string) ([]by
 	if err != nil {
 		return nil, fmt.Errorf("cannot query %q: %s", apiURL, err)
 	}
+	return readResponseBody(resp, apiURL)
+}
+
+func readResponseBody(resp *http.Response, apiURL string) ([]byte, error) {
 	data, err := ioutil.ReadAll(resp.Body)
 	_ = resp.Body.Close()
 	if err != nil {
@@ -135,3 +156,40 @@ func appendNonEmptyQueryArg(apiURL, arg string) string {
 	}
 	return apiURL + fmt.Sprintf("%spageToken=%s", prefix, url.QueryEscape(arg))
 }
+
+func getCurrentZone() (string, error) {
+	// See https://cloud.google.com/compute/docs/storing-retrieving-metadata#default
+	data, err := getGCEMetadata("instance/zone")
+	if err != nil {
+		return "", err
+	}
+	parts := strings.Split(string(data), "/")
+	if len(parts) != 4 {
+		return "", fmt.Errorf("unexpected data returned from GCE; it must contain something like `projects/projectnum/zones/zone`; data: %q", data)
+	}
+	return parts[3], nil
+}
+
+func getCurrentProject() (string, error) {
+	// See https://cloud.google.com/compute/docs/storing-retrieving-metadata#default
+	data, err := getGCEMetadata("project/project-id")
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
+func getGCEMetadata(path string) ([]byte, error) {
+	// See https://cloud.google.com/compute/docs/storing-retrieving-metadata#default
+	metadataURL := "http://metadata.google.internal/computeMetadata/v1/" + path
+	req, err := http.NewRequest("GET", metadataURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create http request for %q: %s", metadataURL, err)
+	}
+	req.Header.Set("Metadata-Flavor", "Google")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot obtain response to %q: %s", metadataURL, err)
+	}
+	return readResponseBody(resp, metadataURL)
+}

lib/promscrape/discovery/gce/gce.go

@@ -8,15 +8,45 @@ import (
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config
 type SDConfig struct {
-	Project string `yaml:"project"`
-	Zone    string `yaml:"zone"`
-	Filter  string `yaml:"filter"`
+	Project string   `yaml:"project"`
+	Zone    ZoneYAML `yaml:"zone"`
+	Filter  string   `yaml:"filter"`
 	// RefreshInterval time.Duration `yaml:"refresh_interval"`
 	// refresh_interval is obtained from `-promscrape.gceSDCheckInterval` command-line option.
 	Port         *int    `yaml:"port"`
 	TagSeparator *string `yaml:"tag_separator"`
 }
 
+// ZoneYAML holds info about zones.
+type ZoneYAML struct {
+	zones []string
+}
+
+// UnmarshalYAML implements yaml.Unmarshaler
+func (z *ZoneYAML) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var v interface{}
+	if err := unmarshal(&v); err != nil {
+		return err
+	}
+	var zones []string
+	switch v.(type) {
+	case string:
+		zones = []string{v.(string)}
+	case []interface{}:
+		for _, vv := range v.([]interface{}) {
+			zone, ok := vv.(string)
+			if !ok {
+				return fmt.Errorf("unexpected zone type detected: %T; contents: %#v", vv, vv)
+			}
+			zones = append(zones, zone)
+		}
+	default:
+		return fmt.Errorf("unexpected type unmarshaled for ZoneYAML: %T; contents: %#v", v, v)
+	}
+	z.zones = zones
+	return nil
+}
+
 // GetLabels returns gce labels according to sdc.
 func GetLabels(sdc *SDConfig) ([]map[string]string, error) {
 	cfg, err := getAPIConfig(sdc)