diff --git a/README.md b/README.md index bedcb4456..3d5f54da0 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,7 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la * [static_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) * [file_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) * [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) +* [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) In the future other `*_sd_config` types will be supported. diff --git a/app/vmagent/README.md b/app/vmagent/README.md index 3fa50e711..0ed7a0721 100644 --- a/app/vmagent/README.md +++ b/app/vmagent/README.md @@ -133,11 +133,12 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details. * `kubernetes_sd_configs` - for scraping targets in Kubernetes (k8s). See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details. +* `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE). + See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details. The following service discovery mechanisms will be added to `vmagent` soon: * [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) -* [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) * [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) * [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) diff --git a/docs/Single-server-VictoriaMetrics.md b/docs/Single-server-VictoriaMetrics.md index bedcb4456..3d5f54da0 100644 --- a/docs/Single-server-VictoriaMetrics.md +++ b/docs/Single-server-VictoriaMetrics.md @@ -260,6 +260,7 @@ Currently the following [scrape_config](https://prometheus.io/docs/prometheus/la * [static_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config) * [file_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) * [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) +* [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) In the future other `*_sd_config` types will be supported. diff --git a/docs/vmagent.md b/docs/vmagent.md index 3fa50e711..0ed7a0721 100644 --- a/docs/vmagent.md +++ b/docs/vmagent.md @@ -133,11 +133,12 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh See [these docs](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config) for details. * `kubernetes_sd_configs` - for scraping targets in Kubernetes (k8s). See [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) for details. +* `gce_sd_configs` - for scraping targets in Google Compute Engine (GCE). + See [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) for details. The following service discovery mechanisms will be added to `vmagent` soon: * [ec2_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#ec2_sd_config) -* [gce_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config) * [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) * [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) diff --git a/go.mod b/go.mod index 3026814f0..2c4bf9f14 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/valyala/histogram v1.0.1 github.com/valyala/quicktemplate v1.4.1 golang.org/x/net v0.0.0-20200421231249-e086a090c8fd // indirect + golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f golang.org/x/tools v0.0.0-20200423205358-59e73619c742 // indirect google.golang.org/api v0.22.0 diff --git a/lib/promscrape/config.go b/lib/promscrape/config.go index 28ceda4d5..1a7939c58 100644 --- a/lib/promscrape/config.go +++ b/lib/promscrape/config.go @@ -14,6 +14,7 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes" "gopkg.in/yaml.v2" ) @@ -60,6 +61,7 @@ type ScrapeConfig struct { StaticConfigs []StaticConfig `yaml:"static_configs"` FileSDConfigs []FileSDConfig `yaml:"file_sd_configs"` KubernetesSDConfigs []kubernetes.SDConfig `yaml:"kubernetes_sd_configs"` + GCESDConfigs []gce.SDConfig `yaml:"gce_sd_configs"` RelabelConfigs []promrelabel.RelabelConfig `yaml:"relabel_configs"` MetricRelabelConfigs []promrelabel.RelabelConfig `yaml:"metric_relabel_configs"` SampleLimit int `yaml:"sample_limit"` @@ -147,6 +149,14 @@ func (cfg *Config) kubernetesSDConfigsCount() int { return n } +func (cfg *Config) gceSDConfigsCount() int { + n := 0 + for i := range cfg.ScrapeConfigs { + n += len(cfg.ScrapeConfigs[i].GCESDConfigs) + } + return n +} + func (cfg *Config) fileSDConfigsCount() int { n := 0 for i := range cfg.ScrapeConfigs { @@ -168,6 +178,19 @@ func (cfg *Config) getKubernetesSDScrapeWork() []ScrapeWork { return dst } +// getGCESDScrapeWork returns `gce_sd_configs` ScrapeWork from cfg. +func (cfg *Config) getGCESDScrapeWork() []ScrapeWork { + var dst []ScrapeWork + for i := range cfg.ScrapeConfigs { + sc := &cfg.ScrapeConfigs[i] + for j := range sc.GCESDConfigs { + sdc := &sc.GCESDConfigs[j] + dst = appendGCEScrapeWork(dst, sdc, sc.swc) + } + } + return dst +} + // getFileSDScrapeWork returns `file_sd_configs` ScrapeWork from cfg. func (cfg *Config) getFileSDScrapeWork(prev []ScrapeWork) []ScrapeWork { // Create a map for the previous scrape work. @@ -297,13 +320,25 @@ func appendKubernetesScrapeWork(dst []ScrapeWork, sdc *kubernetes.SDConfig, base logger.Errorf("error when discovering kubernetes nodes for `job_name` %q: %s; skipping it", swc.jobName, err) return dst } + return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "kubernetes_sd_config") +} + +func appendGCEScrapeWork(dst []ScrapeWork, sdc *gce.SDConfig, swc *scrapeWorkConfig) []ScrapeWork { + targetLabels, err := gce.GetLabels(sdc) + if err != nil { + logger.Errorf("error when discovering gce nodes for `job_name` %q: %s; skippint it", swc.jobName, err) + return dst + } + return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "gce_sd_config") +} + +func appendScrapeWorkForTargetLabels(dst []ScrapeWork, swc *scrapeWorkConfig, targetLabels []map[string]string, sectionName string) []ScrapeWork { for _, metaLabels := range targetLabels { target := metaLabels["__address__"] var err error dst, err = appendScrapeWork(dst, swc, target, nil, metaLabels) if err != nil { - logger.Errorf("error when parsing `kubernetes_sd_config` target %q with role %q for `job_name` %q: %s; skipping it", - target, sdc.Role, swc.jobName, err) + logger.Errorf("error when parsing `%s` target %q for `job_name` %q: %s; skipping it", sectionName, target, swc.jobName, err) continue } } diff --git a/lib/promscrape/discovery/gce/api.go b/lib/promscrape/discovery/gce/api.go new file mode 100644 index 000000000..96da0e011 --- /dev/null +++ b/lib/promscrape/discovery/gce/api.go @@ -0,0 +1,97 @@ +package gce + +import ( + "context" + "fmt" + "net/http" + "net/url" + "sync" + "time" + + "golang.org/x/oauth2/google" +) + +type apiConfig struct { + client *http.Client + apiURL string + project string + tagSeparator string + port int +} + +func getAPIConfig(sdc *SDConfig) (*apiConfig, error) { + apiConfigMapLock.Lock() + defer apiConfigMapLock.Unlock() + + if !hasAPIConfigMapCleaner { + hasAPIConfigMapCleaner = true + go apiConfigMapCleaner() + } + + e := apiConfigMap[sdc] + if e != nil { + e.lastAccessTime = time.Now() + return e.cfg, nil + } + cfg, err := newAPIConfig(sdc) + if err != nil { + return nil, err + } + apiConfigMap[sdc] = &apiConfigMapEntry{ + cfg: cfg, + lastAccessTime: time.Now(), + } + return cfg, nil +} + +func apiConfigMapCleaner() { + tc := time.NewTicker(15 * time.Minute) + for currentTime := range tc.C { + apiConfigMapLock.Lock() + for k, e := range apiConfigMap { + if currentTime.Sub(e.lastAccessTime) > 10*time.Minute { + delete(apiConfigMap, k) + } + } + apiConfigMapLock.Unlock() + } +} + +type apiConfigMapEntry struct { + cfg *apiConfig + lastAccessTime time.Time +} + +var ( + apiConfigMap = make(map[*SDConfig]*apiConfigMapEntry) + apiConfigMapLock sync.Mutex + hasAPIConfigMapCleaner bool +) + +func newAPIConfig(sdc *SDConfig) (*apiConfig, error) { + ctx := context.Background() + client, err := google.DefaultClient(ctx, "https://www.googleapis.com/auth/compute.readonly") + if err != nil { + return nil, fmt.Errorf("cannot create oauth2 client for gce: %s", err) + } + // See https://cloud.google.com/compute/docs/reference/rest/v1/instances/list + apiURL := fmt.Sprintf("https://compute.googleapis.com/compute/v1/projects/%s/zones/%s/instances", sdc.Project, sdc.Zone) + if len(sdc.Filter) > 0 { + apiURL += fmt.Sprintf("?filter=%s", url.QueryEscape(sdc.Filter)) + } + tagSeparator := "," + if sdc.TagSeparator != nil { + tagSeparator = *sdc.TagSeparator + } + port := 80 + if sdc.Port != nil { + port = *sdc.Port + } + return &apiConfig{ + client: client, + apiURL: apiURL, + project: sdc.Project, + tagSeparator: tagSeparator, + port: port, + }, nil +} diff --git a/lib/promscrape/discovery/gce/gce.go b/lib/promscrape/discovery/gce/gce.go new file mode 100644 index 000000000..ea68627f6 --- /dev/null +++ b/lib/promscrape/discovery/gce/gce.go @@ -0,0 +1,31 @@ +package gce + +import ( + "fmt" +) + +// SDConfig represents service discovery config for gce. +// +// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config +type SDConfig struct { + Project string `yaml:"project"` + Zone string `yaml:"zone"` + Filter string `yaml:"filter"` + // RefreshInterval time.Duration `yaml:"refresh_interval"` + // refresh_interval is obtained from `-promscrape.gceSDCheckInterval` command-line option. + Port *int `yaml:"port"` + TagSeparator *string `yaml:"tag_separator"` +} + +// GetLabels returns gce labels according to sdc. +func GetLabels(sdc *SDConfig) ([]map[string]string, error) { + cfg, err := getAPIConfig(sdc) + if err != nil { + return nil, fmt.Errorf("cannot get API config: %s", err) + } + ms, err := getInstancesLabels(cfg) + if err != nil { + return nil, fmt.Errorf("error when fetching instances data from GCE: %s", err) + } + return ms, nil +} diff --git a/lib/promscrape/discovery/gce/instance.go b/lib/promscrape/discovery/gce/instance.go new file mode 100644 index 000000000..d76416672 --- /dev/null +++ b/lib/promscrape/discovery/gce/instance.go @@ -0,0 +1,170 @@ +package gce + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" +) + +// getInstancesLabels returns labels for gce instances obtained from the given cfg +func getInstancesLabels(cfg *apiConfig) ([]map[string]string, error) { + insts, err := getInstances(cfg) + if err != nil { + return nil, err + } + var ms []map[string]string + for _, inst := range insts { + ms = inst.appendTargetLabels(ms, cfg.project, cfg.tagSeparator, cfg.port) + } + return ms, nil +} + +func getInstances(cfg *apiConfig) ([]Instance, error) { + var result []Instance + pageToken := "" + for { + insts, nextPageToken, err := getInstancesPage(cfg, pageToken) + if err != nil { + return nil, err + } + result = append(result, insts...) + if len(nextPageToken) == 0 { + return result, nil + } + pageToken = nextPageToken + } +} + +func getInstancesPage(cfg *apiConfig, pageToken string) ([]Instance, string, error) { + apiURL := cfg.apiURL + if len(pageToken) > 0 { + // See https://cloud.google.com/compute/docs/reference/rest/v1/instances/list about pageToken + prefix := "?" + if strings.Contains(apiURL, "?") { + prefix = "&" + } + apiURL += fmt.Sprintf("%spageToken=%s", prefix, url.QueryEscape(pageToken)) + } + resp, err := cfg.client.Get(apiURL) + if err != nil { + return nil, "", fmt.Errorf("cannot obtain instances data from API server: %s", err) + } + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, "", fmt.Errorf("cannot read instances data from API server: %s", err) + } + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("unexpected status code when reading instances data from API server; got %d; want %d; response body: %q", + resp.StatusCode, http.StatusOK, data) + } + il, err := parseInstanceList(data) + if err != nil { + return nil, "", fmt.Errorf("cannot parse instances response from API server: %s", err) + } + return il.Items, il.NextPageToken, nil +} + +// InstanceList is response to https://cloud.google.com/compute/docs/reference/rest/v1/instances/list +type InstanceList struct { + Items []Instance + NextPageToken string +} + +// Instance is instance from https://cloud.google.com/compute/docs/reference/rest/v1/instances/list +type Instance struct { + ID string `json:"id"` + Name string + Status string + MachineType string + Zone string + NetworkInterfaces []NetworkInterface + Tags TagList + Metadata MetadataList + Labels discoveryutils.SortedLabels +} + +// NetworkInterface is network interface from https://cloud.google.com/compute/docs/reference/rest/v1/instances/list +type NetworkInterface struct { + Network string + Subnetwork string + NetworkIP string + AccessConfigs []AccessConfig +} + +// AccessConfig is access config from https://cloud.google.com/compute/docs/reference/rest/v1/instances/list +type AccessConfig struct { + Type string + NatIP string +} + +// TagList is tag list from https://cloud.google.com/compute/docs/reference/rest/v1/instances/list +type TagList struct { + Items []string +} + +// MetadataList is metadataList from https://cloud.google.com/compute/docs/reference/rest/v1/instances/list +type MetadataList struct { + Items []MetadataEntry +} + +// MetadataEntry is a single entry from metadata +type MetadataEntry struct { + Key string + Value string +} + +// parseInstanceList parses InstanceList from data. +func parseInstanceList(data []byte) (*InstanceList, error) { + var il InstanceList + if err := json.Unmarshal(data, &il); err != nil { + return nil, fmt.Errorf("cannot unmarshal InstanceList from %q: %s", data, err) + } + return &il, nil +} + +func (inst *Instance) appendTargetLabels(ms []map[string]string, project, tagSeparator string, port int) []map[string]string { + if len(inst.NetworkInterfaces) == 0 { + return ms + } + iface := inst.NetworkInterfaces[0] + addr := discoveryutils.JoinHostPort(iface.NetworkIP, port) + m := map[string]string{ + "__address__": addr, + "__meta_gce_instance_id": inst.ID, + "__meta_gce_instance_status": inst.Status, + "__meta_gce_instance_name": inst.Name, + "__meta_gce_machine_type": inst.MachineType, + "__meta_gce_network": iface.Network, + "__meta_gce_private_ip": iface.NetworkIP, + "__meta_gce_project": project, + "__meta_gce_subnetwork": iface.Subnetwork, + "__meta_gce_zone": inst.Zone, + } + if len(inst.Tags.Items) > 0 { + // We surround the separated list with the separator as well. This way regular expressions + // in relabeling rules don't have to consider tag positions. + m["__meta_gce_tags"] = tagSeparator + strings.Join(inst.Tags.Items, tagSeparator) + tagSeparator + } + for _, item := range inst.Metadata.Items { + key := discoveryutils.SanitizeLabelName(item.Key) + m["__meta_gce_metadata_"+key] = item.Value + } + for _, label := range inst.Labels { + name := discoveryutils.SanitizeLabelName(label.Name) + m["__meta_gce_label_"+name] = label.Value + } + if len(iface.AccessConfigs) > 0 { + ac := iface.AccessConfigs[0] + if ac.Type == "ONE_TO_ONE_NAT" { + m["__meta_gce_public_ip"] = ac.NatIP + } + } + ms = append(ms, m) + return ms +} diff --git a/lib/promscrape/discovery/gce/instance_test.go b/lib/promscrape/discovery/gce/instance_test.go new file mode 100644 index 000000000..a5426d81e --- /dev/null +++ b/lib/promscrape/discovery/gce/instance_test.go @@ -0,0 +1,179 @@ +package gce + +import ( + "reflect" + "testing" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" +) + +func TestParseInstanceListFailure(t *testing.T) { + f := func(s string) { + t.Helper() + il, err := parseInstanceList([]byte(s)) + if err == nil { + t.Fatalf("expecting non-nil error") + } + if il != nil { + t.Fatalf("unexpected non-nil InstanceList: %v", il) + } + } + f(``) + f(`[1,23]`) + f(`{"items":[{"metadata":1}]}`) +} + +func TestParseInstanceListSuccess(t *testing.T) { + data := ` +{ + "id": "projects/victoriametrics-test/zones/us-east1-b/instances", + "items": [ + { + "id": "7897352091592122", + "creationTimestamp": "2020-02-16T07:10:14.357-08:00", + "name": "play-1m-1-vmagent", + "tags": { + "items": [ + "play", + "play-1m-1", + "vmagent" + ], + "fingerprint": "O44NvJ36CCo=" + }, + "machineType": "https://www.googleapis.com/compute/v1/projects/victoriametrics-test/zones/us-east1-b/machineTypes/f1-micro", + "status": "RUNNING", + "zone": "https://www.googleapis.com/compute/v1/projects/victoriametrics-test/zones/us-east1-b", + "networkInterfaces": [ + { + "network": "https://www.googleapis.com/compute/v1/projects/victoriametrics-test/global/networks/default", + "subnetwork": "https://www.googleapis.com/compute/v1/projects/victoriametrics-test/regions/us-east1/subnetworks/play-1m-1-snw", + "networkIP": "10.11.2.7", + "name": "nic0", + "fingerprint": "O4eNOfaplJ4=", + "kind": "compute#networkInterface" + } + ], + "disks": [ + { + "type": "PERSISTENT", + "mode": "READ_WRITE", + "source": "https://www.googleapis.com/compute/v1/projects/victoriametrics-test/zones/us-east1-b/disks/play-1m-1-vmagent", + "deviceName": "boot", + "index": 0, + "boot": true, + "autoDelete": true, + "licenses": [ + "https://www.googleapis.com/compute/v1/projects/cos-cloud-shielded/global/licenses/shielded-cos", + "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/licenses/cos", + "https://www.googleapis.com/compute/v1/projects/cos-cloud/global/licenses/cos-pcid" + ], + "interface": "SCSI", + "guestOsFeatures": [ + { + "type": "VIRTIO_SCSI_MULTIQUEUE" + }, + { + "type": "UEFI_COMPATIBLE" + } + ], + "diskSizeGb": "10", + "kind": "compute#attachedDisk" + } + ], + "metadata": { + "fingerprint": "BAFZwTyaAxQ=", + "items": [ + { + "key": "gce-container-declaration", + "value": "foobar" + } + ], + "kind": "compute#metadata" + }, + "serviceAccounts": [ + { + "email": "12-compute@developer.gserviceaccount.com", + "scopes": [ + "https://www.googleapis.com/auth/devstorage.read_write", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly" + ] + } + ], + "selfLink": "https://www.googleapis.com/compute/v1/projects/victoriametrics-test/zones/us-east1-b/instances/play-1m-1-vmagent", + "scheduling": { + "onHostMaintenance": "MIGRATE", + "automaticRestart": true, + "preemptible": false + }, + "cpuPlatform": "Intel Haswell", + "labels": { + "goog-dm": "play-deployment", + "cluster_num": "1", + "cluster_retention": "1m", + "env": "play", + "type": "vmagent" + }, + "labelFingerprint": "-CXeRXMQiVc=", + "startRestricted": false, + "deletionProtection": false, + "shieldedInstanceConfig": { + "enableSecureBoot": false, + "enableVtpm": true, + "enableIntegrityMonitoring": true + }, + "shieldedInstanceIntegrityPolicy": { + "updateAutoLearnPolicy": true + }, + "fingerprint": "hd3NB2-9QIg=", + "kind": "compute#instance" + } + ] +} +` + il, err := parseInstanceList([]byte(data)) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if len(il.Items) != 1 { + t.Fatalf("unexpected length of InstanceList.Items; got %d; want %d", len(il.Items), 1) + } + inst := il.Items[0] + + // Check inst.appendTargetLabels() + project := "proj-1" + tagSeparator := "," + port := 80 + labelss := inst.appendTargetLabels(nil, project, tagSeparator, port) + var sortedLabelss [][]prompbmarshal.Label + for _, labels := range labelss { + sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels)) + } + expectedLabelss := [][]prompbmarshal.Label{ + discoveryutils.GetSortedLabels(map[string]string{ + "__address__": "10.11.2.7:80", + "__meta_gce_instance_id": "7897352091592122", + "__meta_gce_instance_name": "play-1m-1-vmagent", + "__meta_gce_instance_status": "RUNNING", + "__meta_gce_label_cluster_num": "1", + "__meta_gce_label_cluster_retention": "1m", + "__meta_gce_label_env": "play", + "__meta_gce_label_goog_dm": "play-deployment", + "__meta_gce_label_type": "vmagent", + "__meta_gce_machine_type": "https://www.googleapis.com/compute/v1/projects/victoriametrics-test/zones/us-east1-b/machineTypes/f1-micro", + "__meta_gce_metadata_gce_container_declaration": "foobar", + "__meta_gce_network": "https://www.googleapis.com/compute/v1/projects/victoriametrics-test/global/networks/default", + "__meta_gce_private_ip": "10.11.2.7", + "__meta_gce_project": "proj-1", + "__meta_gce_subnetwork": "https://www.googleapis.com/compute/v1/projects/victoriametrics-test/regions/us-east1/subnetworks/play-1m-1-snw", + "__meta_gce_tags": ",play,play-1m-1,vmagent,", + "__meta_gce_zone": "https://www.googleapis.com/compute/v1/projects/victoriametrics-test/zones/us-east1-b", + }), + } + if !reflect.DeepEqual(sortedLabelss, expectedLabelss) { + t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, expectedLabelss) + } +} diff --git a/lib/promscrape/discovery/kubernetes/common_types.go b/lib/promscrape/discovery/kubernetes/common_types.go index 38c483312..039fce6a9 100644 --- a/lib/promscrape/discovery/kubernetes/common_types.go +++ b/lib/promscrape/discovery/kubernetes/common_types.go @@ -1,13 +1,10 @@ package kubernetes import ( - "encoding/json" "fmt" "net/url" - "sort" "strings" - "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" ) @@ -18,8 +15,8 @@ type ObjectMeta struct { Name string Namespace string UID string - Labels SortedLabels - Annotations SortedLabels + Labels discoveryutils.SortedLabels + Annotations discoveryutils.SortedLabels OwnerReferences []OwnerReference } @@ -36,33 +33,6 @@ func (om *ObjectMeta) registerLabelsAndAnnotations(prefix string, m map[string]s } } -// SortedLabels represents sorted labels. -type SortedLabels []prompbmarshal.Label - -// UnmarshalJSON unmarshals JSON from data. -func (sls *SortedLabels) UnmarshalJSON(data []byte) error { - var m map[string]string - if err := json.Unmarshal(data, &m); err != nil { - return err - } - *sls = getSortedLabels(m) - return nil -} - -func getSortedLabels(m map[string]string) SortedLabels { - a := make([]prompbmarshal.Label, 0, len(m)) - for k, v := range m { - a = append(a, prompbmarshal.Label{ - Name: k, - Value: v, - }) - } - sort.Slice(a, func(i, j int) bool { - return a[i].Name < a[j].Name - }) - return a -} - // OwnerReference represents OwnerReferense from k8s API. // // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ownerreference-v1-meta diff --git a/lib/promscrape/discovery/kubernetes/endpoints_test.go b/lib/promscrape/discovery/kubernetes/endpoints_test.go index cb8bb698e..0decc0800 100644 --- a/lib/promscrape/discovery/kubernetes/endpoints_test.go +++ b/lib/promscrape/discovery/kubernetes/endpoints_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" ) func TestParseEndpointsListFailure(t *testing.T) { @@ -85,10 +86,10 @@ func TestParseEndpointsListSuccess(t *testing.T) { labelss := endpoint.appendTargetLabels(nil, nil, nil) var sortedLabelss [][]prompbmarshal.Label for _, labels := range labelss { - sortedLabelss = append(sortedLabelss, getSortedLabels(labels)) + sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels)) } expectedLabelss := [][]prompbmarshal.Label{ - getSortedLabels(map[string]string{ + discoveryutils.GetSortedLabels(map[string]string{ "__address__": "172.17.0.2:8443", "__meta_kubernetes_endpoint_address_target_kind": "Pod", "__meta_kubernetes_endpoint_address_target_name": "coredns-6955765f44-lnp6t", diff --git a/lib/promscrape/discovery/kubernetes/ingress_test.go b/lib/promscrape/discovery/kubernetes/ingress_test.go index 8a1fbae75..575abb5f8 100644 --- a/lib/promscrape/discovery/kubernetes/ingress_test.go +++ b/lib/promscrape/discovery/kubernetes/ingress_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" ) func TestParseIngressListFailure(t *testing.T) { @@ -83,10 +84,10 @@ func TestParseIngressListSuccess(t *testing.T) { labelss := ig.appendTargetLabels(nil) var sortedLabelss [][]prompbmarshal.Label for _, labels := range labelss { - sortedLabelss = append(sortedLabelss, getSortedLabels(labels)) + sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels)) } expectedLabelss := [][]prompbmarshal.Label{ - getSortedLabels(map[string]string{ + discoveryutils.GetSortedLabels(map[string]string{ "__address__": "foobar", "__meta_kubernetes_ingress_annotation_kubectl_kubernetes_io_last_applied_configuration": `{"apiVersion":"networking.k8s.io/v1beta1","kind":"Ingress","metadata":{"annotations":{},"name":"test-ingress","namespace":"default"},"spec":{"backend":{"serviceName":"testsvc","servicePort":80}}}` + "\n", "__meta_kubernetes_ingress_annotationpresent_kubectl_kubernetes_io_last_applied_configuration": "true", diff --git a/lib/promscrape/discovery/kubernetes/kubernetes.go b/lib/promscrape/discovery/kubernetes/kubernetes.go index 1c1b212be..fd1af2a3e 100644 --- a/lib/promscrape/discovery/kubernetes/kubernetes.go +++ b/lib/promscrape/discovery/kubernetes/kubernetes.go @@ -6,30 +6,6 @@ import ( "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" ) -// GetLabels returns labels for the given k8s role and the given cfg. -func GetLabels(ac *promauth.Config, sdc *SDConfig) ([]map[string]string, error) { - cfg := &apiConfig{ - Server: sdc.APIServer, - AuthConfig: ac, - Namespaces: sdc.Namespaces.Names, - Selectors: sdc.Selectors, - } - switch sdc.Role { - case "node": - return getNodesLabels(cfg) - case "service": - return getServicesLabels(cfg) - case "pod": - return getPodsLabels(cfg) - case "endpoints": - return getEndpointsLabels(cfg) - case "ingress": - return getIngressesLabels(cfg) - default: - return nil, fmt.Errorf("unexpected `role`: %q; must be one of `node`, `service`, `pod`, `endpoints` or `ingress`; skipping it", sdc.Role) - } -} - // SDConfig represents kubernetes-based service discovery config. // // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config @@ -58,3 +34,27 @@ type Selector struct { Label string `yaml:"label"` Field string `yaml:"field"` } + +// GetLabels returns labels for the given k8s role and the given ac and sdc. +func GetLabels(ac *promauth.Config, sdc *SDConfig) ([]map[string]string, error) { + cfg := &apiConfig{ + Server: sdc.APIServer, + AuthConfig: ac, + Namespaces: sdc.Namespaces.Names, + Selectors: sdc.Selectors, + } + switch sdc.Role { + case "node": + return getNodesLabels(cfg) + case "service": + return getServicesLabels(cfg) + case "pod": + return getPodsLabels(cfg) + case "endpoints": + return getEndpointsLabels(cfg) + case "ingress": + return getIngressesLabels(cfg) + default: + return nil, fmt.Errorf("unexpected `role`: %q; must be one of `node`, `service`, `pod`, `endpoints` or `ingress`; skipping it", sdc.Role) + } +} diff --git a/lib/promscrape/discovery/kubernetes/node_test.go b/lib/promscrape/discovery/kubernetes/node_test.go index 00ce2c89e..c632b8123 100644 --- a/lib/promscrape/discovery/kubernetes/node_test.go +++ b/lib/promscrape/discovery/kubernetes/node_test.go @@ -3,6 +3,8 @@ package kubernetes import ( "reflect" "testing" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" ) func TestParseNodeListFailure(t *testing.T) { @@ -236,7 +238,7 @@ func TestParseNodeListSuccess(t *testing.T) { if meta.Name != "m01" { t.Fatalf("unexpected ObjectMeta.Name; got %q; want %q", meta.Name, "m01") } - expectedLabels := getSortedLabels(map[string]string{ + expectedLabels := discoveryutils.GetSortedLabels(map[string]string{ "beta.kubernetes.io/arch": "amd64", "beta.kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64", @@ -251,7 +253,7 @@ func TestParseNodeListSuccess(t *testing.T) { if !reflect.DeepEqual(meta.Labels, expectedLabels) { t.Fatalf("unexpected ObjectMeta.Labels\ngot\n%v\nwant\n%v", meta.Labels, expectedLabels) } - expectedAnnotations := getSortedLabels(map[string]string{ + expectedAnnotations := discoveryutils.GetSortedLabels(map[string]string{ "kubeadm.alpha.kubernetes.io/cri-socket": "/var/run/dockershim.sock", "node.alpha.kubernetes.io/ttl": "0", "volumes.kubernetes.io/controller-managed-attach-detach": "true", @@ -275,8 +277,8 @@ func TestParseNodeListSuccess(t *testing.T) { } // Check node.appendTargetLabels() - labels := getSortedLabels(node.appendTargetLabels(nil)[0]) - expectedLabels = getSortedLabels(map[string]string{ + labels := discoveryutils.GetSortedLabels(node.appendTargetLabels(nil)[0]) + expectedLabels = discoveryutils.GetSortedLabels(map[string]string{ "instance": "m01", "__address__": "172.17.0.2:10250", "__meta_kubernetes_node_name": "m01", diff --git a/lib/promscrape/discovery/kubernetes/pod_test.go b/lib/promscrape/discovery/kubernetes/pod_test.go index fb3842828..7462f0014 100644 --- a/lib/promscrape/discovery/kubernetes/pod_test.go +++ b/lib/promscrape/discovery/kubernetes/pod_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" ) func TestParsePodListFailure(t *testing.T) { @@ -240,10 +241,10 @@ func TestParsePodListSuccess(t *testing.T) { labelss := pod.appendTargetLabels(nil) var sortedLabelss [][]prompbmarshal.Label for _, labels := range labelss { - sortedLabelss = append(sortedLabelss, getSortedLabels(labels)) + sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels)) } expectedLabels := [][]prompbmarshal.Label{ - getSortedLabels(map[string]string{ + discoveryutils.GetSortedLabels(map[string]string{ "__address__": "172.17.0.2:1234", "__meta_kubernetes_namespace": "kube-system", diff --git a/lib/promscrape/discovery/kubernetes/service_test.go b/lib/promscrape/discovery/kubernetes/service_test.go index caf7215e3..2f3c1f7e7 100644 --- a/lib/promscrape/discovery/kubernetes/service_test.go +++ b/lib/promscrape/discovery/kubernetes/service_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" ) func TestParseServiceListFailure(t *testing.T) { @@ -100,7 +101,7 @@ func TestParseServiceListSuccess(t *testing.T) { if meta.Name != "kube-dns" { t.Fatalf("unexpected ObjectMeta.Name; got %q; want %q", meta.Name, "kube-dns") } - expectedLabels := getSortedLabels(map[string]string{ + expectedLabels := discoveryutils.GetSortedLabels(map[string]string{ "k8s-app": "kube-dns", "kubernetes.io/cluster-service": "true", "kubernetes.io/name": "KubeDNS", @@ -108,7 +109,7 @@ func TestParseServiceListSuccess(t *testing.T) { if !reflect.DeepEqual(meta.Labels, expectedLabels) { t.Fatalf("unexpected ObjectMeta.Labels\ngot\n%v\nwant\n%v", meta.Labels, expectedLabels) } - expectedAnnotations := getSortedLabels(map[string]string{ + expectedAnnotations := discoveryutils.GetSortedLabels(map[string]string{ "prometheus.io/port": "9153", "prometheus.io/scrape": "true", }) @@ -148,10 +149,10 @@ func TestParseServiceListSuccess(t *testing.T) { labelss := service.appendTargetLabels(nil) var sortedLabelss [][]prompbmarshal.Label for _, labels := range labelss { - sortedLabelss = append(sortedLabelss, getSortedLabels(labels)) + sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels)) } expectedLabelss := [][]prompbmarshal.Label{ - getSortedLabels(map[string]string{ + discoveryutils.GetSortedLabels(map[string]string{ "__address__": "kube-dns.kube-system.svc:53", "__meta_kubernetes_namespace": "kube-system", "__meta_kubernetes_service_name": "kube-dns", @@ -174,7 +175,7 @@ func TestParseServiceListSuccess(t *testing.T) { "__meta_kubernetes_service_annotationpresent_prometheus_io_port": "true", "__meta_kubernetes_service_annotationpresent_prometheus_io_scrape": "true", }), - getSortedLabels(map[string]string{ + discoveryutils.GetSortedLabels(map[string]string{ "__address__": "kube-dns.kube-system.svc:53", "__meta_kubernetes_namespace": "kube-system", "__meta_kubernetes_service_name": "kube-dns", @@ -197,7 +198,7 @@ func TestParseServiceListSuccess(t *testing.T) { "__meta_kubernetes_service_annotationpresent_prometheus_io_port": "true", "__meta_kubernetes_service_annotationpresent_prometheus_io_scrape": "true", }), - getSortedLabels(map[string]string{ + discoveryutils.GetSortedLabels(map[string]string{ "__address__": "kube-dns.kube-system.svc:9153", "__meta_kubernetes_namespace": "kube-system", "__meta_kubernetes_service_name": "kube-dns", diff --git a/lib/promscrape/discoveryutils/utils.go b/lib/promscrape/discoveryutils/utils.go index 58b21e306..18c7d00e0 100644 --- a/lib/promscrape/discoveryutils/utils.go +++ b/lib/promscrape/discoveryutils/utils.go @@ -1,9 +1,13 @@ package discoveryutils import ( + "encoding/json" "net" "regexp" + "sort" "strconv" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal" ) // SanitizeLabelName replaces anything that doesn't match @@ -25,3 +29,31 @@ func JoinHostPort(host string, port int) string { portStr := strconv.Itoa(port) return net.JoinHostPort(host, portStr) } + +// SortedLabels represents sorted labels. +type SortedLabels []prompbmarshal.Label + +// UnmarshalJSON unmarshals JSON from data. +func (sls *SortedLabels) UnmarshalJSON(data []byte) error { + var m map[string]string + if err := json.Unmarshal(data, &m); err != nil { + return err + } + *sls = GetSortedLabels(m) + return nil +} + +// GetSortedLabels returns SortedLabels built from m. +func GetSortedLabels(m map[string]string) SortedLabels { + a := make([]prompbmarshal.Label, 0, len(m)) + for k, v := range m { + a = append(a, prompbmarshal.Label{ + Name: k, + Value: v, + }) + } + sort.Slice(a, func(i, j int) bool { + return a[i].Name < a[j].Name + }) + return a +} diff --git a/lib/promscrape/scraper.go b/lib/promscrape/scraper.go index 643d55d74..32d96fbb0 100644 --- a/lib/promscrape/scraper.go +++ b/lib/promscrape/scraper.go @@ -18,10 +18,13 @@ var ( configCheckInterval = flag.Duration("promscrape.configCheckInterval", 0, "Interval for checking for changes in '-promscrape.config' file. "+ "By default the checking is disabled. Send SIGHUP signal in order to force config check for changes") fileSDCheckInterval = flag.Duration("promscrape.fileSDCheckInterval", 30*time.Second, "Interval for checking for changes in 'file_sd_config'. "+ - "See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config") + "See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config for details") kubernetesSDCheckInterval = flag.Duration("promscrape.kubernetesSDCheckInterval", 30*time.Second, "Interval for checking for changes in Kubernetes API server. "+ "This works only if `kubernetes_sd_configs` is configured in '-promscrape.config' file. "+ "See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details") + gceSDCheckInterval = flag.Duration("promscrape.gceSDCheckInterval", time.Minute, "Interval for checking for changes in gce. "+ + "This works only if `gce_sd_configs` is configured in '-promscrape.config' file. "+ + "See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details") promscrapeConfigFile = flag.String("promscrape.config", "", "Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. "+ "See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config for details") ) @@ -89,6 +92,11 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest) defer wg.Done() runKubernetesSDScrapers(cfg, pushData, stopCh) }() + wg.Add(1) + go func() { + defer wg.Done() + runGCESDScrapers(cfg, pushData, stopCh) + }() waitForChans: select { @@ -194,6 +202,51 @@ var ( kubernetesSDReloads = metrics.NewCounter(`vm_promscrape_reloads_total{type="kubernetes_sd"}`) ) +func runGCESDScrapers(cfg *Config, pushData func(wr *prompbmarshal.WriteRequest), stopCh <-chan struct{}) { + if cfg.gceSDConfigsCount() == 0 { + return + } + sws := cfg.getGCESDScrapeWork() + ticker := time.NewTicker(*gceSDCheckInterval) + defer ticker.Stop() + mustStop := false + for !mustStop { + localStopCh := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func(sws []ScrapeWork) { + defer wg.Done() + logger.Infof("starting %d scrapers for `gce_sd_config` targets", len(sws)) + gceSDTargets.Set(uint64(len(sws))) + runScrapeWorkers(sws, pushData, localStopCh) + gceSDTargets.Set(0) + logger.Infof("stopped all the %d scrapers for `gce_sd_config` targets", len(sws)) + }(sws) + waitForChans: + select { + case <-ticker.C: + swsNew := cfg.getGCESDScrapeWork() + if equalStaticConfigForScrapeWorks(swsNew, sws) { + // Nothing changed, continue waiting for updated scrape work + goto waitForChans + } + logger.Infof("restarting scrapers for changed `gce_sd_config` targets") + sws = swsNew + case <-stopCh: + mustStop = true + } + + close(localStopCh) + wg.Wait() + gceSDReloads.Inc() + } +} + +var ( + gceSDTargets = metrics.NewCounter(`vm_promscrape_targets{type="gce_sd"}`) + gceSDReloads = metrics.NewCounter(`vm_promscrape_reloads_total{type="gce_sd"}`) +) + func runFileSDScrapers(cfg *Config, pushData func(wr *prompbmarshal.WriteRequest), stopCh <-chan struct{}) { if cfg.fileSDConfigsCount() == 0 { return