diff --git a/lib/awsapi/config.go b/lib/awsapi/config.go index c4106a0691..522c582da5 100644 --- a/lib/awsapi/config.go +++ b/lib/awsapi/config.go @@ -87,6 +87,7 @@ func NewConfig(region, roleARN, accessKey, secretKey string) (*Config, error) { // GetEC2APIResponse performs EC2 API request with ghe given action. // // filtersQueryString must contain an optional percent-encoded query string for aws filters. +// This string can be obtained by calling GetFiltersQueryString(). // See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html for examples. // See also https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Filter.html func (cfg *Config) GetEC2APIResponse(action, filtersQueryString, nextPageToken string) ([]byte, error) { @@ -424,3 +425,30 @@ func buildAPIEndpoint(customEndpoint, region, service string) string { } return endpoint } + +// GetFiltersQueryString returns query string formed from the given filters. +// +// If whitelist isn't nil, then filters which don't fall into whitelist isn't returned. +func GetFiltersQueryString(filters []Filter, whitelist map[string]bool) string { + // See how to build filters query string at examples at https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html + var args []string + for i, f := range filters { + if whitelist != nil && !whitelist[f.Name] { + continue + } + args = append(args, fmt.Sprintf("Filter.%d.Name=%s", i+1, url.QueryEscape(f.Name))) + for j, v := range f.Values { + args = append(args, fmt.Sprintf("Filter.%d.Value.%d=%s", i+1, j+1, url.QueryEscape(v))) + } + } + return strings.Join(args, "&") +} + +// Filter is ec2 filter. +// +// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html +// and https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Filter.html +type Filter struct { + Name string `yaml:"name"` + Values []string `yaml:"values"` +} diff --git a/lib/promscrape/discovery/ec2/api.go b/lib/promscrape/discovery/ec2/api.go index 841af481a5..014cea0863 100644 --- a/lib/promscrape/discovery/ec2/api.go +++ b/lib/promscrape/discovery/ec2/api.go @@ -1,9 +1,6 @@ package ec2 import ( - "fmt" - "net/url" - "strings" "sync" "github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi" @@ -11,9 +8,9 @@ import ( ) type apiConfig struct { - awsConfig *awsapi.Config - filtersQueryString string - port int + awsConfig *awsapi.Config + filters []awsapi.Filter + port int // A map from AZ name to AZ id. azMap map[string]string @@ -31,7 +28,6 @@ func getAPIConfig(sdc *SDConfig) (*apiConfig, error) { } func newAPIConfig(sdc *SDConfig) (*apiConfig, error) { - fqs := getFiltersQueryString(sdc.Filters) port := 80 if sdc.Port != nil { port = *sdc.Port @@ -41,21 +37,9 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) { return nil, err } cfg := &apiConfig{ - awsConfig: awsCfg, - filtersQueryString: fqs, - port: port, + awsConfig: awsCfg, + filters: sdc.Filters, + port: port, } return cfg, nil } - -func getFiltersQueryString(filters []Filter) string { - // See how to build filters query string at examples at https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html - var args []string - for i, f := range filters { - args = append(args, fmt.Sprintf("Filter.%d.Name=%s", i+1, url.QueryEscape(f.Name))) - for j, v := range f.Values { - args = append(args, fmt.Sprintf("Filter.%d.Value.%d=%s", i+1, j+1, url.QueryEscape(v))) - } - } - return strings.Join(args, "&") -} diff --git a/lib/promscrape/discovery/ec2/az.go b/lib/promscrape/discovery/ec2/az.go index bd096660e7..ed299c4549 100644 --- a/lib/promscrape/discovery/ec2/az.go +++ b/lib/promscrape/discovery/ec2/az.go @@ -4,6 +4,7 @@ import ( "encoding/xml" "fmt" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" ) @@ -29,7 +30,8 @@ func getAZMap(cfg *apiConfig) map[string]string { func getAvailabilityZones(cfg *apiConfig) ([]AvailabilityZone, error) { // See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html - data, err := cfg.awsConfig.GetEC2APIResponse("DescribeAvailabilityZones", "", "") + azFilters := awsapi.GetFiltersQueryString(cfg.filters, azFiltersWhitelist) + data, err := cfg.awsConfig.GetEC2APIResponse("DescribeAvailabilityZones", azFilters, "") if err != nil { return nil, fmt.Errorf("cannot obtain availability zones: %w", err) } @@ -40,6 +42,20 @@ func getAvailabilityZones(cfg *apiConfig) ([]AvailabilityZone, error) { return azr.AvailabilityZoneInfo.Items, nil } +// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html +var azFiltersWhitelist = map[string]bool{ + "group-name": true, + "message": true, + "opt-in-status": true, + "parent-zoneID": true, + "parent-zoneName": true, + "region-name": true, + "state": true, + "zone-id": true, + "zone-type": true, + "zone-name": true, +} + // AvailabilityZonesResponse represents the response for https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html type AvailabilityZonesResponse struct { AvailabilityZoneInfo AvailabilityZoneInfo `xml:"availabilityZoneInfo"` diff --git a/lib/promscrape/discovery/ec2/ec2.go b/lib/promscrape/discovery/ec2/ec2.go index 9f8c610591..5cca0bc3b3 100644 --- a/lib/promscrape/discovery/ec2/ec2.go +++ b/lib/promscrape/discovery/ec2/ec2.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth" ) @@ -26,17 +27,8 @@ type SDConfig struct { RoleARN string `yaml:"role_arn,omitempty"` // RefreshInterval time.Duration `yaml:"refresh_interval"` // refresh_interval is obtained from `-promscrape.ec2SDCheckInterval` command-line option. - Port *int `yaml:"port,omitempty"` - Filters []Filter `yaml:"filters,omitempty"` -} - -// Filter is ec2 filter. -// -// See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html -// and https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_Filter.html -type Filter struct { - Name string `yaml:"name"` - Values []string `yaml:"values"` + Port *int `yaml:"port,omitempty"` + Filters []awsapi.Filter `yaml:"filters,omitempty"` } // GetLabels returns ec2 labels according to sdc. diff --git a/lib/promscrape/discovery/ec2/instance.go b/lib/promscrape/discovery/ec2/instance.go index 6d74dc8d0a..89b0369e05 100644 --- a/lib/promscrape/discovery/ec2/instance.go +++ b/lib/promscrape/discovery/ec2/instance.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/awsapi" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" ) @@ -28,8 +29,9 @@ func getReservations(cfg *apiConfig) ([]Reservation, error) { // See https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html var rs []Reservation pageToken := "" + instanceFilters := awsapi.GetFiltersQueryString(cfg.filters, nil) for { - data, err := cfg.awsConfig.GetEC2APIResponse("DescribeInstances", cfg.filtersQueryString, pageToken) + data, err := cfg.awsConfig.GetEC2APIResponse("DescribeInstances", instanceFilters, pageToken) if err != nil { return nil, fmt.Errorf("cannot obtain instances: %w", err) }