From f686174329fd08aac4a955d919c03d55088d00d6 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Tue, 2 Mar 2021 13:46:26 +0200 Subject: [PATCH] lib/promscrape/discovery/ec2: follow-up after f6114345de736a946747d3f66e9879886ed6e563 --- docs/CHANGELOG.md | 1 + lib/promscrape/discovery/ec2/api.go | 39 +++++++++-------------- lib/promscrape/discovery/ec2/api_test.go | 1 - lib/protoparser/prometheus/parser_test.go | 4 +-- 4 files changed, 18 insertions(+), 27 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5692a47526..bfafd52223 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -18,6 +18,7 @@ * FEATURE: add `increase_pure(m[d])` function to MetricsQL. It works the same as `increase(m[d])` except of various edge cases. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/962) for details. * FEATURE: increase accuracy for `buckets_limit(limit, buckets)` results for small `limit` values. See [MetricsQL docs](https://victoriametrics.github.io/MetricsQL.html) for details. * FEATURE: vmagent: initial support for Windows build with `CGO_ENABLED=0 GOOS=windows go build -mod=vendor ./app/vmagent`. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/70) and [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1036). +* FEATURE: vmagent: support WebIdentityToken auth in EC2 service discovery. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1080) for details. * FEATURE: vmalert: properly process query params in `-datasource.url` and `-remoteRead.url` command-line flags. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1087) for details. * BUGFIX: vmagent: properly apply `-remoteWrite.rateLimit` when `-remoteWrite.queues` is greater than 1. Previously there was a data race, which could prevent from proper rate limiting. diff --git a/lib/promscrape/discovery/ec2/api.go b/lib/promscrape/discovery/ec2/api.go index c407a2afa8..e0e6b30a35 100644 --- a/lib/promscrape/discovery/ec2/api.go +++ b/lib/promscrape/discovery/ec2/api.go @@ -12,17 +12,10 @@ import ( "sync" "time" + "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils" ) -const ( - awsAccessKeyEnv = "AWS_ACCESS_KEY_ID" - awsSecretKeyEnv = "AWS_SECRET_ACCESS_KEY" - awsRegionEnv = "AWS_REGION" - awsRoleARNEnv = "AWS_ROLE_ARN" - awsWITPath = "AWS_WEB_IDENTITY_TOKEN_FILE" -) - type apiConfig struct { region string roleARN string @@ -83,17 +76,16 @@ func newAPIConfig(sdc *SDConfig) (*apiConfig, error) { cfg.ec2Endpoint = buildAPIEndpoint(sdc.Endpoint, region, "ec2") cfg.stsEndpoint = buildAPIEndpoint(sdc.Endpoint, region, "sts") - envARN := os.Getenv(awsRoleARNEnv) - if envARN != "" { - cfg.roleARN = envARN + if cfg.roleARN == "" { + cfg.roleARN = os.Getenv("AWS_ROLE_ARN") } - cfg.webTokenPath = os.Getenv(awsWITPath) + cfg.webTokenPath = os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE") if cfg.webTokenPath != "" && cfg.roleARN == "" { - return nil, fmt.Errorf("roleARN is missing for %q, set it with cfg or env var %q", awsWITPath, awsRoleARNEnv) + return nil, fmt.Errorf("roleARN is missing for AWS_WEB_IDENTITY_TOKEN_FILE=%q, set it either in `ec2_sd_config` or via env var AWS_ROLE_ARN", cfg.webTokenPath) } // explicitly set credentials has priority over env variables - cfg.defaultAccessKey = os.Getenv(awsAccessKeyEnv) - cfg.defaultSecretKey = os.Getenv(awsSecretKeyEnv) + cfg.defaultAccessKey = os.Getenv("AWS_ACCESS_KEY_ID") + cfg.defaultSecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY") if len(sdc.AccessKey) > 0 { cfg.defaultAccessKey = sdc.AccessKey } @@ -120,11 +112,10 @@ func getFiltersQueryString(filters []Filter) string { } func getDefaultRegion() (string, error) { - envRegion := os.Getenv(awsRegionEnv) + envRegion := os.Getenv("AWS_REGION") if envRegion != "" { return envRegion, nil } - data, err := getMetadataByPath("dynamic/instance-identity/document") if err != nil { return "", err @@ -199,12 +190,12 @@ func getAPICredentials(cfg *apiConfig) (*apiCredentials, error) { acNew = ac } if len(acNew.AccessKeyID) == 0 { - return nil, fmt.Errorf("missing `access_key`, you can set it with %s env var, "+ - "directly at `ec2_sd_config` as `access_key` or use instance iam role", awsAccessKeyEnv) + return nil, fmt.Errorf("missing `access_key`, you can set it with env var AWS_ACCESS_KEY_ID, " + + "directly at `ec2_sd_config` as `access_key` or use instance iam role") } if len(acNew.SecretAccessKey) == 0 { - return nil, fmt.Errorf("missing `secret_key`, you can set it with %s env var,"+ - "directly at `ec2_sd_config` as `secret_key` or use instance iam role", awsSecretKeyEnv) + return nil, fmt.Errorf("missing `secret_key`, you can set it with env var AWS_SECRET_ACCESS_KEY," + + "directly at `ec2_sd_config` as `secret_key` or use instance iam role") } return acNew, nil } @@ -293,7 +284,7 @@ func getMetadataByPath(apiPath string) ([]byte, error) { // https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/ func getRoleWebIdentityCredentials(stsEndpoint, roleARN string, token string) (*apiCredentials, error) { data, err := getSTSAPIResponse("AssumeRoleWithWebIdentity", stsEndpoint, roleARN, func(apiURL string) (*http.Request, error) { - apiURL += fmt.Sprintf("&WebIdentityToken=%s", token) + apiURL += fmt.Sprintf("&WebIdentityToken=%s", url.QueryEscape(token)) return http.NewRequest("GET", apiURL, nil) }) if err != nil { @@ -328,7 +319,7 @@ func parseARNCredentials(data []byte, role string) (*apiCredentials, error) { case "AssumeRoleWithWebIdentity": cred = arr.AssumeRoleWithWebIdentityResult.Credentials default: - return nil, fmt.Errorf("bug, unexpected role: %q", role) + logger.Panicf("BUG: unexpected role: %q", role) } return &apiCredentials{ AccessKeyID: cred.AccessKeyID, @@ -374,7 +365,7 @@ func buildAPIEndpoint(customEndpoint, region, service string) string { return endpoint } -// getSTSAPIResponse makes request to aws sts api with role_arn +// getSTSAPIResponse makes request to aws sts api with roleARN // and returns temporary credentials with expiration time // // See https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html diff --git a/lib/promscrape/discovery/ec2/api_test.go b/lib/promscrape/discovery/ec2/api_test.go index cd4b4a6cc1..1a1769f841 100644 --- a/lib/promscrape/discovery/ec2/api_test.go +++ b/lib/promscrape/discovery/ec2/api_test.go @@ -64,7 +64,6 @@ func TestParseARNCredentialsFailure(t *testing.T) { } func TestParseARNCredentialsSuccess(t *testing.T) { - f := func(data, role string, credsExpected *apiCredentials) { t.Helper() creds, err := parseARNCredentials([]byte(data), role) diff --git a/lib/protoparser/prometheus/parser_test.go b/lib/protoparser/prometheus/parser_test.go index e8d2bbb35b..eef1497130 100644 --- a/lib/protoparser/prometheus/parser_test.go +++ b/lib/protoparser/prometheus/parser_test.go @@ -190,11 +190,11 @@ cassandra_token_ownership_ratio 78.9`, &Rows{ Metric: "mssql_sql_server_active_transactions_sec", Tags: []Tag{ { - Key: "loginname", + Key: "loginname", Value: "domain\\somelogin", }, { - Key: "env", + Key: "env", Value: "develop", }, },