From d60f3e76105c8002b46c306658bef6230ce61b5b Mon Sep 17 00:00:00 2001 From: Matt Merkes Date: Mon, 18 Nov 2024 15:38:16 -0800 Subject: [PATCH] Fixes EC2 SDK v2 client configuration to assume role properly --- go.mod | 16 ++--- go.sum | 28 ++++---- pkg/providers/v1/aws.go | 18 +++-- pkg/resourcemanagers/topology.go | 2 + pkg/resourcemanagers/topology_test.go | 2 +- pkg/services/aws_ec2.go | 11 ++- pkg/services/aws_sts.go | 97 +++++++++++++++++++++++++++ 7 files changed, 145 insertions(+), 29 deletions(-) create mode 100644 pkg/services/aws_sts.go diff --git a/go.mod b/go.mod index 880ea93de8..2c008a7bdb 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.7 require ( github.com/aws/aws-sdk-go v1.55.5 - github.com/aws/aws-sdk-go-v2 v1.32.2 + github.com/aws/aws-sdk-go-v2 v1.32.5 github.com/aws/aws-sdk-go-v2/config v1.28.0 github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2 github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.27.2 @@ -30,18 +30,18 @@ require ( github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.41 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/service/ec2 v1.186.0 - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect - github.com/aws/smithy-go v1.22.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 + github.com/aws/smithy-go v1.22.1 github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect diff --git a/go.sum b/go.sum index decaa59330..9ebc19794f 100644 --- a/go.sum +++ b/go.sum @@ -8,18 +8,18 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= -github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= +github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/config v1.28.0 h1:FosVYWcqEtWNxHn8gB/Vs6jOlNwSoyOCA/g/sxyySOQ= github.com/aws/aws-sdk-go-v2/config v1.28.0/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc= github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/service/ec2 v1.186.0 h1:n2l2WeV+lEABrGwG/4MsE0WFEbd3j7yKsmZzbnEm5CY= @@ -28,18 +28,18 @@ github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2 h1:VDQaVwGOokbd3VUbHF+wupiffdrb github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2/go.mod h1:lvUlMghKYmSxSfv0vU7pdU/8jSY+s0zpG8xXhaGKCw0= github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.27.2 h1:Zru9Iy2JPM5+uRnFnoqeOZzi8JIVIHJ0ua6JdeDHcyg= github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.27.2/go.mod h1:PtQC3XjutCYFCn1+i8+wtpDaXvEK+vXF2gyLIKAmh4A= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 h1:wtpJ4zcwrSbwhECWQoI/g6WM9zqCcSpHDJIWSbMLOu4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5/go.mod h1:qu/W9HXQbbQ4+1+JcZp0ZNPV31ym537ZJN+fiS7Ti8E= github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI= -github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo= -github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= -github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= -github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLbc2iVROyaNEwBHbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= diff --git a/pkg/providers/v1/aws.go b/pkg/providers/v1/aws.go index 7a37f35d81..003824d97e 100644 --- a/pkg/providers/v1/aws.go +++ b/pkg/providers/v1/aws.go @@ -27,6 +27,7 @@ import ( "strings" "time" + stscredsv2 "github.com/aws/aws-sdk-go-v2/credentials/stscreds" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" @@ -464,6 +465,7 @@ func (c *Cloud) CurrentNodeName(ctx context.Context, hostname string) (types.Nod func init() { registerMetrics() cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { + ctx := context.Background() cfg, err := readAWSCloudConfig(config) if err != nil { return nil, fmt.Errorf("unable to read AWS cloud provider config file: %v", err) @@ -492,6 +494,7 @@ func init() { } var creds *credentials.Credentials + var credsV2 *stscredsv2.AssumeRoleProvider if cfg.Global.RoleARN != "" { stsClient, err := getSTSClient(sess, cfg.Global.RoleARN, cfg.Global.SourceARN) if err != nil { @@ -505,10 +508,16 @@ func init() { RoleARN: cfg.Global.RoleARN, }), }) + + stsClientv2, err := services.NewStsV2Client(ctx, regionName, cfg.Global.RoleARN, cfg.Global.SourceARN) + if err != nil { + return nil, fmt.Errorf("unable to create sts v2 client: %v", err) + } + credsV2 = stscredsv2.NewAssumeRoleProvider(stsClientv2, cfg.Global.RoleARN) } aws := newAWSSDKProvider(creds, cfg) - return newAWSCloud2(*cfg, aws, aws, creds) + return newAWSCloud2(*cfg, aws, aws, creds, credsV2) }) } @@ -564,12 +573,13 @@ func azToRegion(az string) (string, error) { } func newAWSCloud(cfg config.CloudConfig, awsServices Services) (*Cloud, error) { - return newAWSCloud2(cfg, awsServices, nil, nil) + return newAWSCloud2(cfg, awsServices, nil, nil, nil) } // newAWSCloud creates a new instance of AWSCloud. // AWSProvider and instanceId are primarily for tests -func newAWSCloud2(cfg config.CloudConfig, awsServices Services, provider config.SDKProvider, credentials *credentials.Credentials) (*Cloud, error) { +func newAWSCloud2(cfg config.CloudConfig, awsServices Services, provider config.SDKProvider, credentials *credentials.Credentials, credentialsV2 *stscredsv2.AssumeRoleProvider) (*Cloud, error) { + ctx := context.Background() // We have some state in the Cloud object // Log so that if we are building multiple Cloud objects, it is obvious! klog.Infof("Building AWS cloudprovider") @@ -589,7 +599,7 @@ func newAWSCloud2(cfg config.CloudConfig, awsServices Services, provider config. return nil, fmt.Errorf("error creating AWS EC2 client: %v", err) } - ec2v2, err := services.NewEc2SdkV2(regionName) + ec2v2, err := services.NewEc2SdkV2(ctx, regionName, credentialsV2) if err != nil { return nil, fmt.Errorf("error creating AWS EC2v2 client: %v", err) } diff --git a/pkg/resourcemanagers/topology.go b/pkg/resourcemanagers/topology.go index ea99dbefd2..0f5cfd37ca 100644 --- a/pkg/resourcemanagers/topology.go +++ b/pkg/resourcemanagers/topology.go @@ -81,6 +81,8 @@ func (t *instanceTopologyManager) GetNodeTopology(ctx context.Context, instanceT case "UnauthorizedOperation": // Gracefully handle the DecribeInstanceTopology access missing error klog.Warningf("Not authorized to perform: ec2:DescribeInstanceTopology, permission missing: %q", err) + // Mark region as unsupported to back off on attempts to get network topology. + t.addUnsupported(region) return nil, nil case "RequestLimitExceeded": // Gracefully handle request throttling diff --git a/pkg/resourcemanagers/topology_test.go b/pkg/resourcemanagers/topology_test.go index 1eae0304ad..d889a3d420 100644 --- a/pkg/resourcemanagers/topology_test.go +++ b/pkg/resourcemanagers/topology_test.go @@ -85,7 +85,7 @@ func TestGetNodeTopology(t *testing.T) { } } - mockedEc2SdkV2.AssertNumberOfCalls(t, "DescribeInstanceTopology", 2) + mockedEc2SdkV2.AssertNumberOfCalls(t, "DescribeInstanceTopology", 1) }) t.Run("Should handle exceeding request limits for DescribeInstanceTopology", func(t *testing.T) { diff --git a/pkg/services/aws_ec2.go b/pkg/services/aws_ec2.go index 5d0d2981c9..a2848bac36 100644 --- a/pkg/services/aws_ec2.go +++ b/pkg/services/aws_ec2.go @@ -19,7 +19,9 @@ package services import ( "context" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" ) @@ -40,12 +42,17 @@ type ec2SdkV2 struct { } // NewEc2SdkV2 is a constructor for Ec2SdkV2 that creates a default EC2 client. -func NewEc2SdkV2(region string) (Ec2SdkV2, error) { - cfg, err := config.LoadDefaultConfig(context.TODO()) +func NewEc2SdkV2(ctx context.Context, region string, assumeRoleProvider *stscreds.AssumeRoleProvider) (Ec2SdkV2, error) { + cfg, err := config.LoadDefaultConfig(ctx) if err != nil { return nil, err } + // Don't override the default creds if the assume role provider isn't set. + if assumeRoleProvider != nil { + cfg.Credentials = aws.NewCredentialsCache(assumeRoleProvider) + } + client := ec2.NewFromConfig(cfg, func(o *ec2.Options) { o.Region = region }) diff --git a/pkg/services/aws_sts.go b/pkg/services/aws_sts.go new file mode 100644 index 0000000000..5f96968616 --- /dev/null +++ b/pkg/services/aws_sts.go @@ -0,0 +1,97 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" + "k8s.io/klog/v2" +) + +const headerSourceArn = "x-amz-source-arn" +const headerSourceAccount = "x-amz-source-account" + +type withStsHeadersMiddleware struct { + headers map[string]string +} + +func (*withStsHeadersMiddleware) ID() string { + return "withStsHeadersMiddleware" +} + +func (m *withStsHeadersMiddleware) HandleBuild(ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler) ( + out middleware.BuildOutput, metadata middleware.Metadata, err error, +) { + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return out, metadata, fmt.Errorf("unrecognized transport type %T", in.Request) + } + + for k, v := range m.headers { + req.Header.Set(k, v) + } + return next.HandleBuild(ctx, in) +} + +// WithStsHeadersMiddleware provides middleware to set custom headers for STS calls +func WithStsHeadersMiddleware(headers map[string]string) func(*sts.Options) { + return func(o *sts.Options) { + o.APIOptions = append(o.APIOptions, func(s *middleware.Stack) error { + return s.Build.Add(&withStsHeadersMiddleware{ + headers: headers, + }, middleware.After) + }) + } +} + +// NewStsV2Client provides a new STS client. +func NewStsV2Client(ctx context.Context, region, roleARN, sourceARN string) (*sts.Client, error) { + klog.Infof("Using AWS assumed role %v", roleARN) + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return nil, err + } + + parsedSourceArn, err := arn.Parse(roleARN) + if err != nil { + return nil, err + } + + sourceAcct := parsedSourceArn.AccountID + + reqHeaders := map[string]string{ + headerSourceAccount: sourceAcct, + } + if sourceARN != "" { + reqHeaders[headerSourceArn] = sourceARN + } + + // Create the STS client with the custom middleware + // svc := s3.NewFromConfig(cfg, WithHeader("x-user-header", "...")) + stsClient := sts.NewFromConfig(cfg, func(o *sts.Options) { + o.Region = region + }, WithStsHeadersMiddleware(reqHeaders)) + + klog.V(4).Infof("configuring STS client with extra headers, %v", reqHeaders) + return stsClient, nil +}