Skip to content

Commit

Permalink
Fixes EC2 SDK v2 client configuration to assume role properly
Browse files Browse the repository at this point in the history
  • Loading branch information
mmerkes committed Nov 19, 2024
1 parent 58e1fac commit d60f3e7
Show file tree
Hide file tree
Showing 7 changed files with 145 additions and 29 deletions.
16 changes: 8 additions & 8 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.22.7

require (
github.com/aws/aws-sdk-go v1.55.5
github.com/aws/aws-sdk-go-v2 v1.32.2
github.com/aws/aws-sdk-go-v2 v1.32.5
github.com/aws/aws-sdk-go-v2/config v1.28.0
github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2
github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.27.2
Expand All @@ -30,18 +30,18 @@ require (
github.com/NYTimes/gziphandler v1.1.1 // indirect
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.41
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ec2 v1.186.0
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect
github.com/aws/smithy-go v1.22.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.1
github.com/aws/smithy-go v1.22.1
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
Expand Down
28 changes: 14 additions & 14 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,18 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI=
github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo=
github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
github.com/aws/aws-sdk-go-v2/config v1.28.0 h1:FosVYWcqEtWNxHn8gB/Vs6jOlNwSoyOCA/g/sxyySOQ=
github.com/aws/aws-sdk-go-v2/config v1.28.0/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc=
github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8=
github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.186.0 h1:n2l2WeV+lEABrGwG/4MsE0WFEbd3j7yKsmZzbnEm5CY=
Expand All @@ -28,18 +28,18 @@ github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2 h1:VDQaVwGOokbd3VUbHF+wupiffdrb
github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2/go.mod h1:lvUlMghKYmSxSfv0vU7pdU/8jSY+s0zpG8xXhaGKCw0=
github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.27.2 h1:Zru9Iy2JPM5+uRnFnoqeOZzi8JIVIHJ0ua6JdeDHcyg=
github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.27.2/go.mod h1:PtQC3XjutCYFCn1+i8+wtpDaXvEK+vXF2gyLIKAmh4A=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 h1:wtpJ4zcwrSbwhECWQoI/g6WM9zqCcSpHDJIWSbMLOu4=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5/go.mod h1:qu/W9HXQbbQ4+1+JcZp0ZNPV31ym537ZJN+fiS7Ti8E=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk=
github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI=
github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo=
github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo=
github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLbc2iVROyaNEwBHbU=
github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg=
github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
Expand Down
18 changes: 14 additions & 4 deletions pkg/providers/v1/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"strings"
"time"

stscredsv2 "github.com/aws/aws-sdk-go-v2/credentials/stscreds"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
Expand Down Expand Up @@ -464,6 +465,7 @@ func (c *Cloud) CurrentNodeName(ctx context.Context, hostname string) (types.Nod
func init() {
registerMetrics()
cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) {
ctx := context.Background()
cfg, err := readAWSCloudConfig(config)
if err != nil {
return nil, fmt.Errorf("unable to read AWS cloud provider config file: %v", err)
Expand Down Expand Up @@ -492,6 +494,7 @@ func init() {
}

var creds *credentials.Credentials
var credsV2 *stscredsv2.AssumeRoleProvider
if cfg.Global.RoleARN != "" {
stsClient, err := getSTSClient(sess, cfg.Global.RoleARN, cfg.Global.SourceARN)
if err != nil {
Expand All @@ -505,10 +508,16 @@ func init() {
RoleARN: cfg.Global.RoleARN,
}),
})

stsClientv2, err := services.NewStsV2Client(ctx, regionName, cfg.Global.RoleARN, cfg.Global.SourceARN)
if err != nil {
return nil, fmt.Errorf("unable to create sts v2 client: %v", err)
}
credsV2 = stscredsv2.NewAssumeRoleProvider(stsClientv2, cfg.Global.RoleARN)
}

aws := newAWSSDKProvider(creds, cfg)
return newAWSCloud2(*cfg, aws, aws, creds)
return newAWSCloud2(*cfg, aws, aws, creds, credsV2)
})
}

Expand Down Expand Up @@ -564,12 +573,13 @@ func azToRegion(az string) (string, error) {
}

func newAWSCloud(cfg config.CloudConfig, awsServices Services) (*Cloud, error) {
return newAWSCloud2(cfg, awsServices, nil, nil)
return newAWSCloud2(cfg, awsServices, nil, nil, nil)
}

// newAWSCloud creates a new instance of AWSCloud.
// AWSProvider and instanceId are primarily for tests
func newAWSCloud2(cfg config.CloudConfig, awsServices Services, provider config.SDKProvider, credentials *credentials.Credentials) (*Cloud, error) {
func newAWSCloud2(cfg config.CloudConfig, awsServices Services, provider config.SDKProvider, credentials *credentials.Credentials, credentialsV2 *stscredsv2.AssumeRoleProvider) (*Cloud, error) {
ctx := context.Background()
// We have some state in the Cloud object
// Log so that if we are building multiple Cloud objects, it is obvious!
klog.Infof("Building AWS cloudprovider")
Expand All @@ -589,7 +599,7 @@ func newAWSCloud2(cfg config.CloudConfig, awsServices Services, provider config.
return nil, fmt.Errorf("error creating AWS EC2 client: %v", err)
}

ec2v2, err := services.NewEc2SdkV2(regionName)
ec2v2, err := services.NewEc2SdkV2(ctx, regionName, credentialsV2)
if err != nil {
return nil, fmt.Errorf("error creating AWS EC2v2 client: %v", err)
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/resourcemanagers/topology.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ func (t *instanceTopologyManager) GetNodeTopology(ctx context.Context, instanceT
case "UnauthorizedOperation":
// Gracefully handle the DecribeInstanceTopology access missing error
klog.Warningf("Not authorized to perform: ec2:DescribeInstanceTopology, permission missing: %q", err)
// Mark region as unsupported to back off on attempts to get network topology.
t.addUnsupported(region)
return nil, nil
case "RequestLimitExceeded":
// Gracefully handle request throttling
Expand Down
2 changes: 1 addition & 1 deletion pkg/resourcemanagers/topology_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func TestGetNodeTopology(t *testing.T) {
}
}

mockedEc2SdkV2.AssertNumberOfCalls(t, "DescribeInstanceTopology", 2)
mockedEc2SdkV2.AssertNumberOfCalls(t, "DescribeInstanceTopology", 1)
})

t.Run("Should handle exceeding request limits for DescribeInstanceTopology", func(t *testing.T) {
Expand Down
11 changes: 9 additions & 2 deletions pkg/services/aws_ec2.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ package services
import (
"context"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"
)
Expand All @@ -40,12 +42,17 @@ type ec2SdkV2 struct {
}

// NewEc2SdkV2 is a constructor for Ec2SdkV2 that creates a default EC2 client.
func NewEc2SdkV2(region string) (Ec2SdkV2, error) {
cfg, err := config.LoadDefaultConfig(context.TODO())
func NewEc2SdkV2(ctx context.Context, region string, assumeRoleProvider *stscreds.AssumeRoleProvider) (Ec2SdkV2, error) {
cfg, err := config.LoadDefaultConfig(ctx)
if err != nil {
return nil, err
}

// Don't override the default creds if the assume role provider isn't set.
if assumeRoleProvider != nil {
cfg.Credentials = aws.NewCredentialsCache(assumeRoleProvider)
}

client := ec2.NewFromConfig(cfg, func(o *ec2.Options) {
o.Region = region
})
Expand Down
97 changes: 97 additions & 0 deletions pkg/services/aws_sts.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package services

import (
"context"
"fmt"

"github.com/aws/aws-sdk-go-v2/aws/arn"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/aws/smithy-go/middleware"
smithyhttp "github.com/aws/smithy-go/transport/http"
"k8s.io/klog/v2"
)

const headerSourceArn = "x-amz-source-arn"
const headerSourceAccount = "x-amz-source-account"

type withStsHeadersMiddleware struct {
headers map[string]string
}

func (*withStsHeadersMiddleware) ID() string {
return "withStsHeadersMiddleware"
}

func (m *withStsHeadersMiddleware) HandleBuild(ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler) (
out middleware.BuildOutput, metadata middleware.Metadata, err error,
) {
req, ok := in.Request.(*smithyhttp.Request)
if !ok {
return out, metadata, fmt.Errorf("unrecognized transport type %T", in.Request)
}

for k, v := range m.headers {
req.Header.Set(k, v)
}
return next.HandleBuild(ctx, in)
}

// WithStsHeadersMiddleware provides middleware to set custom headers for STS calls
func WithStsHeadersMiddleware(headers map[string]string) func(*sts.Options) {
return func(o *sts.Options) {
o.APIOptions = append(o.APIOptions, func(s *middleware.Stack) error {
return s.Build.Add(&withStsHeadersMiddleware{
headers: headers,
}, middleware.After)
})
}
}

// NewStsV2Client provides a new STS client.
func NewStsV2Client(ctx context.Context, region, roleARN, sourceARN string) (*sts.Client, error) {
klog.Infof("Using AWS assumed role %v", roleARN)
cfg, err := config.LoadDefaultConfig(ctx)
if err != nil {
return nil, err
}

parsedSourceArn, err := arn.Parse(roleARN)
if err != nil {
return nil, err
}

sourceAcct := parsedSourceArn.AccountID

reqHeaders := map[string]string{
headerSourceAccount: sourceAcct,
}
if sourceARN != "" {
reqHeaders[headerSourceArn] = sourceARN
}

// Create the STS client with the custom middleware
// svc := s3.NewFromConfig(cfg, WithHeader("x-user-header", "..."))
stsClient := sts.NewFromConfig(cfg, func(o *sts.Options) {
o.Region = region
}, WithStsHeadersMiddleware(reqHeaders))

klog.V(4).Infof("configuring STS client with extra headers, %v", reqHeaders)
return stsClient, nil
}

0 comments on commit d60f3e7

Please sign in to comment.