From ea854fc41188577d6f4c1bbf12c9f53b2d867313 Mon Sep 17 00:00:00 2001 From: danmx Date: Wed, 12 Aug 2020 22:51:28 +0200 Subject: [PATCH 1/2] fix(pre-commit): adding Gazelle run during fmt phase Signed-off-by: danmx --- CHANGELOG.md | 1 + scripts/pre-commit.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 979e52c..925ac18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### Fix - **bazel:** docker rules +- **pre-commit:** adding Gazelle run during fmt phase diff --git a/scripts/pre-commit.sh b/scripts/pre-commit.sh index 8d4f1d9..c94cddd 100755 --- a/scripts/pre-commit.sh +++ b/scripts/pre-commit.sh @@ -7,7 +7,7 @@ test(){ } fmt() { - bazelisk run :gofmt + bazelisk run :gazelle && bazelisk run :gofmt } lint() { From 246b1a6922c5dc9020424638c6f184a707ca7317 Mon Sep 17 00:00:00 2001 From: danmx Date: Thu, 13 Aug 2020 19:06:07 +0200 Subject: [PATCH 2/2] aws: customizing default retryer for AWS api calls Signed-off-by: danmx --- CHANGELOG.md | 3 +++ cmd/list.go | 3 +++ cmd/session.go | 3 +++ cmd/ssh.go | 3 +++ pkg/aws/BUILD.bazel | 2 ++ pkg/aws/aws.go | 28 ++++++++++++++++++++++++---- pkg/aws/log/BUILD.bazel | 12 ++++++++++++ pkg/aws/log/log.go | 19 +++++++++++++++++++ pkg/list/list.go | 2 ++ pkg/session/session.go | 2 ++ pkg/session/session_test.go | 6 ++++++ pkg/ssh/ssh.go | 2 ++ pkg/ssh/ssh_test.go | 9 +++++++++ 13 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 pkg/aws/log/BUILD.bazel create mode 100644 pkg/aws/log/log.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 925ac18..71a9ae7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ ## [Unreleased] +### Aws +- customizing default retryer for AWS api calls + ### Chore - **deps:** bumping Go and tidying dependecies - **deps:** update module aws/aws-sdk-go to v1.34.2 diff --git a/cmd/list.go b/cmd/list.go index 53b3597..9f460e4 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -81,6 +81,7 @@ Filter format examples: listType := cfg.GetString("list-type") instanceIDs := cfg.GetStringSlice("filters.instance.ids") mfaToken := cfg.GetString("mfa") + trace := log.IsLevelEnabled(log.TraceLevel) // hack to get map[string]string from args // https://github.com/spf13/viper/issues/608 if cmd.Flags().Changed("session-filters") { @@ -113,6 +114,7 @@ Filter format examples: "instanceIDs": instanceIDs, "sessionFilters": sessionFilters, "tags": tags, + "trace": trace, }).Debug("List inputs") input := &list.StartInput{ OutputFormat: &outputFormat, @@ -122,6 +124,7 @@ Filter format examples: Filters: &filters, Interactive: &interactive, Type: &listType, + Trace: &trace, } err := list.Start(input) if err != nil { diff --git a/cmd/session.go b/cmd/session.go index 5a9cf06..05726b1 100644 --- a/cmd/session.go +++ b/cmd/session.go @@ -36,12 +36,14 @@ var sessionCmd = &cobra.Command{ profile := cfg.GetString("profile") region := cfg.GetString("region") mfaToken := cfg.GetString("mfa") + trace := log.IsLevelEnabled(log.TraceLevel) log.WithFields(log.Fields{ "target": target, "type": targetType, "region": region, "profile": profile, "mfa": mfaToken, + "trace": trace, }).Debug("Session inputs") input := &session.StartInput{ Target: &target, @@ -49,6 +51,7 @@ var sessionCmd = &cobra.Command{ Region: ®ion, Profile: &profile, MFAToken: &mfaToken, + Trace: &trace, } // returns err return session.Start(input) diff --git a/cmd/ssh.go b/cmd/ssh.go index fc899d2..6d9cb2b 100644 --- a/cmd/ssh.go +++ b/cmd/ssh.go @@ -52,6 +52,7 @@ var ( genKeyPair := cfg.GetBool("gen-key-pair") genKeyDir := cfg.GetString("gen-key-dir") mfaToken := cfg.GetString("mfa") + trace := log.IsLevelEnabled(log.TraceLevel) if genKeyPair { stat, err := os.Stat(genKeyDir) if !(err == nil && stat.IsDir()) { @@ -79,6 +80,7 @@ var ( "os-user": OSUser, "gen-key-pair": genKeyPair, "gen-key-dir": genKeyDir, + "trace": trace, }).Debug("ssh inputs") input := &ssh.StartInput{ Target: &target, @@ -90,6 +92,7 @@ var ( Region: ®ion, Profile: &profile, MFAToken: &mfaToken, + Trace: &trace, } // returns err return ssh.Start(input) diff --git a/pkg/aws/BUILD.bazel b/pkg/aws/BUILD.bazel index d357b36..707fc23 100644 --- a/pkg/aws/BUILD.bazel +++ b/pkg/aws/BUILD.bazel @@ -12,8 +12,10 @@ go_library( importpath = "github.com/danmx/sigil/pkg/aws", visibility = ["//visibility:public"], deps = [ + "//pkg/aws/log:go_default_library", "//pkg/os:go_default_library", "@com_github_aws_aws_sdk_go//aws:go_default_library", + "@com_github_aws_aws_sdk_go//aws/client:go_default_library", "@com_github_aws_aws_sdk_go//aws/credentials/stscreds:go_default_library", "@com_github_aws_aws_sdk_go//aws/session:go_default_library", "@com_github_aws_aws_sdk_go//service/ec2:go_default_library", diff --git a/pkg/aws/aws.go b/pkg/aws/aws.go index edb3643..a3dce13 100644 --- a/pkg/aws/aws.go +++ b/pkg/aws/aws.go @@ -7,18 +7,19 @@ import ( "os/exec" "os/signal" "syscall" + "time" + logger "github.com/danmx/sigil/pkg/aws/log" sigilOS "github.com/danmx/sigil/pkg/os" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" log "github.com/sirupsen/logrus" ) -const execEnvVar = "AWS_EXECUTION_ENV" - // Provider contains necessary components like the session type Provider struct { filters Filters @@ -32,6 +33,7 @@ type Config struct { Region string Profile string MFAToken string + Trace bool } // Filters grouped per type @@ -82,8 +84,15 @@ type SessionFilters struct { } const ( - maxResults int64 = 50 - pluginName string = "session-manager-plugin" + execEnvVar = "AWS_EXECUTION_ENV" + maxResults int64 = 50 + pluginName = "session-manager-plugin" + // API calls retry configuration + numMaxRetries = 10 + minRetryDelay = 1 * time.Second + minThrottleDelay = 2 * time.Second + maxRetryDelay = client.DefaultRetryerMaxRetryDelay + maxThrottleDelay = client.DefaultRetryerMaxThrottleDelay // TargetTypeInstanceID points to an instance ID type TargetTypeInstanceID = "instance-id" // TargetTypePrivateDNS points to a private DNS type @@ -102,6 +111,17 @@ func (p *Provider) NewWithConfig(c *Config) error { Profile: c.Profile, } awsConfig := aws.NewConfig() + if c.Trace { + awsConfig.LogLevel = aws.LogLevel(aws.LogDebugWithRequestRetries) + awsConfig.Logger = logger.NewTraceLogger() + } + awsConfig.Retryer = client.DefaultRetryer{ + NumMaxRetries: numMaxRetries, + MinRetryDelay: minRetryDelay, + MinThrottleDelay: minThrottleDelay, + MaxRetryDelay: maxRetryDelay, + MaxThrottleDelay: maxThrottleDelay, + } awsConfig.Region = aws.String(c.Region) options.Config = *awsConfig sess, err := session.NewSessionWithOptions(options) diff --git a/pkg/aws/log/BUILD.bazel b/pkg/aws/log/BUILD.bazel new file mode 100644 index 0000000..a264816 --- /dev/null +++ b/pkg/aws/log/BUILD.bazel @@ -0,0 +1,12 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["log.go"], + importpath = "github.com/danmx/sigil/pkg/aws/log", + visibility = ["//visibility:public"], + deps = [ + "@com_github_aws_aws_sdk_go//aws:go_default_library", + "@com_github_sirupsen_logrus//:go_default_library", + ], +) diff --git a/pkg/aws/log/log.go b/pkg/aws/log/log.go new file mode 100644 index 0000000..0e3d218 --- /dev/null +++ b/pkg/aws/log/log.go @@ -0,0 +1,19 @@ +package aws + +import ( + "github.com/aws/aws-sdk-go/aws" + log "github.com/sirupsen/logrus" +) + +// A traceLogger provides a minimalistic logger satisfying the aws.Logger interface. +type traceLogger struct{} + +// newTraceLogger returns a Logger which will write log messages to current logger +func NewTraceLogger() aws.Logger { + return &traceLogger{} +} + +// Log logs the parameters to the stdlib logger. See log.Println. +func (l traceLogger) Log(args ...interface{}) { + log.Trace(args) //nolint:govet // AWS Logger to Logrus +} diff --git a/pkg/list/list.go b/pkg/list/list.go index c08f4f8..0c51f83 100644 --- a/pkg/list/list.go +++ b/pkg/list/list.go @@ -46,6 +46,7 @@ type StartInput struct { Region *string Profile *string Filters *aws.Filters + Trace *bool } // Start will output a ist of all available EC2 instances @@ -56,6 +57,7 @@ func Start(input *StartInput) error { Region: *input.Region, Profile: *input.Profile, MFAToken: *input.MFAToken, + Trace: *input.Trace, }) if err != nil { log.Error(err) diff --git a/pkg/session/session.go b/pkg/session/session.go index 810bddb..26ddcd8 100644 --- a/pkg/session/session.go +++ b/pkg/session/session.go @@ -18,6 +18,7 @@ type StartInput struct { MFAToken *string Region *string Profile *string + Trace *bool } // Start will start a session in chosen instance @@ -30,6 +31,7 @@ func (input *StartInput) start(provider aws.CloudInstances) error { Region: *input.Region, Profile: *input.Profile, MFAToken: *input.MFAToken, + Trace: *input.Trace, }) if err != nil { log.Error("Failed to generate new provider") diff --git a/pkg/session/session_test.go b/pkg/session/session_test.go index 1ff9db3..c98c167 100644 --- a/pkg/session/session_test.go +++ b/pkg/session/session_test.go @@ -21,10 +21,12 @@ func TestStart(t *testing.T) { mfa := "123456" region := "eu-west-1" profile := "west" + trace := false input := StartInput{ MFAToken: &mfa, Region: ®ion, Profile: &profile, + Trace: &trace, } // Instance ID target := "i-xxxxxxxxxxxxxxxx1" @@ -36,6 +38,7 @@ func TestStart(t *testing.T) { Region: *input.Region, Profile: *input.Profile, MFAToken: *input.MFAToken, + Trace: *input.Trace, })).Return(nil), m.EXPECT().StartSession(gomock.Eq(*input.TargetType), gomock.Eq(*input.Target)).Return(nil), ) @@ -48,6 +51,7 @@ func TestStart(t *testing.T) { Region: *input.Region, Profile: *input.Profile, MFAToken: *input.MFAToken, + Trace: *input.Trace, })).Return(nil), m.EXPECT().StartSession(gomock.Eq(*input.TargetType), gomock.Eq(*input.Target)).Return(nil), ) @@ -60,6 +64,7 @@ func TestStart(t *testing.T) { Region: *input.Region, Profile: *input.Profile, MFAToken: *input.MFAToken, + Trace: *input.Trace, })).Return(nil), m.EXPECT().StartSession(gomock.Eq(*input.TargetType), gomock.Eq(*input.Target)).Return(nil), ) @@ -71,6 +76,7 @@ func TestStart(t *testing.T) { Region: *input.Region, Profile: *input.Profile, MFAToken: *input.MFAToken, + Trace: *input.Trace, })).Return(nil), m.EXPECT().StartSession(gomock.Eq(*input.TargetType), gomock.Eq(*input.Target)).Return(nil), ) diff --git a/pkg/ssh/ssh.go b/pkg/ssh/ssh.go index e05851a..31d91a5 100644 --- a/pkg/ssh/ssh.go +++ b/pkg/ssh/ssh.go @@ -31,6 +31,7 @@ type StartInput struct { MFAToken *string Region *string Profile *string + Trace *bool } // Start will start ssh session @@ -43,6 +44,7 @@ func (input *StartInput) start(provider aws.CloudSSH) error { Region: *input.Region, Profile: *input.Profile, MFAToken: *input.MFAToken, + Trace: *input.Trace, }) if err != nil { log.Error(err) diff --git a/pkg/ssh/ssh_test.go b/pkg/ssh/ssh_test.go index 81ba154..4eb3a42 100644 --- a/pkg/ssh/ssh_test.go +++ b/pkg/ssh/ssh_test.go @@ -30,6 +30,7 @@ func TestStart(t *testing.T) { pubKey := path.Join(os.TempDir(), "sigil_test.pub") osUser := "ec2-user" genKey := true + trace := false input := StartInput{ MFAToken: &mfa, Region: ®ion, @@ -40,6 +41,7 @@ func TestStart(t *testing.T) { PublicKey: &pubKey, OSUser: &osUser, GenKeyPair: &genKey, + Trace: &trace, } gomock.InOrder( @@ -47,6 +49,7 @@ func TestStart(t *testing.T) { Region: *input.Region, Profile: *input.Profile, MFAToken: *input.MFAToken, + Trace: *input.Trace, })).Return(nil), m.EXPECT().StartSSH( gomock.Eq(*input.TargetType), @@ -72,6 +75,7 @@ func TestStart(t *testing.T) { PublicKey: &pubKey, OSUser: &osUser, GenKeyPair: &genKey, + Trace: &trace, } gomock.InOrder( @@ -79,6 +83,7 @@ func TestStart(t *testing.T) { Region: *input.Region, Profile: *input.Profile, MFAToken: *input.MFAToken, + Trace: *input.Trace, })).Return(nil), m.EXPECT().StartSSH( gomock.Eq(*input.TargetType), @@ -104,6 +109,7 @@ func TestStart(t *testing.T) { PublicKey: &pubKey, OSUser: &osUser, GenKeyPair: &genKey, + Trace: &trace, } gomock.InOrder( @@ -111,6 +117,7 @@ func TestStart(t *testing.T) { Region: *input.Region, Profile: *input.Profile, MFAToken: *input.MFAToken, + Trace: *input.Trace, })).Return(nil), m.EXPECT().StartSSH( gomock.Eq(*input.TargetType), @@ -135,6 +142,7 @@ func TestStart(t *testing.T) { PublicKey: &pubKey, OSUser: &osUser, GenKeyPair: &genKey, + Trace: &trace, } gomock.InOrder( @@ -142,6 +150,7 @@ func TestStart(t *testing.T) { Region: *input.Region, Profile: *input.Profile, MFAToken: *input.MFAToken, + Trace: *input.Trace, })).Return(nil), m.EXPECT().StartSSH( gomock.Eq(*input.TargetType),