From d02cb7bbebd52dee58c814ed1acd9cd79b8c293f Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Sat, 20 Jul 2024 11:48:41 +0000 Subject: [PATCH 01/17] Added utilities for parsing AWS provider configuration from the provider's configuration file. --- go.mod | 18 +- go.sum | 14 + tfutils/aws_config.go | 273 ++++++++++++++++++ tfutils/aws_config_test.go | 177 ++++++++++++ tfutils/plan.go | 4 +- .../config_from_provider/ca-bundle.crt | 97 +++++++ tfutils/testdata/config_from_provider/test.tf | 49 ++++ tfutils/testdata/providers.tf | 91 ++++++ tfutils/testdata/subfolder/more_providers.tf | 6 + 9 files changed, 721 insertions(+), 8 deletions(-) create mode 100644 tfutils/aws_config.go create mode 100644 tfutils/aws_config_test.go create mode 100644 tfutils/testdata/config_from_provider/ca-bundle.crt create mode 100644 tfutils/testdata/config_from_provider/test.tf create mode 100644 tfutils/testdata/providers.tf create mode 100644 tfutils/testdata/subfolder/more_providers.tf diff --git a/go.mod b/go.mod index 2a994869..3c655982 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,10 @@ go 1.22.4 require ( connectrpc.com/connect v1.16.2 + github.com/aws/aws-sdk-go-v2 v1.30.3 + github.com/aws/aws-sdk-go-v2/config v1.27.27 + github.com/aws/aws-sdk-go-v2/credentials v1.17.27 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 github.com/charmbracelet/bubbles v0.18.0 github.com/charmbracelet/bubbletea v0.26.6 @@ -13,9 +17,11 @@ require ( github.com/getsentry/sentry-go v0.28.1 github.com/go-jose/go-jose/v4 v4.0.3 github.com/google/uuid v1.6.0 + github.com/hashicorp/hcl/v2 v2.21.0 github.com/hexops/gotextdiff v1.0.3 github.com/jedib0t/go-pretty/v6 v6.5.9 github.com/mattn/go-isatty v0.0.20 + github.com/mitchellh/go-homedir v1.1.0 github.com/muesli/reflow v0.3.0 github.com/muesli/termenv v0.15.2 github.com/overmindtech/aws-source v0.0.0-20240718232007-6b71c2ec6ebd @@ -37,22 +43,21 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/trace v1.28.0 + golang.org/x/net v0.26.0 golang.org/x/oauth2 v0.21.0 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/agext/levenshtein v1.2.2 // indirect github.com/alecthomas/chroma/v2 v2.8.0 // indirect github.com/alecthomas/kingpin/v2 v2.3.2 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/auth0/go-jwt-middleware/v2 v2.2.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect - github.com/aws/aws-sdk-go-v2/config v1.27.27 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect @@ -103,6 +108,7 @@ require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/btree v1.1.2 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -117,7 +123,7 @@ require ( github.com/micahhausler/aws-iam-policy v0.4.2 // indirect github.com/microcosm-cc/bluemonday v1.0.25 // indirect github.com/miekg/dns v1.1.61 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect @@ -145,6 +151,7 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yuin/goldmark v1.5.4 // indirect github.com/yuin/goldmark-emoji v1.0.2 // indirect + github.com/zclconf/go-cty v1.14.4 // indirect go.opentelemetry.io/otel/log v0.3.0 // indirect go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect @@ -152,7 +159,6 @@ require ( golang.org/x/crypto v0.25.0 // indirect golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/mod v0.18.0 // indirect - golang.org/x/net v0.26.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect diff --git a/go.sum b/go.sum index 7b3e784d..135af77f 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/MrAlias/otel-schema-utils v0.2.1-alpha h1:dSeMM04tO+EY1JLof0bL8rIDkaMV3yBiUFdcSeIfbqI= github.com/MrAlias/otel-schema-utils v0.2.1-alpha/go.mod h1:i5gQR7dVLC4XxJuPITWxpWnjGRICZY50OMXXtrQTDeQ= +github.com/agext/levenshtein v1.2.2 h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXvaqE= +github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/alecthomas/assert/v2 v2.2.1 h1:XivOgYcduV98QCahG8T5XTezV5bylXe+lBxLG2K2ink= github.com/alecthomas/assert/v2 v2.2.1/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/chroma/v2 v2.8.0 h1:w9WJUjFFmHHB2e8mRpL9jjy3alYDlU0QLDezj1xE264= @@ -14,6 +16,8 @@ github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= +github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/auth0/go-jwt-middleware/v2 v2.2.1 h1:pqxEIwlCztD0T9ZygGfOrw4NK/F9iotnCnPJVADKbkE= @@ -154,6 +158,8 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= +github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -168,6 +174,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/hcl/v2 v2.21.0 h1:lve4q/o/2rqwYOgUg3y3V2YPyD1/zkCLGjIV74Jit14= +github.com/hashicorp/hcl/v2 v2.21.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= @@ -210,6 +218,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -315,6 +325,10 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark-emoji v1.0.2 h1:c/RgTShNgHTtc6xdz2KKI74jJr6rWi7FPgnP9GAsO5s= github.com/yuin/goldmark-emoji v1.0.2/go.mod h1:RhP/RWpexdp+KHs7ghKnifRoIs/Bq4nDS7tRbCkOwKY= +github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8= +github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= go.opentelemetry.io/contrib/detectors/aws/ec2 v1.28.0 h1:d+y/wygENfwEbVpo7c3A9GfnMhoTiepQcthQSh+Mc9g= go.opentelemetry.io/contrib/detectors/aws/ec2 v1.28.0/go.mod h1:gxGqapN+BNTBkKvKZFQJ1mfhQss7suB5gDmPwzJJWhQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= diff --git a/tfutils/aws_config.go b/tfutils/aws_config.go new file mode 100644 index 00000000..149095f3 --- /dev/null +++ b/tfutils/aws_config.go @@ -0,0 +1,273 @@ +package tfutils + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/hcl/v2/hclparse" + "github.com/mitchellh/go-homedir" + "golang.org/x/net/http/httpproxy" +) + +// This struct allows us to parse any HCL file that contains an AWS provider +// using the gohcl library. +type ProviderFile struct { + Providers []AWSProvider `hcl:"provider,block"` + + // Throw any additional stuff into here so it doesn't fail + Remain hcl.Body `hcl:",remain"` +} + +// This struct represents an AWS provider block in a terraform file. It is +// intended to be used with the gohcl library to parse HCL files. +// +// The fields are based on the AWS provider configuration documentation: +// https://registry.terraform.io/providers/hashicorp/aws/latest/docs#provider-configuration +type AWSProvider struct { + Name string `hcl:"name,label"` + AccessKey string `hcl:"access_key,optional"` + SecretKey string `hcl:"secret_key,optional"` + Token string `hcl:"token,optional"` + Region string `hcl:"region,optional"` + CustomCABundle string `hcl:"custom_ca_bundle,optional"` + EC2MetadataServiceEndpoint string `hcl:"ec2_metadata_service_endpoint,optional"` + EC2MetadataServiceEndpointMode string `hcl:"ec2_metadata_service_endpoint_mode,optional"` + SkipMetadataAPICheck bool `hcl:"skip_metadata_api_check,optional"` + HTTPProxy string `hcl:"http_proxy,optional"` + HTTPSProxy string `hcl:"https_proxy,optional"` + NoProxy string `hcl:"no_proxy,optional"` + MaxRetries int `hcl:"max_retries,optional"` + Profile string `hcl:"profile,optional"` + RetryMode string `hcl:"retry_mode,optional"` + SharedConfigFiles []string `hcl:"shared_config_files,optional"` + SharedCredentialsFiles []string `hcl:"shared_credentials_files,optional"` + UseDualStackEndpoint bool `hcl:"use_dualstack_endpoint,optional"` + UseFIPSEndpoint bool `hcl:"use_fips_endpoint,optional"` + + AssumeRole *AssumeRole `hcl:"assume_role,block"` + AssumeRoleWithWebIdentity *AssumeRoleWithWebIdentity `hcl:"assume_role_with_web_identity,block"` + + // Throw any additional stuff into here so it doesn't fail + Remain hcl.Body `hcl:",remain"` +} + +// Fields that are used for assuming a role, see: +// https://registry.terraform.io/providers/hashicorp/aws/latest/docs#assuming-an-iam-role +type AssumeRole struct { + Duration string `hcl:"duration,optional"` + ExternalID string `hcl:"external_id,optional"` + Policy string `hcl:"policy,optional"` + PolicyARNs []string `hcl:"policy_arns,optional"` + RoleARN string `hcl:"role_arn,optional"` + SessionName string `hcl:"session_name,optional"` + SourceIdentity string `hcl:"source_identity,optional"` + Tags map[string]string `hcl:"tags,optional"` + TransitiveTagKeys []string `hcl:"transitive_tag_keys,optional"` + + // Throw any additional stuff into here so it doesn't fail + Remain hcl.Body `hcl:",remain"` +} + +// Fields that are used for assuming a role with web identity, see: +// https://registry.terraform.io/providers/hashicorp/aws/latest/docs#assuming-an-iam-role-using-a-web-identity +type AssumeRoleWithWebIdentity struct { + Duration string `hcl:"duration,optional"` + Policy string `hcl:"policy,optional"` + PolicyARNs []string `hcl:"policy_arns,optional"` + RoleARN string `hcl:"role_arn,optional"` + SessionName string `hcl:"session_name,optional"` + WebIdentityToken string `hcl:"web_identity_token,optional"` + WebIdentityTokenFile string `hcl:"web_identity_token_file,optional"` + + // Throw any additional stuff into here so it doesn't fail + Remain hcl.Body `hcl:",remain"` +} + +// Parses AWS provider config from all terraform files in the given directory, +// recursing into subdirectories. Returns a list of AWS providers and a list of +// files that were parsed. +func ParseAWSProviders(dir string) ([]AWSProvider, []string, error) { + files := make([]string, 0) + + // Get all files matching *.tf from everywhere under the directory + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("error searching for terraform files: %w", err) + } + if !info.IsDir() && filepath.Ext(info.Name()) == ".tf" { + files = append(files, path) + } + return nil + }) + + if err != nil { + return nil, nil, err + } + + parser := hclparse.NewParser() + awsProviders := make([]AWSProvider, 0) + + // Iterate over the files + for _, file := range files { + b, err := os.ReadFile(file) + if err != nil { + return nil, files, fmt.Errorf("error reading terraform file: (%v) %w", file, err) + } + + // Parse the HCL file + parsedFile, diag := parser.ParseHCL(b, file) + if diag.HasErrors() { + return nil, files, fmt.Errorf("error parsing terraform file: (%v) %w", file, diag) + } + + providerFile := ProviderFile{} + diag = gohcl.DecodeBody(parsedFile.Body, &hcl.EvalContext{}, &providerFile) + if diag.HasErrors() { + return nil, files, fmt.Errorf("error decoding terraform file: (%v) %w", file, diag) + } + + for _, provider := range providerFile.Providers { + if provider.Name == "aws" { + awsProviders = append(awsProviders, provider) + } + } + } + + return awsProviders, files, nil +} + +// ConfigFromProvider creates an aws.Config from an AWSProvider that uses the +// provided HTTP client. This client will be modified with proxy settings if +// they are present in the provider. +func ConfigFromProvider(ctx context.Context, provider AWSProvider) (aws.Config, error) { + var options []func(*config.LoadOptions) error + + if provider.AccessKey != "" { + options = append(options, config.WithCredentialsProvider(credentials.StaticCredentialsProvider{ + Value: aws.Credentials{ + AccessKeyID: provider.AccessKey, + SecretAccessKey: provider.SecretKey, + SessionToken: provider.Token, + }, + })) + } + + if provider.Region != "" { + options = append(options, config.WithRegion(provider.Region)) + } + + if provider.CustomCABundle != "" { + bundlePath := os.ExpandEnv(provider.CustomCABundle) + bundlePath, err := homedir.Expand(bundlePath) + if err != nil { + return aws.Config{}, fmt.Errorf("expanding custom CA bundle path: %w", err) + } + + bundle, err := os.ReadFile(bundlePath) + if err != nil { + return aws.Config{}, fmt.Errorf("reading custom CA bundle: %w", err) + } + + options = append(options, config.WithCustomCABundle(bytes.NewReader(bundle))) + } + + if provider.EC2MetadataServiceEndpoint != "" { + options = append(options, config.WithEC2IMDSEndpoint(provider.EC2MetadataServiceEndpoint)) + } + + if provider.EC2MetadataServiceEndpointMode != "" { + var mode imds.EndpointModeState + + switch { + case len(provider.EC2MetadataServiceEndpointMode) == 0: + mode = imds.EndpointModeStateUnset + case strings.EqualFold(provider.EC2MetadataServiceEndpointMode, "IPv6"): + mode = imds.EndpointModeStateIPv4 + case strings.EqualFold(provider.EC2MetadataServiceEndpointMode, "IPv4"): + mode = imds.EndpointModeStateIPv6 + default: + return aws.Config{}, fmt.Errorf("unknown EC2 IMDS endpoint mode, must be either IPv6 or IPv4") + } + + options = append(options, config.WithEC2IMDSEndpointMode(mode)) + } + + if provider.SkipMetadataAPICheck { + options = append(options, config.WithEC2IMDSClientEnableState(imds.ClientDisabled)) + } + + proxyConfig := httpproxy.FromEnvironment() + + if provider.HTTPProxy != "" { + proxyConfig.HTTPProxy = provider.HTTPProxy + } + + if provider.HTTPSProxy != "" { + proxyConfig.HTTPSProxy = provider.HTTPSProxy + } + + if provider.NoProxy != "" { + proxyConfig.NoProxy = provider.NoProxy + } + + // Always append the HTTP client that is configured with all our required + // proxy settings + // TODO: Can we inherit a transport here for things like OTEL? + httpClient := awshttp.NewBuildableClient() + httpClient.WithTransportOptions(func(t *http.Transport) { + t.Proxy = func(r *http.Request) (*url.URL, error) { + return proxyConfig.ProxyFunc()(r.URL) + } + }) + options = append(options, config.WithHTTPClient(httpClient)) + + if provider.MaxRetries != 0 { + options = append(options, config.WithRetryMaxAttempts(provider.MaxRetries)) + } + + if provider.Profile != "" { + options = append(options, config.WithSharedConfigProfile(provider.Profile)) + } + + if provider.RetryMode != "" { + switch { + case strings.EqualFold(provider.RetryMode, "standard"): + options = append(options, config.WithRetryMode(aws.RetryModeStandard)) + case strings.EqualFold(provider.RetryMode, "adaptive"): + options = append(options, config.WithRetryMode(aws.RetryModeAdaptive)) + default: + return aws.Config{}, fmt.Errorf("unknown retry mode: %s. Must be 'standard' or 'adaptive'", provider.RetryMode) + } + } + + if len(provider.SharedConfigFiles) != 0 { + options = append(options, config.WithSharedConfigFiles(provider.SharedConfigFiles)) + } + + if len(provider.SharedCredentialsFiles) != 0 { + options = append(options, config.WithSharedCredentialsFiles(provider.SharedCredentialsFiles)) + } + + if provider.UseDualStackEndpoint { + options = append(options, config.WithUseDualStackEndpoint(aws.DualStackEndpointStateEnabled)) + } + + if provider.UseFIPSEndpoint { + options = append(options, config.WithUseFIPSEndpoint(aws.FIPSEndpointStateEnabled)) + } + + return config.LoadDefaultConfig(ctx, options...) +} diff --git a/tfutils/aws_config_test.go b/tfutils/aws_config_test.go new file mode 100644 index 00000000..28e623b3 --- /dev/null +++ b/tfutils/aws_config_test.go @@ -0,0 +1,177 @@ +package tfutils + +import ( + "context" + "testing" +) + +func TestParseAWSProviders(t *testing.T) { + providers, files, err := ParseAWSProviders("testdata") + if err != nil { + t.Errorf("Error parsing AWS providers: %v", err) + } + + if len(files) != 3 { + t.Errorf("Expected 3 files, got %d", len(files)) + } + + if len(providers) != 5 { + t.Fatalf("Expected 5 providers, got %d", len(providers)) + } + + if providers[1].Region != "us-east-1" { + t.Errorf("Expected region us-east-1, got %s", providers[0].Region) + } + + if providers[2].AssumeRole.RoleARN != "arn:aws:iam::123456789012:role/ROLE_NAME" { + t.Errorf("Expected role arn arn:aws:iam::123456789012:role/ROLE_NAME, got %s", providers[2].AssumeRole.RoleARN) + } + + if providers[2].AssumeRole.SessionName != "SESSION_NAME" { + t.Errorf("Expected session name SESSION_NAME, got % s", providers[2].AssumeRole.SessionName) + } + + if providers[2].AssumeRole.ExternalID != "EXTERNAL_ID" { + t.Errorf("Expected external id EXTERNAL_ID, got %s", providers[2].AssumeRole.ExternalID) + } + + if providers[3].AccessKey != "access_key" { + t.Errorf("Expected access key access_key, got %s", providers[3].AccessKey) + } + + if providers[3].SecretKey != "secret_key" { + t.Errorf("Expected secret key secret_key, got %s", providers[3].SecretKey) + } + + if providers[3].Token != "token" { + t.Errorf("Expected token token, got %s", providers[3].Token) + } + + if providers[3].Region != "region" { + t.Errorf("Expected region region, got %s", providers[3].Region) + } + + if providers[3].CustomCABundle != "testdata/providers.tf" { + t.Errorf("Expected custom ca bundle testdata/providers.tf, got %s", providers[3].CustomCABundle) + } + + if providers[3].EC2MetadataServiceEndpoint != "ec2_metadata_service_endpoint" { + t.Errorf("Expected ec2 metadata service endpoint ec2_metadata_service_endpoint, got %s", providers[3].EC2MetadataServiceEndpoint) + } + + if providers[3].EC2MetadataServiceEndpointMode != "ipv6" { + t.Errorf("Expected ec2 metadata service endpoint mode ipv6, got %s", providers[3].EC2MetadataServiceEndpointMode) + } + + if providers[3].SkipMetadataAPICheck != true { + t.Errorf("Expected skip metadata api check true, got %t", providers[3].SkipMetadataAPICheck) + } + + if providers[3].HTTPProxy != "http_proxy" { + t.Errorf("Expected http proxy http_proxy, got %s", providers[3].HTTPProxy) + } + + if providers[3].HTTPSProxy != "https_proxy" { + t.Errorf("Expected https proxy https_proxy, got %s", providers[3].HTTPSProxy) + } + + if providers[3].NoProxy != "no_proxy" { + t.Errorf("Expected no proxy no_proxy, got %s", providers[3].NoProxy) + } + + if providers[3].MaxRetries != 10 { + t.Errorf("Expected max retries 10, got %d", providers[3].MaxRetries) + } + + if providers[3].Profile != "profile" { + t.Errorf("Expected profile profile, got %s", providers[3].Profile) + } + + if providers[3].RetryMode != "standard" { + t.Errorf("Expected retry mode standard, got %s", providers[3].RetryMode) + } + + if len(providers[3].SharedConfigFiles) != 1 { + t.Errorf("Expected 1 shared config file, got %d", len(providers[3].SharedConfigFiles)) + } + + if providers[3].SharedConfigFiles[0] != "shared_config_files" { + t.Errorf("Expected shared config file shared_config_files, got %s", providers[3].SharedConfigFiles[0]) + } + + if len(providers[3].SharedCredentialsFiles) != 1 { + t.Errorf("Expected 1 shared credentials file, got %d", len(providers[3].SharedCredentialsFiles)) + } + + if providers[3].SharedCredentialsFiles[0] != "shared_credentials_files" { + t.Errorf("Expected shared credentials file shared_credentials_files, got %s", providers[3].SharedCredentialsFiles[0]) + } + + if providers[3].UseDualStackEndpoint != false { + t.Errorf("Expected use dual stack endpoint false, got %t", providers[3].UseDualStackEndpoint) + } + + if providers[3].UseFIPSEndpoint != false { + t.Errorf("Expected use fips endpoint false, got %t", providers[3].UseFIPSEndpoint) + } + + if providers[3].AssumeRoleWithWebIdentity.RoleARN != "arn:aws:iam::123456789012:role/ROLE_NAME" { + t.Errorf("Expected role arn arn:aws:iam::123456789012:role/ROLE_NAME, got %s", providers[3].AssumeRoleWithWebIdentity.RoleARN) + } + + if providers[3].AssumeRoleWithWebIdentity.SessionName != "SESSION_NAME" { + t.Errorf("Expected session name SESSION_NAME, got %s", providers[3].AssumeRoleWithWebIdentity.SessionName) + } + + if providers[3].AssumeRoleWithWebIdentity.WebIdentityTokenFile != "/Users/tf_user/secrets/web-identity-token" { + t.Errorf("Expected web identity token file /Users/tf_user/secrets/web-identity-token, got %s", providers[3].AssumeRoleWithWebIdentity.WebIdentityTokenFile) + } + + if providers[3].AssumeRoleWithWebIdentity.WebIdentityToken != "web_identity_token" { + t.Errorf("Expected web identity token web_identity_token, got %s", providers[3].AssumeRoleWithWebIdentity.WebIdentityToken) + } + + if providers[3].AssumeRoleWithWebIdentity.Duration != "1s" { + t.Errorf("Expected duration 1s, got %s", providers[3].AssumeRoleWithWebIdentity.Duration) + } + + if providers[3].AssumeRoleWithWebIdentity.Policy != "policy" { + t.Errorf("Expected policy policy, got %s", providers[3].AssumeRoleWithWebIdentity.Policy) + } + + if len(providers[3].AssumeRoleWithWebIdentity.PolicyARNs) != 1 { + t.Errorf("Expected 1 policy arn, got %d", len(providers[3].AssumeRoleWithWebIdentity.PolicyARNs)) + } + + if providers[3].AssumeRoleWithWebIdentity.PolicyARNs[0] != "policy_arns" { + t.Errorf("Expected policy arn policy_arns, got %s", providers[3].AssumeRoleWithWebIdentity.PolicyARNs[0]) + } + + if providers[4].Region != "us-west-2" { + t.Errorf("Expected region us-west-2, got %s", providers[4].Region) + } + + if providers[4].AccessKey != "my-access-key" { + t.Errorf("Expected access key my-access-key, got %s", providers[4].AccessKey) + } + + if providers[4].SecretKey != "my-secret-key" { + t.Errorf("Expected secret key my-secret-key, got %s", providers[4].SecretKey) + } +} + +func TestConfigFromProvider(t *testing.T) { + // Make sure the providers we have created can all be turned into configs + // without any issues + providers, _, err := ParseAWSProviders("testdata/config_from_provider") + if err != nil { + t.Fatalf("Error parsing AWS providers: %v", err) + } + + for _, provider := range providers { + _, err := ConfigFromProvider(context.Background(), provider) + if err != nil { + t.Errorf("Error converting provider to config: %v", err) + } + } +} diff --git a/tfutils/plan.go b/tfutils/plan.go index de9b7b07..ddfc6271 100644 --- a/tfutils/plan.go +++ b/tfutils/plan.go @@ -27,7 +27,7 @@ type Plan struct { ResourceChanges []ResourceChange `json:"resource_changes,omitempty"` OutputChanges map[string]Change `json:"output_changes,omitempty"` PriorState State `json:"prior_state,omitempty"` - Config config `json:"configuration,omitempty"` + Config planConfig `json:"configuration,omitempty"` RelevantAttributes []ResourceAttr `json:"relevant_attributes,omitempty"` Checks json.RawMessage `json:"checks,omitempty"` Timestamp string `json:"timestamp,omitempty"` @@ -35,7 +35,7 @@ type Plan struct { } // Config represents the complete configuration source -type config struct { +type planConfig struct { ProviderConfigs map[string]ProviderConfig `json:"provider_config,omitempty"` RootModule ConfigModule `json:"root_module,omitempty"` } diff --git a/tfutils/testdata/config_from_provider/ca-bundle.crt b/tfutils/testdata/config_from_provider/ca-bundle.crt new file mode 100644 index 00000000..f2da5248 --- /dev/null +++ b/tfutils/testdata/config_from_provider/ca-bundle.crt @@ -0,0 +1,97 @@ +Certificate: + Data: + Version: 1 (0x0) + Serial Number: + 02:ad:66:7e:4e:45:fe:5e:57:6f:3c:98:19:5e:dd:c0 + Signature Algorithm: md2WithRSAEncryption + Issuer: C=US, O=RSA Data Security, Inc., OU=Secure Server Certification Authority + Validity + Not Before: Nov 9 00:00:00 1994 GMT + Not After : Jan 7 23:59:59 2010 GMT + Subject: C=US, O=RSA Data Security, Inc., OU=Secure Server Certification Authority + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public Key: (1000 bit) + Modulus (1000 bit): + 00:92:ce:7a:c1:ae:83:3e:5a:aa:89:83:57:ac:25: + 01:76:0c:ad:ae:8e:2c:37:ce:eb:35:78:64:54:03: + e5:84:40:51:c9:bf:8f:08:e2:8a:82:08:d2:16:86: + 37:55:e9:b1:21:02:ad:76:68:81:9a:05:a2:4b:c9: + 4b:25:66:22:56:6c:88:07:8f:f7:81:59:6d:84:07: + 65:70:13:71:76:3e:9b:77:4c:e3:50:89:56:98:48: + b9:1d:a7:29:1a:13:2e:4a:11:59:9c:1e:15:d5:49: + 54:2c:73:3a:69:82:b1:97:39:9c:6d:70:67:48:e5: + dd:2d:d6:c8:1e:7b + Exponent: 65537 (0x10001) + Signature Algorithm: md2WithRSAEncryption + 65:dd:7e:e1:b2:ec:b0:e2:3a:e0:ec:71:46:9a:19:11:b8:d3: + c7:a0:b4:03:40:26:02:3e:09:9c:e1:12:b3:d1:5a:f6:37:a5: + b7:61:03:b6:5b:16:69:3b:c6:44:08:0c:88:53:0c:6b:97:49: + c7:3e:35:dc:6c:b9:bb:aa:df:5c:bb:3a:2f:93:60:b6:a9:4b: + 4d:f2:20:f7:cd:5f:7f:64:7b:8e:dc:00:5c:d7:fa:77:ca:39: + 16:59:6f:0e:ea:d3:b5:83:7f:4d:4d:42:56:76:b4:c9:5f:04: + f8:38:f8:eb:d2:5f:75:5f:cd:7b:fc:e5:8e:80:7c:fc:50 +MD5 Fingerprint=74:7B:82:03:43:F0:00:9E:6B:B3:EC:47:BF:85:A5:93 +-----BEGIN CERTIFICATE----- +MIICNDCCAaECEAKtZn5ORf5eV288mBle3cAwDQYJKoZIhvcNAQECBQAwXzELMAkG +A1UEBhMCVVMxIDAeBgNVBAoTF1JTQSBEYXRhIFNlY3VyaXR5LCBJbmMuMS4wLAYD +VQQLEyVTZWN1cmUgU2VydmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTk0 +MTEwOTAwMDAwMFoXDTEwMDEwNzIzNTk1OVowXzELMAkGA1UEBhMCVVMxIDAeBgNV +BAoTF1JTQSBEYXRhIFNlY3VyaXR5LCBJbmMuMS4wLAYDVQQLEyVTZWN1cmUgU2Vy +dmVyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIGbMA0GCSqGSIb3DQEBAQUAA4GJ +ADCBhQJ+AJLOesGugz5aqomDV6wlAXYMra6OLDfO6zV4ZFQD5YRAUcm/jwjiioII +0haGN1XpsSECrXZogZoFokvJSyVmIlZsiAeP94FZbYQHZXATcXY+m3dM41CJVphI +uR2nKRoTLkoRWZweFdVJVCxzOmmCsZc5nG1wZ0jl3S3WyB57AgMBAAEwDQYJKoZI +hvcNAQECBQADfgBl3X7hsuyw4jrg7HFGmhkRuNPHoLQDQCYCPgmc4RKz0Vr2N6W3 +YQO2WxZpO8ZECAyIUwxrl0nHPjXcbLm7qt9cuzovk2C2qUtN8iD3zV9/ZHuO3ABc +1/p3yjkWWW8O6tO1g39NTUJWdrTJXwT4OPjr0l91X817/OWOgHz8UA== +-----END CERTIFICATE----- + + +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 419 (0x1a3) + Signature Algorithm: md5WithRSAEncryption + Issuer: C=US, O=GTE Corporation, CN=GTE CyberTrust Root + Validity + Not Before: Feb 23 23:01:00 1996 GMT + Not After : Feb 23 23:59:00 2006 GMT + Subject: C=US, O=GTE Corporation, CN=GTE CyberTrust Root + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public Key: (1024 bit) + Modulus (1024 bit): + 00:b8:e6:4f:ba:db:98:7c:71:7c:af:44:b7:d3:0f: + 46:d9:64:e5:93:c1:42:8e:c7:ba:49:8d:35:2d:7a: + e7:8b:bd:e5:05:31:59:c6:b1:2f:0a:0c:fb:9f:a7: + 3f:a2:09:66:84:56:1e:37:29:1b:87:e9:7e:0c:ca: + 9a:9f:a5:7f:f5:15:94:a3:d5:a2:46:82:d8:68:4c: + d1:37:15:06:68:af:bd:f8:b0:b3:f0:29:f5:95:5a: + 09:16:61:77:0a:22:25:d4:4f:45:aa:c7:bd:e5:96: + df:f9:d4:a8:8e:42:cc:24:c0:1e:91:27:4a:b5:6d: + 06:80:63:39:c4:a2:5e:38:03 + Exponent: 65537 (0x10001) + Signature Algorithm: md5WithRSAEncryption + 12:b3:75:c6:5f:1d:e1:61:55:80:00:d4:81:4b:7b:31:0f:23: + 63:e7:3d:f3:03:f9:f4:36:a8:bb:d9:e3:a5:97:4d:ea:2b:29: + e0:d6:6a:73:81:e6:c0:89:a3:d3:f1:e0:a5:a5:22:37:9a:63: + c2:48:20:b4:db:72:e3:c8:f6:d9:7c:be:b1:af:53:da:14:b4: + 21:b8:d6:d5:96:e3:fe:4e:0c:59:62:b6:9a:4a:f9:42:dd:8c: + 6f:81:a9:71:ff:f4:0a:72:6d:6d:44:0e:9d:f3:74:74:a8:d5: + 34:49:e9:5e:9e:e9:b4:7a:e1:e5:5a:1f:84:30:9c:d3:9f:a5: + 25:d8 +MD5 Fingerprint=C4:D7:F0:B2:A3:C5:7D:61:67:F0:04:CD:43:D3:BA:58 +-----BEGIN CERTIFICATE----- +MIIB+jCCAWMCAgGjMA0GCSqGSIb3DQEBBAUAMEUxCzAJBgNVBAYTAlVTMRgwFgYD +VQQKEw9HVEUgQ29ycG9yYXRpb24xHDAaBgNVBAMTE0dURSBDeWJlclRydXN0IFJv +b3QwHhcNOTYwMjIzMjMwMTAwWhcNMDYwMjIzMjM1OTAwWjBFMQswCQYDVQQGEwJV +UzEYMBYGA1UEChMPR1RFIENvcnBvcmF0aW9uMRwwGgYDVQQDExNHVEUgQ3liZXJU +cnVzdCBSb290MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC45k+625h8cXyv +RLfTD0bZZOWTwUKOx7pJjTUteueLveUFMVnGsS8KDPufpz+iCWaEVh43KRuH6X4M +ypqfpX/1FZSj1aJGgthoTNE3FQZor734sLPwKfWVWgkWYXcKIiXUT0Wqx73llt/5 +1KiOQswkwB6RJ0q1bQaAYznEol44AwIDAQABMA0GCSqGSIb3DQEBBAUAA4GBABKz +dcZfHeFhVYAA1IFLezEPI2PnPfMD+fQ2qLvZ46WXTeorKeDWanOB5sCJo9Px4KWl +IjeaY8JIILTbcuPI9tl8vrGvU9oUtCG41tWW4/5ODFlitppK+ULdjG+BqXH/9Apy +bW1EDp3zdHSo1TRJ6V6e6bR64eVaH4QwnNOfpSXY +-----END CERTIFICATE----- \ No newline at end of file diff --git a/tfutils/testdata/config_from_provider/test.tf b/tfutils/testdata/config_from_provider/test.tf new file mode 100644 index 00000000..c194ee63 --- /dev/null +++ b/tfutils/testdata/config_from_provider/test.tf @@ -0,0 +1,49 @@ +# This exists to test the ConfigFromProvider method, we have to omit a few +# things since when we are constructing the AWS config it actually does real +# validation like making sure a profile exists in the shared config files, etc. +# So we have to omit those fields in the test file. +provider "aws" { + alias = "everything" + access_key = "access_key" + secret_key = "secret_key" + token = "token" + region = "region" + custom_ca_bundle = "testdata/config_from_provider/ca-bundle.crt" + ec2_metadata_service_endpoint = "ec2_metadata_service_endpoint" + ec2_metadata_service_endpoint_mode = "ipv6" + skip_metadata_api_check = true + http_proxy = "http_proxy" + https_proxy = "https_proxy" + no_proxy = "no_proxy" + max_retries = 10 +# profile = "profile" + retry_mode = "standard" + shared_config_files = ["shared_config_files"] + shared_credentials_files = ["shared_credentials_files"] + s3_us_east_1_regional_endpoint = "s3_us_east_1_regional_endpoint" + use_dualstack_endpoint = false + use_fips_endpoint = false + + assume_role { + role_arn = "arn:aws:iam::123456789012:role/ROLE_NAME" + session_name = "SESSION_NAME" + external_id = "EXTERNAL_ID" + duration = "1s" + policy = "policy" + policy_arns = ["policy_arns"] + tags = { + key = "value" + } + } + + assume_role_with_web_identity { + role_arn = "arn:aws:iam::123456789012:role/ROLE_NAME" + session_name = "SESSION_NAME" + web_identity_token_file = "/Users/tf_user/secrets/web-identity-token" + web_identity_token = "web_identity_token" + duration = "1s" + policy = "policy" + policy_arns = ["policy_arns"] + } + +} \ No newline at end of file diff --git a/tfutils/testdata/providers.tf b/tfutils/testdata/providers.tf new file mode 100644 index 00000000..5db851df --- /dev/null +++ b/tfutils/testdata/providers.tf @@ -0,0 +1,91 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.16" + } + } + + required_version = ">= 1.2.0" +} + +// Provider that should be ignored +provider "google" { + project = "acme-app" + region = "us-central1" +} + +// This should also be ignonred +variable "image_id" { + type = string +} + +// This should be ignored too +resource "aws_instance" "app_server" { + ami = "ami-830c94e3" + instance_type = "t2.micro" + + tags = { + Name = "ExampleAppServerInstance" + } +} + +provider "aws" { + region = "us-east-1" +} + +provider "aws" { + alias = "assume_role" + + assume_role { + role_arn = "arn:aws:iam::123456789012:role/ROLE_NAME" + session_name = "SESSION_NAME" + external_id = "EXTERNAL_ID" + } +} + +provider "aws" { + alias = "everything" + access_key = "access_key" + secret_key = "secret_key" + token = "token" + region = "region" + custom_ca_bundle = "testdata/providers.tf" + ec2_metadata_service_endpoint = "ec2_metadata_service_endpoint" + ec2_metadata_service_endpoint_mode = "ipv6" + skip_metadata_api_check = true + http_proxy = "http_proxy" + https_proxy = "https_proxy" + no_proxy = "no_proxy" + max_retries = 10 + profile = "profile" + retry_mode = "standard" + shared_config_files = ["shared_config_files"] + shared_credentials_files = ["shared_credentials_files"] + s3_us_east_1_regional_endpoint = "s3_us_east_1_regional_endpoint" + use_dualstack_endpoint = false + use_fips_endpoint = false + + assume_role { + role_arn = "arn:aws:iam::123456789012:role/ROLE_NAME" + session_name = "SESSION_NAME" + external_id = "EXTERNAL_ID" + duration = "1s" + policy = "policy" + policy_arns = ["policy_arns"] + tags = { + key = "value" + } + } + + assume_role_with_web_identity { + role_arn = "arn:aws:iam::123456789012:role/ROLE_NAME" + session_name = "SESSION_NAME" + web_identity_token_file = "/Users/tf_user/secrets/web-identity-token" + web_identity_token = "web_identity_token" + duration = "1s" + policy = "policy" + policy_arns = ["policy_arns"] + } + +} diff --git a/tfutils/testdata/subfolder/more_providers.tf b/tfutils/testdata/subfolder/more_providers.tf new file mode 100644 index 00000000..a64e9c2d --- /dev/null +++ b/tfutils/testdata/subfolder/more_providers.tf @@ -0,0 +1,6 @@ +provider "aws" { + alias = "subdir" + region = "us-west-2" + access_key = "my-access-key" + secret_key = "my-secret-key" +} \ No newline at end of file From 9201ca842fb08e38c56a9408c60cb41ee3f43ee6 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Mon, 22 Jul 2024 07:48:11 +0000 Subject: [PATCH 02/17] Added WIP variable handling --- tfutils/aws_config.go | 24 +++++++++++++++++++++--- tfutils/aws_config_test.go | 4 ++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/tfutils/aws_config.go b/tfutils/aws_config.go index 149095f3..9b538d09 100644 --- a/tfutils/aws_config.go +++ b/tfutils/aws_config.go @@ -96,14 +96,24 @@ type AssumeRoleWithWebIdentity struct { Remain hcl.Body `hcl:",remain"` } +// Loads the eval context from the following locations: +// +// - `-var-file=FILENAME` files: These should be passed as file paths +// - `-var 'NAME=VALUE'` arguments: These should be passed as a list of strings +// - Environment Variables: These should be passed as a []strings (from `os.Environ()`), +// variables beginning with TF_VAR_ will be used +func LoadEvalContext(varsFiles []string, args []string, env []string) (*hcl.EvalContext, error) { + return nil, nil +} + // Parses AWS provider config from all terraform files in the given directory, // recursing into subdirectories. Returns a list of AWS providers and a list of // files that were parsed. -func ParseAWSProviders(dir string) ([]AWSProvider, []string, error) { +func ParseAWSProviders(terraformDir string, evalContext *hcl.EvalContext) ([]AWSProvider, []string, error) { files := make([]string, 0) // Get all files matching *.tf from everywhere under the directory - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + err := filepath.Walk(terraformDir, func(path string, info os.FileInfo, err error) error { if err != nil { return fmt.Errorf("error searching for terraform files: %w", err) } @@ -120,6 +130,14 @@ func ParseAWSProviders(dir string) ([]AWSProvider, []string, error) { parser := hclparse.NewParser() awsProviders := make([]AWSProvider, 0) + // TODO: We need to also make sure we have all the variables and inputs set + // up so that dynamic values can be used. These could come from -vars-file + // or the Environment. It's also possible to have a provider in a module + // that sets parameters based on the input variables of the module + // + // * [ ] Parse tfvars files and arguments + // * [ ] Parse environment variables + // Iterate over the files for _, file := range files { b, err := os.ReadFile(file) @@ -134,7 +152,7 @@ func ParseAWSProviders(dir string) ([]AWSProvider, []string, error) { } providerFile := ProviderFile{} - diag = gohcl.DecodeBody(parsedFile.Body, &hcl.EvalContext{}, &providerFile) + diag = gohcl.DecodeBody(parsedFile.Body, evalContext, &providerFile) if diag.HasErrors() { return nil, files, fmt.Errorf("error decoding terraform file: (%v) %w", file, diag) } diff --git a/tfutils/aws_config_test.go b/tfutils/aws_config_test.go index 28e623b3..f98c309b 100644 --- a/tfutils/aws_config_test.go +++ b/tfutils/aws_config_test.go @@ -6,7 +6,7 @@ import ( ) func TestParseAWSProviders(t *testing.T) { - providers, files, err := ParseAWSProviders("testdata") + providers, files, err := ParseAWSProviders("testdata", nil) if err != nil { t.Errorf("Error parsing AWS providers: %v", err) } @@ -163,7 +163,7 @@ func TestParseAWSProviders(t *testing.T) { func TestConfigFromProvider(t *testing.T) { // Make sure the providers we have created can all be turned into configs // without any issues - providers, _, err := ParseAWSProviders("testdata/config_from_provider") + providers, _, err := ParseAWSProviders("testdata/config_from_provider", nil) if err != nil { t.Fatalf("Error parsing AWS providers: %v", err) } From 2ee46429f0f6116f5ab6a0a058a592a48accf42e Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Mon, 22 Jul 2024 08:25:26 +0000 Subject: [PATCH 03/17] Fix command signatures --- tfutils/aws_config.go | 44 ++++++++--- tfutils/aws_config_test.go | 158 ++++++++++++++++++------------------- 2 files changed, 112 insertions(+), 90 deletions(-) diff --git a/tfutils/aws_config.go b/tfutils/aws_config.go index 9b538d09..15acd4e5 100644 --- a/tfutils/aws_config.go +++ b/tfutils/aws_config.go @@ -106,10 +106,21 @@ func LoadEvalContext(varsFiles []string, args []string, env []string) (*hcl.Eval return nil, nil } +type ProviderResult struct { + Provider *AWSProvider + Error error + FilePath string +} + // Parses AWS provider config from all terraform files in the given directory, // recursing into subdirectories. Returns a list of AWS providers and a list of -// files that were parsed. -func ParseAWSProviders(terraformDir string, evalContext *hcl.EvalContext) ([]AWSProvider, []string, error) { +// files that were parsed. This will return an error only if there was an error +// loading the files. ProviderResults will be returned for: +// +// * Files that could not be parsed at all (just an error) +// * Files that contained an AWS provider that we couldn't fully evaluate (with an error) +// * Files that contained an AWS provider that we could fully evaluate (with no error) +func ParseAWSProviders(terraformDir string, evalContext *hcl.EvalContext) ([]ProviderResult, error) { files := make([]string, 0) // Get all files matching *.tf from everywhere under the directory @@ -124,11 +135,11 @@ func ParseAWSProviders(terraformDir string, evalContext *hcl.EvalContext) ([]AWS }) if err != nil { - return nil, nil, err + return nil, err } parser := hclparse.NewParser() - awsProviders := make([]AWSProvider, 0) + results := make([]ProviderResult, 0) // TODO: We need to also make sure we have all the variables and inputs set // up so that dynamic values can be used. These could come from -vars-file @@ -142,29 +153,44 @@ func ParseAWSProviders(terraformDir string, evalContext *hcl.EvalContext) ([]AWS for _, file := range files { b, err := os.ReadFile(file) if err != nil { - return nil, files, fmt.Errorf("error reading terraform file: (%v) %w", file, err) + results = append(results, ProviderResult{ + Error: fmt.Errorf("error reading terraform file: (%v) %w", file, err), + FilePath: file, + }) + continue } // Parse the HCL file parsedFile, diag := parser.ParseHCL(b, file) if diag.HasErrors() { - return nil, files, fmt.Errorf("error parsing terraform file: (%v) %w", file, diag) + results = append(results, ProviderResult{ + Error: fmt.Errorf("error parsing terraform file: (%v) %w", file, diag), + FilePath: file, + }) + continue } providerFile := ProviderFile{} diag = gohcl.DecodeBody(parsedFile.Body, evalContext, &providerFile) if diag.HasErrors() { - return nil, files, fmt.Errorf("error decoding terraform file: (%v) %w", file, diag) + results = append(results, ProviderResult{ + Error: fmt.Errorf("error decoding terraform file: (%v) %w", file, diag), + FilePath: file, + }) + continue } for _, provider := range providerFile.Providers { if provider.Name == "aws" { - awsProviders = append(awsProviders, provider) + results = append(results, ProviderResult{ + Provider: &provider, //nolint:all // this is not relevant for go1.22 and later + FilePath: file, + }) } } } - return awsProviders, files, nil + return results, nil } // ConfigFromProvider creates an aws.Config from an AWSProvider that uses the diff --git a/tfutils/aws_config_test.go b/tfutils/aws_config_test.go index f98c309b..65818040 100644 --- a/tfutils/aws_config_test.go +++ b/tfutils/aws_config_test.go @@ -6,170 +6,166 @@ import ( ) func TestParseAWSProviders(t *testing.T) { - providers, files, err := ParseAWSProviders("testdata", nil) + results, err := ParseAWSProviders("testdata", nil) if err != nil { - t.Errorf("Error parsing AWS providers: %v", err) + t.Errorf("Error parsing AWS results: %v", err) } - if len(files) != 3 { - t.Errorf("Expected 3 files, got %d", len(files)) + if len(results) != 5 { + t.Fatalf("Expected 5 results, got %d", len(results)) } - if len(providers) != 5 { - t.Fatalf("Expected 5 providers, got %d", len(providers)) + if results[1].Provider.Region != "us-east-1" { + t.Errorf("Expected region us-east-1, got %s", results[0].Provider.Region) } - if providers[1].Region != "us-east-1" { - t.Errorf("Expected region us-east-1, got %s", providers[0].Region) + if results[2].Provider.AssumeRole.RoleARN != "arn:aws:iam::123456789012:role/ROLE_NAME" { + t.Errorf("Expected role arn arn:aws:iam::123456789012:role/ROLE_NAME, got %s", results[2].Provider.AssumeRole.RoleARN) } - if providers[2].AssumeRole.RoleARN != "arn:aws:iam::123456789012:role/ROLE_NAME" { - t.Errorf("Expected role arn arn:aws:iam::123456789012:role/ROLE_NAME, got %s", providers[2].AssumeRole.RoleARN) + if results[2].Provider.AssumeRole.SessionName != "SESSION_NAME" { + t.Errorf("Expected session name SESSION_NAME, got % s", results[2].Provider.AssumeRole.SessionName) } - if providers[2].AssumeRole.SessionName != "SESSION_NAME" { - t.Errorf("Expected session name SESSION_NAME, got % s", providers[2].AssumeRole.SessionName) + if results[2].Provider.AssumeRole.ExternalID != "EXTERNAL_ID" { + t.Errorf("Expected external id EXTERNAL_ID, got %s", results[2].Provider.AssumeRole.ExternalID) } - if providers[2].AssumeRole.ExternalID != "EXTERNAL_ID" { - t.Errorf("Expected external id EXTERNAL_ID, got %s", providers[2].AssumeRole.ExternalID) + if results[3].Provider.AccessKey != "access_key" { + t.Errorf("Expected access key access_key, got %s", results[3].Provider.AccessKey) } - if providers[3].AccessKey != "access_key" { - t.Errorf("Expected access key access_key, got %s", providers[3].AccessKey) + if results[3].Provider.SecretKey != "secret_key" { + t.Errorf("Expected secret key secret_key, got %s", results[3].Provider.SecretKey) } - if providers[3].SecretKey != "secret_key" { - t.Errorf("Expected secret key secret_key, got %s", providers[3].SecretKey) + if results[3].Provider.Token != "token" { + t.Errorf("Expected token token, got %s", results[3].Provider.Token) } - if providers[3].Token != "token" { - t.Errorf("Expected token token, got %s", providers[3].Token) + if results[3].Provider.Region != "region" { + t.Errorf("Expected region region, got %s", results[3].Provider.Region) } - if providers[3].Region != "region" { - t.Errorf("Expected region region, got %s", providers[3].Region) + if results[3].Provider.CustomCABundle != "testdata/providers.tf" { + t.Errorf("Expected custom ca bundle testdata/providers.tf, got %s", results[3].Provider.CustomCABundle) } - if providers[3].CustomCABundle != "testdata/providers.tf" { - t.Errorf("Expected custom ca bundle testdata/providers.tf, got %s", providers[3].CustomCABundle) + if results[3].Provider.EC2MetadataServiceEndpoint != "ec2_metadata_service_endpoint" { + t.Errorf("Expected ec2 metadata service endpoint ec2_metadata_service_endpoint, got %s", results[3].Provider.EC2MetadataServiceEndpoint) } - if providers[3].EC2MetadataServiceEndpoint != "ec2_metadata_service_endpoint" { - t.Errorf("Expected ec2 metadata service endpoint ec2_metadata_service_endpoint, got %s", providers[3].EC2MetadataServiceEndpoint) + if results[3].Provider.EC2MetadataServiceEndpointMode != "ipv6" { + t.Errorf("Expected ec2 metadata service endpoint mode ipv6, got %s", results[3].Provider.EC2MetadataServiceEndpointMode) } - if providers[3].EC2MetadataServiceEndpointMode != "ipv6" { - t.Errorf("Expected ec2 metadata service endpoint mode ipv6, got %s", providers[3].EC2MetadataServiceEndpointMode) + if results[3].Provider.SkipMetadataAPICheck != true { + t.Errorf("Expected skip metadata api check true, got %t", results[3].Provider.SkipMetadataAPICheck) } - if providers[3].SkipMetadataAPICheck != true { - t.Errorf("Expected skip metadata api check true, got %t", providers[3].SkipMetadataAPICheck) + if results[3].Provider.HTTPProxy != "http_proxy" { + t.Errorf("Expected http proxy http_proxy, got %s", results[3].Provider.HTTPProxy) } - if providers[3].HTTPProxy != "http_proxy" { - t.Errorf("Expected http proxy http_proxy, got %s", providers[3].HTTPProxy) + if results[3].Provider.HTTPSProxy != "https_proxy" { + t.Errorf("Expected https proxy https_proxy, got %s", results[3].Provider.HTTPSProxy) } - if providers[3].HTTPSProxy != "https_proxy" { - t.Errorf("Expected https proxy https_proxy, got %s", providers[3].HTTPSProxy) + if results[3].Provider.NoProxy != "no_proxy" { + t.Errorf("Expected no proxy no_proxy, got %s", results[3].Provider.NoProxy) } - if providers[3].NoProxy != "no_proxy" { - t.Errorf("Expected no proxy no_proxy, got %s", providers[3].NoProxy) + if results[3].Provider.MaxRetries != 10 { + t.Errorf("Expected max retries 10, got %d", results[3].Provider.MaxRetries) } - if providers[3].MaxRetries != 10 { - t.Errorf("Expected max retries 10, got %d", providers[3].MaxRetries) + if results[3].Provider.Profile != "profile" { + t.Errorf("Expected profile profile, got %s", results[3].Provider.Profile) } - if providers[3].Profile != "profile" { - t.Errorf("Expected profile profile, got %s", providers[3].Profile) + if results[3].Provider.RetryMode != "standard" { + t.Errorf("Expected retry mode standard, got %s", results[3].Provider.RetryMode) } - if providers[3].RetryMode != "standard" { - t.Errorf("Expected retry mode standard, got %s", providers[3].RetryMode) + if len(results[3].Provider.SharedConfigFiles) != 1 { + t.Errorf("Expected 1 shared config file, got %d", len(results[3].Provider.SharedConfigFiles)) } - if len(providers[3].SharedConfigFiles) != 1 { - t.Errorf("Expected 1 shared config file, got %d", len(providers[3].SharedConfigFiles)) + if results[3].Provider.SharedConfigFiles[0] != "shared_config_files" { + t.Errorf("Expected shared config file shared_config_files, got %s", results[3].Provider.SharedConfigFiles[0]) } - if providers[3].SharedConfigFiles[0] != "shared_config_files" { - t.Errorf("Expected shared config file shared_config_files, got %s", providers[3].SharedConfigFiles[0]) + if len(results[3].Provider.SharedCredentialsFiles) != 1 { + t.Errorf("Expected 1 shared credentials file, got %d", len(results[3].Provider.SharedCredentialsFiles)) } - if len(providers[3].SharedCredentialsFiles) != 1 { - t.Errorf("Expected 1 shared credentials file, got %d", len(providers[3].SharedCredentialsFiles)) + if results[3].Provider.SharedCredentialsFiles[0] != "shared_credentials_files" { + t.Errorf("Expected shared credentials file shared_credentials_files, got %s", results[3].Provider.SharedCredentialsFiles[0]) } - if providers[3].SharedCredentialsFiles[0] != "shared_credentials_files" { - t.Errorf("Expected shared credentials file shared_credentials_files, got %s", providers[3].SharedCredentialsFiles[0]) + if results[3].Provider.UseDualStackEndpoint != false { + t.Errorf("Expected use dual stack endpoint false, got %t", results[3].Provider.UseDualStackEndpoint) } - if providers[3].UseDualStackEndpoint != false { - t.Errorf("Expected use dual stack endpoint false, got %t", providers[3].UseDualStackEndpoint) + if results[3].Provider.UseFIPSEndpoint != false { + t.Errorf("Expected use fips endpoint false, got %t", results[3].Provider.UseFIPSEndpoint) } - if providers[3].UseFIPSEndpoint != false { - t.Errorf("Expected use fips endpoint false, got %t", providers[3].UseFIPSEndpoint) + if results[3].Provider.AssumeRoleWithWebIdentity.RoleARN != "arn:aws:iam::123456789012:role/ROLE_NAME" { + t.Errorf("Expected role arn arn:aws:iam::123456789012:role/ROLE_NAME, got %s", results[3].Provider.AssumeRoleWithWebIdentity.RoleARN) } - if providers[3].AssumeRoleWithWebIdentity.RoleARN != "arn:aws:iam::123456789012:role/ROLE_NAME" { - t.Errorf("Expected role arn arn:aws:iam::123456789012:role/ROLE_NAME, got %s", providers[3].AssumeRoleWithWebIdentity.RoleARN) + if results[3].Provider.AssumeRoleWithWebIdentity.SessionName != "SESSION_NAME" { + t.Errorf("Expected session name SESSION_NAME, got %s", results[3].Provider.AssumeRoleWithWebIdentity.SessionName) } - if providers[3].AssumeRoleWithWebIdentity.SessionName != "SESSION_NAME" { - t.Errorf("Expected session name SESSION_NAME, got %s", providers[3].AssumeRoleWithWebIdentity.SessionName) + if results[3].Provider.AssumeRoleWithWebIdentity.WebIdentityTokenFile != "/Users/tf_user/secrets/web-identity-token" { + t.Errorf("Expected web identity token file /Users/tf_user/secrets/web-identity-token, got %s", results[3].Provider.AssumeRoleWithWebIdentity.WebIdentityTokenFile) } - if providers[3].AssumeRoleWithWebIdentity.WebIdentityTokenFile != "/Users/tf_user/secrets/web-identity-token" { - t.Errorf("Expected web identity token file /Users/tf_user/secrets/web-identity-token, got %s", providers[3].AssumeRoleWithWebIdentity.WebIdentityTokenFile) + if results[3].Provider.AssumeRoleWithWebIdentity.WebIdentityToken != "web_identity_token" { + t.Errorf("Expected web identity token web_identity_token, got %s", results[3].Provider.AssumeRoleWithWebIdentity.WebIdentityToken) } - if providers[3].AssumeRoleWithWebIdentity.WebIdentityToken != "web_identity_token" { - t.Errorf("Expected web identity token web_identity_token, got %s", providers[3].AssumeRoleWithWebIdentity.WebIdentityToken) + if results[3].Provider.AssumeRoleWithWebIdentity.Duration != "1s" { + t.Errorf("Expected duration 1s, got %s", results[3].Provider.AssumeRoleWithWebIdentity.Duration) } - if providers[3].AssumeRoleWithWebIdentity.Duration != "1s" { - t.Errorf("Expected duration 1s, got %s", providers[3].AssumeRoleWithWebIdentity.Duration) + if results[3].Provider.AssumeRoleWithWebIdentity.Policy != "policy" { + t.Errorf("Expected policy policy, got %s", results[3].Provider.AssumeRoleWithWebIdentity.Policy) } - if providers[3].AssumeRoleWithWebIdentity.Policy != "policy" { - t.Errorf("Expected policy policy, got %s", providers[3].AssumeRoleWithWebIdentity.Policy) + if len(results[3].Provider.AssumeRoleWithWebIdentity.PolicyARNs) != 1 { + t.Errorf("Expected 1 policy arn, got %d", len(results[3].Provider.AssumeRoleWithWebIdentity.PolicyARNs)) } - if len(providers[3].AssumeRoleWithWebIdentity.PolicyARNs) != 1 { - t.Errorf("Expected 1 policy arn, got %d", len(providers[3].AssumeRoleWithWebIdentity.PolicyARNs)) + if results[3].Provider.AssumeRoleWithWebIdentity.PolicyARNs[0] != "policy_arns" { + t.Errorf("Expected policy arn policy_arns, got %s", results[3].Provider.AssumeRoleWithWebIdentity.PolicyARNs[0]) } - if providers[3].AssumeRoleWithWebIdentity.PolicyARNs[0] != "policy_arns" { - t.Errorf("Expected policy arn policy_arns, got %s", providers[3].AssumeRoleWithWebIdentity.PolicyARNs[0]) + if results[4].Provider.Region != "us-west-2" { + t.Errorf("Expected region us-west-2, got %s", results[4].Provider.Region) } - if providers[4].Region != "us-west-2" { - t.Errorf("Expected region us-west-2, got %s", providers[4].Region) + if results[4].Provider.AccessKey != "my-access-key" { + t.Errorf("Expected access key my-access-key, got %s", results[4].Provider.AccessKey) } - if providers[4].AccessKey != "my-access-key" { - t.Errorf("Expected access key my-access-key, got %s", providers[4].AccessKey) - } - - if providers[4].SecretKey != "my-secret-key" { - t.Errorf("Expected secret key my-secret-key, got %s", providers[4].SecretKey) + if results[4].Provider.SecretKey != "my-secret-key" { + t.Errorf("Expected secret key my-secret-key, got %s", results[4].Provider.SecretKey) } } func TestConfigFromProvider(t *testing.T) { // Make sure the providers we have created can all be turned into configs // without any issues - providers, _, err := ParseAWSProviders("testdata/config_from_provider", nil) + results, err := ParseAWSProviders("testdata/config_from_provider", nil) if err != nil { t.Fatalf("Error parsing AWS providers: %v", err) } - for _, provider := range providers { - _, err := ConfigFromProvider(context.Background(), provider) + for _, provider := range results { + _, err := ConfigFromProvider(context.Background(), *provider.Provider) if err != nil { t.Errorf("Error converting provider to config: %v", err) } From c969cc92e93b9a6942bee2bd9da0b107cbec51ea Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Mon, 22 Jul 2024 14:10:06 +0200 Subject: [PATCH 04/17] Frontend changes to use new tf/provider parser to connect to AWS --- cmd/tea.go | 173 +---------------- cmd/tea_initialisesources.go | 364 +++++++++++++---------------------- cmd/tea_plan.go | 15 -- cmd/tea_submitplan.go | 5 + cmd/tea_terraform.go | 3 + cmd/terraform_apply.go | 5 - cmd/terraform_plan.go | 10 +- 7 files changed, 146 insertions(+), 429 deletions(-) diff --git a/cmd/tea.go b/cmd/tea.go index 2f9c0fbf..83715f34 100644 --- a/cmd/tea.go +++ b/cmd/tea.go @@ -5,32 +5,20 @@ package cmd import ( "context" "fmt" - "net/http" "net/url" "os" "time" - "github.com/aws/aws-sdk-go-v2/service/sts" tea "github.com/charmbracelet/bubbletea" - "github.com/overmindtech/aws-source/proc" "github.com/overmindtech/cli/tracing" - "github.com/overmindtech/sdp-go/auth" - stdlibsource "github.com/overmindtech/stdlib-source/sources" log "github.com/sirupsen/logrus" - "github.com/sourcegraph/conc/pool" "github.com/spf13/cobra" "github.com/spf13/viper" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "golang.org/x/oauth2" ) type OvermindCommandHandler func(ctx context.Context, args []string, oi OvermindInstance, token *oauth2.Token) error -type terraformStoredConfig struct { - Config string `json:"aws-config"` - Profile string `json:"aws-profile"` -} - // viperGetApp fetches and validates the configured app url func viperGetApp(ctx context.Context) (string, error) { app := viper.GetString("app") @@ -103,6 +91,7 @@ func CmdWrapper(action string, requiredScopes []string, commandModel func(args [ timeout: timeout, app: app, requiredScopes: requiredScopes, + args: args, apiKey: viper.GetString("api-key"), tasks: map[string]tea.Model{}, } @@ -140,163 +129,3 @@ func CmdWrapper(action string, requiredScopes []string, commandModel func(args [ } } } - -func InitializeSources(ctx context.Context, oi OvermindInstance, aws_config, aws_profile string, token *oauth2.Token) (func(), error) { - hostname, err := os.Hostname() - if err != nil { - hostname = "localhost" - } - - natsNamePrefix := "overmind-cli" - - openapiUrl := *oi.ApiUrl - openapiUrl.Path = "/api" - tokenClient := auth.NewOAuthTokenClientWithContext( - ctx, - openapiUrl.String(), - "", - oauth2.StaticTokenSource(token), - ) - - natsOptions := auth.NATSOptions{ - NumRetries: 3, - RetryDelay: 1 * time.Second, - Servers: []string{oi.NatsUrl.String()}, - ConnectionName: fmt.Sprintf("%v.%v", natsNamePrefix, hostname), - ConnectionTimeout: (10 * time.Second), // TODO: Make configurable - MaxReconnects: -1, - ReconnectWait: 1 * time.Second, - ReconnectJitter: 1 * time.Second, - TokenClient: tokenClient, - } - - awsAuthConfig := proc.AwsAuthConfig{ - // TODO: ask user to select regions - Regions: []string{}, - } - - switch aws_config { - case "profile_input", "aws_profile": - awsAuthConfig.Strategy = "sso-profile" - awsAuthConfig.Profile = aws_profile - case "defaults": - awsAuthConfig.Strategy = "defaults" - case "managed": - // TODO: not implemented yet - } - - all_regions := []string{ - "us-east-2", - "us-east-1", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-2", - "ap-southeast-3", - "ap-southeast-4", - "ap-south-1", - "ap-northeast-3", - "ap-northeast-2", - "ap-southeast-1", - "ap-southeast-2", - "ap-northeast-1", - "ca-central-1", - "ca-west-1", - "eu-central-1", - "eu-west-1", - "eu-west-2", - "eu-south-1", - "eu-west-3", - "eu-south-2", - "eu-north-1", - "eu-central-2", - "il-central-1", - "me-south-1", - "me-central-1", - "sa-east-1"} - configCtx, configCancel := context.WithTimeout(ctx, 10*time.Second) - defer configCancel() - - region_checkers := pool. - NewWithResults[string](). - WithContext(configCtx). - WithMaxGoroutines(len(all_regions)). - WithFirstError() - - for _, r := range all_regions { - r := r // loopvar saver; TODO: update golangci-lint or vscode validator to understand this is not required anymore - lf := log.Fields{ - "region": r, - "strategy": awsAuthConfig.Strategy, - } - - region_checkers.Go(func(ctx context.Context) (string, error) { - cfg, err := awsAuthConfig.GetAWSConfig(r) - if err != nil { - log.WithError(err).WithFields(lf).Debug("skipping region") - return "", err - } - - // Add OTel instrumentation - cfg.HTTPClient = &http.Client{ - Transport: otelhttp.NewTransport(http.DefaultTransport), - } - - // Work out what account we're using. This will be used in item scopes - stsClient := sts.NewFromConfig(cfg) - - _, err = stsClient.GetCallerIdentity(configCtx, &sts.GetCallerIdentityInput{}) - if err != nil { - - if awsAuthConfig.TargetRoleARN != "" { - lf["targetRoleARN"] = awsAuthConfig.TargetRoleARN - lf["externalID"] = awsAuthConfig.ExternalID - } - log.WithError(err).WithFields(lf).Debug("skipping region") - return "", err - } - return r, nil - }) - } - - working_regions, err := region_checkers.Wait() - // errors are only relevant if no region remained - if len(working_regions) == 0 { - return func() {}, fmt.Errorf("no regions available: %w", err) - } - - awsAuthConfig.Regions = append(awsAuthConfig.Regions, working_regions...) - log.WithField("regions", awsAuthConfig.Regions).Debug("Using regions") - - awsEngine, err := proc.InitializeAwsSourceEngine(ctx, natsOptions, awsAuthConfig, 2_000) - if err != nil { - return func() {}, fmt.Errorf("failed to initialize AWS source engine: %w", err) - } - - // todo: pass in context with timeout to abort timely and allow Ctrl-C to work - err = awsEngine.Start() - if err != nil { - return func() {}, fmt.Errorf("failed to start AWS source engine: %w", err) - } - - stdlibEngine, err := stdlibsource.InitializeEngine(natsOptions, 2_000, true) - if err != nil { - return func() { - _ = awsEngine.Stop() - }, fmt.Errorf("failed to initialize stdlib source engine: %w", err) - } - - // todo: pass in context with timeout to abort timely and allow Ctrl-C to work - err = stdlibEngine.Start() - if err != nil { - return func() { - _ = awsEngine.Stop() - }, fmt.Errorf("failed to start stdlib source engine: %w", err) - } - - return func() { - _ = awsEngine.Stop() - _ = stdlibEngine.Stop() - }, nil -} diff --git a/cmd/tea_initialisesources.go b/cmd/tea_initialisesources.go index 3accf6cd..979dc7eb 100644 --- a/cmd/tea_initialisesources.go +++ b/cmd/tea_initialisesources.go @@ -1,19 +1,18 @@ package cmd import ( - "bytes" "context" - "encoding/json" - "errors" "fmt" "os" "strings" + "time" - "connectrpc.com/connect" + "github.com/aws/aws-sdk-go-v2/aws" tea "github.com/charmbracelet/bubbletea" - "github.com/charmbracelet/huh" - "github.com/overmindtech/sdp-go" - log "github.com/sirupsen/logrus" + "github.com/overmindtech/aws-source/proc" + "github.com/overmindtech/cli/tfutils" + "github.com/overmindtech/sdp-go/auth" + stdlibsource "github.com/overmindtech/stdlib-source/sources" "github.com/spf13/viper" "golang.org/x/oauth2" ) @@ -23,17 +22,14 @@ type loadSourcesConfigMsg struct { oi OvermindInstance action string token *oauth2.Token + tfArgs []string } -type askForAwsConfigMsg struct { - // an optional message when requesting a new config to explain why a new - // config is required. This is used for example when a source does not start - // up correctly. - retryMsg string -} -type configStoredMsg struct{} -type sourceInitialisationFailedMsg struct{ err error } +type stdlibSourceInitialisedMsg struct{} +type awsSourceInitialisedMsg struct{} + type sourcesInitialisedMsg struct{} +type sourceInitialisationFailedMsg struct{ err error } // this tea.Model either fetches the AWS auth config from the ConfigService or // interrogates the user. Results get stored in the ConfigService. Send a @@ -48,11 +44,7 @@ type initialiseSourcesModel struct { action string token *oauth2.Token - awsConfigForm *huh.Form // is set if the user needs to be interrogated about their aws_config - awsConfigFormDone bool // gets set to true once the form result has been processed - profileInputForm *huh.Form // is set if the user needs to be interrogated about their profile_input - profileInputFormDone bool // gets set to true once the form result has been processed - + useManagedSources bool awsSourceRunning bool stdlibSourceRunning bool @@ -91,101 +83,37 @@ func (m initialiseSourcesModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.token = msg.token m.status = taskStatusRunning - cmds = append(cmds, m.loadSourcesConfigCmd) + if viper.GetBool("only-use-managed-sources") { + m.useManagedSources = true + cmds = append(cmds, func() tea.Msg { return sourcesInitialisedMsg{} }) + } else { + cmds = append(cmds, m.startStdlibSourceCmd(m.ctx, m.oi, m.token)) + cmds = append(cmds, m.startAwsSourceCmd(m.ctx, m.oi, m.token, msg.tfArgs)) + } if os.Getenv("CI") == "" { cmds = append(cmds, m.spinner.Tick) } - case askForAwsConfigMsg: - // load the config that was injected above. If it's not there, prompt the user. - aws_config := viper.GetString("aws-config") - aws_profile := viper.GetString("aws-profile") - - if aws_config == "" || viper.GetBool("reset-stored-config") { - aws_config = "aborted" - options := []huh.Option[string]{} - aws_profile_env := os.Getenv("AWS_PROFILE") - // TODO: add a "managed" option - if aws_profile == aws_profile_env && aws_profile != "" { - // the value of $AWS_PROFILE was not overridden on the commandline - options = append(options, - huh.NewOption("Use the default settings", "defaults"), - huh.NewOption(fmt.Sprintf("Use $AWS_PROFILE (currently: '%v')", aws_profile_env), "aws_profile"), - huh.NewOption("Select a different AWS auth profile", "profile_input"), - ) - } else { - if aws_profile != "" { - // used --aws-profile on the command line, with a value different from $AWS_PROFILE - options = append(options, - huh.NewOption("Use the default settings", "defaults"), - huh.NewOption(fmt.Sprintf("Use the selected AWS profile (currently: '%v')", aws_profile), "aws_profile"), - huh.NewOption("Select a different AWS auth profile", "profile_input"), - ) - } else { - options = append(options, - huh.NewOption("Use the default settings", "defaults"), - huh.NewOption("Select an AWS auth profile", "profile_input"), - ) - } - } - - // TODO: what URL needs to get opened here? - // TODO: how to wait for a source to be configured? - // options = append(options, - // huh.NewOption("Run managed source (opens browser)", "managed"), - // ) - - selector := huh.NewSelect[string](). - Key("aws-config"). - Title("Choose how to access your AWS account (read-only):"). - Options(options...) - m.awsConfigForm = huh.NewForm(huh.NewGroup(selector)) - m.awsConfigFormDone = false - if msg.retryMsg != "" { - m.errorHints = append(m.errorHints, msg.retryMsg) - } - cmds = append(cmds, selector.Focus()) - } else { - m.awsConfigFormDone = true - - if aws_config == "profile_input" && aws_profile == "" { - input := huh.NewInput(). - Key("aws-profile"). - Title("Input the name of the AWS profile to use:") - m.profileInputForm = huh.NewForm( - huh.NewGroup(input), - ) - cmds = append(cmds, input.Focus()) - } else { - if cmdSpan != nil { - cmdSpan.AddEvent("Used stored AWS config") - } - - cmds = append(cmds, m.storeConfigCmd(aws_config, aws_profile)) - cmds = append(cmds, m.startSourcesCmd(aws_config, aws_profile)) - } + case stdlibSourceInitialisedMsg: + m.stdlibSourceRunning = true + if cmdSpan != nil { + cmdSpan.AddEvent("stdlib source initialised") } - case configStoredMsg: - m.title = "Configuring AWS Access (config stored)" - case sourcesInitialisedMsg: + if m.awsSourceRunning { + cmds = append(cmds, func() tea.Msg { return sourcesInitialisedMsg{} }) + } + case awsSourceInitialisedMsg: m.awsSourceRunning = true - m.stdlibSourceRunning = true - m.status = taskStatusDone - if cmdSpan != nil { - cmdSpan.AddEvent("Sources initialised") + cmdSpan.AddEvent("aws source initialised") } + if m.stdlibSourceRunning { + cmds = append(cmds, func() tea.Msg { return sourcesInitialisedMsg{} }) + } + case sourcesInitialisedMsg: + m.status = taskStatusDone case sourceInitialisationFailedMsg: m.status = taskStatusError - errorHint := "Error initialising sources" - switch viper.GetString("aws-config") { - case "defaults": - errorHint += " with default settings" - case "aws_profile": - errorHint += fmt.Sprintf(" with AWS profile from environment '%v'", viper.GetString("aws-profile")) - case "profile_input": - errorHint += fmt.Sprintf(" with selected AWS profile '%v'", viper.GetString("aws-profile")) - } - m.errorHints = append(m.errorHints, errorHint) + m.errorHints = append(m.errorHints, "Error initialising sources") cmds = append(cmds, func() tea.Msg { // create a fatalError for aborting the CLI and common error detail // reporting, but don't pass in the spinner ID, to avoid double @@ -207,78 +135,6 @@ func (m initialiseSourcesModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.taskModel, taskCmd = m.taskModel.Update(msg) cmds = append(cmds, taskCmd) - // process the form if it is not yet done - if m.awsConfigForm != nil && !m.awsConfigFormDone { - switch m.awsConfigForm.State { - case huh.StateAborted: - m.awsConfigFormDone = true - // well, shucks - return m, tea.Quit - case huh.StateNormal: - // pass on messages while the form is active - form, cmd := m.awsConfigForm.Update(msg) - if f, ok := form.(*huh.Form); ok { - m.awsConfigForm = f - } - if cmd != nil { - cmds = append(cmds, cmd) - } - case huh.StateCompleted: - m.awsConfigFormDone = true - - // store the result locally - aws_config := m.awsConfigForm.GetString("aws-config") - viper.Set("aws-config", aws_config) - - // ask the next question if required - if aws_config == "profile_input" { - input := huh.NewInput(). - Key("aws-profile"). - Title("Input the name of the AWS profile to use:") - m.profileInputForm = huh.NewForm( - huh.NewGroup(input), - ) - cmds = append(cmds, input.Focus()) - } else { - // no input required; skip the next question - m.profileInputFormDone = true - aws_profile := viper.GetString("aws-profile") - cmds = append(cmds, m.storeConfigCmd(aws_config, aws_profile)) - cmds = append(cmds, m.startSourcesCmd(aws_config, aws_profile)) - } - } - } - - // process the form if it exists and is not yet done - if m.profileInputForm != nil && !m.profileInputFormDone { - switch m.profileInputForm.State { - case huh.StateAborted: - m.profileInputFormDone = true - // well, shucks - return m, tea.Quit - case huh.StateNormal: - // pass on messages while the form is active - form, cmd := m.profileInputForm.Update(msg) - if f, ok := form.(*huh.Form); ok { - m.profileInputForm = f - } - if cmd != nil { - cmds = append(cmds, cmd) - } - case huh.StateCompleted: - m.profileInputFormDone = true - - if cmdSpan != nil { - cmdSpan.AddEvent("User provided AWS config") - } - - // store the result - viper.Set("aws-profile", m.profileInputForm.GetString("aws-profile")) - cmds = append(cmds, m.storeConfigCmd(viper.GetString("aws-config"), viper.GetString("aws-profile"))) - cmds = append(cmds, m.startSourcesCmd(viper.GetString("aws-config"), viper.GetString("aws-profile"))) - } - } - return m, tea.Batch(cmds...) } @@ -287,82 +143,122 @@ func (m initialiseSourcesModel) View() string { for _, hint := range m.errorHints { bits = append(bits, wrap(fmt.Sprintf(" %v %v", RenderErr(), hint), m.width, 2)) } - if m.awsConfigForm != nil && !m.awsConfigFormDone { - bits = append(bits, m.awsConfigForm.View()) - } - if m.profileInputForm != nil && !m.profileInputFormDone { - bits = append(bits, m.profileInputForm.View()) - } - if m.awsSourceRunning { - bits = append(bits, wrap(fmt.Sprintf(" %v AWS Source: running", RenderOk()), m.width, 4)) - } - if m.stdlibSourceRunning { - bits = append(bits, wrap(fmt.Sprintf(" %v stdlib Source: running", RenderOk()), m.width, 4)) + if m.useManagedSources { + bits = append(bits, wrap(fmt.Sprintf(" %v Using managed sources", RenderOk()), m.width, 2)) + } else { + if m.awsSourceRunning { + bits = append(bits, wrap(fmt.Sprintf(" %v AWS Source: running", RenderOk()), m.width, 4)) + } + if m.stdlibSourceRunning { + bits = append(bits, wrap(fmt.Sprintf(" %v stdlib Source: running", RenderOk()), m.width, 4)) + } } return strings.Join(bits, "\n") } -func (m initialiseSourcesModel) loadSourcesConfigCmd() tea.Msg { - ctx := m.ctx - configClient := AuthenticatedConfigClient(ctx, m.oi) - cfgValue, err := configClient.GetConfig(ctx, &connect.Request[sdp.GetConfigRequest]{ - Msg: &sdp.GetConfigRequest{ - Key: "cli terraform", - }, - }) +func natsOptions(ctx context.Context, oi OvermindInstance, token *oauth2.Token) auth.NATSOptions { + hostname, err := os.Hostname() if err != nil { - var cErr *connect.Error - if !errors.As(err, &cErr) || cErr.Code() != connect.CodeNotFound { - return fatalError{id: m.spinner.ID(), err: fmt.Errorf("failed to get stored config: %w", err)} - } - } - if cfgValue != nil { - viper.SetConfigType("json") - err = viper.MergeConfig(bytes.NewBuffer([]byte(cfgValue.Msg.GetValue()))) - if err != nil { - return fatalError{id: m.spinner.ID(), err: fmt.Errorf("failed to merge stored config: %w", err)} - } + hostname = "localhost" } - return askForAwsConfigMsg{} + natsNamePrefix := "overmind-cli" + + openapiUrl := *oi.ApiUrl + openapiUrl.Path = "/api" + tokenClient := auth.NewOAuthTokenClientWithContext( + ctx, + openapiUrl.String(), + "", + oauth2.StaticTokenSource(token), + ) + + return auth.NATSOptions{ + NumRetries: 3, + RetryDelay: 1 * time.Second, + Servers: []string{oi.NatsUrl.String()}, + ConnectionName: fmt.Sprintf("%v.%v", natsNamePrefix, hostname), + ConnectionTimeout: (10 * time.Second), // TODO: Make configurable + MaxReconnects: -1, + ReconnectWait: 1 * time.Second, + ReconnectJitter: 1 * time.Second, + TokenClient: tokenClient, + } } -func (m initialiseSourcesModel) storeConfigCmd(aws_config, aws_profile string) tea.Cmd { +func (m initialiseSourcesModel) startStdlibSourceCmd(ctx context.Context, oi OvermindInstance, token *oauth2.Token) tea.Cmd { return func() tea.Msg { - ctx := m.ctx - configClient := AuthenticatedConfigClient(ctx, m.oi) + natsOptions := natsOptions(ctx, oi, token) - jsonBuf, err := json.Marshal(terraformStoredConfig{ - Config: aws_config, - Profile: aws_profile, - }) + // ignore returned context. Cancellation of sources is handled by the process exiting for now. + // should sources require more teardown, we'll have to figure something out. + + stdlibEngine, err := stdlibsource.InitializeEngine(natsOptions, 2_000, true) if err != nil { - return otherError{id: m.spinner.ID(), err: fmt.Errorf("failed to marshal config: %w", err)} + return fatalError{id: m.spinner.ID(), err: fmt.Errorf("failed to initialize stdlib source engine: %w", err)} } - _, err = configClient.SetConfig(ctx, &connect.Request[sdp.SetConfigRequest]{ - Msg: &sdp.SetConfigRequest{ - Key: "cli terraform", - Value: string(jsonBuf), - }, - }) + + // todo: pass in context with timeout to abort timely and allow Ctrl-C to work + err = stdlibEngine.Start() + if err != nil { - return otherError{id: m.spinner.ID(), err: fmt.Errorf("failed to upload config: %w", err)} + return fatalError{id: m.spinner.ID(), err: fmt.Errorf("failed to start stdlib source engine: %w", err)} } - - return configStoredMsg{} + return stdlibSourceInitialisedMsg{} } } -func (m initialiseSourcesModel) startSourcesCmd(aws_config, aws_profile string) tea.Cmd { +func (m initialiseSourcesModel) startAwsSourceCmd(ctx context.Context, oi OvermindInstance, token *oauth2.Token, tfArgs []string) tea.Cmd { return func() tea.Msg { - // ignore returned context. Cancellation of sources is handled by the process exiting for now. - // should sources require more teardown, we'll have to figure something out. - _, err := InitializeSources(m.ctx, m.oi, aws_config, aws_profile, m.token) + varFiles := []string{} + vars := []string{} + + for _, arg := range tfArgs { + if strings.HasPrefix(arg, "-var-file=") { + varFiles = append(varFiles, strings.TrimPrefix(arg, "-var-file=")) + } else if strings.HasPrefix(arg, "-var=") { + vars = append(vars, strings.TrimPrefix(arg, "-var=")) + } else if strings.HasPrefix(arg, "--var-file=") { + varFiles = append(varFiles, strings.TrimPrefix(arg, "--var-file=")) + } else if strings.HasPrefix(arg, "--var=") { + vars = append(vars, strings.TrimPrefix(arg, "--var=")) + } + } + + tfEval, err := tfutils.LoadEvalContext(varFiles, vars, os.Environ()) + if err != nil { + return sourceInitialisationFailedMsg{fmt.Errorf("failed to load variables from the environment: %w", err)} + } + + providers, err := tfutils.ParseAWSProviders(".", tfEval) + if err != nil { + return sourceInitialisationFailedMsg{fmt.Errorf("failed to parse providers: %w", err)} + } + configs := []aws.Config{} + + for _, p := range providers { + if p.Error != nil { + return sourceInitialisationFailedMsg{fmt.Errorf("error when parsing provider: %w", p.Error)} + } + c, err := tfutils.ConfigFromProvider(ctx, *p.Provider) + if err != nil { + return sourceInitialisationFailedMsg{fmt.Errorf("error when converting provider to config: %w", err)} + } + configs = append(configs, c) + } + + natsOptions := natsOptions(ctx, oi, token) + + awsEngine, err := proc.InitializeAwsSourceEngine(ctx, natsOptions, 2_000, configs...) + if err != nil { + return sourceInitialisationFailedMsg{fmt.Errorf("failed to initialize AWS source engine: %w", err)} + } + + // todo: pass in context with timeout to abort timely and allow Ctrl-C to work + err = awsEngine.Start() if err != nil { - log.WithError(err).Error("failed to initialise sources") - viper.Set("reset-stored-config", true) - return askForAwsConfigMsg{retryMsg: fmt.Sprintf("Error initialising sources: %v", err)} + return sourceInitialisationFailedMsg{fmt.Errorf("failed to start AWS source engine: %w", err)} } - return sourcesInitialisedMsg{} + return awsSourceInitialisedMsg{} } } diff --git a/cmd/tea_plan.go b/cmd/tea_plan.go index bbc059d5..8eaaf3c5 100644 --- a/cmd/tea_plan.go +++ b/cmd/tea_plan.go @@ -3,7 +3,6 @@ package cmd import ( "context" "fmt" - "os" "os/exec" "strings" @@ -81,20 +80,6 @@ func (m runPlanModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return nil } - // inject the profile, if configured - if aws_config := viper.GetString("aws-config"); aws_config == "profile_input" || aws_config == "aws_profile" { - // override the AWS_PROFILE value in the environment with the - // provided value from the config; this might be redundant if - // viper picked it up from the environment in the first place, - // but we wouldn't know that. - if aws_profile := viper.GetString("aws-profile"); aws_profile != "" { - // copy the current environment, as a non-nil Env value instructs exec.Cmd to not inherit the parent's environment - c.Env = os.Environ() - // set the AWS_PROFILE value as last entry, which will override any previous value - c.Env = append(c.Env, fmt.Sprintf("AWS_PROFILE=%v", aws_profile)) - } - } - if viper.GetString("ovm-test-fake") != "" { c = exec.CommandContext(m.ctx, "bash", "-c", "for i in $(seq 25); do echo fake terraform plan progress line $i of 25; sleep .1; done") } diff --git a/cmd/tea_submitplan.go b/cmd/tea_submitplan.go index 2adb3b5c..9207fd5f 100644 --- a/cmd/tea_submitplan.go +++ b/cmd/tea_submitplan.go @@ -811,6 +811,11 @@ func (m submitPlanModel) submitPlanCmd() tea.Msg { } func (m submitPlanModel) FinalReport() string { + if m.Status() == taskStatusPending { + // hasn't started yet + return "" + } + bits := []string{} if m.blastRadiusItems > 0 { bits = append(bits, styleH1().Render("Blast Radius")) diff --git a/cmd/tea_terraform.go b/cmd/tea_terraform.go index 82546cf5..19331815 100644 --- a/cmd/tea_terraform.go +++ b/cmd/tea_terraform.go @@ -35,6 +35,7 @@ type cmdModel struct { apiKey string oi OvermindInstance // loaded from instanceLoadedMsg requiredScopes []string + args []string // UI state tasks map[string]tea.Model @@ -207,6 +208,7 @@ func (m *cmdModel) tokenChecks(token *oauth2.Token) tea.Cmd { oi: m.oi, action: m.action, token: token, + tfArgs: m.args, } } } @@ -261,6 +263,7 @@ func (m *cmdModel) tokenChecks(token *oauth2.Token) tea.Cmd { oi: m.oi, action: m.action, token: token, + tfArgs: m.args, } } } diff --git a/cmd/terraform_apply.go b/cmd/terraform_apply.go index 79a13372..9e3fb878 100644 --- a/cmd/terraform_apply.go +++ b/cmd/terraform_apply.go @@ -291,11 +291,6 @@ func (m tfApplyModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return nil } - // inject the profile, if configured - if aws_profile := viper.GetString("aws-profile"); aws_profile != "" { - c.Env = append(c.Env, fmt.Sprintf("AWS_PROFILE=%v", aws_profile)) - } - _, span := tracing.Tracer().Start(m.ctx, "terraform apply") // nolint:spancheck // will be ended in the tea.Exec cleanup func if viper.GetString("ovm-test-fake") != "" { diff --git a/cmd/terraform_plan.go b/cmd/terraform_plan.go index b8e81691..721cf37a 100644 --- a/cmd/terraform_plan.go +++ b/cmd/terraform_plan.go @@ -159,9 +159,13 @@ func getTicketLinkFromPlan(planFile string) (string, error) { } func addTerraformBaseFlags(cmd *cobra.Command) { - cmd.PersistentFlags().Bool("reset-stored-config", false, "Set this to reset the sources config stored in Overmind and input fresh values.") - cmd.PersistentFlags().String("aws-config", "", "The chosen AWS config method, best set through the initial wizard when running the CLI. Options: 'profile_input', 'aws_profile', 'defaults', 'managed'.") - cmd.PersistentFlags().String("aws-profile", "", "Set this to the name of the AWS profile to use.") + cmd.PersistentFlags().Bool("reset-stored-config", false, "[deprecated: this is now autoconfigured from local terraform files] Set this to reset the sources config stored in Overmind and input fresh values.") + cmd.PersistentFlags().String("aws-config", "", "[deprecated: this is now autoconfigured from local terraform files] The chosen AWS config method, best set through the initial wizard when running the CLI. Options: 'profile_input', 'aws_profile', 'defaults', 'managed'.") + cmd.PersistentFlags().String("aws-profile", "", "[deprecated: this is now autoconfigured from local terraform files] Set this to the name of the AWS profile to use.") + cobra.CheckErr(cmd.PersistentFlags().MarkHidden("reset-stored-config")) + cobra.CheckErr(cmd.PersistentFlags().MarkHidden("aws-config")) + cobra.CheckErr(cmd.PersistentFlags().MarkHidden("aws-profile")) + cmd.PersistentFlags().Bool("only-use-managed-sources", false, "Set this to skip local autoconfiguration and only use the managed sources as configured in Overmind.") } func init() { From 5c077de4da4da9e93acbaedddc9d7130d9db95d0 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Mon, 22 Jul 2024 12:22:00 +0000 Subject: [PATCH 05/17] Added support for variables --- cmd/tea_initialisesources.go | 17 +- tfutils/aws_config.go | 311 ++++++++++++++++++++++++--- tfutils/aws_config_test.go | 219 +++++++++++++++++++ tfutils/testdata/invalid_vars.tfvars | 3 + tfutils/testdata/test_vars.tfvars | 29 +++ tfutils/testdata/tfvars.json | 5 + 6 files changed, 537 insertions(+), 47 deletions(-) create mode 100644 tfutils/testdata/invalid_vars.tfvars create mode 100644 tfutils/testdata/test_vars.tfvars create mode 100644 tfutils/testdata/tfvars.json diff --git a/cmd/tea_initialisesources.go b/cmd/tea_initialisesources.go index 979dc7eb..ab7a24de 100644 --- a/cmd/tea_initialisesources.go +++ b/cmd/tea_initialisesources.go @@ -210,22 +210,7 @@ func (m initialiseSourcesModel) startStdlibSourceCmd(ctx context.Context, oi Ove func (m initialiseSourcesModel) startAwsSourceCmd(ctx context.Context, oi OvermindInstance, token *oauth2.Token, tfArgs []string) tea.Cmd { return func() tea.Msg { - varFiles := []string{} - vars := []string{} - - for _, arg := range tfArgs { - if strings.HasPrefix(arg, "-var-file=") { - varFiles = append(varFiles, strings.TrimPrefix(arg, "-var-file=")) - } else if strings.HasPrefix(arg, "-var=") { - vars = append(vars, strings.TrimPrefix(arg, "-var=")) - } else if strings.HasPrefix(arg, "--var-file=") { - varFiles = append(varFiles, strings.TrimPrefix(arg, "--var-file=")) - } else if strings.HasPrefix(arg, "--var=") { - vars = append(vars, strings.TrimPrefix(arg, "--var=")) - } - } - - tfEval, err := tfutils.LoadEvalContext(varFiles, vars, os.Environ()) + tfEval, err := tfutils.LoadEvalContext(tfArgs, os.Environ()) if err != nil { return sourceInitialisationFailedMsg{fmt.Errorf("failed to load variables from the environment: %w", err)} } diff --git a/tfutils/aws_config.go b/tfutils/aws_config.go index 15acd4e5..2b664915 100644 --- a/tfutils/aws_config.go +++ b/tfutils/aws_config.go @@ -19,6 +19,8 @@ import ( "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclparse" "github.com/mitchellh/go-homedir" + "github.com/zclconf/go-cty/cty" + ctyjson "github.com/zclconf/go-cty/cty/json" "golang.org/x/net/http/httpproxy" ) @@ -96,14 +98,280 @@ type AssumeRoleWithWebIdentity struct { Remain hcl.Body `hcl:",remain"` } -// Loads the eval context from the following locations: +// Loads the eval context in the same way that Terraform does, this means it +// supports TF_VAR_* environment variables, terraform.tfvars, +// terraform.tfvars.json, *.auto.tfvars, and *.auto.tfvars.json files, and -var +// and -var-file arguments. These are processed in the order that Terraform uses +// and should result in the same set of variables being loaded. // -// - `-var-file=FILENAME` files: These should be passed as file paths -// - `-var 'NAME=VALUE'` arguments: These should be passed as a list of strings -// - Environment Variables: These should be passed as a []strings (from `os.Environ()`), -// variables beginning with TF_VAR_ will be used -func LoadEvalContext(varsFiles []string, args []string, env []string) (*hcl.EvalContext, error) { - return nil, nil +// The args parameter should contain the raw arguments that were passed to +// terraform. This includes: -var and -var-file arguments, and should be passed +// as a list of strings. +// +// The env parameter should contain the environment variables that were present +// when Terraform was run. These should be passed as a []strings (from +// `os.Environ()`), variables beginning with TF_VAR_ will be used. +func LoadEvalContext(args []string, env []string) (*hcl.EvalContext, error) { + // Note that Terraform has a hierarchy of variable sources, which we need + // to respect, with later sources taking precedence over earlier ones: + // + // * Environment variables + // * The terraform.tfvars file, if present. + // * The terraform.tfvars.json file, if present. + // * Any *.auto.tfvars or *.auto.tfvars.json files, processed in lexical + // order of their filenames. + // * Any -var and -var-file options on the command line, in the order they + // are provided. (This includes variables set by an HCP Terraform workspace.) + evalCtx := hcl.EvalContext{ + Variables: make(map[string]cty.Value), + } + + // Parse environment variables. Note that if a root module variable uses a + // type constraint to require a complex value (list, set, map, object, or + // tuple), Terraform will instead attempt to parse its value using the same + // syntax used within variable definitions files, which requires careful + // attention to the string escaping rules in your shell: + // + // ```shell + // export TF_VAR_availability_zone_names='["us-west-1b","us-west-1d"]' + // ``` + // + for _, envVar := range env { + // If the key starts with TF_VAR_, we need to strip that off, and we + // also want to filter on only these variables + if strings.HasPrefix(envVar, "TF_VAR_") { + err := ParseFlagValue(envVar[7:], &evalCtx) + if err != nil { + return nil, err + } + } else { + continue + } + } + + // Parse the terraform.tfvars file, if present. + if _, err := os.Stat("terraform.tfvars"); err == nil { + // Parse the HCL file + err = ParseTFVarsFile("terraform.tfvars", &evalCtx) + if err != nil { + return nil, err + } + } + + // Parse the terraform.tfvars.json file, if present. + if _, err := os.Stat("terraform.tfvars.json"); err == nil { + // Parse the JSON file + err = ParseTFVarsJSONFile("terraform.tfvars.json", &evalCtx) + if err != nil { + return nil, err + } + } + + // Parse *.auto.tfvars or *.auto.tfvars.json files, processed in lexical + // order of their filenames. + matches, _ := filepath.Glob("*.auto.tfvars") + for _, file := range matches { + // Parse the HCL file + err := ParseTFVarsFile(file, &evalCtx) + if err != nil { + return nil, err + } + } + + matches, _ = filepath.Glob("*.auto.tfvars.json") + for _, file := range matches { + // Parse the JSON file + err := ParseTFVarsJSONFile(file, &evalCtx) + if err != nil { + return nil, err + } + } + + // Parse vars from args, this means the var files and raw vars, in the order + // they are provided + err := ParseVarsArgs(args, &evalCtx) + if err != nil { + return nil, err + } + + return &evalCtx, nil +} + +// Parses a given TF Vars file into the given eval context +func ParseTFVarsFile(file string, dest *hcl.EvalContext) error { + // Read the file + b, err := os.ReadFile(file) + if err != nil { + return fmt.Errorf("error reading terraform vars file: %w", err) + } + + // Parse the HCL file + parser := hclparse.NewParser() + parsedFile, diag := parser.ParseHCL(b, file) + if diag.HasErrors() { + return fmt.Errorf("error parsing terraform vars file: %w", diag) + } + + // Decode the body + var vars map[string]cty.Value + diag = gohcl.DecodeBody(parsedFile.Body, nil, &vars) + if diag.HasErrors() { + return fmt.Errorf("error decoding terraform vars file: %w", diag) + } + + // Merge the vars into the eval context + for k, v := range vars { + dest.Variables[k] = v + } + + return nil +} + +// Parses a given TF Vars JSON file into the given eval context. In this each +// key becomes a variable as par the Hashicorp docs: +// https://developer.hashicorp.com/terraform/language/values/variables#variable-definitions-tfvars-files +func ParseTFVarsJSONFile(file string, dest *hcl.EvalContext) error { + // Read the file + b, err := os.ReadFile(file) + if err != nil { + return fmt.Errorf("error reading terraform vars file: %w", err) + } + + // Read the type structure form the file + ctyType, err := ctyjson.ImpliedType(b) + if err != nil { + return fmt.Errorf("error unmarshalling terraform vars file: %w", err) + } + + // Unmarshal the values + ctyValue, err := ctyjson.Unmarshal(b, ctyType) + if err != nil { + return fmt.Errorf("error unmarshalling terraform vars file: %w", err) + } + + // Extract the variables + for k, v := range ctyValue.AsValueMap() { + dest.Variables[k] = v + } + + return nil +} + +// Parses either a `json` or `tfvars` formatted vars file ands adds these +// variables to the context +func ParseVarsFile(path string, dest *hcl.EvalContext) error { + switch { + case strings.HasSuffix(path, ".json"): + return ParseTFVarsJSONFile(path, dest) + case strings.HasSuffix(path, ".tfvars"): + return ParseTFVarsFile(path, dest) + default: + return fmt.Errorf("unsupported vars file format: %s", path) + } +} + +// Parses the os.Args for -var and -var-file arguments and adds them to the eval +// context. +func ParseVarsArgs(args []string, dest *hcl.EvalContext) error { + // We are going to parse the whole argument as HCL here since you can + // include arrays, maps etc. + for i, arg := range args { + switch { + case strings.HasPrefix(arg, "-var="): + err := ParseFlagValue(arg[5:], dest) + if err != nil { + return err + } + case arg == "-var": + // If the flag is just -var, we need to use the next arg as the value + // and skip this one + if i+1 < len(args) { + err := ParseFlagValue(args[i+1], dest) + if err != nil { + return err + } + } else { + continue + } + case strings.HasPrefix(arg, "-var-file="): + err := ParseVarsFile(arg[10:], dest) + if err != nil { + return err + } + case arg == "-var-file": + // If the flag is just -var-file, we need to use the next arg as the value + // and skip this one + if i+1 < len(args) { + err := ParseVarsFile(args[i+1], dest) + if err != nil { + return err + } + } else { + continue + } + default: + continue + } + + } + + return nil +} + +// Parses the value of a -var flag. The value should already be extracted here +// i.e. the text after the = sign, or after the space if the = sign isn't used, +// so you should be passing in "foo=var" or "[1,2,3]" etc. +// +// Terraform allows a user to specify string values without quotes, +// which isn't valid HCL, but everything else needs to be valid HCL. For +// example you can set a string like this: +// +// -var="foo=bar" +// +// But this isn't valid HCL since the string isn't quoted. However if +// you want to set a list, map etc, you need to use valid HCL syntax. +// e.g. +// +// -var="foo=[1,2,3]" +// +// In order to handle this we're going to try to parse as HCL, then +// fall back to basic string parsing if that doesn't work, which seems +// to be how the Terraform works +func ParseFlagValue(value string, dest *hcl.EvalContext) error { + err := func() error { + // Parse argument as HCL + parser := hclparse.NewParser() + parsedFile, diag := parser.ParseHCL([]byte(value), "") + if diag.HasErrors() { + return fmt.Errorf("error parsing terraform vars file: %w", diag) + } + + // Decode the body + var vars map[string]cty.Value + diag = gohcl.DecodeBody(parsedFile.Body, nil, &vars) + if diag.HasErrors() { + return fmt.Errorf("error decoding terraform vars file: %w", diag) + } + + // Merge the vars into the eval context + for k, v := range vars { + dest.Variables[k] = v + } + + return nil + }() + + if err != nil { + // Fall back to string parsing + parts := strings.SplitN(value, "=", 2) + if len(parts) != 2 { + return fmt.Errorf("invalid variable argument: %s", value) + } + + dest.Variables[parts[0]] = cty.StringVal(parts[1]) + } + + return nil } type ProviderResult struct { @@ -113,27 +381,16 @@ type ProviderResult struct { } // Parses AWS provider config from all terraform files in the given directory, -// recursing into subdirectories. Returns a list of AWS providers and a list of -// files that were parsed. This will return an error only if there was an error -// loading the files. ProviderResults will be returned for: +// without recursion as we don't yet support providers in submodules. Returns a +// list of AWS providers and a list of files that were parsed. This will return +// an error only if there was an error loading the files. ProviderResults will +// be returned for: // // * Files that could not be parsed at all (just an error) // * Files that contained an AWS provider that we couldn't fully evaluate (with an error) // * Files that contained an AWS provider that we could fully evaluate (with no error) func ParseAWSProviders(terraformDir string, evalContext *hcl.EvalContext) ([]ProviderResult, error) { - files := make([]string, 0) - - // Get all files matching *.tf from everywhere under the directory - err := filepath.Walk(terraformDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return fmt.Errorf("error searching for terraform files: %w", err) - } - if !info.IsDir() && filepath.Ext(info.Name()) == ".tf" { - files = append(files, path) - } - return nil - }) - + files, err := filepath.Glob(filepath.Join(terraformDir, "*.tf")) if err != nil { return nil, err } @@ -141,14 +398,6 @@ func ParseAWSProviders(terraformDir string, evalContext *hcl.EvalContext) ([]Pro parser := hclparse.NewParser() results := make([]ProviderResult, 0) - // TODO: We need to also make sure we have all the variables and inputs set - // up so that dynamic values can be used. These could come from -vars-file - // or the Environment. It's also possible to have a provider in a module - // that sets parameters based on the input variables of the module - // - // * [ ] Parse tfvars files and arguments - // * [ ] Parse environment variables - // Iterate over the files for _, file := range files { b, err := os.ReadFile(file) diff --git a/tfutils/aws_config_test.go b/tfutils/aws_config_test.go index 65818040..b34161dc 100644 --- a/tfutils/aws_config_test.go +++ b/tfutils/aws_config_test.go @@ -3,6 +3,9 @@ package tfutils import ( "context" "testing" + + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" ) func TestParseAWSProviders(t *testing.T) { @@ -171,3 +174,219 @@ func TestConfigFromProvider(t *testing.T) { } } } + +func TestParseTFVarsFile(t *testing.T) { + t.Run("with a good file", func(t *testing.T) { + evalCtx := hcl.EvalContext{ + Variables: make(map[string]cty.Value), + } + + err := ParseTFVarsFile("testdata/test_vars.tfvars", &evalCtx) + if err != nil { + t.Fatalf("Error parsing TF vars file: %v", err) + } + + if evalCtx.Variables["simple_string"].Type() != cty.String { + t.Errorf("Expected simple_string to be a string, got %s", evalCtx.Variables["simple_string"].Type()) + } + + if evalCtx.Variables["simple_string"].AsString() != "example_string" { + t.Errorf("Expected simple_string to be example_string, got %s", evalCtx.Variables["simple_string"].AsString()) + } + + if evalCtx.Variables["example_number"].Type() != cty.Number { + t.Errorf("Expected example_number to be a number, got %s", evalCtx.Variables["example_number"].Type()) + } + + if evalCtx.Variables["example_number"].AsBigFloat().String() != "42" { + t.Errorf("Expected example_number to be 42, got %s", evalCtx.Variables["example_number"].AsBigFloat().String()) + } + + if evalCtx.Variables["example_boolean"].Type() != cty.Bool { + t.Errorf("Expected example_boolean to be a bool, got %s", evalCtx.Variables["example_boolean"].Type()) + } + + if values := evalCtx.Variables["example_list"].AsValueSlice(); len(values) == 3 { + if values[0].AsString() != "item1" { + t.Errorf("Expected first item to be item1, got %s", values[0].AsString()) + } + } else { + t.Errorf("Expected example_list to have 3 elements, got %d", len(values)) + } + + if m := evalCtx.Variables["example_map"].AsValueMap(); len(m) == 2 { + if m["key1"].AsString() != "value1" { + t.Errorf("Expected key1 to be value1, got %s", m["key1"].AsString()) + } + } else { + t.Errorf("Expected example_map to have 2 elements, got %d", len(m)) + } + }) + + t.Run("with a file that doesn't exist", func(t *testing.T) { + evalCtx := hcl.EvalContext{ + Variables: make(map[string]cty.Value), + } + + err := ParseTFVarsFile("testdata/nonexistent.tfvars", &evalCtx) + if err == nil { + t.Fatalf("Expected error parsing nonexistent file, got nil") + } + }) + + t.Run("with a file that has invalid syntax", func(t *testing.T) { + evalCtx := hcl.EvalContext{ + Variables: make(map[string]cty.Value), + } + + err := ParseTFVarsFile("testdata/invalid_vars.tfvars", &evalCtx) + if err == nil { + t.Fatalf("Expected error parsing invalid syntax file, got nil") + } + }) +} + +func TestParseTFVarsJSONFile(t *testing.T) { + t.Run("with a good file", func(t *testing.T) { + evalCtx := hcl.EvalContext{ + Variables: make(map[string]cty.Value), + } + + err := ParseTFVarsJSONFile("testdata/tfvars.json", &evalCtx) + if err != nil { + t.Fatalf("Error parsing TF vars file: %v", err) + } + + if evalCtx.Variables["string"].Type() != cty.String { + t.Errorf("Expected string to be a string, got %s", evalCtx.Variables["string"].Type()) + } + + if evalCtx.Variables["string"].AsString() != "example_string" { + t.Errorf("Expected string to be example_string, got %s", evalCtx.Variables["string"].AsString()) + } + + if values := evalCtx.Variables["list"].AsValueSlice(); len(values) == 2 { + if values[0].AsString() != "item1" { + t.Errorf("Expected first item to be item1, got %s", values[0].AsString()) + } + } else { + t.Errorf("Expected list to have 2 elements, got %d", len(values)) + } + }) + + t.Run("with a file that doesn't exist", func(t *testing.T) { + evalCtx := hcl.EvalContext{ + Variables: make(map[string]cty.Value), + } + + err := ParseTFVarsJSONFile("testdata/nonexistent.json", &evalCtx) + if err == nil { + t.Fatalf("Expected error parsing nonexistent file, got nil") + } + }) +} + +func TestParseFlagValue(t *testing.T) { + // There are a number of ways to supply ags, for example: + // + // terraform apply + // terraform apply -var "image_id=ami-abc123" + // terraform apply -var 'name=value' + // terraform apply -var='image_id_list=["ami-abc123","ami-def456"]' -var="instance_type=t2.micro" + // terraform apply -var='image_id_map={"us-east-1":"ami-abc123","us-east-2":"ami-def456"}' + + tests := []struct { + Name string + Value string + }{ + { + Name: "with =", + Value: "image_id=ami-abc123", + }, + { + Name: "with a space", + Value: "image_id=ami-abc123", + }, + { + Name: "with a list", + Value: "image_id_list=[\"ami-abc123\",\"ami-def456\"]", + }, + { + Name: "with a map", + Value: "image_id_map={\"us-east-1\":\"ami-abc123\",\"us-east-2\":\"ami-def456\"}", + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + evalCtx := hcl.EvalContext{ + Variables: make(map[string]cty.Value), + } + + err := ParseFlagValue(test.Value, &evalCtx) + if err != nil { + t.Fatalf("Error parsing vars args: %v", err) + } + }) + } +} + +func TestParseVarsArgs(t *testing.T) { + tests := []struct { + Name string + Args []string + }{ + { + Name: "with a single var", + Args: []string{"-var", "image_id=ami-abc123"}, + }, + { + Name: "with multiple vars", + Args: []string{"-var", "image_id=ami-abc123", "-var", "instance_type=t2.micro"}, + }, + { + Name: "with a vars file", + Args: []string{"-var-file", "testdata/test_vars.tfvars"}, + }, + { + Name: "with a vars json file", + Args: []string{"-var-file", "testdata/tfvars.json"}, + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + evalCtx := hcl.EvalContext{ + Variables: make(map[string]cty.Value), + } + + err := ParseVarsArgs(test.Args, &evalCtx) + if err != nil { + t.Fatalf("Error parsing vars args: %v", err) + } + }) + } +} + +func TestLoadEvalContext(t *testing.T) { + args := []string{ + "plan", + "-var", "image_id=args", + "-var", "instance_type=t2.micro", + "-var-file", "testdata/tfvars.json", + "-var-file=testdata/test_vars.tfvars", + } + + env := []string{ + "TF_VAR_image_id=environment", + } + + evalCtx, err := LoadEvalContext(args, env) + if err != nil { + t.Fatal(err) + } + + if evalCtx.Variables["image_id"].AsString() != "args" { + t.Errorf("Expected image_id to be args, got %s", evalCtx.Variables["image_id"].AsString()) + } +} diff --git a/tfutils/testdata/invalid_vars.tfvars b/tfutils/testdata/invalid_vars.tfvars new file mode 100644 index 00000000..fbd3da59 --- /dev/null +++ b/tfutils/testdata/invalid_vars.tfvars @@ -0,0 +1,3 @@ +this is not valid hcl + +And therefore shouldn't parse \ No newline at end of file diff --git a/tfutils/testdata/test_vars.tfvars b/tfutils/testdata/test_vars.tfvars new file mode 100644 index 00000000..4c2b99d6 --- /dev/null +++ b/tfutils/testdata/test_vars.tfvars @@ -0,0 +1,29 @@ +# String variable +simple_string="example_string" + +# Number variable +example_number = 42 + +# Boolean variable +example_boolean = true + +# List of strings +example_list = ["item1", "item2", "item3"] + +# Map of strings +example_map = { + key1 = "value1" + key2 = "value2" +} + +# Complex map (nested maps) +complex_map = { + nested_map1 = { + nested_key1 = "nested_value1" + nested_key2 = "nested_value2" + } + nested_map2 = { + nested_key1 = "nested_value3" + nested_key2 = "nested_value4" + } +} diff --git a/tfutils/testdata/tfvars.json b/tfutils/testdata/tfvars.json new file mode 100644 index 00000000..6c179368 --- /dev/null +++ b/tfutils/testdata/tfvars.json @@ -0,0 +1,5 @@ +{ + "string": "example_string", + "list": ["item1", "item2"] +} + \ No newline at end of file From 375dd9be246ecdb7daf06823af7c9aadb8dd56b4 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Mon, 22 Jul 2024 13:18:23 +0000 Subject: [PATCH 06/17] Added provider alias --- tfutils/aws_config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tfutils/aws_config.go b/tfutils/aws_config.go index 2b664915..51b7d80e 100644 --- a/tfutils/aws_config.go +++ b/tfutils/aws_config.go @@ -40,6 +40,7 @@ type ProviderFile struct { // https://registry.terraform.io/providers/hashicorp/aws/latest/docs#provider-configuration type AWSProvider struct { Name string `hcl:"name,label"` + Alias string `hcl:"alias,optional"` AccessKey string `hcl:"access_key,optional"` SecretKey string `hcl:"secret_key,optional"` Token string `hcl:"token,optional"` From 1481212df993ffcd74678189de09f4a0531d0f58 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Mon, 22 Jul 2024 15:54:16 +0200 Subject: [PATCH 07/17] Improve formatting with multiple providers --- cmd/tea_initialisesources.go | 26 ++++++++++++++++++++++---- main.tf | 4 ++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/cmd/tea_initialisesources.go b/cmd/tea_initialisesources.go index ab7a24de..3d31952d 100644 --- a/cmd/tea_initialisesources.go +++ b/cmd/tea_initialisesources.go @@ -14,6 +14,8 @@ import ( "github.com/overmindtech/sdp-go/auth" stdlibsource "github.com/overmindtech/stdlib-source/sources" "github.com/spf13/viper" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "golang.org/x/oauth2" ) @@ -26,7 +28,9 @@ type loadSourcesConfigMsg struct { } type stdlibSourceInitialisedMsg struct{} -type awsSourceInitialisedMsg struct{} +type awsSourceInitialisedMsg struct { + providers []tfutils.ProviderResult +} type sourcesInitialisedMsg struct{} type sourceInitialisationFailedMsg struct{ err error } @@ -46,6 +50,7 @@ type initialiseSourcesModel struct { useManagedSources bool awsSourceRunning bool + awsProviders []tfutils.ProviderResult stdlibSourceRunning bool errorHints []string @@ -103,8 +108,11 @@ func (m initialiseSourcesModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { } case awsSourceInitialisedMsg: m.awsSourceRunning = true + m.awsProviders = msg.providers if cmdSpan != nil { - cmdSpan.AddEvent("aws source initialised") + cmdSpan.AddEvent("aws source initialised", trace.WithAttributes( + attribute.Int("ovm.aws.providers", len(msg.providers)), + )) } if m.stdlibSourceRunning { cmds = append(cmds, func() tea.Msg { return sourcesInitialisedMsg{} }) @@ -147,7 +155,17 @@ func (m initialiseSourcesModel) View() string { bits = append(bits, wrap(fmt.Sprintf(" %v Using managed sources", RenderOk()), m.width, 2)) } else { if m.awsSourceRunning { - bits = append(bits, wrap(fmt.Sprintf(" %v AWS Source: running", RenderOk()), m.width, 4)) + bits = append(bits, wrap(fmt.Sprintf(" %v AWS Source: running with %v providers", RenderOk(), len(m.awsProviders)), m.width, 4)) + for _, p := range m.awsProviders { + label := p.Provider.Alias + if label == "" { + label = p.Provider.Name + } + bits = append(bits, fmt.Sprintf(" %v (from %v)", label, p.FilePath)) + if p.Provider.Region != "" { + bits = append(bits, fmt.Sprintf(" region: %v", p.Provider.Region)) + } + } } if m.stdlibSourceRunning { bits = append(bits, wrap(fmt.Sprintf(" %v stdlib Source: running", RenderOk()), m.width, 4)) @@ -244,6 +262,6 @@ func (m initialiseSourcesModel) startAwsSourceCmd(ctx context.Context, oi Overmi if err != nil { return sourceInitialisationFailedMsg{fmt.Errorf("failed to start AWS source engine: %w", err)} } - return awsSourceInitialisedMsg{} + return awsSourceInitialisedMsg{providers: providers} } } diff --git a/main.tf b/main.tf index bc45c567..790d88e1 100644 --- a/main.tf +++ b/main.tf @@ -12,6 +12,10 @@ terraform { } provider "aws" {} +provider "aws" { + alias = "aliased" + region = "us-east-1" +} variable "bucket_postfix" { type = string From 49b055741ab62cb9b94dd29180fef1f56f78306b Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Tue, 23 Jul 2024 07:45:27 +0000 Subject: [PATCH 08/17] Added provider rendering style --- cmd/tea_initialisesources.go | 69 ++++++++++++++++++++++--- cmd/tea_initialisesources_test.go | 50 ++++++++++++++++++ tfutils/aws_config.go | 84 +++++++++++++++---------------- 3 files changed, 153 insertions(+), 50 deletions(-) create mode 100644 cmd/tea_initialisesources_test.go diff --git a/cmd/tea_initialisesources.go b/cmd/tea_initialisesources.go index 3d31952d..2fcff772 100644 --- a/cmd/tea_initialisesources.go +++ b/cmd/tea_initialisesources.go @@ -9,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" "github.com/overmindtech/aws-source/proc" "github.com/overmindtech/cli/tfutils" "github.com/overmindtech/sdp-go/auth" @@ -17,6 +18,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "golang.org/x/oauth2" + "gopkg.in/yaml.v3" ) type loadSourcesConfigMsg struct { @@ -157,14 +159,7 @@ func (m initialiseSourcesModel) View() string { if m.awsSourceRunning { bits = append(bits, wrap(fmt.Sprintf(" %v AWS Source: running with %v providers", RenderOk(), len(m.awsProviders)), m.width, 4)) for _, p := range m.awsProviders { - label := p.Provider.Alias - if label == "" { - label = p.Provider.Name - } - bits = append(bits, fmt.Sprintf(" %v (from %v)", label, p.FilePath)) - if p.Provider.Region != "" { - bits = append(bits, fmt.Sprintf(" region: %v", p.Provider.Region)) - } + bits = append(bits, renderProviderResult(p, 6)...) } } if m.stdlibSourceRunning { @@ -174,6 +169,64 @@ func (m initialiseSourcesModel) View() string { return strings.Join(bits, "\n") } +// Prints details of a provider with a given indent +func renderProviderResult(result tfutils.ProviderResult, indent int) []string { + output := make([]string, 0) + + indentString := strings.Repeat(" ", indent) + + style := lipgloss.NewStyle() + + if result.Error != nil { + style.Foreground(ColorPalette.BgDanger) + } + + var providerName string + + if result.Provider != nil { + if result.Provider.Alias != "" { + providerName = result.Provider.Alias + } else { + providerName = result.Provider.Name + } + } else { + providerName = "Unknown" + } + + // Print the heading i.e. name (from file.tf) + output = append(output, fmt.Sprintf("%v%v (%v)", indentString, style.Render(providerName), result.FilePath)) + + // Increase indent since everything should come under this heading + indent += 2 + indentString = strings.Repeat(" ", indent) + + if result.Error == nil { + if result.Provider != nil { + // Create a local copy of the provider so that we can redact + // sensitive information. Note that this won't be a deep copy, but + // there isn't anything to redact in the nested structs so this is + // okay + provider := *result.Provider + + if provider.SecretKey != "" { + provider.SecretKey = "REDACTED" + } + + out, err := yaml.Marshal(provider) + if err != nil { + output = append(output, fmt.Sprintf("%vFailed to marshal provider: %v", indentString, err)) + } else { + // Print the provider details with additional indentation + output = append(output, fmt.Sprintf("%v%v", indentString, strings.ReplaceAll(string(out), "\n", "\n"+indentString))) + } + } + } else { + output = append(output, fmt.Sprintf("%vError: %v", indentString, result.Error)) + } + + return output +} + func natsOptions(ctx context.Context, oi OvermindInstance, token *oauth2.Token) auth.NATSOptions { hostname, err := os.Hostname() if err != nil { diff --git a/cmd/tea_initialisesources_test.go b/cmd/tea_initialisesources_test.go new file mode 100644 index 00000000..6f99a055 --- /dev/null +++ b/cmd/tea_initialisesources_test.go @@ -0,0 +1,50 @@ +package cmd + +import ( + "fmt" + "testing" + + "github.com/overmindtech/cli/tfutils" +) + +func TestPrintProviderResult(t *testing.T) { + results := []tfutils.ProviderResult{ + { + Provider: &tfutils.AWSProvider{ + Name: "aws", + AccessKey: "OOH SECRET", + SecretKey: "EVEN MORE SECRET", + Region: "us-west-2", + }, + FilePath: "/path/to/aws.tf", + }, + { + Error: fmt.Errorf("failed to parse provider"), + FilePath: "/path/to/bad.tf", + }, + { + Provider: &tfutils.AWSProvider{ + Alias: "dev", + Region: "us-west-2", + SharedConfigFiles: []string{ + "/path/to/credentials", + }, + AssumeRole: &tfutils.AssumeRole{ + Duration: "12h", + ExternalID: "external-id", + Policy: "policy", + RoleARN: "arn:aws:iam::123456789012:role/role-name", + }, + }, + }, + } + + for _, result := range results { + // This doesn't test anything, it's just used to visually confirm the + // results in the debug window + str := renderProviderResult(result, 0) + for _, line := range str { + fmt.Println(line) + } + } +} diff --git a/tfutils/aws_config.go b/tfutils/aws_config.go index 51b7d80e..2b40be81 100644 --- a/tfutils/aws_config.go +++ b/tfutils/aws_config.go @@ -39,64 +39,64 @@ type ProviderFile struct { // The fields are based on the AWS provider configuration documentation: // https://registry.terraform.io/providers/hashicorp/aws/latest/docs#provider-configuration type AWSProvider struct { - Name string `hcl:"name,label"` - Alias string `hcl:"alias,optional"` - AccessKey string `hcl:"access_key,optional"` - SecretKey string `hcl:"secret_key,optional"` - Token string `hcl:"token,optional"` - Region string `hcl:"region,optional"` - CustomCABundle string `hcl:"custom_ca_bundle,optional"` - EC2MetadataServiceEndpoint string `hcl:"ec2_metadata_service_endpoint,optional"` - EC2MetadataServiceEndpointMode string `hcl:"ec2_metadata_service_endpoint_mode,optional"` - SkipMetadataAPICheck bool `hcl:"skip_metadata_api_check,optional"` - HTTPProxy string `hcl:"http_proxy,optional"` - HTTPSProxy string `hcl:"https_proxy,optional"` - NoProxy string `hcl:"no_proxy,optional"` - MaxRetries int `hcl:"max_retries,optional"` - Profile string `hcl:"profile,optional"` - RetryMode string `hcl:"retry_mode,optional"` - SharedConfigFiles []string `hcl:"shared_config_files,optional"` - SharedCredentialsFiles []string `hcl:"shared_credentials_files,optional"` - UseDualStackEndpoint bool `hcl:"use_dualstack_endpoint,optional"` - UseFIPSEndpoint bool `hcl:"use_fips_endpoint,optional"` - - AssumeRole *AssumeRole `hcl:"assume_role,block"` - AssumeRoleWithWebIdentity *AssumeRoleWithWebIdentity `hcl:"assume_role_with_web_identity,block"` + Name string `hcl:"name,label" yaml:"name,omitempty"` + Alias string `hcl:"alias,optional" yaml:"alias,omitempty"` + AccessKey string `hcl:"access_key,optional" yaml:"access_key,omitempty"` + SecretKey string `hcl:"secret_key,optional" yaml:"secret_key,omitempty"` + Token string `hcl:"token,optional" yaml:"token,omitempty"` + Region string `hcl:"region,optional" yaml:"region,omitempty"` + CustomCABundle string `hcl:"custom_ca_bundle,optional" yaml:"custom_ca_bundle,omitempty"` + EC2MetadataServiceEndpoint string `hcl:"ec2_metadata_service_endpoint,optional" yaml:"ec2_metadata_service_endpoint,omitempty"` + EC2MetadataServiceEndpointMode string `hcl:"ec2_metadata_service_endpoint_mode,optional" yaml:"ec2_metadata_service_endpoint_mode,omitempty"` + SkipMetadataAPICheck bool `hcl:"skip_metadata_api_check,optional" yaml:"skip_metadata_api_check,omitempty"` + HTTPProxy string `hcl:"http_proxy,optional" yaml:"http_proxy,omitempty"` + HTTPSProxy string `hcl:"https_proxy,optional" yaml:"https_proxy,omitempty"` + NoProxy string `hcl:"no_proxy,optional" yaml:"no_proxy,omitempty"` + MaxRetries int `hcl:"max_retries,optional" yaml:"max_retries,omitempty"` + Profile string `hcl:"profile,optional" yaml:"profile,omitempty"` + RetryMode string `hcl:"retry_mode,optional" yaml:"retry_mode,omitempty"` + SharedConfigFiles []string `hcl:"shared_config_files,optional" yaml:"shared_config_files,omitempty"` + SharedCredentialsFiles []string `hcl:"shared_credentials_files,optional" yaml:"shared_credentials_files,omitempty"` + UseDualStackEndpoint bool `hcl:"use_dualstack_endpoint,optional" yaml:"use_dualstack_endpoint,omitempty"` + UseFIPSEndpoint bool `hcl:"use_fips_endpoint,optional" yaml:"use_fips_endpoint,omitempty"` + + AssumeRole *AssumeRole `hcl:"assume_role,block" yaml:"assume_role,omitempty"` + AssumeRoleWithWebIdentity *AssumeRoleWithWebIdentity `hcl:"assume_role_with_web_identity,block" yaml:"assume_role_with_web_identity,omitempty"` // Throw any additional stuff into here so it doesn't fail - Remain hcl.Body `hcl:",remain"` + Remain hcl.Body `hcl:",remain" yaml:"-"` } // Fields that are used for assuming a role, see: // https://registry.terraform.io/providers/hashicorp/aws/latest/docs#assuming-an-iam-role type AssumeRole struct { - Duration string `hcl:"duration,optional"` - ExternalID string `hcl:"external_id,optional"` - Policy string `hcl:"policy,optional"` - PolicyARNs []string `hcl:"policy_arns,optional"` - RoleARN string `hcl:"role_arn,optional"` - SessionName string `hcl:"session_name,optional"` - SourceIdentity string `hcl:"source_identity,optional"` - Tags map[string]string `hcl:"tags,optional"` - TransitiveTagKeys []string `hcl:"transitive_tag_keys,optional"` + Duration string `hcl:"duration,optional" yaml:"duration,omitempty"` + ExternalID string `hcl:"external_id,optional" yaml:"external_id,omitempty"` + Policy string `hcl:"policy,optional" yaml:"policy,omitempty"` + PolicyARNs []string `hcl:"policy_arns,optional" yaml:"policy_arns,omitempty"` + RoleARN string `hcl:"role_arn,optional" yaml:"role_arn,omitempty"` + SessionName string `hcl:"session_name,optional" yaml:"session_name,omitempty"` + SourceIdentity string `hcl:"source_identity,optional" yaml:"source_identity,omitempty"` + Tags map[string]string `hcl:"tags,optional" yaml:"tags,omitempty"` + TransitiveTagKeys []string `hcl:"transitive_tag_keys,optional" yaml:"transitive_tag_keys,omitempty"` // Throw any additional stuff into here so it doesn't fail - Remain hcl.Body `hcl:",remain"` + Remain hcl.Body `hcl:",remain" yaml:"-"` } // Fields that are used for assuming a role with web identity, see: // https://registry.terraform.io/providers/hashicorp/aws/latest/docs#assuming-an-iam-role-using-a-web-identity type AssumeRoleWithWebIdentity struct { - Duration string `hcl:"duration,optional"` - Policy string `hcl:"policy,optional"` - PolicyARNs []string `hcl:"policy_arns,optional"` - RoleARN string `hcl:"role_arn,optional"` - SessionName string `hcl:"session_name,optional"` - WebIdentityToken string `hcl:"web_identity_token,optional"` - WebIdentityTokenFile string `hcl:"web_identity_token_file,optional"` + Duration string `hcl:"duration,optional" yaml:"duration,omitempty"` + Policy string `hcl:"policy,optional" yaml:"policy,omitempty"` + PolicyARNs []string `hcl:"policy_arns,optional" yaml:"policy_arns,omitempty"` + RoleARN string `hcl:"role_arn,optional" yaml:"role_arn,omitempty"` + SessionName string `hcl:"session_name,optional" yaml:"session_name,omitempty"` + WebIdentityToken string `hcl:"web_identity_token,optional" yaml:"web_identity_token,omitempty"` + WebIdentityTokenFile string `hcl:"web_identity_token_file,optional" yaml:"web_identity_token_file,omitempty"` // Throw any additional stuff into here so it doesn't fail - Remain hcl.Body `hcl:",remain"` + Remain hcl.Body `hcl:",remain" yaml:"-"` } // Loads the eval context in the same way that Terraform does, this means it From 6c31a69b12302f147f74186e61d809ebfb5bca49 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 23 Jul 2024 08:19:55 +0000 Subject: [PATCH 09/17] Update github.com/overmindtech/aws-source digest to 718c447 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3c655982..aaa62a3d 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/muesli/reflow v0.3.0 github.com/muesli/termenv v0.15.2 - github.com/overmindtech/aws-source v0.0.0-20240718232007-6b71c2ec6ebd + github.com/overmindtech/aws-source v0.0.0-20240723080057-718c447a98d5 github.com/overmindtech/sdp-go v0.80.0 github.com/overmindtech/stdlib-source v0.0.0-20240718131945-5d4742e6c9cb github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c diff --git a/go.sum b/go.sum index 135af77f..94605447 100644 --- a/go.sum +++ b/go.sum @@ -246,8 +246,8 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/openrdap/rdap v0.9.2-0.20240517203139-eb57b3a8dedd h1:UuQycBx6K0lB0/IfHePshOYjlrptkF4FoApFP2Y4s3k= github.com/openrdap/rdap v0.9.2-0.20240517203139-eb57b3a8dedd/go.mod h1:391Ww1JbjG4FHOlvQqCd6n25CCCPE64JzC5cCYPxhyM= -github.com/overmindtech/aws-source v0.0.0-20240718232007-6b71c2ec6ebd h1:InUBHKndqd3OZvuRd8PxQUOPvlytv/h8CALr6g+r5H4= -github.com/overmindtech/aws-source v0.0.0-20240718232007-6b71c2ec6ebd/go.mod h1:IU5Z+hFouu6ljSAi8zAEIm7mbaEYw0OuIqiYyELVyzw= +github.com/overmindtech/aws-source v0.0.0-20240723080057-718c447a98d5 h1:x+zDf5S0Ae3Y5Evk0UySxeRMRcPr4JhxfRfJzLcAiIQ= +github.com/overmindtech/aws-source v0.0.0-20240723080057-718c447a98d5/go.mod h1:IU5Z+hFouu6ljSAi8zAEIm7mbaEYw0OuIqiYyELVyzw= github.com/overmindtech/discovery v0.27.6 h1:p+xMEIST0fk6HfbejXR+Ea59+JA5zjCO4zvRqelRqwE= github.com/overmindtech/discovery v0.27.6/go.mod h1:A3wvNM6VTo7qfGExWQ+fgh+NfLh35nuPJ7wF4pLYEQI= github.com/overmindtech/sdp-go v0.80.0 h1:VD2x5UlBmgVsIaFoNzOMmrVcvDeFdIDNIMHGLEBIbOg= From a558a779d3fb0da5465d17f80cb1e83ef3c3f143 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 23 Jul 2024 11:16:27 +0200 Subject: [PATCH 10/17] Adjust test results to match new non-recursing strategy --- tfutils/aws_config_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tfutils/aws_config_test.go b/tfutils/aws_config_test.go index b34161dc..e8932182 100644 --- a/tfutils/aws_config_test.go +++ b/tfutils/aws_config_test.go @@ -14,8 +14,8 @@ func TestParseAWSProviders(t *testing.T) { t.Errorf("Error parsing AWS results: %v", err) } - if len(results) != 5 { - t.Fatalf("Expected 5 results, got %d", len(results)) + if len(results) != 3 { + t.Fatalf("Expected 3 results, got %d", len(results)) } if results[1].Provider.Region != "us-east-1" { From 6bf9083076704f4b72e5a5ad26f9480803e24ff4 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 23 Jul 2024 11:27:35 +0200 Subject: [PATCH 11/17] Extend tests --- tfutils/aws_config.go | 4 ++++ tfutils/aws_config_test.go | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tfutils/aws_config.go b/tfutils/aws_config.go index 2b40be81..0ed86c68 100644 --- a/tfutils/aws_config.go +++ b/tfutils/aws_config.go @@ -277,6 +277,10 @@ func ParseVarsArgs(args []string, dest *hcl.EvalContext) error { // We are going to parse the whole argument as HCL here since you can // include arrays, maps etc. for i, arg := range args { + // normalize `--foo` arguments to `-foo` + if strings.HasPrefix(arg, "--") { + arg = arg[1:] + } switch { case strings.HasPrefix(arg, "-var="): err := ParseFlagValue(arg[5:], dest) diff --git a/tfutils/aws_config_test.go b/tfutils/aws_config_test.go index e8932182..fe5e2486 100644 --- a/tfutils/aws_config_test.go +++ b/tfutils/aws_config_test.go @@ -372,12 +372,13 @@ func TestLoadEvalContext(t *testing.T) { args := []string{ "plan", "-var", "image_id=args", - "-var", "instance_type=t2.micro", + "--var", "instance_type=t2.micro", "-var-file", "testdata/tfvars.json", "-var-file=testdata/test_vars.tfvars", } env := []string{ + "TF_VAR_something=else", "TF_VAR_image_id=environment", } @@ -386,6 +387,14 @@ func TestLoadEvalContext(t *testing.T) { t.Fatal(err) } + t.Log(evalCtx) + + if evalCtx.Variables["instance_type"].AsString() != "t2.micro" { + t.Errorf("Expected instance_type to be t2.micro, got %s", evalCtx.Variables["instance_type"].AsString()) + } + if evalCtx.Variables["something"].AsString() != "else" { + t.Errorf("Expected something to be else, got %s", evalCtx.Variables["something"].AsString()) + } if evalCtx.Variables["image_id"].AsString() != "args" { t.Errorf("Expected image_id to be args, got %s", evalCtx.Variables["image_id"].AsString()) } From 920235de18b2cda34b68494882977db083d11667 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 23 Jul 2024 11:59:19 +0200 Subject: [PATCH 12/17] Put variables where terraform expects them ... into an object called `var`. --- tfutils/aws_config.go | 26 +++++++++++++++++++++----- tfutils/aws_config_test.go | 14 ++++++++------ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/tfutils/aws_config.go b/tfutils/aws_config.go index 0ed86c68..811b1e9c 100644 --- a/tfutils/aws_config.go +++ b/tfutils/aws_config.go @@ -127,6 +127,8 @@ func LoadEvalContext(args []string, env []string) (*hcl.EvalContext, error) { Variables: make(map[string]cty.Value), } + evalCtx.Variables["var"] = cty.ObjectVal(map[string]cty.Value{}) + // Parse environment variables. Note that if a root module variable uses a // type constraint to require a complex value (list, set, map, object, or // tuple), Terraform will instead attempt to parse its value using the same @@ -222,12 +224,22 @@ func ParseTFVarsFile(file string, dest *hcl.EvalContext) error { // Merge the vars into the eval context for k, v := range vars { - dest.Variables[k] = v + setVariable(k, v, dest) } return nil } +// setVariable sets a variable in the given eval context +func setVariable(key string, value cty.Value, dest *hcl.EvalContext) { + variables := dest.Variables["var"].AsValueMap() + if variables == nil { + variables = map[string]cty.Value{} + } + variables[key] = value + dest.Variables["var"] = cty.ObjectVal(variables) +} + // Parses a given TF Vars JSON file into the given eval context. In this each // key becomes a variable as par the Hashicorp docs: // https://developer.hashicorp.com/terraform/language/values/variables#variable-definitions-tfvars-files @@ -252,7 +264,7 @@ func ParseTFVarsJSONFile(file string, dest *hcl.EvalContext) error { // Extract the variables for k, v := range ctyValue.AsValueMap() { - dest.Variables[k] = v + setVariable(k, v, dest) } return nil @@ -359,9 +371,14 @@ func ParseFlagValue(value string, dest *hcl.EvalContext) error { } // Merge the vars into the eval context + variables := dest.Variables["var"].AsValueMap() + if variables == nil { + variables = map[string]cty.Value{} + } for k, v := range vars { - dest.Variables[k] = v + variables[k] = v } + dest.Variables["var"] = cty.ObjectVal(variables) return nil }() @@ -372,8 +389,7 @@ func ParseFlagValue(value string, dest *hcl.EvalContext) error { if len(parts) != 2 { return fmt.Errorf("invalid variable argument: %s", value) } - - dest.Variables[parts[0]] = cty.StringVal(parts[1]) + setVariable(parts[0], cty.StringVal(parts[1]), dest) } return nil diff --git a/tfutils/aws_config_test.go b/tfutils/aws_config_test.go index fe5e2486..1a6d39a3 100644 --- a/tfutils/aws_config_test.go +++ b/tfutils/aws_config_test.go @@ -389,13 +389,15 @@ func TestLoadEvalContext(t *testing.T) { t.Log(evalCtx) - if evalCtx.Variables["instance_type"].AsString() != "t2.micro" { - t.Errorf("Expected instance_type to be t2.micro, got %s", evalCtx.Variables["instance_type"].AsString()) + variables := evalCtx.Variables["var"].AsValueMap() + + if variables["instance_type"].AsString() != "t2.micro" { + t.Errorf("Expected instance_type to be t2.micro, got %s", variables["instance_type"].AsString()) } - if evalCtx.Variables["something"].AsString() != "else" { - t.Errorf("Expected something to be else, got %s", evalCtx.Variables["something"].AsString()) + if variables["something"].AsString() != "else" { + t.Errorf("Expected something to be else, got %s", variables["something"].AsString()) } - if evalCtx.Variables["image_id"].AsString() != "args" { - t.Errorf("Expected image_id to be args, got %s", evalCtx.Variables["image_id"].AsString()) + if variables["image_id"].AsString() != "args" { + t.Errorf("Expected image_id to be args, got %s", variables["image_id"].AsString()) } } From 05fa043a5b70af4a03f413eb818248ada3889a7d Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 23 Jul 2024 14:08:49 +0200 Subject: [PATCH 13/17] Use terraform-config-inspect to load variable defaults from the current module --- go.mod | 7 +++--- go.sum | 2 ++ tfutils/aws_config.go | 50 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index aaa62a3d..59a8424a 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.27.27 github.com/aws/aws-sdk-go-v2/credentials v1.17.27 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 - github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 github.com/charmbracelet/bubbles v0.18.0 github.com/charmbracelet/bubbletea v0.26.6 github.com/charmbracelet/glamour v0.7.0 @@ -29,13 +28,13 @@ require ( github.com/overmindtech/stdlib-source v0.0.0-20240718131945-5d4742e6c9cb github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c github.com/sirupsen/logrus v1.9.3 - github.com/sourcegraph/conc v0.3.0 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 github.com/uptrace/opentelemetry-go-extra/otellogrus v0.3.1 github.com/xiam/dig v0.0.0-20191116195832-893b5fb5093b + github.com/zclconf/go-cty v1.14.4 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 go.opentelemetry.io/otel v1.28.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 @@ -89,6 +88,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect github.com/aws/smithy-go v1.20.3 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect @@ -112,6 +112,7 @@ require ( github.com/gorilla/css v1.0.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hashicorp/terraform-config-inspect v0.0.0-20240701073647-9fc3669f7553 // indirect github.com/iancoleman/strcase v0.2.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -141,6 +142,7 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -151,7 +153,6 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yuin/goldmark v1.5.4 // indirect github.com/yuin/goldmark-emoji v1.0.2 // indirect - github.com/zclconf/go-cty v1.14.4 // indirect go.opentelemetry.io/otel/log v0.3.0 // indirect go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect diff --git a/go.sum b/go.sum index 94605447..88523051 100644 --- a/go.sum +++ b/go.sum @@ -176,6 +176,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl/v2 v2.21.0 h1:lve4q/o/2rqwYOgUg3y3V2YPyD1/zkCLGjIV74Jit14= github.com/hashicorp/hcl/v2 v2.21.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= +github.com/hashicorp/terraform-config-inspect v0.0.0-20240701073647-9fc3669f7553 h1:ApSEBSu6EhcJWCdwSMd1VbQUeJDtB1jAOHfIxjZyMTc= +github.com/hashicorp/terraform-config-inspect v0.0.0-20240701073647-9fc3669f7553/go.mod h1:Gz/z9Hbn+4KSp8A2FBtNszfLSdT2Tn/uAKGuVqqWmDI= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= diff --git a/tfutils/aws_config.go b/tfutils/aws_config.go index 811b1e9c..3de736a3 100644 --- a/tfutils/aws_config.go +++ b/tfutils/aws_config.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/hcl/v2/hclparse" + "github.com/hashicorp/terraform-config-inspect/tfconfig" "github.com/mitchellh/go-homedir" "github.com/zclconf/go-cty/cty" ctyjson "github.com/zclconf/go-cty/cty/json" @@ -99,6 +100,36 @@ type AssumeRoleWithWebIdentity struct { Remain hcl.Body `hcl:",remain" yaml:"-"` } +// restore the default value to a cty value after tfconfig has +// passed it through JSON to "void the caller needing to deal with +// cty" +func ctyFromTfconfig(v interface{}) cty.Value { + switch def := v.(type) { + case bool: + return cty.BoolVal(def) + case float64: + return cty.NumberFloatVal(def) + case int: + return cty.NumberIntVal(int64(def)) + case string: + return cty.StringVal(def) + case []interface{}: + d := make([]cty.Value, 0, len(def)) + for _, v := range def { + d = append(d, ctyFromTfconfig(v)) + } + return cty.ListVal(d) + case map[string]interface{}: + d := map[string]cty.Value{} + for k, v := range def { + d[k] = ctyFromTfconfig(v) + } + return cty.ObjectVal(d) + default: + return cty.NilVal + } +} + // Loads the eval context in the same way that Terraform does, this means it // supports TF_VAR_* environment variables, terraform.tfvars, // terraform.tfvars.json, *.auto.tfvars, and *.auto.tfvars.json files, and -var @@ -127,7 +158,24 @@ func LoadEvalContext(args []string, env []string) (*hcl.EvalContext, error) { Variables: make(map[string]cty.Value), } - evalCtx.Variables["var"] = cty.ObjectVal(map[string]cty.Value{}) + // Parse variable declarations from the Terraform configuration. This will + // supply any default values from variables that are declared in the root + // module. + mod, diags := tfconfig.LoadModule(".") + if diags.HasErrors() { + return nil, fmt.Errorf("error loading terraform module: %w", diags) + } + if mod.Diagnostics.HasErrors() { + return nil, fmt.Errorf("loaded terraform module with errors: %w", mod.Diagnostics) + } + + vars := map[string]cty.Value{} + for _, v := range mod.Variables { + if v.Default != nil { + vars[v.Name] = ctyFromTfconfig(v.Default) + } + } + evalCtx.Variables["var"] = cty.ObjectVal(vars) // Parse environment variables. Note that if a root module variable uses a // type constraint to require a complex value (list, set, map, object, or From d54ab72c8497100a6fe54f3a73c3743275f63e21 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 23 Jul 2024 14:46:10 +0200 Subject: [PATCH 14/17] Ignore errors to allow functional providers to be loaded into the CLI This is a tough tradeoff between offering more functionality (providers configured through data sources) and implementation complexity (lifting a full terraform parser and evaluator to load data sources). For now we err on the simple side and expose provider parse errors when `data` references are used and continue processing as long as there is any valid provider. --- cmd/tea_initialisesources.go | 5 ++++- tfutils/aws_config.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/tea_initialisesources.go b/cmd/tea_initialisesources.go index 2fcff772..b7c183b3 100644 --- a/cmd/tea_initialisesources.go +++ b/cmd/tea_initialisesources.go @@ -294,7 +294,10 @@ func (m initialiseSourcesModel) startAwsSourceCmd(ctx context.Context, oi Overmi for _, p := range providers { if p.Error != nil { - return sourceInitialisationFailedMsg{fmt.Errorf("error when parsing provider: %w", p.Error)} + // skip providers that had errors. This allows us to use + // providers we _could_ detect, while still failing if there is + // a true syntax error and no providers are available at all. + continue } c, err := tfutils.ConfigFromProvider(ctx, *p.Provider) if err != nil { diff --git a/tfutils/aws_config.go b/tfutils/aws_config.go index 3de736a3..2b00ada1 100644 --- a/tfutils/aws_config.go +++ b/tfutils/aws_config.go @@ -495,7 +495,10 @@ func ParseAWSProviders(terraformDir string, evalContext *hcl.EvalContext) ([]Pro Error: fmt.Errorf("error decoding terraform file: (%v) %w", file, diag), FilePath: file, }) - continue + // continue with using the providers that were parsed. This could be + // e.g. providers that are configured using `data` references. If + // there is a true syntax error, then no providers will be parsed at + // all, and we'll fail later. } for _, provider := range providerFile.Providers { From 6778ea451eea7e5e6678ccdad2563e0b3aeff252 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 23 Jul 2024 15:14:56 +0200 Subject: [PATCH 15/17] More test fixups --- tfutils/aws_config_test.go | 130 ++----------------------------------- 1 file changed, 5 insertions(+), 125 deletions(-) diff --git a/tfutils/aws_config_test.go b/tfutils/aws_config_test.go index 1a6d39a3..99abe1c6 100644 --- a/tfutils/aws_config_test.go +++ b/tfutils/aws_config_test.go @@ -18,10 +18,14 @@ func TestParseAWSProviders(t *testing.T) { t.Fatalf("Expected 3 results, got %d", len(results)) } - if results[1].Provider.Region != "us-east-1" { + if results[0].Provider.Region != "us-east-1" { t.Errorf("Expected region us-east-1, got %s", results[0].Provider.Region) } + if results[1].Provider.Region != "" { + t.Errorf("Expected region to be empty, got %s", results[1].Provider.Region) + } + if results[2].Provider.AssumeRole.RoleARN != "arn:aws:iam::123456789012:role/ROLE_NAME" { t.Errorf("Expected role arn arn:aws:iam::123456789012:role/ROLE_NAME, got %s", results[2].Provider.AssumeRole.RoleARN) } @@ -33,130 +37,6 @@ func TestParseAWSProviders(t *testing.T) { if results[2].Provider.AssumeRole.ExternalID != "EXTERNAL_ID" { t.Errorf("Expected external id EXTERNAL_ID, got %s", results[2].Provider.AssumeRole.ExternalID) } - - if results[3].Provider.AccessKey != "access_key" { - t.Errorf("Expected access key access_key, got %s", results[3].Provider.AccessKey) - } - - if results[3].Provider.SecretKey != "secret_key" { - t.Errorf("Expected secret key secret_key, got %s", results[3].Provider.SecretKey) - } - - if results[3].Provider.Token != "token" { - t.Errorf("Expected token token, got %s", results[3].Provider.Token) - } - - if results[3].Provider.Region != "region" { - t.Errorf("Expected region region, got %s", results[3].Provider.Region) - } - - if results[3].Provider.CustomCABundle != "testdata/providers.tf" { - t.Errorf("Expected custom ca bundle testdata/providers.tf, got %s", results[3].Provider.CustomCABundle) - } - - if results[3].Provider.EC2MetadataServiceEndpoint != "ec2_metadata_service_endpoint" { - t.Errorf("Expected ec2 metadata service endpoint ec2_metadata_service_endpoint, got %s", results[3].Provider.EC2MetadataServiceEndpoint) - } - - if results[3].Provider.EC2MetadataServiceEndpointMode != "ipv6" { - t.Errorf("Expected ec2 metadata service endpoint mode ipv6, got %s", results[3].Provider.EC2MetadataServiceEndpointMode) - } - - if results[3].Provider.SkipMetadataAPICheck != true { - t.Errorf("Expected skip metadata api check true, got %t", results[3].Provider.SkipMetadataAPICheck) - } - - if results[3].Provider.HTTPProxy != "http_proxy" { - t.Errorf("Expected http proxy http_proxy, got %s", results[3].Provider.HTTPProxy) - } - - if results[3].Provider.HTTPSProxy != "https_proxy" { - t.Errorf("Expected https proxy https_proxy, got %s", results[3].Provider.HTTPSProxy) - } - - if results[3].Provider.NoProxy != "no_proxy" { - t.Errorf("Expected no proxy no_proxy, got %s", results[3].Provider.NoProxy) - } - - if results[3].Provider.MaxRetries != 10 { - t.Errorf("Expected max retries 10, got %d", results[3].Provider.MaxRetries) - } - - if results[3].Provider.Profile != "profile" { - t.Errorf("Expected profile profile, got %s", results[3].Provider.Profile) - } - - if results[3].Provider.RetryMode != "standard" { - t.Errorf("Expected retry mode standard, got %s", results[3].Provider.RetryMode) - } - - if len(results[3].Provider.SharedConfigFiles) != 1 { - t.Errorf("Expected 1 shared config file, got %d", len(results[3].Provider.SharedConfigFiles)) - } - - if results[3].Provider.SharedConfigFiles[0] != "shared_config_files" { - t.Errorf("Expected shared config file shared_config_files, got %s", results[3].Provider.SharedConfigFiles[0]) - } - - if len(results[3].Provider.SharedCredentialsFiles) != 1 { - t.Errorf("Expected 1 shared credentials file, got %d", len(results[3].Provider.SharedCredentialsFiles)) - } - - if results[3].Provider.SharedCredentialsFiles[0] != "shared_credentials_files" { - t.Errorf("Expected shared credentials file shared_credentials_files, got %s", results[3].Provider.SharedCredentialsFiles[0]) - } - - if results[3].Provider.UseDualStackEndpoint != false { - t.Errorf("Expected use dual stack endpoint false, got %t", results[3].Provider.UseDualStackEndpoint) - } - - if results[3].Provider.UseFIPSEndpoint != false { - t.Errorf("Expected use fips endpoint false, got %t", results[3].Provider.UseFIPSEndpoint) - } - - if results[3].Provider.AssumeRoleWithWebIdentity.RoleARN != "arn:aws:iam::123456789012:role/ROLE_NAME" { - t.Errorf("Expected role arn arn:aws:iam::123456789012:role/ROLE_NAME, got %s", results[3].Provider.AssumeRoleWithWebIdentity.RoleARN) - } - - if results[3].Provider.AssumeRoleWithWebIdentity.SessionName != "SESSION_NAME" { - t.Errorf("Expected session name SESSION_NAME, got %s", results[3].Provider.AssumeRoleWithWebIdentity.SessionName) - } - - if results[3].Provider.AssumeRoleWithWebIdentity.WebIdentityTokenFile != "/Users/tf_user/secrets/web-identity-token" { - t.Errorf("Expected web identity token file /Users/tf_user/secrets/web-identity-token, got %s", results[3].Provider.AssumeRoleWithWebIdentity.WebIdentityTokenFile) - } - - if results[3].Provider.AssumeRoleWithWebIdentity.WebIdentityToken != "web_identity_token" { - t.Errorf("Expected web identity token web_identity_token, got %s", results[3].Provider.AssumeRoleWithWebIdentity.WebIdentityToken) - } - - if results[3].Provider.AssumeRoleWithWebIdentity.Duration != "1s" { - t.Errorf("Expected duration 1s, got %s", results[3].Provider.AssumeRoleWithWebIdentity.Duration) - } - - if results[3].Provider.AssumeRoleWithWebIdentity.Policy != "policy" { - t.Errorf("Expected policy policy, got %s", results[3].Provider.AssumeRoleWithWebIdentity.Policy) - } - - if len(results[3].Provider.AssumeRoleWithWebIdentity.PolicyARNs) != 1 { - t.Errorf("Expected 1 policy arn, got %d", len(results[3].Provider.AssumeRoleWithWebIdentity.PolicyARNs)) - } - - if results[3].Provider.AssumeRoleWithWebIdentity.PolicyARNs[0] != "policy_arns" { - t.Errorf("Expected policy arn policy_arns, got %s", results[3].Provider.AssumeRoleWithWebIdentity.PolicyARNs[0]) - } - - if results[4].Provider.Region != "us-west-2" { - t.Errorf("Expected region us-west-2, got %s", results[4].Provider.Region) - } - - if results[4].Provider.AccessKey != "my-access-key" { - t.Errorf("Expected access key my-access-key, got %s", results[4].Provider.AccessKey) - } - - if results[4].Provider.SecretKey != "my-secret-key" { - t.Errorf("Expected secret key my-secret-key, got %s", results[4].Provider.SecretKey) - } } func TestConfigFromProvider(t *testing.T) { From e61eefc294d446998296277e03326f4b97b33747 Mon Sep 17 00:00:00 2001 From: Dylan Ratcliffe Date: Tue, 23 Jul 2024 13:39:33 +0000 Subject: [PATCH 16/17] Added some multi-stage decoding so that we only deocde the AWS config and not any other providers --- tfutils/aws_config.go | 50 ++++++++++++++++++++++++++--------- tfutils/testdata/providers.tf | 6 +++++ 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/tfutils/aws_config.go b/tfutils/aws_config.go index 2b00ada1..b687eeaf 100644 --- a/tfutils/aws_config.go +++ b/tfutils/aws_config.go @@ -25,6 +25,20 @@ import ( "golang.org/x/net/http/httpproxy" ) +// A minimal struct that will decode the bare minimum to allow us to avoid +// looking at things we don't want to +type basicProviderFile struct { + Providers []genericProvider `hcl:"provider,block"` + Remain hcl.Body `hcl:",remain"` +} + +// Bare minimum provider block that allows us to parse the provider name and +// nothing else, then pass the remaining to a more specific scope +type genericProvider struct { + Name string `hcl:"name,label"` + Remain hcl.Body `hcl:",remain"` +} + // This struct allows us to parse any HCL file that contains an AWS provider // using the gohcl library. type ProviderFile struct { @@ -488,25 +502,37 @@ func ParseAWSProviders(terraformDir string, evalContext *hcl.EvalContext) ([]Pro continue } - providerFile := ProviderFile{} - diag = gohcl.DecodeBody(parsedFile.Body, evalContext, &providerFile) + // First decode really minimally to find just the AWS providers + basicFile := basicProviderFile{} + diag = gohcl.DecodeBody(parsedFile.Body, evalContext, &basicFile) if diag.HasErrors() { results = append(results, ProviderResult{ Error: fmt.Errorf("error decoding terraform file: (%v) %w", file, diag), FilePath: file, }) - // continue with using the providers that were parsed. This could be - // e.g. providers that are configured using `data` references. If - // there is a true syntax error, then no providers will be parsed at - // all, and we'll fail later. + continue } - for _, provider := range providerFile.Providers { - if provider.Name == "aws" { - results = append(results, ProviderResult{ - Provider: &provider, //nolint:all // this is not relevant for go1.22 and later - FilePath: file, - }) + for _, genericProvider := range basicFile.Providers { + switch genericProvider.Name { + case "aws": + awsProvider := AWSProvider{ + // Since this was already decoded we need to use it here + Name: genericProvider.Name, + } + diag = gohcl.DecodeBody(genericProvider.Remain, evalContext, &awsProvider) + if diag.HasErrors() { + results = append(results, ProviderResult{ + Error: fmt.Errorf("error decoding terraform file: (%v) %w", file, diag), + FilePath: file, + }) + continue + } else { + results = append(results, ProviderResult{ + Provider: &awsProvider, //nolint:all // this is not relevant for go1.22 and later + FilePath: file, + }) + } } } } diff --git a/tfutils/testdata/providers.tf b/tfutils/testdata/providers.tf index 5db851df..2767c697 100644 --- a/tfutils/testdata/providers.tf +++ b/tfutils/testdata/providers.tf @@ -30,6 +30,12 @@ resource "aws_instance" "app_server" { } } +# Example kube provider using data and functions which we don't support reading +provider "kubernetes" { + host = data.aws_eks_cluster.core_eks.endpoint + token = data.aws_eks_cluster_auth.core_eks.token +} + provider "aws" { region = "us-east-1" } From 66a0cceaadd07ca4a14349dc0663b46c85a5b849 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Tue, 23 Jul 2024 15:38:26 +0200 Subject: [PATCH 17/17] Fixed setVariable to not segfault --- tfutils/aws_config.go | 43 +++++++++++++++++++++-------------- tfutils/aws_config_test.go | 46 ++++++++++++++++++++++++-------------- 2 files changed, 55 insertions(+), 34 deletions(-) diff --git a/tfutils/aws_config.go b/tfutils/aws_config.go index b687eeaf..38905b56 100644 --- a/tfutils/aws_config.go +++ b/tfutils/aws_config.go @@ -285,16 +285,17 @@ func ParseTFVarsFile(file string, dest *hcl.EvalContext) error { } // Merge the vars into the eval context - for k, v := range vars { - setVariable(k, v, dest) - } - + setVariables(dest, vars) return nil } // setVariable sets a variable in the given eval context -func setVariable(key string, value cty.Value, dest *hcl.EvalContext) { - variables := dest.Variables["var"].AsValueMap() +func setVariable(dest *hcl.EvalContext, key string, value cty.Value) { + variablesValue, ok := dest.Variables["var"] + if !ok { + variablesValue = cty.EmptyObjectVal + } + variables := variablesValue.AsValueMap() if variables == nil { variables = map[string]cty.Value{} } @@ -302,6 +303,22 @@ func setVariable(key string, value cty.Value, dest *hcl.EvalContext) { dest.Variables["var"] = cty.ObjectVal(variables) } +// setVariables sets multiple variables in the given eval context +func setVariables(dest *hcl.EvalContext, variables map[string]cty.Value) { + variablesValue, ok := dest.Variables["var"] + if !ok { + variablesValue = cty.EmptyObjectVal + } + variablesDest := variablesValue.AsValueMap() + if variablesDest == nil { + variablesDest = map[string]cty.Value{} + } + for k, v := range variables { + variablesDest[k] = v + } + dest.Variables["var"] = cty.ObjectVal(variablesDest) +} + // Parses a given TF Vars JSON file into the given eval context. In this each // key becomes a variable as par the Hashicorp docs: // https://developer.hashicorp.com/terraform/language/values/variables#variable-definitions-tfvars-files @@ -326,7 +343,7 @@ func ParseTFVarsJSONFile(file string, dest *hcl.EvalContext) error { // Extract the variables for k, v := range ctyValue.AsValueMap() { - setVariable(k, v, dest) + setVariable(dest, k, v) } return nil @@ -433,15 +450,7 @@ func ParseFlagValue(value string, dest *hcl.EvalContext) error { } // Merge the vars into the eval context - variables := dest.Variables["var"].AsValueMap() - if variables == nil { - variables = map[string]cty.Value{} - } - for k, v := range vars { - variables[k] = v - } - dest.Variables["var"] = cty.ObjectVal(variables) - + setVariables(dest, vars) return nil }() @@ -451,7 +460,7 @@ func ParseFlagValue(value string, dest *hcl.EvalContext) error { if len(parts) != 2 { return fmt.Errorf("invalid variable argument: %s", value) } - setVariable(parts[0], cty.StringVal(parts[1]), dest) + setVariable(dest, parts[0], cty.StringVal(parts[1])) } return nil diff --git a/tfutils/aws_config_test.go b/tfutils/aws_config_test.go index 99abe1c6..b4f78133 100644 --- a/tfutils/aws_config_test.go +++ b/tfutils/aws_config_test.go @@ -66,27 +66,33 @@ func TestParseTFVarsFile(t *testing.T) { t.Fatalf("Error parsing TF vars file: %v", err) } - if evalCtx.Variables["simple_string"].Type() != cty.String { - t.Errorf("Expected simple_string to be a string, got %s", evalCtx.Variables["simple_string"].Type()) + if !evalCtx.Variables["var"].Type().IsObjectType() { + t.Errorf("Expected var to be an object, got %s", evalCtx.Variables["var"].Type()) } - if evalCtx.Variables["simple_string"].AsString() != "example_string" { - t.Errorf("Expected simple_string to be example_string, got %s", evalCtx.Variables["simple_string"].AsString()) + variables := evalCtx.Variables["var"].AsValueMap() + + if variables["simple_string"].Type() != cty.String { + t.Errorf("Expected simple_string to be a string, got %s", variables["simple_string"].Type()) + } + + if variables["simple_string"].AsString() != "example_string" { + t.Errorf("Expected simple_string to be example_string, got %s", variables["simple_string"].AsString()) } - if evalCtx.Variables["example_number"].Type() != cty.Number { - t.Errorf("Expected example_number to be a number, got %s", evalCtx.Variables["example_number"].Type()) + if variables["example_number"].Type() != cty.Number { + t.Errorf("Expected example_number to be a number, got %s", variables["example_number"].Type()) } - if evalCtx.Variables["example_number"].AsBigFloat().String() != "42" { - t.Errorf("Expected example_number to be 42, got %s", evalCtx.Variables["example_number"].AsBigFloat().String()) + if variables["example_number"].AsBigFloat().String() != "42" { + t.Errorf("Expected example_number to be 42, got %s", variables["example_number"].AsBigFloat().String()) } - if evalCtx.Variables["example_boolean"].Type() != cty.Bool { - t.Errorf("Expected example_boolean to be a bool, got %s", evalCtx.Variables["example_boolean"].Type()) + if variables["example_boolean"].Type() != cty.Bool { + t.Errorf("Expected example_boolean to be a bool, got %s", variables["example_boolean"].Type()) } - if values := evalCtx.Variables["example_list"].AsValueSlice(); len(values) == 3 { + if values := variables["example_list"].AsValueSlice(); len(values) == 3 { if values[0].AsString() != "item1" { t.Errorf("Expected first item to be item1, got %s", values[0].AsString()) } @@ -94,7 +100,7 @@ func TestParseTFVarsFile(t *testing.T) { t.Errorf("Expected example_list to have 3 elements, got %d", len(values)) } - if m := evalCtx.Variables["example_map"].AsValueMap(); len(m) == 2 { + if m := variables["example_map"].AsValueMap(); len(m) == 2 { if m["key1"].AsString() != "value1" { t.Errorf("Expected key1 to be value1, got %s", m["key1"].AsString()) } @@ -137,15 +143,21 @@ func TestParseTFVarsJSONFile(t *testing.T) { t.Fatalf("Error parsing TF vars file: %v", err) } - if evalCtx.Variables["string"].Type() != cty.String { - t.Errorf("Expected string to be a string, got %s", evalCtx.Variables["string"].Type()) + if !evalCtx.Variables["var"].Type().IsObjectType() { + t.Errorf("Expected var to be an object, got %s", evalCtx.Variables["var"].Type()) + } + + variables := evalCtx.Variables["var"].AsValueMap() + + if variables["string"].Type() != cty.String { + t.Errorf("Expected string to be a string, got %s", variables["string"].Type()) } - if evalCtx.Variables["string"].AsString() != "example_string" { - t.Errorf("Expected string to be example_string, got %s", evalCtx.Variables["string"].AsString()) + if variables["string"].AsString() != "example_string" { + t.Errorf("Expected string to be example_string, got %s", variables["string"].AsString()) } - if values := evalCtx.Variables["list"].AsValueSlice(); len(values) == 2 { + if values := variables["list"].AsValueSlice(); len(values) == 2 { if values[0].AsString() != "item1" { t.Errorf("Expected first item to be item1, got %s", values[0].AsString()) }