diff --git a/Project.toml b/Project.toml index 1875e52459..b0fde53ccb 100644 --- a/Project.toml +++ b/Project.toml @@ -5,16 +5,23 @@ version = "1.0.0" [deps] AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +IniFile = "83e8ac13-25f8-5344-8a64-a9f2b223428f" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +Retry = "20febd7b-183b-5ae2-ac4a-720e7ce64774" SymDict = "2da68c74-98d7-5633-99d6-8493888d7b1e" [compat] AWSCore = "0.6" HTTP = "0.8" +IniFile = "0.5" JSON = "0.18, 0.19, 0.20, 0.21" +Mocking = "0.7" OrderedCollections = "1" +Retry = "0.3, 0.4" SymDict = "0.3" julia = "1" diff --git a/src/AWS.jl b/src/AWS.jl index 0ae1d098c4..f7dfd1f0a6 100644 --- a/src/AWS.jl +++ b/src/AWS.jl @@ -5,6 +5,7 @@ using SymDict export @service, AWSServices, RestJSONService, JSONService, RestXMLService, QueryService +include("AWSCredentials.jl") include("AWSMetadataUtilities.jl") macro service(module_name::Symbol) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl new file mode 100644 index 0000000000..93f18c8004 --- /dev/null +++ b/src/AWSCredentials.jl @@ -0,0 +1,611 @@ +using AWSCore: Services +using IniFile +using HTTP +using Dates +using Mocking +using JSON + + +export AWSCredentials, + aws_account_number, + aws_get_region, + aws_get_role_details, + aws_user_arn, + check_credentials, + dot_aws_config, + dot_aws_credentials, + dot_aws_credentials_file, + dot_aws_config_file, + ec2_instance_credentials, + ecs_instance_credentials, + env_var_credentials, + localhost_is_ec2, + localhost_maybe_ec2, + localhost_is_lambda + +""" +When you interact with AWS, you specify your [AWS Security Credentials](http://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html) +to verify who you are and whether you have permission to access the resources that you are requesting. +AWS uses the security credentials to authenticate and authorize your requests. +The fields `access_key_id` and `secret_key` hold the access keys used to authenticate API requests +(see [Creating, Modifying, and Viewing Access Keys](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey)). +[Temporary Security Credentials](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html) require the extra session `token` field. +The `user_arn` and `account_number` fields are used to cache the result of the [`aws_user_arn`](@ref) and [`aws_account_number`](@ref) functions. + +AWSCore searches for credentials in a series of possible locations and stops as soon as it finds credentials. +The order of precedence for this search is as follows: + +1. Passing credentials directly to the `AWSCredentials` constructor +2. [Environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) +3. Shared credential file [(~/.aws/credentials)](http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) +4. AWS config file [(~/.aws/config)](http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) +5. Assume Role provider via the aws config file +6. Instance metadata service on an Amazon EC2 instance that has an IAM role configured + +Once the credentials are found, the method by which they were accessed is stored in the `renew` field +and the DateTime at which they will expire is stored in the `expiry` field. +This allows the credentials to be refreshed as needed using [`check_credentials`](@ref). +If `renew` is set to `nothing`, no attempt will be made to refresh the credentials. +Any renewal function is expected to return `nothing` on failure or a populated `AWSCredentials` object on success. +The `renew` field of the returned `AWSCredentials` will be discarded and does not need to be set. + +To specify the profile to use from `~/.aws/credentials`, do, for example, `AWSCredentials(profile="profile-name")`. +""" +mutable struct AWSCredentials + access_key_id::String + secret_key::String + token::String + user_arn::String + account_number::String + expiry::DateTime + renew::Union{Function, Nothing} # Function which can be used to refresh credentials + + function AWSCredentials( + access_key_id, + secret_key, + token="", + user_arn="", + account_number=""; + expiry=typemax(DateTime), + renew=nothing, + ) + return new(access_key_id, secret_key, token, user_arn, account_number, expiry, renew) + end +end + + +function Base.show(io::IO,c::AWSCredentials) + println(io, + c.user_arn, + isempty(c.user_arn) ? "" : " ", + "(", + c.account_number, + isempty(c.account_number) ? "" : ", ", + c.access_key_id, + isempty(c.secret_key) ? "" : ", $(c.secret_key[1:3])...", + isempty(c.token) ? "" : ", $(c.token[1:3])...", + c.expiry, + ")" + ) +end + + +function Base.copyto!(dest::AWSCredentials, src::AWSCredentials) + for f in fieldnames(typeof(dest)) + setfield!(dest, f, getfield(src, f)) + end +end + + +dot_aws_config_file() = get(ENV, "AWS_CONFIG_FILE", joinpath(homedir(), ".aws", "config")) +dot_aws_credentials_file() = get(ENV, "AWS_SHARED_CREDENTIALS_FILE", joinpath(homedir(), ".aws", "credentials")) +localhost_maybe_ec2() = localhost_is_ec2() || isfile("/sys/devices/virtual/dmi/id/product_uuid") +localhost_is_lambda() = haskey(ENV, "LAMBDA_TASK_ROOT") +_aws_get_profile() = get(ENV, "AWS_DEFAULT_PROFILE", get(ENV, "AWS_PROFILE", "default")) + + +""" + AWSCredentials(; profile=nothing) -> Union{AWSCredentials, Nothing} + +Create an AWSCredentials object, given a provided profile (if not provided "default" will be +used). + +Checks credential locations in the order: + 1. Environment Variables + 2. ~/.aws/credentials + 3. ~/.aws/config + 4. EC2 or ECS metadata + +# Keywords +- `profile::AbstractString`: Specific profile used to search for AWSCredentials + +# Throws +- `error("Can't find AWS Credentials")`: AWSCredentials could not be found +""" +function AWSCredentials(; profile=nothing) + creds = nothing + credential_function = () -> nothing + + if profile == nothing + profile = get(ENV, "AWS_PROFILE", get(ENV, "AWS_DEFAULT_PROFILE", nothing)) + end + + # Define our search options, expected to be callable with no arguments. Should return + # `nothing` when credentials are not able to be located + functions = [ + env_var_credentials, + () -> dot_aws_credentials(profile), + () -> dot_aws_config(profile), + ecs_instance_credentials, + ec2_instance_credentials + ] + + # Loop through our search locations until we get credentials back + for f in functions + credential_function = f + creds = credential_function() + creds === nothing || break + end + + creds === nothing && error("Can't find AWS credentials!") + creds.renew = credential_function + + return creds +end + + +""" + localhost_is_ec2() -> Bool + +Determine if the machine executing this code is running on an EC2 instance. +""" +function localhost_is_ec2() + # Checking to see if you are running on an EC2 instance is a complicated problem due to + # a large amount of caveats. Below is a list of methods to implement to work through + # most of these problems: + # + # 1. Check the `hostname -d`; this will not work if using non-Amazon DNS + # 2. Check metadata with EC2 internal domain name `curl -s + # http://instance-data.ec2.internal`; this will not work with a VPC (legacy EC2 only) + # 3. Check `sudo dmidecode -s bios-version`; this requires `dmidecode` on the instance + # 4. Check `/sys/devices/virtual/dmi/id/bios_version`; this may not work depending on + # the instance, Amazon does not document this file however so it's quite unreliable + # 5. Check `http://169.254.169.254`; This is a link-local address for metadata, + # apparently other cloud providers make this metadata URL available now as well so it's + # not guaranteed that you're on an EC2 instance + # Or check a specific endpoint of the instance metadata such as: + # ims_local_hostname = String(HTTP.get("http://169.254.169.254/latest/meta-data/local-hostname").body) + # but with a fast timeout and cache the result. + # See https://docs.aws.amazon.com/en_us/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html + # 6. When checking the UUID, check for little-endian representation, + # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html + + # This is not guarenteed to work on Windows as RNG can make the UUID begin with EC2 on a + # non-EC2 instance + if @mock Sys.iswindows() + command = `wmic path win32_computersystemproduct get uuid` + result = @mock Base.read(command, String) + instance_uuid = strip(split(result, "\n")[2]) + + return instance_uuid[1:3] == "EC2" + end + + # Note: try catch required for open calls on files of mode 400 (-r--------) + # Note: This will not work on new m5 and c5 instances because they use a new hypervisor + # stack and the kernel does not create files in sysfs + hypervisor_uuid = "/sys/hypervisor/uuid" + if _can_read_file(hypervisor_uuid) + return true + end + + # Note: Works if you are running as root + product_uuid = "/sys/devices/virtual/dmi/id/product_uuid" + if _can_read_file(product_uuid) && _begins_with_ec2(product_uuid) + return true + end + + # Check additional values under /sys/devices/virtual/dmi/id for the key "EC2" + # These work for the new m5 and c5 (nitro hypervisor) when root isn't available + # filenames = ["bios_vendor", "board_vendor", "chassis_asset_tag", "chassis_version", "sys_vendor", "uevent", "modalias"] + # all return "Amazon EC2" except the last two + sys_vendor = "/sys/devices/virtual/dmi/id/sys_vendor" + if _can_read_file(sys_vendor) && _ends_with_ec2(sys_vendor) + return true + end + + return false +end + +_can_read_file(file_name::String) = return isfile(file_name) && try isreadable(open(file_name, "r")) catch e; false; end +_begins_with_ec2(file_name::String) = return uppercase(String(read(file_name, 3))) == "EC2" +_ends_with_ec2(file_name::String) = return endswith(strip(uppercase(read(file_name, String))), "EC2") + + +""" + check_credentials( + aws_creds::AWSCredentials, force_refresh::Bool=false + ) -> AWSCredentials + +Checks current AWSCredentials, refreshing them if they are soon to expire. If +`force_refresh` is `true` the credentials will be renewed immediately + +# Arguments +- `aws_creds::AWSCredentials`: AWSCredentials to be checked / refreshed + +# Keywords +- `force_refresh::Bool=false`: `true` to refresh the credentials + +# Throws +- `error("Can't find AWS credentials!")`: If no credentials can be found +""" +function check_credentials(aws_creds::AWSCredentials; force_refresh::Bool=false) + if force_refresh || _will_expire(aws_creds) + credential_method = aws_creds.renew + + if credential_method !== nothing + new_aws_creds = credential_method() + + new_aws_creds === nothing && error("Can't find AWS credentials!") + copyto!(aws_creds, new_aws_creds) + + # Ensure credential_method is not overwritten by the new credentials + aws_creds.renew = credential_method + end + end + + return aws_creds +end + + +function _will_expire(aws_creds::AWSCredentials) + return now(UTC) >= aws_creds.expiry - Minute(5) +end + + +""" + _ec2_metadata(metadata_endpoint::String) -> Union{String, Nothing} + +Retrieve the EC2 meta data from the local AWS endpoint. Return the EC2 metadata request +body, or `nothing` if not running on an EC2 instance. + +# Arguments +- `metadata_endpoint::String`: AWS internal meta data endpoint to hit + +# Throws +- `StatusError`: If the response status is >= 300 +- `ParsingError`: Invalid HTTP request target +""" +function _ec2_metadata(metadata_endpoint::String) + try + request = @mock http_get( + "http://169.254.169.254/latest/meta-data/$metadata_endpoint" + ) + + return String(request.body) + catch e + e isa IOError || rethrow(e) + end + + return nothing +end + + +""" + ec2_instance_credentials() -> AWSCredentials + +Parse the EC2 metadata to retrieve AWSCredentials. +""" +function ec2_instance_credentials() + info = _ec2_metadata("iam/info") + info = JSON.parse(info) + + name = _ec2_metadata("iam/security-credentials/") + creds = _ec2_metadata("iam/security-credentials/$name") + new_creds = JSON.parse(creds) + + expiry = DateTime(strip(new_creds["Expiration"], 'Z')) + + return AWSCredentials( + new_creds["AccessKeyId"], + new_creds["SecretAccessKey"], + new_creds["Token"], + info["InstanceProfileArn"]; + expiry=expiry, + renew=ec2_instance_credentials + ) +end + + +""" + ecs_instance_credentials() -> Union{AWSCredential, Nothing} + +Retrieve credentials from the local endpoint. Return `nothing` if not running on an ECS +instance. + +More information can be found at: +https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html + +# Returns +- `AWSCredentials`: AWSCredentials from `ECS` credentials URI, `nothing` if the Env Var is + not set (not running on an ECS container instance) + +# Throws +- `StatusError`: If the response status is >= 300 +- `ParsingError`: Invalid HTTP request target +""" +function ecs_instance_credentials() + if !haskey(ENV, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + return nothing + end + + uri = ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] + + response = @mock http_get("http://169.254.170.2$uri") + new_creds = String(response.body) + new_creds = JSON.parse(new_creds) + + expiry = DateTime(rstrip(new_creds["Expiration"], 'Z')) + + return AWSCredentials( + new_creds["AccessKeyId"], + new_creds["SecretAccessKey"], + new_creds["Token"], + new_creds["RoleArn"]; + expiry=expiry, + renew=ecs_instance_credentials + ) +end + + +""" + env_var_credentials() -> Union{AWSCredential, Nothing} + +Use AWS environmental variables (e.g. AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, etc.) +to create AWSCredentials. +""" +function env_var_credentials() + if haskey(ENV, "AWS_ACCESS_KEY_ID") && haskey(ENV, "AWS_SECRET_ACCESS_KEY") + return AWSCredentials( + ENV["AWS_ACCESS_KEY_ID"], + ENV["AWS_SECRET_ACCESS_KEY"], + get(ENV, "AWS_SESSION_TOKEN", ""), + get(ENV, "AWS_USER_ARN", ""); + renew=env_var_credentials + ) + end + + return nothing +end + + +""" + dot_aws_credentials(profile=nothing) -> Union{AWSCredential, Nothing} + +Retrieve AWSCredentials from the `~/.aws/credentials` file + +# Arguments +- `profile`: Specific profile used to get AWSCredentials, default is `nothing` +""" +function dot_aws_credentials(profile=nothing) + credential_file = @mock dot_aws_credentials_file() + + if isfile(credential_file) + ini = read(Inifile(), credential_file) + access_key, secret_key, token = _aws_get_credential_details( + profile === nothing ? _aws_get_profile() : profile, + ini, + ) + + if access_key !== nothing + return AWSCredentials(access_key, secret_key, token) + end + end + + return nothing +end + + +""" + dot_aws_config(profile=nothing) -> Union{AWSCredential, Nothing} + +Retrieve AWSCredentials for the default or specified profile from the `~/.aws/config` file. +If this fails try to retrieve credentials from `_aws_get_role()`, otherwise return `nothing` + +# Arguments +- `profile`: Specific profile used to get AWSCredentials, default is `nothing` +""" +function dot_aws_config(profile=nothing) + config_file = dot_aws_config_file() + + if isfile(config_file) + ini = read(Inifile(), config_file) + p = profile === nothing ? _aws_get_profile() : profile + access_key, secret_key, token = _aws_get_credential_details(p, ini) + + if access_key !== nothing + return AWSCredentials(access_key, secret_key, token) + else + return _aws_get_role(p, ini) + end + end + + return nothing +end + + +""" + _aws_get_credential_details(profile::AbstractString, ini::Inifile) -> Tuple + +Get `AWSCredentials` for the specified `profile` from the `inifile`. If targeting the +`~/.aws/config` file, with a non-default `profile`, you must specify `config=true` otherwise +the default credentials will be returned. + +# Arguments +- `profile::AbstractString`: Specific profile used to get AWSCredentials +- `ini::Inifile`: Inifile to look into for the `profile` credentials +""" +function _aws_get_credential_details(profile::AbstractString, ini::Inifile) + access_key = _get_ini_value(ini, profile, "aws_access_key_id") + secret_key = _get_ini_value(ini, profile, "aws_secret_access_key") + token = _get_ini_value(ini, profile, "aws_session_token"; default_value="") + + return (access_key, secret_key, token) +end + + +""" + aws_get_region(profile::AbstractString, ini::Inifile) -> String + +Retrieve the AWS Region for a given profile, returns `us-east-1` as a default. + +# Arguments +- `profile::AbstractString`: Specific profile used to get the region +- `ini::Inifile`: Inifile to look in for the region +""" +function aws_get_region(profile::AbstractString, ini::Inifile) + region = get(ENV, "AWS_DEFAULT_REGION", "us-east-1") + region = _get_ini_value(ini, profile, "region"; default_value=region) + + return region +end + + +""" + aws_user_arn(aws::AWSConfig) -> String + +Retrieve the `User ARN` from the `AWSConfig`, if not present query STS to update the +`user_arn`. + +# Arguments +- `aws::AWSConfig`: SymbolDict used to retrieve the user arn +""" +function aws_user_arn(aws::AWSConfig) + creds = aws[:creds] + + if isempty(creds.user_arn) + _update_creds(aws) + end + + return creds.user_arn +end + + +""" + aws_account_number(aws::AWSConfig) -> String + +Retrieve the `AWS account number` from the `AWSConfig`, if not present query STS to update +the `AWS account number`. + +# Arguments +- `aws::AWSConfig`: SymbolDict used to retrieve the AWS account number +""" +function aws_account_number(aws::AWSConfig) + creds = aws[:creds] + + if isempty(creds.account_number) + _update_creds(aws) + end + + return creds.account_number +end + + +""" + _update_creds(aws::AWSConfig) -> AWSConfig + + +Update the `user_arn` and `account_number` from Security Token Services. +""" +function _update_creds(aws::AWSConfig) + r = Services.sts(aws, "GetCallerIdentity", []) + creds = aws[:creds] + + creds.user_arn = r["Arn"] + creds.account_number = r["Account"] + + return creds +end + + +""" + _aws_get_role(role::AbstractString, ini::Inifile) -> Union{AWSCredentials, Nothing} + +Retrieve the `AWSCredentials` for a given role from Security Token Services (STS). + +# Arguments +- `role::AbstractString`: Name of the `role` +- `ini::Inifile`: Inifile to look into to find the `role` +""" +function _aws_get_role(role::AbstractString, ini::Inifile) + source_profile, role_arn = aws_get_role_details(role, ini) + source_profile === nothing && return nothing + credentials = nothing + + for f in [dot_aws_credentials, dot_aws_config] + credentials = f(source_profile) + credentials === nothing || break + end + + credentials === nothing && return nothing + config = AWSConfig(:creds => credentials, :region => aws_get_region(source_profile, ini)) + + role = Services.sts( + config, + "AssumeRole", + RoleArn=role_arn, + RoleSessionName=replace(role, r"[^\w+=,.@-]" => s"-"), + ) + + role_creds = role["Credentials"] + + return AWSCredentials( + role_creds["AccessKeyId"], + role_creds["SecretAccessKey"], + role_creds["SessionToken"]; + expiry=unix2datetime(role_creds["Expiration"]) + ) +end + + +""" + aws_get_role_details(profile::AbstractString, ini::Inifile) -> Tuple + +Return a tuple of `profile` details and the `role arn`. + +# Arguments +- `profile::AbstractString`: Specific profile to get role details about +- `ini::Inifile`: Inifile to look into to find the role details +""" +function aws_get_role_details(profile::AbstractString, ini::Inifile) + role_arn = _get_ini_value(ini, profile, "role_arn") + source_profile = _get_ini_value(ini, profile, "source_profile") + + return (source_profile, role_arn) +end + + +""" + _get_ini_value( + ini::Inifile, profile::AbstractString, key::AbstractString; + default_value=nothing + ) -> String + +Get the value for `key` in the `ini` file for a given `profile`. + +# Arguments +- `ini::Inifile`: Inifile to look for `key` in +- `profile::AbstractString`: Given profile to find the `key` for +- `key::AbstractString`: Name of the `key` to get + +# Keywords +- `default_value`: If the `key` is not found, default to this value +""" +function _get_ini_value( + ini::Inifile, profile::AbstractString, key::AbstractString; + default_value=nothing +) + value = get(ini, profile, key, default_value) + value = get(ini, "profile $profile", key, value) + + return value +end diff --git a/src/AWSExceptions.jl b/src/AWSExceptions.jl index 42b0a16987..64f04ca486 100644 --- a/src/AWSExceptions.jl +++ b/src/AWSExceptions.jl @@ -1,6 +1,8 @@ module AWSExceptions -export ProtocolNotDefined, InvalidFileName +using HTTP + +export AWSException, ProtocolNotDefined, InvalidFileName struct ProtocolNotDefined <: Exception message::String @@ -12,4 +14,50 @@ struct InvalidFileName <: Exception end show(io::IO, e::InvalidFileName) = println(io, e.message) + +struct AWSException <: Exception + code::String + message::String + info::Union{String, Nothing} + cause::HTTP.StatusError +end +show(io::IO, e::AWSException) = println( + io, + string(e.code, isempty(e.message) ? "" : (" -- " * e.message), "\n", e.cause) +) + + +function AWSException(e::HTTP.StatusError) + code = string(http_status(e)) + message = "AWSException" + info = Dict(String, Dict) + + # Extract API error code from Lambda-style JSON error message... + if occursin(r"json$", content_type(e)) + info = JSON.parse(http_message(e)) + end + + # Extract API error code from JSON error message... + if occursin(r"^application/x-amz-json-1\.[01]$", content_type(e)) + info = JSON.parse(http_message(e)) + if haskey(info, "__type") + code = split(info["__type"], "#")[end] + end + end + + # Extract API error code from XML error message... + error_content_types = ["", "application/xml", "text/xml"] + if content_type(e) in error_content_types && length(http_message(e)) > 0 + info = parse_xml(http_message(e)) + end + + info = get(info, "Errors", info) + info = get(info, "Error", info) + code = get(info, "Code", code) + message = get(info, "Message", message) + message = get(info, "message", message) + + return AWSException(code, message, info, e) +end + end diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl new file mode 100644 index 0000000000..69377c1bf2 --- /dev/null +++ b/test/AWSCredentials.jl @@ -0,0 +1,551 @@ +macro test_ecode(error_codes, expr) + quote + try + $expr + @test false + catch e + if e isa AWSException + @test ecode(e) in [$error_codes;] + else + rethrow(e) + end + end + end +end + +@testset "Load Credentials" begin + user = AWS.aws_user_arn(aws) # TODO: Remove qualification when dep on AWSCore is removed + @test occursin(r"^arn:aws:(iam|sts)::[0-9]+:[^:]+$", user) + aws[:region] = "us-east-1" + + @test_ecode( + "InvalidAction", + AWSCore.Services.iam("GetFoo", Dict("ContentType" => "JSON")) + ) + + @test_ecode( + ["AccessDenied", "NoSuchEntity"], + AWSCore.Services.iam("GetUser", Dict("UserName" => "notauser", "ContentType" => "JSON")) + ) + + @test_ecode( + "ValidationError", + AWSCore.Services.iam("GetUser", Dict("UserName" => "@#!%%!", "ContentType" => "JSON")) + ) + + @test_ecode( + ["AccessDenied", "EntityAlreadyExists"], + AWSCore.Services.iam("CreateUser", Dict("UserName" => "root", "ContentType" => "JSON")) + ) +end + +@testset "NoAuth" begin + pub_request1 = Dict{Symbol, Any}( + :service => "s3", + :headers => Dict{String, String}("Range" => "bytes=0-0"), + :content => "", + :resource => "/invenia-static-website-content/invenia_ca/index.html", + :url => "https://s3.us-east-1.amazonaws.com/invenia-static-website-content/invenia_ca/index.html", + :verb => "GET", + :region => "us-east-1", + :creds => nothing, + ) + pub_request2 = Dict{Symbol, Any}( + :service => "s3", + :headers => Dict{String, String}("Range" => "bytes=0-0"), + :content => "", + :resource => "ryft-public-sample-data/AWS-x86-AMI-queries.json", + :url => "https://s3.amazonaws.com/ryft-public-sample-data/AWS-x86-AMI-queries.json", + :verb => "GET", + :region => "us-east-1", + :creds => nothing, + ) + + try + response = AWSCore.do_request(pub_request1) + @test response == "<" + catch e + @test_ecode( + ["AccessDenied", "NoSuchEntity"], + AWSCore.do_request(pub_request1) + ) + end + + try + response = AWSCore.do_request(pub_request2) + @test response == UInt8['['] + catch e + @test_ecode( + ["AccessDenied", "NoSuchEntity"], + AWSCore.do_request(pub_request2) + ) + end +end + +@testset "AWSCredentials" begin + @testset "Defaults" begin + creds = AWS.AWSCredentials("access_key_id" ,"secret_key") + @test creds.token == "" + @test creds.user_arn == "" + @test creds.account_number == "" + @test creds.expiry == typemax(DateTime) + @test creds.renew == nothing + end + + @testset "Renewal" begin + # Credentials shouldn't throw an error if no renew function is supplied + creds = AWS.AWSCredentials("access_key_id", "secret_key", renew=nothing) + # TODO: Remove qualification when dep on AWSCore is removed + newcreds = AWS.check_credentials(creds, force_refresh = true) + + # Creds should remain unchanged if no renew function exists + @test creds === newcreds + @test creds.access_key_id == "access_key_id" + @test creds.secret_key == "secret_key" + @test creds.renew == nothing + + # Creds should error if the renew function returns nothing + creds = AWS.AWSCredentials("access_key_id", "secret_key", renew = () -> nothing) + # TODO: Remove qualification when dep on AWSCore is removed + @test_throws ErrorException AWS.check_credentials(creds, force_refresh=true) + + # Creds should remain unchanged + @test creds.access_key_id == "access_key_id" + @test creds.secret_key == "secret_key" + + # Creds should take on value of a returned AWSCredentials except renew function + function gen_credentials() + i = 0 + () -> (i += 1; AWSCredentials("NEW_ID_$i", "NEW_KEY_$i")) + end + + creds = AWS.AWSCredentials( + "access_key_id", + "secret_key", + renew=gen_credentials(), + expiry=now(UTC), + ) + + @test creds.renew !== nothing + renewed = creds.renew() + + @test creds.access_key_id == "access_key_id" + @test creds.secret_key == "secret_key" + @test creds.expiry <= now(UTC) + @test AWS._will_expire(creds) + + @test renewed.access_key_id === "NEW_ID_1" + @test renewed.secret_key == "NEW_KEY_1" + @test renewed.renew === nothing + @test renewed.expiry == typemax(DateTime) + @test !AWS._will_expire(renewed) + renew = creds.renew + + # Check renewal on time out + # TODO: Remove qualification when dep on AWSCore is removed + newcreds = AWS.check_credentials(creds, force_refresh=false) + @test creds === newcreds + @test creds.access_key_id == "NEW_ID_2" + @test creds.secret_key == "NEW_KEY_2" + @test creds.renew !== nothing + @test creds.renew === renew + @test creds.expiry == typemax(DateTime) + @test !AWS._will_expire(creds) + + # Check renewal doesn't happen if not forced or timed out + # TODO: Remove qualification when dep on AWSCore is removed + newcreds = AWS.check_credentials(creds, force_refresh=false) + @test creds === newcreds + @test creds.access_key_id == "NEW_ID_2" + @test creds.secret_key == "NEW_KEY_2" + @test creds.renew !== nothing + @test creds.renew === renew + @test creds.expiry == typemax(DateTime) + + # Check forced renewal works + # TODO: Remove qualification when dep on AWSCore is removed + newcreds = AWS.check_credentials(creds, force_refresh=true) + @test creds === newcreds + @test creds.access_key_id == "NEW_ID_3" + @test creds.secret_key == "NEW_KEY_3" + @test creds.renew !== nothing + @test creds.renew === renew + @test creds.expiry == typemax(DateTime) + end + + mktempdir() do dir + config_file = joinpath(dir, "config") + creds_file = joinpath(dir, "creds") + write( + config_file, + """ + [profile test] + output = json + region = us-east-1 + + [profile test:dev] + source_profile = test + role_arn = arn:aws:iam::123456789000:role/Dev + + [profile test:sub-dev] + source_profile = test:dev + role_arn = arn:aws:iam::123456789000:role/SubDev + + [profile test2] + aws_access_key_id = WRONG_ACCESS_ID + aws_secret_access_key = WRONG_ACCESS_KEY + output = json + region = us-east-1 + + [profile test3] + source_profile = test:dev + role_arn = arn:aws:iam::123456789000:role/test3 + + [profile test4] + aws_access_key_id = RIGHT_ACCESS_ID4 + aws_secret_access_key = RIGHT_ACCESS_KEY4 + source_profile = test:dev + role_arn = arn:aws:iam::123456789000:role/test3 + """ + ) + + write( + creds_file, + """ + [test] + aws_access_key_id = TEST_ACCESS_ID + aws_secret_access_key = TEST_ACCESS_KEY + + [test2] + aws_access_key_id = RIGHT_ACCESS_ID2 + aws_secret_access_key = RIGHT_ACCESS_KEY2 + + [test3] + aws_access_key_id = RIGHT_ACCESS_ID3 + aws_secret_access_key = RIGHT_ACCESS_KEY3 + """ + ) + + withenv( + "AWS_SHARED_CREDENTIALS_FILE" => creds_file, + "AWS_CONFIG_FILE" => config_file, + "AWS_DEFAULT_PROFILE" => "test", + "AWS_ACCESS_KEY_ID" => nothing + ) do + + @testset "Loading" begin + # Check credentials load + config = aws_config() + creds = config[:creds] + + # TODO: Resolve test after AWSConfig is rewritten + @test_broken creds isa AWS.AWSCredentials + + @test creds.access_key_id == "TEST_ACCESS_ID" + @test creds.secret_key == "TEST_ACCESS_KEY" + @test creds.renew !== nothing + + # Check credential file takes precedence over config + ENV["AWS_DEFAULT_PROFILE"] = "test2" + config = aws_config() + creds = config[:creds] + + @test creds.access_key_id == "RIGHT_ACCESS_ID2" + @test creds.secret_key == "RIGHT_ACCESS_KEY2" + + # Check credentials take precedence over role + ENV["AWS_DEFAULT_PROFILE"] = "test3" + config = aws_config() + creds = config[:creds] + + @test creds.access_key_id == "RIGHT_ACCESS_ID3" + @test creds.secret_key == "RIGHT_ACCESS_KEY3" + + ENV["AWS_DEFAULT_PROFILE"] = "test4" + config = aws_config() + creds = config[:creds] + + @test creds.access_key_id == "RIGHT_ACCESS_ID4" + @test creds.secret_key == "RIGHT_ACCESS_KEY4" + + end + + @testset "Refresh" begin + ENV["AWS_DEFAULT_PROFILE"] = "test" + # Check credentials refresh on timeout + config = aws_config() + creds = config[:creds] + creds.access_key_id = "EXPIRED_ACCESS_ID" + creds.secret_key = "EXPIRED_ACCESS_KEY" + creds.expiry = now(UTC) + + @test creds.renew !== nothing + renew = creds.renew + + # TODO: Resolve test after AWSConfig is rewritten + @test_broken renew() isa AWS.AWSCredentials + + # TODO: Remove qualification when dep on AWSCore is removed + creds = AWSCore.check_credentials(config[:creds]) + + @test creds.access_key_id == "TEST_ACCESS_ID" + @test creds.secret_key == "TEST_ACCESS_KEY" + @test creds.expiry > now(UTC) + + # Check renew function remains unchanged + @test creds.renew !== nothing + @test creds.renew === renew + + # Check force_refresh + creds.access_key_id = "WRONG_ACCESS_KEY" + # TODO: Remove qualification when dep on AWSCore is removed + creds = AWSCore.check_credentials(creds, force_refresh = true) + @test creds.access_key_id == "TEST_ACCESS_ID" + end + + @testset "Profile" begin + # Check profile kwarg + ENV["AWS_DEFAULT_PROFILE"] = "test" + creds = AWS.AWSCredentials(profile="test2") + @test creds.access_key_id == "RIGHT_ACCESS_ID2" + @test creds.secret_key == "RIGHT_ACCESS_KEY2" + + config = aws_config(profile="test2") + creds = config[:creds] + @test creds.access_key_id == "RIGHT_ACCESS_ID2" + @test creds.secret_key == "RIGHT_ACCESS_KEY2" + + # Check profile persists on renewal + creds.access_key_id = "WRONG_ACCESS_ID2" + creds.secret_key = "WRONG_ACCESS_KEY2" + # TODO: Remove qualification when dep on AWSCore is removed + creds = AWSCore.check_credentials(creds, force_refresh=true) + + @test creds.access_key_id == "RIGHT_ACCESS_ID2" + @test creds.secret_key == "RIGHT_ACCESS_KEY2" + end + + @testset "Assume Role" begin + # Check we try to assume a role + ENV["AWS_DEFAULT_PROFILE"] = "test:dev" + + @test_ecode( + "InvalidClientTokenId", + aws_config() + ) + + # Check we try to assume a role + ENV["AWS_DEFAULT_PROFILE"] = "test:sub-dev" + let oldout = stdout + r,w = redirect_stdout() + + @test_ecode( + "InvalidClientTokenId", + aws_config() + ) + redirect_stdout(oldout) + close(w) + output = String(read(r)) + occursin("Assuming \"test:dev\"", output) + occursin("Assuming \"test\"", output) + close(r) + end + end + end + end +end + +@testset "Retrieving AWS Credentials" begin + test_values = Dict{String, Any}( + "Default-Profile" => "default", + "Test-Profile" => "test", + "Test-Config-Profile" => "profile test", + + # Default profile values, needs to match due to AWSCredentials.jl:239 + "AccessKeyId" => "Default-Key", + "SecretAccessKey" => "Default-Secret", + + "Test-AccessKeyId" => "Test-Key", + "Test-SecretAccessKey" => "Test-Secret", + + "Token" => "Test-Token", + "InstanceProfileArn" => "Test-Arn", + "RoleArn" => "Test-Arn", + "Expiration" => now(), + + "URI" => "/Test-URI/", + "Security-Credentials" => "Test-Security-Credentials" + ) + + http_get_patch = @patch function http_get(url::String) + security_credentials = test_values["Security-Credentials"] + uri = test_values["URI"] + + if url == "http://169.254.169.254/latest/meta-data/iam/info" + instance_profile_arn = test_values["InstanceProfileArn"] + return HTTP.Response("{\"InstanceProfileArn\": \"$instance_profile_arn\"}") + elseif url == "http://169.254.169.254/latest/meta-data/iam/security-credentials/" + return HTTP.Response(test_values["Security-Credentials"]) + elseif url == "http://169.254.169.254/latest/meta-data/iam/security-credentials/$security_credentials" || url == "http://169.254.170.2$uri" + my_dict = JSON.json(test_values) + response = HTTP.Response(my_dict) + return response + else + return nothing + end + end + + @testset "~/.aws/config - Default Profile" begin + mktemp() do config_file, config_io + write( + config_io, + """ + [$(test_values["Default-Profile"])] + aws_access_key_id=$(test_values["AccessKeyId"]) + aws_secret_access_key=$(test_values["SecretAccessKey"]) + """ + ) + close(config_io) + + withenv("AWS_CONFIG_FILE" => config_file) do + # TODO: Remove qualification when dep on AWSCore is removed + default_profile = AWS.dot_aws_config() + + @test default_profile.access_key_id == test_values["AccessKeyId"] + @test default_profile.secret_key == test_values["SecretAccessKey"] + end + end + end + + @testset "~/.aws/config - Specified Profile" begin + mktemp() do config_file, config_io + write( + config_io, + """ + [$(test_values["Test-Config-Profile"])] + aws_access_key_id=$(test_values["Test-AccessKeyId"]) + aws_secret_access_key=$(test_values["Test-SecretAccessKey"]) + """ + ) + close(config_io) + + withenv("AWS_CONFIG_FILE" => config_file) do + # TODO: Remove qualification when dep on AWSCore is removed + specified_result = AWS.dot_aws_config(test_values["Test-Profile"]) + + @test specified_result.access_key_id == test_values["Test-AccessKeyId"] + @test specified_result.secret_key == test_values["Test-SecretAccessKey"] + end + end + end + + @testset "~/.aws/creds - Default Profile" begin + mktemp() do creds_file, creds_io + write( + creds_io, + """ + [$(test_values["Default-Profile"])] + aws_access_key_id=$(test_values["AccessKeyId"]) + aws_secret_access_key=$(test_values["SecretAccessKey"]) + """ + ) + close(creds_io) + + withenv("AWS_SHARED_CREDENTIALS_FILE" => creds_file) do + # TODO: Remove qualification when dep on AWSCore is removed + specified_result = AWS.dot_aws_credentials() + + @test specified_result.access_key_id == test_values["AccessKeyId"] + @test specified_result.secret_key == test_values["SecretAccessKey"] + end + end + end + + @testset "~/.aws/creds - Specified Profile" begin + mktemp() do creds_file, creds_io + write( + creds_io, + """ + [$(test_values["Test-Profile"])] + aws_access_key_id=$(test_values["Test-AccessKeyId"]) + aws_secret_access_key=$(test_values["Test-SecretAccessKey"]) + """ + ) + close(creds_io) + + withenv("AWS_SHARED_CREDENTIALS_FILE" => creds_file) do + # TODO: Remove qualification when dep on AWSCore is removed + specified_result = AWS.dot_aws_credentials(test_values["Test-Profile"]) + + @test specified_result.access_key_id == test_values["Test-AccessKeyId"] + @test specified_result.secret_key == test_values["Test-SecretAccessKey"] + end + end + end + + @testset "Environment Variables" begin + withenv( + "AWS_ACCESS_KEY_ID" => test_values["AccessKeyId"], + "AWS_SECRET_ACCESS_KEY" => test_values["SecretAccessKey"] + ) do + # TODO: Remove qualification when dep on AWSCore is removed + aws_creds = AWS.env_var_credentials() + @test aws_creds.access_key_id == test_values["AccessKeyId"] + @test aws_creds.secret_key == test_values["SecretAccessKey"] + end + end + + @testset "Instance - EC2" begin + apply([http_get_patch]) do + # TODO: Remove qualification when dep on AWSCore is removed + result = AWS.ec2_instance_credentials() + @test result.access_key_id == test_values["AccessKeyId"] + @test result.secret_key == test_values["SecretAccessKey"] + @test result.token == test_values["Token"] + @test result.user_arn == test_values["InstanceProfileArn"] + @test result.expiry == test_values["Expiration"] + # TODO: Remove qualification when dep on AWSCore is removed + @test result.renew == AWS.ec2_instance_credentials + end + end + + @testset "Instance - ECS" begin + withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => test_values["URI"]) do + apply([http_get_patch]) do + # TODO: Remove qualification when dep on AWSCore is removed + result = AWS.ecs_instance_credentials() + @test result.access_key_id == test_values["AccessKeyId"] + @test result.secret_key == test_values["SecretAccessKey"] + @test result.token == test_values["Token"] + @test result.user_arn == test_values["RoleArn"] + @test result.expiry == test_values["Expiration"] + # TODO: Remove qualification when dep on AWSCore is removed + @test result.renew == AWS.ecs_instance_credentials + end + end + end + + @testset "Helper functions" begin + @testset "Check Credentials - EnvVars" begin + withenv( + "AWS_ACCESS_KEY_ID" => test_values["AccessKeyId"], + "AWS_SECRET_ACCESS_KEY" => test_values["SecretAccessKey"] + ) do + testAWSCredentials = AWS.AWSCredentials( + test_values["AccessKeyId"], + test_values["SecretAccessKey"], + expiry=Dates.now() - Minute(10), + # TODO: Remove qualification when dep on AWSCore is removed + renew=AWS.env_var_credentials + ) + + # TODO: Remove qualification when dep on AWSCore is removed + result = AWS.check_credentials(testAWSCredentials, force_refresh=true) + @test result.access_key_id == testAWSCredentials.access_key_id + @test result.secret_key == testAWSCredentials.secret_key + @test result.expiry == typemax(DateTime) + @test result.renew == testAWSCredentials.renew + end + end + end +end diff --git a/test/AWSExceptions.jl b/test/AWSExceptions.jl new file mode 100644 index 0000000000..a18030321b --- /dev/null +++ b/test/AWSExceptions.jl @@ -0,0 +1,59 @@ +@testset "AWSException" begin + @testset "XML Request" begin + code = "NoSuchKey" + message = "The resource you requested does not exist" + resource = "/mybucket/myfoto.jpg" + requestId = "4442587FB7D0A2F9" + + body = """ + + + $code + $message + $resource + $requestId + + """ + headers = ["Content-Type" => "application/xml"] + status_code = 400 + + # This does not actually send a request, just creates the object to test with + req = HTTP.Request("GET", "https://amazon.ca", headers, body) + resp = HTTP.Response(status_code, headers; body=body, request=req) + ex = AWSException(HTTP.StatusError(status_code, resp)) + + @test ex.code == code + @test ex.message == ex.info["Message"] == message + @test ex.info["Resource"] == resource + @test ex.info["RequestId"] == requestId + @test String(ex.cause.response.body) == body + @test ex.cause.status == status_code + @test ex.cause.response.headers == headers + end + + + @testset "JSON Request" begin + code = "InvalidSignatureException" + message = "Signature expired: ..." + body = """ + { + "__type": "$code", + "message": "$message" + } + """ + headers = ["Content-Type" => "application/x-amz-json-1.1"] + status_code = 400 + + # This does not actually send a request, just creates the object to test with + req = HTTP.Request("GET", "https://amazon.ca", headers, body) + resp = HTTP.Response(status_code, headers; body=body, request=req) + ex = AWSException(HTTP.StatusError(status_code, resp)) + + @test ex.code == code + @test ex.message == ex.info["message"] == message + @test ex.info["__type"] == code + @test String(ex.cause.response.body) == body + @test ex.cause.status == status_code + @test ex.cause.response.headers == headers + end +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index a7bce4de80..274cdbd042 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,11 +1,25 @@ using AWS +using AWS: AWSCredentials +using AWSCore +using AWSCore: Services using AWS.AWSMetadataUtilities: _documentation_cleaning, _filter_latest_service_version, _generate_low_level_definition, _generate_high_level_definition, _generate_high_level_definitions, _get_aws_sdk_js_files, _get_service_and_version, _get_function_parameters, InvalidFileName, ProtocolNotDefined +using Dates +using HTTP using JSON using OrderedCollections: OrderedDict +using Mocking +using Retry using Test -include("AWS.jl") -include("AWSMetadataUtilities.jl") +Mocking.activate() +aws = aws_config() + +@testset "AWS.jl" begin + include("AWS.jl") + include("AWSCredentials.jl") + include("AWSExceptions.jl") + include("AWSMetadataUtilities.jl") +end