From 19e20b6f62529333018559ae4947ab96353dca78 Mon Sep 17 00:00:00 2001 From: Matt Brzezinski Date: Wed, 11 Sep 2019 09:33:36 -0500 Subject: [PATCH] Moved AWS4AuthLayer from AWSAuth.jl into AWSCore.jl Moved AWS4AuthLayer into this package from AWSAuth.jl. Originally wanted to move AWSCredentials into AWSAuth.jl however this would create a circular dependency. Instead we're moving AWS4AuthLayer into AWSCore.jl. This now allows us to clean up HTTP.jl and remove AWS functionality from it. --- Project.toml | 4 +- src/AWSCore.jl | 20 +- src/AWSCredentials.jl | 662 ++++++++++++++++++++++++---------------- src/AWSException.jl | 1 - src/deprecations.jl | 2 + src/signaturev4.jl | 205 +++++++++++++ test/arn.jl | 8 + test/aws4.jl | 201 ++++++++++++ test/credentials.jl | 526 +++++++++++++++++++++++++++++++ test/exceptions.jl | 20 ++ test/jltest.aws.example | 4 - test/localhost.jl | 20 ++ test/runtests.jl | 609 ++---------------------------------- test/signaturev4.jl | 129 ++++++++ test/xml.jl | 144 +++++++++ 15 files changed, 1676 insertions(+), 879 deletions(-) create mode 100644 src/deprecations.jl create mode 100644 src/signaturev4.jl create mode 100644 test/arn.jl create mode 100644 test/aws4.jl create mode 100644 test/credentials.jl create mode 100644 test/exceptions.jl delete mode 100644 test/jltest.aws.example create mode 100644 test/localhost.jl create mode 100644 test/signaturev4.jl create mode 100644 test/xml.jl diff --git a/Project.toml b/Project.toml index 4855fee..a2b7159 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "AWSCore" uuid = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" -version = "0.6.2" +version = "0.6.3" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" @@ -11,12 +11,14 @@ IniFile = "83e8ac13-25f8-5344-8a64-a9f2b223428f" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" LazyJSON = "fc18253b-5e1b-504c-a4a2-9ece4944c004" MbedTLS = "739be429-bea8-5141-9913-cc70e7f3736d" +Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533" Retry = "20febd7b-183b-5ae2-ac4a-720e7ce64774" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" SymDict = "2da68c74-98d7-5633-99d6-8493888d7b1e" XMLDict = "228000da-037f-5747-90a9-8195ccbf91a5" [compat] +Mocking = "0.7" julia = "1" [extras] diff --git a/src/AWSCore.jl b/src/AWSCore.jl index b09ed55..0e57c54 100644 --- a/src/AWSCore.jl +++ b/src/AWSCore.jl @@ -8,8 +8,8 @@ module AWSCore -export AWSException, AWSConfig, AWSRequest, - aws_config, default_aws_config +export AWSException, AWSConfig, AWSRequest, SignatureV4, aws_config, default_aws_config, + http_get using Base64 using Dates @@ -49,14 +49,15 @@ It contains the following keys: """ const AWSRequest = SymbolDict - include("http.jl") include("AWSException.jl") include("AWSCredentials.jl") +include("deprecations.jl") include("names.jl") include("mime.jl") - - +include("signaturev4.jl") +include("sign.jl") +include("Services.jl") #------------------------------------------------------------------------------# # Configuration. @@ -106,7 +107,6 @@ as follows. However, putting access credentials in source code is discouraged. aws = aws_config(creds = AWSCredentials("AKIAXXXXXXXXXXXXXXXX", "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")) ``` - """ function aws_config(;profile=nothing, creds=AWSCredentials(profile=profile), @@ -120,6 +120,8 @@ global _default_aws_config = nothing # Union{AWSConfig,Nothing} """ + default_aws_config() + `default_aws_config` returns a global shared [`AWSConfig`](@ref) object obtained by calling [`aws_config`](@ref) with no optional arguments. """ @@ -138,7 +140,6 @@ end Convert nested `Vector{Pair}` maps in `args` into `Dict{String,Any}` maps. """ function aws_args_dict(args) - result = stringdict(args) dictlike(t) = (t <: AbstractDict @@ -557,11 +558,6 @@ global debug_level = 0 function set_debug_level(n) global debug_level = n end - - -include("Services.jl") - - end # module AWSCore diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 46e81fb..e64c47a 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -1,32 +1,35 @@ -#==============================================================================# -# AWSCredentials.jl -# -# Load AWS Credentials from: -# - EC2 Instance Profile, -# - Environment variables, or -# - ~/.aws/credentials file. -# -# Copyright OC Technology Pty Ltd 2014 - All rights reserved -#==============================================================================# +using Base +using IniFile +using HTTP +using Dates +using Mocking +using JSON + export AWSCredentials, - localhost_is_lambda, + aws_account_number, + aws_get_region, + aws_get_role_details, + aws_user_arn, + check_credentials, + dot_aws_config, + dot_aws_credentials, + dot_aws_credentials_file, + dot_aws_config_file, + ec2_instance_credentials, + ecs_instance_credentials, + env_var_credentials, localhost_is_ec2, localhost_maybe_ec2, - aws_user_arn, - aws_account_number, - check_credentials + localhost_is_lambda """ When you interact with AWS, you specify your [AWS Security Credentials](http://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html) to verify who you are and whether you have permission to access the resources that you are requesting. AWS uses the security credentials to authenticate and authorize your requests. - The fields `access_key_id` and `secret_key` hold the access keys used to authenticate API requests (see [Creating, Modifying, and Viewing Access Keys](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey)). - [Temporary Security Credentials](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html) require the extra session `token` field. - The `user_arn` and `account_number` fields are used to cache the result of the [`aws_user_arn`](@ref) and [`aws_account_number`](@ref) functions. AWSCore searches for credentials in a series of possible locations and stop as soon as it finds credentials. @@ -43,7 +46,6 @@ Once the credentials are found, the method by which they were accessed is stored and the DateTime at which they will expire is stored in the `expiry` field. This allows the credentials to be refreshed as needed using [`check_credentials`](@ref). If `renew` is set to `nothing`, no attempt will be made to refresh the credentials. - Any renewal function is expected to return `nothing` on failure or a populated `AWSCredentials` object on success. The `renew` field of the returned `AWSCredentials` will be discarded and does not need to be set. @@ -56,392 +58,470 @@ mutable struct AWSCredentials user_arn::String account_number::String expiry::DateTime - renew::Union{Function, Nothing} + renew::Union{Function, Nothing} # Function which can be used to refresh credentials + + function AWSCredentials( + access_key_id, + secret_key, + token="", + user_arn="", + account_number=""; + expiry=typemax(DateTime), + renew=nothing, + ) + return new(access_key_id, secret_key, token, user_arn, account_number, expiry, renew) + end +end + + +function Base.show(io::IO,c::AWSCredentials) + println(io, + c.user_arn, + isempty(c.user_arn) ? "" : " ", + "(", + c.account_number, + isempty(c.account_number) ? "" : ", ", + c.access_key_id, + isempty(c.secret_key) ? "" : ", $(c.secret_key[1:3])...", + isempty(c.token) ? "" : ", $(c.token[1:3])...", + c.expiry, + ")" + ) +end - function AWSCredentials(access_key_id,secret_key, - token="", user_arn="", account_number=""; - expiry=typemax(DateTime), - renew=nothing) - new(access_key_id, secret_key, token, user_arn, account_number, expiry, renew) + +function Base.copyto!(dest::AWSCredentials, src::AWSCredentials) + for f in fieldnames(typeof(dest)) + setfield!(dest, f, getfield(src, f)) end end + +dot_aws_config_file() = get(ENV, "AWS_CONFIG_FILE", joinpath(homedir(), ".aws", "config")) +dot_aws_credentials_file() = get(ENV, "AWS_SHARED_CREDENTIALS_FILE", joinpath(homedir(), ".aws", "credentials")) +localhost_maybe_ec2() = localhost_is_ec2() || isfile("/sys/devices/virtual/dmi/id/product_uuid") +localhost_is_lambda() = haskey(ENV, "LAMBDA_TASK_ROOT") +_aws_get_profile() = get(ENV, "AWS_DEFAULT_PROFILE", get(ENV, "AWS_PROFILE", "default")) + + +""" + AWSCredentials(;profile=nothing) -> Union{AWSCredentials, Nothing} + +Create an AWSCredentials object, given a provided profile (if not provided "default" will be +used). + +Checks credential locations in the order: + 1. Environment Variables + 2. ~/.aws/credentials + 3. ~/.aws/config + 4. EC2 or ECS metadata + +# Keywords +- `profile::AbstractString`: Specific profile used to search for AWSCredentials + +# Throws +- `error("Can't find AWS Credentials")`: AWSCredentials could not be found +""" function AWSCredentials(;profile=nothing) creds = nothing - renew = Nothing + credential_function = () -> nothing - # Define our search options + if profile == nothing + profile = get(ENV, "AWS_PROFILE", get(ENV, "AWS_DEFAULT_PROFILE", nothing)) + end + + # Define our search options, expected to be callable with no arguments. Should return + # `nothing` when credentials are not able to be located functions = [ - env_instance_credentials, + env_var_credentials, () -> dot_aws_credentials(profile), () -> dot_aws_config(profile), - instance_credentials, + ecs_instance_credentials, + ec2_instance_credentials ] # Loop through our search locations until we get credentials back for f in functions - renew = f - creds = renew() + credential_function = f + creds = credential_function() creds === nothing || break end creds === nothing && error("Can't find AWS credentials!") - creds.renew = renew - - if debug_level > 0 - display(creds) - println() - end + creds.renew = credential_function return creds end -will_expire(cr::AWSCredentials) = now(UTC) >= cr.expiry - Minute(5) """ - check_credentials(cr::AWSCredentials; force_refresh::Bool=false) + localhost_is_ec2() -> Bool -Checks current AWSCredentials, refreshing them if they are soon to expire. -If force_refresh is `true` the credentials will be renewed immediately. +Determine if the machine executing this code is running on an EC2 instance. """ -function check_credentials(cr::AWSCredentials; force_refresh::Bool=false) - if force_refresh || will_expire(cr) - if debug_level > 0 - println("Renewing credentials... ") - end - renew = cr.renew - - if renew !== nothing - new_creds = renew() - - new_creds === nothing && error("Can't find AWS credentials!") - copyto!(cr, new_creds) +function localhost_is_ec2() + # Checking to see if you are running on an EC2 instance is a complicated problem due to + # a large amount of caveats. Below is a list of methods to implement to work through + # most of these problems: + # + # 1. Check the `hostname -d`; this will not work if using non-Amazon DNS + # 2. Check metadata with EC2 internal domain name `curl -s + # http://instance-data.ec2.internal`; this will not work with a VPC (legacy EC2 only) + # 3. Check `sudo dmidecode -s bios-version`; this requires `dmidecode` on the instance + # 4. Check `/sys/devices/virtual/dmi/id/bios_version`; this may not work depending on + # the instance, Amazon does not document this file however so it's quite unreliable + # 5. Check `http://169.254.169.254`; This is a link-local address for metadata, + # apparently other cloud providers make this metadata URL available now as well so it's + # not guaranteed that you're on an EC2 instance + # 6. When checking the UUID, check for little-endian representation, + # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html + + # This is not guarenteed to work on Windows as RNG can make the UUID begin with EC2 on a + # non-EC2 instance + if @mock Sys.iswindows() + command = `wmic path win32_computersystemproduct get uuid` + result = @mock Base.read(command, String) + instance_uuid = strip(split(result, "\n")[2]) + + return instance_uuid[1:3] == "EC2" + end - # Ensure renewal function is not overwritten by the new credentials - cr.renew = renew - else - if debug_level > 0 - println("Credentials cannot be renewed...") - end - end + # Note: This will not work on new m5 and c5 instances because they use a new hypervisor + # stack and the kernel does not create files in sysfs + hypervisor_uuid = "/sys/hypervisor/uuid" + if isfile(hypervisor_uuid) && _begins_with_ec2(hypervisor_uuid) + return true end - return cr -end + product_uuid = "/sys/devices/virtual/dmi/id/product_uuid" + if isreadable(open(product_uuid, "r")) && _begins_with_ec2(product_uuid) + return true + end -function Base.show(io::IO,c::AWSCredentials) - println(io, string(c.user_arn, - c.user_arn == "" ? "" : " ", - "(", - c.account_number, - c.account_number == "" ? "" : ", ", - c.access_key_id, - c.secret_key == "" ? "" : ", $(c.secret_key[1:3])...", - c.token == "" ? "" : ", $(c.token[1:3])..."), - ")") + return false end -function Base.copyto!(dest::AWSCredentials, src::AWSCredentials) - for f in fieldnames(typeof(dest)) - setfield!(dest, f, getfield(src, f)) - end -end -import Base: copy! -Base.@deprecate copy!(dest::AWSCredentials, src::AWSCredentials) copyto!(dest, src) +_begins_with_ec2(file_name::String) = return uppercase(String(read(file_name, 3))) == "EC2" """ -Is Julia running in an AWS Lambda sandbox? -""" -localhost_is_lambda() = haskey(ENV, "LAMBDA_TASK_ROOT") + check_credentials( + aws_creds::AWSCredentials, force_refresh::Bool=false + ) -> AWSCredentials +Checks current AWSCredentials, refreshing them if they are soon to expire. If +`force_refresh` is `true` the credentials will be renewed immediately +# Arguments +- `aws_creds::AWSCredentials`: AWSCredentials to be checked / refreshed + +# Keywords +- `force_refresh::Bool=false`: `true` to refresh the credentials + +# Throws +- `error("Can't find AWS credentials!")`: If no credentials can be found """ -Is Julia running on an EC2 virtual machine? -""" -function localhost_is_ec2() +function check_credentials(aws_creds::AWSCredentials; force_refresh::Bool=false) + if force_refresh || _will_expire(aws_creds) + credential_method = aws_creds.renew - if localhost_is_lambda() - return false - end + if credential_method !== nothing + new_aws_creds = credential_method() - if isfile("/sys/hypervisor/uuid") && - String(read("/sys/hypervisor/uuid",3)) == "ec2" - return true - end + new_aws_creds === nothing && error("Can't find AWS credentials!") + copyto!(aws_creds, new_aws_creds) - if isfile("/sys/devices/virtual/dmi/id/product_uuid") - try - # product_uuid is not world readable! - # https://patchwork.kernel.org/patch/6461521/ - # https://github.com/JuliaCloud/AWSCore.jl/issues/24 - if String(read("/sys/devices/virtual/dmi/id/product_uuid")) == "EC2" - return true - end - catch + # Ensure credential_method is not overwritten by the new credentials + aws_creds.renew = credential_method end end - return false + return aws_creds end -localhost_maybe_ec2() = localhost_is_ec2() || - isfile("/sys/devices/virtual/dmi/id/product_uuid") - -""" - aws_user_arn(::AWSConfig) - -Unique -[Amazon Resource Name] -(http://docs.aws.amazon.com/IAM/latest/UserGuide/id_users.html) -for configrued user. -e.g. `"arn:aws:iam::account-ID-without-hyphens:user/Bob"` -""" -function aws_user_arn(aws::AWSConfig) - creds = aws[:creds] - if creds.user_arn == "" - r = Services.sts(aws, "GetCallerIdentity", []) - creds.user_arn = r["Arn"] - creds.account_number = r["Account"] - end - return creds.user_arn +function _will_expire(aws_creds::AWSCredentials) + return now(UTC) >= aws_creds.expiry - Minute(5) end """ - aws_account_number(::AWSConfig) - -12-digit [AWS Account Number](http://docs.aws.amazon.com/general/latest/gr/acct-identifiers.html). -""" -function aws_account_number(aws::AWSConfig) - creds = aws[:creds] - if creds.account_number == "" - aws_user_arn(aws) - end - return creds.account_number -end + _ec2_metadata(metadata_endpoint::String) -> Union{String, Nothing} +Retrieve the EC2 meta data from the local AWS endpoint. Return the EC2 metadata request +body, or `nothing` if not running on an EC2 instance. -""" - ec2_metadata(key) +# Arguments +- `metadata_endpoint::String`: AWS internal meta data endpoint to hit -Fetch [EC2 meta-data] -(http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) -for `key`. +# Throws +- `StatusError`: If the response status is >= 300 +- `ParsingError`: Invalid HTTP request target """ -function ec2_metadata(key) +function _ec2_metadata(metadata_endpoint::String) + try + request = @mock http_get( + "http://169.254.169.254/latest/meta-data/$metadata_endpoint" + ) - @assert localhost_maybe_ec2() + return String(request.body) + catch e + e isa IOError || rethrow(e) + end - String(http_get("http://169.254.169.254/latest/meta-data/$key").body) + return nothing end -function instance_credentials() - if haskey(ENV, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") - return ecs_instance_credentials() - elseif localhost_maybe_ec2() - return ec2_instance_credentials() - else - return nothing - end -end """ -Load [Instance Profile Credentials] -(http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials) -for EC2 virtual machine. + ec2_instance_credentials() -> AWSCredentials + +Parse the EC2 metadata to retrieve AWSCredentials. """ function ec2_instance_credentials() + info = _ec2_metadata("iam/info") + info = JSON.parse(info) - @assert localhost_maybe_ec2() - - info = ec2_metadata("iam/info") - info = LazyJSON.value(info) - - name = ec2_metadata("iam/security-credentials/") - creds = ec2_metadata("iam/security-credentials/$name") - new_creds = LazyJSON.value(creds) - - if debug_level > 0 - print("Loading AWSCredentials from EC2 metadata... ") - end + name = _ec2_metadata("iam/security-credentials/") + creds = _ec2_metadata("iam/security-credentials/$name") + new_creds = JSON.parse(creds) expiry = DateTime(strip(new_creds["Expiration"], 'Z')) - AWSCredentials(new_creds["AccessKeyId"], - new_creds["SecretAccessKey"], - new_creds["Token"], - info["InstanceProfileArn"]; - expiry = expiry) + return AWSCredentials( + new_creds["AccessKeyId"], + new_creds["SecretAccessKey"], + new_creds["Token"], + info["InstanceProfileArn"]; + expiry=expiry, + renew=ec2_instance_credentials + ) end """ -Load [ECS Task Credentials] -(http://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html) + ecs_instance_credentials() -> Union{AWSCredential, Nothing} + +Retrieve credentials from the local endpoint. Return `nothing` if not running on an ECS +instance. + +More information can be found at: +https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html + +# Returns +- `AWSCredentials`: AWSCredentials from `ECS` credentials URI, `nothing` if the Env Var is + not set (not running on an ECS container instance) + +# Throws +- `StatusError`: If the response status is >= 300 +- `ParsingError`: Invalid HTTP request target """ function ecs_instance_credentials() - - @assert haskey(ENV, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + if !haskey(ENV, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + return nothing + end uri = ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] - new_creds = String(http_get("http://169.254.170.2$uri").body) + response = @mock http_get("http://169.254.170.2$uri") + new_creds = String(response.body) new_creds = LazyJSON.value(new_creds) - if debug_level > 0 - print("Loading AWSCredentials from ECS metadata... ") - end - expiry = DateTime(strip(new_creds["Expiration"], 'Z')) - AWSCredentials(new_creds["AccessKeyId"], - new_creds["SecretAccessKey"], - new_creds["Token"], - new_creds["RoleArn"]; - expiry = expiry) + return AWSCredentials( + new_creds["AccessKeyId"], + new_creds["SecretAccessKey"], + new_creds["Token"], + new_creds["RoleArn"]; + expiry=expiry, + renew=ecs_instance_credentials + ) end """ -Load Credentials from [environment variables] -(http://docs.aws.amazon.com/cli/latest/userguide/cli-environment.html) -`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` etc. -(e.g. in Lambda sandbox). -""" -function env_instance_credentials() - - if haskey(ENV, "AWS_ACCESS_KEY_ID") - if debug_level > 0 - print("Loading AWSCredentials from ENV[\"AWS_ACCESS_KEY_ID\"]... ") - end + env_var_credentials() -> Union{AWSCredential, Nothing} +Use AWS environmental variables (e.g. AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, etc.) +to create AWSCredentials. +""" +function env_var_credentials() + if haskey(ENV, "AWS_ACCESS_KEY_ID") && haskey(ENV, "AWS_SECRET_ACCESS_KEY") return AWSCredentials( ENV["AWS_ACCESS_KEY_ID"], ENV["AWS_SECRET_ACCESS_KEY"], get(ENV, "AWS_SESSION_TOKEN", ""), get(ENV, "AWS_USER_ARN", ""); - renew = env_instance_credentials + renew=env_var_credentials ) - else - return nothing end + + return nothing end -using IniFile +""" + dot_aws_credentials(profile=nothing) -> Union{AWSCredential, Nothing} -function dot_aws_credentials_file() - get(ENV, "AWS_SHARED_CREDENTIALS_FILE", joinpath(homedir(), ".aws", "credentials")) -end +Retrieve AWSCredentials from the `~/.aws/credentials` file +# Arguments +- `profile`: Specific profile used to get AWSCredentials, default is `nothing` """ -Try to load Credentials from [AWS CLI ~/.aws/credentials file] -(http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) -""" -function dot_aws_credentials(profile = nothing) - creds = nothing - credential_file = dot_aws_credentials_file() +function dot_aws_credentials(profile=nothing) + credential_file = @mock dot_aws_credentials_file() - ini = nothing if isfile(credential_file) ini = read(Inifile(), credential_file) - key, key_id, token = aws_get_credential_details( - profile === nothing ? aws_get_profile() : profile, + access_key, secret_key, token = _aws_get_credential_details( + profile === nothing ? _aws_get_profile() : profile, ini, - false ) - if key !== :notfound - creds = AWSCredentials(key_id, key, token) + if access_key !== nothing + return AWSCredentials(access_key, secret_key, token) end end - return creds + return nothing end -dot_aws_config_file() = get(ENV, "AWS_CONFIG_FILE", joinpath(homedir(), ".aws", "config")) """ -Try to load Credentials or assume a role via the [AWS CLI ~/.aws/config file] -(http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) + dot_aws_config(profile=nothing) -> Union{AWSCredential, Nothing} + +Retrieve AWSCredentials for the default or specified profile from the `~/.aws/config` file. +If this fails try to retrieve credentials from `_aws_get_role()`, otherwise return `nothing` + +# Arguments +- `profile`: Specific profile used to get AWSCredentials, default is `nothing` """ -function dot_aws_config(profile = nothing) - creds = nothing +function dot_aws_config(profile=nothing) config_file = dot_aws_config_file() - ini = nothing if isfile(config_file) ini = read(Inifile(), config_file) - p = profile === nothing ? aws_get_profile() : profile - key, key_id, token = aws_get_credential_details(p, ini, true) + p = profile === nothing ? _aws_get_profile() : profile + access_key, secret_key, token = _aws_get_credential_details(p, ini) - if key !== :notfound - creds = AWSCredentials(key_id, key, token) + if access_key !== nothing + return AWSCredentials(access_key, secret_key, token) else - creds = aws_get_role(p, ini) + return _aws_get_role(p, ini) end end - return creds + return nothing end -function aws_get_role_details(profile::AbstractString, ini::Inifile) - if debug_level > 0 - println("Loading \"$profile\" Profile from " * - dot_aws_config_file() * "... ") - end - role_arn = get(ini, profile, "role_arn") - source_profile = get(ini, profile, "source_profile") +""" + _aws_get_credential_details(profile::AbstractString, ini::Inifile) -> Tuple + +Get `AWSCredentials` for the specified `profile` from the `inifile`. If targeting the +`~/.aws/config` file, with a non-default `profile`, you must specify `config=true` otherwise +the default credentials will be returned. - profile = "profile $profile" - role_arn = get(ini, profile, "role_arn", role_arn) - source_profile = get(ini, profile, "source_profile", source_profile) +# Arguments +- `profile::AbstractString`: Specific profile used to get AWSCredentials +- `ini::Inifile`: Inifile to look into for the `profile` credentials +""" +function _aws_get_credential_details(profile::AbstractString, ini::Inifile) + access_key = _get_ini_value(ini, profile, "aws_access_key_id") + secret_key = _get_ini_value(ini, profile, "aws_secret_access_key") + token = _get_ini_value(ini, profile, "aws_session_token"; default_value="") - (source_profile, role_arn) + return (access_key, secret_key, token) end -function aws_get_credential_details(profile::AbstractString, ini::Inifile, config::Bool) - if debug_level > 0 - filename = config ? dot_aws_config_file() : dot_aws_credentials_file() - println("Loading \"$profile\" AWSCredentials from " * filename - * "... ") - end - key_id = get(ini, profile, "aws_access_key_id") - key = get(ini, profile, "aws_secret_access_key") - token = get(ini, profile, "aws_session_token", "") +""" + aws_get_region(profile::AbstractString, ini::Inifile) -> String + +Retrieve the AWS Region for a given profile, returns `us-east-1` as a default. + +# Arguments +- `profile::AbstractString`: Specific profile used to get the region +- `ini::Inifile`: Inifile to look in for the region +""" +function aws_get_region(profile::AbstractString, ini::Inifile) + region = get(ENV, "AWS_DEFAULT_REGION", "us-east-1") + region = _get_ini_value(ini, profile, "region"; default_value=region) - if config - profile = "profile $profile" - key_id = get(ini, profile, "aws_access_key_id", key_id) - key = get(ini, profile, "aws_secret_access_key", key) - token = get(ini, profile, "aws_session_token", token) + return region +end + + +""" + aws_user_arn(aws::AWSConfig) -> String + +Retrieve the `User ARN` from the `AWSConfig`, if not present query STS to update the +`user_arn`. + +# Arguments +- `aws::AWSConfig`: SymbolDict used to retrieve the user arn +""" +function aws_user_arn(aws::AWSConfig) + creds = aws[:creds] + + if isempty(creds.user_arn) + _update_creds(aws) end - (key, key_id, token) + return creds.user_arn end -function aws_get_profile() - get(ENV, "AWS_DEFAULT_PROFILE", get(ENV, "AWS_PROFILE", "default")) + +""" + aws_account_number(aws::AWSConfig) -> String + +Retrieve the `AWS account number` from the `AWSConfig`, if not present query STS to update +the `AWS account number`. + +# Arguments +- `aws::AWSConfig`: SymbolDict used to retrieve the AWS account number +""" +function aws_account_number(aws::AWSConfig) + creds = aws[:creds] + + if isempty(creds.account_number) + _update_creds(aws) + end + + return creds.account_number end -function aws_get_region(profile::AbstractString, ini::Inifile) - region = get(ENV, "AWS_DEFAULT_REGION", "us-east-1") - region = get(ini, profile, "region", region) - region = get(ini, "profile $profile", "region", region) +""" + _update_creds(aws::AWSConfig) -> AWSConfig + + +Update the `user_arn` and `account_number` from Security Token Services. +""" +function _update_creds(aws::AWSConfig) + r = Services.sts(aws, "GetCallerIdentity", []) + creds = aws[:creds] + + creds.user_arn = r["Arn"] + creds.account_number = r["Account"] + + return creds end -function aws_get_role(role::AbstractString, ini::Inifile) - source_profile, role_arn = aws_get_role_details(role, ini) - source_profile === :notfound && return nothing - if debug_level > 0 - println("Assuming \"$source_profile\"... ") - end +""" + _aws_get_role(role::AbstractString, ini::Inifile) -> Union{AWSCredentials, Nothing} + +Retrieve the `AWSCredentials` for a given role from Security Token Services (STS). + +# Arguments +- `role::AbstractString`: Name of the `role` +- `ini::Inifile`: Inifile to look into to find the `role` +""" +function _aws_get_role(role::AbstractString, ini::Inifile) + source_profile, role_arn = aws_get_role_details(role, ini) + source_profile === nothing && return nothing credentials = nothing for f in [dot_aws_credentials, dot_aws_config] @@ -450,7 +530,6 @@ function aws_get_role(role::AbstractString, ini::Inifile) end credentials === nothing && return nothing - config = AWSConfig(:creds=>credentials, :region=>aws_get_region(source_profile, ini)) role = Services.sts( @@ -459,14 +538,57 @@ function aws_get_role(role::AbstractString, ini::Inifile) RoleArn=role_arn, RoleSessionName=replace(role, r"[^\w+=,.@-]" => s"-"), ) + role_creds = role["Credentials"] - AWSCredentials(role_creds["AccessKeyId"], + return AWSCredentials( + role_creds["AccessKeyId"], role_creds["SecretAccessKey"], role_creds["SessionToken"]; - expiry = unix2datetime(role_creds["Expiration"])) + expiry=unix2datetime(role_creds["Expiration"]) + ) end -#==============================================================================# -# End of file. -#==============================================================================# + +""" + aws_get_role_details(profile::AbstractString, ini::Inifile) -> Tuple + +Return a tuple of `profile` details and the `role arn`. + +# Arguments +- `profile::AbstractString`: Specific profile to get role details about +- `ini::Inifile`: Inifile to look into to find the role details +""" +function aws_get_role_details(profile::AbstractString, ini::Inifile) + role_arn = _get_ini_value(ini, profile, "role_arn") + source_profile = _get_ini_value(ini, profile, "source_profile") + + return (source_profile, role_arn) +end + + +""" + _get_ini_value( + ini::Inifile, profile::AbstractString, key::AbstractString; + default_value=nothing + ) -> String + +Get the value for `key` in the `ini` file for a given `profile`. + +# Arguments +- `ini::Inifile`: Inifile to look for `key` in +- `profile::AbstractString`: Given profile to find the `key` for +- `key::AbstractString`: Name of the `key` to get + +# Keywords +- `default_value`: If the `key` is not found, default to this value +""" +function _get_ini_value( + ini::Inifile, profile::AbstractString, key::AbstractString; + default_value=nothing +) + value = get(ini, profile, key, default_value) + value = get(ini, "profile $profile", key, value) + + return value +end diff --git a/src/AWSException.jl b/src/AWSException.jl index 0a249ca..88723e6 100644 --- a/src/AWSException.jl +++ b/src/AWSException.jl @@ -22,7 +22,6 @@ end function AWSException(e::HTTP.StatusError) - code = string(http_status(e)) message = "AWSException" info = Dict() diff --git a/src/deprecations.jl b/src/deprecations.jl new file mode 100644 index 0000000..4380819 --- /dev/null +++ b/src/deprecations.jl @@ -0,0 +1,2 @@ +# https://github.com/JuliaCloud/AWSCore.jl/commit/b719614b812fbec385833e607a2f8772d6009b59 +Base.@deprecate copy!(dest::AWSCredentials, src::AWSCredentials) copyto!(dest, src) \ No newline at end of file diff --git a/src/signaturev4.jl b/src/signaturev4.jl new file mode 100644 index 0000000..b474620 --- /dev/null +++ b/src/signaturev4.jl @@ -0,0 +1,205 @@ +module SignatureV4 + +using AWSCore +using Base64 +using Dates +using HTTP +using HTTP.Pairs +using HTTP.URIs +using HTTP: Headers, Layer, Request +using IniFile +using MbedTLS + +export AWS4AuthLayer, sign_signature! + +""" + AWS4AuthLayer{Next} <: HTTP.Layer + +Abstract type used by [`HTTP.request`](@ref) to add an +[AWS Signature v4](http://docs.aws.amazon.com/general/latest/gr/signature-version-4.html) +authentication layer to the request. + +Historically this layer has been placed in-between the `MessageLayer` and `RetryLayer` in the HTTP +stack. The request stack can be found +[here](https://github.com/JuliaWeb/HTTP.jl/blob/v0.8.6/src/HTTP.jl#L534). + +An example of how to layer insert can be found below: +```julia +using AWSCore +using HTTP + +stack = HTTP.stack() + +# This will insert the AWS4AuthLayer before the RetryLayer +insert(stack, RetryLayer, AWS4AuthLayer) +``` +""" +abstract type AWS4AuthLayer{Next<:Layer} <: Layer{Next} end + +""" + HTTP.request(::Type{AWS4AuthLayer}, url::HTTP.URI, req::HTTP.Request, body) -> HTTP.Response + +Perform the given request, adding a layer of AWS authentication using +[AWS Signature v4](http://docs.aws.amazon.com/general/latest/gr/signature-version-4.html). +An "Authorization" header to the request. +""" +function HTTP.request(::Type{AWS4AuthLayer{Next}}, url::URI, req::Request, body; kw...) where Next + if !haskey(kw, :aws_access_key_id) + creds = AWSCredentials() + creds = (aws_access_key_id=creds.access_key_id, aws_secret_access_key=creds.secret_key) + kw = merge(creds, kw) + end + sign_signature!(req.method, url, req.headers, req.body; kw...) + return HTTP.request(Next, url, req, body; kw...) +end + +# Normalize whitespace to the form required in the canonical headers. +# Note that the expected format for multiline headers seems not to be explicitly +# documented, but Amazon provides a test case for it, so we'll match that behavior. +# We replace each `\n` with a `,` and remove all whitespace around the newlines, +# then any remaining contiguous whitespace is replaced with a single space. +function _normalize_ws(s::AbstractString) + if any(isequal('\n'), s) + return join(map(_normalize_ws, split(s, '\n')), ',') + end + + return replace(strip(s), r"\s+" => " ") +end + +""" + sign_signature!(method::String, url::HTTP.URI, headers::HTTP.Headers, body; kwargs...) + +Add an "Authorization" header to `headers`, modifying it in place. +The header contains a computed signature based on the given credentials as well as +metadata about what was used to compute the signature. +For more information, see the AWS documentation on the +[Signature v4](http://docs.aws.amazon.com/general/latest/gr/signature-version-4.html) +process. + +# Keyword arguments + +All keyword arguments to this function are optional, as they have default values. + +* `body_sha256`: A precomputed SHA-256 sum of `body` +* `body_md5`: A precomputed MD5 sum of `body` +* `timestamp`: The timestamp used in request signing (defaults to now in UTC) +* `aws_service`: The AWS service for the request (determined from the URL) +* `aws_region`: The AWS region for the request (determined from the URL) +* `aws_access_key_id`: AWS access key (read from the environment) +* `aws_secret_access_key`: AWS secret access key (read from the environment) +* `aws_session_token`: AWS session token (read from the environment, or empty) +* `token_in_signature`: Use `aws_session_token` when computing the signature (`true`) +* `include_md5`: Add the "Content-MD5" header to `headers` (`true`) +* `include_sha256`: Add the "x-amz-content-sha256" header to `headers` (`true`) +""" +function sign_signature!( + method::String, + url::URI, + headers::Headers, + body::Vector{UInt8}; + body_sha256::Vector{UInt8}=digest(MD_SHA256, body), + body_md5::Vector{UInt8}=digest(MD_MD5, body), + t::Union{DateTime,Nothing}=nothing, + timestamp::DateTime=now(Dates.UTC), + aws_service::String=String(split(url.host, ".")[1]), + aws_region::String=String(split(url.host, ".")[2]), + aws_access_key_id::String=ENV["AWS_ACCESS_KEY_ID"], + aws_secret_access_key::String=ENV["AWS_SECRET_ACCESS_KEY"], + aws_session_token::String=get(ENV, "AWS_SESSION_TOKEN", ""), + token_in_signature=true, + include_md5=true, + include_sha256=true, + kw..., +) + # ISO8601 date/time strings for time of request... + date = Dates.format(timestamp, dateformat"yyyymmdd") + datetime = Dates.format(timestamp, dateformat"yyyymmdd\THHMMSS\Z") + + # Authentication scope... + scope = [date, aws_region, aws_service, "aws4_request"] + + # Signing key generated from today's scope string... + signing_key = string("AWS4", aws_secret_access_key) + for element in scope + signing_key = digest(MD_SHA256, element, signing_key) + end + + # Authentication scope string... + scope = join(scope, "/") + + # SHA256 hash of content... + content_hash = bytes2hex(body_sha256) + + # HTTP headers... + rmkv(headers, "Authorization") + setkv(headers, "host", url.host) + setkv(headers, "x-amz-date", datetime) + include_md5 && setkv(headers, "Content-MD5", base64encode(body_md5)) + if (aws_service == "s3" && method == "PUT") || include_sha256 + # This header is required for S3 PUT requests. See the documentation at + # https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + setkv(headers, "x-amz-content-sha256", content_hash) + end + if aws_session_token != "" + setkv(headers, "x-amz-security-token", aws_session_token) + end + + # Sort and lowercase() Headers to produce canonical form... + unique_header_keys = Vector{String}() + normalized_headers = Dict{String,Vector{String}}() + for (k, v) in sort!([lowercase(k) => v for (k, v) in headers], by=first) + # Some services want the token included as part of the signature + if k == "x-amz-security-token" && !token_in_signature + continue + end + # In Amazon's examples, they exclude Content-Length from signing. This does not + # appear to be addressed in the documentation, so we'll just mimic the example. + if k == "content-length" + continue + end + if !haskey(normalized_headers, k) + normalized_headers[k] = Vector{String}() + push!(unique_header_keys, k) + end + push!(normalized_headers[k], _normalize_ws(v)) + end + canonical_headers = map(unique_header_keys) do k + string(k, ':', join(normalized_headers[k], ',')) + end + signed_headers = join(unique_header_keys, ';') + + # Sort Query String... + query = sort!(collect(queryparams(url.query)), by=first) + + # Paths for requests to S3 should be escaped but not normalized. See + # http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html#canonical-request + # Note that escapepath escapes ~ per RFC 1738, but Amazon includes an example in their + # signature v4 test suite where ~ remains unescaped. We follow the spec here and thus + # deviate from Amazon's example in this case. + path = escapepath(aws_service == "s3" ? url.path : URIs.normpath(url.path)) + + # Create hash of canonical request... + canonical_form = join([method, + path, + escapeuri(query), + join(canonical_headers, "\n"), + "", + signed_headers, + content_hash], "\n") + canonical_hash = bytes2hex(digest(MD_SHA256, canonical_form)) + + # Create and sign "String to Sign"... + string_to_sign = "AWS4-HMAC-SHA256\n$datetime\n$scope\n$canonical_hash" + signature = bytes2hex(digest(MD_SHA256, string_to_sign, signing_key)) + + # Append Authorization header... + setkv(headers, "Authorization", string( + "AWS4-HMAC-SHA256 ", + "Credential=$aws_access_key_id/$scope, ", + "SignedHeaders=$signed_headers, ", + "Signature=$signature" + )) + + return headers +end +end # module AWS4AuthRequest diff --git a/test/arn.jl b/test/arn.jl new file mode 100644 index 0000000..dd97936 --- /dev/null +++ b/test/arn.jl @@ -0,0 +1,8 @@ +@testset "ARN" begin + @test arn(aws,"s3","foo/bar") == "arn:aws:s3:::foo/bar" + @test arn(aws,"s3","foo") == "arn:aws:s3:::foo" + @test arn(aws,"sqs", "au-test-queue", "ap-southeast-2", "1234") == + "arn:aws:sqs:ap-southeast-2:1234:au-test-queue" + @test arn(aws,"sns","*","*",1234) == "arn:aws:sns:*:1234:*" + @test arn(aws,"iam","role/foo-role", "", 1234) == "arn:aws:iam::1234:role/foo-role" +end diff --git a/test/aws4.jl b/test/aws4.jl new file mode 100644 index 0000000..102e827 --- /dev/null +++ b/test/aws4.jl @@ -0,0 +1,201 @@ +# Based on https://docs.aws.amazon.com/general/latest/gr/signature-v4-test-suite.html +# and https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + +header_keys(headers) = sort!(map(first, headers)) +const required_headers = ["Authorization", "host", "x-amz-date"] + +function test_sign!(method, headers, params, body=""; opts...) + SignatureV4.sign_signature!(method, + URI("https://example.amazonaws.com/" * params), + headers, + Vector{UInt8}(body); + timestamp=DateTime(2015, 8, 30, 12, 36), + aws_service="service", + aws_region="us-east-1", + # NOTE: These are the example credentials as specified in the AWS docs, + # they are not real + aws_access_key_id="AKIDEXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + include_md5=false, + include_sha256=false, + opts...) + return headers +end + +function test_auth_string(headers, sig, key="AKIDEXAMPLE", date="20150830", service="service") + d = [ + "AWS4-HMAC-SHA256 Credential" => "$key/$date/us-east-1/$service/aws4_request", + "SignedHeaders" => headers, + "Signature" => sig, + ] + join(map(p->join(p, '='), d), ", ") +end + +@testset "AWS Signature Version 4" begin + # The signature for requests with no headers where the path ends up as simply / + slash_only_sig = "5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31" + noheaders = [ + ("get-vanilla", "", slash_only_sig), + ("get-vanilla-empty-query-key", "?Param1=value1", "a67d582fa61cc504c4bae71f336f98b97f1ea3c7a6bfe1b6e45aec72011b9aeb"), + ("get-utf8", "ሴ", "8318018e0b0f223aa2bbf98705b62bb787dc9c0e678f255a891fd03141be5d85"), + ("get-relative", "example/..", slash_only_sig), + ("get-relative-relative", "example1/example2/../..", slash_only_sig), + ("get-slash", "/", slash_only_sig), + ("get-slash-dot-slash", "./", slash_only_sig), + ("get-slashes", "example/", "9a624bd73a37c9a373b5312afbebe7a714a789de108f0bdfe846570885f57e84"), + ("get-slash-pointless-dot", "./example", "ef75d96142cf21edca26f06005da7988e4f8dc83a165a80865db7089db637ec5"), + ("get-space", "example space/", "652487583200325589f1fba4c7e578f72c47cb61beeca81406b39ddec1366741"), + ("post-vanilla", "", "5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b"), + ("post-vanilla-empty-query-value", "?Param1=value1", "28038455d6de14eafc1f9222cf5aa6f1a96197d7deb8263271d420d138af7f11"), + ] + + @testset "$name" for (name, p, sig) in noheaders + m = startswith(name, "get") ? "GET" : "POST" + headers = test_sign!(m, Headers([]), p) + @test header_keys(headers) == required_headers + d = Dict(headers) + @test d["x-amz-date"] == "20150830T123600Z" + @test d["host"] == "example.amazonaws.com" + @test d["Authorization"] == test_auth_string("host;x-amz-date", sig) + end + + yesheaders = [ + ("get-header-key-duplicate", "", "", + Headers(["My-Header1" => "value2", + "My-Header1" => "value2", + "My-Header1" => "value1"]), + "host;my-header1;x-amz-date", + "c9d5ea9f3f72853aea855b47ea873832890dbdd183b4468f858259531a5138ea"), + ("get-header-value-multiline", "", "", + Headers(["My-Header1" => "value1\n value2\n value3"]), + "host;my-header1;x-amz-date", + "ba17b383a53190154eb5fa66a1b836cc297cc0a3d70a5d00705980573d8ff790"), + ("get-header-value-order", "", "", + Headers(["My-Header1" => "value4", + "My-Header1" => "value1", + "My-Header1" => "value3", + "My-Header1" => "value2"]), + "host;my-header1;x-amz-date", + "08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01"), + ("get-header-value-trim", "", "", + Headers(["My-Header1" => " value1", + "My-Header2" => " \"a b c\""]), + "host;my-header1;my-header2;x-amz-date", + "acc3ed3afb60bb290fc8d2dd0098b9911fcaa05412b367055dee359757a9c736"), + ("post-header-key-sort", "", "", + Headers(["My-Header1" => "value1"]), + "host;my-header1;x-amz-date", + "c5410059b04c1ee005303aed430f6e6645f61f4dc9e1461ec8f8916fdf18852c"), + ("post-header-value-case", "", "", + Headers(["My-Header1" => "VALUE1"]), + "host;my-header1;x-amz-date", + "cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d"), + ("post-x-www-form-urlencoded", "", "Param1=value1", + Headers(["Content-Type" => "application/x-www-form-urlencoded", + "Content-Length" => "13"]), + "content-type;host;x-amz-date", + "ff11897932ad3f4e8b18135d722051e5ac45fc38421b1da7b9d196a0fe09473a"), + ("post-x-www-form-urlencoded-parameters", "", "Param1=value1", + Headers(["Content-Type" => "application/x-www-form-urlencoded; charset=utf8", + "Content-Length" => "13"]), + "content-type;host;x-amz-date", + "1a72ec8f64bd914b0e42e42607c7fbce7fb2c7465f63e3092b3b0d39fa77a6fe"), + ] + + @testset "$name" for (name, p, body, h, sh, sig) in yesheaders + hh = sort(map(first, h)) + m = startswith(name, "get") ? "GET" : "POST" + test_sign!(m, h, p, body) + @test header_keys(h) == sort(vcat(required_headers, hh)) + d = Dict(h) # collapses duplicates but we don't care here + @test d["x-amz-date"] == "20150830T123600Z" + @test d["host"] == "example.amazonaws.com" + @test d["Authorization"] == test_auth_string(sh, sig) + end + + @testset "AWS Security Token Service" begin + # Not a real security token, provided by AWS as an example + token = string("AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwd", + "QWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/k", + "McGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXD", + "vp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64", + "lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2I", + "CCR/oLxBA==") + + @testset "Token included in signature" begin + sh = "host;x-amz-date;x-amz-security-token" + sig = "85d96828115b5dc0cfc3bd16ad9e210dd772bbebba041836c64533a82be05ead" + h = test_sign!("POST", Headers([]), "", aws_session_token=token) + d = Dict(h) + @test d["Authorization"] == test_auth_string(sh, sig) + @test haskey(d, "x-amz-security-token") + end + + @testset "Token not included in signature" begin + sh = "host;x-amz-date" + sig = "5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b" + h = test_sign!("POST", Headers([]), "", aws_session_token=token, token_in_signature=false) + d = Dict(h) + @test d["Authorization"] == test_auth_string(sh, sig) + @test haskey(d, "x-amz-security-token") + end + end + + @testset "AWS Simple Storage Service" begin + s3url = "https://examplebucket.s3.amazonaws.com" + opts = (timestamp=DateTime(2013, 5, 24), + aws_service="s3", + aws_region="us-east-1", + # NOTE: These are the example credentials as specified in the AWS docs, + # they are not real + aws_access_key_id="AKIAIOSFODNN7EXAMPLE", + aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + include_md5=false) + + @testset "GET Object" begin + sh = "host;range;x-amz-content-sha256;x-amz-date" + sig = "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41" + h = Headers(["Range" => "bytes=0-9"]) + SignatureV4.sign_signature!("GET", URI(s3url * "/test.txt"), h, UInt8[]; opts...) + d = Dict(h) + @test d["Authorization"] == test_auth_string(sh, sig, opts.aws_access_key_id, "20130524", "s3") + @test haskey(d, "x-amz-content-sha256") # required for S3 requests + end + + @testset "PUT Object" begin + sh = "date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class" + sig = "98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd" + h = Headers(["Date" => "Fri, 24 May 2013 00:00:00 GMT", + "x-amz-storage-class" => "REDUCED_REDUNDANCY"]) + SignatureV4.sign_signature!("PUT", URI(s3url * "/test\$file.text"), h, UInt8[]; + # Override the SHA-256 of the request body, since the actual body is not provided + # for this example in the documentation, only the SHA + body_sha256=hex2bytes("44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072"), + opts...) + d = Dict(h) + @test d["Authorization"] == test_auth_string(sh, sig, opts.aws_access_key_id, "20130524", "s3") + @test haskey(d, "x-amz-content-sha256") + end + + @testset "GET Bucket Lifecycle" begin + sh = "host;x-amz-content-sha256;x-amz-date" + sig = "fea454ca298b7da1c68078a5d1bdbfbbe0d65c699e0f91ac7a200a0136783543" + h = Headers([]) + SignatureV4.sign_signature!("GET", URI(s3url * "/?lifecycle"), h, UInt8[]; opts...) + d = Dict(h) + @test d["Authorization"] == test_auth_string(sh, sig, opts.aws_access_key_id, "20130524", "s3") + @test haskey(d, "x-amz-content-sha256") + end + + @testset "GET Bucket (List Objects)" begin + sh = "host;x-amz-content-sha256;x-amz-date" + sig = "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7" + h = Headers([]) + SignatureV4.sign_signature!("GET", URI(s3url * "/?max-keys=2&prefix=J"), h, UInt8[]; opts...) + d = Dict(h) + @test d["Authorization"] == test_auth_string(sh, sig, opts.aws_access_key_id, "20130524", "s3") + @test haskey(d, "x-amz-content-sha256") + end + end +end + diff --git a/test/credentials.jl b/test/credentials.jl new file mode 100644 index 0000000..d4b6efd --- /dev/null +++ b/test/credentials.jl @@ -0,0 +1,526 @@ +macro test_ecode(error_codes, expr) + quote + try + $expr + @test false + catch e + if e isa AWSException + ecode(e) in [$error_codes;] + end + end + end +end + +@testset "Load Credentials" begin + user = aws_user_arn(aws) + @test occursin(r"^arn:aws:(iam|sts)::[0-9]+:[^:]+$", user) + aws[:region] = "us-east-1" + + @test_ecode( + "InvalidAction", + AWSCore.Services.iam("GetFoo", Dict("ContentType" => "JSON")) + ) + + @test_ecode( + ["AccessDenied", "NoSuchEntity"], + AWSCore.Services.iam("GetUser", Dict("UserName" => "notauser", "ContentType" => "JSON")) + ) + + @test_ecode( + "ValidationError", + AWSCore.Services.iam("GetUser", Dict("UserName" => "@#!%%!", "ContentType" => "JSON")) + ) + + @test_ecode( + ["AccessDenied", "EntityAlreadyExists"], + AWSCore.Services.iam("CreateUser", Dict("UserName" => "root", "ContentType" => "JSON")) + ) +end + +@testset "NoAuth" begin + pub_request1 = Dict{Symbol, Any}( + :service => "s3", + :headers => Dict{String, String}("Range" => "bytes=0-0"), + :content => "", + :resource => "/invenia-static-website-content/invenia_ca/index.html", + :url => "https://s3.us-east-1.amazonaws.com/invenia-static-website-content/invenia_ca/index.html", + :verb => "GET", + :region => "us-east-1", + :creds => nothing, + ) + pub_request2 = Dict{Symbol, Any}( + :service => "s3", + :headers => Dict{String, String}("Range" => "bytes=0-0"), + :content => "", + :resource => "ryft-public-sample-data/AWS-x86-AMI-queries.json", + :url => "https://s3.amazonaws.com/ryft-public-sample-data/AWS-x86-AMI-queries.json", + :verb => "GET", + :region => "us-east-1", + :creds => nothing, + ) + + try + response = AWSCore.do_request(pub_request1) + @test response == "<" + catch e + @test_ecode( + ["AccessDenied", "NoSuchEntity"], + AWSCore.do_request(pub_request1) + ) + end + + try + response = AWSCore.do_request(pub_request2) + @test response == UInt8['['] + catch e + @test_ecode( + ["AccessDenied", "NoSuchEntity"], + AWSCore.do_request(pub_request2) + ) + end +end + +@testset "AWSCredentials" begin + @testset "Defaults" begin + creds = AWSCredentials("access_key_id" ,"secret_key") + @test creds.token == "" + @test creds.user_arn == "" + @test creds.account_number == "" + @test creds.expiry == typemax(DateTime) + @test creds.renew == nothing + end + + @testset "Renewal" begin + # Credentials shouldn't throw an error if no renew function is supplied + creds = AWSCredentials("access_key_id", "secret_key", renew=nothing) + newcreds = check_credentials(creds, force_refresh = true) + + # Creds should remain unchanged if no renew function exists + @test creds === newcreds + @test creds.access_key_id == "access_key_id" + @test creds.secret_key == "secret_key" + @test creds.renew == nothing + + # Creds should error if the renew function returns nothing + creds = AWSCredentials("access_key_id", "secret_key", renew = () -> nothing) + @test_throws ErrorException check_credentials(creds, force_refresh=true) + + # Creds should remain unchanged + @test creds.access_key_id == "access_key_id" + @test creds.secret_key == "secret_key" + + # Creds should take on value of a returned AWSCredentials except renew function + function gen_credentials() + i = 0 + () -> (i += 1; AWSCredentials("NEW_ID_$i", "NEW_KEY_$i")) + end + + creds = AWSCredentials( + "access_key_id", + "secret_key", + renew=gen_credentials(), + expiry=now(UTC), + ) + + @test creds.renew !== nothing + renewed = creds.renew() + + @test creds.access_key_id == "access_key_id" + @test creds.secret_key == "secret_key" + @test creds.expiry <= now(UTC) + @test AWSCore._will_expire(creds) + + @test renewed.access_key_id === "NEW_ID_1" + @test renewed.secret_key == "NEW_KEY_1" + @test renewed.renew === nothing + @test renewed.expiry == typemax(DateTime) + @test !AWSCore._will_expire(renewed) + renew = creds.renew + + # Check renewal on time out + newcreds = check_credentials(creds, force_refresh=false) + @test creds === newcreds + @test creds.access_key_id == "NEW_ID_2" + @test creds.secret_key == "NEW_KEY_2" + @test creds.renew !== nothing + @test creds.renew === renew + @test creds.expiry == typemax(DateTime) + @test !AWSCore._will_expire(creds) + + # Check renewal doesn't happen if not forced or timed out + newcreds = check_credentials(creds, force_refresh=false) + @test creds === newcreds + @test creds.access_key_id == "NEW_ID_2" + @test creds.secret_key == "NEW_KEY_2" + @test creds.renew !== nothing + @test creds.renew === renew + @test creds.expiry == typemax(DateTime) + + # Check forced renewal works + newcreds = check_credentials(creds, force_refresh=true) + @test creds === newcreds + @test creds.access_key_id == "NEW_ID_3" + @test creds.secret_key == "NEW_KEY_3" + @test creds.renew !== nothing + @test creds.renew === renew + @test creds.expiry == typemax(DateTime) + end + + mktempdir() do dir + config_file = joinpath(dir, "config") + creds_file = joinpath(dir, "creds") + write( + config_file, + """ + [profile test] + output = json + region = us-east-1 + + [profile test:dev] + source_profile = test + role_arn = arn:aws:iam::123456789000:role/Dev + + [profile test:sub-dev] + source_profile = test:dev + role_arn = arn:aws:iam::123456789000:role/SubDev + + [profile test2] + aws_access_key_id = WRONG_ACCESS_ID + aws_secret_access_key = WRONG_ACCESS_KEY + output = json + region = us-east-1 + + [profile test3] + source_profile = test:dev + role_arn = arn:aws:iam::123456789000:role/test3 + + [profile test4] + aws_access_key_id = RIGHT_ACCESS_ID4 + aws_secret_access_key = RIGHT_ACCESS_KEY4 + source_profile = test:dev + role_arn = arn:aws:iam::123456789000:role/test3 + """ + ) + + write( + creds_file, + """ + [test] + aws_access_key_id = TEST_ACCESS_ID + aws_secret_access_key = TEST_ACCESS_KEY + + [test2] + aws_access_key_id = RIGHT_ACCESS_ID2 + aws_secret_access_key = RIGHT_ACCESS_KEY2 + + [test3] + aws_access_key_id = RIGHT_ACCESS_ID3 + aws_secret_access_key = RIGHT_ACCESS_KEY3 + """ + ) + + withenv( + "AWS_SHARED_CREDENTIALS_FILE" => creds_file, + "AWS_CONFIG_FILE" => config_file, + "AWS_DEFAULT_PROFILE" => "test", + "AWS_ACCESS_KEY_ID" => nothing + ) do + + @testset "Loading" begin + # Check credentials load + config = aws_config() + creds = config[:creds] + @test creds isa AWSCredentials + + @test creds.access_key_id == "TEST_ACCESS_ID" + @test creds.secret_key == "TEST_ACCESS_KEY" + @test creds.renew !== nothing + + # Check credential file takes precedence over config + ENV["AWS_DEFAULT_PROFILE"] = "test2" + config = aws_config() + creds = config[:creds] + + @test creds.access_key_id == "RIGHT_ACCESS_ID2" + @test creds.secret_key == "RIGHT_ACCESS_KEY2" + + # Check credentials take precedence over role + ENV["AWS_DEFAULT_PROFILE"] = "test3" + config = aws_config() + creds = config[:creds] + + @test creds.access_key_id == "RIGHT_ACCESS_ID3" + @test creds.secret_key == "RIGHT_ACCESS_KEY3" + + ENV["AWS_DEFAULT_PROFILE"] = "test4" + config = aws_config() + creds = config[:creds] + + @test creds.access_key_id == "RIGHT_ACCESS_ID4" + @test creds.secret_key == "RIGHT_ACCESS_KEY4" + + end + + @testset "Refresh" begin + ENV["AWS_DEFAULT_PROFILE"] = "test" + # Check credentials refresh on timeout + config = aws_config() + creds = config[:creds] + creds.access_key_id = "EXPIRED_ACCESS_ID" + creds.secret_key = "EXPIRED_ACCESS_KEY" + creds.expiry = now(UTC) + + @test creds.renew !== nothing + renew = creds.renew + @test renew() isa AWSCredentials + + creds = check_credentials(config[:creds]) + + @test creds.access_key_id == "TEST_ACCESS_ID" + @test creds.secret_key == "TEST_ACCESS_KEY" + @test creds.expiry > now(UTC) + + # Check renew function remains unchanged + @test creds.renew !== nothing + @test creds.renew === renew + + # Check force_refresh + creds.access_key_id = "WRONG_ACCESS_KEY" + creds = check_credentials(creds, force_refresh = true) + @test creds.access_key_id == "TEST_ACCESS_ID" + end + + @testset "Profile" begin + # Check profile kwarg + ENV["AWS_DEFAULT_PROFILE"] = "test" + creds = AWSCredentials(profile="test2") + @test creds.access_key_id == "RIGHT_ACCESS_ID2" + @test creds.secret_key == "RIGHT_ACCESS_KEY2" + + config = aws_config(profile="test2") + creds = config[:creds] + @test creds.access_key_id == "RIGHT_ACCESS_ID2" + @test creds.secret_key == "RIGHT_ACCESS_KEY2" + + # Check profile persists on renewal + creds.access_key_id = "WRONG_ACCESS_ID2" + creds.secret_key = "WRONG_ACCESS_KEY2" + creds = check_credentials(creds, force_refresh=true) + + @test creds.access_key_id == "RIGHT_ACCESS_ID2" + @test creds.secret_key == "RIGHT_ACCESS_KEY2" + end + + @testset "Assume Role" begin + # Check we try to assume a role + ENV["AWS_DEFAULT_PROFILE"] = "test:dev" + + @test_ecode( + "InvalidClientTokenId", + aws_config() + ) + + # Check we try to assume a role + ENV["AWS_DEFAULT_PROFILE"] = "test:sub-dev" + let oldout = stdout + r,w = redirect_stdout() + + @test_ecode( + "InvalidClientTokenId", + aws_config() + ) + redirect_stdout(oldout) + close(w) + output = String(read(r)) + occursin("Assuming \"test:dev\"", output) + occursin("Assuming \"test\"", output) + close(r) + end + end + end + end +end + +@testset "Retrieving AWS Credentials" begin + test_values = Dict{String, Any}( + "Default-Profile" => "default", + "Test-Profile" => "test", + "Test-Config-Profile" => "profile test", + + # Default profile values, needs to match due to AWSCredentials.jl:239 + "AccessKeyId" => "Default-Key", + "SecretAccessKey" => "Default-Secret", + + "Test-AccessKeyId" => "Test-Key", + "Test-SecretAccessKey" => "Test-Secret", + + "Token" => "Test-Token", + "InstanceProfileArn" => "Test-Arn", + "RoleArn" => "Test-Arn", + "Expiration" => now(), + + "URI" => "/Test-URI/", + "Security-Credentials" => "Test-Security-Credentials" + ) + + http_get_patch = @patch function http_get(url::String) + security_credentials = test_values["Security-Credentials"] + uri = test_values["URI"] + + if url == "http://169.254.169.254/latest/meta-data/iam/info" + instance_profile_arn = test_values["InstanceProfileArn"] + return HTTP.Response("{\"InstanceProfileArn\": \"$instance_profile_arn\"}") + elseif url == "http://169.254.169.254/latest/meta-data/iam/security-credentials/" + return HTTP.Response(test_values["Security-Credentials"]) + elseif url == "http://169.254.169.254/latest/meta-data/iam/security-credentials/$security_credentials" || url == "http://169.254.170.2$uri" + my_dict = JSON.json(test_values) + response = HTTP.Response(my_dict) + return response + else + return nothing + end + end + + @testset "~/.aws/config - Default Profile" begin + mktemp() do config_file, config_io + write( + config_io, + """ + [$(test_values["Default-Profile"])] + aws_access_key_id=$(test_values["AccessKeyId"]) + aws_secret_access_key=$(test_values["SecretAccessKey"]) + """ + ) + close(config_io) + + withenv("AWS_CONFIG_FILE" => config_file) do + default_profile = dot_aws_config() + + @test default_profile.access_key_id == test_values["AccessKeyId"] + @test default_profile.secret_key == test_values["SecretAccessKey"] + end + end + end + + @testset "~/.aws/config - Specified Profile" begin + mktemp() do config_file, config_io + write( + config_io, + """ + [$(test_values["Test-Config-Profile"])] + aws_access_key_id=$(test_values["Test-AccessKeyId"]) + aws_secret_access_key=$(test_values["Test-SecretAccessKey"]) + """ + ) + close(config_io) + + withenv("AWS_CONFIG_FILE" => config_file) do + specified_result = dot_aws_config(test_values["Test-Profile"]) + + @test specified_result.access_key_id == test_values["Test-AccessKeyId"] + @test specified_result.secret_key == test_values["Test-SecretAccessKey"] + end + end + end + + @testset "~/.aws/creds - Default Profile" begin + mktemp() do creds_file, creds_io + write( + creds_io, + """ + [$(test_values["Default-Profile"])] + aws_access_key_id=$(test_values["AccessKeyId"]) + aws_secret_access_key=$(test_values["SecretAccessKey"]) + """ + ) + close(creds_io) + + withenv("AWS_SHARED_CREDENTIALS_FILE" => creds_file) do + specified_result = dot_aws_credentials() + + @test specified_result.access_key_id == test_values["AccessKeyId"] + @test specified_result.secret_key == test_values["SecretAccessKey"] + end + end + end + + @testset "~/.aws/creds - Specified Profile" begin + mktemp() do creds_file, creds_io + write( + creds_io, + """ + [$(test_values["Test-Profile"])] + aws_access_key_id=$(test_values["Test-AccessKeyId"]) + aws_secret_access_key=$(test_values["Test-SecretAccessKey"]) + """ + ) + close(creds_io) + + withenv("AWS_SHARED_CREDENTIALS_FILE" => creds_file) do + specified_result = dot_aws_credentials(test_values["Test-Profile"]) + + @test specified_result.access_key_id == test_values["Test-AccessKeyId"] + @test specified_result.secret_key == test_values["Test-SecretAccessKey"] + end + end + end + + @testset "Environment Variables" begin + withenv( + "AWS_ACCESS_KEY_ID" => test_values["AccessKeyId"], + "AWS_SECRET_ACCESS_KEY" => test_values["SecretAccessKey"] + ) do + aws_creds = env_var_credentials() + @test aws_creds.access_key_id == test_values["AccessKeyId"] + @test aws_creds.secret_key == test_values["SecretAccessKey"] + end + end + + @testset "Instance - EC2" begin + apply([http_get_patch]) do + result = ec2_instance_credentials() + @test result.access_key_id == test_values["AccessKeyId"] + @test result.secret_key == test_values["SecretAccessKey"] + @test result.token == test_values["Token"] + @test result.user_arn == test_values["InstanceProfileArn"] + @test result.expiry == test_values["Expiration"] + @test result.renew == ec2_instance_credentials + end + end + + @testset "Instance - ECS" begin + withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => test_values["URI"]) do + apply([http_get_patch]) do + result = ecs_instance_credentials() + @test result.access_key_id == test_values["AccessKeyId"] + @test result.secret_key == test_values["SecretAccessKey"] + @test result.token == test_values["Token"] + @test result.user_arn == test_values["RoleArn"] + @test result.expiry == test_values["Expiration"] + @test result.renew == ecs_instance_credentials + end + end + end + + @testset "Helper functions" begin + @testset "Check Credentials - EnvVars" begin + withenv( + "AWS_ACCESS_KEY_ID" => test_values["AccessKeyId"], + "AWS_SECRET_ACCESS_KEY" => test_values["SecretAccessKey"] + ) do + testAWSCredentials = AWSCredentials( + test_values["AccessKeyId"], + test_values["SecretAccessKey"], + expiry=Dates.now() - Minute(10), + renew=env_var_credentials + ) + + result = check_credentials(testAWSCredentials, force_refresh=true) + @test result.access_key_id == testAWSCredentials.access_key_id + @test result.secret_key == testAWSCredentials.secret_key + @test result.expiry == typemax(DateTime) + @test result.renew == testAWSCredentials.renew + end + end + end +end \ No newline at end of file diff --git a/test/exceptions.jl b/test/exceptions.jl new file mode 100644 index 0000000..bc51897 --- /dev/null +++ b/test/exceptions.jl @@ -0,0 +1,20 @@ +@testset "AWSException" begin + code = "InvalidSignatureException" + message = "Signature expired: ..." + body = """ + { + "__type": "$code", + "message": "$message" + } + """ + headers = ["Content-Type" => "application/x-amz-json-1.1"] + status_code = 400 + + # This does not actually send a request, just creates the object to test with + req = HTTP.Request("GET", "https://amazon.ca", headers, body) + resp = HTTP.Response(status_code, headers; body=body, request=req) + ex = AWSException(HTTP.StatusError(status_code, resp)) + + @test ex.code == code + @test ex.message == message +end \ No newline at end of file diff --git a/test/jltest.aws.example b/test/jltest.aws.example deleted file mode 100644 index 47f0948..0000000 --- a/test/jltest.aws.example +++ /dev/null @@ -1,4 +0,0 @@ -arn:aws:iam::[account number here]:user/ocaws.jl.test -[Access Key here] -[Secred Key here] -ap-southeast-2 diff --git a/test/localhost.jl b/test/localhost.jl new file mode 100644 index 0000000..8afdb60 --- /dev/null +++ b/test/localhost.jl @@ -0,0 +1,20 @@ +@testset "Localhost" begin + wmic_command = "wmic path win32_computersystemproduct get uuid" + + windows_patch = @patch Sys.iswindows() = return true + + wmic_patch = @patch read(command::Cmd, ::Type{String}) = + return "UUID\nEC2D1284-E32E-FB5E-20E4-F43F6E01CA7A" + + @testset "EC2 - Windows" begin + apply([windows_patch, wmic_patch]) do + @test localhost_is_ec2() + end + end +end + +if Sys.iswindows() + @testset "Windows - Is EC2" begin + @test localhost_is_ec2() + end +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 97cf7dd..e00a8f6 100755 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,601 +4,28 @@ # Copyright OC Technology Pty Ltd 2014 - All rights reserved #==============================================================================# -using Test -using Dates using AWSCore -using SymDict +using Dates +using HTTP +using HTTP: Headers, URI +using IniFile +using JSON +using Mocking using Retry +using SymDict +using Test using XMLDict -using HTTP - -AWSCore.set_debug_level(1) - - -@testset "AWSCore" begin +using .SignatureV4 +Mocking.activate() aws = aws_config() -@testset "NoAuth" begin - pub_request1 = Dict{Symbol, Any}( - :service => "s3", - :headers => Dict{String, String}("Range" => "bytes=0-0"), - :content => "", - :resource => "/invenia-static-website-content/invenia_ca/index.html", - :url => "https://s3.us-east-1.amazonaws.com/invenia-static-website-content/invenia_ca/index.html", - :verb => "GET", - :region => "us-east-1", - :creds => nothing, - ) - pub_request2 = Dict{Symbol, Any}( - :service => "s3", - :headers => Dict{String, String}("Range" => "bytes=0-0"), - :content => "", - :resource => "ryft-public-sample-data/AWS-x86-AMI-queries.json", - :url => "https://s3.amazonaws.com/ryft-public-sample-data/AWS-x86-AMI-queries.json", - :verb => "GET", - :region => "us-east-1", - :creds => nothing, - ) - response = nothing - try - response = AWSCore.do_request(pub_request1) - catch e - println(e) - @test ecode(e) in ["AccessDenied", "NoSuchEntity"] - try - response = AWSCore.do_request(pub_request2) - catch e - println(e) - @test ecode(e) in ["AccessDenied", "NoSuchEntity"] - end - end - @test response == "<" || response == UInt8['['] -end -@testset "Load Credentials" begin - user = aws_user_arn(aws) - - @test occursin(r"^arn:aws:iam::[0-9]+:[^:]+$", user) - - println("Authenticated as: $user") - - aws[:region] = "us-east-1" - - println("Testing exceptions...") - try - AWSCore.Services.iam("GetFoo", Dict("ContentType" => "JSON")) - @test false - catch e - println(e) - @test ecode(e) == "InvalidAction" - end - - try - AWSCore.Services.iam("GetUser", Dict("UserName" => "notauser", - "ContentType" => "JSON")) - @test false - catch e - println(e) - @test ecode(e) in ["AccessDenied", "NoSuchEntity"] - end - - try - AWSCore.Services.iam("GetUser", Dict("UserName" => "@#!%%!", - "ContentType" => "JSON")) - @test false - catch e - println(e) - @test ecode(e) == "ValidationError" - end - - try - AWSCore.Services.iam("CreateUser", Dict("UserName" => "root", - "ContentType" => "JSON")) - @test false - catch e - println(e) - @test ecode(e) in ["AccessDenied", "EntityAlreadyExists"] - end -end - -@testset "AWSCredentials" begin - @testset "Defaults" begin - creds = AWSCredentials("access_key_id" ,"secret_key") - @test creds.token == "" - @test creds.user_arn == "" - @test creds.account_number == "" - @test creds.expiry == typemax(DateTime) - @test creds.renew == nothing - end - - @testset "Renewal" begin - # Credentials shouldn't throw an error if no renew function is supplied - creds = AWSCredentials("access_key_id", "secret_key", renew = nothing) - newcreds = check_credentials(creds, force_refresh = true) - # Creds should remain unchanged if no renew function exists - @test creds === newcreds - @test creds.access_key_id == "access_key_id" - @test creds.secret_key == "secret_key" - @test creds.renew == nothing - - # Creds should error if the renew function returns nothing - creds = AWSCredentials("access_key_id", "secret_key", renew = () -> nothing) - @test_throws ErrorException check_credentials(creds, force_refresh = true) - # Creds should remain unchanged - @test creds.access_key_id == "access_key_id" - @test creds.secret_key == "secret_key" - - # Creds should take on value of a returned AWSCredentials except renew function - function gen_credentials() - i = 0 - () -> (i += 1; AWSCredentials("NEW_ID_$i", "NEW_KEY_$i")) - end - - creds = AWSCredentials( - "access_key_id", - "secret_key", - renew = gen_credentials(), - expiry = now(UTC), - ) - - @test creds.renew !== nothing - renewed = creds.renew() - - @test creds.access_key_id == "access_key_id" - @test creds.secret_key == "secret_key" - @test creds.expiry <= now(UTC) - @test AWSCore.will_expire(creds) - - @test renewed.access_key_id === "NEW_ID_1" - @test renewed.secret_key == "NEW_KEY_1" - @test renewed.renew === nothing - @test renewed.expiry == typemax(DateTime) - @test !AWSCore.will_expire(renewed) - renew = creds.renew - - # Check renewal on time out - newcreds = check_credentials(creds, force_refresh = false) - @test creds === newcreds - @test creds.access_key_id == "NEW_ID_2" - @test creds.secret_key == "NEW_KEY_2" - @test creds.renew !== nothing - @test creds.renew === renew - @test creds.expiry == typemax(DateTime) - @test !AWSCore.will_expire(creds) - - # Check renewal doesn't happen if not forced or timed out - newcreds = check_credentials(creds, force_refresh = false) - @test creds === newcreds - @test creds.access_key_id == "NEW_ID_2" - @test creds.secret_key == "NEW_KEY_2" - @test creds.renew !== nothing - @test creds.renew === renew - @test creds.expiry == typemax(DateTime) - - # Check forced renewal works - newcreds = check_credentials(creds, force_refresh = true) - @test creds === newcreds - @test creds.access_key_id == "NEW_ID_3" - @test creds.secret_key == "NEW_KEY_3" - @test creds.renew !== nothing - @test creds.renew === renew - @test creds.expiry == typemax(DateTime) - end - - mktemp() do config_file, config_io - write(config_io, """[profile test] - output = json - region = us-east-1 - - [profile test:dev] - source_profile = test - role_arn = arn:aws:iam::123456789000:role/Dev - - [profile test:sub-dev] - source_profile = test:dev - role_arn = arn:aws:iam::123456789000:role/SubDev - - [profile test2] - aws_access_key_id = WRONG_ACCESS_ID - aws_secret_access_key = WRONG_ACCESS_KEY - output = json - region = us-east-1 - - [profile test3] - source_profile = test:dev - role_arn = arn:aws:iam::123456789000:role/test3 - - [profile test4] - source_profile = test:dev - role_arn = arn:aws:iam::123456789000:role/test3 - aws_access_key_id = RIGHT_ACCESS_ID4 - aws_secret_access_key = RIGHT_ACCESS_KEY4 - """) - close(config_io) - - mktemp() do creds_file, creds_io - write(creds_io, """[test] - aws_access_key_id = TEST_ACCESS_ID - aws_secret_access_key = TEST_ACCESS_KEY - - [test2] - aws_access_key_id = RIGHT_ACCESS_ID2 - aws_secret_access_key = RIGHT_ACCESS_KEY2 - - [test3] - aws_access_key_id = RIGHT_ACCESS_ID3 - aws_secret_access_key = RIGHT_ACCESS_KEY3 - """) - close(creds_io) - - withenv( - "AWS_SHARED_CREDENTIALS_FILE" => creds_file, - "AWS_CONFIG_FILE" => config_file, - "AWS_DEFAULT_PROFILE" => "test", - "AWS_ACCESS_KEY_ID" => nothing - ) do - - @testset "Loading" begin - # Check credentials load - config = aws_config() - creds = config[:creds] - @test creds isa AWSCredentials - - @test creds.access_key_id == "TEST_ACCESS_ID" - @test creds.secret_key == "TEST_ACCESS_KEY" - @test creds.renew !== nothing - - # Check credential file takes precedence over config - ENV["AWS_DEFAULT_PROFILE"] = "test2" - config = aws_config() - creds = config[:creds] - - @test creds.access_key_id == "RIGHT_ACCESS_ID2" - @test creds.secret_key == "RIGHT_ACCESS_KEY2" - - # Check credentials take precedence over role - ENV["AWS_DEFAULT_PROFILE"] = "test3" - config = aws_config() - creds = config[:creds] - - @test creds.access_key_id == "RIGHT_ACCESS_ID3" - @test creds.secret_key == "RIGHT_ACCESS_KEY3" - - ENV["AWS_DEFAULT_PROFILE"] = "test4" - config = aws_config() - creds = config[:creds] - - @test creds.access_key_id == "RIGHT_ACCESS_ID4" - @test creds.secret_key == "RIGHT_ACCESS_KEY4" - end - - @testset "Refresh" begin - ENV["AWS_DEFAULT_PROFILE"] = "test" - # Check credentials refresh on timeout - config = aws_config() - creds = config[:creds] - creds.access_key_id = "EXPIRED_ACCESS_ID" - creds.secret_key = "EXPIRED_ACCESS_KEY" - creds.expiry = now(UTC) - - @test creds.renew !== nothing - renew = creds.renew - @test renew() isa AWSCredentials - - creds = check_credentials(config[:creds]) - - @test creds.access_key_id == "TEST_ACCESS_ID" - @test creds.secret_key == "TEST_ACCESS_KEY" - @test creds.expiry > now(UTC) - - # Check renew function remains unchanged - @test creds.renew !== nothing - @test creds.renew === renew - - # Check force_refresh - creds.access_key_id = "WRONG_ACCESS_KEY" - creds = check_credentials(creds, force_refresh = true) - @test creds.access_key_id == "TEST_ACCESS_ID" - end - - @testset "Profile" begin - # Check profile kwarg - ENV["AWS_DEFAULT_PROFILE"] = "test" - creds = AWSCredentials(profile="test2") - @test creds.access_key_id == "RIGHT_ACCESS_ID2" - @test creds.secret_key == "RIGHT_ACCESS_KEY2" - - config = aws_config(profile="test2") - creds = config[:creds] - @test creds.access_key_id == "RIGHT_ACCESS_ID2" - @test creds.secret_key == "RIGHT_ACCESS_KEY2" - - # Check profile persists on renewal - creds.access_key_id = "WRONG_ACCESS_ID2" - creds.secret_key = "WRONG_ACCESS_KEY2" - creds = check_credentials(creds, force_refresh=true) - - @test creds.access_key_id == "RIGHT_ACCESS_ID2" - @test creds.secret_key == "RIGHT_ACCESS_KEY2" - end - - @testset "Assume Role" begin - # Check we try to assume a role - ENV["AWS_DEFAULT_PROFILE"] = "test:dev" - - try - aws_config() - @test false - catch e - @test e isa AWSException - @test ecode(e) == "InvalidClientTokenId" - end - - # Check we try to assume a role - ENV["AWS_DEFAULT_PROFILE"] = "test:sub-dev" - let oldout = stdout - r,w = redirect_stdout() - try - aws_config() - @test false - catch e - @test e isa AWSException - @test ecode(e) == "InvalidClientTokenId" - end - redirect_stdout(oldout) - close(w) - output = String(read(r)) - occursin("Assuming \"test:dev\"", output) - occursin("Assuming \"test\"", output) - close(r) - end - end - end - end - end -end - -@testset "XML Parsing" begin - XML(x)=parse_xml(x) - - xml = """ - - - - http://queue.amazonaws.com/123456789012/testQueue - - - - - 7a62c49f-347e-4fc4-9331-6e8e7a96aa73 - - - - """ - - @assert XML(xml)["CreateQueueResult"]["QueueUrl"] == - "http://queue.amazonaws.com/123456789012/testQueue" - - xml = """ - - - - 2015-12-23T22:45:36Z - arn:aws:iam::012541411202:root - 012541411202 - 2015-09-15T01:07:23Z - - - - 837446c9-abaf-11e5-9f63-65ae4344bd73 - - - """ - - @test XML(xml)["GetUserResult"]["User"]["Arn"] == "arn:aws:iam::012541411202:root" - - - xml = """ - - - - ReceiveMessageWaitTimeSeconds - 2 - - - VisibilityTimeout - 30 - - - ApproximateNumberOfMessages - 0 - - - ApproximateNumberOfMessagesNotVisible - 0 - - - CreatedTimestamp - 1286771522 - - - LastModifiedTimestamp - 1286771522 - - - QueueArn - arn:aws:sqs:us-east-1:123456789012:qfoo - - - MaximumMessageSize - 8192 - - - MessageRetentionPeriod - 345600 - - - - 1ea71be5-b5a2-4f9d-b85a-945d8d08cd0b - - - """ - - d = Dict(a["Name"] => a["Value"] for a in XML(xml)["GetQueueAttributesResult"]["Attribute"]) - - @test d["MessageRetentionPeriod"] == "345600" - @test d["CreatedTimestamp"] == "1286771522" - - - xml = """ - - - - bcaf1ffd86f461ca5fb16fd081034f - webfile - - - - quotes - 2006-02-03T16:45:09.000Z - - - samples - 2006-02-03T16:41:58.000Z - - - - """ - - @test map(b->b["Name"], XML(xml)["Buckets"]["Bucket"]) == ["quotes", "samples"] - - - xml = """ - - - Domain1 - Domain2 - TWV0ZXJpbmdUZXN0RG9tYWluMS0yMDA3MDYwMTE2NTY= - - - eb13162f-1b95-4511-8b12-489b86acfd28 - 0.0000219907 - - - """ - - @test XML(xml)["ListDomainsResult"]["DomainName"] == ["Domain1", "Domain2"] -end - -@testset "AWS Signature Version 4" begin - function aws4_request_headers_test() - - r = @SymDict( - creds = AWSCredentials( - "AKIDEXAMPLE", - "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" - ), - region = "us-east-1", - verb = "POST", - service = "iam", - url = "http://iam.amazonaws.com/", - content = "Action=ListUsers&Version=2010-05-08", - headers = Dict( - "Content-Type" => - "application/x-www-form-urlencoded; charset=utf-8", - "Host" => "iam.amazonaws.com" - ) - ) - - AWSCore.sign!(r, DateTime("2011-09-09T23:36:00")) - - h = r[:headers] - out = join(["$k: $(h[k])\n" for k in sort(collect(keys(h)))]) - - expected = ( - "Authorization: AWS4-HMAC-SHA256 " * - "Credential=AKIDEXAMPLE/20110909/us-east-1/iam/aws4_request, " * - "SignedHeaders=content-md5;content-type;host;" * - "x-amz-content-sha256;x-amz-date, " * - "Signature=1a6db936024345449ef4507f890c5161" * - "bbfa2ff2490866653bb8b58b7ba1554a\n" * - "Content-MD5: r2d9jRneykOuUqFWSFXKCg==\n" * - "Content-Type: application/x-www-form-urlencoded; " * - "charset=utf-8\n" * - "Host: iam.amazonaws.com\n" * - "x-amz-content-sha256: b6359072c78d70ebee1e81adcbab4f01" * - "bf2c23245fa365ef83fe8f1f955085e2\n" * - "x-amz-date: 20110909T233600Z\n") - - @test out == expected - end - - aws4_request_headers_test() -end - -@testset "ARN" begin - @test arn(aws,"s3","foo/bar") == "arn:aws:s3:::foo/bar" - @test arn(aws,"s3","foo") == "arn:aws:s3:::foo" - @test arn(aws,"sqs", "au-test-queue", "ap-southeast-2", "1234") == - "arn:aws:sqs:ap-southeast-2:1234:au-test-queue" - - @test arn(aws,"sns","*","*",1234) == "arn:aws:sns:*:1234:*" - @test arn(aws,"iam","role/foo-role", "", 1234) == - "arn:aws:iam::1234:role/foo-role" -end - -@testset "Misc" begin - @test HTTP.escapepath("invocations/function:f:PROD") == - "invocations/function%3Af%3APROD" -end - -@testset "Exception" begin - code = "InvalidSignatureException" - message = "Signature expired: ..." - body = """ - { - "__type": "$code", - "message": "$message" - } - """ - headers = ["Content-Type" => "application/x-amz-json-1.1"] - status_code = 400 - - # This does not actually send a request, just creates the object to test with - req = HTTP.Request("GET", "https://amazon.ca", headers, body) - resp = HTTP.Response(status_code, headers; body=body, request=req) - ex = AWSException(HTTP.StatusError(status_code, resp)) - - @test ex.code == code - @test ex.message == message -end - -instance_type = get(ENV, "AWSCORE_INSTANCE_TYPE", "") -if instance_type == "EC2" - @testset "EC2" begin - @test_nowarn AWSCore.ec2_metadata("instance-id") - @test startswith(AWSCore.ec2_metadata("instance-id"), "i-") - - @test AWSCore.localhost_maybe_ec2() - @test AWSCore.localhost_is_ec2() - @test_nowarn AWSCore.ec2_instance_credentials() - ec2_creds = AWSCore.ec2_instance_credentials() - @test ec2_creds !== nothing - - default_creds = AWSCredentials() - @test default_creds.access_key_id == ec2_creds.access_key_id - @test default_creds.secret_key == ec2_creds.secret_key - end -elseif instance_type == "ECS" - @testset "ECS" begin - @test_nowarn AWSCore.ecs_instance_credentials() - ecs_creds = AWSCore.ecs_instance_credentials() - @test ecs_creds !== nothing - - default_creds = AWSCredentials() - @test default_creds.access_key_id == ecs_creds.access_key_id - @test default_creds.secret_key == ecs_creds.secret_key - end +@testset "AWSCore" begin + include("aws4.jl") + include("arn.jl") + include("credentials.jl") + include("exceptions.jl") + include("localhost.jl") + include("signaturev4.jl") + include("xml.jl") end - -end # testset "AWSCore" diff --git a/test/signaturev4.jl b/test/signaturev4.jl new file mode 100644 index 0000000..1ef33f1 --- /dev/null +++ b/test/signaturev4.jl @@ -0,0 +1,129 @@ +# Julia 1.0: Type definition not allowed inside a local scope, therefore we must define our +# TestLayer outside of the @testset +abstract type TestLayer{Next <: HTTP.Layer} <: HTTP.Layer{Next} end + +@testset "AWS Signature Version 4" begin + r = @SymDict( + creds = AWSCredentials("AKIDEXAMPLE","wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"), + region = "us-east-1", + verb = "POST", + service = "iam", + url = "http://iam.amazonaws.com/", + content = "Action=ListUsers&Version=2010-05-08", + headers = Dict( + "Content-Type" => "application/x-www-form-urlencoded; charset=utf-8", + "Host" => "iam.amazonaws.com" + ) + ) + + AWSCore.sign!(r, DateTime("2011-09-09T23:36:00")) + + h = r[:headers] + out = join(["$k: $(h[k])\n" for k in sort(collect(keys(h)))]) + + expected = ( + "Authorization: AWS4-HMAC-SHA256 " * + "Credential=AKIDEXAMPLE/20110909/us-east-1/iam/aws4_request, " * + "SignedHeaders=content-md5;content-type;host;" * + "x-amz-content-sha256;x-amz-date, " * + "Signature=1a6db936024345449ef4507f890c5161bbfa2ff2490866653bb8b58b7ba1554a\n" * + "Content-MD5: r2d9jRneykOuUqFWSFXKCg==\n" * + "Content-Type: application/x-www-form-urlencoded; charset=utf-8\n" * + "Host: iam.amazonaws.com\n" * + "x-amz-content-sha256: b6359072c78d70ebee1e81adcbab4f01bf2c23245fa365ef83fe8f1f955085e2\n" * + "x-amz-date: 20110909T233600Z\n" + ) + + @test out == expected +end + +@testset "HTTP Request - AWS4AuthLayer" begin + test_access_key = "TEST_ACCESS_KEY" + test_secret_key = "TEST_SECRET_KEY" + + function HTTP.request(::Type{TestLayer{Next}}, io::IO, req, body; kw...) where Next + @test kw[:aws_access_key_id] == test_access_key + @test kw[:aws_secret_access_key] == test_secret_key + return HTTP.request(Next, io, req, body; kw...) + end + + function _create_stack() + custom_stack = insert(stack(), StreamLayer, TestLayer) + custom_stack = insert(custom_stack, RetryLayer, SignatureV4.AWS4AuthLayer) + result = HTTP.request(custom_stack, "GET", "http://httpbin.org/ip") + @test result.status == 200 + end + + @testset "Environment Variables" begin + withenv( + "AWS_ACCESS_KEY_ID" => test_access_key, + "AWS_SECRET_ACCESS_KEY" => test_secret_key, + ) do + _create_stack() + end + end + + @testset "Credentials File - Default" begin + mktemp() do creds_file, creds_io + write(creds_io, """ + [default] + aws_access_key_id=$(test_access_key) + aws_secret_access_key=$(test_secret_key) + """) + close(creds_io) + + withenv( + "AWS_ACCESS_KEY_ID" => nothing, + "AWS_SECRET_ACCESS_KEY" => nothing, + "AWS_SHARED_CREDENTIALS_FILE" => creds_file, + ) do + _create_stack() + end + end + end + + @testset "Credentials File - Specified Profile" begin + aws_profile = "test" + + mktemp() do creds_file, creds_io + write(creds_io, """ + [$(aws_profile)] + aws_access_key_id=$(test_access_key) + aws_secret_access_key=$(test_secret_key) + """) + close(creds_io) + + withenv( + "AWS_PROFILE" => aws_profile, + "AWS_ACCESS_KEY_ID" => nothing, + "AWS_SECRET_ACCESS_KEY" => nothing, + "AWS_SHARED_CREDENTIALS_FILE" => creds_file, + ) do + _create_stack() + end + end + end + + @testset "Configuration File" begin + mktemp() do config_file, config_io + write(config_io, """ + [default] + aws_access_key_id=$(test_access_key) + aws_secret_access_key=$(test_secret_key) + """) + close(config_io) + + withenv( + "AWS_ACCESS_KEY_ID" => nothing, + "AWS_SECRET_ACCESS_KEY" => nothing, + "AWS_CONFIG_FILE" => config_file, + ) do + cred_patch = @patch dot_aws_credentials_file() = "" + + apply([cred_patch]) do + _create_stack() + end + end + end + end +end diff --git a/test/xml.jl b/test/xml.jl new file mode 100644 index 0000000..a79479a --- /dev/null +++ b/test/xml.jl @@ -0,0 +1,144 @@ +@testset "QueueURL" begin + expected = "http://queue.amazonaws.com/123456789012/testQueue" + + xml = """ + + + + http://queue.amazonaws.com/123456789012/testQueue + + + + + 7a62c49f-347e-4fc4-9331-6e8e7a96aa73 + + + + """ + + @assert parse_xml(xml)["CreateQueueResult"]["QueueUrl"] == expected +end + +@testset "User ARN" begin + expected = "arn:aws:iam::012541411202:root" + + xml = """ + + + + 2015-12-23T22:45:36Z + arn:aws:iam::012541411202:root + 012541411202 + 2015-09-15T01:07:23Z + + + + 837446c9-abaf-11e5-9f63-65ae4344bd73 + + + """ + + @test parse_xml(xml)["GetUserResult"]["User"]["Arn"] == expected +end + +@testset "Domain Names" begin + expected = ["Domain1", "Domain2"] + + xml = """ + + + Domain1 + Domain2 + TWV0ZXJpbmdUZXN0RG9tYWluMS0yMDA3MDYwMTE2NTY= + + + eb13162f-1b95-4511-8b12-489b86acfd28 + 0.0000219907 + + + """ + + @test parse_xml(xml)["ListDomainsResult"]["DomainName"] == expected +end + +@testset "Bucket Names" begin + expected = ["quotes", "samples"] + + xml = """ + + + + bcaf1ffd86f461ca5fb16fd081034f + webfile + + + + quotes + 2006-02-03T16:45:09.000Z + + + samples + 2006-02-03T16:41:58.000Z + + + + """ + + @test map(b->b["Name"], parse_xml(xml)["Buckets"]["Bucket"]) == expected +end + +@testset "Attributes" begin + expected_retention_period = "345600" + expected_timestamp = "1286771522" + + xml = """ + + + + ReceiveMessageWaitTimeSeconds + 2 + + + VisibilityTimeout + 30 + + + ApproximateNumberOfMessages + 0 + + + ApproximateNumberOfMessagesNotVisible + 0 + + + CreatedTimestamp + 1286771522 + + + LastModifiedTimestamp + 1286771522 + + + QueueArn + arn:aws:sqs:us-east-1:123456789012:qfoo + + + MaximumMessageSize + 8192 + + + MessageRetentionPeriod + 345600 + + + + 1ea71be5-b5a2-4f9d-b85a-945d8d08cd0b + + + """ + + d = Dict(a["Name"] => a["Value"] for a in parse_xml(xml)["GetQueueAttributesResult"]["Attribute"]) + + @test d["MessageRetentionPeriod"] == expected_retention_period + @test d["CreatedTimestamp"] == expected_timestamp +end \ No newline at end of file