Merge #69

69: Bugfix and testing upgrade for renewable AWSCredentials r=omus a=morris25

Fixes timestamp format for ec2 and ecr credentials. Also fixes renewal to not overwrite `renew` function.
Solves #68. 
Also addresses #70 and the remaining comments from #52 to enable testing.

Co-authored-by: morris25 <sam.morrison@invenia.ca>

.travis.yml

@@ -10,6 +10,10 @@ julia:
 matrix:
   allow_failures:
     - julia: nightly
+    - julia: 1.0
+      stage: "Verify AWSS3"
+    - julia: 1.0
+      stage: "Verify AWSSQS"
 branches:
   only:
     - master
@@ -25,6 +29,16 @@ env:
     - secure: Lrh6AxUp1yFdsB1XlvNHFk+5pOMGQR8bJadUtnqXsuk97ieINI5w/da6wBPIH4WkZBGpQr+uJLe10PTxDcaNeZOC1LcuSK+v5J88pSiFYi10BggYWC2Kb2XEVBjd9BHi/RxAkiHXziBEaqpuX2WcwY2hgAOU+rpt3M9U9SxDITXNG68PpVVqDJ7GSZvZeJZnLrjTPKLlFR8EkqrE23bTaGvzqn50fwk7U3n0F04slkmg2ORScWkgBSFmaATBbiSU9Pz7bB4KuGarNC0Q0eTeoZcThRVYVwse3oLnYLMXkUuIDF5QwDfA5HCC4ad5yXdVenwHZSPS/sSnRM+fVAwkU8CL7hFC9zX2X/fBExWet0LufaCh4L6pax3dKV2m6q3Uq9M1z8X33ry2xqS96rpdD4bo+FxYrzjyAotn46w4fPE7yb3YZgf+dbHTDfVjiPH70hIKUhKiyzyKMTwdT63WNIy1/qbREge4jPVXgnIM3/jAu0THf74cT+RwN4xQnt7a50Yn2c/I+37kO2B+4qkyh4p75eIfPXqak1vdDc9tUqxZa7lPdIkofT3jp7TaJ+4/OLf2OrpA2XFjx20el4/TdB9TiJtGv0qTrujY4wh5BALxGVoSEihj3ahkqV/FQ6RyWJsMggd3yoaING61nCkDXUvLk/44yzeYavNrUaMzWpc=
 jobs:
   include:
+    - stage: "Verify AWSS3"
+      julia: 1.0
+      os: linux
+      script:
+        - julia --project=tmps3 -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.add("AWSS3"); Pkg.test("AWSS3")'
+    - stage: "Verify AWSSQS"
+      julia: 1.0
+      os: linux
+      script:
+        - julia --project=tmpsqs -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.add("AWSSQS"); Pkg.test("AWSSQS")'
     - stage: "Documentation"
       julia: 1.0
       os: linux

src/AWSCore.jl

@@ -109,7 +109,7 @@ aws = aws_config(creds = AWSCredentials("AKIAXXXXXXXXXXXXXXXX",
 
 """
 function aws_config(;profile=nothing,
-                     creds=RenewableAWSCredentials(profile=profile),
+                     creds=AWSCredentials(profile=profile),
                      region=get(ENV, "AWS_DEFAULT_REGION", "us-east-1"),
                      args...)
     @SymDict(creds, region, args...)
@@ -432,7 +432,7 @@ function do_request(r::AWSRequest)
                                "RequestExpired")
 
             # Reload local system credentials if needed...
-            get_credentials(r[:creds], force_refresh=true)
+            check_credentials(r[:creds], force_refresh=true)
 
         end
 

src/AWSCredentials.jl

@@ -15,10 +15,7 @@ export AWSCredentials,
        localhost_maybe_ec2,
        aws_user_arn,
        aws_account_number,
-       get_credentials,
-       RenewableAWSCredentials
-
-const DEFAULT_DURATION = Hour(1)
+       check_credentials
 
 """
 When you interact with AWS, you specify your [AWS Security Credentials](http://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html)
@@ -42,7 +39,13 @@ The order of precedence for this search is as follows:
 5. Assume Role provider via the aws config file
 6. Instance metadata service on an Amazon EC2 instance that has an IAM role configured.
 
-Each of those locations is discussed in more detail below.
+Once the credentials are found, the method by which they were accessed is stored in the `renew` field
+and the DateTime at which they will expire is stored in the `expiry` field.
+This allows the credentials to be refreshed as needed using [`check_credentials`](@ref).
+If `renew` is set to `nothing`, no attempt will be made to refresh the credentials.
+
+Any renewal function is expected to return `nothing` on failure or a populated `AWSCredentials` object on success.
+The `renew` field of the returned `AWSCredentials` will be discarded and does not need to be set.
 
 To specify the profile to use from `~/.aws/credentials`, do, for example, `AWSCredentials(profile="profile-name")`.
 """
@@ -53,30 +56,19 @@ mutable struct AWSCredentials
     user_arn::String
     account_number::String
     expiry::DateTime
+    renew::Union{Function, Nothing}
 
     function AWSCredentials(access_key_id,secret_key,
                             token="", user_arn="", account_number="";
-                            expiry=now(UTC) + DEFAULT_DURATION)
-        new(access_key_id, secret_key, token, user_arn, account_number, expiry)
+                            expiry=typemax(DateTime),
+                            renew=nothing)
+        new(access_key_id, secret_key, token, user_arn, account_number, expiry, renew)
     end
 end
 
-"""
-Holds an [`AWSCredentials`](@ref) object and ensures it is renewed before it can expire.
-Use [`get_credentials`](@ref) to access the credentials.
-
-The `RenewableAWSCredentials()` constructor tries to load local Credentials from
-environment variables, `~/.aws/credentials`, `~/.aws/config` or EC2 instance credentials.
-To specify the profile to use from `~/.aws/credentials`, do, for example, `RenewableAWSCredentials(profile="profile-name")`.
-"""
-struct RenewableAWSCredentials
-    creds::AWSCredentials
-    renew::Function
-end
-
-function RenewableAWSCredentials(;profile=nothing)
+function AWSCredentials(;profile=nothing)
     creds = nothing
-    renew = nothing
+    renew = Nothing
 
     # Define our search options
     functions = [
@@ -94,38 +86,47 @@ function RenewableAWSCredentials(;profile=nothing)
     end
 
     creds === nothing && error("Can't find AWS credentials!")
+    creds.renew = renew
 
     if debug_level > 0
         display(creds)
         println()
     end
 
-    return RenewableAWSCredentials(creds, renew)
+    return creds
 end
 
-@deprecate AWSCredentials(;profile=nothing) get_credentials(RenewableAWSCredentials(profile=profile))
+will_expire(cr::AWSCredentials) = now(UTC) >= cr.expiry - Minute(5)
 
 """
-    get_credentials(cr::RenewableAWSCredentials; force_refresh::Bool=false)
+    check_credentials(cr::AWSCredentials; force_refresh::Bool=false)
 
-Gets current AWSCredentials, refreshing them if they are soon to expire.
+Checks current AWSCredentials, refreshing them if they are soon to expire.
 If force_refresh is `true` the credentials will be renewed immediately.
 """
-function get_credentials(cr::RenewableAWSCredentials; force_refresh::Bool=false)
-    if force_refresh || now(UTC) >= cr.creds.expiry - Minute(5)
+function check_credentials(cr::AWSCredentials; force_refresh::Bool=false)
+    if force_refresh || will_expire(cr)
         if debug_level > 0
             println("Renewing credentials... ")
         end
-        copyto!(cr.creds, cr.renew())
-    end
-    return cr.creds
-end
+        renew = cr.renew
+
+        if renew !== nothing
+            new_creds = renew()
 
-function get_credentials(cr::AWSCredentials; force_refresh::Bool=false)
-    if force_refresh
-        copyto!(cr, get_credentials(RenewableAWSCredentials()))
+            new_creds === nothing && error("Can't find AWS credentials!")
+            copyto!(cr, new_creds)
+
+            # Ensure renewal function is not overwritten by the new credentials
+            cr.renew = renew
+        else
+            if debug_level > 0
+                println("Credentials cannot be renewed...")
+            end
+        end
     end
-    cr
+
+    return cr
 end
 
 function Base.show(io::IO,c::AWSCredentials)
@@ -198,16 +199,12 @@ for configrued user.
 e.g. `"arn:aws:iam::account-ID-without-hyphens:user/Bob"`
 """
 function aws_user_arn(aws::AWSConfig)
-
-    creds = get_credentials(aws[:creds])
-
+    creds = aws[:creds]
     if creds.user_arn == ""
-
         r = Services.sts(aws, "GetCallerIdentity", [])
         creds.user_arn = r["Arn"]
         creds.account_number = r["Account"]
     end
-
     return creds.user_arn
 end
 
@@ -218,7 +215,7 @@ end
 12-digit [AWS Account Number](http://docs.aws.amazon.com/general/latest/gr/acct-identifiers.html).
 """
 function aws_account_number(aws::AWSConfig)
-    creds = get_credentials(aws[:creds])
+    creds = aws[:creds]
     if creds.account_number == ""
         aws_user_arn(aws)
     end
@@ -270,11 +267,13 @@ function ec2_instance_credentials()
         print("Loading AWSCredentials from EC2 metadata... ")
     end
 
+    expiry = DateTime(strip(new_creds["Expiration"], 'Z'))
+
     AWSCredentials(new_creds["AccessKeyId"],
                    new_creds["SecretAccessKey"],
                    new_creds["Token"],
                    info["InstanceProfileArn"];
-                   expiry = unix2datetime(new_creds["Expiration"]))
+                   expiry = expiry)
 end
 
 
@@ -295,11 +294,13 @@ function ecs_instance_credentials()
         print("Loading AWSCredentials from ECS metadata... ")
     end
 
+    expiry = DateTime(strip(new_creds["Expiration"], 'Z'))
+
     AWSCredentials(new_creds["AccessKeyId"],
                    new_creds["SecretAccessKey"],
                    new_creds["Token"],
                    new_creds["RoleArn"];
-                   expiry = unix2datetime(new_creds["Expiration"]))
+                   expiry = expiry)
 end
 
 
@@ -320,7 +321,8 @@ function env_instance_credentials()
             ENV["AWS_ACCESS_KEY_ID"],
             ENV["AWS_SECRET_ACCESS_KEY"],
             get(ENV, "AWS_SESSION_TOKEN", ""),
-            get(ENV, "AWS_USER_ARN", "")
+            get(ENV, "AWS_USER_ARN", "");
+            renew = env_instance_credentials
         )
     else
         return nothing

src/sign.jl

@@ -34,7 +34,7 @@ function sign_aws2!(r::AWSRequest, t)
     r[:headers]["Content-Type"] =
         "application/x-www-form-urlencoded; charset=utf-8"
 
-    creds = get_credentials(r[:creds])
+    creds = check_credentials(r[:creds])
     query["AWSAccessKeyId"] = creds.access_key_id
     query["Expires"] = Dates.format(t + Dates.Second(120),
                                    dateformat"yyyy-mm-dd\THH:MM:SS\Z")
@@ -70,7 +70,7 @@ function sign_aws4!(r::AWSRequest, t)
     # Authentication scope...
     scope = [date, r[:region], r[:service], "aws4_request"]
 
-    creds = get_credentials(r[:creds])
+    creds = check_credentials(r[:creds])
     # Signing key generated from today's scope string...
     signing_key = string("AWS4", creds.secret_key)
     for element in scope

test/aws/awscore_test.yml

@@ -22,3 +22,74 @@ Resources:
               - cloudformation:DescribeStacks
             Resource:
               - !Sub arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${AWS::StackName}/*
+
+  # Policies to support testing on AWSS3 and AWSSQS
+  S3TestPolicy:
+    Type: AWS::IAM::Policy
+    Properties:
+      PolicyName: S3TestPolicy
+      Users:
+        - !Sub ${PublicCIUser}
+      PolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Action:
+              - s3:ListAllMyBuckets
+            Resource: "*"
+          - Effect: Allow
+            Action:
+              - s3:GetBucketPolicyStatus
+              - s3:ListBucketByTags
+              - s3:GetBucketTagging
+              - s3:PutBucketTagging
+              - s3:ListBucketVersions
+              - s3:CreateBucket
+              - s3:ListBucket
+              - s3:GetBucketVersioning
+              - s3:DeleteBucket
+              - s3:PutBucketVersioning
+            Resource:
+              - arn:aws:s3:::ocaws.jl.test.*
+          - Effect: Allow
+            Action:
+              - s3:DeleteObjectTagging
+              - s3:PutObject
+              - s3:GetObject
+              - s3:DeleteObjectVersion
+              - s3:GetObjectVersionTagging
+              - s3:PutObjectVersionTagging
+              - s3:GetObjectTagging
+              - s3:PutObjectTagging
+              - s3:DeleteObjectVersionTagging
+              - s3:DeleteObject
+              - s3:GetObjectVersion
+            Resource:
+              - arn:aws:s3:::ocaws.jl.test.*/*
+  SQSTestPolicy:
+    Type: AWS::IAM::Policy
+    Properties:
+      PolicyName: SQSTestPolicy
+      Users:
+        - !Sub ${PublicCIUser}
+      PolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Action:
+              - sqs:ListQueues
+            Resource: "*"
+          - Effect: Allow
+            Action:
+              - sqs:GetQueueAttributes
+              - sqs:GetQueueUrl
+              - sqs:ReceiveMessage
+              - sqs:CreateQueue
+              - sqs:DeleteMessage
+              - sqs:DeleteMessageBatch
+              - sqs:DeleteQueue
+              - sqs:SendMessage
+              - sqs:SendMessageBatch
+              - sqs:SetQueueAttributes
+            Resource:
+              - !Sub arn:aws:sqs:*:${AWS::AccountId}:ocaws-jl-test-queue-*