Merge #116

116: Return both body and header for all AWS requests r=mattBrzezinski a=mattBrzezinski

`AWSS3.jl` is having issues where the multi-part upload is not working properly, it attempts to return request headers back as both the `part number` and `etag` are required to successfully complete a multi-part upload.

`AWSCore.jl` currently does not return headers unless the operation is a `HEAD` type request. This merge request returns back the response `body` and `headers` for all requests, which accommodates for these requirements.

This change will allow use to begin resolving [AWSS3.jl #84](JuliaCloud/AWSS3.jl#84).

Co-authored-by: Matt Brzezinski <matt.brzezinski@invenia.ca>
Co-authored-by: Matt.jl <matt.brzezinski@invenia.ca>

Project.toml

@@ -1,6 +1,6 @@
 name = "AWSCore"
 uuid = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598"
-version = "0.6.9"
+version = "0.6.10"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

src/AWSCore.jl

@@ -225,7 +225,6 @@ end
 Process request for AWS "query" service protocol.
 """
 function service_query(aws::AWSConfig; args...)
-
     request = Dict{Symbol,Any}(args)
 
     request[:verb] = "POST"
@@ -257,7 +256,6 @@ end
 Process request for AWS "json" service protocol.
 """
 function service_json(aws::AWSConfig; args...)
-
     request = Dict{Symbol,Any}(args)
 
     request[:verb] = "POST"
@@ -301,7 +299,6 @@ end
 Process request for AWS "rest_json" service protocol.
 """
 function service_rest_json(aws::AWSConfig; args...)
-
     request = Dict{Symbol,Any}(args)
     args = Dict(request[:args])
 
@@ -323,7 +320,6 @@ end
 Process request for AWS "rest_xml" service protocol.
 """
 function service_rest_xml(aws::AWSConfig; args...)
-
     request = Dict{Symbol,Any}(args)
     args = stringdict(request[:args])
 
@@ -387,12 +383,11 @@ end
 
 
 """
-    do_request(::AWSRequest)
+    do_request(::AWSRequest; return_headers=false)
 
-Submit an API request, return the result.
+Submit an API request, return the response body and headers if `return_headers=true`.
 """
-function do_request(r::AWSRequest)
-
+function do_request(r::AWSRequest; return_headers=false)
     response = nothing
 
     # Try request 3 times to deal with possible Redirect and ExiredToken...
@@ -507,7 +502,7 @@ function do_request(r::AWSRequest)
 
     # Return raw data if requested...
     if get(r, :return_raw, false)
-        return response.body
+        return (return_headers ? (response.body, response.headers) : response.body)
     end
 
     # Parse response data according to mimetype...
@@ -521,17 +516,17 @@ function do_request(r::AWSRequest)
     body = String(copy(response.body))
 
     if occursin(r"/xml", mime)
-        return parse_xml(body)
+        return (return_headers ? (parse_xml(body), Dict(response.headers)) : parse_xml(body))
     end
 
     if occursin(r"/x-amz-json-1.[01]$", mime)
         if isempty(response.body)
             return nothing
         end
         if get(r, :ordered_json_dict, true)
-            return JSON.parse(body, dicttype=OrderedDict)
+            return (return_headers ? (JSON.parse(body, dicttype=OrderedDict), Dict(response.headers)) : JSON.parse(body, dicttype=OrderedDict))
         else
-            return JSON.parse(body)
+            return (return_headers ? (JSON.parse(body), Dict(response.headers)) : JSON.parse(body))
         end
     end
 
@@ -551,15 +546,16 @@ function do_request(r::AWSRequest)
         catch e
             @ignore if typeof(e) == KeyError end
         end
-        return info
+
+        return (return_headers ? (info, Dict(response.headers)) : info)
     end
 
     if occursin(r"^text/", mime)
-        return body
+        return (return_headers ? (body, Dict(response.headers)) : body)
     end
 
     # Return raw data by default...
-    return response.body
+    return (return_headers ? (response.body, nothing) : response.body)
 end
 
 

test/aws4.jl

@@ -221,17 +221,20 @@ end
         expected_data = "{\n    \"zarr_format\": 2\n}"
 
         @test result["Contents"][1]["Key"] == expected_content_key
-        @test String(AWSCore.Services.s3(
+
+        key_result = AWSCore.Services.s3(
             aws_google,
             "GET", "/{Bucket}/{Key+}",
             Bucket="cmip6",
-            Key="$(data_prefix)/.zgroup")
-        ) == expected_data
+            Key="$(data_prefix)/.zgroup"
+        )
+
+        @test String(key_result) == expected_data
     end
 
     @testset "Accessing OTC Object Store" begin
         aws_otc = aws_config(creds=nothing, region="eu-de", service_name="obs", service_host="otc.t-systems.com")
-        result = AWSCore.Services.s3(aws_otc, "GET", "/{Bucket}?list-type=2", Dict(:Bucket=>"obs-esdc-v2.0.0",:prefix=>"",:delimiter=>"/"))
+        result = AWSCore.Services.s3(aws_otc, "GET", "/{Bucket}?list-type=2", Dict(:Bucket=>"obs-esdc-v2.0.0", :prefix=>"",:delimiter=>"/"))
 
         @test result["CommonPrefixes"][1]["Prefix"] == "esdc-8d-0.0083deg-184x60x60-2.0.0_colombia.zarr/"
     end