Merge #89

89: Use delay when retrying throttled requests r=mattBrzezinski a=morris25 Uses exponential backoff to avoid more throttling errors. Co-authored-by: morris25 <sam.morrison@invenia.ca>
JuliaCloud · Oct 22, 2019 · 82a615b · 82a615b
2 parents 6f43e72 + 877b8b4
commit 82a615b
Showing 1 changed file with 18 additions and 13 deletions.
diff --git a/src/AWSCore.jl b/src/AWSCore.jl
@@ -447,8 +447,7 @@ function do_request(r::AWSRequest)
         # https://github.com/boto/botocore/blob/master/botocore/data/_retry.json
         # Recommended for SDKs at:
         # https://docs.aws.amazon.com/general/latest/gr/api-retries.html
-        # Also BadDigest error and CRC32 thing
-        @retry if e isa AWSException && (
+        @delay_retry if e isa AWSException && (
                   http_status(e.cause) == 429 ||
                   ecode(e) in ("Throttling",
                                "ThrottlingException",
@@ -458,25 +457,31 @@ function do_request(r::AWSRequest)
                                "ProvisionedThroughputExceededException",
                                "LimitExceededException",
                                "RequestThrottled",
-                               "RequestTimeout",
-                               "BadDigest",
-                               "RequestTimeoutException",
-                               "PriorRequestNotComplete") ||
-                  header(e.cause, "crc32body") == "x-amz-crc32")
+                               "PriorRequestNotComplete"))
             if debug_level > 1
                 cause = "throttling"
 
-                if header(e.cause, "crc32body") == "x-amz-crc32"
-                    cause = "CRC32"
-                elseif ecode(e) in ("RequestTimeout",
-                                    "BadDigest",
-                                    "RequestTimeoutException",
-                                    "PriorRequestNotComplete")
+                if ecode(e) == "PriorRequestNotComplete"
                     cause = ecode(e)
                 end
                 println("Caught $e during request $(dump_aws_request(r)), retrying due to $cause...")
             end
         end
+
+        # Handle BadDigest error and CRC32 thing
+        @retry if e isa AWSException && (
+            header(e.cause, "crc32body") == "x-amz-crc32" ||
+            ecode(e) in ("BadDigest", "RequestTimeout", "RequestTimeoutException")
+        )
+            if debug_level > 1
+                cause = if header(e.cause, "crc32body") == "x-amz-crc32"
+                    "CRC32"
+                else
+                    ecode(e)
+                end
+                println("Caught $e during request $(dump_aws_request(r)), retrying due to $cause...")
+            end
+        end
     end
 
     if debug_level > 1