Skip to content

Commit

Permalink
Extend retryablehttp limits in SDK base client
Browse files Browse the repository at this point in the history
Explicitly set three fields in the `retryablehttp.Client` struct.

`RetryWaitMin` is the shortest wait time before attempting to retry the
request, and is the same as the default, but set explicitly here for
clarity.

`RetryWaitMax` is the longest wait time before attempting to retry the
request. The default is 30 seconds, we extend this to 61 second in
consideration of potential API throttling for especially long-lasting
eventual consistency issues.

`RetryMax` is the maximum number of retries for a given API request, and
is given the most consideration here. The default is 4, which is far too
little for our purposes. Here, we set it to 16, which in conjunction
with the above two field values, and when implemented using an
exponential backoff algorithm, should take approximately 10 minutes to
exhaust.

However, in consideration of _really_ long lasting retries, we inspect
the context and increase this if it looks like the retries will be
exhausted before the context deadlines. The goal is to try and deadline
the context just before the retries are exhausted.

For compatibility, since we don't strictly require a deadline to execute
a request, we don't require one here.
  • Loading branch information
manicminer committed Mar 26, 2024
1 parent 1caa14b commit bee96b6
Showing 1 changed file with 14 additions and 2 deletions.
16 changes: 14 additions & 2 deletions sdk/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ func (c *Client) Execute(ctx context.Context, req *Request) (*Response, error) {
}

// Instantiate a RetryableHttp client and configure its CheckRetry func
r := c.retryableClient(func(ctx context.Context, r *http.Response, err error) (bool, error) {
r := c.retryableClient(ctx, func(ctx context.Context, r *http.Response, err error) (bool, error) {
// First check for badly malformed responses
if r == nil {
if req.IsIdempotent() {
Expand Down Expand Up @@ -647,7 +647,7 @@ func (c *Client) ExecutePaged(ctx context.Context, req *Request) (*Response, err
}

// retryableClient instantiates a new *retryablehttp.Client having the provided checkRetry func
func (c *Client) retryableClient(checkRetry retryablehttp.CheckRetry) (r *retryablehttp.Client) {
func (c *Client) retryableClient(ctx context.Context, checkRetry retryablehttp.CheckRetry) (r *retryablehttp.Client) {
r = retryablehttp.NewClient()

r.Backoff = func(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
Expand All @@ -672,6 +672,18 @@ func (c *Client) retryableClient(checkRetry retryablehttp.CheckRetry) (r *retrya
r.CheckRetry = checkRetry
r.ErrorHandler = RetryableErrorHandler
r.Logger = log.Default()
r.RetryWaitMin = 1 * time.Second
r.RetryWaitMax = 61 * time.Second

// Default RetryMax of 16 takes approx 10 minutes to iterate
r.RetryMax = 16

// Extend the RetryMax if the context timeout exceeds 10 minutes
if deadline, ok := ctx.Deadline(); ok {
if timeout := deadline.Sub(time.Now()); timeout > 10*time.Minute {
r.RetryMax = int(math.Round(timeout.Minutes())) + 6
}
}

tlsConfig := tls.Config{
MinVersion: tls.VersionTLS12,
Expand Down

0 comments on commit bee96b6

Please sign in to comment.