Skip to content
This repository has been archived by the owner on Apr 19, 2023. It is now read-only.

Feature/Bug Fix: Use exponential back off for gdrive download and gdrive download query #309

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Feature/Bug Fix: Use exponential back off for gdrive download and gdrive download query #309

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions drive/download.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ type DownloadArgs struct {
Delete bool
Stdout bool
Timeout time.Duration
Try int
}

func (self *Drive) Download(args DownloadArgs) error {
Expand All @@ -31,6 +32,11 @@ func (self *Drive) Download(args DownloadArgs) error {

f, err := self.service.Files.Get(args.Id).Fields("id", "name", "size", "mimeType", "md5Checksum").Do()
if err != nil {
if isBackendOrRateLimitError(err) && args.Try < MaxErrorRetries {
exponentialBackoffSleep(args.Try)
args.Try++
return self.Download(args)
}
return fmt.Errorf("Failed to get file: %s", err)
}

Expand Down Expand Up @@ -72,6 +78,7 @@ type DownloadQueryArgs struct {
Force bool
Skip bool
Recursive bool
Try int
}

func (self *Drive) DownloadQuery(args DownloadQueryArgs) error {
Expand All @@ -81,6 +88,11 @@ func (self *Drive) DownloadQuery(args DownloadQueryArgs) error {
}
files, err := self.listAllFiles(listArgs)
if err != nil {
if isBackendOrRateLimitError(err) && args.Try < MaxErrorRetries {
exponentialBackoffSleep(args.Try)
args.Try++
return self.DownloadQuery(args)
}
return fmt.Errorf("Failed to list files: %s", err)
}

Expand Down Expand Up @@ -110,6 +122,11 @@ func (self *Drive) DownloadQuery(args DownloadQueryArgs) error {
func (self *Drive) downloadRecursive(args DownloadArgs) error {
f, err := self.service.Files.Get(args.Id).Fields("id", "name", "size", "mimeType", "md5Checksum").Do()
if err != nil {
if isBackendOrRateLimitError(err) && args.Try < MaxErrorRetries {
exponentialBackoffSleep(args.Try)
args.Try++
return self.downloadRecursive(args)
}
return fmt.Errorf("Failed to get file: %s", err)
}

Expand All @@ -131,6 +148,10 @@ func (self *Drive) downloadBinary(f *drive.File, args DownloadArgs) (int64, int6
if err != nil {
if isTimeoutError(err) {
return 0, 0, fmt.Errorf("Failed to download file: timeout, no data was transferred for %v", args.Timeout)
} else if isBackendOrRateLimitError(err) && (args.Try < MaxErrorRetries) {
exponentialBackoffSleep(args.Try)
args.Try++
return self.downloadBinary(f, args)
}
return 0, 0, fmt.Errorf("Failed to download file: %s", err)
}
Expand Down Expand Up @@ -230,6 +251,11 @@ func (self *Drive) downloadDirectory(parent *drive.File, args DownloadArgs) erro
}
files, err := self.listAllFiles(listArgs)
if err != nil {
if isBackendOrRateLimitError(err) && args.Try < MaxErrorRetries {
exponentialBackoffSleep(args.Try)
args.Try++
return self.downloadDirectory(parent, args)
}
return fmt.Errorf("Failed listing files: %s", err)
}

Expand Down