From 93a7c25da52995887d20c5997eeead10fd4d478d Mon Sep 17 00:00:00 2001 From: "jeff.yin" Date: Tue, 12 Nov 2019 11:28:55 +0800 Subject: [PATCH 1/7] remove vendor/google.golang.org , replace by not vendor --- vendor/{google.golang.org => _google.golang.org}/api/LICENSE | 0 .../api/drive/v3/drive-api.json | 0 .../api/drive/v3/drive-gen.go | 0 .../api/gensupport/backoff.go | 0 .../api/gensupport/buffer.go | 0 .../api/gensupport/doc.go | 0 .../api/gensupport/json.go | 0 .../api/gensupport/media.go | 0 .../api/gensupport/params.go | 0 .../api/gensupport/resumable.go | 0 .../api/gensupport/retry.go | 0 .../api/googleapi/googleapi.go | 0 .../api/googleapi/internal/uritemplates/LICENSE | 0 .../api/googleapi/internal/uritemplates/uritemplates.go | 0 .../api/googleapi/internal/uritemplates/utils.go | 0 .../api/googleapi/types.go | 0 16 files changed, 0 insertions(+), 0 deletions(-) rename vendor/{google.golang.org => _google.golang.org}/api/LICENSE (100%) rename vendor/{google.golang.org => _google.golang.org}/api/drive/v3/drive-api.json (100%) rename vendor/{google.golang.org => _google.golang.org}/api/drive/v3/drive-gen.go (100%) rename vendor/{google.golang.org => _google.golang.org}/api/gensupport/backoff.go (100%) rename vendor/{google.golang.org => _google.golang.org}/api/gensupport/buffer.go (100%) rename vendor/{google.golang.org => _google.golang.org}/api/gensupport/doc.go (100%) rename vendor/{google.golang.org => _google.golang.org}/api/gensupport/json.go (100%) rename vendor/{google.golang.org => _google.golang.org}/api/gensupport/media.go (100%) rename vendor/{google.golang.org => _google.golang.org}/api/gensupport/params.go (100%) rename vendor/{google.golang.org => _google.golang.org}/api/gensupport/resumable.go (100%) rename vendor/{google.golang.org => _google.golang.org}/api/gensupport/retry.go (100%) rename vendor/{google.golang.org => _google.golang.org}/api/googleapi/googleapi.go (100%) rename vendor/{google.golang.org => _google.golang.org}/api/googleapi/internal/uritemplates/LICENSE (100%) rename vendor/{google.golang.org => _google.golang.org}/api/googleapi/internal/uritemplates/uritemplates.go (100%) rename vendor/{google.golang.org => _google.golang.org}/api/googleapi/internal/uritemplates/utils.go (100%) rename vendor/{google.golang.org => _google.golang.org}/api/googleapi/types.go (100%) diff --git a/vendor/google.golang.org/api/LICENSE b/vendor/_google.golang.org/api/LICENSE similarity index 100% rename from vendor/google.golang.org/api/LICENSE rename to vendor/_google.golang.org/api/LICENSE diff --git a/vendor/google.golang.org/api/drive/v3/drive-api.json b/vendor/_google.golang.org/api/drive/v3/drive-api.json similarity index 100% rename from vendor/google.golang.org/api/drive/v3/drive-api.json rename to vendor/_google.golang.org/api/drive/v3/drive-api.json diff --git a/vendor/google.golang.org/api/drive/v3/drive-gen.go b/vendor/_google.golang.org/api/drive/v3/drive-gen.go similarity index 100% rename from vendor/google.golang.org/api/drive/v3/drive-gen.go rename to vendor/_google.golang.org/api/drive/v3/drive-gen.go diff --git a/vendor/google.golang.org/api/gensupport/backoff.go b/vendor/_google.golang.org/api/gensupport/backoff.go similarity index 100% rename from vendor/google.golang.org/api/gensupport/backoff.go rename to vendor/_google.golang.org/api/gensupport/backoff.go diff --git a/vendor/google.golang.org/api/gensupport/buffer.go b/vendor/_google.golang.org/api/gensupport/buffer.go similarity index 100% rename from vendor/google.golang.org/api/gensupport/buffer.go rename to vendor/_google.golang.org/api/gensupport/buffer.go diff --git a/vendor/google.golang.org/api/gensupport/doc.go b/vendor/_google.golang.org/api/gensupport/doc.go similarity index 100% rename from vendor/google.golang.org/api/gensupport/doc.go rename to vendor/_google.golang.org/api/gensupport/doc.go diff --git a/vendor/google.golang.org/api/gensupport/json.go b/vendor/_google.golang.org/api/gensupport/json.go similarity index 100% rename from vendor/google.golang.org/api/gensupport/json.go rename to vendor/_google.golang.org/api/gensupport/json.go diff --git a/vendor/google.golang.org/api/gensupport/media.go b/vendor/_google.golang.org/api/gensupport/media.go similarity index 100% rename from vendor/google.golang.org/api/gensupport/media.go rename to vendor/_google.golang.org/api/gensupport/media.go diff --git a/vendor/google.golang.org/api/gensupport/params.go b/vendor/_google.golang.org/api/gensupport/params.go similarity index 100% rename from vendor/google.golang.org/api/gensupport/params.go rename to vendor/_google.golang.org/api/gensupport/params.go diff --git a/vendor/google.golang.org/api/gensupport/resumable.go b/vendor/_google.golang.org/api/gensupport/resumable.go similarity index 100% rename from vendor/google.golang.org/api/gensupport/resumable.go rename to vendor/_google.golang.org/api/gensupport/resumable.go diff --git a/vendor/google.golang.org/api/gensupport/retry.go b/vendor/_google.golang.org/api/gensupport/retry.go similarity index 100% rename from vendor/google.golang.org/api/gensupport/retry.go rename to vendor/_google.golang.org/api/gensupport/retry.go diff --git a/vendor/google.golang.org/api/googleapi/googleapi.go b/vendor/_google.golang.org/api/googleapi/googleapi.go similarity index 100% rename from vendor/google.golang.org/api/googleapi/googleapi.go rename to vendor/_google.golang.org/api/googleapi/googleapi.go diff --git a/vendor/google.golang.org/api/googleapi/internal/uritemplates/LICENSE b/vendor/_google.golang.org/api/googleapi/internal/uritemplates/LICENSE similarity index 100% rename from vendor/google.golang.org/api/googleapi/internal/uritemplates/LICENSE rename to vendor/_google.golang.org/api/googleapi/internal/uritemplates/LICENSE diff --git a/vendor/google.golang.org/api/googleapi/internal/uritemplates/uritemplates.go b/vendor/_google.golang.org/api/googleapi/internal/uritemplates/uritemplates.go similarity index 100% rename from vendor/google.golang.org/api/googleapi/internal/uritemplates/uritemplates.go rename to vendor/_google.golang.org/api/googleapi/internal/uritemplates/uritemplates.go diff --git a/vendor/google.golang.org/api/googleapi/internal/uritemplates/utils.go b/vendor/_google.golang.org/api/googleapi/internal/uritemplates/utils.go similarity index 100% rename from vendor/google.golang.org/api/googleapi/internal/uritemplates/utils.go rename to vendor/_google.golang.org/api/googleapi/internal/uritemplates/utils.go diff --git a/vendor/google.golang.org/api/googleapi/types.go b/vendor/_google.golang.org/api/googleapi/types.go similarity index 100% rename from vendor/google.golang.org/api/googleapi/types.go rename to vendor/_google.golang.org/api/googleapi/types.go From 68d336cbf4ca3a11bc907144ff58999967b10e8c Mon Sep 17 00:00:00 2001 From: "jeff.yin" Date: Tue, 12 Nov 2019 11:29:05 +0800 Subject: [PATCH 2/7] support teamdrives --- drive/delete.go | 6 +++--- drive/download.go | 13 +++++++++---- drive/export.go | 2 +- drive/info.go | 5 +++-- drive/list.go | 12 ++++++++---- drive/mkdir.go | 5 +++-- drive/sync.go | 11 ++++++----- drive/sync_download.go | 9 +++++---- drive/sync_upload.go | 19 ++++++++++--------- drive/update.go | 7 ++++--- drive/upload.go | 9 +++++---- 11 files changed, 57 insertions(+), 41 deletions(-) diff --git a/drive/delete.go b/drive/delete.go index 314672ce..5b412a8d 100644 --- a/drive/delete.go +++ b/drive/delete.go @@ -12,7 +12,7 @@ type DeleteArgs struct { } func (self *Drive) Delete(args DeleteArgs) error { - f, err := self.service.Files.Get(args.Id).Fields("name", "mimeType").Do() + f, err := self.service.Files.Get(args.Id).SupportsAllDrives(true).Fields("name", "mimeType").Do() if err != nil { return fmt.Errorf("Failed to get file: %s", err) } @@ -21,7 +21,7 @@ func (self *Drive) Delete(args DeleteArgs) error { return fmt.Errorf("'%s' is a directory, use the 'recursive' flag to delete directories", f.Name) } - err = self.service.Files.Delete(args.Id).Do() + err = self.service.Files.Delete(args.Id).SupportsAllDrives(true).Do() if err != nil { return fmt.Errorf("Failed to delete file: %s", err) } @@ -31,7 +31,7 @@ func (self *Drive) Delete(args DeleteArgs) error { } func (self *Drive) deleteFile(fileId string) error { - err := self.service.Files.Delete(fileId).Do() + err := self.service.Files.Delete(fileId).SupportsAllDrives(true).Do() if err != nil { return fmt.Errorf("Failed to delete file: %s", err) } diff --git a/drive/download.go b/drive/download.go index e4a41412..795d4b68 100644 --- a/drive/download.go +++ b/drive/download.go @@ -29,7 +29,7 @@ func (self *Drive) Download(args DownloadArgs) error { return self.downloadRecursive(args) } - f, err := self.service.Files.Get(args.Id).Fields("id", "name", "size", "mimeType", "md5Checksum").Do() + f, err := self.service.Files.Get(args.Id).SupportsAllDrives(true).Fields("id", "name", "size", "mimeType", "md5Checksum").Do() if err != nil { return fmt.Errorf("Failed to get file: %s", err) } @@ -108,7 +108,8 @@ func (self *Drive) DownloadQuery(args DownloadQueryArgs) error { } func (self *Drive) downloadRecursive(args DownloadArgs) error { - f, err := self.service.Files.Get(args.Id).Fields("id", "name", "size", "mimeType", "md5Checksum").Do() + fmt.Println("jeff_debug downloadRecursive") + f, err := self.service.Files.Get(args.Id).SupportsAllDrives(true).Fields("id", "name", "size", "mimeType", "md5Checksum").Do() if err != nil { return fmt.Errorf("Failed to get file: %s", err) } @@ -126,8 +127,8 @@ func (self *Drive) downloadRecursive(args DownloadArgs) error { func (self *Drive) downloadBinary(f *drive.File, args DownloadArgs) (int64, int64, error) { // Get timeout reader wrapper and context timeoutReaderWrapper, ctx := getTimeoutReaderWrapperContext(args.Timeout) - - res, err := self.service.Files.Get(f.Id).Context(ctx).Download() + fmt.Println("jeff_debug downloadBinary") + res, err := self.service.Files.Get(f.Id).SupportsAllDrives(true).Context(ctx).Download() if err != nil { if isTimeoutError(err) { return 0, 0, fmt.Errorf("Failed to download file: timeout, no data was transferred for %v", args.Timeout) @@ -140,6 +141,7 @@ func (self *Drive) downloadBinary(f *drive.File, args DownloadArgs) (int64, int6 // Path to file fpath := filepath.Join(args.Path, f.Name) + fmt.Println("fpath:", fpath) if !args.Stdout { fmt.Fprintf(args.Out, "Downloading %s -> %s\n", f.Name, fpath) @@ -197,6 +199,8 @@ func (self *Drive) saveFile(args saveFileArgs) (int64, int64, error) { // Download to tmp file tmpPath := args.fpath + ".incomplete" + fmt.Println("tmpPath:", tmpPath) + // Create new file outFile, err := os.Create(tmpPath) if err != nil { @@ -224,6 +228,7 @@ func (self *Drive) saveFile(args saveFileArgs) (int64, int64, error) { } func (self *Drive) downloadDirectory(parent *drive.File, args DownloadArgs) error { + fmt.Println("jeff_debug downloadDirectory") listArgs := listAllFilesArgs{ query: fmt.Sprintf("'%s' in parents", parent.Id), fields: []googleapi.Field{"nextPageToken", "files(id,name)"}, diff --git a/drive/export.go b/drive/export.go index 3fdd45a6..2b7356b9 100644 --- a/drive/export.go +++ b/drive/export.go @@ -25,7 +25,7 @@ type ExportArgs struct { } func (self *Drive) Export(args ExportArgs) error { - f, err := self.service.Files.Get(args.Id).Fields("name", "mimeType").Do() + f, err := self.service.Files.Get(args.Id).SupportsTeamDrives(true).Fields("name", "mimeType").Do() if err != nil { return fmt.Errorf("Failed to get file: %s", err) } diff --git a/drive/info.go b/drive/info.go index c6f44715..2a1a20ca 100644 --- a/drive/info.go +++ b/drive/info.go @@ -2,8 +2,9 @@ package drive import ( "fmt" - "google.golang.org/api/drive/v3" "io" + + "google.golang.org/api/drive/v3" ) type FileInfoArgs struct { @@ -13,7 +14,7 @@ type FileInfoArgs struct { } func (self *Drive) Info(args FileInfoArgs) error { - f, err := self.service.Files.Get(args.Id).Fields("id", "name", "size", "createdTime", "modifiedTime", "md5Checksum", "mimeType", "parents", "shared", "description", "webContentLink", "webViewLink").Do() + f, err := self.service.Files.Get(args.Id).SupportsAllDrives(true).Fields("id", "name", "size", "createdTime", "modifiedTime", "md5Checksum", "mimeType", "parents", "shared", "description", "webContentLink", "webViewLink").Do() if err != nil { return fmt.Errorf("Failed to get file: %s", err) } diff --git a/drive/list.go b/drive/list.go index ab8aca56..ff52ef58 100644 --- a/drive/list.go +++ b/drive/list.go @@ -2,11 +2,12 @@ package drive import ( "fmt" + "io" + "text/tabwriter" + "golang.org/x/net/context" "google.golang.org/api/drive/v3" "google.golang.org/api/googleapi" - "io" - "text/tabwriter" ) type ListFilesArgs struct { @@ -73,10 +74,13 @@ func (self *Drive) listAllFiles(args listAllFilesArgs) ([]*drive.File, error) { } controlledStop := fmt.Errorf("Controlled stop") - - err := self.service.Files.List().Q(args.query).Fields(args.fields...).OrderBy(args.sortOrder).PageSize(pageSize).Pages(context.TODO(), func(fl *drive.FileList) error { + fmt.Println(args.query) + err := self.service.Files.List().Corpora("allDrives").IncludeItemsFromAllDrives(true).SupportsAllDrives(true).Q(args.query).Fields(args.fields...).OrderBy(args.sortOrder).PageSize(pageSize).Pages(context.TODO(), func(fl *drive.FileList) error { files = append(files, fl.Files...) + fmt.Println("jeff_debug listAllFiles") + fmt.Println(fl.Files) + fmt.Println(files) // Stop when we have all the files we need if args.maxFiles > 0 && len(files) >= int(args.maxFiles) { return controlledStop diff --git a/drive/mkdir.go b/drive/mkdir.go index 05d51914..1fabef04 100644 --- a/drive/mkdir.go +++ b/drive/mkdir.go @@ -2,8 +2,9 @@ package drive import ( "fmt" - "google.golang.org/api/drive/v3" "io" + + "google.golang.org/api/drive/v3" ) const DirectoryMimeType = "application/vnd.google-apps.folder" @@ -35,7 +36,7 @@ func (self *Drive) mkdir(args MkdirArgs) (*drive.File, error) { dstFile.Parents = args.Parents // Create directory - f, err := self.service.Files.Create(dstFile).Do() + f, err := self.service.Files.Create(dstFile).SupportsAllDrives(true).Do() if err != nil { return nil, fmt.Errorf("Failed to create directory: %s", err) } diff --git a/drive/sync.go b/drive/sync.go index 35ab16eb..5d3c69bb 100644 --- a/drive/sync.go +++ b/drive/sync.go @@ -2,16 +2,17 @@ package drive import ( "fmt" - "github.com/sabhiram/go-git-ignore" - "github.com/soniakeys/graph" - "google.golang.org/api/drive/v3" - "google.golang.org/api/googleapi" "io" "os" "path/filepath" "strings" "text/tabwriter" "time" + + "github.com/sabhiram/go-git-ignore" + "github.com/soniakeys/graph" + "google.golang.org/api/drive/v3" + "google.golang.org/api/googleapi" ) const DefaultIgnoreFile = ".gdriveignore" @@ -86,7 +87,7 @@ func (self *Drive) prepareSyncFiles(localPath string, root *drive.File, cmp File } func (self *Drive) isSyncFile(id string) (bool, error) { - f, err := self.service.Files.Get(id).Fields("appProperties").Do() + f, err := self.service.Files.Get(id).SupportsAllDrives(true).Fields("appProperties").Do() if err != nil { return false, fmt.Errorf("Failed to get file: %s", err) } diff --git a/drive/sync_download.go b/drive/sync_download.go index 58cd49c6..9a88b611 100644 --- a/drive/sync_download.go +++ b/drive/sync_download.go @@ -3,13 +3,14 @@ package drive import ( "bytes" "fmt" - "google.golang.org/api/drive/v3" - "google.golang.org/api/googleapi" "io" "os" "path/filepath" "sort" "time" + + "google.golang.org/api/drive/v3" + "google.golang.org/api/googleapi" ) type DownloadSyncArgs struct { @@ -85,7 +86,7 @@ func (self *Drive) DownloadSync(args DownloadSyncArgs) error { func (self *Drive) getSyncRoot(rootId string) (*drive.File, error) { fields := []googleapi.Field{"id", "name", "mimeType", "appProperties"} - f, err := self.service.Files.Get(rootId).Fields(fields...).Do() + f, err := self.service.Files.Get(rootId).SupportsAllDrives(true).Fields(fields...).Do() if err != nil { return nil, fmt.Errorf("Failed to find root dir: %s", err) } @@ -191,7 +192,7 @@ func (self *Drive) downloadRemoteFile(id, fpath string, args DownloadSyncArgs, t // Get timeout reader wrapper and context timeoutReaderWrapper, ctx := getTimeoutReaderWrapperContext(args.Timeout) - res, err := self.service.Files.Get(id).Context(ctx).Download() + res, err := self.service.Files.Get(id).SupportsAllDrives(true).Context(ctx).Download() if err != nil { if isBackendOrRateLimitError(err) && try < MaxErrorRetries { exponentialBackoffSleep(try) diff --git a/drive/sync_upload.go b/drive/sync_upload.go index f1e43a49..bd2b76d9 100644 --- a/drive/sync_upload.go +++ b/drive/sync_upload.go @@ -3,13 +3,14 @@ package drive import ( "bytes" "fmt" - "google.golang.org/api/drive/v3" - "google.golang.org/api/googleapi" "io" "os" "path/filepath" "sort" "time" + + "google.golang.org/api/drive/v3" + "google.golang.org/api/googleapi" ) type UploadSyncArgs struct { @@ -96,7 +97,7 @@ func (self *Drive) UploadSync(args UploadSyncArgs) error { func (self *Drive) prepareSyncRoot(args UploadSyncArgs) (*drive.File, error) { fields := []googleapi.Field{"id", "name", "mimeType", "appProperties"} - f, err := self.service.Files.Get(args.RootId).Fields(fields...).Do() + f, err := self.service.Files.Get(args.RootId).SupportsAllDrives(true).Fields(fields...).Do() if err != nil { return nil, fmt.Errorf("Failed to find root dir: %s", err) } @@ -128,7 +129,7 @@ func (self *Drive) prepareSyncRoot(args UploadSyncArgs) (*drive.File, error) { AppProperties: map[string]string{"sync": "true", "syncRoot": "true"}, } - f, err = self.service.Files.Update(f.Id, dstFile).Fields(fields...).Do() + f, err = self.service.Files.Update(f.Id, dstFile).SupportsAllDrives(true).Fields(fields...).Do() if err != nil { return nil, fmt.Errorf("Failed to update root directory: %s", err) } @@ -268,7 +269,7 @@ func (self *Drive) createMissingRemoteDir(args createMissingRemoteDirArgs) (*dri return dstFile, nil } - f, err := self.service.Files.Create(dstFile).Do() + f, err := self.service.Files.Create(dstFile).SupportsAllDrives(true).Do() if err != nil { if isBackendOrRateLimitError(err) && args.try < MaxErrorRetries { exponentialBackoffSleep(args.try) @@ -311,7 +312,7 @@ func (self *Drive) uploadMissingFile(parentId string, lf *LocalFile, args Upload // Wrap reader in timeout reader reader, ctx := getTimeoutReaderContext(progressReader, args.Timeout) - _, err = self.service.Files.Create(dstFile).Fields("id", "name", "size", "md5Checksum").Context(ctx).Media(reader, chunkSize).Do() + _, err = self.service.Files.Create(dstFile).SupportsAllDrives(true).Fields("id", "name", "size", "md5Checksum").Context(ctx).Media(reader, chunkSize).Do() if err != nil { if isBackendOrRateLimitError(err) && try < MaxErrorRetries { exponentialBackoffSleep(try) @@ -352,7 +353,7 @@ func (self *Drive) updateChangedFile(cf *changedFile, args UploadSyncArgs, try i // Wrap reader in timeout reader reader, ctx := getTimeoutReaderContext(progressReader, args.Timeout) - _, err = self.service.Files.Update(cf.remote.file.Id, dstFile).Context(ctx).Media(reader, chunkSize).Do() + _, err = self.service.Files.Update(cf.remote.file.Id, dstFile).SupportsAllDrives(true).Context(ctx).Media(reader, chunkSize).Do() if err != nil { if isBackendOrRateLimitError(err) && try < MaxErrorRetries { exponentialBackoffSleep(try) @@ -373,7 +374,7 @@ func (self *Drive) deleteRemoteFile(rf *RemoteFile, args UploadSyncArgs, try int return nil } - err := self.service.Files.Delete(rf.file.Id).Do() + err := self.service.Files.Delete(rf.file.Id).SupportsAllDrives(true).Do() if err != nil { if isBackendOrRateLimitError(err) && try < MaxErrorRetries { exponentialBackoffSleep(try) @@ -389,7 +390,7 @@ func (self *Drive) deleteRemoteFile(rf *RemoteFile, args UploadSyncArgs, try int func (self *Drive) dirIsEmpty(id string) (bool, error) { query := fmt.Sprintf("'%s' in parents", id) - fileList, err := self.service.Files.List().Q(query).Do() + fileList, err := self.service.Files.List().Corpora("allDrives").IncludeItemsFromAllDrives(true).SupportsAllDrives(true).Q(query).Do() if err != nil { return false, fmt.Errorf("Empty dir check failed: ", err) } diff --git a/drive/update.go b/drive/update.go index f496f52b..0d397e50 100644 --- a/drive/update.go +++ b/drive/update.go @@ -2,12 +2,13 @@ package drive import ( "fmt" - "google.golang.org/api/drive/v3" - "google.golang.org/api/googleapi" "io" "mime" "path/filepath" "time" + + "google.golang.org/api/drive/v3" + "google.golang.org/api/googleapi" ) type UpdateArgs struct { @@ -64,7 +65,7 @@ func (self *Drive) Update(args UpdateArgs) error { fmt.Fprintf(args.Out, "Uploading %s\n", args.Path) started := time.Now() - f, err := self.service.Files.Update(args.Id, dstFile).Fields("id", "name", "size").Context(ctx).Media(reader, chunkSize).Do() + f, err := self.service.Files.Update(args.Id, dstFile).SupportsAllDrives(true).Fields("id", "name", "size").Context(ctx).Media(reader, chunkSize).Do() if err != nil { if isTimeoutError(err) { return fmt.Errorf("Failed to upload file: timeout, no data was transferred for %v", args.Timeout) diff --git a/drive/upload.go b/drive/upload.go index a4482e20..6b7889a0 100644 --- a/drive/upload.go +++ b/drive/upload.go @@ -2,13 +2,14 @@ package drive import ( "fmt" - "google.golang.org/api/drive/v3" - "google.golang.org/api/googleapi" "io" "mime" "os" "path/filepath" "time" + + "google.golang.org/api/drive/v3" + "google.golang.org/api/googleapi" ) type UploadArgs struct { @@ -184,7 +185,7 @@ func (self *Drive) uploadFile(args UploadArgs) (*drive.File, int64, error) { fmt.Fprintf(args.Out, "Uploading %s\n", args.Path) started := time.Now() - f, err := self.service.Files.Create(dstFile).Fields("id", "name", "size", "md5Checksum", "webContentLink").Context(ctx).Media(reader, chunkSize).Do() + f, err := self.service.Files.Create(dstFile).SupportsAllDrives(true).Fields("id", "name", "size", "md5Checksum", "webContentLink").Context(ctx).Media(reader, chunkSize).Do() if err != nil { if isTimeoutError(err) { return nil, 0, fmt.Errorf("Failed to upload file: timeout, no data was transferred for %v", args.Timeout) @@ -239,7 +240,7 @@ func (self *Drive) UploadStream(args UploadStreamArgs) error { fmt.Fprintf(args.Out, "Uploading %s\n", dstFile.Name) started := time.Now() - f, err := self.service.Files.Create(dstFile).Fields("id", "name", "size", "webContentLink").Context(ctx).Media(reader, chunkSize).Do() + f, err := self.service.Files.Create(dstFile).SupportsAllDrives(true).Fields("id", "name", "size", "webContentLink").Context(ctx).Media(reader, chunkSize).Do() if err != nil { if isTimeoutError(err) { return fmt.Errorf("Failed to upload file: timeout, no data was transferred for %v", args.Timeout) From e79a23060ececd1ac9e64f9d3edcfbeeb5015e45 Mon Sep 17 00:00:00 2001 From: "jeff.yin" Date: Wed, 13 Nov 2019 14:09:05 +0800 Subject: [PATCH 3/7] do extra Query during recursive downlaod --- drive/download.go | 49 +++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/drive/download.go b/drive/download.go index 795d4b68..2ef4b1e3 100644 --- a/drive/download.go +++ b/drive/download.go @@ -12,16 +12,17 @@ import ( ) type DownloadArgs struct { - Out io.Writer - Progress io.Writer - Id string - Path string - Force bool - Skip bool - Recursive bool - Delete bool - Stdout bool - Timeout time.Duration + Out io.Writer + Progress io.Writer + Id string + Path string + Force bool + Skip bool + Recursive bool + Delete bool + Stdout bool + Timeout time.Duration + RecursiveExtraQuery string // add extra query } func (self *Drive) Download(args DownloadArgs) error { @@ -65,13 +66,14 @@ func (self *Drive) Download(args DownloadArgs) error { } type DownloadQueryArgs struct { - Out io.Writer - Progress io.Writer - Query string - Path string - Force bool - Skip bool - Recursive bool + Out io.Writer + Progress io.Writer + Query string + Path string + Force bool + Skip bool + Recursive bool + RecursiveExtraQuery string } func (self *Drive) DownloadQuery(args DownloadQueryArgs) error { @@ -85,11 +87,12 @@ func (self *Drive) DownloadQuery(args DownloadQueryArgs) error { } downloadArgs := DownloadArgs{ - Out: args.Out, - Progress: args.Progress, - Path: args.Path, - Force: args.Force, - Skip: args.Skip, + Out: args.Out, + Progress: args.Progress, + Path: args.Path, + Force: args.Force, + Skip: args.Skip, + RecursiveExtraQuery: args.RecursiveExtraQuery, } for _, f := range files { @@ -230,7 +233,7 @@ func (self *Drive) saveFile(args saveFileArgs) (int64, int64, error) { func (self *Drive) downloadDirectory(parent *drive.File, args DownloadArgs) error { fmt.Println("jeff_debug downloadDirectory") listArgs := listAllFilesArgs{ - query: fmt.Sprintf("'%s' in parents", parent.Id), + query: fmt.Sprintf("'%s' in parents %s", parent.Id, args.RecursiveExtraQuery), fields: []googleapi.Field{"nextPageToken", "files(id,name)"}, } files, err := self.listAllFiles(listArgs) From db14009b0b012dce9177e3c43149261fad79e881 Mon Sep 17 00:00:00 2001 From: "jeff.yin" Date: Mon, 18 Nov 2019 16:33:59 +0800 Subject: [PATCH 4/7] remove debug println --- drive/download.go | 3 --- drive/list.go | 3 --- 2 files changed, 6 deletions(-) diff --git a/drive/download.go b/drive/download.go index 2ef4b1e3..eb124f77 100644 --- a/drive/download.go +++ b/drive/download.go @@ -111,7 +111,6 @@ func (self *Drive) DownloadQuery(args DownloadQueryArgs) error { } func (self *Drive) downloadRecursive(args DownloadArgs) error { - fmt.Println("jeff_debug downloadRecursive") f, err := self.service.Files.Get(args.Id).SupportsAllDrives(true).Fields("id", "name", "size", "mimeType", "md5Checksum").Do() if err != nil { return fmt.Errorf("Failed to get file: %s", err) @@ -130,7 +129,6 @@ func (self *Drive) downloadRecursive(args DownloadArgs) error { func (self *Drive) downloadBinary(f *drive.File, args DownloadArgs) (int64, int64, error) { // Get timeout reader wrapper and context timeoutReaderWrapper, ctx := getTimeoutReaderWrapperContext(args.Timeout) - fmt.Println("jeff_debug downloadBinary") res, err := self.service.Files.Get(f.Id).SupportsAllDrives(true).Context(ctx).Download() if err != nil { if isTimeoutError(err) { @@ -231,7 +229,6 @@ func (self *Drive) saveFile(args saveFileArgs) (int64, int64, error) { } func (self *Drive) downloadDirectory(parent *drive.File, args DownloadArgs) error { - fmt.Println("jeff_debug downloadDirectory") listArgs := listAllFilesArgs{ query: fmt.Sprintf("'%s' in parents %s", parent.Id, args.RecursiveExtraQuery), fields: []googleapi.Field{"nextPageToken", "files(id,name)"}, diff --git a/drive/list.go b/drive/list.go index ff52ef58..98fad930 100644 --- a/drive/list.go +++ b/drive/list.go @@ -77,10 +77,7 @@ func (self *Drive) listAllFiles(args listAllFilesArgs) ([]*drive.File, error) { fmt.Println(args.query) err := self.service.Files.List().Corpora("allDrives").IncludeItemsFromAllDrives(true).SupportsAllDrives(true).Q(args.query).Fields(args.fields...).OrderBy(args.sortOrder).PageSize(pageSize).Pages(context.TODO(), func(fl *drive.FileList) error { files = append(files, fl.Files...) - fmt.Println("jeff_debug listAllFiles") - fmt.Println(fl.Files) - fmt.Println(files) // Stop when we have all the files we need if args.maxFiles > 0 && len(files) >= int(args.maxFiles) { return controlledStop From 488fe0175139b2adad90f9da53dd5fae805a4ef6 Mon Sep 17 00:00:00 2001 From: "jeff.yin" Date: Mon, 18 Nov 2019 18:01:04 +0800 Subject: [PATCH 5/7] support multidownload --- drive/download.go | 34 ++++++++++++++++++++++++---------- drive/drive.go | 10 +++++++--- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/drive/download.go b/drive/download.go index eb124f77..9551bb80 100644 --- a/drive/download.go +++ b/drive/download.go @@ -27,12 +27,14 @@ type DownloadArgs struct { func (self *Drive) Download(args DownloadArgs) error { if args.Recursive { - return self.downloadRecursive(args) + self.downloadRecursive(args) + self.waitGroup.Wait() + return self.downloadErr } f, err := self.service.Files.Get(args.Id).SupportsAllDrives(true).Fields("id", "name", "size", "mimeType", "md5Checksum").Do() if err != nil { - return fmt.Errorf("Failed to get file: %s", err) + return fmt.Errorf("Failed to get file: %s, err:", args.Id, err) } if isDir(f) { @@ -107,21 +109,33 @@ func (self *Drive) DownloadQuery(args DownloadQueryArgs) error { } } - return nil + self.waitGroup.Wait() + + return self.downloadErr } func (self *Drive) downloadRecursive(args DownloadArgs) error { f, err := self.service.Files.Get(args.Id).SupportsAllDrives(true).Fields("id", "name", "size", "mimeType", "md5Checksum").Do() if err != nil { - return fmt.Errorf("Failed to get file: %s", err) + return fmt.Errorf("Failed to get file: %s, err:", args.Id, err) } - if isDir(f) { - return self.downloadDirectory(f, args) - } else if isBinary(f) { - _, _, err = self.downloadBinary(f, args) - return err - } + self.waitGroup.Add(1) + go func() { + if isDir(f) { + err = self.downloadDirectory(f, args) + } else if isBinary(f) { + _, _, err = self.downloadBinary(f, args) + } + + if err != nil { + fmt.Errorf("%s \n %s", self.downloadErr, err.Error()) + //fmt.Println("Failed to download:", err) + } + + self.waitGroup.Done() + + }() return nil } diff --git a/drive/drive.go b/drive/drive.go index 696f5d55..48dd3f2c 100644 --- a/drive/drive.go +++ b/drive/drive.go @@ -1,12 +1,16 @@ package drive import ( - "google.golang.org/api/drive/v3" "net/http" + "sync" + + "google.golang.org/api/drive/v3" ) type Drive struct { - service *drive.Service + service *drive.Service + waitGroup sync.WaitGroup + downloadErr error } func New(client *http.Client) (*Drive, error) { @@ -15,5 +19,5 @@ func New(client *http.Client) (*Drive, error) { return nil, err } - return &Drive{service}, nil + return &Drive{service, sync.WaitGroup{}, nil}, nil } From e7749227e5421abf6898fe24ebf27dd2a268f7b9 Mon Sep 17 00:00:00 2001 From: "jeff.yin" Date: Tue, 19 Nov 2019 17:44:10 +0800 Subject: [PATCH 6/7] It's can set AsyncDownload or not --- drive/download.go | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/drive/download.go b/drive/download.go index 9551bb80..2d72c523 100644 --- a/drive/download.go +++ b/drive/download.go @@ -23,6 +23,7 @@ type DownloadArgs struct { Stdout bool Timeout time.Duration RecursiveExtraQuery string // add extra query + IsAsyncDownload bool } func (self *Drive) Download(args DownloadArgs) error { @@ -76,6 +77,7 @@ type DownloadQueryArgs struct { Skip bool Recursive bool RecursiveExtraQuery string + IsAsyncDownload bool } func (self *Drive) DownloadQuery(args DownloadQueryArgs) error { @@ -95,6 +97,7 @@ func (self *Drive) DownloadQuery(args DownloadQueryArgs) error { Force: args.Force, Skip: args.Skip, RecursiveExtraQuery: args.RecursiveExtraQuery, + IsAsyncDownload: args.IsAsyncDownload, } for _, f := range files { @@ -120,24 +123,31 @@ func (self *Drive) downloadRecursive(args DownloadArgs) error { return fmt.Errorf("Failed to get file: %s, err:", args.Id, err) } - self.waitGroup.Add(1) - go func() { - if isDir(f) { - err = self.downloadDirectory(f, args) - } else if isBinary(f) { - _, _, err = self.downloadBinary(f, args) - } - - if err != nil { - fmt.Errorf("%s \n %s", self.downloadErr, err.Error()) - //fmt.Println("Failed to download:", err) - } + if args.IsAsyncDownload { + self.waitGroup.Add(1) + go func() { + self.doDownloadRecursive(f, args) + self.waitGroup.Done() + }() + } else { + self.doDownloadRecursive(f, args) + } - self.waitGroup.Done() + return nil +} - }() +func (self *Drive) doDownloadRecursive(f *drive.File, args DownloadArgs) { + var err error + if isDir(f) { + err = self.downloadDirectory(f, args) + } else if isBinary(f) { + _, _, err = self.downloadBinary(f, args) + } - return nil + if err != nil { + fmt.Errorf("%s \n %s", self.downloadErr, err.Error()) + //fmt.Println("Failed to download:", err) + } } func (self *Drive) downloadBinary(f *drive.File, args DownloadArgs) (int64, int64, error) { From 75981bf78a00fffd90cd12c477b44771018b280e Mon Sep 17 00:00:00 2001 From: "jeff.yin" Date: Mon, 25 Nov 2019 13:59:39 +0800 Subject: [PATCH 7/7] It can set limit of per second download --- drive/download.go | 23 +++++++++++++++++++++++ drive/drive.go | 16 ++++++++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/drive/download.go b/drive/download.go index 2d72c523..6fc2ba45 100644 --- a/drive/download.go +++ b/drive/download.go @@ -3,6 +3,7 @@ package drive import ( "fmt" "io" + "math" "os" "path/filepath" "time" @@ -24,10 +25,12 @@ type DownloadArgs struct { Timeout time.Duration RecursiveExtraQuery string // add extra query IsAsyncDownload bool + LimitPerSec int64 } func (self *Drive) Download(args DownloadArgs) error { if args.Recursive { + self.ResetDownloadTime() self.downloadRecursive(args) self.waitGroup.Wait() return self.downloadErr @@ -78,6 +81,7 @@ type DownloadQueryArgs struct { Recursive bool RecursiveExtraQuery string IsAsyncDownload bool + LimitPerSec int64 } func (self *Drive) DownloadQuery(args DownloadQueryArgs) error { @@ -98,8 +102,11 @@ func (self *Drive) DownloadQuery(args DownloadQueryArgs) error { Skip: args.Skip, RecursiveExtraQuery: args.RecursiveExtraQuery, IsAsyncDownload: args.IsAsyncDownload, + LimitPerSec: args.LimitPerSec, } + self.ResetDownloadTime() + for _, f := range files { if isDir(f) && args.Recursive { err = self.downloadDirectory(f, downloadArgs) @@ -118,6 +125,22 @@ func (self *Drive) DownloadQuery(args DownloadQueryArgs) error { } func (self *Drive) downloadRecursive(args DownloadArgs) error { + self.downlaodCount++ + perid := time.Now().Unix() - self.downloadStartUnix + if perid < 1 { + perid = 1 + } + limit := perid * args.LimitPerSec + if limit < 0 { + limit = math.MaxInt64 + } + for self.downlaodCount > limit { + gap := self.downlaodCount - limit + time.Sleep(time.Duration(gap) * time.Second) + perid = time.Now().Unix() - self.downloadStartUnix + limit = perid * args.LimitPerSec + } + f, err := self.service.Files.Get(args.Id).SupportsAllDrives(true).Fields("id", "name", "size", "mimeType", "md5Checksum").Do() if err != nil { return fmt.Errorf("Failed to get file: %s, err:", args.Id, err) diff --git a/drive/drive.go b/drive/drive.go index 48dd3f2c..3bf0270f 100644 --- a/drive/drive.go +++ b/drive/drive.go @@ -3,14 +3,17 @@ package drive import ( "net/http" "sync" + "time" "google.golang.org/api/drive/v3" ) type Drive struct { - service *drive.Service - waitGroup sync.WaitGroup - downloadErr error + service *drive.Service + downloadStartUnix int64 + downlaodCount int64 + waitGroup sync.WaitGroup + downloadErr error } func New(client *http.Client) (*Drive, error) { @@ -19,5 +22,10 @@ func New(client *http.Client) (*Drive, error) { return nil, err } - return &Drive{service, sync.WaitGroup{}, nil}, nil + return &Drive{service, 0, 0, sync.WaitGroup{}, nil}, nil +} + +func (d *Drive) ResetDownloadTime() { + d.downloadStartUnix = time.Now().Unix() + d.downlaodCount = 0 }