From 86ef8841c7295a1d05f5958378b68419a8565253 Mon Sep 17 00:00:00 2001 From: Grzegorz Lukasik <32371993+hauserx@users.noreply.github.com> Date: Mon, 2 Dec 2024 20:05:38 +0100 Subject: [PATCH] Use lock to fix issue with parallel invocations (#638) Fixes issues with errors like under Linux: ``` Opening zip "/(...)/downloads/sha256/c97f(...)/bin/bazel (deleted)": open(): No such file or directory FATAL: Failed to open '/(...)/downloads/sha256/c97f(...)/bin/bazel (deleted)' as a zip file: (error: 2): No such file or directory ``` and below ones under Windows: ``` could not download Bazel: failed to download bazel: failed to move (...) (...)\bin\bazel.exe: Access is denied. ``` See: https://github.com/bazelbuild/bazelisk/issues/436 Tested using `parallel_bazelisk_bug.go` script under Linux and Windows that this change fixes issue. --- MODULE.bazel | 1 + core/BUILD | 1 + core/core.go | 33 ++++++++++++++++++++++++++++++++- go.mod | 1 + go.sum | 10 ++++++++++ 5 files changed, 45 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 841ac99d..cee279bd 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -19,6 +19,7 @@ go_deps.from_file(go_mod = "//:go.mod") use_repo( go_deps, "com_github_bgentry_go_netrc", + "com_github_gofrs_flock", "com_github_hashicorp_go_version", "com_github_mitchellh_go_homedir", "org_golang_x_term", diff --git a/core/BUILD b/core/BUILD index 32940a8e..18efeafb 100644 --- a/core/BUILD +++ b/core/BUILD @@ -15,6 +15,7 @@ go_library( "//platforms", "//versions", "//ws", + "@com_github_gofrs_flock//:flock", "@com_github_mitchellh_go_homedir//:go-homedir", ], ) diff --git a/core/core.go b/core/core.go index 8b9eb7b2..1a150615 100644 --- a/core/core.go +++ b/core/core.go @@ -5,6 +5,7 @@ package core import ( "bufio" + "context" "crypto/rand" "crypto/sha256" "encoding/json" @@ -21,12 +22,14 @@ import ( "sort" "strings" "syscall" + "time" "github.com/bazelbuild/bazelisk/config" "github.com/bazelbuild/bazelisk/httputil" "github.com/bazelbuild/bazelisk/platforms" "github.com/bazelbuild/bazelisk/versions" "github.com/bazelbuild/bazelisk/ws" + "github.com/gofrs/flock" "github.com/mitchellh/go-homedir" ) @@ -451,6 +454,34 @@ func atomicWriteFile(path string, contents []byte, perm os.FileMode) error { return nil } +// lockedRenameIfDstAbsent executes os.Rename under file lock to avoid issues +// of multiple bazelisk processes renaming file to the same destination file. +// See https://github.com/bazelbuild/bazelisk/issues/436. +func lockedRenameIfDstAbsent(src, dst string) error { + lockFile := dst + ".lock" + fileLock := flock.New(lockFile) + + // Do not wait for lock forever to avoid hanging in any scenarios. This + // makes the lock best-effort. + lockCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + ok, err := fileLock.TryLockContext(lockCtx, 50*time.Millisecond) + if !ok || err != nil { + log.Printf("WARNING: Unable to create lock during rename to %s, this may cause issues for parallel bazel executions: %s\n", lockFile, err) + } else { + defer func() { + _ = fileLock.Unlock() + }() + } + + if _, err := os.Stat(dst); err == nil { + return nil + } + + return os.Rename(src, dst) +} + func downloadBazelToCAS(version string, bazeliskHome string, repos *Repositories, config config.Config, downloader DownloadFunc) (string, string, error) { downloadsDir := filepath.Join(bazeliskHome, "downloads") temporaryDownloadDir := filepath.Join(downloadsDir, "_tmp") @@ -509,7 +540,7 @@ func downloadBazelToCAS(version string, bazeliskHome string, repos *Repositories if err := os.Rename(tmpDestPath, tmpPathInCorrectDirectory); err != nil { return "", "", fmt.Errorf("failed to move %s to %s: %w", tmpDestPath, tmpPathInCorrectDirectory, err) } - if err := os.Rename(tmpPathInCorrectDirectory, pathToBazelInCAS); err != nil { + if err := lockedRenameIfDstAbsent(tmpPathInCorrectDirectory, pathToBazelInCAS); err != nil { return "", "", fmt.Errorf("failed to move %s to %s: %w", tmpPathInCorrectDirectory, pathToBazelInCAS, err) } diff --git a/go.mod b/go.mod index 92540ba8..2c67a74b 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.22 require ( github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d + github.com/gofrs/flock v0.12.1 github.com/hashicorp/go-version v1.7.0 github.com/mitchellh/go-homedir v1.1.0 golang.org/x/term v0.26.0 diff --git a/go.sum b/go.sum index ad8f4f8a..5a88d84f 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,20 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= +github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=