Update to blobfuse 2.3.0 release

.github/workflows/code-coverage.yml

@@ -89,6 +89,7 @@ jobs:
         echo "\"spn-tenant\"": "\"${{ secrets.AZTEST_TENANT }}\"", >> $cnfFile
         echo "\"spn-secret\"": "\"${{ secrets.AZTEST_SECRET }}\"", >> $cnfFile
         echo "\"skip-msi\"": "true", >> $cnfFile
+        echo "\"skip-azcli\"": "true", >> $cnfFile
         echo "\"proxy-address\"": "\"\"" >> $cnfFile
         echo "}" >> $cnfFile
     
@@ -646,6 +647,7 @@ jobs:
           "spn-tenant": "${{ secrets.AZTEST_TENANT }}",
           "spn-secret": "${{ secrets.AZTEST_SECRET }}",
           "skip-msi": true,
+          "skip-azcli": true,
           "proxy-address": ""
         }
         "@

.github/workflows/unit-test.yml

@@ -71,6 +71,7 @@ jobs:
           echo "\"block-key\"": "\"${{ env.BLOB_KEY }}\"", >> $cnfFile
           echo "\"endpoint\"": "\"${{ env.BLOB_ENDPOINT }}\"", >> $cnfFile
           echo "\"skip-msi\"": "true", >> $cnfFile
+          echo "\"skip-azcli\"": "true", >> $cnfFile
           echo "\"proxy-address\"": "\"\"" >> $cnfFile
           echo "}" >> $cnfFile
 
@@ -179,7 +180,7 @@ jobs:
           version: latest
 
           # Optional: golangci-lint command line arguments.
-          args: --tests=false --timeout=5m --skip-dirs test,common/stats_collector,common/stats_monitor --skip-files component/libfuse/libfuse2_handler_test_wrapper.go --out-format=github-actions
+          args: --tests=false --timeout=5m --exclude-dirs test,common/stats_collector,common/stats_monitor --exclude-files component/libfuse/libfuse2_handler_test_wrapper.go --out-format=github-actions
 
       - name: Notice file check
         run: |

CHANGELOG.md

@@ -68,7 +68,6 @@ This version is based on [blobfuse2 2.2.0](https://github.com/Azure/azure-storag
 - improved performance of directory listing
 - merged upstream version 2.2.0
 
-
 ## **1.0.1** ##
 
 January 19th 2024

TSG.md

@@ -3,13 +3,10 @@
 Please ensure logging is turned on DEBUG mode when trying to reproduce an issue.
 This can help in many instances to understand what the underlying issue is.
 
-A useful setting in your configuration file to utilize when debugging is `sdk-trace: true` under the azstorage component. This will log all outgoing REST calls.
-
 # Cloudfuse Health Monitor
 
 The health monitor customers gain more insight into how their Cloudfuse instance is behaving with the rest of their machine. Visit [here](tools/health-monitor/README.md) to set it up.
 
-
 # Common Mount Problems
 
 **1. Error: fusermount: failed to open /etc/fuse.conf: Permission denied**
@@ -26,15 +23,17 @@ If are you using 'allow-other: true' config then make sure `user_allow_other` is
 There might be something wrong about the storage config, please double check the storage account name, account key and container/filesystem name. errno = 1**
 
 Possible causes are:
+
 - Invalid account, or access key
-- Non-existing container (The container must be created prior to Cloudfuse mount) 
+- Non-existing container (The container must be created prior to Cloudfuse mount)
 - Windows line-endings (CRLF) - fix it by running dos2unix
 - Use of HTTP while 'Secure Transfer (HTTPS)' is enabled on a Storage account
 - Enabled VNET Security rule that blocks VM from connecting to the Storage account. Ensure you can connect to your Storage account using AzCopy or Azure CLI
 - DNS issues/timeouts - add the Storage account resolution to /etc/hosts to bypass the DNS lookup
 - If using a proxy endpoint - ensure that you use the correct transfer protocol HTTP vs HTTPS
 
 **4. For MSI or SPN auth, Http Status Code = 403 in the response. Authorization error**
+
 - Verify your storage account Access roles. Make sure you have both Contributor and Storage Blob Contributor roles for the MSI or SPN identity.
 - In the case of a private AAD endpoint (private MSI endpoitns) ensure that your env variables are configured correctly.
 
@@ -51,11 +50,10 @@ FUSE allows mounting filesystem in user space, and is only accessible by the use
 
     sudo cloudfuse2 mount /home/myuser/mount_dir/ --config-file=config.yaml --allow-other
 
-
 **7. fusermount: command not found**
 
 You try to unmount the blob storage, but the recommended command is not found. Whilst `umount` may work instead, fusermount is the recommended method, so install the fuse package, for example on Ubuntu 20+:
-    
+
     sudo apt install fuse3
 please note the fuse version (2 or 3) is dependent on the linux distribution you're using. Refer to fuse version for your distro.
 
@@ -79,7 +77,7 @@ b) DNS setting cannot be set inside Guest OS VM NIC.
 
 For Custom DNS server defined check the following:
 
-Custom DNS Server forwards all requests to 168.63.129.16 
+Custom DNS Server forwards all requests to 168.63.129.16
 
 Yes – you should be able to consume Azure Private DNS zones correctly.
 
@@ -91,31 +89,37 @@ a) DNS has Root Hits only – In this case is the best to have a forwarder confi
 
 b) DNS Forwarders to another DNS Server (not Azure Provided DNS) – In this case you need to create a conditional forwarder to original PaaS domain zone (i.e. Storage you should configure blob.core.windows.net conditional forwarder to 168.63.129.16). Keep in mind using that approach will make all DNS requests to storage account with or without private endpoint to be resolved by Azure Provided DNS. By having multiple Custom DNS Serves in Azure will help to get better high availability for requests coming from On-Prem.
 
-
 **9. Cloudfuse killed by OOM**
 
 The "OOM Killer" or "Out of Memory Killer" is a process that the Linux kernel employs when the system is critically low on memory. Based on its algorithm it kills one or more process to free up some memory space. Cloudfuse could be one such process. To investigate Cloudfuse was killed by OOM or not run following command:
 
-``` dmesg -T | egrep -i 'killed process'```
+```dmesg -T | egrep -i 'killed process'```
 
 If Cloudfuse pid is listed in the output then OOM has sent a SIGKILL to Cloudfuse. If Cloudfuse was not running as a service it will not restart automatically and user has to manually mount again. If this keeps happening then user need to monitor the system and investigate why system is getting low on memory. VM might need an upgrade here if the such high usage is expected.
 
-
 **10. Unable to access HNS enabled storage account behind a private end point**
 
 For HNS account, always add `type: adls` under `azstorage` section in your config file. Avoid using `endpoint` unless your storage account is behind a private endpoint. Cloudfuse uses both blob and dfs endpoints to connect to storage account. User has to expose both these endpoints over private-endpoint for Cloudfuse to function properly.
 
 To create a private-endpoint for DFS in Azure portal: Go to your storage account -> Networking -> Private Endpoint connections. Click `+ Private endpoint`, fill in Subscription, Resource Group, Name, Network Interface Name and Region. Click next and under Target sub-resource select `dfs`. Click Virtual network and select virtual network and Subnet. Click DNS. Select Yes for Integrate with private DNS. Select the Subscription and Resource Group for your private link DNS. Select Next, Next and select Create.
 
+**11. Failed to initialize new pipeline [config error in azstorage [account name not provided]]'**
+
+Make sure the configuration file has `azstorage` section in your config file.
+
+The [Cloudfuse base configuration file](https://github.com/Seagate/cloudfuse/blob/main/setup/baseConfig.yaml) contains a list of all settings and a brief explanation of each setting. Use the [sample file cache configuration file](https://github.com/Seagate/cloudfuse/blob/main/sampleFileCacheConfigAzure.yaml) or the [sample block cache configuration file](https://github.com/Seagate/cloudfuse/blob/main/sampleBlockCacheConfig.yaml) to get started quickly by using some basic settings for each of those scenarios.
+
 # Common Problems after a Successful Mount
+
 **1. Errno 24: Failed to open file /mnt/tmp/root/filex in file cache.  errno = 24 OR Too many files Open error**
-Errno 24 in Linux corresponds to 'Too many files open' error which can occur when an application opens more files than it is allowed on the system. Cloudfuse typically allows 20 files less than the ulimit value set in Linux. Usually the Linux limit is 1024 per process (e.g. Cloudfuse in this case will allow 1004 open file descriptors at a time). Recommended approach is to edit the /etc/security/limits.conf in Ubuntu and add these two lines, 
-* soft nofile 16384
-* hard nofile 16384
+Errno 24 in Linux corresponds to 'Too many files open' error which can occur when an application opens more files than it is allowed on the system. Cloudfuse typically allows 20 files less than the ulimit value set in Linux. Usually the Linux limit is 1024 per process (e.g. Cloudfuse in this case will allow 1004 open file descriptors at a time). Recommended approach is to edit the /etc/security/limits.conf in Ubuntu and add these two lines,
+
+- soft nofile 16384
+- hard nofile 16384
 
 16384 here refers to the number of allowed open files
-you must reboot after editing this file for Cloudfuse to pick up the new limits. You may increase the limit via the command `ulimit -n 16834` however this does not appear in work in Ubuntu. 
- 
+you must reboot after editing this file for Cloudfuse to pick up the new limits. You may increase the limit via the command `ulimit -n 16834` however this does not appear in work in Ubuntu.
+
 **2. Input/output error**
 If you mounted a Blob container successfully, but failed to create a directory, or upload a file, it may be that you mounted a Blob container from a Premium (Page) Blob account which does not support Block blob. Cloudfuse uses Block Blobs as files hence requires accounts that support Block blobs.
 
@@ -147,6 +151,7 @@ Below are the steps to automate this at pod creation:
 1.Create a new configmap in the cluster which contains the new configuration about the script.
 
 2.Create a DaemonSet with the new configmap which could apply the configuration changes to every node in the cluster.
+
 ```
 Example:
 configmap fiie: (testcm.yaml)
@@ -202,36 +207,34 @@ hostPath:
 path: /etc
 type: Directory
 ```
-**4. File contents are not in sync with storage** 
 
-Please refer to the file cache component setting `timeout-sec`.
+**4. File contents are not in sync with storage**
 
+Please refer to the file cache component setting `timeout-sec`.
 
 **5. failed to unmount /path/<mount dir>**
 
 Unmount fails when a file is open or a user or process is cd'd into the mount directory or its sub directories. Please ensure no files are in use and try the unmount command again. Even umount -f will not work if the mounted files /directories are in use.
 umount -l does a lazy unmount meaning it will unmount automatically when the mounted files are no longer in use.
 
-**6. Cloudfuse mounts but not functioning at all** 
+**6. Cloudfuse mounts but not functioning at all**
 
-https://github.com/Azure/azure-storage-fuse/issues/803
+<https://github.com/Azure/azure-storage-fuse/issues/803>
 There are cases where anti-malware / anti-virus software block the fuse functionality and in such case though mount command is successful and Cloudfuse binary is running, the fuse functionality will not work. One way to identify that you are hitting this issue is turn on the debug logs and mount Cloudfuse. If you do not see any logs coming from Cloudfuse and potentially you have run into this issue. Stop the anti-virus software and try again.
 In such cases we have seen mounting through /etc/fstab works, because that executes mount command before the anti-malware software kicks in.
 
 **7. file cache temp directory not empty**
 
 To ensure that you don't have leftover files in your file cache temp dir, unmount rather than killing
-Cloudfuse. If Cloudfuse is killed without unmounting you can also set `cleanup-on-start` in your config file on the next mount to clear the temp dir. 
-
+Cloudfuse. If Cloudfuse is killed without unmounting you can also set `cleanup-on-start` in your config file on the next mount to clear the temp dir.
 
 **8. Unable to modify existing file (error: invalid argument)**
 <!-- Uncomment when cgofuse supports fuse3
 By default `writeback-cache` is enabled for libfuse3 and this may result in append/write operations to fail. Either you can disable writeback-cache, which might hurt the performance or you can configure Cloudfuse to ignore open flags given by user and make it work with writeback-cache.
 -->
 To disable writeback-cache : Add `disable-writeback-cache: true` under libfuse section in your config file.
 
-To make it work with writeback-cache : Add `ignore-open-flags: true` under libfuse section in your config file. 
-
+To make it work with writeback-cache : Add `ignore-open-flags: true` under libfuse section in your config file.
 
 **9. Unable to list files/directories for non-HNS (flat-namespace) accounts**
 
@@ -259,7 +262,7 @@ If your workflow involves updating the file directly on container (not using Clo
 -->
 
 # Problems with build
+
 Make sure you have correctly setup your GO dev environment. Ensure you have installed fuse2 for example:
 
     sudo apt-get install fuse libfuse-dev -y
-

cmd/doc_test.go

@@ -67,7 +67,7 @@ func (suite *docTestSuite) TestDocsGeneration() {
 	suite.assert.NoError(err)
 	files, err := os.ReadDir(opDir)
 	suite.assert.NoError(err)
-	suite.assert.NotZero(len(files))
+	suite.assert.NotEmpty(files)
 }
 
 func (suite *docTestSuite) TestOutputDirCreationError() {

cmd/mount.go

@@ -79,6 +79,7 @@ type mountOptions struct {
 	ProfilerIP        string         `config:"profiler-ip"`
 	MonitorOpt        monitorOptions `config:"health_monitor"`
 	WaitForMount      time.Duration  `config:"wait-for-mount"`
+	LazyWrite         bool           `config:"lazy-write"`
 
 	// v1 support
 	Streaming      bool     `config:"streaming"`
@@ -246,6 +247,8 @@ var mountCmd = &cobra.Command{
 	RunE: func(_ *cobra.Command, args []string) error {
 		options.MountPath = common.ExpandPath(args[0])
 		configFileProvided := options.ConfigFile != ""
+		common.MountPath = options.MountPath
+
 		configFileExists := true
 
 		if options.ConfigFile == "" {
@@ -446,6 +449,7 @@ var mountCmd = &cobra.Command{
 		var pipeline *internal.Pipeline
 
 		log.Crit("Starting Cloudfuse Mount : %s on [%s]", common.CloudfuseVersion, common.GetCurrentDistro())
+		log.Info("Mount Command: %s", os.Args)
 		log.Crit("Logging level set to : %s", logLevel.String())
 		log.Debug("Mount allowed on nonempty path : %v", options.NonEmpty)
 
@@ -678,6 +682,9 @@ func init() {
 		"Test mount configuration, credentials, etc., but don't make any changes to the container or the local file system. Implies foreground.")
 	config.BindPFlag("dry-run", mountCmd.Flags().Lookup("dry-run"))
 
+	mountCmd.PersistentFlags().Bool("lazy-write", false, "Async write to storage container after file handle is closed.")
+	config.BindPFlag("lazy-write", mountCmd.PersistentFlags().Lookup("lazy-write"))
+
 	mountCmd.PersistentFlags().String("default-working-dir", "", "Default working directory for storing log files and other cloudfuse information")
 	mountCmd.PersistentFlags().Lookup("default-working-dir").Hidden = true
 	config.BindPFlag("default-working-dir", mountCmd.PersistentFlags().Lookup("default-working-dir"))

cmd/root_test.go

@@ -159,6 +159,8 @@ func (suite *rootCmdSuite) TestGetRemoteVersionCurrentSame() {
 }
 
 func (suite *rootCmdSuite) testExecute() {
+	suite.T().Helper()
+
 	defer suite.cleanupTest()
 	buf := new(bytes.Buffer)
 	rootCmd.SetOut(buf)

common/log/base_logger.go

@@ -215,10 +215,11 @@ func (l *BaseLogger) logEvent(lvl string, format string, args ...interface{}) {
 	// Only log if the log level matches the log request
 	_, fn, ln, _ := runtime.Caller(3)
 	msg := fmt.Sprintf(format, args...)
-	msg = fmt.Sprintf("%s : %s[%d] : %s [%s (%d)]: %s",
+	msg = fmt.Sprintf("%s : %s[%d] : [%s] %s [%s (%d)]: %s",
 		time.Now().Format(unixDateMilli),
 		l.fileConfig.LogTag,
 		l.procPID,
+		common.MountPath,
 		lvl,
 		filepath.Base(fn), ln,
 		msg)

common/log/sys_logger_linux.go

@@ -114,7 +114,7 @@ func getSyslogLevel(lvl common.LogLevel) syslog.Priority {
 func (l *SysLogger) write(lvl string, format string, args ...interface{}) {
 	_, fn, ln, _ := runtime.Caller(3)
 	msg := fmt.Sprintf(format, args...)
-	l.logger.Print(lvl, " [", filepath.Base(fn), " (", ln, ")]: ", msg)
+	l.logger.Print("[", common.MountPath, "] ", lvl, " [", filepath.Base(fn), " (", ln, ")]: ", msg)
 }
 
 func (l *SysLogger) Debug(format string, args ...interface{}) {

common/types.go

@@ -62,6 +62,8 @@ const (
 	CfuseStats = "cloudfuse_stats"
 
 	FuseAllowedFlags = "invalid FUSE options. Allowed FUSE configurations are: `-o attr_timeout=TIMEOUT`, `-o negative_timeout=TIMEOUT`, `-o entry_timeout=TIMEOUT` `-o allow_other`, `-o allow_root`, `-o umask=PERMISSIONS -o default_permissions`, `-o ro`"
+
+	UserAgentHeader = "User-Agent"
 )
 
 var GitCommit = "**local_build**"
@@ -94,6 +96,8 @@ func GetDefaultWorkDir() string {
 	return val
 }
 
+var MountPath string
+
 // LogLevel enum
 type LogLevel int
 

common/version.go

@@ -40,26 +40,26 @@ type Version struct {
 }
 
 // To keep the code simple, we assume we only use a simple subset of semantic versions.
-// Namely, the version is either a normal stable version, or a pre-release version with '-preview' attached.
-// Examples: 10.1.0, 11.2.0-preview.1
+// Namely, the version is either a normal stable version, or a pre-release version with '~preview' or '-preview' attached.
+// Examples: 10.1.0, 11.2.0-preview.1, 11.2.0~preview.1
 func ParseVersion(raw string) (*Version, error) {
 	const standardError = "invalid version string"
 
 	rawSegments := strings.Split(raw, ".")
-	if !(len(rawSegments) == 3 || (len(rawSegments) == 4 && strings.Contains(rawSegments[2], "-"))) {
+	if !(len(rawSegments) == 3 || (len(rawSegments) == 4 && (strings.Contains(rawSegments[2], "-") || strings.Contains(rawSegments[2], "~")))) {
 		return nil, errors.New(standardError)
 	}
 
 	v := &Version{segments: make([]int64, 4), original: raw}
 	for i, str := range rawSegments {
 		//For any case such as SemVer-preview.1, SemVer-beta.1, SemVer-alpha.1 this would be true, and we assume the version to be a preview version.
-		if strings.Contains(str, "-") {
+		if strings.Contains(str, "-") || strings.Contains(str, "~") {
 			if i != 2 {
 				return nil, errors.New(standardError)
 			}
 			v.preview = true
 			//Splitting the string into two pieces and extracting SemVer which is always at 0th index
-			str = strings.Split(str, "-")[0]
+			str = strings.Split(strings.Split(str, "-")[0], "~")[0]
 		}
 
 		val, err := strconv.ParseInt(str, 10, 64)

common/version_test.go

@@ -58,6 +58,14 @@ func (vSuite *versionTestSuite) TestVersionEquality() {
 	v1, _ = ParseVersion("10.0.0-beta.5")
 	v2, _ = ParseVersion("10.0.0-beta.5")
 	assert.Equal(0, v1.compare(*v2))
+
+	v1, _ = ParseVersion("10.0.0~preview.1")
+	v2, _ = ParseVersion("10.0.0~preview.1")
+	assert.Equal(0, v1.compare(*v2))
+
+	v1, _ = ParseVersion("10.0.0~beta.5")
+	v2, _ = ParseVersion("10.0.0~beta.5")
+	assert.Equal(0, v1.compare(*v2))
 }
 
 func (vSuite *versionTestSuite) TestVersionSuperiority() {
@@ -82,6 +90,18 @@ func (vSuite *versionTestSuite) TestVersionSuperiority() {
 	v1, _ = ParseVersion("15.5.5-preview.6")
 	v2, _ = ParseVersion("15.5.5-preview.3")
 	assert.Equal(1, v1.compare(*v2))
+
+	v1, _ = ParseVersion("15.5.6")
+	v2, _ = ParseVersion("15.5.6~preview.3")
+	assert.Equal(1, v1.compare(*v2))
+
+	v1, _ = ParseVersion("15.5.6~preview.6")
+	v2, _ = ParseVersion("15.5.6~preview.3")
+	assert.Equal(1, v1.compare(*v2))
+
+	v1, _ = ParseVersion("15.5.7~preview.6")
+	v2, _ = ParseVersion("15.5.7-preview.3")
+	assert.Equal(1, v1.compare(*v2))
 }
 
 func (vSuite *versionTestSuite) TestVersionInferiority() {
@@ -106,6 +126,18 @@ func (vSuite *versionTestSuite) TestVersionInferiority() {
 	v1, _ = ParseVersion("15.5.5-preview.3")
 	v2, _ = ParseVersion("15.5.5-preview.6")
 	assert.Equal(v1.compare(*v2), -1)
+
+	v1, _ = ParseVersion("15.5.6~preview.6")
+	v2, _ = ParseVersion("15.5.6")
+	assert.Equal(v1.compare(*v2), -1)
+
+	v1, _ = ParseVersion("15.5.6~preview.3")
+	v2, _ = ParseVersion("15.5.6~preview.6")
+	assert.Equal(v1.compare(*v2), -1)
+
+	v1, _ = ParseVersion("15.5.7-preview.3")
+	v2, _ = ParseVersion("15.5.7~preview.6")
+	assert.Equal(v1.compare(*v2), -1)
 }
 
 func TestVersionTestSuite(t *testing.T) {