Final cleanup of config and small helper function

component/s3storage/client.go

@@ -80,6 +80,16 @@ var (
 	errNoBucketInAccount  = errors.New("Bucket Error: No bucket exists in S3 account. Please create a bucket in your account.")
 )
 
+// getSymlinkBool returns true if the symlink flag is set in the metadata map, false otherwise.
+func getSymlinkBool(metadata map[string]*string) bool {
+	isSymlink := false
+	sym, ok := metadata[symlinkKey]
+	if ok && sym != nil {
+		isSymlink = (*sym == "true")
+	}
+	return isSymlink
+}
+
 // Configure : Initialize the awsS3Client
 func (cl *Client) Configure(cfg Config) error {
 	log.Trace("Client::Configure : initialize awsS3Client")
@@ -595,12 +605,8 @@ func (cl *Client) ReadInBuffer(name string, offset int64, length int64, data []b
 // Upload from a file handle to an object.
 // The metadata parameter is not used.
 func (cl *Client) WriteFromFile(name string, metadata map[string]*string, fi *os.File) error {
-	isSymlink := false
-	sym, ok := metadata[symlinkKey]
-	if ok && sym != nil {
-		isSymlink = (*sym == "true")
+	isSymlink := getSymlinkBool(metadata)
 
-	}
 	log.Trace("Client::WriteFromFile : file %s -> name %s", fi.Name(), name)
 	// track time for performance testing
 	defer log.TimeTrack(time.Now(), "Client::WriteFromFile", name)
@@ -646,11 +652,7 @@ func (cl *Client) WriteFromFile(name string, metadata map[string]*string, fi *os
 // name is the file path.
 func (cl *Client) WriteFromBuffer(name string, metadata map[string]*string, data []byte) error {
 	log.Trace("Client::WriteFromBuffer : name %s", name)
-	isSymlink := false
-	sym, ok := metadata[symlinkKey]
-	if ok && sym != nil {
-		isSymlink = (*sym == "true")
-	}
+	isSymlink := getSymlinkBool(metadata)
 
 	// convert byte array to io.Reader
 	dataReader := bytes.NewReader(data)
@@ -772,11 +774,8 @@ func (cl *Client) Write(options internal.WriteFileOptions) error {
 		// case 1: file consists of no parts (small file)
 
 		// get the existing object data
-		isSymlink := false
-		sym, ok := options.Metadata[symlinkKey]
-		if ok && sym != nil {
-			isSymlink = (*sym == "true")
-		}
+		isSymlink := getSymlinkBool(options.Metadata)
+
 		oldData, _ := cl.ReadBuffer(name, 0, 0, isSymlink)
 		// update the data with the new data
 		// if we're only overwriting existing data

setup/baseConfig.yaml

@@ -32,6 +32,7 @@
 foreground: true|false <run cloudfuse in foreground or background>
 
 # Common configurations
+read-only: true|false <mount in read only mode - used for Streaming and FUSE>
 allow-other: true|false <allow other users to access the mounted directory - used for FUSE and File Cache>
 nonempty: true|false <allow mounting on non-empty directory>
 restricted-characters-windows: true|false <allows filenames with restricted characters to appear on Windows - used for AzStorage and S3Storage>
@@ -151,10 +152,20 @@ azstorage:
   max-concurrency: <number of parallel upload/download threads. Default - 32>
   tier: hot|cool|cold|premium|archive|none <blob-tier to be set while uploading a blob. Default - none>
   block-list-on-mount-sec: <time list api to be blocked after mount (in sec). Default - 0 sec>
+  max-retries: <number of retries to attempt for any operation failure. Default - 5>
+  max-retry-timeout-sec: <maximum timeout allowed for a given retry (in sec). Default - 900 sec>
+  retry-backoff-sec: <retry backoff between two tries (in sec). Default - 4 sec>
+  max-retry-delay-sec: <maximum delay between two tries (in sec). Default - 60 sec>
+  http-proxy: ip-address:port <http proxy to be used for connection>
+  https-proxy: ip-address:port <https proxy to be used for connection>
+  fail-unsupported-op: true|false <for block blob account return failure for unsupported operations like chmod and chown>
   auth-resource: <resource string to be used during OAuth token retrieval>
   update-md5: true|false <set md5 sum on upload. Impacts performance. works only when file-cache component is part of the pipeline>
   validate-md5: true|false <validate md5 on download. Impacts performance. works only when file-cache component is part of the pipeline>
+  virtual-directory: true|false <support virtual directories without existence of a special marker blob. Default - true>
   disable-compression: true|false <disable transport layer content encoding like gzip, set this flag to true if blobs have content-encoding set in container>
+  max-results-for-list: <maximum number of results returned in a single list API call while getting file attributes. Default - 2>
+  telemetry : <additional information that customer want to push in user-agent>
   honour-acl: true|false <honour ACLs on files and directories when mounted using MSI Auth and object-ID is provided in config>
   cpk-enabled: true|false <enable client provided key encryption>
   cpk-encryption-key: <customer provided base64-encoded AES-256 encryption key value>

setup/devConfig.yaml

@@ -165,7 +165,9 @@ azstorage:
   auth-resource: <resource string to be used during OAuth token retrieval>
   update-md5: true|false <set md5 sum on upload. Impacts performance. works only when file-cache component is part of the pipeline>
   validate-md5: true|false <validate md5 on download. Impacts performance. works only when file-cache component is part of the pipeline>
+  virtual-directory: true|false <support virtual directories without existence of a special marker blob. Default - true>
   disable-compression: true|false <disable transport layer content encoding like gzip, set this flag to true if blobs have content-encoding set in container>
+  max-results-for-list: <maximum number of results returned in a single list API call while getting file attributes. Default - 2>
   telemetry : <additional information that customer want to push in user-agent>
   honour-acl: true|false <honour ACLs on files and directories when mounted using MSI Auth and object-ID is provided in config>
   cpk-enabled: true|false <enable client provided key encryption>
@@ -174,9 +176,7 @@ azstorage:
 
 # S3 storage configuration
 s3storage:
-  # Required
   bucket-name: <name of the bucket to be mounted>
-  # Optional
   key-id: <S3 access key ID. Default - use credentials from environment variables or shared profile (see README)>
   secret-key: <S3 secret access key. Default - use credentials from environment variables or shared profile (see README)>
   region: <S3 region. Default - us-east-1>