From ed042116a70761ec204b5dcfaf973d42d1b3aae3 Mon Sep 17 00:00:00 2001 From: Luo Date: Wed, 7 Aug 2024 23:21:29 +0800 Subject: [PATCH] support user-defined metadata in CreateMultipartUpload operation. (#100) --- .../core/model/internal/UploadMetadata.java | 4 ++++ .../request/CreateMultipartUploadOptions.java | 3 +++ .../CompleteMultipartUploadService.java | 1 + .../service/CreateMultipartUploadService.java | 1 + .../test/MultipartUploadIntegrationTest.java | 8 +++++++ .../CreateMultipartUploadController.java | 1 + .../s3/rest/handler/PutObjectController.java | 14 ++--------- .../robothy/s3/rest/utils/RequestUtils.java | 24 +++++++++++++++++++ 8 files changed, 44 insertions(+), 12 deletions(-) diff --git a/local-s3-core/src/main/java/com/robothy/s3/core/model/internal/UploadMetadata.java b/local-s3-core/src/main/java/com/robothy/s3/core/model/internal/UploadMetadata.java index 6f60983..d00a4bc 100644 --- a/local-s3-core/src/main/java/com/robothy/s3/core/model/internal/UploadMetadata.java +++ b/local-s3-core/src/main/java/com/robothy/s3/core/model/internal/UploadMetadata.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.robothy.s3.core.converters.deserializer.UploadPartMetadataMapConverter; + +import java.util.Map; import java.util.NavigableMap; import java.util.Optional; import java.util.concurrent.ConcurrentSkipListMap; @@ -22,6 +24,8 @@ public class UploadMetadata { private String[][] tagging; + private Map userMetadata; + @JsonDeserialize(converter = UploadPartMetadataMapConverter.class) @Builder.Default private NavigableMap parts = new ConcurrentSkipListMap<>(); diff --git a/local-s3-core/src/main/java/com/robothy/s3/core/model/request/CreateMultipartUploadOptions.java b/local-s3-core/src/main/java/com/robothy/s3/core/model/request/CreateMultipartUploadOptions.java index 0986cf6..8b87b1a 100644 --- a/local-s3-core/src/main/java/com/robothy/s3/core/model/request/CreateMultipartUploadOptions.java +++ b/local-s3-core/src/main/java/com/robothy/s3/core/model/request/CreateMultipartUploadOptions.java @@ -1,5 +1,6 @@ package com.robothy.s3.core.model.request; +import java.util.Map; import java.util.Optional; import lombok.Builder; import lombok.Getter; @@ -15,4 +16,6 @@ public class CreateMultipartUploadOptions { public Optional getTagging() { return Optional.ofNullable(tagging); } + + private Map userMetadata; } diff --git a/local-s3-core/src/main/java/com/robothy/s3/core/service/CompleteMultipartUploadService.java b/local-s3-core/src/main/java/com/robothy/s3/core/service/CompleteMultipartUploadService.java index bebbf87..d85bdf5 100644 --- a/local-s3-core/src/main/java/com/robothy/s3/core/service/CompleteMultipartUploadService.java +++ b/local-s3-core/src/main/java/com/robothy/s3/core/service/CompleteMultipartUploadService.java @@ -72,6 +72,7 @@ default CompleteMultipartUploadAns completeMultipartUpload(String bucket, String .content(in) .contentType(uploadMetadata.getContentType()) .tagging(uploadMetadata.getTagging().orElse(null)) + .userMetadata(uploadMetadata.getUserMetadata()) .build(); putObjectAns = putObject(bucket, key, putObjectOptions); diff --git a/local-s3-core/src/main/java/com/robothy/s3/core/service/CreateMultipartUploadService.java b/local-s3-core/src/main/java/com/robothy/s3/core/service/CreateMultipartUploadService.java index ebb67d1..5c1fb3b 100644 --- a/local-s3-core/src/main/java/com/robothy/s3/core/service/CreateMultipartUploadService.java +++ b/local-s3-core/src/main/java/com/robothy/s3/core/service/CreateMultipartUploadService.java @@ -33,6 +33,7 @@ default String createMultipartUpload(String bucket, String key, CreateMultipartU .contentType(options.getContentType()) .createDate(System.currentTimeMillis()) .tagging(options.getTagging().orElse(null)) + .userMetadata(options.getUserMetadata()) .build()); return uploadId; } diff --git a/local-s3-interationtest/src/test/java/com/robothy/s3/test/MultipartUploadIntegrationTest.java b/local-s3-interationtest/src/test/java/com/robothy/s3/test/MultipartUploadIntegrationTest.java index ca7ce6d..8d1a82b 100644 --- a/local-s3-interationtest/src/test/java/com/robothy/s3/test/MultipartUploadIntegrationTest.java +++ b/local-s3-interationtest/src/test/java/com/robothy/s3/test/MultipartUploadIntegrationTest.java @@ -28,6 +28,8 @@ import java.io.IOException; import java.net.URL; import java.util.List; +import java.util.Map; + import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.sync.RequestBody; @@ -56,6 +58,8 @@ void multipartUpload(AmazonS3 s3) throws IOException { s3.createBucket(bucket); ObjectMetadata objectMetadata1 = new ObjectMetadata(); objectMetadata1.setContentType("plain/text"); + objectMetadata1.addUserMetadata("x-filename", "a.txt"); + objectMetadata1.addUserMetadata("x-other", "other-value"); InitiateMultipartUploadResult initResult = s3.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucket, key1, objectMetadata1)); assertNotNull(initResult.getUploadId()); @@ -100,6 +104,10 @@ void multipartUpload(AmazonS3 s3) throws IOException { S3Object object = s3.getObject(bucket, key1); assertEquals("HelloWorld", new String(object.getObjectContent().readAllBytes())); assertEquals("plain/text", object.getObjectMetadata().getContentType()); + Map userMetadata = object.getObjectMetadata().getUserMetadata(); + assertEquals(2, userMetadata.size()); + assertEquals("a.txt", userMetadata.get("x-filename")); + assertEquals("other-value", userMetadata.get("x-other")); assertThrows(AmazonS3Exception.class, () -> { s3.copyPart(new CopyPartRequest().withUploadId(initResult.getUploadId()) diff --git a/local-s3-rest/src/main/java/com/robothy/s3/rest/handler/CreateMultipartUploadController.java b/local-s3-rest/src/main/java/com/robothy/s3/rest/handler/CreateMultipartUploadController.java index 1d2cb61..c7bcc55 100644 --- a/local-s3-rest/src/main/java/com/robothy/s3/rest/handler/CreateMultipartUploadController.java +++ b/local-s3-rest/src/main/java/com/robothy/s3/rest/handler/CreateMultipartUploadController.java @@ -35,6 +35,7 @@ public void handle(HttpRequest request, HttpResponse response) throws Exception String contentType = request.header("content-type").orElse("octet/stream"); String uploadId = uploadService.createMultipartUpload(bucket, key, CreateMultipartUploadOptions.builder() .tagging(RequestUtils.extractTagging(request).orElse(null)) + .userMetadata(RequestUtils.extractUserMetadata(request)) .contentType(contentType).build()); InitiateMultipartUploadResult result = InitiateMultipartUploadResult.builder() .bucket(bucket) diff --git a/local-s3-rest/src/main/java/com/robothy/s3/rest/handler/PutObjectController.java b/local-s3-rest/src/main/java/com/robothy/s3/rest/handler/PutObjectController.java index 7fce753..e38e360 100644 --- a/local-s3-rest/src/main/java/com/robothy/s3/rest/handler/PutObjectController.java +++ b/local-s3-rest/src/main/java/com/robothy/s3/rest/handler/PutObjectController.java @@ -41,7 +41,7 @@ public void handle(HttpRequest request, HttpResponse response) throws Exception .size(decodedBody.getDecodedContentLength()) .content(decodedBody.getDecodedBody()) .tagging(RequestUtils.extractTagging(request).orElse(null)) - .userMetadata(extractUserMetadata(request)) + .userMetadata(RequestUtils.extractUserMetadata(request)) .build(); PutObjectAns ans = objectService.putObject(bucketName, key, options); @@ -58,16 +58,6 @@ public void handle(HttpRequest request, HttpResponse response) throws Exception ResponseUtils.addAmzRequestId(response); } - Map extractUserMetadata(HttpRequest request) { - Map userMetadata = new HashMap<>(); - request.getHeaders() - .forEach((k, v) -> { - if (k.toString().startsWith(AmzHeaderNames.X_AMZ_META_PREFIX)) { - String metaName = RequestAssertions.assertUserMetadataHeaderIsValid(k.toString()); - userMetadata.put(metaName, v); - } - }); - return userMetadata; - } + } diff --git a/local-s3-rest/src/main/java/com/robothy/s3/rest/utils/RequestUtils.java b/local-s3-rest/src/main/java/com/robothy/s3/rest/utils/RequestUtils.java index a90bb69..92a6a49 100644 --- a/local-s3-rest/src/main/java/com/robothy/s3/rest/utils/RequestUtils.java +++ b/local-s3-rest/src/main/java/com/robothy/s3/rest/utils/RequestUtils.java @@ -2,12 +2,15 @@ import com.robothy.netty.http.HttpRequest; import com.robothy.s3.core.exception.LocalS3InvalidArgumentException; +import com.robothy.s3.rest.assertions.RequestAssertions; import com.robothy.s3.rest.constants.AmzHeaderNames; import com.robothy.s3.rest.constants.AmzHeaderValues; import com.robothy.s3.rest.model.request.DecodedAmzRequestBody; import io.netty.buffer.ByteBufInputStream; import io.netty.handler.codec.http.HttpHeaderNames; import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; import org.apache.commons.lang3.StringUtils; @@ -86,4 +89,25 @@ public static Optional extractTagging(HttpRequest request) { return Optional.of(tagSet); } + + /** + * Extract user metadata from headers. User metadata in headers that start with {@linkplain AmzHeaderNames#X_AMZ_META_PREFIX}. + * + *

User-defined object metadata + * + * @param request HTTP request + * @return fetched user metadata. + */ + public static Map extractUserMetadata(HttpRequest request) { + Map userMetadata = new HashMap<>(); + request.getHeaders() + .forEach((k, v) -> { + if (k.toString().startsWith(AmzHeaderNames.X_AMZ_META_PREFIX)) { + String metaName = RequestAssertions.assertUserMetadataHeaderIsValid(k.toString()); + userMetadata.put(metaName, v); + } + }); + return userMetadata; + } + }