Skip to content

Commit

Permalink
CRC64NVME checksum algo (#853)
Browse files Browse the repository at this point in the history
  • Loading branch information
@TingDaoK
TingDaoK authored Dec 6, 2024
1 parent 91f9370 commit 32cbb5a
Show file tree
Hide file tree
Showing 5 changed files with 118 additions and 32 deletions.
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,7 @@ jobs:
cd crt/aws-c-http/tests/py_localhost/
Start-Process -NoNewWindow python .\server.py
Start-Process -NoNewWindow python .\non_tls_server.py
cd ../../../../
python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
python builder.pyz localhost-test -p ${{ env.PACKAGE_NAME }} downstream
Expand Down
2 changes: 1 addition & 1 deletion crt/aws-c-common
Submodule aws-c-common updated 3 files
+1 −1 .github/workflows/clang-tidy.yml
+12 −6 .github/workflows/codecov.yml
+2 −3 cmake/AwsPrebuildDependency.cmake
2 changes: 1 addition & 1 deletion crt/aws-c-s3
Submodule aws-c-s3 updated 32 files
+94 −37 .github/workflows/ci.yml
+18 −13 .github/workflows/codecov.yml
+39 −14 include/aws/s3/private/s3_checksums.h
+4 −0 include/aws/s3/private/s3_client_impl.h
+1 −1 include/aws/s3/private/s3_meta_request_impl.h
+3 −0 include/aws/s3/private/s3_request.h
+5 −9 include/aws/s3/private/s3_request_messages.h
+8 −35 include/aws/s3/private/s3_util.h
+4 −3 include/aws/s3/s3_client.h
+73 −3 source/s3_auto_ranged_put.c
+1 −1 source/s3_checksum_stream.c
+183 −105 source/s3_checksums.c
+6 −6 source/s3_chunk_stream.c
+6 −27 source/s3_client.c
+3,158 −2,863 source/s3_endpoint_resolver/aws_s3_endpoint_rule_set.c
+49 −30 source/s3_meta_request.c
+78 −38 source/s3_request_messages.c
+12 −20 source/s3_util.c
+11 −4 tests/CMakeLists.txt
+14 −11 tests/s3_checksum_stream_test.c
+6 −41 tests/s3_checksums_crc32_tests.c
+6 −41 tests/s3_checksums_crc32c_tests.c
+148 −0 tests/s3_checksums_crc64nvme_tests.c
+6 −38 tests/s3_checksums_sha1_tests.c
+6 −54 tests/s3_checksums_sha256_tests.c
+1 −3 tests/s3_checksums_test_case_helper.h
+35 −12 tests/s3_data_plane_tests.c
+4 −1 tests/s3_list_objects_tests.c
+203 −0 tests/s3_mock_server_tests.c
+5 −0 tests/s3_request_messages_tests.c
+63 −1 tests/s3_tester.c
+1 −0 tests/s3_tester.h
5 changes: 4 additions & 1 deletion src/main/java/software/amazon/awssdk/crt/s3/ChecksumAlgorithm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ public enum ChecksumAlgorithm {

SHA1(3),

SHA256(4);
SHA256(4),

CRC64NVME(5);

ChecksumAlgorithm(int nativeValue) {
this.nativeValue = nativeValue;
Expand All @@ -46,6 +48,7 @@ private static Map<Integer, ChecksumAlgorithm> buildEnumMapping() {
enumMapping.put(CRC32.getNativeValue(), CRC32);
enumMapping.put(SHA1.getNativeValue(), SHA1);
enumMapping.put(SHA256.getNativeValue(), SHA256);
enumMapping.put(CRC64NVME.getNativeValue(), CRC64NVME);
return enumMapping;
}

Expand Down
140 changes: 111 additions & 29 deletions src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,8 +276,8 @@ public void testS3ClientCreateDestroyHttpProxyOptions() {
proxyOptions.setAuthorizationUsername("username");
proxyOptions.setAuthorizationPassword("password");
try (S3Client client = createS3Client(new S3ClientOptions()
.withRegion(REGION)
.withProxyOptions(proxyOptions), elg)) {
.withRegion(REGION)
.withProxyOptions(proxyOptions), elg)) {
}
}
}
Expand Down Expand Up @@ -449,7 +449,9 @@ public void testS3GetErrorFinishedResponseContextHasAllData() {
S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() {

@Override
public int onResponseBody(ByteBuffer bodyBytesIn, long objectRangeStart, long objectRangeEnd) { return 0; }
public int onResponseBody(ByteBuffer bodyBytesIn, long objectRangeStart, long objectRangeEnd) {
return 0;
}

@Override
public void onFinished(S3FinishedResponseContext context) {
Expand Down Expand Up @@ -864,12 +866,13 @@ private byte[] createTestPayload(int size) {
}

private String uploadObjectPathInit(String objectPath) {
return UPLOAD_DIR+objectPath;
return UPLOAD_DIR + objectPath;
}

private void testS3PutHelper(boolean useFile, boolean unknownContentLength, String objectPath, boolean s3express,
int contentLength, boolean contentMD5) throws IOException {
S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION).withEnableS3Express(s3express).withComputeContentMd5(contentMD5);
S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION).withEnableS3Express(s3express)
.withComputeContentMd5(contentMD5);
Path uploadFilePath = Files.createTempFile("testS3PutFilePath", ".txt");
try (S3Client client = createS3Client(clientOptions)) {
CompletableFuture<Integer> onFinishedFuture = new CompletableFuture<>();
Expand Down Expand Up @@ -933,7 +936,7 @@ public long getLength() {
.withMetaRequestType(MetaRequestType.PUT_OBJECT).withHttpRequest(httpRequest)
.withResponseHandler(responseHandler);

if(!contentMD5) {
if (!contentMD5) {
ChecksumConfig checksumConfig = new ChecksumConfig().withChecksumAlgorithm(ChecksumAlgorithm.SHA1)
.withChecksumLocation(ChecksumLocation.TRAILER).withValidateChecksum(true);
metaRequestOptions = metaRequestOptions.withChecksumConfig(checksumConfig);
Expand Down Expand Up @@ -1030,7 +1033,8 @@ public void testS3PutNonexistentFilePath() throws IOException {
new HttpHeader("Host", ENDPOINT),
new HttpHeader("Content-Length", String.valueOf(1024)),
};
HttpRequest httpRequest = new HttpRequest("PUT", uploadObjectPathInit("/put_nonexistent_file"), headers, null);
HttpRequest httpRequest = new HttpRequest("PUT", uploadObjectPathInit("/put_nonexistent_file"), headers,
null);

S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions()
.withMetaRequestType(MetaRequestType.PUT_OBJECT)
Expand Down Expand Up @@ -1112,7 +1116,8 @@ public long getLength() {
HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT),
new HttpHeader("Content-Length", Integer.valueOf(payload.capacity()).toString()), };

HttpRequest httpRequest = new HttpRequest("PUT", uploadObjectPathInit("/put_object_test_128MB"), headers, payloadStream);
HttpRequest httpRequest = new HttpRequest("PUT", uploadObjectPathInit("/put_object_test_128MB"), headers,
payloadStream);

S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions()
.withMetaRequestType(MetaRequestType.PUT_OBJECT)
Expand Down Expand Up @@ -1156,7 +1161,7 @@ public long getLength() {
new HttpHeader("Content-Length", Integer.valueOf(payloadResume.capacity()).toString()), };

HttpRequest httpRequestResume = new HttpRequest("PUT",
uploadObjectPathInit("/put_object_test_128MB"), headersResume, payloadStreamResume);
uploadObjectPathInit("/put_object_test_128MB"), headersResume, payloadStreamResume);

CompletableFuture<Integer> onFinishedFutureResume = new CompletableFuture<>();
CompletableFuture<Void> onProgressFutureResume = new CompletableFuture<>();
Expand All @@ -1183,13 +1188,17 @@ public long getLength() {
}
}

@Test
public void testS3PutTrailerChecksums() {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
private void testS3RoundTripWithChecksumHelper(ChecksumAlgorithm algo, ChecksumLocation location, boolean MPU,
boolean provide_full_object_checksum) throws IOException {

S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION);
if (MPU) {
clientOptions.withPartSize(5 * 1024 * 1024);
clientOptions.withMultipartUploadThreshold(5 * 1024 * 1024);
} else {
clientOptions.withPartSize(10 * 1024 * 1024);
clientOptions.withMultipartUploadThreshold(20 * 1024 * 1024);
}
try (S3Client client = createS3Client(clientOptions)) {
CompletableFuture<Integer> onPutFinishedFuture = new CompletableFuture<>();
S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() {
Expand All @@ -1212,7 +1221,7 @@ public void onFinished(S3FinishedResponseContext context) {
}
};

final ByteBuffer payload = ByteBuffer.wrap(createTestPayload(1024 * 1024));
final ByteBuffer payload = ByteBuffer.wrap(createTestPayload(10 * 1024 * 1024));

HttpRequestBodyStream payloadStream = new HttpRequestBodyStream() {
@Override
Expand All @@ -1231,14 +1240,36 @@ public long getLength() {
return payload.capacity();
}
};
ArrayList<HttpHeader> headers = new ArrayList<HttpHeader>();
headers.add(new HttpHeader("Host", ENDPOINT));
headers.add(new HttpHeader("Content-Length", Integer.valueOf(payload.capacity()).toString()));
if (provide_full_object_checksum) {
switch (algo) {
case CRC32:
headers.add(new HttpHeader("x-amz-checksum-crc32", "1BObvg=="));
break;
case CRC64NVME:
headers.add(new HttpHeader("x-amz-checksum-crc64nvme", "fIa08UXfyzk="));
break;

HttpHeader[] headers = { new HttpHeader("Host", ENDPOINT),
new HttpHeader("Content-Length", Integer.valueOf(payload.capacity()).toString()), };
default:
Assert.fail("Unsupported checksum algorithm for full object checksum");
break;
}
}

String objectPath = uploadObjectPathInit("/prefix/round_trip/java_round_trip_test_fc.txt");
HttpRequest httpRequest = new HttpRequest("PUT", objectPath, headers, payloadStream);
ChecksumConfig config = new ChecksumConfig().withChecksumAlgorithm(ChecksumAlgorithm.CRC32)
.withChecksumLocation(ChecksumLocation.TRAILER);
String objectPath = uploadObjectPathInit(
"/prefix/round_trip/java_round_trip_test_fc_" + location.name() + "_" + algo.name())
+ (MPU ? "_mpu" : "") + (provide_full_object_checksum ? "_full_object" : "");
HttpRequest httpRequest = new HttpRequest("PUT", objectPath, headers.toArray(new HttpHeader[0]),
payloadStream);
ChecksumConfig config = new ChecksumConfig();

if (!provide_full_object_checksum) {
/* If the checksum provided for the full object via header, skip the checksum from client. */
config.withChecksumAlgorithm(algo)
.withChecksumLocation(location);
}
S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions()
.withMetaRequestType(MetaRequestType.PUT_OBJECT).withHttpRequest(httpRequest)
.withResponseHandler(responseHandler)
Expand Down Expand Up @@ -1276,17 +1307,18 @@ public void onFinished(S3FinishedResponseContext context) {
new RuntimeException("Checksum was not validated"));
return;
}
if (context.getChecksumAlgorithm() != ChecksumAlgorithm.CRC32) {
if (context.getChecksumAlgorithm() != algo) {
onGetFinishedFuture.completeExceptionally(
new RuntimeException("Checksum was not validated via CRC32"));
new RuntimeException("Checksum was not validated via expected algo: " + algo.name()));
return;
}
onGetFinishedFuture.complete(Integer.valueOf(context.getErrorCode()));
}
};
ArrayList<ChecksumAlgorithm> algorList = new ArrayList<ChecksumAlgorithm>();
algorList.add(ChecksumAlgorithm.CRC32);
algorList.add(ChecksumAlgorithm.CRC64NVME);
algorList.add(ChecksumAlgorithm.CRC32C);
algorList.add(ChecksumAlgorithm.CRC32);
algorList.add(ChecksumAlgorithm.SHA1);
algorList.add(ChecksumAlgorithm.SHA256);
ChecksumConfig validateChecksumConfig = new ChecksumConfig().withValidateChecksum(true)
Expand All @@ -1304,6 +1336,54 @@ public void onFinished(S3FinishedResponseContext context) {
}
}

@Test
public void testS3PutTrailerChecksums() throws Exception {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC64NVME, ChecksumLocation.TRAILER, false, false);
}

@Test
public void testS3PutHeaderChecksums() throws Exception {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC64NVME, ChecksumLocation.HEADER, false, false);
}

@Test
public void testS3RoundTripWithFullObjectMPU() throws Exception {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC64NVME, ChecksumLocation.NONE, true, true);
}

@Test
public void testS3RoundTripWithFullObjectSinglePart() throws Exception {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC64NVME, ChecksumLocation.NONE, false, true);
}

@Test
public void testS3RoundTripWithFullObjectMPUCRC32() throws Exception {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC32, ChecksumLocation.NONE, true, true);
}

@Test
public void testS3RoundTripWithFullObjectSinglePartCRC32() throws Exception {
skipIfAndroid();
skipIfNetworkUnavailable();
Assume.assumeTrue(hasAwsCredentials());
testS3RoundTripWithChecksumHelper(ChecksumAlgorithm.CRC32, ChecksumLocation.NONE, false, true);
}

@Test
public void testS3GetS3ExpressOverride() throws Exception {
skipIfNetworkUnavailable();
Expand Down Expand Up @@ -1396,7 +1476,7 @@ public void onFinished(S3FinishedResponseContext context) {
}
}

private void putS3ExpressHelper(String region, S3Client client) throws Exception{
private void putS3ExpressHelper(String region, S3Client client) throws Exception {

CompletableFuture<Integer> onFinishedFuture = new CompletableFuture<>();
S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() {
Expand All @@ -1420,7 +1500,8 @@ public void onFinished(S3FinishedResponseContext context) {
};

HttpHeader[] headers = {
new HttpHeader("Host", region.equals("us-east-1")? S3EXPRESS_ENDPOINT_USE1_AZ4 : S3EXPRESS_ENDPOINT_USW2_AZ1),
new HttpHeader("Host",
region.equals("us-east-1") ? S3EXPRESS_ENDPOINT_USE1_AZ4 : S3EXPRESS_ENDPOINT_USW2_AZ1),
};
HttpRequest httpRequest;
String path = uploadObjectPathInit("/put_object_test_10MB.txt");
Expand Down Expand Up @@ -1507,7 +1588,6 @@ public void testS3Copy() {
String COPY_SOURCE_KEY = "crt-canary-obj.txt";
String X_AMZ_COPY_SOURCE_HEADER = "x-amz-copy-source";


S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION);
try (S3Client client = createS3Client(clientOptions)) {
CompletableFuture<Integer> onFinishedFuture = new CompletableFuture<>();
Expand Down Expand Up @@ -1733,7 +1813,8 @@ public void onFinished(S3FinishedResponseContext context) {
concurrentSlots.release();

if (context.getErrorCode() != 0) {
onFinishedFuture.completeExceptionally(makeExceptionFromFinishedResponseContext(context));
onFinishedFuture
.completeExceptionally(makeExceptionFromFinishedResponseContext(context));
return;
}

Expand Down Expand Up @@ -1862,7 +1943,8 @@ public void onFinished(S3FinishedResponseContext context) {
concurrentSlots.release();

if (context.getErrorCode() != 0) {
onFinishedFuture.completeExceptionally(makeExceptionFromFinishedResponseContext(context));
onFinishedFuture
.completeExceptionally(makeExceptionFromFinishedResponseContext(context));
return;
}

Expand Down

0 comments on commit 32cbb5a

Please sign in to comment.