diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e09440..2b52e93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,11 +6,11 @@ on: - 'main' env: - BUILDER_VERSION: v0.9.63 + BUILDER_VERSION: v0.9.66 BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-checksums - LINUX_BASE_IMAGE: ubuntu-18-x64 + LINUX_BASE_IMAGE: ubuntu-22-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -18,7 +18,7 @@ env: jobs: linux-compat: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest strategy: fail-fast: false matrix: @@ -38,8 +38,9 @@ jobs: aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - linux-compiler-compat: - runs-on: ubuntu-22.04 # latest + + linux-compiler-compat-old-compilers: + runs-on: ubuntu-24.04 # latest strategy: matrix: compiler: @@ -49,11 +50,33 @@ jobs: - clang-9 - clang-10 - clang-11 + - clang-12 - gcc-4.8 - gcc-5 - gcc-6 - gcc-7 - gcc-8 + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-ubuntu-18-x64 build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} + + linux-compiler-compat: + runs-on: ubuntu-24.04 # latest + strategy: + matrix: + compiler: + - clang-13 + - clang-14 + - clang-15 + - clang-16 + - clang-17 + - clang-18 + - gcc-11 + - gcc-12 + - gcc-13 steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} @@ -62,7 +85,7 @@ jobs: ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} clang-sanitizers: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest strategy: matrix: sanitizers: [",thread", ",address,undefined"] @@ -71,10 +94,10 @@ jobs: - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=clang-11 --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}" + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=clang-16 --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}" linux-shared-libs: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} @@ -83,7 +106,7 @@ jobs: ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DBUILD_SHARED_LIBS=ON linux-no-cpu-extensions: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} @@ -178,7 +201,7 @@ jobs: cross_compile: name: Cross Compile ${{matrix.arch}} - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest strategy: matrix: arch: [linux-armv6, linux-armv7, linux-arm64, android-armv7] @@ -193,10 +216,56 @@ jobs: # Test downstream repos. # This should not be required because we can run into a chicken and egg problem if there is a change that needs some fix in a downstream repo. downstream: - runs-on: ubuntu-22.04 # latest + runs-on: ubuntu-24.04 # latest steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build downstream -p ${{ env.PACKAGE_NAME }} + + freebsd: + runs-on: ubuntu-24.04 # latest + steps: + - uses: actions/checkout@v4 + - name: Build ${{ env.PACKAGE_NAME }} + consumers + id: test + uses: cross-platform-actions/action@v0.24.0 + with: + operating_system: freebsd + architecture: x86-64 + version: '14.0' + cpu_count: 4 + shell: bash + run: | + sudo pkg install -y python3 net/py-urllib3 + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + + openbsd: + runs-on: ubuntu-24.04 # latest + strategy: + fail-fast: false + matrix: + # OpenBSD only supports the two most recent releases + version: ['7.4', '7.5'] + steps: + # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install + - uses: actions/checkout@v4 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + consumers + uses: cross-platform-actions/action@v0.24.0 + with: + operating_system: openbsd + architecture: x86-64 + version: ${{ matrix.version }} + cpu_count: 4 + shell: bash + environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + run: | + sudo pkg_add awscli py3-pip py3-urllib3 + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} diff --git a/bin/benchmark/main.c b/bin/benchmark/main.c index 4dd0dfb..07d74f7 100644 --- a/bin/benchmark/main.c +++ b/bin/benchmark/main.c @@ -55,6 +55,10 @@ static void s_runcrc64(struct aws_byte_cursor checksum_this) { (void)crc; } +#define KB_TO_BYTES(kb) ((kb) * 1024) +#define MB_TO_BYTES(mb) ((mb) * 1024 * 1024) +#define GB_TO_BYTES(gb) ((gb) * 1024 * 1024 * 1024ULL) + int main(void) { fprintf(stdout, "hw features for this run:\n"); @@ -97,7 +101,8 @@ int main(void) { // get buffer sizes large enough that all the simd code paths get hit hard, but // also measure the smaller buffer paths since they often can't be optimized as thoroughly. - size_t buffer_sizes[] = {8, 16, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536}; + size_t buffer_sizes[] = {8, 16, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, + KB_TO_BYTES(32), KB_TO_BYTES(64), KB_TO_BYTES(256), MB_TO_BYTES(1), MB_TO_BYTES(10), MB_TO_BYTES(100), GB_TO_BYTES(1)}; size_t buffer_sizes_len = AWS_ARRAY_SIZE(buffer_sizes); // warm it up to factor out the cpuid checks: @@ -115,9 +120,10 @@ int main(void) { aws_high_res_clock_get_ticks(&end_time); fprintf( stdout, - "buffer size %zu (bytes), latency: %" PRIu64 " ns\n", + "buffer size %zu (bytes), latency: %" PRIu64 " ns throughput: %f GiB/s\n", buffer_sizes[k], - end_time - start_time); + end_time - start_time, + (buffer_sizes[k] * 1000000000.0 /* ns -> sec factor */ / GB_TO_BYTES(1)) / (end_time - start_time)); aws_byte_buf_clean_up(&x_bytes); } fprintf(stdout, "\n"); diff --git a/source/intel/intrin/crc32c_sse42_avx512.c b/source/intel/intrin/crc32c_sse42_avx512.c index 02de641..3bf8aff 100644 --- a/source/intel/intrin/crc32c_sse42_avx512.c +++ b/source/intel/intrin/crc32c_sse42_avx512.c @@ -17,8 +17,7 @@ # include -AWS_ALIGNED_TYPEDEF(const uint64_t, zalign_8, 64); -AWS_ALIGNED_TYPEDEF(const uint64_t, zalign_2, 16); +AWS_ALIGNED_TYPEDEF(const uint64_t, aligned_512_u64[8], 64); // This macro uses casting to ensure the compiler actually uses the unaligned load instructions # define load_zmm(ptr) _mm512_loadu_si512((const uint8_t *)(const void *)(ptr)) @@ -48,13 +47,13 @@ static uint32_t s_checksums_crc32c_avx512_impl(const uint8_t *input, int length, * k6 = ( x ^ ( 128 - 32 ) mod P(x) << 32 )' << 1 */ - static zalign_8 k1k2[8] = { + static aligned_512_u64 k1k2 = { 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86}; - static zalign_8 k3k4[8] = { + static aligned_512_u64 k3k4 = { 0x740eef02, 0x9e4addf8, 0x740eef02, 0x9e4addf8, 0x740eef02, 0x9e4addf8, 0x740eef02, 0x9e4addf8}; - static zalign_8 k9k10[8] = { + static aligned_512_u64 k9k10 = { 0x6992cea2, 0x0d3b6092, 0x6992cea2, 0x0d3b6092, 0x6992cea2, 0x0d3b6092, 0x6992cea2, 0x0d3b6092}; - static zalign_8 k1k4[8] = { + static aligned_512_u64 k1k4 = { 0x1c291d04, 0xddc0152b, 0x3da6d0cb, 0xba4fc28e, 0xf20c0dfe, 0x493c7d27, 0x00000000, 0x00000000}; __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8; diff --git a/tests/crc64_test.c b/tests/crc64_test.c index af6d859..00acb2f 100644 --- a/tests/crc64_test.c +++ b/tests/crc64_test.c @@ -129,7 +129,7 @@ AWS_TEST_CASE(test_crc64nvme, s_test_crc64nvme) static int s_test_large_buffer_crc64(struct aws_allocator *allocator, void *ctx) { (void)ctx; -#if SIZE_BITS == 32 +#if SIZE_BITS == 32 || defined(__OpenBSD__) /* openbsd fails to allocate big buffer */ (void)allocator; return AWS_OP_SKIP; #else diff --git a/tests/crc_test.c b/tests/crc_test.c index 8bebcb2..8db0d2b 100644 --- a/tests/crc_test.c +++ b/tests/crc_test.c @@ -186,7 +186,7 @@ AWS_TEST_CASE(test_crc32, s_test_crc32) static int s_test_large_buffer_crc32(struct aws_allocator *allocator, void *ctx) { (void)ctx; -#if SIZE_BITS == 32 +#if SIZE_BITS == 32 || defined(__OpenBSD__) /* openbsd fails to allocate big buffer */ (void)allocator; return AWS_OP_SKIP; #else