Merge pull request #3487 from facebook/dev

release v1.5.4

.circleci/config.yml

@@ -7,8 +7,6 @@ jobs:
     #       preinstalled to reduce installation time.
     docker:
       - image: fbopensource/zstd-circleci-primary:0.0.1
-    # TODO: Re-enable aarch64 build:
-    #       make aarch64build     && make clean
     steps:
       - checkout
       - run:
@@ -36,6 +34,7 @@ jobs:
             make ppc64build V=1; make clean
             make ppcbuild   V=1; make clean
             make armbuild   V=1; make clean
+            make aarch64build V=1; make clean
             make -C tests test-legacy test-longmatch; make clean
             make -C lib libzstd-nomt; make clean
   # This step should only be run in a cron job

.circleci/images/primary/Dockerfile

@@ -1,4 +1,4 @@
-FROM circleci/buildpack-deps:bionic
+FROM circleci/buildpack-deps:focal
 
 RUN sudo dpkg --add-architecture i386
 RUN sudo apt-get -y -qq update

.cirrus.yml

@@ -2,9 +2,8 @@ task:
   name: FreeBSD (shortest)
   freebsd_instance:
     matrix:
+      image_family: freebsd-13-0
       image_family: freebsd-12-2
-      # The stable 11.3 image causes "Agent is not responding" so use a snapshot
-      image_family: freebsd-11-3-snap
   install_script: pkg install -y gmake coreutils
   script: |
     MOREFLAGS="-Werror" gmake -j all

.github/ISSUE_TEMPLATE/bug_report.md

@@ -26,8 +26,8 @@ If applicable, add screenshots and charts to help explain your problem.
 **Desktop (please complete the following information):**
  - OS: [e.g. Mac]
  - Version [e.g. 22]
- - Compiler [e.g gcc]
- - Flags [e.g O2]
+ - Compiler [e.g. gcc]
+ - Flags [e.g. O2]
  - Other relevant hardware specs [e.g. Dual-core]
  - Build system [e.g. Makefile]
 

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/dev-long-tests.yml

@@ -13,7 +13,7 @@ jobs:
   make-all:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make all
       run: make all
 
@@ -24,57 +24,85 @@ jobs:
       DEVNULLRIGHTS: 1
       READFROMBLOCKDEVICE: 1
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: make test
       run: make test
 
   # lasts ~26mn
   make-test-osx:
     runs-on: macos-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: OS-X test
       run: make test # make -c lib all doesn't work because of the fact that it's not a tty
 
   no-intrinsics-fuzztest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: no intrinsics fuzztest
       run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest
 
   tsan-zstreamtest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: thread sanitizer zstreamtest
       run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
 
+  ubsan-zstreamtest:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: undefined behavior sanitizer zstreamtest
+      run: CC=clang make uasan-test-zstream
+
   # lasts ~15mn
   tsan-fuzztest:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: thread sanitizer fuzztest
       run: CC=clang make tsan-fuzztest
 
+
+  big-tests-zstreamtest32:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: zstream tests in 32bit mode, with big tests
+      run: |
+        sudo apt-get -qqq update
+        make libc6install
+        CC=clang make -C tests test-zstream32 FUZZER_FLAGS="--big-tests"
+
   # lasts ~23mn
   gcc-8-asan-ubsan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: gcc-8 + ASan + UBSan + Test Zstd
+      # See https://askubuntu.com/a/1428822
       run: |
+        echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -qqq update
         make gcc8install
         CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
 
+  clang-asan-ubsan-testzstd:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: clang + ASan + UBSan + Test Zstd
+      run: CC=clang make -j uasan-test-zstd </dev/null V=1
+
   gcc-asan-ubsan-testzstd-32bit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: ASan + UBSan + Test Zstd, 32bit mode
       run: |
+        sudo apt-get -qqq update
         make libc6install
         make -j uasan-test-zstd32 V=1
 
@@ -85,39 +113,67 @@ jobs:
   gcc-8-asan-ubsan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: gcc-8 + ASan + UBSan + Fuzz Test
+      # See https://askubuntu.com/a/1428822
       run: |
+        echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee -a /etc/apt/sources.list
+        sudo apt-get -qqq update
         make gcc8install
         CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
 
+  clang-asan-ubsan-fuzz:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: clang + ASan + UBSan + Fuzz Test
+      run: CC=clang FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
+
   gcc-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: ASan + UBSan + Fuzz Test 32bit
       run: |
+        sudo apt-get -qqq update
         make libc6install
         CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
 
+  clang-asan-ubsan-fuzz32:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: clang + ASan + UBSan + Fuzz Test 32bit
+      run: |
+        sudo apt-get -qqq update
+        make libc6install
+        CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
+
   asan-ubsan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: ASan + UBSan + Regression Test
       run: make -j uasanregressiontest
 
+  clang-ubsan-regression:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - name: clang + ASan + UBSan + Regression Test
+      run: CC=clang make -j uasanregressiontest
+
   msan-regression:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: MSan + Regression Test
       run: make -j msanregressiontest
 
   clang-msan-fuzz:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: clang + MSan + Fuzz Test
       run: |
         sudo apt-get -qqq update
@@ -128,7 +184,7 @@ jobs:
   clang-msan-testzstd:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: clang + MSan + Test Zstd
       run: |
         sudo apt-get update
@@ -138,7 +194,7 @@ jobs:
   armfuzz:
       runs-on: ubuntu-latest
       steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
       - name: Qemu ARM emulation + Fuzz Test
         run: |
           sudo apt-get -qqq update
@@ -148,67 +204,42 @@ jobs:
   valgrind-fuzz-test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
     - name: valgrind + fuzz test stack mode    # ~ 7mn
       shell: 'script -q -e -c "bash {0}"'
       run: |
+        sudo apt-get -qqq update
         make valgrindinstall
-        make -C tests valgrindTest
+        make -C tests test-valgrind
         make clean
         make -C tests test-fuzzer-stackmode
 
   mingw-long-test:
     runs-on: windows-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include: [
-          { compiler: clang, platform: x64, action: build, script: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion -Wno-unused-command-line-argument -Wno-implicit-int-float-conversion' make -j allzstd V=1"},
-          { compiler: gcc, platform: x64, action: test, script: ""},
-        ]
+    defaults:
+      run:
+        shell: msys2 {0}
     steps:
-    - uses: actions/checkout@v2
-    - name: Mingw long test
+    - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # tag=v3
+    - uses: msys2/setup-msys2@v2
+      with:
+        msystem: MINGW64
+        install: make
+        update: true
+    # Based on https://ariya.io/2020/07/on-github-actions-with-msys2
+    - name: install mingw gcc
+      run: pacman --noconfirm -S gcc
+    - name: MINGW64 gcc fuzztest
       run: |
-        $env:PATH_ORIGINAL = $env:PATH
-        $env:PATH_MINGW32 = "C:\msys64\mingw32\bin"
-        $env:PATH_MINGW64 = "C:\msys64\mingw64\bin"
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw32\bin\make.exe
-        COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw64\bin\make.exe
-        IF ("${{matrix.platform}}" -eq "x64")
-        {
-            $env:PATH = $env:PATH_MINGW64 + ";" + $env:PATH_ORIGINAL
-        }
-        ELSEIF ("${{matrix.platform}}" -eq "x86")
-        {
-            $env:PATH = $env:PATH_MINGW32 + ";" + $env:PATH_ORIGINAL
-        }
-        IF ("${{matrix.action}}" -eq "build")
-        {
-          make -v
-          sh -c "${{matrix.compiler}} -v"
-          ECHO "Building zlib to static link"
-          $env:CC = "${{matrix.compiler}}"
-          sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib"
-          sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a"
-          ECHO "Building zstd"
-          $env:CPPFLAGS = "-I../../zlib"
-          $env:LDFLAGS = "../../zlib/libz.a"
-          sh -c "${{matrix.script}}"
-        }
-        ELSEIF ("${{matrix.action}}" -eq "test")
-        {
-            ECHO "Testing ${{matrix.compiler}} ${{matrix.platform}}"
-            $env:CC = "gcc"
-            $env:CXX = "g++"
-            MKDIR build\cmake\build
-            CD build\cmake\build
-            $env:FUZZERTEST = "-T2mn"
-            $env:ZSTREAM_TESTTIME = "-T2mn"
-            cmake -G "Visual Studio 14 2015 Win64" ..
-            cd ..\..\..
-            make clean
-        }
+        export CC="gcc"
+        export CXX="g++"
+        export FUZZERTEST="-T2mn"
+        export ZSTREAM_TESTTIME="-T2mn"
+        echo "Testing $CC $CXX MINGW64"
+        make -v
+        $CC --version
+        $CXX --version
+        make -C tests fuzztest
 
   # lasts ~20mn
   oss-fuzz:
@@ -233,7 +264,7 @@ jobs:
         dry-run: false
         sanitizer: ${{ matrix.sanitizer }}
     - name: Upload Crash
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # tag=v3.1.2
       if: failure() && steps.build.outcome == 'success'
       with:
         name: ${{ matrix.sanitizer }}-artifacts