Merge branch 'v8_1' into v9_0

.github/actions/build/action.yml

@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: (c) 2024 Advanced Micro Devices, Inc.
+
+name: Build
+description: Build TCPDirect
+inputs:
+  tcpdirect_tree:
+    required: true
+    type: string
+    description: Path to the TCPDirect source tree to build
+  onload_tree:
+    required: true
+    type: string
+    description: Path to the Onload source tree to build TCPDirect with
+  build_target:
+    required: false
+    default: all
+    type: choice
+    options:
+      - all
+      - shim
+    description: TCPDirect target to build, all or shim
+  debug:
+    required: false
+    default: false
+    type: boolean
+    description: Whether to build debug binaries
+runs:
+  using: "composite"
+  steps:
+    - name: Input Validation
+      shell: sh
+      env:
+        BUILD_TARGET: ${{ inputs.build_target }}
+      run: |
+        if [ "$BUILD_TARGET" != "all" ] && [ "$BUILD_TARGET" != "shim" ]; then
+          echo "Input Error: build_target must be either all or shim!"
+          exit 1
+        fi
+
+    - name: Build
+      shell: sh
+      env:
+        ONLOAD_TREE: ${{ inputs.onload_tree }}
+        ZF_DEVEL: ${{ inputs.build_target == 'shim' && 1 || 0 }}
+        NDEBUG: ${{ fromJSON(inputs.debug) && 0 || 1 }}
+        BUILD_TARGET: ${{ inputs.build_target }}
+      working-directory: ${{ inputs.tcpdirect_tree }}
+      run: make -j $(nproc) "$BUILD_TARGET"

.github/actions/install-dependencies/action.yml

@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: (c) 2024 Advanced Micro Devices, Inc.
+
+name: Install dependencies
+description: Install build and test dependencies for TCPDirect
+runs:
+  using: "composite"
+  steps:
+    - name: Install dependencies
+      shell: sh
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y build-essential perl

.github/actions/test/action.yml

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: (c) 2024 Advanced Micro Devices, Inc.
+
+name: Test
+description: Run the TCPDirect unit tests
+inputs:
+  tcpdirect_tree:
+    required: true
+    type: string
+    description: Path to the TCPDirect source tree to build
+  onload_tree:
+    required: true
+    type: string
+    description: Path to the Onload source tree to build TCPDirect with
+  debug:
+    required: false
+    default: true
+    type: boolean
+    description: Whether to build debug binaries
+  test_timeout_multiplier:
+    description: Value to multiply with default test timeouts
+    default: 1
+    required: false
+    type: number
+  run_slow_tests:
+    description: Whether the unit tests should run tests marked as slow
+    default: true
+    required: false
+    type: boolean
+runs:
+  using: "composite"
+  steps:
+    - name: Run tests
+      shell: sh
+      working-directory: ${{ inputs.tcpdirect_tree }}
+      env:
+        ONLOAD_TREE: ${{ inputs.onload_tree }}
+        NDEBUG: ${{ fromJSON(inputs.debug) && 0 || 1 }}
+        TIMEOUT_MULTIPLIER: ${{ inputs.test_timeout_multiplier }}
+        ZF_RUN_SLOW_TESTS: ${{ fromJSON(inputs.run_slow_tests) && 1 || 0 }}
+        ZF_DEVEL: 1
+      run: UT_OUTPUT="$GITHUB_STEP_SUMMARY" make -j $(nproc) test

.github/workflows/perform_build.yml

@@ -1,19 +1,16 @@
-# SPDX-License-Identifier: BSD-2-Clause
-# X-SPDX-Copyright-Text: (c) Copyright 2024 Advanced Micro Devices, Inc.
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: (c) 2024 Advanced Micro Devices, Inc.
 
 name: "perform_build"
-
 on:
   push:
   pull_request:
     types: [opened]
-
 jobs:
   extract_branch_info:
     runs-on: ubuntu-latest
     timeout-minutes: 1
     outputs:
-      tcpdirectBranch: ${{ steps.tcpdirectBranch.outputs.branch }}
       onloadPublicTreeish: ${{ steps.onloadPublicTreeish.outputs.tag }}
       packetDrillBranch: ${{ steps.packetDrillBranch.outputs.branch }}
     steps:
@@ -28,10 +25,6 @@ jobs:
            sudo apt-get install -y python3 python3-pip jq
            pip3 install yq
 
-      - name: Extract TCPDirect branch name
-        id: tcpdirectBranch
-        run: echo "branch=$(yq -r '.products.TCPDirect.version' < $GITHUB_WORKSPACE/tcpdirect/versions.yaml)" >> "$GITHUB_OUTPUT"
-
       - name: Extract public onload tree-ish
         id: onloadPublicTreeish
         run: echo "tag=$(yq -r '.products.Onload.public_tree_ish' < $GITHUB_WORKSPACE/tcpdirect/versions.yaml)" >> "$GITHUB_OUTPUT"
@@ -40,50 +33,55 @@ jobs:
         id: packetDrillBranch
         run: echo "branch=$(yq -r '.products.Packetdrill.version' < $GITHUB_WORKSPACE/tcpdirect/versions.yaml)" >> "$GITHUB_OUTPUT"
 
-
   perform_build_and_test:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     needs: extract_branch_info
+    env:
+      TCPDIRECT_TREE: ${{ github.workspace }}/tcpdirect
+      ONLOAD_TREE: ${{ github.workspace }}/onload
     steps:
-
-      - name: tcpdirect
+      - name: Checkout TCPDirect
         uses: actions/checkout@v4
         with:
-          path: tcpdirect
+          path: ${{ env.TCPDIRECT_TREE }}
 
-      - name: onload checkout public tree-ish
+      - name: Checkout Onload
         uses: actions/checkout@v4
         with:
           repository: ${{ github.repository_owner }}/onload
-          path: onload
+          path: ${{ env.ONLOAD_TREE }}
           ref: ${{ needs.extract_branch_info.outputs.onloadPublicTreeish }}
 
-      - name: packetdrill checkout
+      - name: Checkout packetdrill
         uses: actions/checkout@v4
         continue-on-error: true
         with:
           repository: ${{ github.repository_owner }}/packetdrill-tcpdirect
           path: packetdrill-tcpdirect
-          ssh-key: ${{ secrets.PACKET_DRILL_PRIVATE_KEY }}
           ref: ${{ needs.extract_branch_info.outputs.packetDrillBranch }}
 
-      - name: Install TCPDirect Deps
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y build-essential perl
+      - name: Install the TCPDirect build and test dependencies
+        uses: ./tcpdirect/.github/actions/install-dependencies
 
-      - name: build
-        run: |
-          cd $GITHUB_WORKSPACE/tcpdirect
-          ONLOAD_TREE=$GITHUB_WORKSPACE/onload NDEBUG=1 make -j $(nproc)
+      - name: Build the base TCPDirect library
+        uses: ./tcpdirect/.github/actions/build
+        with:
+          tcpdirect_tree: ${{ env.TCPDIRECT_TREE }}
+          onload_tree: ${{ env.ONLOAD_TREE }}
+          build_target: all
+          debug: true
 
-      - name: shim
-        run: |
-          cd $GITHUB_WORKSPACE/tcpdirect
-          ONLOAD_TREE=$GITHUB_WORKSPACE/onload NDEBUG=1 ZF_DEVEL=1 TEST_THREAD_NAME=zf make -j $(nproc) shim
+      - name: Build the TCPDirect socket shim
+        uses: ./tcpdirect/.github/actions/build
+        with:
+          tcpdirect_tree: ${{ env.TCPDIRECT_TREE }}
+          onload_tree: ${{ env.ONLOAD_TREE }}
+          build_target: shim
+          debug: true
 
-      - name: tests
-        run: |
-          cd $GITHUB_WORKSPACE/tcpdirect
-          ONLOAD_TREE=$GITHUB_WORKSPACE/onload UT_OUTPUT=$GITHUB_STEP_SUMMARY NDEBUG=1 ZF_DEVEL=1 TEST_THREAD_NAME=zf make -j $(nproc) -k test
+      - name: Run the TCPDirect unit tests
+        uses: ./tcpdirect/.github/actions/test
+        with:
+          tcpdirect_tree: ${{ env.TCPDIRECT_TREE }}
+          onload_tree: ${{ env.ONLOAD_TREE }}

ci/test.groovy

@@ -155,8 +155,10 @@ nm.slack_notify() {
             export CC=${CC}
             export ONLOAD_TREE=\$PWD/onload
             export ZF_DEVEL=1
+	    export ZF_RUN_UNSTABLE_TESTS=1
+	    export ZF_RUN_SLOW_TESTS=1
             export TEST_THREAD_NAME=zf
-            make -k -C tcpdirect test_jenkins
+            make -k -C tcpdirect test
           """, returnStatus: true)
           if (rc != 0) {
             unstable("not all tests passed")

src/tests/zf_unit/Makefile.inc

@@ -12,7 +12,6 @@ UT_LOOP := zftest zfinit zfudprx zf_superbuf_reuse zftcprx zftcpcopyrx \
 UT_LOOP2 := zftest zfinit zfudprx zf_superbuf_reuse zftcprx zftcpcopyrx \
             zftcptx zfmulticast zftcp_muxer \
             zfds efrxtimestamping zftcprx_on14495
-UT_LOOP_JENKINS := zftimestamping
 UT_MISC := zfpool zfmuxer zfrxtable zfzockbuf \
           zfmultizock zftimers zflisten zfbackingsock zfzockfree \
           zfquiesce_waittw zfquiesce_nowaittw zfstackfree \
@@ -28,6 +27,14 @@ UT_ALTS := zfalternatives
 UT_OTHER := zftcppingpong2
 UT_SCRIPTS := zfudppingpong.sh zf_tcp_sanity.sh packetdrill.sh zfsend.sh zfudpttl.sh
 
+# List of tests to skip unless ZF_RUN_SLOW_TESTS=1 (defaults to 1)
+# NOTE: `zfovl` is not actually slow, but rather unstable on slow machines.
+UT_SLOW := zf_superbuf_fallback zf_superbuf_stack_destroy zf_superbuf_locality \
+           zf_superbuf_n_sleep zf_superbuf_sleepy_stacks zftcptimeouts \
+	   zfreactorloop zfstackfree zfrxtable zftcprx zfovl
+
+# List of tests to skip unless ZF_RUN_UNSTABLE_TESTS=1 (defaults to 0)
+UT_UNSTABLE := zftimestamping $(UT_SCRIPTS)
 
 # sort is required to remove duplicates
 UT_ALL := $(sort \
@@ -38,8 +45,7 @@ UT_ALL := $(sort \
   $(UT_B2B) \
   $(UT_B2B_SB) \
   $(UT_ALTS) \
-  $(UT_OTHER) \
-  $(UT_LOOP_JENKINS))
+  $(UT_OTHER))
 
 
 ifdef ONLOAD_TREE
@@ -112,7 +118,7 @@ target_variants_ = $(1) $(foreach v, $(variants), $(1)_$v)
 target_variants = $(foreach t,$(1),$(call target_variants_,$(t)))
 target_explode = $(call target_variants,$(1)) $(call target_variants,$(2:%=run_%))
 
-UT_NONSCRIPT_LISTS := testloop testloop2  testjenkins testmisc testdelegated testb2b testb2bsb testalts testrxx3
+UT_NONSCRIPT_LISTS := testloop testloop2 testmisc testdelegated testb2b testb2bsb testalts testrxx3
 UT_SCRIPT_LISTS := testscript testpacketdrill
 UT_ALL_LISTS = $(UT_NONSCRIPT_LISTS) $(UT_SCRIPT_LISTS)
 
@@ -128,14 +134,12 @@ UT_ALL_NONSCRIPT_TARGETS := $(UT_ALL_LOOP_TARGETS) $(UT_ALL_MISC_TARGETS) \
 		  $(UT_ALL_B2B_SB_TARGETS)
 UT_ALL_SCRIPT_TARGETS := $(call target_explode,testscript,$(UT_SCRIPTS)) \
                          $(call target_variants,testpacketdrill,$(UT_SCRIPTS))
-UT_ALL_JENKINS_LOOP_TARGETS := $(call target_explode,testjenkins,$(UT_LOOP_JENKINS))
 UT_ALL_TARGETS := $(UT_ALL_NONSCRIPT_TARGETS) \
                   $(UT_ALL_SCRIPT_TARGETS)
 
 
 # Associate targets of each list with their base configuration
 $(UT_ALL_LOOP_TARGETS) : ATTR_LIST += $(TEST_LOOP_ZF_ATTR)
-$(UT_ALL_JENKINS_LOOP_TARGETS) : ATTR_LIST += $(TEST_LOOP_ZF_ATTR)
 $(UT_ALL_MISC_TARGETS) : ATTR_LIST += $(TEST_LOOP_ZF_ATTR)
 # zfdelegated needs the send queue to be smaller than the TCP window, so that
 # send space is the limiting factor. Because the send queue is a fixed number
@@ -181,7 +185,6 @@ TEST_TIME_OUT=120
 
 $(call target_variants,testloop): $(UT_LOOP:%=$(UT_BINDIR)/%)
 $(call target_variants,testloop2): $(UT_LOOP2:%=$(UT_BINDIR)/%)
-$(call target_variants,testjenkins): $(UT_LOOP_JENKINS:%=$(UT_BINDIR)/%)
 $(call target_variants,testmisc): $(UT_MISC:%=$(UT_BINDIR)/%)
 $(call target_variants,testdelegated): $(UT_DELEGATED:%=$(UT_BINDIR)/%)
 $(call target_variants,testb2bsb): $(UT_B2B_SB:%=$(UT_BINDIR)/%)
@@ -204,8 +207,13 @@ buildattrs = ZF_ATTR="$(subst $(space),;,$(ATTR_LIST))"
 $(call target_variants,$(UT_NONSCRIPT_LISTS)):
 	@echo "Test target: $@"
 	sudo env $(UT_ENV) $(ENV) $(call buildattrs) \
-		UT_OUTPUT=$(UT_OUTPUT) TEST_TIME_OUT=$(TEST_TIME_OUT) TEST_TARGET="$@" \
-		timeout $(HARNESS_TIME_OUT) \
+		UT_OUTPUT=$(UT_OUTPUT) TEST_TARGET="$@" \
+		ZF_SLOW_TESTS="${UT_SLOW}" \
+		ZF_UNSTABLE_TESTS="${UT_UNSTABLE}" \
+		ZF_RUN_SLOW_TESTS=$${ZF_RUN_SLOW_TESTS:-1} \
+		ZF_RUN_UNSTABLE_TESTS=$${ZF_RUN_UNSTABLE_TESTS:-0} \
+		TEST_TIME_OUT=$$(($${TIMEOUT_MULTIPLIER:-1} * $(TEST_TIME_OUT))) \
+		timeout $$(($${TIMEOUT_MULTIPLIER:-1} * $(HARNESS_TIME_OUT))) \
 		prove --exec \
 		"$(NETNS_SCRIPT)" \
 		$(PROVE_FLAGS) \
@@ -214,8 +222,13 @@ $(call target_variants,$(UT_NONSCRIPT_LISTS)):
 $(call target_variants,$(UT_SCRIPT_LISTS)):
 	@echo "Test target: $@"
 	sudo env $(UT_ENV) $(ENV) EXTRA_$(call buildattrs) \
-		UT_OUTPUT=$(UT_OUTPUT) TEST_TIME_OUT=$(TEST_TIME_OUT) TEST_TARGET="$@" \
-		timeout $(HARNESS_TIME_OUT) \
+		UT_OUTPUT=$(UT_OUTPUT) TEST_TARGET="$@" \
+		ZF_SLOW_TESTS="${UT_SLOW}" \
+		ZF_UNSTABLE_TESTS="${UT_UNSTABLE}" \
+		ZF_RUN_SLOW_TESTS=$${ZF_RUN_SLOW_TESTS:-1} \
+		ZF_RUN_UNSTABLE_TESTS=$${ZF_RUN_UNSTABLE_TESTS:-0} \
+		TEST_TIME_OUT=$$(($${TIMEOUT_MULTIPLIER:-1} * $(TEST_TIME_OUT))) \
+		timeout $$(($${TIMEOUT_MULTIPLIER:-1} * $(HARNESS_TIME_OUT))) \
 		prove --exec \
 		"$(NETNS_SCRIPT)" \
 		$(PROVE_FLAGS) \
@@ -234,12 +247,9 @@ test: testloop \
       testloop_vlan testloop_txvi \
       testb2bsb testmisc \
       testdelegated_smallmtu \
-      testb2b testalts testalts_vlan testalts_txvi testpacketdrill
-
-test_jenkins: testjenkins \
-              testscript_x3 \
-              testscript
-
+      testb2b testalts testalts_vlan \
+      testalts_txvi testpacketdrill \
+      testscript_x3 testscript
 
 .PHONY: zf_unit test
 zf_unit: $(UT_BINS)

src/tests/zf_unit/netns.sh

@@ -23,6 +23,26 @@ shift
 
 rm -f /dev/shm/zf_emu_*
 
+name_in_list()
+{
+  name=$1
+  list=$2
+  if [ "$(echo "$list" | sed -e "s/ /\n/g" | grep "$name")" = "$name" ]; then
+    return 0;
+  else
+    return 1;
+  fi
+}
+
+testapp_name=$(basename $1)
+if [ "${ZF_RUN_SLOW_TESTS:-1}" = "0" ] && (name_in_list "$testapp_name" "$ZF_SLOW_TESTS"); then
+  echo "1..0 # SKIP not running $testapp_name as it is considered slow. Set ZF_RUN_SLOW_TESTS=1 to run it."
+  exit 0
+elif [ "${ZF_RUN_UNSTABLE_TESTS:-0}" = "0" ] && (name_in_list "$testapp_name" "$ZF_UNSTABLE_TESTS"); then
+  echo "1..0 # SKIP not running $testapp_name as it is considered unstable. Set ZF_RUN_UNSTABLE_TESTS=1 to run it."
+  exit 0
+fi
+
 output="$(timeout ${TEST_TIME_OUT:-120} $@)"
 rc="$?"
 echo "$output"