Merge pull request #15018 from omoerbeek/rec-bulktest-ubicloud

rec: add bigger (and v6 enabled) bulk test on ubicloud

.github/workflows/build-and-test-all.yml

@@ -623,6 +623,7 @@ jobs:
         threads: [1, 2, 3, 4, 8]
         mthreads: [2048]
         shards: [1, 2, 1024]
+        IPv6: [0]
     container:
       image: "${{ needs.get-runner-container-image.outputs.id }}:${{ needs.get-runner-container-image.outputs.tag }}"
       env:
@@ -644,7 +645,7 @@ jobs:
           path: /opt/pdns-recursor
       - run: inv install-clang-runtime
       - run: inv install-rec-bulk-deps
-      - run: inv test-bulk-recursor ${{ matrix.threads }} ${{ matrix.mthreads }} ${{ matrix.shards }}
+      - run: inv test-bulk-recursor 100 ${{ matrix.threads }} ${{ matrix.mthreads }} ${{ matrix.shards }} ${{ matrix.IPv6 }}
       - run: inv generate-coverage-info /opt/pdns-recursor/sbin/pdns_recursor $GITHUB_WORKSPACE
         if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
       - name: Coveralls Parallel recursor bulk
@@ -657,6 +658,51 @@ jobs:
           allow-empty: true
           fail-on-error: false
 
+  test-recursor-ubicloud-bulk:
+    if: ${{ vars.REC_BULKTEST_USE_UBICLOUD == '1' || github.repository == 'PowerDNS/pdns' }}
+    name: 'test rec ubicloud bulk'
+    needs:
+      - build-recursor
+    runs-on: ubicloud-standard-8-ubuntu-2404
+    strategy:
+      matrix:
+        sanitizers: [ubsan+asan] # TSAN disabled for now
+        threads: [8]
+        mthreads: [2048]
+        shards: [1024]
+        IPv6: [0, 1]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 5
+          submodules: recursive
+          ref: ${{ inputs.branch-name }}
+      - run: echo "normalized-branch-name=${{ inputs.branch-name || github.ref_name }}" | tr "/" "-" >> "$GITHUB_ENV"
+      - name: Fetch the binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: pdns-recursor-full-${{ matrix.sanitizers }}-${{ env.normalized-branch-name }}
+          path: /opt/pdns-recursor
+      - run: build-scripts/gh-actions-setup-inv-no-dist-upgrade
+      - run: inv install-rec-bulk-ubicloud-deps
+      - run: inv test-bulk-recursor 50000 ${{ matrix.threads }} ${{ matrix.mthreads }} ${{ matrix.shards }} ${{ matrix.IPv6 }}
+        env:
+          UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1:suppressions=${{ github.workspace }}/build-scripts/UBSan.supp'
+          ASAN_OPTIONS: detect_leaks=0
+          TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/recursordist/recursor-tsan.supp"
+      #  Disabled, it gives us: "/bin/bash: line 1: llvm-profdata-13: command not found" due to mismatch between deb and ubuntu versions
+      #- run: inv generate-coverage-info /opt/pdns-recursor/sbin/pdns_recursor $GITHUB_WORKSPACE
+      #  if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
+      #- name: Coveralls Parallel recursor bulk
+      #  if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
+      #  uses: coverallsapp/github-action@v2
+      #  with:
+      #    flag-name: rec-regression-bulk-full-${{ matrix.sanitizers }}
+      #    path-to-lcov: $GITHUB_WORKSPACE/coverage.lcov
+      #    parallel: true
+      #    allow-empty: true
+      #    fail-on-error: false
+
   test-dnsdist-regression:
     needs:
       - build-dnsdist
@@ -736,6 +782,7 @@ jobs:
       - test-recursor-api
       - test-recursor-regression
       - test-recursor-bulk
+      - test-recursor-ubicloud-bulk
     if: success() || failure()
     runs-on: ubuntu-22.04
     steps:

regression-tests/recursor-test

@@ -17,11 +17,17 @@ shards=$5
 : ${TRACE:="fail"}
 : ${DNSBULKTEST:="../pdns/dnsbulktest"}
 
+echo Current working dir is `pwd`
+
 if [ $IPv6 = 1 ]
 then
-        QLA6=" ::"
+    echo \$ dig -6 @k.root-servers.net . SOA
+    dig -6 @k.root-servers.net . SOA # Do we actually have v6 connectivity?
+    QLA6=" ::"
 else
-        QLA6=""
+    echo \$ dig -4 @k.root-servers.net . SOA
+    dig -4 @k.root-servers.net . SOA # Do we actually have v4 connectivity?
+    QLA6=""
 fi
 
 rm -f recursor.pid pdns_recursor.pid
@@ -42,7 +48,7 @@ fi
 
 # warm up the cache
 echo
-echo === First run with limit=$limit threads=$threads mthreads=$mthreads shards=$shards ===
+echo === First run with IPv6=$IPv6 limit=$limit threads=$threads mthreads=$mthreads shards=$shards ===
 ${DNSBULKTEST} --www=false -qe 127.0.0.1 $port $limit < ${CSV} > bulktest.results
 kill -USR1 $(cat pdns_recursor.pid) || true
 ${RECCONTROL} --timeout=20 --socket-dir=. --config-dir=. get-all || true
@@ -51,7 +57,7 @@ sleep 5
 
 # rerun 1 with hot cache
 echo
-echo === Second run with limit=$limit threads=$threads mthreads=$mthreads shards=$shards ===
+echo === Second run with IPv6=$IPv6 limit=$limit threads=$threads mthreads=$mthreads shards=$shards ===
 ${DNSBULKTEST} --www=false -qe 127.0.0.1 $port $limit < ${CSV} > bulktest.results
 kill -USR1 $(cat pdns_recursor.pid) || true
 ${RECCONTROL} --timeout=20 --socket-dir=. --config-dir=. get-all || true
@@ -60,7 +66,7 @@ sleep 5
 
 # rerun 2 with hot cache
 echo
-echo === Third run with limit=$limit threads=$threads mthreads=$mthreads shards=$shards ===
+echo === Third run with IPv6=$IPv6 limit=$limit threads=$threads mthreads=$mthreads shards=$shards ===
 ${DNSBULKTEST} --www=false -qe 127.0.0.1 $port $limit < ${CSV} > bulktest.results
 kill -USR1 $(cat pdns_recursor.pid) || true
 ${RECCONTROL} --timeout=20 --socket-dir=. --config-dir=. get-all || true
@@ -73,7 +79,7 @@ cat recursor.log
 echo "=== END RECURSOR LOG ==="
 sleep 1
 ${RECCONTROL} --timeout=20 --socket-dir=. --config-dir=. ping
-kill $(cat pdns_recursor.pid) 
+kill $(cat pdns_recursor.pid)
 sleep 5
 
 . ./bulktest.results

tasks.py

@@ -60,6 +60,7 @@
 ]
 rec_bulk_deps = [
     'curl',
+    'dnsutils',
     'libboost-all-dev',
     'libcap2',
     'libfstrm0',
@@ -71,6 +72,22 @@
     'pdns-tools',
     'unzip',
 ]
+rec_bulk_ubicloud_deps = [
+    'curl',
+    'dnsutils',
+    'libboost-context1.74.0',
+    'libboost-system1.74.0',
+    'libboost-filesystem1.74.0',
+    'libcap2',
+    'libfstrm0',
+    'libluajit-5.1-2',
+    '"libsnmp[1-9]+"',
+    'libsodium23',
+    'libsystemd0',
+    'moreutils',
+    'pdns-tools',
+    'unzip',
+]
 dnsdist_build_deps = [
     'libcap-dev',
     'libcdb-dev',
@@ -267,6 +284,11 @@ def install_rec_bulk_deps(c): # FIXME: rename this, we do way more than apt-get
     c.sudo('apt-get --no-install-recommends -y install ' + ' '.join(rec_bulk_deps))
     c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
 
+@task
+def install_rec_bulk_ubicloud_deps(c): # FIXME: rename this, we do way more than apt-get
+    c.sudo('apt-get --no-install-recommends -y install ' + ' '.join(rec_bulk_ubicloud_deps))
+    c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
+
 @task
 def install_rec_test_deps(c): # FIXME: rename this, we do way more than apt-get
     c.sudo('apt-get --no-install-recommends install -y ' + ' '.join(rec_bulk_deps) + ' \
@@ -1044,13 +1066,12 @@ def test_regression_recursor(c):
     c.run('PDNSRECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control ./build-scripts/test-recursor')
 
 @task
-def test_bulk_recursor(c, threads, mthreads, shards):
-    # We run an extremely small version of the bulk test, as GH does not seem to be able to handle the UDP load
+def test_bulk_recursor(c, size, threads, mthreads, shards, ipv6):
     with c.cd('regression-tests'):
-        c.run('curl -LO http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip')
+        c.run('curl --no-progress-meter -LO http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip')
         c.run('unzip top-1m.csv.zip -d .')
         c.run('chmod +x /opt/pdns-recursor/bin/* /opt/pdns-recursor/sbin/*')
-        c.run(f'DNSBULKTEST=/usr/bin/dnsbulktest RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control THRESHOLD=95 TRACE=no ./recursor-test 5300 100 {threads} {mthreads} {shards}')
+        c.run(f'DNSBULKTEST=/usr/bin/dnsbulktest RECURSOR=/opt/pdns-recursor/sbin/pdns_recursor RECCONTROL=/opt/pdns-recursor/bin/rec_control IPv6={ipv6} THRESHOLD=95 TRACE=no ./recursor-test 5300 {size} {threads} {mthreads} {shards}')
 
 @task
 def install_swagger_tools(c):