[DO NOT MERGE] Test dynamic sharding step

.github/actions/get-shards/action.yml

@@ -0,0 +1,72 @@
+name: "Get number of shards"
+description: "Get the number of nf-test shards for the current CI job"
+inputs:
+  tags:
+    description: "Tags to pass as argument for nf-test --tag parameter"
+    required: true
+  max_shards:
+    description: "Maximum number of shards allowed"
+    required: true
+  paths:
+    description: "Component paths to test"
+    required: false
+outputs:
+  shard:
+    description: "Array of shard numbers"
+    value: ${{ steps.shards.outputs.shard }}
+  total_shards:
+    description: "Total number of shards"
+    value: ${{ steps.shards.outputs.total_shards }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install nf-test
+      uses: nf-core/setup-nf-test@v1
+      with:
+        version: ${{ env.NFT_VER }}
+        install-pdiff: true
+
+    - name: Get number of shards
+      id: shards
+      shell: bash
+      run: |
+
+        # Prepare tag parameter if tags are provided
+        TAGS=$([ -n "${{ inputs.tags }}" ] && echo "--tag ${{ inputs.tags }}" || echo "")
+        # Run nf-test with dynamic tag parameter
+        nftest_output=$(nf-test test \
+          --dry-run \
+          --profile docker \
+          ${TAGS} \
+          --filter process,workflow \
+          ${{ inputs.paths }})
+
+        echo "nf-test dry-run output: $nftest_output"
+
+        # Default values for shard and total_shards
+        shard="[]"
+        total_shards=0
+
+        # Check if there are related tests
+        if echo "$nftest_output" | grep -q 'No tests to execute'; then
+          echo "No related tests found."
+        else
+          # Extract the number of related tests
+          number_of_shards=$(echo "$nftest_output" | sed -n 's|.*Executed \([0-9]*\) tests.*|\1|p')
+          if [[ -n "$number_of_shards" && "$number_of_shards" -gt 0 ]]; then
+            shards_to_run=$(( $number_of_shards < ${{ inputs.max_shards }} ? $number_of_shards : ${{ inputs.max_shards }} ))
+            shard=$(seq 1 "$shards_to_run" | jq -R . | jq -c -s .)
+            total_shards="$shards_to_run"
+          else
+            echo "Unexpected output format. Falling back to default values."
+          fi
+        fi
+
+        # Write to GitHub Actions outputs
+        echo "shard=$shard" >> $GITHUB_OUTPUT
+        echo "total_shards=$total_shards" >> $GITHUB_OUTPUT
+
+        # Debugging output
+        echo "Final shard array: $shard"
+        echo "Total number of shards: $total_shards"

.github/actions/nf-test-action/action.yml

@@ -74,6 +74,9 @@ runs:
         SENTIEON_AUTH_MECH: "GitHub Actions - token"
       run: |
         NFT_WORKDIR=~ \
+        NFT_DIFF=pdiff \
+        NFT_DIFF_ARGS="--line-numbers --expand-tabs=2"
+
         nf-test test \
           --profile=${{ inputs.profile }} \
           --tap=test.tap \

.github/skip_nf_test.yml

@@ -0,0 +1,104 @@
+conda:
+  modules:
+    - modules/nf-core/angsd/gl
+    - modules/nf-core/annotsv/installannotations
+    - modules/nf-core/happy/sompy
+    - modules/nf-core/backsub
+    - modules/nf-core/bakta/bakta
+    - modules/nf-core/bakta/baktadbdownload
+    - modules/nf-core/bases2fastq
+    - modules/nf-core/bcl2fastq
+    - modules/nf-core/bclconvert
+    - modules/nf-core/celesta
+    - modules/nf-core/cellpose
+    - modules/nf-core/cellranger/count
+    - modules/nf-core/cellranger/mkfastq
+    - modules/nf-core/cellranger/mkgtf
+    - modules/nf-core/cellranger/mkref
+    - modules/nf-core/cellranger/mkvdjref
+    - modules/nf-core/cellranger/multi
+    - modules/nf-core/cellranger/vdj
+    - modules/nf-core/checkqc
+    - modules/nf-core/custom/dumpsoftwareversions
+    - modules/nf-core/deepcell/mesmer
+    - modules/nf-core/deepsomatic
+    - modules/nf-core/deepvariant
+    - modules/nf-core/deepvariant/callvariants
+    - modules/nf-core/deepvariant/makeexamples
+    - modules/nf-core/deepvariant/postprocessvariants
+    - modules/nf-core/deepvariant/rundeepvariant
+    - modules/nf-core/deepvariant/vcfstatsreport
+    - modules/nf-core/doubletdetection
+    - modules/nf-core/ensemblvep/vep
+    - modules/nf-core/fastk/fastk
+    - modules/nf-core/cellrangerarc/mkgtf
+    - modules/nf-core/fastk/histex
+    - modules/nf-core/fastk/merge
+    - modules/nf-core/fcs/fcsadaptor
+    - modules/nf-core/fcs/fcsgx
+    - modules/nf-core/ganon/buildcustom
+    - modules/nf-core/ganon/classify
+    - modules/nf-core/ganon/report
+    - modules/nf-core/ganon/table
+    - modules/nf-core/gatk4/cnnscorevariants
+    - modules/nf-core/gatk4/determinegermlinecontigploidy
+    - modules/nf-core/genescopefk
+    - modules/nf-core/ilastik/multicut
+    - modules/nf-core/ilastik/pixelclassification
+    - modules/nf-core/imputeme/vcftoprs
+    - modules/nf-core/mcstaging/imc2mc
+    - modules/nf-core/mcquant
+    - modules/nf-core/mcstaging/phenoimager2mc
+    - modules/nf-core/merquryfk/katcomp
+    - modules/nf-core/merquryfk/katgc
+    - modules/nf-core/merquryfk/merquryfk
+    - modules/nf-core/merquryfk/ploidyplot
+    - modules/nf-core/molkartgarage/clahe
+    - modules/nf-core/quartonotebook
+    - modules/nf-core/scimap/spatiallda
+    - modules/nf-core/sentieon/bwaindex
+    - modules/nf-core/sentieon/bwamem
+    - modules/nf-core/sentieon/datametrics
+    - modules/nf-core/sentieon/dedup
+    - modules/nf-core/sentieon/qualcal
+    - modules/nf-core/spaceranger/mkgtf
+    - modules/nf-core/spaceranger/mkref
+    - modules/nf-core/spaceranger/count
+    - modules/nf-core/spotiflow
+    - modules/nf-core/svanalyzer/svbenchmark
+    - modules/nf-core/universc
+    - modules/nf-core/vt/decompose
+    - modules/nf-core/wittyer
+    - modules/nf-core/islandpath
+    - modules/nf-core/scimap/mcmicro
+    - modules/nf-core/parabricks/fq2bammeth
+    - modules/nf-core/parabricks/fq2bam
+    - modules/nf-core/xeniumranger/relabel
+    - modules/nf-core/xeniumranger/rename
+    - modules/nf-core/xeniumranger/resegment
+    - modules/nf-core/xeniumranger/import-segmentation
+    - modules/nf-core/parabricks/mutectcaller
+    - modules/nf-core/parabricks/dbsnp
+    - modules/nf-core/parabricks/indexgvcf
+    - modules/nf-core/parabricks/genotypegvcf
+    - modules/nf-core/parabricks/applybqsr
+  subworkflows:
+    - subworkflows/nf-core/vcf_annotate_ensemblvep
+    - subworkflows/nf-core/bcl_demultiplex
+    - subworkflows/nf-core/deepvariant
+    - subworkflows/nf-core/fastq_align_bamcmp_bwa
+    - subworkflows/nf-core/fastq_align_bwa
+    - subworkflows/nf-core/fasta_newick_epang_gappa
+docker_self_hosted:
+  modules:
+    - modules/nf-core/parabricks/fq2bammeth
+    - modules/nf-core/parabricks/fq2bam
+  subworkflows: []
+singularity:
+  modules:
+    - modules/nf-core/deepsomatic
+    - modules/nf-core/universc
+    - modules/nf-core/bases2fastq
+    - modules/nf-core/parabricks/fq2bammeth
+    - modules/nf-core/parabricks/fq2bam
+  subworkflows: []

.github/workflows/lint.yml

@@ -93,7 +93,7 @@ jobs:
           echo ${{ steps.module_names.outputs.result }}
 
   nf-core-lint-modules:
-    runs-on: ${{ github.event.inputs.runners  || 'ubuntu-latest' }}
+    runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
     name: nf-core lint modules
     needs: nf-core-changes
     if: ${{ (needs.nf-core-changes.outputs.modules == 'true') }}

.github/workflows/gpu-tests.yml → .github/workflows/nf-test-gpu.yml

@@ -40,7 +40,8 @@ jobs:
       paths: ${{ steps.list.outputs.components }}
       modules: ${{ steps.outputs.outputs.modules }}
       subworkflows: ${{ steps.outputs.outputs.subworkflows}}
-      # Prod for version bumping
+      shard: ${{ steps.set-shards.outputs.shard }}
+      total_shards: ${{ steps.set-shards.outputs.total_shards }}
     steps:
       - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
         run: |
@@ -66,25 +67,38 @@ jobs:
         run: |
           echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT
           echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
+
+      - name: get number of shards
+        id: set-shards
+        if: ${{ steps.list.outputs.components != '[]' }}
+        uses: ./.github/actions/get-shards
+        env:
+          NFT_VER: ${{ env.NFT_VER }}
+        with:
+          max_shards: 2
+          paths: "${{ join(fromJson(steps.list.outputs.components), ' ') }}"
+          tags: "gpu"
       - name: debug
         run: |
           echo ${{ steps.list.outputs.components }}
           echo ${{ steps.outputs.outputs.modules }}
           echo ${{ steps.outputs.outputs.subworkflows }}
+          echo ${{ steps.set-shards.outputs.shard }}
+          echo ${{ steps.set-shards.outputs.total_shards }}
 
   nf-test-gpu:
     runs-on: "gpu"
     name: "GPU Test | ${{ matrix.profile }} | ${{ matrix.shard }}"
-    needs: nf-test-changes
-    if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }}
+    needs: [nf-test-changes]
+    if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }}
     strategy:
       fail-fast: false
       matrix:
-        shard: [1, 2]
+        shard: ${{ fromJson(needs.nf-test-changes.outputs.shard) }}
         profile: [docker_self_hosted, singularity]
     env:
       NXF_ANSI_LOG: false
-      TOTAL_SHARDS: 2
+      TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }}
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4

.github/workflows/nf-test.yml

@@ -41,9 +41,10 @@ jobs:
     outputs:
       # Expose detected tags as 'modules' and 'workflows' output variables
       paths: ${{ steps.list.outputs.components }}
-      modules: ${{ steps.outputs.outputs.modules }}
-      subworkflows: ${{ steps.outputs.outputs.subworkflows}}
-      # Prod for version bumping
+      modules: ${{ steps.components.outputs.modules }}
+      subworkflows: ${{ steps.components.outputs.subworkflows}}
+      shard: ${{ steps.set-shards.outputs.shard }}
+      total_shards: ${{ steps.set-shards.outputs.total_shards }}
     steps:
       - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner
         run: |
@@ -65,28 +66,44 @@ jobs:
           exclude_tags: "gpu"
 
       - name: Separate modules and subworkflows
-        id: outputs
+        id: components
         run: |
           echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT
           echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
+
+      - name: get number of shards
+        id: set-shards
+        if: ${{ steps.list.outputs.components != '' }}
+        uses: ./.github/actions/get-shards
+        env:
+          NFT_VER: ${{ env.NFT_VER }}
+        with:
+          max_shards: 15
+          paths: "${{ join(fromJson(steps.list.outputs.components ), ' ') }}"
+
       - name: debug
         run: |
           echo ${{ steps.list.outputs.components }}
-          echo ${{ steps.outputs.outputs.modules }}
-          echo ${{ steps.outputs.outputs.subworkflows }}
+          echo ${{ steps.components.outputs.modules }}
+          echo ${{ steps.components.outputs.subworkflows }}
+          echo ${{ steps.set-shards.outputs.shard }}
+          echo ${{ steps.set-shards.outputs.total_shards }}
+
   nf-test:
-    runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }}
+    runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
     name: "Test | ${{ matrix.profile }} | ${{ matrix.shard }}"
-    needs: nf-test-changes
-    if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }}
+    needs: [nf-test-changes]
+    if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }}
     strategy:
       fail-fast: false
       matrix:
-        shard: [1, 2, 3, 4, 5]
+        shard: ${{ fromJson(needs.nf-test-changes.outputs.shard) }}
         profile: [conda, docker_self_hosted, singularity]
     env:
       NXF_ANSI_LOG: false
-      TOTAL_SHARDS: 5
+      TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }}
+    outputs:
+      filtered_paths: ${{ steps.filter.outputs.filtered_paths }}
 
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
@@ -129,6 +146,10 @@ jobs:
 
           echo "filtered_paths=${FILTERED}" >> $GITHUB_OUTPUT
 
+      - name: debug
+        run: |
+          echo filtered_paths = ${{ steps.filter.outputs.filtered_paths }}
+
       - name: Run nf-test Action
         if: ${{steps.filter.outputs.filtered_paths != '[]'}}
         uses: ./.github/actions/nf-test-action
@@ -156,12 +177,19 @@ jobs:
         if: ${{ contains(needs.*.result, 'cancelled') }}
         run: exit 1
 
+      - name: If no tests run, but filtered_paths was not empty and no tests succeeded, fail
+        if: ${{ needs.nf-test.outputs.filtered_paths != '' && !contains(needs.*.result, 'success') }}
+        run: exit 1
+
       - name: All tests ok
         if: ${{ contains(needs.*.result, 'success') }}
         run: exit 0
 
       - name: debug-print
         if: always()
         run: |
-          echo "toJSON(needs) = ${{ toJSON(needs) }}"
-          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
+          echo "::group::DEBUG: `needs` Contents"
+          echo "DEBUG: toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "DEBUG: toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"
+          echo "DEBUG: needs.nf-test.outputs.filtered_paths = ${{ needs.nf-test.outputs.filtered_paths }}"
+          echo "::endgroup::"

.github/workflows/pytest-workflow.yml

@@ -62,7 +62,7 @@ jobs:
           echo ${{ steps.tags.outputs.subworkflows }}
 
   pytest:
-    runs-on: ${{ github.event.inputs.runners  || 'ubuntu-latest' }}
+    runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
     name: pytest
     needs: [pytest-changes]
     if: needs.pytest-changes.outputs.tags != '[]'
@@ -320,7 +320,7 @@ jobs:
             !/home/ubuntu/pytest_workflow_*/*/work/singularity
             !${{ github.workspace }}/.singularity
 
-  confirm-pass:
+  confirm-pass-pytest:
     runs-on: ubuntu-latest
     needs: [pytest-changes, pytest]
     if: always()

modules/nf-core/cellranger/count/main.nf

@@ -11,6 +11,8 @@ process CELLRANGER_COUNT {
     output:
     tuple val(meta), path("**/outs/**"), emit: outs
     path "versions.yml"                , emit: versions
+    // Test modification to trigger GitHub Actions path filtering
+    // This change is to verify the skip_nf_test.yml filtering mechanism
 
     when:
     task.ext.when == null || task.ext.when

modules/nf-core/parabricks/fq2bam/main.nf

@@ -29,6 +29,12 @@ process PARABRICKS_FQ2BAM {
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
         error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead."
     }
+
+
+    // Test modification to trigger GitHub Actions path filtering
+    // This change is to verify the skip_nf_test.yml filtering mechanism
+    // FIXME
+
     def args = task.ext.args ?: ''
     def prefix = task.ext.prefix ?: "${meta.id}"
     def in_fq_command = meta.single_end ? "--in-se-fq $reads" : "--in-fq $reads"