diff --git a/.github/actions/get-shards/action.yml b/.github/actions/get-shards/action.yml new file mode 100644 index 00000000000..c2b511c2672 --- /dev/null +++ b/.github/actions/get-shards/action.yml @@ -0,0 +1,72 @@ +name: "Get number of shards" +description: "Get the number of nf-test shards for the current CI job" +inputs: + tags: + description: "Tags to pass as argument for nf-test --tag parameter" + required: true + max_shards: + description: "Maximum number of shards allowed" + required: true + paths: + description: "Component paths to test" + required: false +outputs: + shard: + description: "Array of shard numbers" + value: ${{ steps.shards.outputs.shard }} + total_shards: + description: "Total number of shards" + value: ${{ steps.shards.outputs.total_shards }} + +runs: + using: "composite" + steps: + - name: Install nf-test + uses: nf-core/setup-nf-test@v1 + with: + version: ${{ env.NFT_VER }} + install-pdiff: true + + - name: Get number of shards + id: shards + shell: bash + run: | + + # Prepare tag parameter if tags are provided + TAGS=$([ -n "${{ inputs.tags }}" ] && echo "--tag ${{ inputs.tags }}" || echo "") + # Run nf-test with dynamic tag parameter + nftest_output=$(nf-test test \ + --dry-run \ + --profile docker \ + ${TAGS} \ + --filter process,workflow \ + ${{ inputs.paths }}) + + echo "nf-test dry-run output: $nftest_output" + + # Default values for shard and total_shards + shard="[]" + total_shards=0 + + # Check if there are related tests + if echo "$nftest_output" | grep -q 'No tests to execute'; then + echo "No related tests found." + else + # Extract the number of related tests + number_of_shards=$(echo "$nftest_output" | sed -n 's|.*Executed \([0-9]*\) tests.*|\1|p') + if [[ -n "$number_of_shards" && "$number_of_shards" -gt 0 ]]; then + shards_to_run=$(( $number_of_shards < ${{ inputs.max_shards }} ? $number_of_shards : ${{ inputs.max_shards }} )) + shard=$(seq 1 "$shards_to_run" | jq -R . | jq -c -s .) + total_shards="$shards_to_run" + else + echo "Unexpected output format. Falling back to default values." + fi + fi + + # Write to GitHub Actions outputs + echo "shard=$shard" >> $GITHUB_OUTPUT + echo "total_shards=$total_shards" >> $GITHUB_OUTPUT + + # Debugging output + echo "Final shard array: $shard" + echo "Total number of shards: $total_shards" diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index 4aa3410ba24..e941bc4160b 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -74,6 +74,9 @@ runs: SENTIEON_AUTH_MECH: "GitHub Actions - token" run: | NFT_WORKDIR=~ \ + NFT_DIFF=pdiff \ + NFT_DIFF_ARGS="--line-numbers --expand-tabs=2" + nf-test test \ --profile=${{ inputs.profile }} \ --tap=test.tap \ diff --git a/.github/skip_nf_test.yml b/.github/skip_nf_test.yml new file mode 100644 index 00000000000..e546c08bfc4 --- /dev/null +++ b/.github/skip_nf_test.yml @@ -0,0 +1,104 @@ +conda: + modules: + - modules/nf-core/angsd/gl + - modules/nf-core/annotsv/installannotations + - modules/nf-core/happy/sompy + - modules/nf-core/backsub + - modules/nf-core/bakta/bakta + - modules/nf-core/bakta/baktadbdownload + - modules/nf-core/bases2fastq + - modules/nf-core/bcl2fastq + - modules/nf-core/bclconvert + - modules/nf-core/celesta + - modules/nf-core/cellpose + - modules/nf-core/cellranger/count + - modules/nf-core/cellranger/mkfastq + - modules/nf-core/cellranger/mkgtf + - modules/nf-core/cellranger/mkref + - modules/nf-core/cellranger/mkvdjref + - modules/nf-core/cellranger/multi + - modules/nf-core/cellranger/vdj + - modules/nf-core/checkqc + - modules/nf-core/custom/dumpsoftwareversions + - modules/nf-core/deepcell/mesmer + - modules/nf-core/deepsomatic + - modules/nf-core/deepvariant + - modules/nf-core/deepvariant/callvariants + - modules/nf-core/deepvariant/makeexamples + - modules/nf-core/deepvariant/postprocessvariants + - modules/nf-core/deepvariant/rundeepvariant + - modules/nf-core/deepvariant/vcfstatsreport + - modules/nf-core/doubletdetection + - modules/nf-core/ensemblvep/vep + - modules/nf-core/fastk/fastk + - modules/nf-core/cellrangerarc/mkgtf + - modules/nf-core/fastk/histex + - modules/nf-core/fastk/merge + - modules/nf-core/fcs/fcsadaptor + - modules/nf-core/fcs/fcsgx + - modules/nf-core/ganon/buildcustom + - modules/nf-core/ganon/classify + - modules/nf-core/ganon/report + - modules/nf-core/ganon/table + - modules/nf-core/gatk4/cnnscorevariants + - modules/nf-core/gatk4/determinegermlinecontigploidy + - modules/nf-core/genescopefk + - modules/nf-core/ilastik/multicut + - modules/nf-core/ilastik/pixelclassification + - modules/nf-core/imputeme/vcftoprs + - modules/nf-core/mcstaging/imc2mc + - modules/nf-core/mcquant + - modules/nf-core/mcstaging/phenoimager2mc + - modules/nf-core/merquryfk/katcomp + - modules/nf-core/merquryfk/katgc + - modules/nf-core/merquryfk/merquryfk + - modules/nf-core/merquryfk/ploidyplot + - modules/nf-core/molkartgarage/clahe + - modules/nf-core/quartonotebook + - modules/nf-core/scimap/spatiallda + - modules/nf-core/sentieon/bwaindex + - modules/nf-core/sentieon/bwamem + - modules/nf-core/sentieon/datametrics + - modules/nf-core/sentieon/dedup + - modules/nf-core/sentieon/qualcal + - modules/nf-core/spaceranger/mkgtf + - modules/nf-core/spaceranger/mkref + - modules/nf-core/spaceranger/count + - modules/nf-core/spotiflow + - modules/nf-core/svanalyzer/svbenchmark + - modules/nf-core/universc + - modules/nf-core/vt/decompose + - modules/nf-core/wittyer + - modules/nf-core/islandpath + - modules/nf-core/scimap/mcmicro + - modules/nf-core/parabricks/fq2bammeth + - modules/nf-core/parabricks/fq2bam + - modules/nf-core/xeniumranger/relabel + - modules/nf-core/xeniumranger/rename + - modules/nf-core/xeniumranger/resegment + - modules/nf-core/xeniumranger/import-segmentation + - modules/nf-core/parabricks/mutectcaller + - modules/nf-core/parabricks/dbsnp + - modules/nf-core/parabricks/indexgvcf + - modules/nf-core/parabricks/genotypegvcf + - modules/nf-core/parabricks/applybqsr + subworkflows: + - subworkflows/nf-core/vcf_annotate_ensemblvep + - subworkflows/nf-core/bcl_demultiplex + - subworkflows/nf-core/deepvariant + - subworkflows/nf-core/fastq_align_bamcmp_bwa + - subworkflows/nf-core/fastq_align_bwa + - subworkflows/nf-core/fasta_newick_epang_gappa +docker_self_hosted: + modules: + - modules/nf-core/parabricks/fq2bammeth + - modules/nf-core/parabricks/fq2bam + subworkflows: [] +singularity: + modules: + - modules/nf-core/deepsomatic + - modules/nf-core/universc + - modules/nf-core/bases2fastq + - modules/nf-core/parabricks/fq2bammeth + - modules/nf-core/parabricks/fq2bam + subworkflows: [] diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a343cc64bf8..f40dc839d78 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -93,7 +93,7 @@ jobs: echo ${{ steps.module_names.outputs.result }} nf-core-lint-modules: - runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} name: nf-core lint modules needs: nf-core-changes if: ${{ (needs.nf-core-changes.outputs.modules == 'true') }} diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/nf-test-gpu.yml similarity index 83% rename from .github/workflows/gpu-tests.yml rename to .github/workflows/nf-test-gpu.yml index 2ada515c1e3..b5b0a8244f3 100644 --- a/.github/workflows/gpu-tests.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -40,7 +40,8 @@ jobs: paths: ${{ steps.list.outputs.components }} modules: ${{ steps.outputs.outputs.modules }} subworkflows: ${{ steps.outputs.outputs.subworkflows}} - # Prod for version bumping + shard: ${{ steps.set-shards.outputs.shard }} + total_shards: ${{ steps.set-shards.outputs.total_shards }} steps: - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner run: | @@ -66,25 +67,38 @@ jobs: run: | echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT + + - name: get number of shards + id: set-shards + if: ${{ steps.list.outputs.components != '[]' }} + uses: ./.github/actions/get-shards + env: + NFT_VER: ${{ env.NFT_VER }} + with: + max_shards: 2 + paths: "${{ join(fromJson(steps.list.outputs.components), ' ') }}" + tags: "gpu" - name: debug run: | echo ${{ steps.list.outputs.components }} echo ${{ steps.outputs.outputs.modules }} echo ${{ steps.outputs.outputs.subworkflows }} + echo ${{ steps.set-shards.outputs.shard }} + echo ${{ steps.set-shards.outputs.total_shards }} nf-test-gpu: runs-on: "gpu" name: "GPU Test | ${{ matrix.profile }} | ${{ matrix.shard }}" - needs: nf-test-changes - if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }} + needs: [nf-test-changes] + if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }} strategy: fail-fast: false matrix: - shard: [1, 2] + shard: ${{ fromJson(needs.nf-test-changes.outputs.shard) }} profile: [docker_self_hosted, singularity] env: NXF_ANSI_LOG: false - TOTAL_SHARDS: 2 + TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index 9774004aa51..797e53a33a6 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -41,9 +41,10 @@ jobs: outputs: # Expose detected tags as 'modules' and 'workflows' output variables paths: ${{ steps.list.outputs.components }} - modules: ${{ steps.outputs.outputs.modules }} - subworkflows: ${{ steps.outputs.outputs.subworkflows}} - # Prod for version bumping + modules: ${{ steps.components.outputs.modules }} + subworkflows: ${{ steps.components.outputs.subworkflows}} + shard: ${{ steps.set-shards.outputs.shard }} + total_shards: ${{ steps.set-shards.outputs.total_shards }} steps: - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner run: | @@ -65,28 +66,44 @@ jobs: exclude_tags: "gpu" - name: Separate modules and subworkflows - id: outputs + id: components run: | echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT + + - name: get number of shards + id: set-shards + if: ${{ steps.list.outputs.components != '' }} + uses: ./.github/actions/get-shards + env: + NFT_VER: ${{ env.NFT_VER }} + with: + max_shards: 15 + paths: "${{ join(fromJson(steps.list.outputs.components ), ' ') }}" + - name: debug run: | echo ${{ steps.list.outputs.components }} - echo ${{ steps.outputs.outputs.modules }} - echo ${{ steps.outputs.outputs.subworkflows }} + echo ${{ steps.components.outputs.modules }} + echo ${{ steps.components.outputs.subworkflows }} + echo ${{ steps.set-shards.outputs.shard }} + echo ${{ steps.set-shards.outputs.total_shards }} + nf-test: - runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} name: "Test | ${{ matrix.profile }} | ${{ matrix.shard }}" - needs: nf-test-changes - if: ${{ needs.nf-test-changes.outputs.modules != '[]' || needs.nf-test-changes.outputs.subworkflows != '[]' }} + needs: [nf-test-changes] + if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }} strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4, 5] + shard: ${{ fromJson(needs.nf-test-changes.outputs.shard) }} profile: [conda, docker_self_hosted, singularity] env: NXF_ANSI_LOG: false - TOTAL_SHARDS: 5 + TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }} + outputs: + filtered_paths: ${{ steps.filter.outputs.filtered_paths }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -129,6 +146,10 @@ jobs: echo "filtered_paths=${FILTERED}" >> $GITHUB_OUTPUT + - name: debug + run: | + echo filtered_paths = ${{ steps.filter.outputs.filtered_paths }} + - name: Run nf-test Action if: ${{steps.filter.outputs.filtered_paths != '[]'}} uses: ./.github/actions/nf-test-action @@ -156,6 +177,10 @@ jobs: if: ${{ contains(needs.*.result, 'cancelled') }} run: exit 1 + - name: If no tests run, but filtered_paths was not empty and no tests succeeded, fail + if: ${{ needs.nf-test.outputs.filtered_paths != '' && !contains(needs.*.result, 'success') }} + run: exit 1 + - name: All tests ok if: ${{ contains(needs.*.result, 'success') }} run: exit 0 @@ -163,5 +188,8 @@ jobs: - name: debug-print if: always() run: | - echo "toJSON(needs) = ${{ toJSON(needs) }}" - echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" + echo "::group::DEBUG: `needs` Contents" + echo "DEBUG: toJSON(needs) = ${{ toJSON(needs) }}" + echo "DEBUG: toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" + echo "DEBUG: needs.nf-test.outputs.filtered_paths = ${{ needs.nf-test.outputs.filtered_paths }}" + echo "::endgroup::" diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 0540ddf7940..ea36486f0b2 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -62,7 +62,7 @@ jobs: echo ${{ steps.tags.outputs.subworkflows }} pytest: - runs-on: ${{ github.event.inputs.runners || 'ubuntu-latest' }} + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} name: pytest needs: [pytest-changes] if: needs.pytest-changes.outputs.tags != '[]' @@ -320,7 +320,7 @@ jobs: !/home/ubuntu/pytest_workflow_*/*/work/singularity !${{ github.workspace }}/.singularity - confirm-pass: + confirm-pass-pytest: runs-on: ubuntu-latest needs: [pytest-changes, pytest] if: always() diff --git a/modules/nf-core/cellranger/count/main.nf b/modules/nf-core/cellranger/count/main.nf index cf94615b80b..4ed33e089ca 100644 --- a/modules/nf-core/cellranger/count/main.nf +++ b/modules/nf-core/cellranger/count/main.nf @@ -11,6 +11,8 @@ process CELLRANGER_COUNT { output: tuple val(meta), path("**/outs/**"), emit: outs path "versions.yml" , emit: versions + // Test modification to trigger GitHub Actions path filtering + // This change is to verify the skip_nf_test.yml filtering mechanism when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/parabricks/fq2bam/main.nf b/modules/nf-core/parabricks/fq2bam/main.nf index 0cd69048048..e7338034d60 100644 --- a/modules/nf-core/parabricks/fq2bam/main.nf +++ b/modules/nf-core/parabricks/fq2bam/main.nf @@ -29,6 +29,12 @@ process PARABRICKS_FQ2BAM { if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." } + + + // Test modification to trigger GitHub Actions path filtering + // This change is to verify the skip_nf_test.yml filtering mechanism + // FIXME + def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def in_fq_command = meta.single_end ? "--in-se-fq $reads" : "--in-fq $reads"