From 62ebfc1af318b3bdc749fe75ad1bce407273ae54 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Thu, 12 Dec 2024 11:35:57 +0100 Subject: [PATCH 1/8] Replace Github environment usage in Download Action. --- .../.github/workflows/download_pipeline.yml | 43 +++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 05397358cf..b828d84be9 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -28,10 +28,14 @@ env: NXF_ANSI_LOG: false jobs: - download: - runs-on: ubuntu-latest + configure: + runs-on: ubuntu-latest{% raw %} + outputs: + REPO_LOWERCASE: ${{ steps.get_repo_properties.outputs.REPO_LOWERCASE }} + REPOTITLE_LOWERCASE: ${{ steps.get_repo_properties.outputs.REPOTITLE_LOWERCASE }} + REPO_BRANCH: ${{ steps.get_repo_properties.outputs.REPO_BRANCH }} steps: - - name: Install Nextflow + - name: Install Nextflow{% endraw %} uses: nf-core/setup-nextflow@v2 - name: Disk space cleanup @@ -53,22 +57,27 @@ jobs: pip install git+https://github.com/nf-core/tools.git@dev - name: Get the repository name and current branch set as environment variable + id: get_repo_properties run: | - echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV} - echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV} - echo "{% raw %}REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> ${GITHUB_ENV} + echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT" + echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> "$GITHUB_OUTPUT" + echo "{% raw %}REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> "$GITHUB_OUTPUT" - name: Make a cache directory for the container images run: | mkdir -p ./singularity_container_images + download: + runs-on: ubuntu-latest + needs: configure + steps: - name: Download the pipeline env: NXF_SINGULARITY_CACHEDIR: ./singularity_container_images run: | - nf-core pipelines download ${{ env.REPO_LOWERCASE }} \ - --revision ${{ env.REPO_BRANCH }} \ - --outdir ./${{ env.REPOTITLE_LOWERCASE }} \ + nf-core pipelines download ${{ needs.configure.outputs.REPO_LOWERCASE }} \ + --revision ${{ needs.configure.outputs.REPO_BRANCH }} \ + --outdir ./${{ needs.configure.outputs.REPOTITLE_LOWERCASE }} \ --compress "none" \ --container-system 'singularity' \ --container-library "quay.io" -l "docker.io" -l "community.wave.seqera.io/library/" \ @@ -76,14 +85,14 @@ jobs: --download-configuration 'yes' - name: Inspect download - run: tree ./${{ env.REPOTITLE_LOWERCASE }}{% endraw %}{% if test_config %}{% raw %} + run: tree ./${{ needs.configure.outputs.REPOTITLE_LOWERCASE }}{% endraw %}{% if test_config %}{% raw %} - name: Count the downloaded number of container images id: count_initial run: | image_count=$(ls -1 ./singularity_container_images | wc -l | xargs) echo "Initial container image count: $image_count" - echo "IMAGE_COUNT_INITIAL=$image_count" >> ${GITHUB_ENV} + echo "IMAGE_COUNT_INITIAL=$image_count" >> "$GITHUB_OUTPUT" - name: Run the downloaded pipeline (stub) id: stub_run_pipeline @@ -91,27 +100,27 @@ jobs: env: NXF_SINGULARITY_CACHEDIR: ./singularity_container_images NXF_SINGULARITY_HOME_MOUNT: true - run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results + run: nextflow run ./${{needs.configure.outputs.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ needs.configure.outputs.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results - name: Run the downloaded pipeline (stub run not supported) id: run_pipeline if: ${{ job.steps.stub_run_pipeline.status == failure() }} env: NXF_SINGULARITY_CACHEDIR: ./singularity_container_images NXF_SINGULARITY_HOME_MOUNT: true - run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results + run: nextflow run ./${{ needs.configure.outputs.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ needs.configure.outputs.REPO_BRANCH }}) -profile test,singularity --outdir ./results - name: Count the downloaded number of container images id: count_afterwards run: | image_count=$(ls -1 ./singularity_container_images | wc -l | xargs) echo "Post-pipeline run container image count: $image_count" - echo "IMAGE_COUNT_AFTER=$image_count" >> ${GITHUB_ENV} + echo "IMAGE_COUNT_AFTER=$image_count" >> "$GITHUB_OUTPUT" - name: Compare container image counts run: | - if [ "${{ env.IMAGE_COUNT_INITIAL }}" -ne "${{ env.IMAGE_COUNT_AFTER }}" ]; then - initial_count=${{ env.IMAGE_COUNT_INITIAL }} - final_count=${{ env.IMAGE_COUNT_AFTER }} + if [ "${{ steps.count_initial.outputs.IMAGE_COUNT_INITIAL }}" -ne "${{ steps.count_afterwards.outputs.IMAGE_COUNT_AFTER }}" ]; then + initial_count=${{ steps.count_initial.outputs.IMAGE_COUNT_INITIAL }} + final_count=${{ steps.count_afterwards.outputs.IMAGE_COUNT_AFTER }} difference=$((final_count - initial_count)) echo "$difference additional container images were \n downloaded at runtime . The pipeline has no support for offline runs!" tree ./singularity_container_images From 060eff51c3562fef14f188678f2a8eea2a8a05a7 Mon Sep 17 00:00:00 2001 From: Matthias Zepper <6963520+MatthiasZepper@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:58:08 +0100 Subject: [PATCH 2/8] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- .../pipeline-template/.github/workflows/download_pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index b828d84be9..d6b9e78d2b 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -61,7 +61,7 @@ jobs: run: | echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT" echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> "$GITHUB_OUTPUT" - echo "{% raw %}REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> "$GITHUB_OUTPUT" + echo "{% raw %}REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> "$GITHUB_OUTPUT"{% endraw %} - name: Make a cache directory for the container images run: | @@ -103,7 +103,7 @@ jobs: run: nextflow run ./${{needs.configure.outputs.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ needs.configure.outputs.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results - name: Run the downloaded pipeline (stub run not supported) id: run_pipeline - if: ${{ job.steps.stub_run_pipeline.status == failure() }} + if: ${{ steps.stub_run_pipeline.outcome == 'failure' }} env: NXF_SINGULARITY_CACHEDIR: ./singularity_container_images NXF_SINGULARITY_HOME_MOUNT: true From a8065da8a87bcc888530e03bb645d12909803f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 12 Dec 2024 10:59:54 +0100 Subject: [PATCH 3/8] Revert "don't set up loggin level to error [no changelog]" --- nf_core/pipelines/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/sync.py b/nf_core/pipelines/sync.py index a75ddef33b..781b4f5f00 100644 --- a/nf_core/pipelines/sync.py +++ b/nf_core/pipelines/sync.py @@ -261,7 +261,7 @@ def make_template_pipeline(self): log.info("Making a new template pipeline using pipeline variables") # Only show error messages from pipeline creation - # logging.getLogger("nf_core.pipelines.create").setLevel(logging.ERROR) + logging.getLogger("nf_core.pipelines.create").setLevel(logging.ERROR) assert self.config_yml_path is not None assert self.config_yml is not None From 4958cf0135b6257d14bd4ef7c84c0e21f12497dc Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 10 Dec 2024 15:40:28 +0100 Subject: [PATCH 4/8] Be more verbose in approval check action --- .../.github/workflows/awsfulltest.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 45c2a0e555..2a3663bdad 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -19,19 +19,29 @@ jobs: if: github.repository == '{{ name }}' && github.event.review.state == 'approved' && github.event.pull_request.base.ref == 'master' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - - uses: octokit/request-action@v2.x + - name: Get PR reviews + uses: octokit/request-action@v2.x if: github.event_name != 'workflow_dispatch' id: check_approvals with: route: GET /repos/{%- raw -%}${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews?per_page=100 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - id: test_variables - if: github.event_name != 'workflow_dispatch' + + - name: Check for approvals + if: steps.check_approvals.outputs == '' + run: | + echo "No approvals found" + exit 1 + + - name: Check for enough approvals (>=2) + id: test_variables + if: github.event_name != 'workflow_dispatch' && steps.check_approvals.outputs != '' run: | JSON_RESPONSE='${{ steps.check_approvals.outputs.data }}'{% endraw %} CURRENT_APPROVALS_COUNT=$(echo $JSON_RESPONSE | jq -c '[.[] | select(.state | contains("APPROVED")) ] | length') test $CURRENT_APPROVALS_COUNT -ge 2 || exit 1 # At least 2 approvals are required + - name: Launch workflow via Seqera Platform uses: seqeralabs/action-tower-launch@v2 # TODO nf-core: You can customise AWS full pipeline tests as required From 4c32e4795214f847760a4e91482f682fa769dd5f Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 10 Dec 2024 16:00:03 +0100 Subject: [PATCH 5/8] fix if clause --- nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 2a3663bdad..cd39124900 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -29,14 +29,14 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Check for approvals - if: steps.check_approvals.outputs == '' + if: ${{ failure() && github.event_name != 'workflow_dispatch' }} run: | - echo "No approvals found" + echo "No review approvals found. At least 2 approvals are required to run this action automatically." exit 1 - name: Check for enough approvals (>=2) id: test_variables - if: github.event_name != 'workflow_dispatch' && steps.check_approvals.outputs != '' + if: github.event_name != 'workflow_dispatch' run: | JSON_RESPONSE='${{ steps.check_approvals.outputs.data }}'{% endraw %} CURRENT_APPROVALS_COUNT=$(echo $JSON_RESPONSE | jq -c '[.[] | select(.state | contains("APPROVED")) ] | length') From e32293d1508ae1590183a16b70e9aedbc964ee59 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 10 Dec 2024 16:01:04 +0100 Subject: [PATCH 6/8] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c23bd96442..603fa77e24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - Add `manifest.contributors` to `nextflow.config` ([#3311](https://github.com/nf-core/tools/pull/3311)) - Update template components ([#3328](https://github.com/nf-core/tools/pull/3328)) - Template: Remove mention of GRCh37 if igenomes is skipped ([#3330](https://github.com/nf-core/tools/pull/3330)) +- Be more verbose in approval check action ([#3338](https://github.com/nf-core/tools/pull/3338)) ### Download From f02d6bb5f96cee457a2fa2526808875349c6afb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 12 Dec 2024 11:14:52 +0100 Subject: [PATCH 7/8] Update nf_core/pipeline-template/.github/workflows/awsfulltest.yml --- nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index cd39124900..6805c83a27 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -23,6 +23,7 @@ jobs: uses: octokit/request-action@v2.x if: github.event_name != 'workflow_dispatch' id: check_approvals + continue-on-error: true with: route: GET /repos/{%- raw -%}${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews?per_page=100 env: From 76ac617b291caa37a53017c8758a0dc938f4c8b1 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 13 Dec 2024 14:11:56 +0100 Subject: [PATCH 8/8] Remove endraw statement that 'nf-core pipelines' create does not like. --- CHANGELOG.md | 2 ++ .../pipeline-template/.github/workflows/download_pipeline.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c9af42a9a..587ba99faf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Use outputs instead of the environment to pass around values between steps in the Download Test Action ([#3351](https://github.com/nf-core/tools/pull/3351)) + ### Download ### Linting diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index d6b9e78d2b..f270dc5411 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -61,7 +61,7 @@ jobs: run: | echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT" echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> "$GITHUB_OUTPUT" - echo "{% raw %}REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> "$GITHUB_OUTPUT"{% endraw %} + echo "{% raw %}REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> "$GITHUB_OUTPUT" - name: Make a cache directory for the container images run: |