From f94e92cb5c4d81178fe803feaf357045ae199f3c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 24 Mar 2022 12:23:05 +0100 Subject: [PATCH 001/136] Bump v2.4dev --- CHANGELOG.md | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 399db4dc61..057c1f44f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # nf-core/tools: Changelog +## v2.4dev + +### Template + +### General + +### Modules + ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] Very minor patch release to fix the full size AWS tests and re-run the template sync, which partially failed due to GitHub pull-requests being down at the time of release. diff --git a/setup.py b/setup.py index 90949a1dd7..5fda0ae5be 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -version = "2.3.2" +version = "2.4dev" with open("README.md") as f: readme = f.read() From 888473a74071f6c57ec3eff446250f24334951d8 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 24 Mar 2022 16:49:28 +0100 Subject: [PATCH 002/136] Fix bug in readme logo path --- CHANGELOG.md | 2 ++ nf_core/create.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 057c1f44f3..454856ccdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Fix bug in pipeline readme logo URL + ### General ### Modules diff --git a/nf_core/create.py b/nf_core/create.py index 3fa3baeda0..d49f245fdd 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -39,8 +39,8 @@ def __init__(self, name, description, author, version="1.0dev", no_git=False, fo self.name = f"nf-core/{self.short_name}" self.name_noslash = self.name.replace("/", "-") self.name_docker = self.name.replace("nf-core", "nfcore") - self.logo_light = f"{self.name}_logo_light.png" - self.logo_dark = f"{self.name}_logo_dark.png" + self.logo_light = f"{self.name_noslash}_logo_light.png" + self.logo_dark = f"{self.name_noslash}_logo_dark.png" self.description = description self.author = author self.version = version From 993de0e3ea06e04213214f864bff487c9e5b9d7a Mon Sep 17 00:00:00 2001 From: Praveen Raj S <43108054+praveenraj2018@users.noreply.github.com> Date: Fri, 25 Mar 2022 15:11:04 +0100 Subject: [PATCH 003/136] Fix: Get file extensions correctly in validate_pair Noticed that when there are more dots in the FASTQ file name, the Path(file).suffixes returns all elements split by dot, which ends in list mismatch, therefore throwing an error. I think we should limit it to the last two items (according to VALID_FORMATS) for the extension check. Hope this PR helps to solve the problem. --- nf_core/pipeline-template/bin/check_samplesheet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/bin/check_samplesheet.py b/nf_core/pipeline-template/bin/check_samplesheet.py index 5473b624c6..d1a9919eec 100755 --- a/nf_core/pipeline-template/bin/check_samplesheet.py +++ b/nf_core/pipeline-template/bin/check_samplesheet.py @@ -98,7 +98,7 @@ def _validate_pair(self, row): if row[self._first_col] and row[self._second_col]: row[self._single_col] = False assert ( - Path(row[self._first_col]).suffixes == Path(row[self._second_col]).suffixes + Path(row[self._first_col]).suffixes[-2:] == Path(row[self._second_col]).suffixes[-2:] ), "FASTQ pairs must have the same file extensions." else: row[self._single_col] = True From 7a2f85dcfe55876498af44a54a28f833e6c7f974 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 28 Mar 2022 13:25:37 +0200 Subject: [PATCH 004/136] Bump minimum version of rich to 10.7.0 --- CHANGELOG.md | 2 ++ requirements.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 454856ccdb..e58c93fc93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### General +- Bumped the minimum version of `rich` from `v10` to `v10.7.0` + ### Modules ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] diff --git a/requirements.txt b/requirements.txt index 2c1d25763d..97ed43df61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,5 +11,5 @@ questionary>=1.8.0 requests requests_cache rich-click>=1.0.0 -rich>=10.0.0 +rich>=10.7.0 tabulate From e0c3bbae544bfd2211cf6fedd4ccd35d86c2263d Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Fri, 1 Apr 2022 10:06:59 +0200 Subject: [PATCH 005/136] Add a last newline to modules.json to avoid prettier error --- nf_core/modules/modules_command.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index a60291fcfa..8caac30bd0 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -301,6 +301,7 @@ def dump_modules_json(self, modules_json): modules_json_path = os.path.join(self.dir, "modules.json") with open(modules_json_path, "w") as fh: json.dump(modules_json, fh, indent=4) + fh.write("\n") def load_lint_config(self): """Parse a pipeline lint config file. From 08352e0df8dccac500642c0878137920ae25d9f5 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Fri, 1 Apr 2022 10:11:30 +0200 Subject: [PATCH 006/136] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e58c93fc93..2db203c576 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ ### Modules +- Add an empty line at the end of the `modules.json` when dumping it to avoid prettier error. + ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] Very minor patch release to fix the full size AWS tests and re-run the template sync, which partially failed due to GitHub pull-requests being down at the time of release. From 2e941aa4d7a305a891c320d766450e6ad77cc949 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 1 Apr 2022 11:08:04 +0200 Subject: [PATCH 007/136] Remove nextflow_config from AWS test CI Tower.nf is now running Nextflow 22.03.0.edge by default, which includes a new strategy to handle retrying jobs killed due to AWS spot policies. This was the reason for these lines of config, so they can now be removed. --- nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 3 --- nf_core/pipeline-template/.github/workflows/awstest.yml | 3 --- 2 files changed, 6 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 6b8c3ce077..c37914276e 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -28,6 +28,3 @@ jobs: "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ github.sha }}{% endraw %}" } profiles: test_full,aws_tower - nextflow_config: | - process.errorStrategy = 'retry' - process.maxRetries = 3 diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 0715c6b9dd..209cabded1 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -23,6 +23,3 @@ jobs: "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-test-${{ github.sha }}{% endraw %}" } profiles: test,aws_tower - nextflow_config: | - process.errorStrategy = 'retry' - process.maxRetries = 3 From ff525456edd5d41b66e38dc812f2425ad255cde9 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 1 Apr 2022 11:14:57 +0200 Subject: [PATCH 008/136] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2db203c576..e2b96fc831 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Fix bug in pipeline readme logo URL +- Removed retry strategy for AWS tests CI, as Nextflow now handles spot instance retries itself ### General From 18044655f52be7937ea804c38a775842315bf782 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Fri, 1 Apr 2022 11:24:59 +0200 Subject: [PATCH 009/136] Add empty line when modules.json is missing and created (lint) --- nf_core/modules/module_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index fcd5a421f6..c2fc542759 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -188,6 +188,7 @@ def create_modules_json(pipeline_dir): modules_json_path = os.path.join(pipeline_dir, "modules.json") with open(modules_json_path, "w") as fh: json.dump(modules_json, fh, indent=4) + fh.write("\n") def find_correct_commit_sha(module_name, module_path, modules_repo): From 094b831c6867b176e550eab30086476995d7d87d Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Fri, 1 Apr 2022 11:30:33 +0200 Subject: [PATCH 010/136] Add last line in params json (launch) --- nf_core/launch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/launch.py b/nf_core/launch.py index 3bf257ff07..ede47d0f22 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -692,6 +692,7 @@ def build_command(self): if self.use_params_file: with open(self.params_out, "w") as fp: json.dump(self.schema_obj.input_params, fp, indent=4) + fp.write("\n") self.nextflow_cmd += ' {} "{}"'.format("-params-file", os.path.relpath(self.params_out)) # Call nextflow with a list of command line flags From 6de9556eea8aa337d1e4eff393237bf0a0aa3ba9 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Fri, 1 Apr 2022 11:34:16 +0200 Subject: [PATCH 011/136] Add last line to nextflow_schema (build) --- nf_core/schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/schema.py b/nf_core/schema.py index e26a46ba61..5784645543 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -171,6 +171,7 @@ def save_schema(self): log.info("Writing schema with {} params: '{}'".format(num_params, self.schema_filename)) with open(self.schema_filename, "w") as fh: json.dump(self.schema, fh, indent=4) + fh.write("\n") def load_input_params(self, params_path): """Load a given a path to a parameters file (JSON/YAML) From f67f88b32fbbf1b4065a4c405eb5f6e229ac2023 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Fri, 1 Apr 2022 11:43:31 +0200 Subject: [PATCH 012/136] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2b96fc831..cf8ad7e990 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,10 +10,10 @@ ### General - Bumped the minimum version of `rich` from `v10` to `v10.7.0` +- Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors. ### Modules -- Add an empty line at the end of the `modules.json` when dumping it to avoid prettier error. ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] From caa6bdf9be034f9fd3b0c15c1cf2b23ce3c3fb0a Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Fri, 1 Apr 2022 11:47:19 +0200 Subject: [PATCH 013/136] Fix prettier --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf8ad7e990..48c2803323 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,6 @@ ### Modules - ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] Very minor patch release to fix the full size AWS tests and re-run the template sync, which partially failed due to GitHub pull-requests being down at the time of release. From aa170b6e913e23c7c5bf74d37ee4684c2eec1776 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 1 Apr 2022 12:16:25 +0200 Subject: [PATCH 014/136] Add .prettierignore file Direct copy of .gitignore. Hopefully can get rid of this file again one day, see https://github.com/prettier/prettier/issues/4708 --- nf_core/pipeline-template/.prettierignore | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 nf_core/pipeline-template/.prettierignore diff --git a/nf_core/pipeline-template/.prettierignore b/nf_core/pipeline-template/.prettierignore new file mode 100644 index 0000000000..5124c9ac77 --- /dev/null +++ b/nf_core/pipeline-template/.prettierignore @@ -0,0 +1,8 @@ +.nextflow* +work/ +data/ +results/ +.DS_Store +testing/ +testing* +*.pyc From d23ebd81f00e5ebf2dfbe0e956576c1b8a992f3d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 1 Apr 2022 12:18:01 +0200 Subject: [PATCH 015/136] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48c2803323..c607ba4d95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Fix bug in pipeline readme logo URL - Removed retry strategy for AWS tests CI, as Nextflow now handles spot instance retries itself +- Add `.prettierignore` file to stop Prettier linting tests from running over test files ### General From a9e3a4b3af1c20ba8431d4d02b03e0461ec462c0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 1 Apr 2022 16:51:15 +0200 Subject: [PATCH 016/136] Escaped test run output before logging it, to avoid a rich `MarkupError` --- CHANGELOG.md | 2 ++ nf_core/modules/test_yml_builder.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c607ba4d95..98673cf208 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ ### Modules +- Escaped test run output before logging it, to avoid a rich ` MarkupError` + ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] Very minor patch release to fix the full size AWS tests and re-run the template sync, which partially failed due to GitHub pull-requests being down at the time of release. diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index d210cf350e..ae7a208ddd 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -330,12 +330,13 @@ def run_tests_workflow(self, command): "It looks like Nextflow is not installed. It is required for most nf-core functions." ) except subprocess.CalledProcessError as e: - raise UserWarning(f"Error running test workflow (exit code {e.returncode})\n[red]{e.output.decode()}") + output = rich.markup.escape(e.output.decode()) + raise UserWarning(f"Error running test workflow (exit code {e.returncode})\n[red]{output}") except Exception as e: raise UserWarning(f"Error running test workflow: {e}") else: log.info("Test workflow finished!") - log.debug(nfconfig_raw) + log.debug(rich.markup.escape(nfconfig_raw)) return tmp_dir, tmp_dir_repeat From ddf660d965b014e44f8a3b0823ccc8ab5a9fe109 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 5 Apr 2022 10:21:58 +0200 Subject: [PATCH 017/136] GH Actions workflow YAML tweaks. - Move the {% endraw %} statements off comment lines and on to the tail of the last statement to get rid of flating comment marker - Remove quotes around the master branch check to try to fix action --- nf_core/pipeline-template/.github/workflows/branch.yml | 5 ++--- nf_core/pipeline-template/.github/workflows/ci.yml | 4 +--- nf_core/pipeline-template/.github/workflows/linting.yml | 4 +--- .../pipeline-template/.github/workflows/linting_comment.yml | 3 +-- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/branch.yml b/nf_core/pipeline-template/.github/workflows/branch.yml index 8f10abac98..e22310ba6f 100644 --- a/nf_core/pipeline-template/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/.github/workflows/branch.yml @@ -13,7 +13,7 @@ jobs: - name: Check PRs if: github.repository == '{{ name }}' run: | - "{ [[ {% raw %}${{github.event.pull_request.head.repo.full_name }}{% endraw %} == {{ name }} ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]]" + { [[ {% raw %}${{github.event.pull_request.head.repo.full_name }}{% endraw %} == {{ name }} ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] # If the above check failed, post a comment on the PR explaining the failure {%- raw %} # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets @@ -41,5 +41,4 @@ jobs: Thanks again for your contribution! repo-token: ${{ secrets.GITHUB_TOKEN }} - allow-repeats: false -# {%- endraw %} + allow-repeats: false {%- endraw %} diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 0cd52074b7..ba22410de6 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -47,6 +47,4 @@ jobs: # For example: adding multiple test runs with different parameters # Remember that you can parallelise this by using strategy.matrix run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results - -# {%- endraw %} + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results {%- endraw %} diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index ca13c70e21..cae7be465b 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -77,6 +77,4 @@ jobs: path: | lint_log.txt lint_results.md - PR_number.txt - -# {%- endraw %} + PR_number.txt {%- endraw %} diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index f3c6e56501..5b91eedce0 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -25,5 +25,4 @@ jobs: with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} number: ${{ steps.pr_number.outputs.pr_number }} - path: linting-logs/lint_results.md -# {%- endraw %} + path: linting-logs/lint_results.md {%- endraw %} From be7bdc55b59611f4dacd461d32a7c843e18ad544 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 6 Apr 2022 15:58:57 +0200 Subject: [PATCH 018/136] Revert using Prettier with the email_template.html file --- .prettierignore | 1 + nf_core/pipeline-template/.prettierignore | 1 + .../assets/email_template.html | 142 ++++++------------ 3 files changed, 44 insertions(+), 100 deletions(-) diff --git a/.prettierignore b/.prettierignore index 069ef5131d..10f3c7f4e7 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,4 @@ +email_template.html docs/api/_build testing diff --git a/nf_core/pipeline-template/.prettierignore b/nf_core/pipeline-template/.prettierignore index 5124c9ac77..d0e7ae5891 100644 --- a/nf_core/pipeline-template/.prettierignore +++ b/nf_core/pipeline-template/.prettierignore @@ -1,3 +1,4 @@ +email_template.html .nextflow* work/ data/ diff --git a/nf_core/pipeline-template/assets/email_template.html b/nf_core/pipeline-template/assets/email_template.html index 24a3ad9d0c..ecff600d44 100644 --- a/nf_core/pipeline-template/assets/email_template.html +++ b/nf_core/pipeline-template/assets/email_template.html @@ -1,111 +1,53 @@ - - - - + + + + - - - {{ name }} Pipeline Report - - -
- + + {{ name }} Pipeline Report + + +
-

{{ name }} v${version}

-

Run Name: $runName

+ - <% if (!success){ out << """ -
-

{{ name }} execution completed unsuccessfully!

+

{{ name }} v${version}

+

Run Name: $runName

+ +<% if (!success){ + out << """ +
+

{{ name }} execution completed unsuccessfully!

The exit status of the task that caused the workflow execution to fail was: $exitStatus.

The full error message was:

- 
${errorReport}
-
- """ } else { out << """ -
+ 
${errorReport}
+
+ """ +} else { + out << """ +
{{ name }} execution completed successfully! -
- """ } %> +
+ """ +} +%> -

The workflow was completed at $dateComplete (duration: $duration)

-

The command used to launch the workflow was as follows:

- 
-$commandLine
+

The workflow was completed at $dateComplete (duration: $duration)

+

The command used to launch the workflow was as follows:

+
$commandLine
-

Pipeline Configuration:

- - - <% out << summary.collect{ k,v -> " - - - - - " }.join("\n") %> - -
- $k - - 
$v
-
+

Pipeline Configuration:

+ + + <% out << summary.collect{ k,v -> "" }.join("\n") %> + +
$k
$v
-

{{ name }}

-

https://github.com/{{ name }}

-
- +

{{ name }}

+

https://github.com/{{ name }}

+ +
+ + From bba49f07b53806178e6b0742eac55332502157a2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 6 Apr 2022 16:02:35 +0200 Subject: [PATCH 019/136] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98673cf208..9fc9e1067d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Fix bug in pipeline readme logo URL +- Fix Prettier formatting bug in completion email HTML template ([#1509](https://github.com/nf-core/tools/issues/1509)) - Removed retry strategy for AWS tests CI, as Nextflow now handles spot instance retries itself - Add `.prettierignore` file to stop Prettier linting tests from running over test files From 2d99a437fa5b9051b0d5ebd1e5e76cf70a30112a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 29 Mar 2022 17:07:27 +0200 Subject: [PATCH 020/136] First stab at writing a central utils function for GitHub API calls. Uses authentication from local gh CLI or a supplied bearer token if found. Done to address nf-core/tools#1496 Bit scared it'll break something, so ideally needs extensive testing. --- nf_core/modules/module_utils.py | 10 +-- nf_core/modules/modules_repo.py | 10 +-- nf_core/sync.py | 88 +++++++------------------ nf_core/utils.py | 111 ++++++++++++++++++++++++++------ 4 files changed, 125 insertions(+), 94 deletions(-) diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index c2fc542759..986d296bbe 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -35,7 +35,7 @@ def module_exist_in_repo(module_name, modules_repo): api_url = ( f"https://api.github.com/repos/{modules_repo.name}/contents/modules/{module_name}?ref={modules_repo.branch}" ) - response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth()) + response = nf_core.utils.call_github_api(api_url, raise_error=False) return not (response.status_code == 404) @@ -65,7 +65,7 @@ def get_module_git_log(module_name, modules_repo=None, per_page=30, page_nbr=1, api_url += f"&since={since}" log.debug(f"Fetching commit history of module '{module_name}' from github API") - response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth()) + response = nf_core.utils.call_github_api(api_url, raise_error=False) if response.status_code == 200: commits = response.json() @@ -101,7 +101,7 @@ def get_commit_info(commit_sha, repo_name="nf-core/modules"): ) api_url = f"https://api.github.com/repos/{repo_name}/commits/{commit_sha}?stats=false" log.debug(f"Fetching commit metadata for commit at {commit_sha}") - response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth()) + response = nf_core.utils.call_github_api(api_url, raise_error=False) if response.status_code == 200: commit = response.json() message = commit["commit"]["message"].partition("\n")[0] @@ -266,7 +266,7 @@ def local_module_equal_to_commit(local_files, module_name, modules_repo, commit_ for i, file in enumerate(files_to_check): # Download remote copy and compare api_url = f"{module_base_url}/{file}" - r = requests.get(url=api_url, auth=nf_core.utils.github_api_auto_auth()) + r = nf_core.utils.call_github_api(api_url, raise_error=False) if r.status_code != 200: log.debug(f"Could not download remote copy of file module {module_name}/{file}") log.debug(api_url) @@ -414,7 +414,7 @@ def verify_pipeline_dir(dir): modules_is_software = False for repo_name in repo_names: api_url = f"https://api.github.com/repos/{repo_name}/contents" - response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth()) + response = nf_core.utils.call_github_api(api_url, raise_error=False) if response.status_code == 404: missing_remote.append(repo_name) if repo_name == "nf-core/software": diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index a625a0db06..0fc7189d64 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -43,7 +43,7 @@ def __init__(self, repo="nf-core/modules", branch=None): def get_default_branch(self): """Get the default branch for a GitHub repo""" api_url = f"https://api.github.com/repos/{self.name}" - response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth()) + response = nf_core.utils.call_github_api(api_url, raise_error=False) if response.status_code == 200: self.branch = response.json()["default_branch"] log.debug(f"Found default branch to be '{self.branch}'") @@ -58,7 +58,7 @@ def verify_modules_repo(self): # Check if repository exist api_url = f"https://api.github.com/repos/{self.name}/branches" - response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth()) + response = nf_core.utils.call_github_api(api_url, raise_error=False) if response.status_code == 200: branches = [branch["name"] for branch in response.json()] if self.branch not in branches: @@ -67,7 +67,7 @@ def verify_modules_repo(self): raise LookupError(f"Repository '{self.name}' is not available on GitHub") api_url = f"https://api.github.com/repos/{self.name}/contents?ref={self.branch}" - response = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth()) + response = nf_core.utils.call_github_api(api_url, raise_error=False) if response.status_code == 200: dir_names = [entry["name"] for entry in response.json() if entry["type"] == "dir"] if "modules" not in dir_names: @@ -86,7 +86,7 @@ def get_modules_file_tree(self): self.modules_avail_module_names """ api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(self.name, self.branch) - r = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth()) + r = nf_core.utils.call_github_api(api_url, raise_error=False) if r.status_code == 404: raise LookupError("Repository / branch not found: {} ({})\n{}".format(self.name, self.branch, api_url)) elif r.status_code != 200: @@ -157,7 +157,7 @@ def download_gh_file(self, dl_filename, api_url): os.makedirs(dl_directory) # Call the GitHub API - r = requests.get(api_url, auth=nf_core.utils.github_api_auto_auth()) + r = nf_core.utils.call_github_api(api_url, raise_error=False) if r.status_code != 200: raise LookupError("Could not fetch {} file: {}\n {}".format(self.name, r.status_code, api_url)) result = r.json() diff --git a/nf_core/sync.py b/nf_core/sync.py index fb1d9f4db0..389da1fc32 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -316,75 +316,31 @@ def make_pull_request(self): ).format(tag=nf_core.__version__) # Make new pull-request - pr_content = { - "title": pr_title, - "body": pr_body_text, - "maintainer_can_modify": True, - "head": self.merge_branch, - "base": self.from_branch, - } - stderr = rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors()) - - while True: + log.debug("Submitting PR to GitHub API") + with requests_cache.disabled(): try: - log.debug("Submitting PR to GitHub API") - returned_data_prettyprint = "" - r_headers_pp = "" - with requests_cache.disabled(): - r = requests.post( - url="https://api.github.com/repos/{}/pulls".format(self.gh_repo), - data=json.dumps(pr_content), - auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), - ) - try: - self.gh_pr_returned_data = json.loads(r.content) - returned_data_prettyprint = json.dumps(dict(self.gh_pr_returned_data), indent=4) - r_headers_pp = json.dumps(dict(r.headers), indent=4) - except: - self.gh_pr_returned_data = r.content - returned_data_prettyprint = r.content - r_headers_pp = r.headers - log.error("Could not parse JSON response from GitHub API!") - stderr.print_exception() - - # Dump the responses to the log just in case.. - log.debug(f"PR response from GitHub. Data:\n{returned_data_prettyprint}\n\nHeaders:\n{r_headers_pp}") - - # PR worked - if r.status_code == 201: - self.pr_url = self.gh_pr_returned_data["html_url"] - log.debug(f"GitHub API PR worked, return code 201") - log.info(f"GitHub PR created: {self.gh_pr_returned_data['html_url']}") - break - - # Returned 403 error - too many simultaneous requests - # https://github.com/nf-core/tools/issues/911 - if r.status_code == 403: - log.debug(f"GitHub API PR failed with 403 error") - wait_time = float(re.sub("[^0-9]", "", str(r.headers.get("Retry-After", 0)))) - if wait_time == 0: - log.debug("Couldn't find 'Retry-After' header, guessing a length of time to wait") - wait_time = random.randrange(10, 60) - log.warning( - f"Got 403 code - probably the abuse protection. Trying again after {wait_time} seconds.." - ) - time.sleep(wait_time) - - # Something went wrong - else: - raise PullRequestException( - f"GitHub API returned code {r.status_code}: \n\n{returned_data_prettyprint}\n\n{r_headers_pp}" - ) - # Don't catch the PullRequestException that we raised inside - except PullRequestException: - raise - # Do catch any other exceptions that we hit - except Exception as e: - stderr.print_exception() - raise PullRequestException( - f"Something went badly wrong - {e}: \n\n{returned_data_prettyprint}\n\n{r_headers_pp}" + r = nf_core.utils.call_github_api( + f"https://api.github.com/repos/{self.gh_repo}/pulls", + post_data={ + "title": pr_title, + "body": pr_body_text, + "maintainer_can_modify": True, + "head": self.merge_branch, + "base": self.from_branch, + }, + auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), + return_ok=[201], + return_retry=[403], ) + except Exception as e: + stderr.print_exception(e) + raise PullRequestException(f"Something went badly wrong - {e}") + else: + self.gh_pr_returned_data = r.json() + self.pr_url = self.gh_pr_returned_data["html_url"] + log.debug(f"GitHub API PR worked, return code 201") + log.info(f"GitHub PR created: {self.gh_pr_returned_data['html_url']}") def close_open_template_merge_prs(self): """Get all template merging branches (starting with 'nf-core-template-merge-') diff --git a/nf_core/utils.py b/nf_core/utils.py index 34660d431c..be59db353a 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -15,6 +15,7 @@ import os import prompt_toolkit import questionary +import random import re import requests import requests_cache @@ -80,17 +81,6 @@ def rich_force_colors(): return None -def github_api_auto_auth(): - try: - with open(os.path.join(os.path.expanduser("~/.config/gh/hosts.yml")), "r") as fh: - auth = yaml.safe_load(fh) - log.debug("Auto-authenticating GitHub API as '@{}'".format(auth["github.com"]["user"])) - return requests.auth.HTTPBasicAuth(auth["github.com"]["user"], auth["github.com"]["oauth_token"]) - except Exception as e: - log.debug(f"Couldn't auto-auth for GitHub: [red]{e}") - return None - - class Pipeline(object): """Object to hold information about a local pipeline. @@ -386,6 +376,91 @@ def poll_nfcore_web_api(api_url, post_data=None): return web_response +def call_github_api(url, post_data=None, return_ok=[200, 201], return_retry=[], auth=None, raise_error=True): + """ + Send a request to the GitHub API. + Uses authentication (to avoid rate limiting) if available. + + Args: + url (str): The full URL of the GitHub API request. + + Returns: + data (dict): The JSON response from the GitHub API. + """ + # Class for Bearer token authentication + # https://stackoverflow.com/a/58055668/713980 + class BearerAuth(requests.auth.AuthBase): + def __init__(self, token): + self.token = token + + def __call__(self, r): + r.headers["authorization"] = f"Bearer {self.token}" + return r + + # Default auth if we're running and the gh CLI tool is installed + if auth is None: + try: + with open(os.path.join(os.path.expanduser("~/.config/gh/hosts.yml")), "r") as fh: + gh_cli_config = yaml.safe_load(fh) + log.debug("Auto-authenticating GitHub API as '@{}'".format(gh_cli_config["github.com"]["user"])) + auth = requests.auth.HTTPBasicAuth( + gh_cli_config["github.com"]["user"], gh_cli_config["github.com"]["oauth_token"] + ) + except Exception as e: + log.debug(f"Couldn't auto-auth for GitHub: [red]{e}") + + # Default auth if we have a GitHub Token (eg. GitHub Actions CI) + if os.environ.get("GITHUB_TOKEN") is not None and auth is None: + auth = BearerAuth(os.environ["GITHUB_TOKEN"]) + + # Start the loop for a retry mechanism + while True: + # GET request + if post_data is None: + r = requests.get(url, auth=auth) + # POST request + else: + r = requests.post(url, json=post_data, auth=auth) + + try: + r_headers_pp = json.dumps(dict(r.headers), indent=4) + r_data_pp = json.dumps(dict(r.json()), indent=4) + except Exception as e: + log.debug(r.headers) + log.debug(r.content) + if raise_error: + raise AssertionError(f"Could not parse JSON response from GitHub API! {e}") + else: + return r + + # Failed but expected - try again + # Added due to 403 error: too many simultaneous requests + # See https://github.com/nf-core/tools/issues/911 + if r.status_code in return_retry: + log.debug(f"GitHub API PR failed - got return code {r.status_code}") + log.debug(r_headers_pp) + log.debug(r_data_pp) + wait_time = float(re.sub("[^0-9]", "", str(r.headers.get("Retry-After", 0)))) + if wait_time == 0: + log.debug("Couldn't find 'Retry-After' header, guessing a length of time to wait") + wait_time = random.randrange(10, 60) + log.warning(f"Got API return code {r.status_code}. Trying again after {wait_time} seconds..") + time.sleep(wait_time) + + # Unexpected error - raise + elif r.status_code not in return_ok: + log.debug(r_headers_pp) + log.debug(r_data_pp) + if raise_error: + raise AssertionError(f"GitHub API PR failed - got return code {r.status_code} from {url}") + else: + return r + + # Success! + else: + return r + + def anaconda_package(dep, dep_channels=["conda-forge", "bioconda", "defaults"]): """Query conda package information. @@ -627,9 +702,9 @@ def prompt_remote_pipeline_name(wfs): else: if pipeline.count("/") == 1: try: - gh_response = requests.get(f"https://api.github.com/repos/{pipeline}") - assert gh_response.json().get("message") != "Not Found" - except AssertionError: + nf_core.utils.call_github_api(f"https://api.github.com/repos/{pipeline}") + except Exception: + # No repo found - pass and raise error at the end pass else: return pipeline @@ -706,7 +781,7 @@ def get_repo_releases_branches(pipeline, wfs): ) # Get releases from GitHub API - rel_r = requests.get(f"https://api.github.com/repos/{pipeline}/releases") + rel_r = nf_core.utils.call_github_api(f"https://api.github.com/repos/{pipeline}/releases") # Check that this repo existed try: @@ -714,13 +789,13 @@ def get_repo_releases_branches(pipeline, wfs): except AssertionError: raise AssertionError(f"Not able to find pipeline '{pipeline}'") except AttributeError: - # When things are working we get a list, which doesn't work with .get() + # Success! We have a list, which doesn't work with .get() which is looking for a dict key wf_releases = list(sorted(rel_r.json(), key=lambda k: k.get("published_at_timestamp", 0), reverse=True)) # Get release tag commit hashes if len(wf_releases) > 0: # Get commit hash information for each release - tags_r = requests.get(f"https://api.github.com/repos/{pipeline}/tags") + tags_r = nf_core.utils.call_github_api(f"https://api.github.com/repos/{pipeline}/tags") for tag in tags_r.json(): for release in wf_releases: if tag["name"] == release["tag_name"]: @@ -731,7 +806,7 @@ def get_repo_releases_branches(pipeline, wfs): raise AssertionError(f"Not able to find pipeline '{pipeline}'") # Get branch information from github api - should be no need to check if the repo exists again - branch_response = requests.get(f"https://api.github.com/repos/{pipeline}/branches") + branch_response = nf_core.utils.call_github_api(f"https://api.github.com/repos/{pipeline}/branches") for branch in branch_response.json(): if ( branch["name"] != "TEMPLATE" From b2b7e3b54614129e0ef2a0e8eefdc2151ca377f4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 30 Mar 2022 14:26:55 +0200 Subject: [PATCH 021/136] Remove superfluous dict() call --- nf_core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index be59db353a..a8be9c85a6 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -424,7 +424,7 @@ def __call__(self, r): try: r_headers_pp = json.dumps(dict(r.headers), indent=4) - r_data_pp = json.dumps(dict(r.json()), indent=4) + r_data_pp = json.dumps(r.json(), indent=4) except Exception as e: log.debug(r.headers) log.debug(r.content) From d516a050d884d991d6036b2f222a32c05c9b8c02 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 30 Mar 2022 22:21:23 +0200 Subject: [PATCH 022/136] Fix sync pytests --- nf_core/sync.py | 2 +- nf_core/utils.py | 4 ++-- tests/test_sync.py | 5 ++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 389da1fc32..d4452ecd2c 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -334,7 +334,7 @@ def make_pull_request(self): return_retry=[403], ) except Exception as e: - stderr.print_exception(e) + stderr.print_exception() raise PullRequestException(f"Something went badly wrong - {e}") else: self.gh_pr_returned_data = r.json() diff --git a/nf_core/utils.py b/nf_core/utils.py index a8be9c85a6..ead6a7fd35 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -417,10 +417,10 @@ def __call__(self, r): while True: # GET request if post_data is None: - r = requests.get(url, auth=auth) + r = requests.get(url=url, auth=auth) # POST request else: - r = requests.post(url, json=post_data, auth=auth) + r = requests.post(url=url, json=post_data, auth=auth) try: r_headers_pp = json.dumps(dict(r.headers), indent=4) diff --git a/tests/test_sync.py b/tests/test_sync.py index 68c96e2929..a0d009638f 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -200,6 +200,9 @@ def __init__(self, data, status_code): self.content = json.dumps(data) self.headers = {"content-encoding": "test", "connection": "fake"} + def json(self): + return json.loads(self.content) + if kwargs["url"] == "https://api.github.com/repos/no_existing_pr/response/pulls": response_data = {"html_url": "great_success"} return MockResponse(response_data, 201) @@ -229,4 +232,4 @@ def test_make_pull_request_bad_response(self, mock_get, mock_post): psync.make_pull_request() raise UserWarning("Should have hit an exception") except nf_core.sync.PullRequestException as e: - assert e.args[0].startswith("GitHub API returned code 404:") + assert e.args[0].startswith("Something went badly wrong - GitHub API PR failed - got return code 404") From 01089e762b62b0cf70b94ea27df6136cfedfbece Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 30 Mar 2022 22:41:05 +0200 Subject: [PATCH 023/136] Better debug logging. Also removed function prefix from calls that were in the same file --- nf_core/utils.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index ead6a7fd35..a1dba30c6a 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -387,6 +387,11 @@ def call_github_api(url, post_data=None, return_ok=[200, 201], return_retry=[], Returns: data (dict): The JSON response from the GitHub API. """ + + auth_mode = None + if auth is not None: + auth_mode = "supplied to function" + # Class for Bearer token authentication # https://stackoverflow.com/a/58055668/713980 class BearerAuth(requests.auth.AuthBase): @@ -398,21 +403,25 @@ def __call__(self, r): return r # Default auth if we're running and the gh CLI tool is installed - if auth is None: + gh_cli_config_fn = os.path.expanduser("~/.config/gh/hosts.yml") + if auth is None and os.path.exists(gh_cli_config_fn): try: - with open(os.path.join(os.path.expanduser("~/.config/gh/hosts.yml")), "r") as fh: + with open(gh_cli_config_fn, "r") as fh: gh_cli_config = yaml.safe_load(fh) - log.debug("Auto-authenticating GitHub API as '@{}'".format(gh_cli_config["github.com"]["user"])) auth = requests.auth.HTTPBasicAuth( gh_cli_config["github.com"]["user"], gh_cli_config["github.com"]["oauth_token"] ) + auth_mode = f"gh CLI config: {gh_cli_config['github.com']['user']}" except Exception as e: - log.debug(f"Couldn't auto-auth for GitHub: [red]{e}") + log.debug(f"Couldn't auto-auth with GitHub CLI auth: [red]{e}") # Default auth if we have a GitHub Token (eg. GitHub Actions CI) if os.environ.get("GITHUB_TOKEN") is not None and auth is None: + auth_mode = "GITHUB_TOKEN" auth = BearerAuth(os.environ["GITHUB_TOKEN"]) + log.debug(f"Fetching GitHub API: {url} (auth: {auth_mode})") + # Start the loop for a retry mechanism while True: # GET request @@ -702,7 +711,7 @@ def prompt_remote_pipeline_name(wfs): else: if pipeline.count("/") == 1: try: - nf_core.utils.call_github_api(f"https://api.github.com/repos/{pipeline}") + call_github_api(f"https://api.github.com/repos/{pipeline}") except Exception: # No repo found - pass and raise error at the end pass @@ -781,7 +790,7 @@ def get_repo_releases_branches(pipeline, wfs): ) # Get releases from GitHub API - rel_r = nf_core.utils.call_github_api(f"https://api.github.com/repos/{pipeline}/releases") + rel_r = call_github_api(f"https://api.github.com/repos/{pipeline}/releases") # Check that this repo existed try: @@ -795,7 +804,7 @@ def get_repo_releases_branches(pipeline, wfs): # Get release tag commit hashes if len(wf_releases) > 0: # Get commit hash information for each release - tags_r = nf_core.utils.call_github_api(f"https://api.github.com/repos/{pipeline}/tags") + tags_r = call_github_api(f"https://api.github.com/repos/{pipeline}/tags") for tag in tags_r.json(): for release in wf_releases: if tag["name"] == release["tag_name"]: @@ -806,7 +815,7 @@ def get_repo_releases_branches(pipeline, wfs): raise AssertionError(f"Not able to find pipeline '{pipeline}'") # Get branch information from github api - should be no need to check if the repo exists again - branch_response = nf_core.utils.call_github_api(f"https://api.github.com/repos/{pipeline}/branches") + branch_response = call_github_api(f"https://api.github.com/repos/{pipeline}/branches") for branch in branch_response.json(): if ( branch["name"] != "TEMPLATE" From 9cdf4bcad7b720c90ba25b5450f1c9f4cbf67c05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 12 Apr 2022 12:14:41 +0200 Subject: [PATCH 024/136] remove md5sum for versions.yml file --- nf_core/modules/test_yml_builder.py | 30 +++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index d210cf350e..3a525dfbaf 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -227,20 +227,22 @@ def create_test_file_dict(self, results_dir, is_repeat=False): test_files = [] for root, dir, file in os.walk(results_dir): for elem in file: - elem = os.path.join(root, elem) - test_file = {"path": elem} # add the key here so that it comes first in the dict - # Check that this isn't an empty file - if self.check_if_empty_file(elem): - if not is_repeat: - self.errors.append(f"Empty file found! '{os.path.basename(elem)}'") - # Add the md5 anyway, linting should fail later and can be manually removed if needed. - # Originally we skipped this if empty, but then it's too easy to miss the warning. - # Equally, if a file is legitimately empty we don't want to prevent this from working. - elem_md5 = self._md5(elem) - test_file["md5sum"] = elem_md5 - # Switch out the results directory path with the expected 'output' directory - test_file["path"] = elem.replace(results_dir, "output") - test_files.append(test_file) + # Check that the file is not versions.yml + if elem != "versions.yml": + elem = os.path.join(root, elem) + test_file = {"path": elem} # add the key here so that it comes first in the dict + # Check that this isn't an empty file + if self.check_if_empty_file(elem): + if not is_repeat: + self.errors.append(f"Empty file found! '{os.path.basename(elem)}'") + # Add the md5 anyway, linting should fail later and can be manually removed if needed. + # Originally we skipped this if empty, but then it's too easy to miss the warning. + # Equally, if a file is legitimately empty we don't want to prevent this from working. + elem_md5 = self._md5(elem) + test_file["md5sum"] = elem_md5 + # Switch out the results directory path with the expected 'output' directory + test_file["path"] = elem.replace(results_dir, "output") + test_files.append(test_file) test_files = sorted(test_files, key=operator.itemgetter("path")) From 10fe141f2530d55f2cd75cf3cd3c53f9718c1f7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 12 Apr 2022 12:36:21 +0200 Subject: [PATCH 025/136] modify changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 399db4dc61..652fcd5db3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Very minor patch release to fix the full size AWS tests and re-run the template - Remove traces of markdownlint in the template ([#1486](https://github.com/nf-core/tools/pull/1486) - Remove accidentally added line in `CHANGELOG.md` in the template ([#1487](https://github.com/nf-core/tools/pull/1487)) - Update linting to check that `.editorconfig` is there and `.yamllint.yml` isn't. +- Remove md5sum for versions.yml ## [v2.3.1 - Mercury Vulture Formatting](https://github.com/nf-core/tools/releases/tag/2.3.1) - [2022-03-23] From 3567b07bc8bd99848ffd93ff3d21d47de78efcb6 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 12 Apr 2022 14:56:49 +0200 Subject: [PATCH 026/136] Remove the graphviz dependency from nf-core --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 001c3261aa..1c12ee3628 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -159,7 +159,7 @@ trace { } dag { enabled = true - file = "${params.tracedir}/pipeline_dag_${trace_timestamp}.svg" + file = "${params.tracedir}/pipeline_dag_${trace_timestamp}.html" } manifest { From 8313a507ae62aee8fade0cce8da9a5c73474c67c Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 12 Apr 2022 20:31:33 +0200 Subject: [PATCH 027/136] mention the Graphviz dependency removal --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 399db4dc61..9f11f6425c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,13 @@ ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] -Very minor patch release to fix the full size AWS tests and re-run the template sync, which partially failed due to GitHub pull-requests being down at the time of release. +This patch release to removes the Graphviz dependency from default pipeline template. + +Also included are patches to fix the full size AWS tests and re-run the template sync, which partially failed due to GitHub pull-requests being down at the time of release. ### Template +- Set the default DAG graphic output to HTML to have a default that does not depend on Graphviz being installed on the host system. - Updated the AWS GitHub actions to let nf-core/tower-action use it's defaults for pipeline and git sha ([#1488](https://github.com/nf-core/tools/pull/1488)) - Add prettier editor extension to `gitpod.yml` in template ([#1485](https://github.com/nf-core/tools/pull/1485)) - Remove traces of markdownlint in the template ([#1486](https://github.com/nf-core/tools/pull/1486) From ae2a0577a3c02e59362672b39330c5c50e0e465f Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 12 Apr 2022 21:33:27 +0200 Subject: [PATCH 028/136] allow all supported DAG file suffixes --- nf_core/lint/nextflow_config.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index b1c3f4f92e..dc36c506e1 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -229,10 +229,12 @@ def nextflow_config(self): # Check that the DAG filename ends in ``.svg`` if "dag.file" in self.nf_config: - if self.nf_config["dag.file"].strip("'\"").endswith(".svg"): - passed.append("Config ``dag.file`` ended with ``.svg``") + dag_file_suffix = os.path.splitext(self.nf_config["dag.file"].strip("'\"")) + allowed_dag_file_suffixes = [".dot", ".html", ".pdf", ".png", ".svg", ".gexf"] + if dag_file_suffix in allowed_dag_file_suffixes: + passed.append(f"Config ``dag.file`` ended with ``.{dag_file_suffix}``") else: - failed.append("Config ``dag.file`` did not end with ``.svg``") + failed.append(f"Config ``dag.file`` ended with ``.{dag_file_suffix}`` but needs to be one of {allowed_dag_file_suffixes!r}") # Check that the minimum nextflowVersion is set properly if "manifest.nextflowVersion" in self.nf_config: From c68fb85e2b8169ae5276e3867717139c358dba92 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 12 Apr 2022 21:35:38 +0200 Subject: [PATCH 029/136] fix black complaint --- nf_core/lint/nextflow_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index dc36c506e1..0421df2dad 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -234,7 +234,9 @@ def nextflow_config(self): if dag_file_suffix in allowed_dag_file_suffixes: passed.append(f"Config ``dag.file`` ended with ``.{dag_file_suffix}``") else: - failed.append(f"Config ``dag.file`` ended with ``.{dag_file_suffix}`` but needs to be one of {allowed_dag_file_suffixes!r}") + failed.append( + f"Config ``dag.file`` ended with ``.{dag_file_suffix}`` but needs to be one of {allowed_dag_file_suffixes!r}" + ) # Check that the minimum nextflowVersion is set properly if "manifest.nextflowVersion" in self.nf_config: From 83fe29ec4e4c0e945f055e553c5e6a8b6af60aa1 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 12 Apr 2022 21:44:05 +0200 Subject: [PATCH 030/136] fix dag.file suffix extraction --- nf_core/lint/nextflow_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index 0421df2dad..b0162191cb 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -229,7 +229,7 @@ def nextflow_config(self): # Check that the DAG filename ends in ``.svg`` if "dag.file" in self.nf_config: - dag_file_suffix = os.path.splitext(self.nf_config["dag.file"].strip("'\"")) + _, dag_file_suffix = os.path.splitext(self.nf_config["dag.file"].strip("'\"")) allowed_dag_file_suffixes = [".dot", ".html", ".pdf", ".png", ".svg", ".gexf"] if dag_file_suffix in allowed_dag_file_suffixes: passed.append(f"Config ``dag.file`` ended with ``.{dag_file_suffix}``") From 32da8e5559426df067bf5f6b35ad1732fa5e0a14 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 12 Apr 2022 22:46:03 +0200 Subject: [PATCH 031/136] move changes in CHANGELOG into new dev section --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f11f6425c..25049cc1ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,17 @@ # nf-core/tools: Changelog -## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] +## dev This patch release to removes the Graphviz dependency from default pipeline template. -Also included are patches to fix the full size AWS tests and re-run the template sync, which partially failed due to GitHub pull-requests being down at the time of release. +- Set the default DAG graphic output to HTML to have a default that does not depend on Graphviz being installed on the host system. + +## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] + +Very minor patch release to fix the full size AWS tests and re-run the template sync, which partially failed due to GitHub pull-requests being down at the time of release. ### Template -- Set the default DAG graphic output to HTML to have a default that does not depend on Graphviz being installed on the host system. - Updated the AWS GitHub actions to let nf-core/tower-action use it's defaults for pipeline and git sha ([#1488](https://github.com/nf-core/tools/pull/1488)) - Add prettier editor extension to `gitpod.yml` in template ([#1485](https://github.com/nf-core/tools/pull/1485)) - Remove traces of markdownlint in the template ([#1486](https://github.com/nf-core/tools/pull/1486) From 3d91fe05b3bf43898df4559e9edc3457aeb92d54 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 12 Apr 2022 23:09:16 +0200 Subject: [PATCH 032/136] describe new behavior and hot to retain status quo * add link to PR --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7015c7f2ad..f09a0772ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,11 @@ ## v2.4dev -This patch release to removes the Graphviz dependency from default pipeline template. +This patch release to removes the Graphviz dependency from default pipeline template by setting the defaul DAG format to HTML. To keep the pevious behaviour, the "dag.file" file extension needs to be changed to ".svg" in the nextflow.config. ### Template -- Set the default DAG graphic output to HTML to have a default that does not depend on Graphviz being installed on the host system. +- Set the default DAG graphic output to HTML to have a default that does not depend on Graphviz being installed on the host system ([#1512](https://github.com/nf-core/tools/pull/1512)). - Fix bug in pipeline readme logo URL - Fix Prettier formatting bug in completion email HTML template ([#1509](https://github.com/nf-core/tools/issues/1509)) - Removed retry strategy for AWS tests CI, as Nextflow now handles spot instance retries itself From e298fe9511a4047cc580b7cefd5d94819c875044 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 12 Apr 2022 23:10:28 +0200 Subject: [PATCH 033/136] fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f09a0772ad..65915dbefe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## v2.4dev -This patch release to removes the Graphviz dependency from default pipeline template by setting the defaul DAG format to HTML. To keep the pevious behaviour, the "dag.file" file extension needs to be changed to ".svg" in the nextflow.config. +This patch release to removes the Graphviz dependency from default pipeline template by setting the defaul DAG format to HTML. To keep the previous behaviour, the "dag.file" file extension needs to be changed to ".svg" in the nextflow.config. ### Template From 8301585fc39af76da44334e5f4ee85b9885f1201 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Wed, 13 Apr 2022 07:26:43 +0200 Subject: [PATCH 034/136] only allow default dag format in lint --- nf_core/lint/nextflow_config.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index b0162191cb..af9dece05a 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -229,14 +229,11 @@ def nextflow_config(self): # Check that the DAG filename ends in ``.svg`` if "dag.file" in self.nf_config: - _, dag_file_suffix = os.path.splitext(self.nf_config["dag.file"].strip("'\"")) - allowed_dag_file_suffixes = [".dot", ".html", ".pdf", ".png", ".svg", ".gexf"] - if dag_file_suffix in allowed_dag_file_suffixes: - passed.append(f"Config ``dag.file`` ended with ``.{dag_file_suffix}``") + default_dag_format = ".html" + if self.nf_config["dag.file"].strip("'\"").endswith(default_dag_format): + passed.append(f"Config ``dag.file`` ended with ``{default_dag_format}``") else: - failed.append( - f"Config ``dag.file`` ended with ``.{dag_file_suffix}`` but needs to be one of {allowed_dag_file_suffixes!r}" - ) + failed.append(f"Config ``dag.file`` did not end with ``{default_dag_format}``") # Check that the minimum nextflowVersion is set properly if "manifest.nextflowVersion" in self.nf_config: From 9b3e7c6996b0535b8a59a6c88363f0e9e2302ce9 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Wed, 13 Apr 2022 07:29:00 +0200 Subject: [PATCH 035/136] Remove obsolete changelog message --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65915dbefe..7be8cec92e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,6 @@ ## v2.4dev -This patch release to removes the Graphviz dependency from default pipeline template by setting the defaul DAG format to HTML. To keep the previous behaviour, the "dag.file" file extension needs to be changed to ".svg" in the nextflow.config. - ### Template - Set the default DAG graphic output to HTML to have a default that does not depend on Graphviz being installed on the host system ([#1512](https://github.com/nf-core/tools/pull/1512)). From 0a6fa6227df35d99b2068e46098516cde1ba6540 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 13 Apr 2022 10:50:24 +0200 Subject: [PATCH 036/136] Add fix-linting workflows --- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/deploy-pypi.yml | 2 +- .github/workflows/fix-linting.yml | 49 +++++++++++++++++ .github/workflows/pytest.yml | 2 +- .github/workflows/sync.yml | 2 +- .github/workflows/tools-api-docs-dev.yml | 2 +- .github/workflows/tools-api-docs-release.yml | 2 +- .../.github/workflows/fix-linting.yml | 55 +++++++++++++++++++ .../.github/workflows/linting.yml | 2 +- 10 files changed, 112 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/fix-linting.yml create mode 100644 nf_core/pipeline-template/.github/workflows/fix-linting.yml diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index e87eca38a8..ab9b3f19b3 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -27,7 +27,7 @@ jobs: # Set up nf-core/tools - name: Set up Python 3.8 - uses: actions/setup-python@v1 + uses: actions/setup-python@v3 with: python-version: 3.8 diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index a16c8a6409..52b7c36369 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -25,7 +25,7 @@ jobs: name: Check out source-code repository - name: Set up Python 3.7 - uses: actions/setup-python@v1 + uses: actions/setup-python@v3 with: python-version: 3.7 diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index c7ddca40de..6f52c0f50f 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -12,7 +12,7 @@ jobs: name: Check out source-code repository - name: Set up Python 3.7 - uses: actions/setup-python@v1 + uses: actions/setup-python@v3 with: python-version: 3.7 diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml new file mode 100644 index 0000000000..44ca255e2b --- /dev/null +++ b/.github/workflows/fix-linting.yml @@ -0,0 +1,49 @@ +name: Fix linting from a comment +on: + issue_comment: + types: [created] + +jobs: + deploy: + # Only run if comment is on a PR with the main repo, and if it contains the magic keywords + if: > + contains(github.event.comment.html_url, '/pull/') && + contains(github.event.comment.body, '@nf-core-bot fix linting') && + github.repository == 'nf-core/tools' + runs-on: ubuntu-latest + steps: + # Use the @nf-core-bot token to check out so we can push later + - uses: actions/checkout@v3 + with: + token: ${{ secrets.nf_core_bot_auth_token }} + + # Action runs on the issue comment, so we don't get the PR by default + # Use the gh cli to check out the PR + - name: Checkout Pull Request + run: gh pr checkout ${{ github.event.issue.number }} + env: + GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} + + - uses: actions/setup-node@v2 + + - name: Install Prettier + run: npm install -g prettier @prettier/plugin-php + + - name: Run 'prettier --write' + run: prettier --write ${GITHUB_WORKSPACE} + + - name: Run Black + uses: psf/black@stable + with: + # Override to remove the default --check flag so that we make changes + options: "--color" + + - name: Commit & push changes + run: | + git config user.email "core@nf-co.re" + git config user.name "nf-core-bot" + git config push.default upstream + git add . + git status + git commit -m "[automated] Fix linting with Prettier" + git push diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index f634f29add..3731c1800c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -20,7 +20,7 @@ jobs: name: Check out source-code repository - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 4e37bc07de..c200a5b072 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -38,7 +38,7 @@ jobs: fetch-depth: "0" - name: Set up Python 3.8 - uses: actions/setup-python@v1 + uses: actions/setup-python@v3 with: python-version: 3.8 diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml index c31c94a721..b9c40787f8 100644 --- a/.github/workflows/tools-api-docs-dev.yml +++ b/.github/workflows/tools-api-docs-dev.yml @@ -12,7 +12,7 @@ jobs: uses: actions/checkout@v2 - name: Set up Python 3.7 - uses: actions/setup-python@v1 + uses: actions/setup-python@v3 with: python-version: 3.7 diff --git a/.github/workflows/tools-api-docs-release.yml b/.github/workflows/tools-api-docs-release.yml index 54c138ed68..a10f7b9b87 100644 --- a/.github/workflows/tools-api-docs-release.yml +++ b/.github/workflows/tools-api-docs-release.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v2 - name: Set up Python 3.7 - uses: actions/setup-python@v1 + uses: actions/setup-python@v3 with: python-version: 3.7 diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml new file mode 100644 index 0000000000..6e82b8b5d7 --- /dev/null +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -0,0 +1,55 @@ +name: Fix linting from a comment +on: + issue_comment: + types: [created] + +jobs: + deploy: + # Only run if comment is on a PR with the main repo, and if it contains the magic keywords + if: > + contains(github.event.comment.html_url, '/pull/') && + contains(github.event.comment.body, '@nf-core-bot fix linting') && + github.repository == '{{ name }}' + runs-on: ubuntu-latest + steps: + # Use the @nf-core-bot token to check out so we can push later + - uses: actions/checkout@v3 + with: + token: ${{ secrets.nf_core_bot_auth_token }} + + # Action runs on the issue comment, so we don't get the PR by default + # Use the gh cli to check out the PR + - name: Checkout Pull Request + run: gh pr checkout ${{ github.event.issue.number }} + env: + GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} + + - uses: actions/setup-node@v2 + + - name: Install Prettier + run: npm install -g prettier @prettier/plugin-php + + # Check that we actually need to fix something + - name: Run 'prettier --check' + id: prettier_status + run: | + if prettier --check ${GITHUB_WORKSPACE}; then + echo "::set-output name=result::pass" + else + echo "::set-output name=result::fail" + fi + + - name: Run 'prettier --write' + if: steps.prettier_status.outputs.result == 'fail' + run: prettier --write ${GITHUB_WORKSPACE} + + - name: Commit & push changes + if: steps.prettier_status.outputs.result == 'fail' + run: | + git config user.email "core@nf-co.re" + git config user.name "nf-core-bot" + git config push.default upstream + git add . + git status + git commit -m "[automated] Fix linting with Prettier" + git push diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index cae7be465b..c52eb5e6a8 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -48,7 +48,7 @@ jobs: wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v3 with: python-version: "3.6" architecture: "x64" From 112e576d94f7a906fb090e84a3b7b74155042dbb Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 13 Apr 2022 10:52:15 +0200 Subject: [PATCH 037/136] Changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98673cf208..0c85e193fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,13 @@ - Fix bug in pipeline readme logo URL - Removed retry strategy for AWS tests CI, as Nextflow now handles spot instance retries itself - Add `.prettierignore` file to stop Prettier linting tests from running over test files +- Add actions workflow to respond to `@nf-core-bot fix linting` comments on pipeline PRs ### General - Bumped the minimum version of `rich` from `v10` to `v10.7.0` - Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors. +- Add actions workflow to respond to `@nf-core-bot fix linting` comments on nf-core/tools PRs ### Modules From e3be16ef3b13e2a2c0d0797d3ee7fdd4d6b612cb Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 13 Apr 2022 10:55:10 +0200 Subject: [PATCH 038/136] Jinja2 raw tags for the template workflow --- nf_core/pipeline-template/.github/workflows/fix-linting.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index 6e82b8b5d7..5ad82fe8f8 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -9,7 +9,7 @@ jobs: if: > contains(github.event.comment.html_url, '/pull/') && contains(github.event.comment.body, '@nf-core-bot fix linting') && - github.repository == '{{ name }}' + github.repository == '{{ name }}' {%- raw %} runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later @@ -52,4 +52,4 @@ jobs: git add . git status git commit -m "[automated] Fix linting with Prettier" - git push + git push {%- endraw %} From cd78ef9bd2c7e2fde4f9b1a4a0571a7ea787f24c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 13 Apr 2022 12:51:26 +0200 Subject: [PATCH 039/136] Update CHANGELOG.md Co-authored-by: Phil Ewels --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 652fcd5db3..65c59935e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Very minor patch release to fix the full size AWS tests and re-run the template - Remove traces of markdownlint in the template ([#1486](https://github.com/nf-core/tools/pull/1486) - Remove accidentally added line in `CHANGELOG.md` in the template ([#1487](https://github.com/nf-core/tools/pull/1487)) - Update linting to check that `.editorconfig` is there and `.yamllint.yml` isn't. -- Remove md5sum for versions.yml +- Don't save md5sum for `versions.yml` when running `nf-core modules create-test-yml` ([#1511](https://github.com/nf-core/tools/pull/1511)) ## [v2.3.1 - Mercury Vulture Formatting](https://github.com/nf-core/tools/releases/tag/2.3.1) - [2022-03-23] From 708c8075bc23e3efcd40844bb24e851f11271383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 13 Apr 2022 14:02:23 +0200 Subject: [PATCH 040/136] Use continue in if statement Co-authored-by: Phil Ewels --- nf_core/modules/test_yml_builder.py | 32 +++++++++++++++-------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index 3a525dfbaf..5ece0da97e 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -228,21 +228,23 @@ def create_test_file_dict(self, results_dir, is_repeat=False): for root, dir, file in os.walk(results_dir): for elem in file: # Check that the file is not versions.yml - if elem != "versions.yml": - elem = os.path.join(root, elem) - test_file = {"path": elem} # add the key here so that it comes first in the dict - # Check that this isn't an empty file - if self.check_if_empty_file(elem): - if not is_repeat: - self.errors.append(f"Empty file found! '{os.path.basename(elem)}'") - # Add the md5 anyway, linting should fail later and can be manually removed if needed. - # Originally we skipped this if empty, but then it's too easy to miss the warning. - # Equally, if a file is legitimately empty we don't want to prevent this from working. - elem_md5 = self._md5(elem) - test_file["md5sum"] = elem_md5 - # Switch out the results directory path with the expected 'output' directory - test_file["path"] = elem.replace(results_dir, "output") - test_files.append(test_file) + if elem = "versions.yml": + continue + + elem = os.path.join(root, elem) + test_file = {"path": elem} # add the key here so that it comes first in the dict + # Check that this isn't an empty file + if self.check_if_empty_file(elem): + if not is_repeat: + self.errors.append(f"Empty file found! '{os.path.basename(elem)}'") + # Add the md5 anyway, linting should fail later and can be manually removed if needed. + # Originally we skipped this if empty, but then it's too easy to miss the warning. + # Equally, if a file is legitimately empty we don't want to prevent this from working. + elem_md5 = self._md5(elem) + test_file["md5sum"] = elem_md5 + # Switch out the results directory path with the expected 'output' directory + test_file["path"] = elem.replace(results_dir, "output") + test_files.append(test_file) test_files = sorted(test_files, key=operator.itemgetter("path")) From 6d4b103035b5f533b8b8bf7a74326dd8b7fa2074 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 13 Apr 2022 14:11:24 +0200 Subject: [PATCH 041/136] Update nf_core/modules/test_yml_builder.py --- nf_core/modules/test_yml_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index 5ece0da97e..2935e99f21 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -228,7 +228,7 @@ def create_test_file_dict(self, results_dir, is_repeat=False): for root, dir, file in os.walk(results_dir): for elem in file: # Check that the file is not versions.yml - if elem = "versions.yml": + if elem == "versions.yml": continue elem = os.path.join(root, elem) From a8baf994f855dfdcc25c9f965175ad5d1c1bf453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 13 Apr 2022 14:17:25 +0200 Subject: [PATCH 042/136] apply refactoring from review suggestions --- nf_core/modules/test_yml_builder.py | 74 ++++++++++++++++++----------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index 5ece0da97e..cf4d5ff1bd 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -76,13 +76,16 @@ def check_inputs(self): style=nf_core.utils.nfcore_question_style, ).ask() self.module_dir = os.path.join("modules", *self.module_name.split("/")) - self.module_test_main = os.path.join("tests", "modules", *self.module_name.split("/"), "main.nf") + self.module_test_main = os.path.join( + "tests", "modules", *self.module_name.split("/"), "main.nf") # First, sanity check that the module directory exists if not os.path.isdir(self.module_dir): - raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") + raise UserWarning( + f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") if not os.path.exists(self.module_test_main): - raise UserWarning(f"Cannot find module test workflow '{self.module_test_main}'") + raise UserWarning( + f"Cannot find module test workflow '{self.module_test_main}'") # Check that we're running tests if no prompts if not self.run_tests and self.no_prompts: @@ -120,14 +123,16 @@ def check_inputs(self): def scrape_workflow_entry_points(self): """Find the test workflow entry points from main.nf""" - log.info(f"Looking for test workflow entry points: '{self.module_test_main}'") + log.info( + f"Looking for test workflow entry points: '{self.module_test_main}'") with open(self.module_test_main, "r") as fh: for line in fh: match = re.match(r"workflow\s+(\S+)\s+{", line) if match: self.entry_points.append(match.group(1)) if len(self.entry_points) == 0: - raise UserWarning("No workflow entry points found in 'self.module_test_main'") + raise UserWarning( + "No workflow entry points found in 'self.module_test_main'") def build_all_tests(self): """ @@ -161,7 +166,8 @@ def build_single_test(self, entry_point): if self.no_prompts: ep_test["name"] = default_val else: - ep_test["name"] = rich.prompt.Prompt.ask("[violet]Test name", default=default_val).strip() + ep_test["name"] = rich.prompt.Prompt.ask( + "[violet]Test name", default=default_val).strip() while ep_test["command"] == "": default_val = ( @@ -170,7 +176,8 @@ def build_single_test(self, entry_point): if self.no_prompts: ep_test["command"] = default_val else: - ep_test["command"] = rich.prompt.Prompt.ask("[violet]Test command", default=default_val).strip() + ep_test["command"] = rich.prompt.Prompt.ask( + "[violet]Test command", default=default_val).strip() while len(ep_test["tags"]) == 0: mod_name_parts = self.module_name.split("/") @@ -186,7 +193,8 @@ def build_single_test(self, entry_point): prompt_tags = rich.prompt.Prompt.ask( "[violet]Test tags[/] (comma separated)", default=",".join(tag_defaults) ).strip() - ep_test["tags"] = [t.strip() for t in prompt_tags.split(",")] + ep_test["tags"] = [t.strip() + for t in prompt_tags.split(",")] ep_test["files"] = self.get_md5_sums(entry_point, ep_test["command"]) @@ -225,25 +233,26 @@ def _md5(self, fname): def create_test_file_dict(self, results_dir, is_repeat=False): """Walk through directory and collect md5 sums""" test_files = [] - for root, dir, file in os.walk(results_dir): - for elem in file: + for root, dir, files in os.walk(results_dir): + for filename in files: # Check that the file is not versions.yml - if elem = "versions.yml": + if filename == "versions.yml": continue - - elem = os.path.join(root, elem) - test_file = {"path": elem} # add the key here so that it comes first in the dict + file_path = os.path.join(root, filename) + # add the key here so that it comes first in the dict + test_file = {"path": file_path} # Check that this isn't an empty file - if self.check_if_empty_file(elem): + if self.check_if_empty_file(file_path): if not is_repeat: - self.errors.append(f"Empty file found! '{os.path.basename(elem)}'") + self.errors.append( + f"Empty file found! '{os.path.basename(file_path)}'") # Add the md5 anyway, linting should fail later and can be manually removed if needed. # Originally we skipped this if empty, but then it's too easy to miss the warning. # Equally, if a file is legitimately empty we don't want to prevent this from working. - elem_md5 = self._md5(elem) - test_file["md5sum"] = elem_md5 + file_md5 = self._md5(file_path) + test_file["md5sum"] = file_md5 # Switch out the results directory path with the expected 'output' directory - test_file["path"] = elem.replace(results_dir, "output") + test_file["path"] = file_path.replace(results_dir, "output") test_files.append(test_file) test_files = sorted(test_files, key=operator.itemgetter("path")) @@ -260,7 +269,8 @@ def get_md5_sums(self, entry_point, command, results_dir=None, results_dir_repea run_this_test = False while results_dir is None: if self.run_tests or run_this_test: - results_dir, results_dir_repeat = self.run_tests_workflow(command) + results_dir, results_dir_repeat = self.run_tests_workflow( + command) else: results_dir = rich.prompt.Prompt.ask( f"[violet]Test output folder with results[/] (leave blank to run test)" @@ -276,7 +286,8 @@ def get_md5_sums(self, entry_point, command, results_dir=None, results_dir_repea # If test was repeated, compare the md5 sums if results_dir_repeat: - test_files_repeat = self.create_test_file_dict(results_dir=results_dir_repeat, is_repeat=True) + test_files_repeat = self.create_test_file_dict( + results_dir=results_dir_repeat, is_repeat=True) # Compare both test.yml files for i in range(len(test_files)): @@ -287,7 +298,8 @@ def get_md5_sums(self, entry_point, command, results_dir=None, results_dir_repea ] = "[ # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ]" if len(test_files) == 0: - raise UserWarning(f"Could not find any test result files in '{results_dir}'") + raise UserWarning( + f"Could not find any test result files in '{results_dir}'") return test_files @@ -310,7 +322,8 @@ def run_tests_workflow(self, command): "message": "Choose software profile", "choices": ["Docker", "Singularity", "Conda"], } - answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) + answer = questionary.unsafe_prompt( + [question], style=nf_core.utils.nfcore_question_style) profile = answer["profile"].lower() if profile in ["singularity", "conda"]: os.environ["PROFILE"] = profile @@ -319,10 +332,12 @@ def run_tests_workflow(self, command): tmp_dir = tempfile.mkdtemp() tmp_dir_repeat = tempfile.mkdtemp() work_dir = tempfile.mkdtemp() - command_repeat = command + f" --outdir {tmp_dir_repeat} -work-dir {work_dir}" + command_repeat = command + \ + f" --outdir {tmp_dir_repeat} -work-dir {work_dir}" command += f" --outdir {tmp_dir} -work-dir {work_dir}" - log.info(f"Running '{self.module_name}' test with command:\n[violet]{command}") + log.info( + f"Running '{self.module_name}' test with command:\n[violet]{command}") try: nfconfig_raw = subprocess.check_output(shlex.split(command)) log.info(f"Repeating test ...") @@ -334,7 +349,8 @@ def run_tests_workflow(self, command): "It looks like Nextflow is not installed. It is required for most nf-core functions." ) except subprocess.CalledProcessError as e: - raise UserWarning(f"Error running test workflow (exit code {e.returncode})\n[red]{e.output.decode()}") + raise UserWarning( + f"Error running test workflow (exit code {e.returncode})\n[red]{e.output.decode()}") except Exception as e: raise UserWarning(f"Error running test workflow: {e}") else: @@ -350,13 +366,15 @@ def print_test_yml(self): if self.test_yml_output_path == "-": console = rich.console.Console() - yaml_str = yaml.dump(self.tests, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) + yaml_str = yaml.dump( + self.tests, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) console.print("\n", Syntax(yaml_str, "yaml"), "\n") return try: log.info(f"Writing to '{self.test_yml_output_path}'") with open(self.test_yml_output_path, "w") as fh: - yaml.dump(self.tests, fh, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) + yaml.dump( + self.tests, fh, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) except FileNotFoundError as e: raise UserWarning("Could not create test.yml file: '{}'".format(e)) From 66d1e98e04d754cde73e00e753f7a05ea71ec9e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 13 Apr 2022 15:31:29 +0200 Subject: [PATCH 043/136] fix linting --- nf_core/modules/test_yml_builder.py | 56 ++++++++++------------------- 1 file changed, 19 insertions(+), 37 deletions(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index 00e1e9d1d0..0366397d8e 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -76,16 +76,13 @@ def check_inputs(self): style=nf_core.utils.nfcore_question_style, ).ask() self.module_dir = os.path.join("modules", *self.module_name.split("/")) - self.module_test_main = os.path.join( - "tests", "modules", *self.module_name.split("/"), "main.nf") + self.module_test_main = os.path.join("tests", "modules", *self.module_name.split("/"), "main.nf") # First, sanity check that the module directory exists if not os.path.isdir(self.module_dir): - raise UserWarning( - f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") + raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") if not os.path.exists(self.module_test_main): - raise UserWarning( - f"Cannot find module test workflow '{self.module_test_main}'") + raise UserWarning(f"Cannot find module test workflow '{self.module_test_main}'") # Check that we're running tests if no prompts if not self.run_tests and self.no_prompts: @@ -123,16 +120,14 @@ def check_inputs(self): def scrape_workflow_entry_points(self): """Find the test workflow entry points from main.nf""" - log.info( - f"Looking for test workflow entry points: '{self.module_test_main}'") + log.info(f"Looking for test workflow entry points: '{self.module_test_main}'") with open(self.module_test_main, "r") as fh: for line in fh: match = re.match(r"workflow\s+(\S+)\s+{", line) if match: self.entry_points.append(match.group(1)) if len(self.entry_points) == 0: - raise UserWarning( - "No workflow entry points found in 'self.module_test_main'") + raise UserWarning("No workflow entry points found in 'self.module_test_main'") def build_all_tests(self): """ @@ -166,8 +161,7 @@ def build_single_test(self, entry_point): if self.no_prompts: ep_test["name"] = default_val else: - ep_test["name"] = rich.prompt.Prompt.ask( - "[violet]Test name", default=default_val).strip() + ep_test["name"] = rich.prompt.Prompt.ask("[violet]Test name", default=default_val).strip() while ep_test["command"] == "": default_val = ( @@ -176,8 +170,7 @@ def build_single_test(self, entry_point): if self.no_prompts: ep_test["command"] = default_val else: - ep_test["command"] = rich.prompt.Prompt.ask( - "[violet]Test command", default=default_val).strip() + ep_test["command"] = rich.prompt.Prompt.ask("[violet]Test command", default=default_val).strip() while len(ep_test["tags"]) == 0: mod_name_parts = self.module_name.split("/") @@ -193,8 +186,7 @@ def build_single_test(self, entry_point): prompt_tags = rich.prompt.Prompt.ask( "[violet]Test tags[/] (comma separated)", default=",".join(tag_defaults) ).strip() - ep_test["tags"] = [t.strip() - for t in prompt_tags.split(",")] + ep_test["tags"] = [t.strip() for t in prompt_tags.split(",")] ep_test["files"] = self.get_md5_sums(entry_point, ep_test["command"]) @@ -244,8 +236,7 @@ def create_test_file_dict(self, results_dir, is_repeat=False): # Check that this isn't an empty file if self.check_if_empty_file(file_path): if not is_repeat: - self.errors.append( - f"Empty file found! '{os.path.basename(file_path)}'") + self.errors.append(f"Empty file found! '{os.path.basename(file_path)}'") # Add the md5 anyway, linting should fail later and can be manually removed if needed. # Originally we skipped this if empty, but then it's too easy to miss the warning. # Equally, if a file is legitimately empty we don't want to prevent this from working. @@ -269,8 +260,7 @@ def get_md5_sums(self, entry_point, command, results_dir=None, results_dir_repea run_this_test = False while results_dir is None: if self.run_tests or run_this_test: - results_dir, results_dir_repeat = self.run_tests_workflow( - command) + results_dir, results_dir_repeat = self.run_tests_workflow(command) else: results_dir = rich.prompt.Prompt.ask( f"[violet]Test output folder with results[/] (leave blank to run test)" @@ -286,8 +276,7 @@ def get_md5_sums(self, entry_point, command, results_dir=None, results_dir_repea # If test was repeated, compare the md5 sums if results_dir_repeat: - test_files_repeat = self.create_test_file_dict( - results_dir=results_dir_repeat, is_repeat=True) + test_files_repeat = self.create_test_file_dict(results_dir=results_dir_repeat, is_repeat=True) # Compare both test.yml files for i in range(len(test_files)): @@ -298,8 +287,7 @@ def get_md5_sums(self, entry_point, command, results_dir=None, results_dir_repea ] = "[ # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead ]" if len(test_files) == 0: - raise UserWarning( - f"Could not find any test result files in '{results_dir}'") + raise UserWarning(f"Could not find any test result files in '{results_dir}'") return test_files @@ -322,8 +310,7 @@ def run_tests_workflow(self, command): "message": "Choose software profile", "choices": ["Docker", "Singularity", "Conda"], } - answer = questionary.unsafe_prompt( - [question], style=nf_core.utils.nfcore_question_style) + answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) profile = answer["profile"].lower() if profile in ["singularity", "conda"]: os.environ["PROFILE"] = profile @@ -332,12 +319,10 @@ def run_tests_workflow(self, command): tmp_dir = tempfile.mkdtemp() tmp_dir_repeat = tempfile.mkdtemp() work_dir = tempfile.mkdtemp() - command_repeat = command + \ - f" --outdir {tmp_dir_repeat} -work-dir {work_dir}" + command_repeat = command + f" --outdir {tmp_dir_repeat} -work-dir {work_dir}" command += f" --outdir {tmp_dir} -work-dir {work_dir}" - log.info( - f"Running '{self.module_name}' test with command:\n[violet]{command}") + log.info(f"Running '{self.module_name}' test with command:\n[violet]{command}") try: nfconfig_raw = subprocess.check_output(shlex.split(command)) log.info(f"Repeating test ...") @@ -349,13 +334,12 @@ def run_tests_workflow(self, command): "It looks like Nextflow is not installed. It is required for most nf-core functions." ) except subprocess.CalledProcessError as e: - output = rich.markup.escape(e.output.decode()) - raise UserWarning(f"Error running test workflow (exit code {e.returncode})\n[red]{output}") + raise UserWarning(f"Error running test workflow (exit code {e.returncode})\n[red]{e.output.decode()}") except Exception as e: raise UserWarning(f"Error running test workflow: {e}") else: log.info("Test workflow finished!") - log.debug(rich.markup.escape(nfconfig_raw)) + log.debug(nfconfig_raw) return tmp_dir, tmp_dir_repeat @@ -366,15 +350,13 @@ def print_test_yml(self): if self.test_yml_output_path == "-": console = rich.console.Console() - yaml_str = yaml.dump( - self.tests, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) + yaml_str = yaml.dump(self.tests, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) console.print("\n", Syntax(yaml_str, "yaml"), "\n") return try: log.info(f"Writing to '{self.test_yml_output_path}'") with open(self.test_yml_output_path, "w") as fh: - yaml.dump( - self.tests, fh, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) + yaml.dump(self.tests, fh, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) except FileNotFoundError as e: raise UserWarning("Could not create test.yml file: '{}'".format(e)) From d7e596ea6db771c520514fe05b0e587c24c2b806 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 13 Apr 2022 16:27:09 +0200 Subject: [PATCH 044/136] Try to rewrite GitHub API calls to use a single shared Session --- nf_core/modules/module_utils.py | 19 ++- nf_core/modules/modules_repo.py | 14 +- nf_core/sync.py | 13 +- nf_core/utils.py | 221 ++++++++++++++++++++------------ 4 files changed, 161 insertions(+), 106 deletions(-) diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index 986d296bbe..23c5ec526f 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -1,7 +1,6 @@ import glob import json import os -import requests import logging import rich import datetime @@ -15,6 +14,8 @@ log = logging.getLogger(__name__) +gh_api = nf_core.utils.gh_api + class ModuleException(Exception): """Exception raised when there was an error with module commands""" @@ -35,7 +36,7 @@ def module_exist_in_repo(module_name, modules_repo): api_url = ( f"https://api.github.com/repos/{modules_repo.name}/contents/modules/{module_name}?ref={modules_repo.branch}" ) - response = nf_core.utils.call_github_api(api_url, raise_error=False) + response = gh_api.get(api_url) return not (response.status_code == 404) @@ -65,7 +66,7 @@ def get_module_git_log(module_name, modules_repo=None, per_page=30, page_nbr=1, api_url += f"&since={since}" log.debug(f"Fetching commit history of module '{module_name}' from github API") - response = nf_core.utils.call_github_api(api_url, raise_error=False) + response = gh_api.get(api_url) if response.status_code == 200: commits = response.json() @@ -80,6 +81,7 @@ def get_module_git_log(module_name, modules_repo=None, per_page=30, page_nbr=1, elif response.status_code == 404: raise LookupError(f"Module '{module_name}' not found in '{modules_repo.name}'\n{api_url}") else: + gh_api.log_content_headers(response) raise LookupError( f"Unable to fetch commit SHA for module {module_name}. API responded with '{response.status_code}'" ) @@ -101,7 +103,7 @@ def get_commit_info(commit_sha, repo_name="nf-core/modules"): ) api_url = f"https://api.github.com/repos/{repo_name}/commits/{commit_sha}?stats=false" log.debug(f"Fetching commit metadata for commit at {commit_sha}") - response = nf_core.utils.call_github_api(api_url, raise_error=False) + response = gh_api.get(api_url) if response.status_code == 200: commit = response.json() message = commit["commit"]["message"].partition("\n")[0] @@ -115,6 +117,7 @@ def get_commit_info(commit_sha, repo_name="nf-core/modules"): elif response.status_code == 404: raise LookupError(f"Commit '{commit_sha}' not found in 'nf-core/modules/'\n{api_url}") else: + gh_api.log_content_headers(response) raise LookupError(f"Unable to fetch metadata for commit SHA {commit_sha}") @@ -266,10 +269,12 @@ def local_module_equal_to_commit(local_files, module_name, modules_repo, commit_ for i, file in enumerate(files_to_check): # Download remote copy and compare api_url = f"{module_base_url}/{file}" - r = nf_core.utils.call_github_api(api_url, raise_error=False) + r = gh_api.get(api_url) + # TODO: Remove debugging + gh_api.log_content_headers(r) if r.status_code != 200: + gh_api.log_content_headers(r) log.debug(f"Could not download remote copy of file module {module_name}/{file}") - log.debug(api_url) else: try: remote_copies[i] = r.content.decode("utf-8") @@ -414,7 +419,7 @@ def verify_pipeline_dir(dir): modules_is_software = False for repo_name in repo_names: api_url = f"https://api.github.com/repos/{repo_name}/contents" - response = nf_core.utils.call_github_api(api_url, raise_error=False) + response = gh_api.get(api_url) if response.status_code == 404: missing_remote.append(repo_name) if repo_name == "nf-core/software": diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 0fc7189d64..b4faba3cbc 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -1,9 +1,7 @@ import os -import requests import base64 -import sys import logging -import nf_core.utils +from nf_core.utils import gh_api log = logging.getLogger(__name__) @@ -43,7 +41,7 @@ def __init__(self, repo="nf-core/modules", branch=None): def get_default_branch(self): """Get the default branch for a GitHub repo""" api_url = f"https://api.github.com/repos/{self.name}" - response = nf_core.utils.call_github_api(api_url, raise_error=False) + response = gh_api.get(api_url) if response.status_code == 200: self.branch = response.json()["default_branch"] log.debug(f"Found default branch to be '{self.branch}'") @@ -58,7 +56,7 @@ def verify_modules_repo(self): # Check if repository exist api_url = f"https://api.github.com/repos/{self.name}/branches" - response = nf_core.utils.call_github_api(api_url, raise_error=False) + response = gh_api.get(api_url) if response.status_code == 200: branches = [branch["name"] for branch in response.json()] if self.branch not in branches: @@ -67,7 +65,7 @@ def verify_modules_repo(self): raise LookupError(f"Repository '{self.name}' is not available on GitHub") api_url = f"https://api.github.com/repos/{self.name}/contents?ref={self.branch}" - response = nf_core.utils.call_github_api(api_url, raise_error=False) + response = gh_api.get(api_url) if response.status_code == 200: dir_names = [entry["name"] for entry in response.json() if entry["type"] == "dir"] if "modules" not in dir_names: @@ -86,7 +84,7 @@ def get_modules_file_tree(self): self.modules_avail_module_names """ api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(self.name, self.branch) - r = nf_core.utils.call_github_api(api_url, raise_error=False) + r = gh_api.get(api_url) if r.status_code == 404: raise LookupError("Repository / branch not found: {} ({})\n{}".format(self.name, self.branch, api_url)) elif r.status_code != 200: @@ -157,7 +155,7 @@ def download_gh_file(self, dl_filename, api_url): os.makedirs(dl_directory) # Call the GitHub API - r = nf_core.utils.call_github_api(api_url, raise_error=False) + r = gh_api.get(api_url) if r.status_code != 200: raise LookupError("Could not fetch {} file: {}\n {}".format(self.name, r.status_code, api_url)) result = r.json() diff --git a/nf_core/sync.py b/nf_core/sync.py index d4452ecd2c..979430b9e3 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -6,13 +6,10 @@ import json import logging import os -import random -import re import requests import requests_cache import rich import shutil -import time import nf_core import nf_core.create @@ -318,9 +315,12 @@ def make_pull_request(self): # Make new pull-request stderr = rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors()) log.debug("Submitting PR to GitHub API") - with requests_cache.disabled(): + gh_api = nf_core.utils.gh_api + gh_api.auth = requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]) + gh_api.return_ok = [201] + with gh_api.disabled(): try: - r = nf_core.utils.call_github_api( + r = gh_api.get_retry( f"https://api.github.com/repos/{self.gh_repo}/pulls", post_data={ "title": pr_title, @@ -329,9 +329,6 @@ def make_pull_request(self): "head": self.merge_branch, "base": self.from_branch, }, - auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), - return_ok=[201], - return_retry=[403], ) except Exception as e: stderr.print_exception() diff --git a/nf_core/utils.py b/nf_core/utils.py index a1dba30c6a..0b6c44d586 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -19,6 +19,7 @@ import re import requests import requests_cache +import rich import shlex import subprocess import sys @@ -296,21 +297,28 @@ def setup_requests_cachedir(): Caching directory will be set up in the user's home directory under a .nfcore_cache subdir. + + Uses requests_cache monkey patching. + Also returns the config dict so that we can use the same setup with a Session. """ pyversion = ".".join(str(v) for v in sys.version_info[0:3]) cachedir = os.path.join(os.getenv("HOME"), os.path.join(".nfcore", "cache_" + pyversion)) + config = { + "cache_name": os.path.join(cachedir, "github_info"), + "expire_after": datetime.timedelta(hours=1), + "backend": "sqlite", + } + try: if not os.path.exists(cachedir): os.makedirs(cachedir) - requests_cache.install_cache( - os.path.join(cachedir, "github_info"), - expire_after=datetime.timedelta(hours=1), - backend="sqlite", - ) + requests_cache.install_cache(**config) except PermissionError: pass + return config + def wait_cli_function(poll_func, poll_every=20): """ @@ -376,98 +384,145 @@ def poll_nfcore_web_api(api_url, post_data=None): return web_response -def call_github_api(url, post_data=None, return_ok=[200, 201], return_retry=[], auth=None, raise_error=True): +class GitHub_API_Session(requests_cache.CachedSession): + """ + Class to provide a single session for interacting with the GitHub API for a run. + Inherits the requests_cache.CachedSession and adds additional functionality, + such as automatically setting up GitHub authentication if we can. """ - Send a request to the GitHub API. - Uses authentication (to avoid rate limiting) if available. - Args: - url (str): The full URL of the GitHub API request. + def __init__(self): + self.auth_mode = None + self.return_ok = [200, 201] + self.return_retry = [403] + self.has_init = False - Returns: - data (dict): The JSON response from the GitHub API. - """ + def lazy_init(self): + """ + Initialise the object. + + Only do this when it's actually being used (due to global import) + """ + log.debug("Initialising GitHub API requests session") + cache_config = setup_requests_cachedir() + super().__init__(**cache_config) + self.setup_github_auth() + self.has_init = True - auth_mode = None - if auth is not None: - auth_mode = "supplied to function" + def setup_github_auth(self, auth=None): + """ + Try to automatically set up GitHub authentication + """ + if auth is not None: + self.auth = auth + self.auth_mode = "supplied to function" + + # Class for Bearer token authentication + # https://stackoverflow.com/a/58055668/713980 + class BearerAuth(requests.auth.AuthBase): + def __init__(self, token): + self.token = token + + def __call__(self, r): + r.headers["authorization"] = f"Bearer {self.token}" + return r + + # Default auth if we're running and the gh CLI tool is installed + gh_cli_config_fn = os.path.expanduser("~/.config/gh/hosts.yml") + if self.auth is None and os.path.exists(gh_cli_config_fn): + try: + with open(gh_cli_config_fn, "r") as fh: + gh_cli_config = yaml.safe_load(fh) + self.auth = requests.auth.HTTPBasicAuth( + gh_cli_config["github.com"]["user"], gh_cli_config["github.com"]["oauth_token"] + ) + self.auth_mode = f"gh CLI config: {gh_cli_config['github.com']['user']}" + except Exception as e: + output = rich.markup.escape(e.output.decode()) + log.debug(f"Couldn't auto-auth with GitHub CLI auth: [red]{output}") - # Class for Bearer token authentication - # https://stackoverflow.com/a/58055668/713980 - class BearerAuth(requests.auth.AuthBase): - def __init__(self, token): - self.token = token + # Default auth if we have a GitHub Token (eg. GitHub Actions CI) + if os.environ.get("GITHUB_TOKEN") is not None and self.auth is None: + self.auth_mode = "Bearer token with GITHUB_TOKEN" + self.auth = BearerAuth(os.environ["GITHUB_TOKEN"]) - def __call__(self, r): - r.headers["authorization"] = f"Bearer {self.token}" - return r + log.debug(f"Using GitHub auth: {self.auth_mode}") - # Default auth if we're running and the gh CLI tool is installed - gh_cli_config_fn = os.path.expanduser("~/.config/gh/hosts.yml") - if auth is None and os.path.exists(gh_cli_config_fn): + def log_content_headers(self, request): + """ + Try to dump everything to the console, useful when things go wrong. + """ + rich.inspect(request) try: - with open(gh_cli_config_fn, "r") as fh: - gh_cli_config = yaml.safe_load(fh) - auth = requests.auth.HTTPBasicAuth( - gh_cli_config["github.com"]["user"], gh_cli_config["github.com"]["oauth_token"] - ) - auth_mode = f"gh CLI config: {gh_cli_config['github.com']['user']}" + log.debug(json.dumps(dict(request.headers), indent=4)) + log.debug(json.dumps(request.json(), indent=4)) except Exception as e: - log.debug(f"Couldn't auto-auth with GitHub CLI auth: [red]{e}") + log.debug(f"Could not parse JSON response from GitHub API! {e}") + log.debug(request.headers) + log.debug(request.content) - # Default auth if we have a GitHub Token (eg. GitHub Actions CI) - if os.environ.get("GITHUB_TOKEN") is not None and auth is None: - auth_mode = "GITHUB_TOKEN" - auth = BearerAuth(os.environ["GITHUB_TOKEN"]) + def safe_get(self, url): + """ + Run a GET request, raise a nice exception with lots of logging if it fails. + """ + if not self.has_init: + self.lazy_init() + request = self.get(url) + if request.status_code not in self.return_ok: + self.log_content_headers(request) + raise AssertionError(f"GitHub API PR failed - got return code {request.status_code} from {url}") + return request + + def get(self, url, **kwargs): + """ + Initialise the session if we haven't already, then call the superclass get method. + """ + if not self.has_init: + self.lazy_init() + return super().get(url, **kwargs) - log.debug(f"Fetching GitHub API: {url} (auth: {auth_mode})") + def get_retry(self, url, post_data=None): + """ + Try to fetch a URL, keep retrying if we get a certain return code. - # Start the loop for a retry mechanism - while True: - # GET request - if post_data is None: - r = requests.get(url=url, auth=auth) - # POST request - else: - r = requests.post(url=url, json=post_data, auth=auth) + Used in nf-core sync code because we get 403 errors: too many simultaneous requests + See https://github.com/nf-core/tools/issues/911 + """ + if not self.has_init: + self.lazy_init() - try: - r_headers_pp = json.dumps(dict(r.headers), indent=4) - r_data_pp = json.dumps(r.json(), indent=4) - except Exception as e: - log.debug(r.headers) - log.debug(r.content) - if raise_error: - raise AssertionError(f"Could not parse JSON response from GitHub API! {e}") + # Start the loop for a retry mechanism + while True: + # GET request + if post_data is None: + r = self.get(url=url) + # POST request else: - return r - - # Failed but expected - try again - # Added due to 403 error: too many simultaneous requests - # See https://github.com/nf-core/tools/issues/911 - if r.status_code in return_retry: - log.debug(f"GitHub API PR failed - got return code {r.status_code}") - log.debug(r_headers_pp) - log.debug(r_data_pp) - wait_time = float(re.sub("[^0-9]", "", str(r.headers.get("Retry-After", 0)))) - if wait_time == 0: - log.debug("Couldn't find 'Retry-After' header, guessing a length of time to wait") - wait_time = random.randrange(10, 60) - log.warning(f"Got API return code {r.status_code}. Trying again after {wait_time} seconds..") - time.sleep(wait_time) - - # Unexpected error - raise - elif r.status_code not in return_ok: - log.debug(r_headers_pp) - log.debug(r_data_pp) - if raise_error: + r = self.post(url=url, json=post_data) + + # Failed but expected - try again + if r.status_code in self.return_retry: + self.log_content_headers(r) + log.debug(f"GitHub API PR failed - got return code {r.status_code}") + wait_time = float(re.sub("[^0-9]", "", str(r.headers.get("Retry-After", 0)))) + if wait_time == 0: + log.debug("Couldn't find 'Retry-After' header, guessing a length of time to wait") + wait_time = random.randrange(10, 60) + log.warning(f"Got API return code {r.status_code}. Trying again after {wait_time} seconds..") + time.sleep(wait_time) + + # Unexpected error - raise + elif r.status_code not in self.return_ok: + self.log_content_headers(r) raise AssertionError(f"GitHub API PR failed - got return code {r.status_code} from {url}") + + # Success! else: return r - # Success! - else: - return r + +# Single session object to use for entire codebase. Not sure if there's a better way to do this? +gh_api = GitHub_API_Session() def anaconda_package(dep, dep_channels=["conda-forge", "bioconda", "defaults"]): @@ -711,7 +766,7 @@ def prompt_remote_pipeline_name(wfs): else: if pipeline.count("/") == 1: try: - call_github_api(f"https://api.github.com/repos/{pipeline}") + gh_api.get(f"https://api.github.com/repos/{pipeline}") except Exception: # No repo found - pass and raise error at the end pass @@ -790,7 +845,7 @@ def get_repo_releases_branches(pipeline, wfs): ) # Get releases from GitHub API - rel_r = call_github_api(f"https://api.github.com/repos/{pipeline}/releases") + rel_r = gh_api.safe_get(f"https://api.github.com/repos/{pipeline}/releases") # Check that this repo existed try: @@ -804,7 +859,7 @@ def get_repo_releases_branches(pipeline, wfs): # Get release tag commit hashes if len(wf_releases) > 0: # Get commit hash information for each release - tags_r = call_github_api(f"https://api.github.com/repos/{pipeline}/tags") + tags_r = gh_api.safe_get(f"https://api.github.com/repos/{pipeline}/tags") for tag in tags_r.json(): for release in wf_releases: if tag["name"] == release["tag_name"]: @@ -815,7 +870,7 @@ def get_repo_releases_branches(pipeline, wfs): raise AssertionError(f"Not able to find pipeline '{pipeline}'") # Get branch information from github api - should be no need to check if the repo exists again - branch_response = call_github_api(f"https://api.github.com/repos/{pipeline}/branches") + branch_response = gh_api.safe_get(f"https://api.github.com/repos/{pipeline}/branches") for branch in branch_response.json(): if ( branch["name"] != "TEMPLATE" From 1a453bb6977dd6ef33355c8c9e9e69994b7c355a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 13 Apr 2022 16:31:10 +0200 Subject: [PATCH 045/136] Use new class more in the sync code --- nf_core/sync.py | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 979430b9e3..e62bea51a6 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -76,6 +76,11 @@ def __init__( self.gh_repo = gh_repo self.pr_url = "" + self.gh_api = nf_core.utils.gh_api + self.gh_api.auth = requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]) + self.gh_api.return_ok = [201] + self.gh_api.lazy_init() + def sync(self): """Find workflow attributes, create a new template pipeline on TEMPLATE""" @@ -315,12 +320,9 @@ def make_pull_request(self): # Make new pull-request stderr = rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors()) log.debug("Submitting PR to GitHub API") - gh_api = nf_core.utils.gh_api - gh_api.auth = requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]) - gh_api.return_ok = [201] - with gh_api.disabled(): + with self.gh_api.cache_disabled(): try: - r = gh_api.get_retry( + r = self.gh_api.get_retry( f"https://api.github.com/repos/{self.gh_repo}/pulls", post_data={ "title": pr_title, @@ -348,11 +350,8 @@ def close_open_template_merge_prs(self): # Look for existing pull-requests list_prs_url = f"https://api.github.com/repos/{self.gh_repo}/pulls" - with requests_cache.disabled(): - list_prs_request = requests.get( - url=list_prs_url, - auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), - ) + with self.gh_api.cache_disabled(): + list_prs_request = self.gh_api.get(list_prs_url) try: list_prs_json = json.loads(list_prs_request.content) list_prs_pp = json.dumps(list_prs_json, indent=4) @@ -390,20 +389,12 @@ def close_open_pr(self, pr): f"This pull-request is now outdated and has been closed in favour of {self.pr_url}\n\n" f"Please use {self.pr_url} to merge in the new changes from the nf-core template as soon as possible." ) - with requests_cache.disabled(): - comment_request = requests.post( - url=pr["comments_url"], - data=json.dumps({"body": comment_text}), - auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), - ) + with self.gh_api.cache_disabled(): + self.gh_api.post(url=pr["comments_url"], data=json.dumps({"body": comment_text})) # Update the PR status to be closed - with requests_cache.disabled(): - pr_request = requests.patch( - url=pr["url"], - data=json.dumps({"state": "closed"}), - auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]), - ) + with self.gh_api.cache_disabled(): + pr_request = self.gh_api.patch(url=pr["url"], data=json.dumps({"state": "closed"})) try: pr_request_json = json.loads(pr_request.content) pr_request_pp = json.dumps(pr_request_json, indent=4) From 868d9412d3122211086d9a8d4fd8a8735d963e1a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 13 Apr 2022 17:09:18 +0200 Subject: [PATCH 046/136] Nearly fix the tests --- nf_core/sync.py | 5 +++-- nf_core/utils.py | 7 +++++-- tests/test_sync.py | 49 +++++++++++++++++++++++++++++++++------------- 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index e62bea51a6..5960048ca1 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -77,7 +77,8 @@ def __init__( self.pr_url = "" self.gh_api = nf_core.utils.gh_api - self.gh_api.auth = requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]) + if self.gh_username and "GITHUB_AUTH_TOKEN" in os.environ: + self.gh_api.auth = requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]) self.gh_api.return_ok = [201] self.gh_api.lazy_init() @@ -322,7 +323,7 @@ def make_pull_request(self): log.debug("Submitting PR to GitHub API") with self.gh_api.cache_disabled(): try: - r = self.gh_api.get_retry( + r = self.gh_api.request_retry( f"https://api.github.com/repos/{self.gh_repo}/pulls", post_data={ "title": pr_title, diff --git a/nf_core/utils.py b/nf_core/utils.py index 0b6c44d586..38e8c96264 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -452,7 +452,10 @@ def log_content_headers(self, request): """ Try to dump everything to the console, useful when things go wrong. """ - rich.inspect(request) + log.debug(f"Requested URL: {request.url}") + log.debug(f"From requests cache: {request.from_cache}") + log.debug(f"Request status code: {request.status_code}") + log.debug(f"Request reason: {request.reason}") try: log.debug(json.dumps(dict(request.headers), indent=4)) log.debug(json.dumps(request.json(), indent=4)) @@ -481,7 +484,7 @@ def get(self, url, **kwargs): self.lazy_init() return super().get(url, **kwargs) - def get_retry(self, url, post_data=None): + def request_retry(self, url, post_data=None): """ Try to fetch a URL, keep retrying if we get a certain return code. diff --git a/tests/test_sync.py b/tests/test_sync.py index a0d009638f..42cfc87918 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -162,69 +162,90 @@ def test_push_template_branch_error(self): except nf_core.sync.PullRequestException as e: assert e.args[0].startswith("Could not push TEMPLATE branch") - def mocked_requests_get(**kwargs): + def mocked_requests_get(url, **kwargs): """Helper function to emulate POST requests responses from the web""" class MockResponse: def __init__(self, data, status_code): + self.url = kwargs.get("url") self.status_code = status_code + self.from_cache = False + self.reason = "Mocked response" + self.data = data self.content = json.dumps(data) + self.headers = {"content-encoding": "test", "connection": "fake"} + + def json(self): + return self.data url_template = "https://api.github.com/repos/{}/response/pulls?head=TEMPLATE&base=None" - if kwargs["url"] == url_template.format("no_existing_pr"): + if url == url_template.format("no_existing_pr"): response_data = [] return MockResponse(response_data, 200) - return MockResponse({"get_url": kwargs["url"]}, 404) + return MockResponse({"html_url": url}, 404) - def mocked_requests_patch(**kwargs): + def mocked_requests_patch(url, **kwargs): """Helper function to emulate POST requests responses from the web""" class MockResponse: def __init__(self, data, status_code): + self.url = kwargs.get("url") self.status_code = status_code + self.from_cache = False + self.reason = "Mocked" self.content = json.dumps(data) + self.headers = {"content-encoding": "test", "connection": "fake"} - if kwargs["url"] == "url_to_update_pr": + if url == "url_to_update_pr": response_data = {"html_url": "great_success"} return MockResponse(response_data, 200) - return MockResponse({"patch_url": kwargs["url"]}, 404) + return MockResponse({"patch_url": url}, 404) - def mocked_requests_post(**kwargs): + def mocked_requests_post(url, **kwargs): """Helper function to emulate POST requests responses from the web""" class MockResponse: def __init__(self, data, status_code): + self.url = kwargs.get("url") self.status_code = status_code + self.from_cache = False + self.reason = "Mocked" + self.data = data self.content = json.dumps(data) self.headers = {"content-encoding": "test", "connection": "fake"} def json(self): - return json.loads(self.content) + return self.data - if kwargs["url"] == "https://api.github.com/repos/no_existing_pr/response/pulls": + if url == "https://api.github.com/repos/no_existing_pr/response/pulls": response_data = {"html_url": "great_success"} return MockResponse(response_data, 201) - return MockResponse({"post_url": kwargs["url"]}, 404) + response_data = {} + return MockResponse(response_data, 404) - @mock.patch("requests.get", side_effect=mocked_requests_get) - @mock.patch("requests.post", side_effect=mocked_requests_post) + @mock.patch("nf_core.utils.gh_api.get", side_effect=mocked_requests_get) + @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) def test_make_pull_request_success(self, mock_get, mock_post): """Try making a PR - successful response""" psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.gh_api.get = mock_get + psync.gh_api.post = mock_post psync.gh_username = "no_existing_pr" psync.gh_repo = "no_existing_pr/response" os.environ["GITHUB_AUTH_TOKEN"] = "test" psync.make_pull_request() assert psync.gh_pr_returned_data["html_url"] == "great_success" - @mock.patch("requests.get", side_effect=mocked_requests_get) - @mock.patch("requests.post", side_effect=mocked_requests_post) + @mock.patch("nf_core.utils.gh_api.get", side_effect=mocked_requests_get) + @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) def test_make_pull_request_bad_response(self, mock_get, mock_post): """Try making a PR and getting a 404 error""" psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.gh_api.get = mock_get + psync.gh_api.post = mock_post psync.gh_username = "bad_url" psync.gh_repo = "bad_url/response" os.environ["GITHUB_AUTH_TOKEN"] = "test" From 7772e5a5c8524870eebd145caf8bae252187a7ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 14 Apr 2022 11:24:14 +0200 Subject: [PATCH 047/136] restore deleted lines Co-authored-by: Phil Ewels --- nf_core/modules/test_yml_builder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index 0366397d8e..b10c64204c 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -334,12 +334,13 @@ def run_tests_workflow(self, command): "It looks like Nextflow is not installed. It is required for most nf-core functions." ) except subprocess.CalledProcessError as e: - raise UserWarning(f"Error running test workflow (exit code {e.returncode})\n[red]{e.output.decode()}") + output = rich.markup.escape(e.output.decode()) + raise UserWarning(f"Error running test workflow (exit code {e.returncode})\n[red]{output}") except Exception as e: raise UserWarning(f"Error running test workflow: {e}") else: log.info("Test workflow finished!") - log.debug(nfconfig_raw) + log.debug(rich.markup.escape(nfconfig_raw)) return tmp_dir, tmp_dir_repeat From ad4acf142cc975d58cf5c19274010b5a76636491 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 14 Apr 2022 14:22:32 +0200 Subject: [PATCH 048/136] Don't allow a .nf-core.yaml file (should be .yml) --- nf_core/lint/files_exist.py | 11 +++++++++++ nf_core/lint/files_unchanged.py | 7 ++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 8b23a88619..8bbf40dd86 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -76,6 +76,7 @@ def files_exist(self): Singularity parameters.settings.json + .nf-core.yaml # NB: Should be yml, not yaml bin/markdown_to_html.r conf/aws.config .github/workflows/push_dockerhub.yml @@ -90,6 +91,15 @@ def files_exist(self): .. code-block:: bash .travis.yml + + .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting + the ``files_exist`` key as follows in your ``.nf-core.yml`` config file. For example: + + .. code-block:: yaml + + lint: + files_exist: + - assets/multiqc_config.yml """ passed = [] @@ -160,6 +170,7 @@ def files_exist(self): files_fail_ifexists = [ "Singularity", "parameters.settings.json", + ".nf-core.yaml", # yml not yaml os.path.join("bin", "markdown_to_html.r"), os.path.join("conf", "aws.config"), os.path.join(".github", "workflows", "push_dockerhub.yml"), diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 887406c6f0..5e3f976c92 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -47,12 +47,13 @@ def files_unchanged(self): .gitignore .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting - the ``files_unchanged`` key as follows in your linting config file. For example: + the ``files_unchanged`` key as follows in your ``.nf-core.yml`` config file. For example: .. code-block:: yaml - files_unchanged: - - .github/workflows/branch.yml + lint: + files_unchanged: + - .github/workflows/branch.yml """ From 253f8e8af9f0cfeaa084b11d484d18225c14325a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 14 Apr 2022 14:24:07 +0200 Subject: [PATCH 049/136] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 090b539637..030f996143 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ This patch release to removes the Graphviz dependency from default pipeline temp - Bumped the minimum version of `rich` from `v10` to `v10.7.0` - Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors. - Add actions workflow to respond to `@nf-core-bot fix linting` comments on nf-core/tools PRs +- Linting: Don't allow a `.nf-core.yaml` file, should be `.yml` ([#1515](https://github.com/nf-core/tools/pull/1515)). ### Modules From 82bbba87e983c848a29f1a828d46e4dde901ab1d Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Wed, 20 Apr 2022 22:03:41 +0200 Subject: [PATCH 050/136] feat: add a modules mulled command --- nf_core/__main__.py | 20 ++++++++++++- nf_core/modules/__init__.py | 1 + nf_core/modules/mulled.py | 58 +++++++++++++++++++++++++++++++++++ requirements.txt | 1 + tests/test_mullled.py | 60 +++++++++++++++++++++++++++++++++++++ 5 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 nf_core/modules/mulled.py create mode 100644 tests/test_mullled.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 8affd23bfe..569a45d657 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -49,7 +49,7 @@ }, { "name": "Developing new modules", - "commands": ["create", "create-test-yml", "lint", "bump-versions"], + "commands": ["create", "create-test-yml", "lint", "bump-versions", "mulled"], }, ], } @@ -674,6 +674,24 @@ def bump_versions(ctx, tool, dir, all, show_all): sys.exit(1) +# nf-core modules mulled +@modules.command() +@click.argument("tools", required=True, nargs=-1, metavar=" <...>") +def mulled(tools): + """ + Generate the name of a BioContainers mulled image version 2. + + When you know the specific dependencies and their versions of a multi-tool container image and you need the name of + that image, this command can generate it for you. + + """ + print( + nf_core.modules.mulled.MulledImageNameGenerator.generate_image_name( + nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(tools) + ) + ) + + # nf-core schema subcommands @nf_core_cli.group() def schema(): diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index f833d52564..6b999aea50 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -9,3 +9,4 @@ from .update import ModuleUpdate from .remove import ModuleRemove from .info import ModuleInfo +from .mulled import MulledImageNameGenerator diff --git a/nf_core/modules/mulled.py b/nf_core/modules/mulled.py new file mode 100644 index 0000000000..b84561a5c1 --- /dev/null +++ b/nf_core/modules/mulled.py @@ -0,0 +1,58 @@ +"""Generate the name of a BioContainers mulled image version 2.""" + + +import logging +import re +from packaging.version import Version, InvalidVersion +from typing import Iterable, Tuple, List + +from galaxy.tool_util.deps.mulled.util import build_target, v2_image_name + + +log = logging.getLogger(__name__) + + +class MulledImageNameGenerator: + """ + Define a service class for generating BioContainers version 2 mulled image names. + + Adapted from https://gist.github.com/natefoo/19cefeedd1942c30f9d88027a61b3f83. + + """ + + _split_pattern = re.compile(r"==?") + + @classmethod + def parse_targets(cls, tools: Iterable[str]) -> List[Tuple[str, str]]: + """ + Parse tool, version pairs from two or more version strings. + + Args: + tools: An iterable of strings that contain tools and their versions. + + """ + result = [] + for spec in tools: + try: + tool, version = cls._split_pattern.split(spec, maxsplit=1) + except ValueError: + raise ValueError( + f"The specification {spec} does not have the expected format or ." + ) from None + try: + Version(version) + except InvalidVersion: + raise ValueError(f"{version} in {spec} is not a PEP440 compliant version specification.") from None + result.append((tool.strip(), version.strip())) + return result + + @classmethod + def generate_image_name(cls, targets: Iterable[Tuple[str, str]]) -> str: + """ + Generate the name of a BioContainers mulled image version 2. + + Args: + targets: One or more tool, version pairs of the multi-tool container image. + + """ + return v2_image_name([build_target(name, version) for name, version in targets]) diff --git a/requirements.txt b/requirements.txt index 97ed43df61..798ed3470b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ click +galaxy-tool-util GitPython jinja2 jsonschema>=3.0 diff --git a/tests/test_mullled.py b/tests/test_mullled.py new file mode 100644 index 0000000000..2206d65eef --- /dev/null +++ b/tests/test_mullled.py @@ -0,0 +1,60 @@ +"""Test the mulled BioContainers image name generation.""" + +import pytest + +from nf_core.modules import MulledImageNameGenerator + + +@pytest.mark.parametrize( + "specs, expected", + [ + (["foo==0.1.2", "bar==1.1"], [("foo", "0.1.2"), ("bar", "1.1")]), + (["foo=0.1.2", "bar=1.1"], [("foo", "0.1.2"), ("bar", "1.1")]), + ], +) +def test_target_parsing(specs, expected): + """""" + assert MulledImageNameGenerator.parse_targets(specs) == expected + + +@pytest.mark.parametrize( + "specs", + [ + ["foo<0.1.2", "bar==1.1"], + ["foo=0.1.2", "bar>1.1"], + ], +) +def test_wrong_specification(specs): + """""" + with pytest.raises(ValueError, match="expected format"): + MulledImageNameGenerator.parse_targets(specs) + + +@pytest.mark.parametrize( + "specs", + [ + ["foo==0a.1.2", "bar==1.1"], + ["foo==0.1.2", "bar==1.b1b"], + ], +) +def test_noncompliant_version(specs): + """""" + with pytest.raises(ValueError, match="PEP440"): + MulledImageNameGenerator.parse_targets(specs) + + +@pytest.mark.parametrize( + "specs, expected", + [ + ( + [("chromap", "0.2.1"), ("samtools", "1.15")], + "mulled-v2-1f09f39f20b1c4ee36581dc81cc323c70e661633:bd74d08a359024829a7aec1638a28607bbcd8a58", + ), + ( + [("pysam", "0.16.0.1"), ("biopython", "1.78")], + "mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f", + ), + ], +) +def test_generate_image_name(specs, expected): + assert MulledImageNameGenerator.generate_image_name(specs) == expected From 4dc3c3e4b1c8fc436739e18288a0d43052605097 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Wed, 20 Apr 2022 22:07:29 +0200 Subject: [PATCH 051/136] chore: add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57a4cff8ea..86a8206b91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ ### Modules - Escaped test run output before logging it, to avoid a rich ` MarkupError` +- Add a new command `nf-core modules mulled` which can generate the name for a multi-tool container image. ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] From 5c4544071d1a85ef60f9ebb033a710da239007f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 21 Apr 2022 10:35:36 +0200 Subject: [PATCH 052/136] print include statement when installing a module --- nf_core/modules/install.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 673e1d7f4e..a72c45da76 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -130,6 +130,14 @@ def install(self, module): if not self.download_module_file(module, version, self.modules_repo, install_folder): return False + # Print include statement + if module == "stringtie/stringtie": + # Only with stringtie the process name is STRINGTIE instead of STRINGTIE_STRINGTIE + module_name = module.upper().split("/")[0] + else: + module_name = "_".join(module.upper().split("/")) + log.info(f"Include statement: include {{ {module_name} }} from '.{os.path.join(*install_folder, module)}/main’") + # Update module.json with newly installed module self.update_modules_json(modules_json, self.modules_repo.name, module, version) return True From cd57124cbafe400374fee0daf7b711947b4c9387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 21 Apr 2022 10:59:50 +0200 Subject: [PATCH 053/136] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fc9e1067d..89d265bf04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Fix Prettier formatting bug in completion email HTML template ([#1509](https://github.com/nf-core/tools/issues/1509)) - Removed retry strategy for AWS tests CI, as Nextflow now handles spot instance retries itself - Add `.prettierignore` file to stop Prettier linting tests from running over test files +- Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520)) ### General From b1e255957470f1ff83b897584f4cc45117b1d2f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 21 Apr 2022 11:04:56 +0200 Subject: [PATCH 054/136] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89d265bf04..378f14f724 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,12 @@ - Fix Prettier formatting bug in completion email HTML template ([#1509](https://github.com/nf-core/tools/issues/1509)) - Removed retry strategy for AWS tests CI, as Nextflow now handles spot instance retries itself - Add `.prettierignore` file to stop Prettier linting tests from running over test files -- Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520)) ### General - Bumped the minimum version of `rich` from `v10` to `v10.7.0` - Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors. +- Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520)) ### Modules From 9bd7ddfbbae4579b597fd02ff4ff6d680276457a Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 21 Apr 2022 11:32:30 +0100 Subject: [PATCH 055/136] Fix #1419 --- CHANGELOG.md | 1 + nf_core/schema.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57a4cff8ea..1d190fe85d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors. - Add actions workflow to respond to `@nf-core-bot fix linting` comments on nf-core/tools PRs - Linting: Don't allow a `.nf-core.yaml` file, should be `.yml` ([#1515](https://github.com/nf-core/tools/pull/1515)). +- Not all definitions in JSON schema have a "properties", leading to an error ([#1419](https://github.com/nf-core/tools/issues/1419)) ### Modules diff --git a/nf_core/schema.py b/nf_core/schema.py index 5784645543..522f48d20b 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -613,6 +613,29 @@ def get_wf_params(self): ) ) + def remove_schema_empty_definitions(self): + """ + Go through top-level schema remove definitions that don't have + any property attributes + """ + # Make copy of schema + schema_no_empty_definitions = copy.deepcopy(self.schema) + + ## Identify and remove empty definitions from the schema + empty_definitions = [] + for d_key, d_schema in list(schema_no_empty_definitions.get("definitions", {}).items()): + if not d_schema['properties']: + del schema_no_empty_definitions["definitions"][d_key] + empty_definitions.append(d_key) + + # Remove "allOf" group with empty definitions from the schema + for d_key in empty_definitions: + allOf = {'$ref': "#/definitions/{}".format(d_key)} + if allOf in schema_no_empty_definitions["allOf"]: + schema_no_empty_definitions["allOf"].remove(allOf) + + self.schema = schema_no_empty_definitions + def remove_schema_notfound_configs(self): """ Go through top-level schema and all definitions sub-schemas to remove @@ -625,6 +648,8 @@ def remove_schema_notfound_configs(self): cleaned_schema, p_removed = self.remove_schema_notfound_configs_single_schema(definition) self.schema["definitions"][d_key] = cleaned_schema params_removed.extend(p_removed) + self.remove_schema_empty_definitions() + return params_removed def remove_schema_notfound_configs_single_schema(self, schema): From 7fb4c697fac3cccad93a2089bfb77f25851854d4 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 21 Apr 2022 12:07:52 +0100 Subject: [PATCH 056/136] Black --- nf_core/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 522f48d20b..788adf73e2 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -624,13 +624,13 @@ def remove_schema_empty_definitions(self): ## Identify and remove empty definitions from the schema empty_definitions = [] for d_key, d_schema in list(schema_no_empty_definitions.get("definitions", {}).items()): - if not d_schema['properties']: + if not d_schema["properties"]: del schema_no_empty_definitions["definitions"][d_key] empty_definitions.append(d_key) # Remove "allOf" group with empty definitions from the schema for d_key in empty_definitions: - allOf = {'$ref': "#/definitions/{}".format(d_key)} + allOf = {"$ref": "#/definitions/{}".format(d_key)} if allOf in schema_no_empty_definitions["allOf"]: schema_no_empty_definitions["allOf"].remove(allOf) From 9fec61b4f2bed1dedde65b7ae4078fe2c1cb23c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 21 Apr 2022 13:52:56 +0200 Subject: [PATCH 057/136] remove remove stringtie special case --- nf_core/modules/install.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index a72c45da76..96f04cb34c 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -131,11 +131,7 @@ def install(self, module): return False # Print include statement - if module == "stringtie/stringtie": - # Only with stringtie the process name is STRINGTIE instead of STRINGTIE_STRINGTIE - module_name = module.upper().split("/")[0] - else: - module_name = "_".join(module.upper().split("/")) + module_name = "_".join(module.upper().split("/")) log.info(f"Include statement: include {{ {module_name} }} from '.{os.path.join(*install_folder, module)}/main’") # Update module.json with newly installed module From 6254c7ca7a0e50b6225b4f82268f672f0e0718c1 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Thu, 21 Apr 2022 15:23:45 +0200 Subject: [PATCH 058/136] tests: add docstrings to test cases --- tests/test_mullled.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_mullled.py b/tests/test_mullled.py index 2206d65eef..5eaa41fc81 100644 --- a/tests/test_mullled.py +++ b/tests/test_mullled.py @@ -13,7 +13,7 @@ ], ) def test_target_parsing(specs, expected): - """""" + """Test that valid specifications are correctly parsed into tool, version pairs.""" assert MulledImageNameGenerator.parse_targets(specs) == expected @@ -25,7 +25,7 @@ def test_target_parsing(specs, expected): ], ) def test_wrong_specification(specs): - """""" + """Test that unexpected version constraints fail.""" with pytest.raises(ValueError, match="expected format"): MulledImageNameGenerator.parse_targets(specs) @@ -38,7 +38,7 @@ def test_wrong_specification(specs): ], ) def test_noncompliant_version(specs): - """""" + """Test that version string that do not comply with PEP440 fail.""" with pytest.raises(ValueError, match="PEP440"): MulledImageNameGenerator.parse_targets(specs) @@ -57,4 +57,5 @@ def test_noncompliant_version(specs): ], ) def test_generate_image_name(specs, expected): + """Test that a known image name is generated from given targets.""" assert MulledImageNameGenerator.generate_image_name(specs) == expected From 4ce681d429260c3399aeb0ec3423c2faa4e51f41 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Thu, 21 Apr 2022 15:31:07 +0200 Subject: [PATCH 059/136] refactor: rename tools parameter to specifications --- .editorconfig | 1 + nf_core/__main__.py | 6 +++--- nf_core/modules/mulled.py | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.editorconfig b/.editorconfig index c29b6c7eee..0ffa5e3ec1 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,6 +7,7 @@ insert_final_newline = true trim_trailing_whitespace = true indent_size = 4 indent_style = space +max_line_length = 120 [*.{md,yml,yaml,html,css,scss,js}] indent_size = 2 diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 569a45d657..49389fd9e6 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -676,8 +676,8 @@ def bump_versions(ctx, tool, dir, all, show_all): # nf-core modules mulled @modules.command() -@click.argument("tools", required=True, nargs=-1, metavar=" <...>") -def mulled(tools): +@click.argument("specifications", required=True, nargs=-1, metavar=" <...>") +def mulled(specifications): """ Generate the name of a BioContainers mulled image version 2. @@ -687,7 +687,7 @@ def mulled(tools): """ print( nf_core.modules.mulled.MulledImageNameGenerator.generate_image_name( - nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(tools) + nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(specifications) ) ) diff --git a/nf_core/modules/mulled.py b/nf_core/modules/mulled.py index b84561a5c1..197913c99e 100644 --- a/nf_core/modules/mulled.py +++ b/nf_core/modules/mulled.py @@ -23,16 +23,16 @@ class MulledImageNameGenerator: _split_pattern = re.compile(r"==?") @classmethod - def parse_targets(cls, tools: Iterable[str]) -> List[Tuple[str, str]]: + def parse_targets(cls, specifications: Iterable[str]) -> List[Tuple[str, str]]: """ - Parse tool, version pairs from two or more version strings. + Parse tool, version pairs from specification strings. Args: - tools: An iterable of strings that contain tools and their versions. + specifications: An iterable of strings that contain tools and their versions. """ result = [] - for spec in tools: + for spec in specifications: try: tool, version = cls._split_pattern.split(spec, maxsplit=1) except ValueError: From 4c45f6a1471685a70964014defd6a651fc05f33c Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Thu, 21 Apr 2022 15:58:01 +0200 Subject: [PATCH 060/136] fix: include image build number --- nf_core/__main__.py | 17 ++++++++++++----- nf_core/modules/mulled.py | 5 +++-- tests/test_mullled.py | 8 ++++++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 49389fd9e6..6d78b35845 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -677,7 +677,15 @@ def bump_versions(ctx, tool, dir, all, show_all): # nf-core modules mulled @modules.command() @click.argument("specifications", required=True, nargs=-1, metavar=" <...>") -def mulled(specifications): +@click.option( + "--build-number", + type=int, + default=0, + show_default=True, + metavar="", + help="The build number for this image. This is an incremental value that starts at zero.", +) +def mulled(specifications, build_number): """ Generate the name of a BioContainers mulled image version 2. @@ -685,11 +693,10 @@ def mulled(specifications): that image, this command can generate it for you. """ - print( - nf_core.modules.mulled.MulledImageNameGenerator.generate_image_name( - nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(specifications) - ) + image_name = nf_core.modules.mulled.MulledImageNameGenerator.generate_image_name( + nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(specifications), build_number=build_number ) + print(image_name) # nf-core schema subcommands diff --git a/nf_core/modules/mulled.py b/nf_core/modules/mulled.py index 197913c99e..fe941ee941 100644 --- a/nf_core/modules/mulled.py +++ b/nf_core/modules/mulled.py @@ -47,12 +47,13 @@ def parse_targets(cls, specifications: Iterable[str]) -> List[Tuple[str, str]]: return result @classmethod - def generate_image_name(cls, targets: Iterable[Tuple[str, str]]) -> str: + def generate_image_name(cls, targets: Iterable[Tuple[str, str]], build_number: int = 0) -> str: """ Generate the name of a BioContainers mulled image version 2. Args: targets: One or more tool, version pairs of the multi-tool container image. + build_number: The build number for this image. This is an incremental value that starts at zero. """ - return v2_image_name([build_target(name, version) for name, version in targets]) + return v2_image_name([build_target(name, version) for name, version in targets], image_build=str(build_number)) diff --git a/tests/test_mullled.py b/tests/test_mullled.py index 5eaa41fc81..cf0a4fcfc0 100644 --- a/tests/test_mullled.py +++ b/tests/test_mullled.py @@ -48,11 +48,15 @@ def test_noncompliant_version(specs): [ ( [("chromap", "0.2.1"), ("samtools", "1.15")], - "mulled-v2-1f09f39f20b1c4ee36581dc81cc323c70e661633:bd74d08a359024829a7aec1638a28607bbcd8a58", + "mulled-v2-1f09f39f20b1c4ee36581dc81cc323c70e661633:bd74d08a359024829a7aec1638a28607bbcd8a58-0", ), ( [("pysam", "0.16.0.1"), ("biopython", "1.78")], - "mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f", + "mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0", + ), + ( + [("samclip", "0.4.0"), ("samtools", "1.15")], + "mulled-v2-d057255d4027721f3ab57f6a599a2ae81cb3cbe3:13051b049b6ae536d76031ba94a0b8e78e364815-0", ), ], ) From 1ff8ade036d8101664f65453c91e8c5275b03e77 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Thu, 21 Apr 2022 16:30:44 +0200 Subject: [PATCH 061/136] refactor: check if the generated image name exists --- nf_core/__main__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 6d78b35845..ce3035845e 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1,10 +1,10 @@ #!/usr/bin/env python """ nf-core: Helper tools for use with nf-core Nextflow pipelines. """ - from rich import print import logging import os import re +import requests import rich.console import rich.logging import rich.traceback @@ -697,6 +697,13 @@ def mulled(specifications, build_number): nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(specifications), build_number=build_number ) print(image_name) + response = requests.get(f"https://quay.io/biocontainers/{image_name}/", allow_redirects=True) + if response.status_code != 200: + log.error( + "The generated multi-tool container image does not seem to exist yet. Are you sure that you provided the " + "right combination of tools and versions?" + ) + sys.exit(1) # nf-core schema subcommands From 1882e87656be142aa0a02c098cb6655b5d819400 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Thu, 21 Apr 2022 16:32:08 +0200 Subject: [PATCH 062/136] chore: undo accidental commit --- .editorconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 0ffa5e3ec1..c29b6c7eee 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,7 +7,6 @@ insert_final_newline = true trim_trailing_whitespace = true indent_size = 4 indent_style = space -max_line_length = 120 [*.{md,yml,yaml,html,css,scss,js}] indent_size = 2 From 893fc9a187a50d4199183c243e76e0bb0b04e1f6 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Thu, 21 Apr 2022 20:31:41 +0200 Subject: [PATCH 063/136] refactor: move check to class, expand error message --- nf_core/__main__.py | 17 ++++++++++------- nf_core/modules/mulled.py | 8 ++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index ce3035845e..ae8b995c99 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -4,7 +4,6 @@ import logging import os import re -import requests import rich.console import rich.logging import rich.traceback @@ -693,15 +692,19 @@ def mulled(specifications, build_number): that image, this command can generate it for you. """ - image_name = nf_core.modules.mulled.MulledImageNameGenerator.generate_image_name( - nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(specifications), build_number=build_number + from nf_core.modules.mulled import MulledImageNameGenerator + + image_name = MulledImageNameGenerator.generate_image_name( + MulledImageNameGenerator.parse_targets(specifications), build_number=build_number ) print(image_name) - response = requests.get(f"https://quay.io/biocontainers/{image_name}/", allow_redirects=True) - if response.status_code != 200: + if not MulledImageNameGenerator.image_exists(image_name): log.error( - "The generated multi-tool container image does not seem to exist yet. Are you sure that you provided the " - "right combination of tools and versions?" + "The generated multi-tool container image name does not seem to exist yet. Please double check that your " + "provided combination of tools and versions exists in the file:\n" + "https://github.com/BioContainers/multi-package-containers/blob/master/combinations/hash.tsv\n" + "If it does not, please add your desired combination as detailed at:\n" + "https://github.com/BioContainers/multi-package-containers\n" ) sys.exit(1) diff --git a/nf_core/modules/mulled.py b/nf_core/modules/mulled.py index fe941ee941..c47f35a559 100644 --- a/nf_core/modules/mulled.py +++ b/nf_core/modules/mulled.py @@ -6,6 +6,7 @@ from packaging.version import Version, InvalidVersion from typing import Iterable, Tuple, List +import requests from galaxy.tool_util.deps.mulled.util import build_target, v2_image_name @@ -57,3 +58,10 @@ def generate_image_name(cls, targets: Iterable[Tuple[str, str]], build_number: i """ return v2_image_name([build_target(name, version) for name, version in targets], image_build=str(build_number)) + + @classmethod + def image_exists(cls, image_name: str) -> bool: + """Check whether a given BioContainers image name exists via a call to the quay.io API.""" + response = requests.get(f"https://quay.io/biocontainers/{image_name}/", allow_redirects=True) + log.debug(response.text) + return response.status_code == 200 From 54d218f2c418a25b892e9c16fcb111aacc8457b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 25 Apr 2022 11:40:08 +0200 Subject: [PATCH 064/136] add new command modules test --- nf_core/__main__.py | 21 ++++++++- nf_core/modules/__init__.py | 1 + nf_core/modules/module_test.py | 81 ++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 nf_core/modules/module_test.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 8affd23bfe..f9118d1322 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -49,7 +49,7 @@ }, { "name": "Developing new modules", - "commands": ["create", "create-test-yml", "lint", "bump-versions"], + "commands": ["create", "create-test-yml", "lint", "bump-versions", "test"], }, ], } @@ -674,6 +674,25 @@ def bump_versions(ctx, tool, dir, all, show_all): sys.exit(1) +# nf-core modules test +@modules.command("test") +@click.pass_context +@click.argument("tool", type=str, required=False, metavar=" or ") +@click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") +def test_module(ctx, tool, no_prompts): + """ + Run module tests locally. + + Given the name of a module, runs the Nextflow test command. + """ + try: + meta_builder = nf_core.modules.ModulesTest(tool, no_prompts) + meta_builder.run() + except UserWarning as e: + log.critical(e) + sys.exit(1) + + # nf-core schema subcommands @nf_core_cli.group() def schema(): diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index f833d52564..c85485c4f1 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -9,3 +9,4 @@ from .update import ModuleUpdate from .remove import ModuleRemove from .info import ModuleInfo +from .module_test import ModulesTest diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py new file mode 100644 index 0000000000..4d9bf95014 --- /dev/null +++ b/nf_core/modules/module_test.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +""" +The ModulesTest class runs the tests locally +""" + +import logging +import questionary +import os + +"""from __future__ import print_function +from rich.syntax import Syntax + +import errno +import gzip +import hashlib +import operator + + +import re +import rich +import shlex +import subprocess +import tempfile +import yaml + +""" +import nf_core.utils +from .modules_repo import ModulesRepo +from .test_yml_builder import ModulesTestYmlBuilder + +log = logging.getLogger(__name__) + + +class ModulesTest(ModulesTestYmlBuilder): + def __init__( + self, + module_name=None, + no_prompts=False, + ): + self.run_tests = True + self.module_name = module_name + self.no_prompts = no_prompts + self.module_dir = None + self.module_test_main = None + self.entry_points = [] + self.tests = [] + self.errors = [] + + def run(self): + """Run test steps""" + if not self.no_prompts: + log.info( + "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" + ) + self.check_inputs_test() + self.scrape_workflow_entry_points() + self.build_all_tests() + if len(self.errors) > 0: + errors = "\n - ".join(self.errors) + raise UserWarning(f"Ran, but found errors:\n - {errors}") + + def check_inputs_test(self): + """Do more complex checks about supplied flags.""" + + # Get the tool name if not specified + if self.module_name is None: + modules_repo = ModulesRepo() + modules_repo.get_modules_file_tree() + self.module_name = questionary.autocomplete( + "Tool name:", + choices=modules_repo.modules_avail_module_names, + style=nf_core.utils.nfcore_question_style, + ).ask() + self.module_dir = os.path.join("modules", *self.module_name.split("/")) + self.module_test_main = os.path.join("tests", "modules", *self.module_name.split("/"), "main.nf") + + # First, sanity check that the module directory exists + if not os.path.isdir(self.module_dir): + raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") + if not os.path.exists(self.module_test_main): + raise UserWarning(f"Cannot find module test workflow '{self.module_test_main}'") From 5097f04ac95616cc1df7135d9cf14b30804f52db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Apr 2022 10:18:15 +0200 Subject: [PATCH 065/136] run tests with pytest --- nf_core/__main__.py | 5 ++- nf_core/modules/module_test.py | 80 ++++++++++++++++++++-------------- 2 files changed, 51 insertions(+), 34 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index f9118d1322..3863ea49c8 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -679,14 +679,15 @@ def bump_versions(ctx, tool, dir, all, show_all): @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") -def test_module(ctx, tool, no_prompts): +@click.option("-a", "--pytest_args", type=str, required=False, multiple=True, help="Additional pytest arguments") +def test_module(ctx, tool, no_prompts, pytest_args): """ Run module tests locally. Given the name of a module, runs the Nextflow test command. """ try: - meta_builder = nf_core.modules.ModulesTest(tool, no_prompts) + meta_builder = nf_core.modules.ModulesTest(tool, no_prompts, pytest_args) meta_builder.run() except UserWarning as e: log.critical(e) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 4d9bf95014..8316d9502d 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -6,45 +6,25 @@ import logging import questionary import os - -"""from __future__ import print_function -from rich.syntax import Syntax - -import errno -import gzip -import hashlib -import operator - - -import re +import pytest +import sys import rich -import shlex -import subprocess -import tempfile -import yaml -""" import nf_core.utils from .modules_repo import ModulesRepo -from .test_yml_builder import ModulesTestYmlBuilder log = logging.getLogger(__name__) - -class ModulesTest(ModulesTestYmlBuilder): +class ModulesTest(object): def __init__( self, module_name=None, no_prompts=False, + pytest_args="", ): - self.run_tests = True self.module_name = module_name self.no_prompts = no_prompts - self.module_dir = None - self.module_test_main = None - self.entry_points = [] - self.tests = [] - self.errors = [] + self.pytest_args = pytest_args def run(self): """Run test steps""" @@ -52,14 +32,11 @@ def run(self): log.info( "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" ) - self.check_inputs_test() - self.scrape_workflow_entry_points() - self.build_all_tests() - if len(self.errors) > 0: - errors = "\n - ".join(self.errors) - raise UserWarning(f"Ran, but found errors:\n - {errors}") + self.check_inputs() + self.set_profile() + self.run_pytests() - def check_inputs_test(self): + def check_inputs(self): """Do more complex checks about supplied flags.""" # Get the tool name if not specified @@ -79,3 +56,42 @@ def check_inputs_test(self): raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") if not os.path.exists(self.module_test_main): raise UserWarning(f"Cannot find module test workflow '{self.module_test_main}'") + + def set_profile(self): + """Set $PROFILE env variable. + The config expects $PROFILE and Nextflow fails if it's not set. + """ + if os.environ.get("PROFILE") is None: + os.environ["PROFILE"] = "" + if self.no_prompts: + log.info( + "Setting env var '$PROFILE' to an empty string as not set.\n" + "Tests will run with Docker by default. " + "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." + ) + else: + question = { + "type": "list", + "name": "profile", + "message": "Choose software profile", + "choices": ["Docker", "Singularity", "Conda"], + } + answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) + profile = answer["profile"].lower() + if profile in ["singularity", "conda"]: + os.environ["PROFILE"] = profile + log.info(f"Setting env var '$PROFILE' to '{profile}'") + + def run_pytests(self): + """Given a module name, run tests.""" + # Print nice divider line + console = rich.console.Console() + console.print("[black]" + "─" * console.width) + + # Set pytest arguments + command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd"] + command_args += self.pytest_args + + # Run pytest + log.info(f"Running pytest for module '{self.module_name}'") + sys.exit(pytest.main(command_args)) From 32274990c5c14d226617d60d1981969d56d8f913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Apr 2022 10:29:36 +0200 Subject: [PATCH 066/136] remove test workflow check --- nf_core/modules/module_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 8316d9502d..ee5e39c4f0 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -25,6 +25,7 @@ def __init__( self.module_name = module_name self.no_prompts = no_prompts self.pytest_args = pytest_args + self.module_dir = None def run(self): """Run test steps""" @@ -49,13 +50,10 @@ def check_inputs(self): style=nf_core.utils.nfcore_question_style, ).ask() self.module_dir = os.path.join("modules", *self.module_name.split("/")) - self.module_test_main = os.path.join("tests", "modules", *self.module_name.split("/"), "main.nf") # First, sanity check that the module directory exists if not os.path.isdir(self.module_dir): raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") - if not os.path.exists(self.module_test_main): - raise UserWarning(f"Cannot find module test workflow '{self.module_test_main}'") def set_profile(self): """Set $PROFILE env variable. From 2e2ea58479e3725b37877e5cbb7ae1e075fb8700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Apr 2022 13:10:10 +0200 Subject: [PATCH 067/136] add code tests --- tests/modules/module_test.py | 15 +++++++++++++++ tests/test_modules.py | 4 ++++ 2 files changed, 19 insertions(+) create mode 100644 tests/modules/module_test.py diff --git a/tests/modules/module_test.py b/tests/modules/module_test.py new file mode 100644 index 0000000000..12a4c85c4d --- /dev/null +++ b/tests/modules/module_test.py @@ -0,0 +1,15 @@ +"""Test the 'modules test' command which runs module pytests.""" +import os +import pytest + +import nf_core.modules + +def test_modules_test_check_inputs(self): + """Test the check_inputs() function - raise UserWarning because module doesn't exist""" + cwd = os.getcwd() + os.chdir(self.nfcore_modules) + meta_builder = nf_core.modules.ModulesTest("none", True, "") + with pytest.raises(UserWarning) as excinfo: + meta_builder.check_inputs() + os.chdir(cwd) + assert "Cannot find directory" in str(excinfo.value) diff --git a/tests/test_modules.py b/tests/test_modules.py index cfa3408e69..00cc89c36d 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -113,3 +113,7 @@ def test_modulesrepo_class(self): test_modules_bump_versions_fail, test_modules_bump_versions_fail_unknown_version, ) + + from .modules.module_test import ( + test_modules_test_check_inputs, + ) From c0b29f5944f86fcf598ce4a6fbf444974b7a53be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Apr 2022 13:29:42 +0200 Subject: [PATCH 068/136] fix linting --- nf_core/modules/module_test.py | 1 + tests/modules/module_test.py | 1 + 2 files changed, 2 insertions(+) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index ee5e39c4f0..5338aeb81e 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -15,6 +15,7 @@ log = logging.getLogger(__name__) + class ModulesTest(object): def __init__( self, diff --git a/tests/modules/module_test.py b/tests/modules/module_test.py index 12a4c85c4d..ac0c36c997 100644 --- a/tests/modules/module_test.py +++ b/tests/modules/module_test.py @@ -4,6 +4,7 @@ import nf_core.modules + def test_modules_test_check_inputs(self): """Test the check_inputs() function - raise UserWarning because module doesn't exist""" cwd = os.getcwd() From bf38c4d03aaf14ea6c72f0f852d268f04ca19e1c Mon Sep 17 00:00:00 2001 From: Nathan Spix <56930974+njspix@users.noreply.github.com> Date: Tue, 26 Apr 2022 09:29:15 -0400 Subject: [PATCH 069/136] Update actions_ci.py In cases where e.g. multiple aligners are specified in the GitHub Actions matrix, trying to pull a 'NXF_VER' key out of the include dictionary can cause line 140 to fail when it shouldn't. The get() method will not error if the key is not present, fixing this problem. --- nf_core/lint/actions_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint/actions_ci.py b/nf_core/lint/actions_ci.py index 89d9becd04..1a02680ece 100644 --- a/nf_core/lint/actions_ci.py +++ b/nf_core/lint/actions_ci.py @@ -137,7 +137,7 @@ def actions_ci(self): # Check that we are testing the minimum nextflow version try: matrix = ciwf["jobs"]["test"]["strategy"]["matrix"]["include"] - assert any([i["NXF_VER"] == self.minNextflowVersion for i in matrix]) + assert any([i.get("NXF_VER") == self.minNextflowVersion for i in matrix]) except (KeyError, TypeError): failed.append("'.github/workflows/ci.yml' does not check minimum NF version") except AssertionError: From 99b5170a5801ab648442157d4a31c4bdcfa0cff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Apr 2022 15:33:13 +0200 Subject: [PATCH 070/136] apply comments from PR review --- nf_core/modules/module_test.py | 39 ++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 5338aeb81e..c6a81f8672 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -17,6 +17,31 @@ class ModulesTest(object): + """ + Class to run module pytests. + + ... + + Attributes + ---------- + module_name : str + name of the tool to run tests for + no_prompts : bool + flat indicating if prompts are used + pytest_args : tuple + additional arguments passed to pytest command + + Methods + ------- + run(): + Run test steps + __check_inputs(): + Check inputs. Ask for module_name if not provided and check that the directory exists + __set_profile(): + Set software profile + __run_pytests(self): + Run pytest + """ def __init__( self, module_name=None, @@ -34,15 +59,17 @@ def run(self): log.info( "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" ) - self.check_inputs() - self.set_profile() - self.run_pytests() + self.__check_inputs() + self.__set_profile() + self.__run_pytests() - def check_inputs(self): + def __check_inputs(self): """Do more complex checks about supplied flags.""" # Get the tool name if not specified if self.module_name is None: + if self.no_prompts: + raise UserWarning(f"Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL.") modules_repo = ModulesRepo() modules_repo.get_modules_file_tree() self.module_name = questionary.autocomplete( @@ -56,7 +83,7 @@ def check_inputs(self): if not os.path.isdir(self.module_dir): raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") - def set_profile(self): + def __set_profile(self): """Set $PROFILE env variable. The config expects $PROFILE and Nextflow fails if it's not set. """ @@ -81,7 +108,7 @@ def set_profile(self): os.environ["PROFILE"] = profile log.info(f"Setting env var '$PROFILE' to '{profile}'") - def run_pytests(self): + def __run_pytests(self): """Given a module name, run tests.""" # Print nice divider line console = rich.console.Console() From b80fd622c1de18d583ff302797fdab5bbcb28b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Apr 2022 15:50:51 +0200 Subject: [PATCH 071/136] fix linting --- nf_core/__main__.py | 2 +- nf_core/modules/module_test.py | 53 ++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 51fb5aebac..45fcff9a60 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -708,7 +708,7 @@ def mulled(specifications, build_number): ) sys.exit(1) - + # nf-core modules test @modules.command("test") @click.pass_context diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index c6a81f8672..3d2aa8d58d 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -18,30 +18,31 @@ class ModulesTest(object): """ - Class to run module pytests. - - ... - - Attributes - ---------- - module_name : str - name of the tool to run tests for - no_prompts : bool - flat indicating if prompts are used - pytest_args : tuple - additional arguments passed to pytest command - - Methods - ------- - run(): - Run test steps - __check_inputs(): - Check inputs. Ask for module_name if not provided and check that the directory exists - __set_profile(): - Set software profile - __run_pytests(self): - Run pytest - """ + Class to run module pytests. + + ... + + Attributes + ---------- + module_name : str + name of the tool to run tests for + no_prompts : bool + flat indicating if prompts are used + pytest_args : tuple + additional arguments passed to pytest command + + Methods + ------- + run(): + Run test steps + __check_inputs(): + Check inputs. Ask for module_name if not provided and check that the directory exists + __set_profile(): + Set software profile + __run_pytests(self): + Run pytest + """ + def __init__( self, module_name=None, @@ -69,7 +70,9 @@ def __check_inputs(self): # Get the tool name if not specified if self.module_name is None: if self.no_prompts: - raise UserWarning(f"Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL.") + raise UserWarning( + f"Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL." + ) modules_repo = ModulesRepo() modules_repo.get_modules_file_tree() self.module_name = questionary.autocomplete( From 6c006fdf2035e0aa5f0ef9a798ced52ad7257a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 27 Apr 2022 11:25:48 +0200 Subject: [PATCH 072/136] apply comments from PR --- nf_core/modules/module_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 3d2aa8d58d..bb39c27073 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -35,11 +35,11 @@ class ModulesTest(object): ------- run(): Run test steps - __check_inputs(): + _check_inputs(): Check inputs. Ask for module_name if not provided and check that the directory exists - __set_profile(): + _set_profile(): Set software profile - __run_pytests(self): + _run_pytests(self): Run pytest """ @@ -60,11 +60,11 @@ def run(self): log.info( "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" ) - self.__check_inputs() - self.__set_profile() - self.__run_pytests() + self._check_inputs() + self._set_profile() + self._run_pytests() - def __check_inputs(self): + def _check_inputs(self): """Do more complex checks about supplied flags.""" # Get the tool name if not specified @@ -86,7 +86,7 @@ def __check_inputs(self): if not os.path.isdir(self.module_dir): raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") - def __set_profile(self): + def _set_profile(self): """Set $PROFILE env variable. The config expects $PROFILE and Nextflow fails if it's not set. """ @@ -111,7 +111,7 @@ def __set_profile(self): os.environ["PROFILE"] = profile log.info(f"Setting env var '$PROFILE' to '{profile}'") - def __run_pytests(self): + def _run_pytests(self): """Given a module name, run tests.""" # Print nice divider line console = rich.console.Console() From 0e4a835fb68ad6e698a161bcd347619bbd3c489e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 27 Apr 2022 12:48:44 +0200 Subject: [PATCH 073/136] fix test --- tests/modules/module_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/module_test.py b/tests/modules/module_test.py index ac0c36c997..fbad1c9fc5 100644 --- a/tests/modules/module_test.py +++ b/tests/modules/module_test.py @@ -11,6 +11,6 @@ def test_modules_test_check_inputs(self): os.chdir(self.nfcore_modules) meta_builder = nf_core.modules.ModulesTest("none", True, "") with pytest.raises(UserWarning) as excinfo: - meta_builder.check_inputs() + meta_builder._check_inputs() os.chdir(cwd) assert "Cannot find directory" in str(excinfo.value) From a8088532f15a2addaed0a2444a0d1312efa84865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 27 Apr 2022 15:26:32 +0200 Subject: [PATCH 074/136] always set profile --- nf_core/modules/module_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index bb39c27073..7f33774776 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -107,9 +107,8 @@ def _set_profile(self): } answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) profile = answer["profile"].lower() - if profile in ["singularity", "conda"]: - os.environ["PROFILE"] = profile - log.info(f"Setting env var '$PROFILE' to '{profile}'") + os.environ["PROFILE"] = profile + log.info(f"Setting env var '$PROFILE' to '{profile}'") def _run_pytests(self): """Given a module name, run tests.""" From 4165a9fecb950488d982af3417a0a1ae44cb94dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 27 Apr 2022 15:29:44 +0200 Subject: [PATCH 075/136] do not abbreviate env var --- nf_core/modules/module_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 7f33774776..db204ba97a 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -94,7 +94,7 @@ def _set_profile(self): os.environ["PROFILE"] = "" if self.no_prompts: log.info( - "Setting env var '$PROFILE' to an empty string as not set.\n" + "Setting environment variable '$PROFILE' to an empty string as not set.\n" "Tests will run with Docker by default. " "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." ) @@ -108,7 +108,7 @@ def _set_profile(self): answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) profile = answer["profile"].lower() os.environ["PROFILE"] = profile - log.info(f"Setting env var '$PROFILE' to '{profile}'") + log.info(f"Setting environment variable '$PROFILE' to '{profile}'") def _run_pytests(self): """Given a module name, run tests.""" From b431f9832b090ad4920349550286ff5190617c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 27 Apr 2022 16:06:53 +0200 Subject: [PATCH 076/136] add new test --- nf_core/modules/module_test.py | 2 +- tests/modules/module_test.py | 11 +++++++++++ tests/test_modules.py | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index db204ba97a..25e4991a4b 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -71,7 +71,7 @@ def _check_inputs(self): if self.module_name is None: if self.no_prompts: raise UserWarning( - f"Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL." + "Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL." ) modules_repo = ModulesRepo() modules_repo.get_modules_file_tree() diff --git a/tests/modules/module_test.py b/tests/modules/module_test.py index fbad1c9fc5..a4559ffde5 100644 --- a/tests/modules/module_test.py +++ b/tests/modules/module_test.py @@ -14,3 +14,14 @@ def test_modules_test_check_inputs(self): meta_builder._check_inputs() os.chdir(cwd) assert "Cannot find directory" in str(excinfo.value) + + +def test_modules_test_no_name_no_prompts(self): + """Test the check_inputs() function - raise UserWarning prompts are deactivated and module name is not provided.""" + cwd = os.getcwd() + os.chdir(self.nfcore_modules) + meta_builder = nf_core.modules.ModulesTest(None, True, "") + with pytest.raises(UserWarning) as excinfo: + meta_builder._check_inputs() + os.chdir(cwd) + assert "Tool name not provided and prompts deactivated." in str(excinfo.value) diff --git a/tests/test_modules.py b/tests/test_modules.py index 00cc89c36d..b12333b51d 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -116,4 +116,5 @@ def test_modulesrepo_class(self): from .modules.module_test import ( test_modules_test_check_inputs, + test_modules_test_no_name_no_prompts, ) From 58df6393e4be08cd3de8465a456206aec7d18767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 27 Apr 2022 16:53:46 +0200 Subject: [PATCH 077/136] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86a8206b91..306425fd2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - Escaped test run output before logging it, to avoid a rich ` MarkupError` - Add a new command `nf-core modules mulled` which can generate the name for a multi-tool container image. +- Add a new command `nf-core modules test` which runs pytests locally. ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] From 530a3839deee338d82a2ecfb3341ed0ca989e3af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 28 Apr 2022 09:57:41 +0200 Subject: [PATCH 078/136] specify test files as test_*.py in pytest.ini --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 57209fee41..652bdf8e53 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,3 +3,4 @@ filterwarnings = ignore::pytest.PytestRemovedIn8Warning:_pytest.nodes:140 testpaths = tests +python_files = test_*.py From b885e5f19184ba5f9e42deae75a01c08a9983b83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 28 Apr 2022 14:31:30 +0200 Subject: [PATCH 079/136] apply comments from PR review --- nf_core/modules/module_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 25e4991a4b..57ca2c6932 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -9,6 +9,7 @@ import pytest import sys import rich +from pathlib import Path import nf_core.utils from .modules_repo import ModulesRepo @@ -52,7 +53,6 @@ def __init__( self.module_name = module_name self.no_prompts = no_prompts self.pytest_args = pytest_args - self.module_dir = None def run(self): """Run test steps""" @@ -80,11 +80,11 @@ def _check_inputs(self): choices=modules_repo.modules_avail_module_names, style=nf_core.utils.nfcore_question_style, ).ask() - self.module_dir = os.path.join("modules", *self.module_name.split("/")) + module_dir = Path("modules") / self.module_name # First, sanity check that the module directory exists - if not os.path.isdir(self.module_dir): - raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") + if not module_dir.is_dir(): + raise UserWarning(f"Cannot find directory '{module_dir}'. Should be TOOL/SUBTOOL or TOOL") def _set_profile(self): """Set $PROFILE env variable. @@ -117,7 +117,7 @@ def _run_pytests(self): console.print("[black]" + "─" * console.width) # Set pytest arguments - command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd"] + command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] command_args += self.pytest_args # Run pytest From f49af39a24209fd08f2b62d6feb91ff503f01f12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 29 Apr 2022 14:12:52 +0200 Subject: [PATCH 080/136] check if profile is available --- nf_core/modules/module_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 57ca2c6932..8ca1027419 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -9,6 +9,8 @@ import pytest import sys import rich +import subprocess +import shlex from pathlib import Path import nf_core.utils @@ -62,6 +64,7 @@ def run(self): ) self._check_inputs() self._set_profile() + self._check_profile() self._run_pytests() def _check_inputs(self): @@ -110,6 +113,14 @@ def _set_profile(self): os.environ["PROFILE"] = profile log.info(f"Setting environment variable '$PROFILE' to '{profile}'") + def _check_profile(self): + """Check if profile is available""" + profile = os.environ.get("PROFILE") + try: + profile_check = subprocess.check_output(shlex.split(f"{profile} --help"), stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + raise UserWarning(f"Error with profile {profile} (exit code {e.returncode})\n[red]{e.output.decode()}") + def _run_pytests(self): """Given a module name, run tests.""" # Print nice divider line From f33f1fb756c1239dfaf6da1c1b37e2c32603de00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 29 Apr 2022 14:41:02 +0200 Subject: [PATCH 081/136] modify linting --- nf_core/modules/module_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 8ca1027419..c2510d3ee9 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -117,7 +117,9 @@ def _check_profile(self): """Check if profile is available""" profile = os.environ.get("PROFILE") try: - profile_check = subprocess.check_output(shlex.split(f"{profile} --help"), stderr=subprocess.STDOUT, shell=True) + profile_check = subprocess.check_output( + shlex.split(f"{profile} --help"), stderr=subprocess.STDOUT, shell=True + ) except subprocess.CalledProcessError as e: raise UserWarning(f"Error with profile {profile} (exit code {e.returncode})\n[red]{e.output.decode()}") From b732386f60fd8f20c79766163a3bbd137622edab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 2 May 2022 12:42:59 +0200 Subject: [PATCH 082/136] Add nf-core/modules folder warning Co-authored-by: Gisela Gabernet --- nf_core/modules/module_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index c2510d3ee9..6507c58219 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -87,7 +87,7 @@ def _check_inputs(self): # First, sanity check that the module directory exists if not module_dir.is_dir(): - raise UserWarning(f"Cannot find directory '{module_dir}'. Should be TOOL/SUBTOOL or TOOL") + raise UserWarning(f"Cannot find directory '{module_dir}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?") def _set_profile(self): """Set $PROFILE env variable. From c182ef3a39f1865776707ac861c5f21475c6c715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 2 May 2022 12:44:22 +0200 Subject: [PATCH 083/136] Add valid profile check Co-authored-by: Fabian Egli --- nf_core/modules/module_test.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 6507c58219..65006a6ced 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -117,9 +117,16 @@ def _check_profile(self): """Check if profile is available""" profile = os.environ.get("PROFILE") try: - profile_check = subprocess.check_output( - shlex.split(f"{profile} --help"), stderr=subprocess.STDOUT, shell=True - ) + + # Make sure the profile read from the environment is a valid Nextflow profile. + valid_nextflow_profiles = ["docker", "singularity", "podman", "shifter", "charliecloud", "conda"] + if profile in valid_nextflow_profiles: + profile_check = subprocess.check_output([profile, "--help"], stderr=subprocess.STDOUT) + else: + raise UserWarning( + f"The profile '{profile}' set in the shell environment is not a valid.\n" + f"Valid Nextflow profiles are {valid_nextflow_profiles}." + ) except subprocess.CalledProcessError as e: raise UserWarning(f"Error with profile {profile} (exit code {e.returncode})\n[red]{e.output.decode()}") From 69247e7b72b8682bc6ca030b5393ee19b0ef7140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 2 May 2022 13:05:39 +0200 Subject: [PATCH 084/136] fix linting --- nf_core/modules/module_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 65006a6ced..781fb3b2f1 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -87,7 +87,9 @@ def _check_inputs(self): # First, sanity check that the module directory exists if not module_dir.is_dir(): - raise UserWarning(f"Cannot find directory '{module_dir}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?") + raise UserWarning( + f"Cannot find directory '{module_dir}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?" + ) def _set_profile(self): """Set $PROFILE env variable. @@ -117,7 +119,6 @@ def _check_profile(self): """Check if profile is available""" profile = os.environ.get("PROFILE") try: - # Make sure the profile read from the environment is a valid Nextflow profile. valid_nextflow_profiles = ["docker", "singularity", "podman", "shifter", "charliecloud", "conda"] if profile in valid_nextflow_profiles: From 1d8e56788e8d791d22dca847a72270084980cab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 2 May 2022 13:14:23 +0200 Subject: [PATCH 085/136] avoid using shell=True when checking profile --- nf_core/modules/module_test.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 781fb3b2f1..75c7cbb333 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -12,6 +12,7 @@ import subprocess import shlex from pathlib import Path +from shutil import which import nf_core.utils from .modules_repo import ModulesRepo @@ -118,18 +119,16 @@ def _set_profile(self): def _check_profile(self): """Check if profile is available""" profile = os.environ.get("PROFILE") - try: - # Make sure the profile read from the environment is a valid Nextflow profile. - valid_nextflow_profiles = ["docker", "singularity", "podman", "shifter", "charliecloud", "conda"] - if profile in valid_nextflow_profiles: - profile_check = subprocess.check_output([profile, "--help"], stderr=subprocess.STDOUT) - else: - raise UserWarning( - f"The profile '{profile}' set in the shell environment is not a valid.\n" - f"Valid Nextflow profiles are {valid_nextflow_profiles}." - ) - except subprocess.CalledProcessError as e: - raise UserWarning(f"Error with profile {profile} (exit code {e.returncode})\n[red]{e.output.decode()}") + # Make sure the profile read from the environment is a valid Nextflow profile. + valid_nextflow_profiles = ["docker", "singularity", "podman", "shifter", "charliecloud", "conda"] + if profile in valid_nextflow_profiles: + if not which(profile): + raise UserWarning(f"The PROFILE '{profile}' set in the shell environment is not available.") + else: + raise UserWarning( + f"The PROFILE '{profile}' set in the shell environment is not valid.\n" + f"Valid Nextflow profiles are {valid_nextflow_profiles}." + ) def _run_pytests(self): """Given a module name, run tests.""" From 8e33499cfbf4ca88c24bdf1df67ad08a8777d968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 2 May 2022 13:54:45 +0200 Subject: [PATCH 086/136] Apply suggestions from code review Co-authored-by: Phil Ewels --- nf_core/modules/module_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 75c7cbb333..8ca12df6d5 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -120,21 +120,21 @@ def _check_profile(self): """Check if profile is available""" profile = os.environ.get("PROFILE") # Make sure the profile read from the environment is a valid Nextflow profile. - valid_nextflow_profiles = ["docker", "singularity", "podman", "shifter", "charliecloud", "conda"] + valid_nextflow_profiles = ["docker", "singularity", "conda"] if profile in valid_nextflow_profiles: if not which(profile): - raise UserWarning(f"The PROFILE '{profile}' set in the shell environment is not available.") + raise UserWarning(f"Command '{profile}' not found - is it installed?") else: raise UserWarning( f"The PROFILE '{profile}' set in the shell environment is not valid.\n" - f"Valid Nextflow profiles are {valid_nextflow_profiles}." + f"Valid Nextflow profiles are '{', '.join(valid_nextflow_profiles)}'." ) def _run_pytests(self): """Given a module name, run tests.""" # Print nice divider line console = rich.console.Console() - console.print("[black]" + "─" * console.width) + console.rule(self.module_name, style="black") # Set pytest arguments command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] From aca376dcf7fc30e1e83c540e5fc04cbee1b92bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 2 May 2022 13:59:54 +0200 Subject: [PATCH 087/136] remove unnecessary libraryes --- nf_core/modules/module_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 8ca12df6d5..35658dd26b 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -9,8 +9,6 @@ import pytest import sys import rich -import subprocess -import shlex from pathlib import Path from shutil import which From 8ade0dfe9be6dfddc5555a54479b31688f3b0c4a Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Wed, 4 May 2022 15:40:42 +0200 Subject: [PATCH 088/136] replace a weird with a normal dash --- nf_core/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index ede47d0f22..b0c3e565f7 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -420,7 +420,7 @@ def prompt_param(self, param_id, param_obj, is_required, answers): # If required and got an empty reponse, ask again while type(answer[param_id]) is str and answer[param_id].strip() == "" and is_required: - log.error("'–-{}' is required".format(param_id)) + log.error("'--{}' is required".format(param_id)) answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) # Ignore if empty From b6bc3f88a41636a5f9280cc1444006137496df0d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 5 May 2022 10:01:39 +0200 Subject: [PATCH 089/136] Handle missing key, log, call after web builder --- nf_core/schema.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 788adf73e2..fcda38b200 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -536,6 +536,7 @@ def build_schema(self, pipeline_dir, no_prompts, web_only, url): self.get_wf_params() self.make_skeleton_schema() self.remove_schema_notfound_configs() + self.remove_schema_empty_definitions() self.add_schema_found_configs() try: self.validate_schema() @@ -555,6 +556,7 @@ def build_schema(self, pipeline_dir, no_prompts, web_only, url): if not self.web_only: self.get_wf_params() self.remove_schema_notfound_configs() + self.remove_schema_empty_definitions() self.add_schema_found_configs() self.save_schema() @@ -624,10 +626,13 @@ def remove_schema_empty_definitions(self): ## Identify and remove empty definitions from the schema empty_definitions = [] for d_key, d_schema in list(schema_no_empty_definitions.get("definitions", {}).items()): - if not d_schema["properties"]: + if not d_schema.get("properties"): del schema_no_empty_definitions["definitions"][d_key] empty_definitions.append(d_key) + if len(empty_definitions): + log.warning(f"Removing empty group: '{', '.join(empty_definitions)}'") + # Remove "allOf" group with empty definitions from the schema for d_key in empty_definitions: allOf = {"$ref": "#/definitions/{}".format(d_key)} @@ -648,7 +653,6 @@ def remove_schema_notfound_configs(self): cleaned_schema, p_removed = self.remove_schema_notfound_configs_single_schema(definition) self.schema["definitions"][d_key] = cleaned_schema params_removed.extend(p_removed) - self.remove_schema_empty_definitions() return params_removed @@ -797,6 +801,7 @@ def get_web_builder_response(self): log.info("Found saved status from nf-core schema builder") try: self.schema = web_response["schema"] + self.remove_schema_empty_definitions() self.validate_schema() except AssertionError as e: raise AssertionError("Response from schema builder did not pass validation:\n {}".format(e)) From 11c22c03e55134b9f8e70e80453d01050fd91c96 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 5 May 2022 10:21:33 +0200 Subject: [PATCH 090/136] Deep copy of schema not needed --- nf_core/schema.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index fcda38b200..05706d0dd1 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -620,26 +620,19 @@ def remove_schema_empty_definitions(self): Go through top-level schema remove definitions that don't have any property attributes """ - # Make copy of schema - schema_no_empty_definitions = copy.deepcopy(self.schema) - - ## Identify and remove empty definitions from the schema + # Identify and remove empty definitions from the schema empty_definitions = [] - for d_key, d_schema in list(schema_no_empty_definitions.get("definitions", {}).items()): + for d_key, d_schema in list(self.schema.get("definitions", {}).items()): if not d_schema.get("properties"): - del schema_no_empty_definitions["definitions"][d_key] + del self.schema["definitions"][d_key] empty_definitions.append(d_key) - - if len(empty_definitions): - log.warning(f"Removing empty group: '{', '.join(empty_definitions)}'") + log.warning(f"Removing empty group: '{d_key}'") # Remove "allOf" group with empty definitions from the schema for d_key in empty_definitions: allOf = {"$ref": "#/definitions/{}".format(d_key)} - if allOf in schema_no_empty_definitions["allOf"]: - schema_no_empty_definitions["allOf"].remove(allOf) - - self.schema = schema_no_empty_definitions + if allOf in self.schema.get("allOf", []): + self.schema["allOf"].remove(allOf) def remove_schema_notfound_configs(self): """ From 07a6a5f7a12357b2f0b4f8cfca6670db500c6c16 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 5 May 2022 10:22:13 +0200 Subject: [PATCH 091/136] Format strings --- nf_core/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 05706d0dd1..20be305f99 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -630,7 +630,7 @@ def remove_schema_empty_definitions(self): # Remove "allOf" group with empty definitions from the schema for d_key in empty_definitions: - allOf = {"$ref": "#/definitions/{}".format(d_key)} + allOf = {"$ref": f"#/definitions/{d_key}"} if allOf in self.schema.get("allOf", []): self.schema["allOf"].remove(allOf) From 092aff3dbbd57edfc2f3c4d6f0889cbee02e052d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 6 May 2022 11:05:33 +0200 Subject: [PATCH 092/136] add documentation for modules mulled command --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index be0054533c..e9d777f0b5 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ A python package with helper tools for the nf-core community. - [`modules create-test-yml` - Create the `test.yml` file for a module](#create-a-module-test-config-file) - [`modules lint` - Check a module against nf-core guidelines](#check-a-module-against-nf-core-guidelines) - [`modules bump-versions` - Bump software versions of modules](#bump-bioconda-and-container-versions-of-modules-in) + - [`modules mulled` - Generate the name for a multi-tool container image](#generate the name for a multi-tool container image) - [Citation](#citation) @@ -1339,6 +1340,25 @@ bump-versions: star/align: "2.6.1d" ``` +### Generate the name for a multi-tool container image + +When you want to use an image of a multi-tool container, and you know the specific dependencies and their versions of that container, you can use the `nf-core modules mulled` helper tool. This tool generates the name of a BioContainers mulled image. + +```console +$ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 + + ,--./,-. + ___ __ __ __ ___ /,-._.--~\ + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' + + nf-core/tools version 2.4 + + +mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 +``` + ## Citation If you use `nf-core tools` in your work, please cite the `nf-core` publication as follows: From 9f9d717731ed02630ca686a5c4fe2514f4a9aa74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 6 May 2022 12:44:06 +0200 Subject: [PATCH 093/136] add documentation for modules test command --- README.md | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e9d777f0b5..04ed950359 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,9 @@ A python package with helper tools for the nf-core community. - [`modules create` - Create a module from the template](#create-a-new-module) - [`modules create-test-yml` - Create the `test.yml` file for a module](#create-a-module-test-config-file) - [`modules lint` - Check a module against nf-core guidelines](#check-a-module-against-nf-core-guidelines) + - [`modules test` - Run the tests for a module](#run-the-tests-for-a-module-using-pytest) - [`modules bump-versions` - Bump software versions of modules](#bump-bioconda-and-container-versions-of-modules-in) - - [`modules mulled` - Generate the name for a multi-tool container image](#generate the name for a multi-tool container image) + - [`modules mulled` - Generate the name for a multi-tool container image](#generate-the-name-for-a-multi-tool-container-image) - [Citation](#citation) @@ -1297,6 +1298,57 @@ INFO Linting module: star/align ╰──────────────────────╯ ``` +### Run the tests for a module using pytest + +To run unit tests of a module that you have installed or the test created by the command [`nf-core mdoules create-test-yml`](#create-a-module-test-config-file), you can use `nf-core modules test` command. This command runs the tests specified in `modules/tests/software///test.yml` file using [pytest](https://pytest-workflow.readthedocs.io/en/stable/). + +You can specify the module name in the form TOOL/SUBTOOL in command line or provide it later by prompts. + +```console +$ nf-core modules test fastqc + ,--./,-. + ___ __ __ __ ___ /,-._.--~\ + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' + + nf-core/tools version 2.4 + +? Choose software profile Docker +INFO Setting environment variable '$PROFILE' to 'docker' +INFO Running pytest for module 'fastqc' + +=============================================================== test session starts ================================================================ +platform darwin -- Python 3.9.12, pytest-7.1.2, pluggy-1.0.0 +rootdir: ~/modules, configfile: pytest.ini +plugins: workflow-1.6.0 +collecting ... +collected 761 items + +fastqc single-end: + command: nextflow run ./tests/modules/fastqc/ -entry test_fastqc_single_end -c ./tests/config/nextflow.config -c ./tests/modules/fastqc/nextflow.config -c ./tests/modules/fastqc/nextflow.config + directory: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_single-end + stdout: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_single-end/log.out + stderr: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_single-end/log.err +'fastqc single-end' done. + +fastqc paired-end: + command: nextflow run ./tests/modules/fastqc/ -entry test_fastqc_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/fastqc/nextflow.config -c ./tests/modules/fastqc/nextflow.config + directory: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_paired-end + stdout: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_paired-end/log.out + stderr: /var/folders/lt/b3cs9y610fg_13q14dckwcvm0000gn/T/pytest_workflow_ahvulf1v/fastqc_paired-end/log.err +'fastqc paired-end' done. + +tests/test_versions_yml.py sssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss [ 17%] +ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss [ 38%] +ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss [ 59%] +sssssssssssssssssssssssssssssssssssssssssssssssssssssssssss..ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss ssss [ 80%] +ssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss [ 98%] +tests/modules/fastqc/test.yml ........ +Keeping temporary directories and logs. Use '--kwd' or '--keep-workflow-wd' to disable this behaviour. +=================================================== 10 passed, 751 skipped, 479 warnings in 50.76s =================================================== +``` + ### Bump bioconda and container versions of modules in If you are contributing to the `nf-core/modules` repository and want to bump bioconda and container versions of certain modules, you can use the `nf-core modules bump-versions` helper tool. This will bump the bioconda version of a single or all modules to the latest version and also fetch the correct Docker and Singularity container tags. From a9ead4af279cee21625335e07f3a2755be5c535a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 6 May 2022 13:48:55 +0200 Subject: [PATCH 094/136] Add biocontainers reference Co-authored-by: Moritz E. Beber --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 04ed950359..949a11b424 100644 --- a/README.md +++ b/README.md @@ -1394,7 +1394,7 @@ bump-versions: ### Generate the name for a multi-tool container image -When you want to use an image of a multi-tool container, and you know the specific dependencies and their versions of that container, you can use the `nf-core modules mulled` helper tool. This tool generates the name of a BioContainers mulled image. +When you want to use an image of a multi-tool container and you know the specific dependencies and their versions of that container, for example, by looking them up in the [BioContainers hash.tsv](https://github.com/BioContainers/multi-package-containers/blob/master/combinations/hash.tsv), you can use the `nf-core modules mulled` helper tool. This tool generates the name of a BioContainers mulled image. ```console $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 From 4fe28fcb527f14faa688ca3088f8f7380e775af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 9 May 2022 10:43:52 +0200 Subject: [PATCH 095/136] replace complex split commands by regex --- nf_core/modules/lint/main_nf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index abc9e48dc1..f7e2ab998a 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -196,7 +196,7 @@ def check_process_section(self, lines): correct_process_labels = ["process_low", "process_medium", "process_high", "process_long"] process_label = [l for l in lines if "label" in l] if len(process_label) > 0: - process_label = process_label[0].split()[1].strip().strip("'").strip('"') + process_label = re.search("process_[A-Za-z]*", process_label[0]).group(0) if not process_label in correct_process_labels: self.warned.append( ( @@ -220,14 +220,14 @@ def check_process_section(self, lines): lspl = l.lstrip("https://").split(":") if len(lspl) == 2: # e.g. 'https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' : - singularity_tag = "_".join(lspl[0].split("/")[-1].strip().rstrip(".img").split("_")[1:]) + singularity_tag = re.search("biocontainers_(.*).img", lspl[0]).group(1) else: # e.g. 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : singularity_tag = lspl[-2].strip() if l.startswith("biocontainers/") or l.startswith("quay.io/"): # e.g. 'quay.io/biocontainers/krona:2.7.1--pl526_5' }" # e.g. 'biocontainers/biocontainers:v1.2.0_cv1' }" - docker_tag = l.split(":")[-1].strip("}").strip() + docker_tag = re.search("\:(.*) \}", l).group(1) # Check that all bioconda packages have build numbers # Also check for newer versions From 754002aad0154dde576270a5c783f7afa5821653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 9 May 2022 15:18:13 +0200 Subject: [PATCH 096/136] don't try to obtain version if the container is not provided --- nf_core/modules/lint/main_nf.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index f7e2ab998a..575c5b0271 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -220,14 +220,18 @@ def check_process_section(self, lines): lspl = l.lstrip("https://").split(":") if len(lspl) == 2: # e.g. 'https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' : - singularity_tag = re.search("biocontainers_(.*).img", lspl[0]).group(1) + if "YOUR-TOOL-HERE" not in lspl[0]: + # Get singularity container version when a container is provided + singularity_tag = re.search("biocontainers_(.*).img", lspl[0]).group(1) else: # e.g. 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : singularity_tag = lspl[-2].strip() if l.startswith("biocontainers/") or l.startswith("quay.io/"): # e.g. 'quay.io/biocontainers/krona:2.7.1--pl526_5' }" # e.g. 'biocontainers/biocontainers:v1.2.0_cv1' }" - docker_tag = re.search("\:(.*) \}", l).group(1) + if "YOUR-TOOL-HERE" not in l: + # Get docker container version when a container is provided + docker_tag = re.search("\:(.*) \}", l).group(1) # Check that all bioconda packages have build numbers # Also check for newer versions From ded22cb06703170051b95071949675f77b745ac1 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 10 May 2022 10:04:02 +0200 Subject: [PATCH 097/136] Better exception handling / logging for gh auth failure --- nf_core/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 38e8c96264..e8ed422dee 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -438,8 +438,9 @@ def __call__(self, r): ) self.auth_mode = f"gh CLI config: {gh_cli_config['github.com']['user']}" except Exception as e: - output = rich.markup.escape(e.output.decode()) - log.debug(f"Couldn't auto-auth with GitHub CLI auth: [red]{output}") + ex_type, ex_value, ex_traceback = sys.exc_info() + output = rich.markup.escape(f"{ex_type.__name__}: {ex_value}") + log.debug(f"Couldn't auto-auth with GitHub CLI auth from '{gh_cli_config_fn}': [red]{output}") # Default auth if we have a GitHub Token (eg. GitHub Actions CI) if os.environ.get("GITHUB_TOKEN") is not None and self.auth is None: From 58d012b68c1a06d83e7fc0cf7a0377dff31f24ed Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 10 May 2022 10:13:14 +0200 Subject: [PATCH 098/136] Use XDG_CONFIG_HOME or ~/.config/nf-core/ for the requests cache dir. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suggested by @Emiller88 👍🏻 --- nf_core/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index e8ed422dee..d56d43ca64 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -49,6 +49,11 @@ ] ) +NFCORE_CONFIG_DIR = os.path.join( + os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME"), ".config")), + "nf-core", +) + def check_if_outdated(current_version=None, remote_version=None, source_url="https://nf-co.re/tools_version"): """ @@ -296,13 +301,13 @@ def setup_requests_cachedir(): """Sets up local caching for faster remote HTTP requests. Caching directory will be set up in the user's home directory under - a .nfcore_cache subdir. + a .config/nf-core/cache_* subdir. Uses requests_cache monkey patching. Also returns the config dict so that we can use the same setup with a Session. """ pyversion = ".".join(str(v) for v in sys.version_info[0:3]) - cachedir = os.path.join(os.getenv("HOME"), os.path.join(".nfcore", "cache_" + pyversion)) + cachedir = os.path.join(NFCORE_CONFIG_DIR, f"cache_{pyversion}") config = { "cache_name": os.path.join(cachedir, "github_info"), From a29fe6134bc70244349293c3da744123a7aa7168 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 10 May 2022 10:17:42 +0200 Subject: [PATCH 099/136] Changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53ae872c3f..95bba01f1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ - Linting: Don't allow a `.nf-core.yaml` file, should be `.yml` ([#1515](https://github.com/nf-core/tools/pull/1515)). - Remove empty JSON schema definition groups to avoid usage errors ([#1419](https://github.com/nf-core/tools/issues/1419)) - Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520)) +- Use [`$XDG_CONFIG_HOME`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) or `~/.config/nf-core` instead of `~/.nfcore` for API cache (the latter can be safely deleted) +- Consolidate GitHub API calls into a shared function that uses authentication from the [`gh` GitHub cli tool](https://cli.github.com/) or `GITHUB_AUTH_TOKEN` to avoid rate limiting ([#1499](https://github.com/nf-core/tools/pull/1499)) ### Modules From cb8d7d80cde8dcd70dbc5d52f9c6c6ff75ccc8fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 10 May 2022 14:32:27 +0200 Subject: [PATCH 100/136] decode bytes-like object for logging --- nf_core/modules/test_yml_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index b10c64204c..5ef6dd19a0 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -340,7 +340,7 @@ def run_tests_workflow(self, command): raise UserWarning(f"Error running test workflow: {e}") else: log.info("Test workflow finished!") - log.debug(rich.markup.escape(nfconfig_raw)) + log.debug(rich.markup.escape(nfconfig_raw.decode("utf-8"))) return tmp_dir, tmp_dir_repeat From 0f73634b749755d3086b25e7a4b55a0d58cc90dc Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 11 May 2022 15:03:55 +0200 Subject: [PATCH 101/136] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- tests/test_sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 42cfc87918..66915009b8 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -228,7 +228,7 @@ def json(self): @mock.patch("nf_core.utils.gh_api.get", side_effect=mocked_requests_get) @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) - def test_make_pull_request_success(self, mock_get, mock_post): + def test_make_pull_request_success(self, mock_post, mock_get): """Try making a PR - successful response""" psync = nf_core.sync.PipelineSync(self.pipeline_dir) psync.gh_api.get = mock_get @@ -241,7 +241,7 @@ def test_make_pull_request_success(self, mock_get, mock_post): @mock.patch("nf_core.utils.gh_api.get", side_effect=mocked_requests_get) @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) - def test_make_pull_request_bad_response(self, mock_get, mock_post): + def test_make_pull_request_bad_response(self, mock_post, mock_get): """Try making a PR and getting a 404 error""" psync = nf_core.sync.PipelineSync(self.pipeline_dir) psync.gh_api.get = mock_get From 40093e03e7470f438d71bc21805750ff22b170f1 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 11 May 2022 15:04:43 +0200 Subject: [PATCH 102/136] Update nf_core/sync.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 5960048ca1..e9f5ff391c 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -79,7 +79,7 @@ def __init__( self.gh_api = nf_core.utils.gh_api if self.gh_username and "GITHUB_AUTH_TOKEN" in os.environ: self.gh_api.auth = requests.auth.HTTPBasicAuth(self.gh_username, os.environ["GITHUB_AUTH_TOKEN"]) - self.gh_api.return_ok = [201] + self.gh_api.return_ok = [200, 201] self.gh_api.lazy_init() def sync(self): From 1aa6b66bdd50d5034e84c4905f64b499c9c954ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 12 May 2022 10:53:35 +0200 Subject: [PATCH 103/136] modify regex based on PR review --- nf_core/modules/lint/main_nf.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 575c5b0271..0cb90b3204 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -196,7 +196,7 @@ def check_process_section(self, lines): correct_process_labels = ["process_low", "process_medium", "process_high", "process_long"] process_label = [l for l in lines if "label" in l] if len(process_label) > 0: - process_label = re.search("process_[A-Za-z]*", process_label[0]).group(0) + process_label = re.search("process_[A-Za-z]+", process_label[0]).group(0) if not process_label in correct_process_labels: self.warned.append( ( @@ -212,26 +212,18 @@ def check_process_section(self, lines): for l in lines: l = l.strip() - l = l.replace('"', "") - l = l.replace("'", "") + l = l.lstrip('"') + l = l.lstrip("'") if re.search("bioconda::", l): bioconda_packages = [b for b in l.split() if "bioconda::" in b] if l.startswith("https://containers") or l.startswith("https://depot"): - lspl = l.lstrip("https://").split(":") - if len(lspl) == 2: - # e.g. 'https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' : - if "YOUR-TOOL-HERE" not in lspl[0]: - # Get singularity container version when a container is provided - singularity_tag = re.search("biocontainers_(.*).img", lspl[0]).group(1) - else: - # e.g. 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : - singularity_tag = lspl[-2].strip() + # e.g. 'https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' : + # e.g. 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : + singularity_tag = re.search("(?:\/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_\.]+)(?:\.img)?['\"]", l).group(1) if l.startswith("biocontainers/") or l.startswith("quay.io/"): # e.g. 'quay.io/biocontainers/krona:2.7.1--pl526_5' }" # e.g. 'biocontainers/biocontainers:v1.2.0_cv1' }" - if "YOUR-TOOL-HERE" not in l: - # Get docker container version when a container is provided - docker_tag = re.search("\:(.*) \}", l).group(1) + docker_tag = re.search("(?:[\/])?(?::)?([A-Za-z\d\-_\.]+)['\"]", l).group(1) # Check that all bioconda packages have build numbers # Also check for newer versions From 6164286395e95dc1a61d128f39ff357f935fc1ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 12 May 2022 12:32:43 +0200 Subject: [PATCH 104/136] apply changes from PR review --- nf_core/modules/lint/main_nf.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 0cb90b3204..288e3466a9 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -211,18 +211,17 @@ def check_process_section(self, lines): self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf)) for l in lines: - l = l.strip() - l = l.lstrip('"') - l = l.lstrip("'") if re.search("bioconda::", l): bioconda_packages = [b for b in l.split() if "bioconda::" in b] + l = l.strip() + l = l.strip(" '\"") if l.startswith("https://containers") or l.startswith("https://depot"): - # e.g. 'https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' : - # e.g. 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : + # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 + # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 singularity_tag = re.search("(?:\/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_\.]+)(?:\.img)?['\"]", l).group(1) if l.startswith("biocontainers/") or l.startswith("quay.io/"): - # e.g. 'quay.io/biocontainers/krona:2.7.1--pl526_5' }" - # e.g. 'biocontainers/biocontainers:v1.2.0_cv1' }" + # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 + # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 docker_tag = re.search("(?:[\/])?(?::)?([A-Za-z\d\-_\.]+)['\"]", l).group(1) # Check that all bioconda packages have build numbers From a3535f59bfd2d9b990d8626c0124354deff955c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 12 May 2022 13:02:34 +0200 Subject: [PATCH 105/136] remove unnecessary strip Co-authored-by: Phil Ewels --- nf_core/modules/lint/main_nf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 288e3466a9..0166fba72d 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -213,7 +213,6 @@ def check_process_section(self, lines): for l in lines: if re.search("bioconda::", l): bioconda_packages = [b for b in l.split() if "bioconda::" in b] - l = l.strip() l = l.strip(" '\"") if l.startswith("https://containers") or l.startswith("https://depot"): # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 From a1156bd29dbc2f45b201282bfd605cc66ce4ff09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 12 May 2022 13:27:02 +0200 Subject: [PATCH 106/136] only decode when nfconfig_raw is not a string Co-authored-by: Phil Ewels --- nf_core/modules/test_yml_builder.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index 5ef6dd19a0..075e6ee302 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -340,7 +340,10 @@ def run_tests_workflow(self, command): raise UserWarning(f"Error running test workflow: {e}") else: log.info("Test workflow finished!") - log.debug(rich.markup.escape(nfconfig_raw.decode("utf-8"))) + try: + log.debug(rich.markup.escape(nfconfig_raw)) + except TypeError: + log.debug(rich.markup.escape(nfconfig_raw.decode("utf-8"))) return tmp_dir, tmp_dir_repeat From ce08fb22bf89e60dc4fb8dd12b43e7615074f9a3 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 12 May 2022 13:32:53 +0200 Subject: [PATCH 107/136] fix linting --- nf_core/modules/test_yml_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index 075e6ee302..fcdde14677 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -341,9 +341,9 @@ def run_tests_workflow(self, command): else: log.info("Test workflow finished!") try: - log.debug(rich.markup.escape(nfconfig_raw)) + log.debug(rich.markup.escape(nfconfig_raw)) except TypeError: - log.debug(rich.markup.escape(nfconfig_raw.decode("utf-8"))) + log.debug(rich.markup.escape(nfconfig_raw.decode("utf-8"))) return tmp_dir, tmp_dir_repeat From 2eea6d6f6536bfb98d414f4a90ebf23c2d9318b1 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 12 May 2022 14:49:08 +0200 Subject: [PATCH 108/136] Make module template and test_yml_builder default commands match. See nf-core/tools#1562 --- nf_core/module-template/tests/test.yml | 2 +- nf_core/modules/test_yml_builder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index b0eb645846..c1782107af 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -1,7 +1,7 @@ ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} - name: "{{ tool }}{{ ' '+subtool if subtool else '' }}" - command: nextflow run ./tests/modules/{{ tool_dir }} -entry test_{{ tool_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ tool_dir }}/nextflow.config + command: nextflow run ./tests/modules/{{ tool_dir }} -entry test_{{ tool_name_underscore }} -c ./tests/config/nextflow.config tags: - "{{ tool }}" # {%- if subtool %} diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index fcdde14677..98dcf3a695 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -165,7 +165,7 @@ def build_single_test(self, entry_point): while ep_test["command"] == "": default_val = ( - f"nextflow run tests/modules/{self.module_name} -entry {entry_point} -c tests/config/nextflow.config" + f"nextflow run ./tests/modules/{self.module_name} -entry {entry_point} -c ./tests/config/nextflow.config" ) if self.no_prompts: ep_test["command"] = default_val From 74441fdfbd7f2319252876cfc456be022522208d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 12 May 2022 14:52:33 +0200 Subject: [PATCH 109/136] Changelog --- CHANGELOG.md | 1 + nf_core/modules/test_yml_builder.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95bba01f1b..6ee6c89dd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Removed retry strategy for AWS tests CI, as Nextflow now handles spot instance retries itself - Add `.prettierignore` file to stop Prettier linting tests from running over test files - Add actions workflow to respond to `@nf-core-bot fix linting` comments on pipeline PRs +- Made module template test command match the default used in `nf-core modules create-test-yml` ([#1562](https://github.com/nf-core/tools/issues/1562)) ### General diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index 98dcf3a695..f96b3d6ca0 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -164,9 +164,7 @@ def build_single_test(self, entry_point): ep_test["name"] = rich.prompt.Prompt.ask("[violet]Test name", default=default_val).strip() while ep_test["command"] == "": - default_val = ( - f"nextflow run ./tests/modules/{self.module_name} -entry {entry_point} -c ./tests/config/nextflow.config" - ) + default_val = f"nextflow run ./tests/modules/{self.module_name} -entry {entry_point} -c ./tests/config/nextflow.config" if self.no_prompts: ep_test["command"] = default_val else: From 479a71d4550b1ef2bf2c7d80d96b7f6cc91dcb83 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 12 May 2022 15:02:54 +0200 Subject: [PATCH 110/136] Module linting: recognise 'shell' block. Means that we properly exit out of the 'when' block when 'shell' is used instead of 'script'. Fixes nf-core/tools#1557 --- CHANGELOG.md | 1 + nf_core/modules/lint/main_nf.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95bba01f1b..c3de92be68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - Escaped test run output before logging it, to avoid a rich ` MarkupError` - Add a new command `nf-core modules mulled` which can generate the name for a multi-tool container image. - Add a new command `nf-core modules test` which runs pytests locally. +- Linting now recognised `shell` blocks to avoid error `when: condition has too many lines` ([#1557](https://github.com/nf-core/tools/issues/1557)) ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 0166fba72d..6504b84566 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -72,6 +72,9 @@ def main_nf(module_lint_object, module): if re.search("script\s*:", l) and state in ["input", "output", "when", "process"]: state = "script" continue + if re.search("shell\s*:", l) and state in ["input", "output", "when", "process"]: + state = "shell" + continue # Perform state-specific linting checks if state == "process" and not _is_empty(module, l): From d61a51d13dd7ddbcba326a1a1d5efccd09399914 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 12 May 2022 15:32:18 +0200 Subject: [PATCH 111/136] Put the -c back for the test pipeline nextflow.config Avoid needing to change loads of module files due to the template update. See https://github.com/nf-core/tools/issues/1562#issuecomment-1124994152 --- nf_core/module-template/tests/test.yml | 2 +- nf_core/modules/test_yml_builder.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index c1782107af..b0eb645846 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -1,7 +1,7 @@ ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} - name: "{{ tool }}{{ ' '+subtool if subtool else '' }}" - command: nextflow run ./tests/modules/{{ tool_dir }} -entry test_{{ tool_name_underscore }} -c ./tests/config/nextflow.config + command: nextflow run ./tests/modules/{{ tool_dir }} -entry test_{{ tool_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ tool_dir }}/nextflow.config tags: - "{{ tool }}" # {%- if subtool %} diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index f96b3d6ca0..4b85371593 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -164,7 +164,9 @@ def build_single_test(self, entry_point): ep_test["name"] = rich.prompt.Prompt.ask("[violet]Test name", default=default_val).strip() while ep_test["command"] == "": - default_val = f"nextflow run ./tests/modules/{self.module_name} -entry {entry_point} -c ./tests/config/nextflow.config" + # Don't think we need the last `-c` flag, but keeping to avoid having to update 100s modules. + # See https://github.com/nf-core/tools/issues/1562 + default_val = f"nextflow run ./tests/modules/{self.module_name} -entry {entry_point} -c ./tests/config/nextflow.config -c ./tests/modules/{self.module_name}/nextflow.config" if self.no_prompts: ep_test["command"] = default_val else: From 200b923267364def54d5c94ccf4a8b0e280bfaa0 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 12 May 2022 15:34:45 +0200 Subject: [PATCH 112/136] follow links when obtaining test output files --- nf_core/modules/test_yml_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index fcdde14677..1739d52cb6 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -225,7 +225,7 @@ def _md5(self, fname): def create_test_file_dict(self, results_dir, is_repeat=False): """Walk through directory and collect md5 sums""" test_files = [] - for root, dir, files in os.walk(results_dir): + for root, dir, files in os.walk(results_dir, followlinks=True): for filename in files: # Check that the file is not versions.yml if filename == "versions.yml": From 54c7b9434bd428379f772007d846e66fe06bb527 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 12 May 2022 15:40:41 +0200 Subject: [PATCH 113/136] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95bba01f1b..4baf9257e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - Escaped test run output before logging it, to avoid a rich ` MarkupError` - Add a new command `nf-core modules mulled` which can generate the name for a multi-tool container image. - Add a new command `nf-core modules test` which runs pytests locally. +- Allow follow links when generating `test.yml` file with `nf-core modules create-test-yml` ([1570](https://github.com/nf-core/tools/pull/1570)) ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] From 266432799c1f83e19b38b0fa4fb513e7d3cb13ba Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 12 May 2022 16:09:07 +0200 Subject: [PATCH 114/136] Module linting: refactor parsing of input channel names. No more split, regexes FTW! Fixes nf-core/tools#1542 --- CHANGELOG.md | 1 + nf_core/modules/lint/main_nf.py | 51 ++++++++++++++++++++++----------- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdcc7ccea1..3c67516d2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ - Add a new command `nf-core modules mulled` which can generate the name for a multi-tool container image. - Add a new command `nf-core modules test` which runs pytests locally. - Linting now recognised `shell` blocks to avoid error `when: condition has too many lines` ([#1557](https://github.com/nf-core/tools/issues/1557)) +- Linting: fix error when using comments after `input` tuple lines ([#1542](https://github.com/nf-core/tools/issues/1542)) ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 6504b84566..4271453687 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -80,7 +80,7 @@ def main_nf(module_lint_object, module): if state == "process" and not _is_empty(module, l): process_lines.append(l) if state == "input" and not _is_empty(module, l): - inputs += _parse_input(module, l) + inputs.extend(_parse_input(module, l)) if state == "output" and not _is_empty(module, l): outputs += _parse_output(module, l) outputs = list(set(outputs)) # remove duplicate 'meta's @@ -242,7 +242,9 @@ def check_process_section(self, lines): else: # Check that required version is available at all if bioconda_version not in response.get("versions"): - self.failed.append(("bioconda_version", "Conda package had unknown version: `{}`", self.main_nf)) + self.failed.append( + ("bioconda_version", f"Conda package had unknown version: `{bioconda_version}`", self.main_nf) + ) continue # No need to test for latest version, continue linting # Check version is latest available last_ver = response.get("latest_version") @@ -260,26 +262,41 @@ def check_process_section(self, lines): return False -def _parse_input(self, line): - input = [] +def _parse_input(self, line_raw): + """ + Return list of input channel names from an input line. + + If more than one elements in channel should work with both of: + tuple val(meta), path(reads) + tuple val(meta), path(reads, stageAs: "input*/*") + + If using a tuple, channel names must be in (parentheses) + """ + inputs = [] + # Remove comments and trailing whitespace + line, *_ = line_raw.partition("//") line = line.strip() + # Tuples with multiple elements if "tuple" in line: - # If more than one elements in channel should work with both of: - # e.g. tuple val(meta), path(reads) - # e.g. tuple val(meta), path(reads, stageAs: "input*/*") - line = line.replace("tuple", "") - line = line.replace(" ", "") - for idx, elem in enumerate(line.split(")")): - if elem: - elem = elem.split("(")[1] - elem = elem.split(",")[0].strip() - input.append(elem) + matches = re.findall("\((\w+)\)", line) + if matches: + inputs.extend(matches) + else: + self.failed.append( + ( + "main_nf_input_tuple", + f"Found tuple but no channel names: `{line}`", + self.main_nf, + ) + ) + # Single element inputs else: if "(" in line: - input.append(line.split("(")[1].replace(")", "")) + match = re.search("\((\w+)\)", line) + inputs.append(match.group(1)) else: - input.append(line.split()[1]) - return input + inputs.append(line.split()[1]) + return inputs def _parse_output(self, line): From 6b9315b25670e90699e7dbc79d513058304a5bd0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 12 May 2022 16:18:13 +0200 Subject: [PATCH 115/136] Mulled: handle invalid versions with a nice error message instead of a traceback --- nf_core/__main__.py | 10 +++++++--- nf_core/modules/mulled.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 45fcff9a60..f4e615635c 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -694,9 +694,13 @@ def mulled(specifications, build_number): """ from nf_core.modules.mulled import MulledImageNameGenerator - image_name = MulledImageNameGenerator.generate_image_name( - MulledImageNameGenerator.parse_targets(specifications), build_number=build_number - ) + try: + image_name = MulledImageNameGenerator.generate_image_name( + MulledImageNameGenerator.parse_targets(specifications), build_number=build_number + ) + except ValueError as e: + log.error(e) + sys.exit(1) print(image_name) if not MulledImageNameGenerator.image_exists(image_name): log.error( diff --git a/nf_core/modules/mulled.py b/nf_core/modules/mulled.py index c47f35a559..745936e86d 100644 --- a/nf_core/modules/mulled.py +++ b/nf_core/modules/mulled.py @@ -43,7 +43,7 @@ def parse_targets(cls, specifications: Iterable[str]) -> List[Tuple[str, str]]: try: Version(version) except InvalidVersion: - raise ValueError(f"{version} in {spec} is not a PEP440 compliant version specification.") from None + raise ValueError(f"Not a PEP440 version spec: '{version}' in '{spec}'") from None result.append((tool.strip(), version.strip())) return result From cf2f82cab54ac24702218189f0913111b4dd9a5f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 12 May 2022 16:43:32 +0200 Subject: [PATCH 116/136] Mulled: Nicer CLI output --- nf_core/__main__.py | 3 ++- nf_core/modules/mulled.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index f4e615635c..13998fd29b 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -701,7 +701,6 @@ def mulled(specifications, build_number): except ValueError as e: log.error(e) sys.exit(1) - print(image_name) if not MulledImageNameGenerator.image_exists(image_name): log.error( "The generated multi-tool container image name does not seem to exist yet. Please double check that your " @@ -711,6 +710,8 @@ def mulled(specifications, build_number): "https://github.com/BioContainers/multi-package-containers\n" ) sys.exit(1) + log.info("Mulled container hash:") + print(image_name) # nf-core modules test diff --git a/nf_core/modules/mulled.py b/nf_core/modules/mulled.py index 745936e86d..9a34ef3355 100644 --- a/nf_core/modules/mulled.py +++ b/nf_core/modules/mulled.py @@ -62,6 +62,12 @@ def generate_image_name(cls, targets: Iterable[Tuple[str, str]], build_number: i @classmethod def image_exists(cls, image_name: str) -> bool: """Check whether a given BioContainers image name exists via a call to the quay.io API.""" - response = requests.get(f"https://quay.io/biocontainers/{image_name}/", allow_redirects=True) - log.debug(response.text) - return response.status_code == 200 + quay_url = f"https://quay.io/biocontainers/{image_name}/" + response = requests.get(quay_url, allow_redirects=True) + log.debug(f"Got response code '{response.status_code}' for URL {quay_url}") + if response.status_code == 200: + log.info(f"Found [link={quay_url}]docker image[/link] on quay.io! :sparkles:") + return True + else: + log.error(f"Was not able to find [link={quay_url}]docker image[/link] on quay.io") + return False From 1a4e91d9bb92ae769c4e9a4e7c84c3c2ae474507 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 12 May 2022 16:49:33 +0200 Subject: [PATCH 117/136] Getting fancy with logging --- nf_core/__main__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 13998fd29b..0c010ae9ab 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -702,12 +702,14 @@ def mulled(specifications, build_number): log.error(e) sys.exit(1) if not MulledImageNameGenerator.image_exists(image_name): - log.error( - "The generated multi-tool container image name does not seem to exist yet. Please double check that your " - "provided combination of tools and versions exists in the file:\n" - "https://github.com/BioContainers/multi-package-containers/blob/master/combinations/hash.tsv\n" - "If it does not, please add your desired combination as detailed at:\n" - "https://github.com/BioContainers/multi-package-containers\n" + log.error("The generated multi-tool container image name does not seem to exist yet.") + log.info( + "Please double check that your provided combination of tools and versions exists in the file: " + "[link=https://github.com/BioContainers/multi-package-containers/blob/master/combinations/hash.tsv]BioContainers/multi-package-containers 'combinations/hash.tsv'[/link]" + ) + log.info( + "If it does not, please add your desired combination as detailed at: " + "https://github.com/BioContainers/multi-package-containers" ) sys.exit(1) log.info("Mulled container hash:") From b6542f0a8dd1c2d1f047ff977eddefed0a9d4e00 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 12 May 2022 17:12:32 +0200 Subject: [PATCH 118/136] Nicer syntax from @Midnighter https://github.com/nf-core/tools/pull/1571#pullrequestreview-970968902 --- nf_core/modules/lint/main_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 4271453687..fb0a5b9fb1 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -274,7 +274,7 @@ def _parse_input(self, line_raw): """ inputs = [] # Remove comments and trailing whitespace - line, *_ = line_raw.partition("//") + line = line_raw.split("//")[0] line = line.strip() # Tuples with multiple elements if "tuple" in line: From 99535d3a9eb98fd25fe30910b1cd54f666e61a33 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 13 May 2022 12:23:45 +0200 Subject: [PATCH 119/136] Richer linting output for modules --- nf_core/modules/lint/__init__.py | 49 ++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index e81eea7f85..6bf5ca8c3c 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -390,46 +390,59 @@ def _s(some_list): return "s" return "" - # Print module linting results header - console.print(Panel("[magenta]Module lint results")) + # Print blank line for spacing + console.print("") # Table of passed tests if len(self.passed) > 0 and show_passed: - console.print( - rich.panel.Panel(r"[!] {} Test{} Passed".format(len(self.passed), _s(self.passed)), style="bold green") - ) - table = Table(style="green", box=rich.box.ROUNDED) + table = Table(style="green", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") table.add_column("Module name", width=max_mod_name_len) table.add_column("File path") table.add_column("Test message") table = format_result(self.passed, table) - console.print(table) - - # Table of warning tests - if len(self.warned) > 0: console.print( rich.panel.Panel( - r"[!] {} Test Warning{}".format(len(self.warned), _s(self.warned)), style="bold yellow" + table, + title=r"[bold][✔] {} Module Test{} Passed".format(len(self.passed), _s(self.passed)), + title_align="left", + style="green", + padding=0, ) ) - table = Table(style="yellow", box=rich.box.ROUNDED) + + # Table of warning tests + if len(self.warned) > 0: + table = Table(style="yellow", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") table.add_column("Module name", width=max_mod_name_len) table.add_column("File path") table.add_column("Test message") table = format_result(self.warned, table) - console.print(table) + console.print( + rich.panel.Panel( + table, + title=r"[bold][!] {} Module Test Warning{}".format(len(self.warned), _s(self.warned)), + title_align="left", + style="yellow", + padding=0, + ) + ) # Table of failing tests if len(self.failed) > 0: - console.print( - rich.panel.Panel(r"[!] {} Test{} Failed".format(len(self.failed), _s(self.failed)), style="bold red") - ) - table = Table(style="red", box=rich.box.ROUNDED) + table = Table(style="red", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") table.add_column("Module name", width=max_mod_name_len) table.add_column("File path") table.add_column("Test message") table = format_result(self.failed, table) - console.print(table) + console.print( + rich.panel.Panel( + table, + title=r"[bold][✗] {} Module Test{} Failed".format(len(self.failed), _s(self.failed)), + title_align="left", + style="red", + padding=0, + ) + ) def print_summary(self): def _s(some_list): From 1e47ba8e555482b4c90dd87f10e6c1fe7d977ba4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 13 May 2022 12:43:48 +0200 Subject: [PATCH 120/136] Linting output consistency between modules + pipeline --- nf_core/lint/__init__.py | 30 +++++------ nf_core/modules/lint/__init__.py | 86 +++++++++++++++----------------- 2 files changed, 55 insertions(+), 61 deletions(-) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index a0dad185b8..c66cf4ff4d 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -326,7 +326,7 @@ def _print_results(self, show_passed): log.debug("Printing final results") # Helper function to format test links nicely - def format_result(test_results, table): + def format_result(test_results, table, row_style=None): """ Given an list of error message IDs and the message texts, return a nicely formatted string for the terminal with appropriate ASCII colours. @@ -338,7 +338,8 @@ def format_result(test_results, table): table.add_row( Markdown( f"[{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}.html): {msg}" - ) + ), + style=row_style, ) return table @@ -347,42 +348,41 @@ def _s(some_list): return "s" return "" - # Print lint results header - console.print(Panel("[magenta]General lint results")) - # Table of passed tests if len(self.passed) > 0 and show_passed: table = Table(style="green", box=rich.box.ROUNDED) - table.add_column(r"[✔] {} Test{} Passed".format(len(self.passed), _s(self.passed)), no_wrap=True) - table = format_result(self.passed, table) + table.add_column(r"[✔] {} Pipeline Test{} Passed".format(len(self.passed), _s(self.passed)), no_wrap=True) + table = format_result(self.passed, table, "green") console.print(table) # Table of fixed tests if len(self.fixed) > 0: table = Table(style="bright_blue", box=rich.box.ROUNDED) - table.add_column(r"[?] {} Test{} Fixed".format(len(self.fixed), _s(self.fixed)), no_wrap=True) - table = format_result(self.fixed, table) + table.add_column(r"[?] {} Pipeline Test{} Fixed".format(len(self.fixed), _s(self.fixed)), no_wrap=True) + table = format_result(self.fixed, table, "bright blue") console.print(table) # Table of ignored tests if len(self.ignored) > 0: table = Table(style="grey58", box=rich.box.ROUNDED) - table.add_column(r"[?] {} Test{} Ignored".format(len(self.ignored), _s(self.ignored)), no_wrap=True) - table = format_result(self.ignored, table) + table.add_column( + r"[?] {} Pipeline Test{} Ignored".format(len(self.ignored), _s(self.ignored)), no_wrap=True + ) + table = format_result(self.ignored, table, "grey58") console.print(table) # Table of warning tests if len(self.warned) > 0: table = Table(style="yellow", box=rich.box.ROUNDED) - table.add_column(r"[!] {} Test Warning{}".format(len(self.warned), _s(self.warned)), no_wrap=True) - table = format_result(self.warned, table) + table.add_column(r"[!] {} Pipeline Test Warning{}".format(len(self.warned), _s(self.warned)), no_wrap=True) + table = format_result(self.warned, table, "yellow") console.print(table) # Table of failing tests if len(self.failed) > 0: table = Table(style="red", box=rich.box.ROUNDED) - table.add_column(r"[✗] {} Test{} Failed".format(len(self.failed), _s(self.failed)), no_wrap=True) - table = format_result(self.failed, table) + table.add_column(r"[✗] {} Pipeline Test{} Failed".format(len(self.failed), _s(self.failed)), no_wrap=True) + table = format_result(self.failed, table, "red") console.print(table) def _print_summary(self): diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 6bf5ca8c3c..7326af8580 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -353,7 +353,7 @@ def _print_results(self, show_passed=False): # Find maximum module name length max_mod_name_len = 40 - for idx, tests in enumerate([self.passed, self.warned, self.failed]): + for tests in [self.passed, self.warned, self.failed]: try: for lint_result in tests: max_mod_name_len = max(len(lint_result.module_name), max_mod_name_len) @@ -369,19 +369,16 @@ def format_result(test_results, table): # TODO: Row styles don't work current as table-level style overrides. # I'd like to make an issue about this on the rich repo so leaving here in case there is a future fix last_modname = False - row_style = None + even_row = False for lint_result in test_results: if last_modname and lint_result.module_name != last_modname: - if row_style: - row_style = None - else: - row_style = "magenta" + even_row = not even_row last_modname = lint_result.module_name table.add_row( Markdown(f"{lint_result.module_name}"), os.path.relpath(lint_result.file_path, self.dir), Markdown(f"{lint_result.message}"), - style=row_style, + style="dim" if even_row else None, ) return table @@ -395,54 +392,51 @@ def _s(some_list): # Table of passed tests if len(self.passed) > 0 and show_passed: - table = Table(style="green", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") - table.add_column("Module name", width=max_mod_name_len) - table.add_column("File path") - table.add_column("Test message") - table = format_result(self.passed, table) - console.print( - rich.panel.Panel( - table, - title=r"[bold][✔] {} Module Test{} Passed".format(len(self.passed), _s(self.passed)), - title_align="left", - style="green", - padding=0, - ) + inner_table = Table(style="green", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") + inner_table.add_column("Module name", width=max_mod_name_len) + inner_table.add_column("File path") + inner_table.add_column("Test message") + inner_table = format_result(self.passed, inner_table) + + wrapper_table = Table(style="green", box=rich.box.ROUNDED) + wrapper_table.add_column( + r"[bold][✔] {} Module Test{} Passed".format(len(self.passed), _s(self.passed)), no_wrap=True ) + wrapper_table.add_row(inner_table, style="green") + + console.print(wrapper_table) # Table of warning tests if len(self.warned) > 0: - table = Table(style="yellow", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") - table.add_column("Module name", width=max_mod_name_len) - table.add_column("File path") - table.add_column("Test message") - table = format_result(self.warned, table) - console.print( - rich.panel.Panel( - table, - title=r"[bold][!] {} Module Test Warning{}".format(len(self.warned), _s(self.warned)), - title_align="left", - style="yellow", - padding=0, - ) + inner_table = Table(style="yellow", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") + inner_table.add_column("Module name", width=max_mod_name_len) + inner_table.add_column("File path") + inner_table.add_column("Test message") + inner_table = format_result(self.warned, inner_table) + + wrapper_table = Table(style="yellow", box=rich.box.ROUNDED) + wrapper_table.add_column( + r"[bold][!] {} Module Test Warning{}".format(len(self.warned), _s(self.warned)), no_wrap=True ) + wrapper_table.add_row(inner_table, style="yellow") + + console.print(wrapper_table) # Table of failing tests if len(self.failed) > 0: - table = Table(style="red", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") - table.add_column("Module name", width=max_mod_name_len) - table.add_column("File path") - table.add_column("Test message") - table = format_result(self.failed, table) - console.print( - rich.panel.Panel( - table, - title=r"[bold][✗] {} Module Test{} Failed".format(len(self.failed), _s(self.failed)), - title_align="left", - style="red", - padding=0, - ) + inner_table = Table(style="red", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") + inner_table.add_column("Module name", width=max_mod_name_len) + inner_table.add_column("File path") + inner_table.add_column("Test message") + inner_table = format_result(self.failed, inner_table) + + wrapper_table = Table(style="red", box=rich.box.ROUNDED) + wrapper_table.add_column( + r"[bold][✗] {} Module Test{} Failed".format(len(self.failed), _s(self.failed)), no_wrap=True ) + wrapper_table.add_row(inner_table, style="red") + + console.print(wrapper_table) def print_summary(self): def _s(some_list): From 74459a0d9749b7a29748f1f5b86bd0d5a1e26e84 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 13 May 2022 12:44:17 +0200 Subject: [PATCH 121/136] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5574e40b52..ede704e668 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520)) - Use [`$XDG_CONFIG_HOME`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) or `~/.config/nf-core` instead of `~/.nfcore` for API cache (the latter can be safely deleted) - Consolidate GitHub API calls into a shared function that uses authentication from the [`gh` GitHub cli tool](https://cli.github.com/) or `GITHUB_AUTH_TOKEN` to avoid rate limiting ([#1499](https://github.com/nf-core/tools/pull/1499)) +- Tweaks to CLI output display of lint results ### Modules From 1504b16592d5e9c4c55e2ae6f3ac68bd2390271d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 13 May 2022 13:03:23 +0200 Subject: [PATCH 122/136] Standardise to panels instead of tables --- nf_core/lint/__init__.py | 80 +++++++++++++++++++++----------- nf_core/modules/lint/__init__.py | 75 ++++++++++++++++-------------- 2 files changed, 91 insertions(+), 64 deletions(-) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index c66cf4ff4d..97293d2a2d 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -8,6 +8,7 @@ from rich.markdown import Markdown from rich.table import Table from rich.panel import Panel +from rich.console import group import datetime import git import json @@ -323,10 +324,14 @@ def _print_results(self, show_passed): summarising the linting results. """ + # Spacing from log messages above + console.print("") + log.debug("Printing final results") # Helper function to format test links nicely - def format_result(test_results, table, row_style=None): + @group() + def format_result(test_results): """ Given an list of error message IDs and the message texts, return a nicely formatted string for the terminal with appropriate ASCII colours. @@ -335,13 +340,9 @@ def format_result(test_results, table, row_style=None): tools_version = __version__ if "dev" in __version__: tools_version = "latest" - table.add_row( - Markdown( - f"[{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}.html): {msg}" - ), - style=row_style, + yield Markdown( + f"[{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}.html): {msg}" ) - return table def _s(some_list): if len(some_list) != 1: @@ -350,40 +351,63 @@ def _s(some_list): # Table of passed tests if len(self.passed) > 0 and show_passed: - table = Table(style="green", box=rich.box.ROUNDED) - table.add_column(r"[✔] {} Pipeline Test{} Passed".format(len(self.passed), _s(self.passed)), no_wrap=True) - table = format_result(self.passed, table, "green") - console.print(table) + console.print( + rich.panel.Panel( + format_result(self.passed), + title=r"[bold][✔] {} Pipeline Test{} Passed".format(len(self.passed), _s(self.passed)), + title_align="left", + style="green", + padding=1, + ) + ) # Table of fixed tests if len(self.fixed) > 0: - table = Table(style="bright_blue", box=rich.box.ROUNDED) - table.add_column(r"[?] {} Pipeline Test{} Fixed".format(len(self.fixed), _s(self.fixed)), no_wrap=True) - table = format_result(self.fixed, table, "bright blue") - console.print(table) + console.print( + rich.panel.Panel( + format_result(self.fixed), + title=r"[bold][?] {} Pipeline Test{} Fixed".format(len(self.fixed), _s(self.fixed)), + title_align="left", + style="bright_blue", + padding=1, + ) + ) # Table of ignored tests if len(self.ignored) > 0: - table = Table(style="grey58", box=rich.box.ROUNDED) - table.add_column( - r"[?] {} Pipeline Test{} Ignored".format(len(self.ignored), _s(self.ignored)), no_wrap=True + console.print( + rich.panel.Panel( + format_result(self.ignored), + title=r"[bold][?] {} Pipeline Test{} Ignored".format(len(self.ignored), _s(self.ignored)), + title_align="left", + style="grey58", + padding=1, + ) ) - table = format_result(self.ignored, table, "grey58") - console.print(table) # Table of warning tests if len(self.warned) > 0: - table = Table(style="yellow", box=rich.box.ROUNDED) - table.add_column(r"[!] {} Pipeline Test Warning{}".format(len(self.warned), _s(self.warned)), no_wrap=True) - table = format_result(self.warned, table, "yellow") - console.print(table) + console.print( + rich.panel.Panel( + format_result(self.warned), + title=r"[bold][!] {} Pipeline Test Warning{}".format(len(self.warned), _s(self.warned)), + title_align="left", + style="yellow", + padding=1, + ) + ) # Table of failing tests if len(self.failed) > 0: - table = Table(style="red", box=rich.box.ROUNDED) - table.add_column(r"[✗] {} Pipeline Test{} Failed".format(len(self.failed), _s(self.failed)), no_wrap=True) - table = format_result(self.failed, table, "red") - console.print(table) + console.print( + rich.panel.Panel( + format_result(self.failed), + title=r"[bold][✗] {} Pipeline Test{} Failed".format(len(self.failed), _s(self.failed)), + title_align="left", + style="red", + padding=1, + ) + ) def _print_summary(self): def _s(some_list): diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 7326af8580..6bea05cb27 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -392,51 +392,54 @@ def _s(some_list): # Table of passed tests if len(self.passed) > 0 and show_passed: - inner_table = Table(style="green", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") - inner_table.add_column("Module name", width=max_mod_name_len) - inner_table.add_column("File path") - inner_table.add_column("Test message") - inner_table = format_result(self.passed, inner_table) - - wrapper_table = Table(style="green", box=rich.box.ROUNDED) - wrapper_table.add_column( - r"[bold][✔] {} Module Test{} Passed".format(len(self.passed), _s(self.passed)), no_wrap=True + table = Table(style="green", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") + table.add_column("Module name", width=max_mod_name_len) + table.add_column("File path") + table.add_column("Test message") + table = format_result(self.passed, table) + console.print( + rich.panel.Panel( + table, + title=r"[bold][✔] {} Module Test{} Passed".format(len(self.passed), _s(self.passed)), + title_align="left", + style="green", + padding=0, + ) ) - wrapper_table.add_row(inner_table, style="green") - - console.print(wrapper_table) # Table of warning tests if len(self.warned) > 0: - inner_table = Table(style="yellow", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") - inner_table.add_column("Module name", width=max_mod_name_len) - inner_table.add_column("File path") - inner_table.add_column("Test message") - inner_table = format_result(self.warned, inner_table) - - wrapper_table = Table(style="yellow", box=rich.box.ROUNDED) - wrapper_table.add_column( - r"[bold][!] {} Module Test Warning{}".format(len(self.warned), _s(self.warned)), no_wrap=True + table = Table(style="yellow", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") + table.add_column("Module name", width=max_mod_name_len) + table.add_column("File path") + table.add_column("Test message") + table = format_result(self.warned, table) + console.print( + rich.panel.Panel( + table, + title=r"[bold][!] {} Module Test Warning{}".format(len(self.warned), _s(self.warned)), + title_align="left", + style="yellow", + padding=0, + ) ) - wrapper_table.add_row(inner_table, style="yellow") - - console.print(wrapper_table) # Table of failing tests if len(self.failed) > 0: - inner_table = Table(style="red", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") - inner_table.add_column("Module name", width=max_mod_name_len) - inner_table.add_column("File path") - inner_table.add_column("Test message") - inner_table = format_result(self.failed, inner_table) - - wrapper_table = Table(style="red", box=rich.box.ROUNDED) - wrapper_table.add_column( - r"[bold][✗] {} Module Test{} Failed".format(len(self.failed), _s(self.failed)), no_wrap=True + table = Table(style="red", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") + table.add_column("Module name", width=max_mod_name_len) + table.add_column("File path") + table.add_column("Test message") + table = format_result(self.failed, table) + console.print( + rich.panel.Panel( + table, + title=r"[bold][✗] {} Module Test{} Failed".format(len(self.failed), _s(self.failed)), + title_align="left", + style="red", + padding=0, + ) ) - wrapper_table.add_row(inner_table, style="red") - - console.print(wrapper_table) def print_summary(self): def _s(some_list): From 1a992c2c4f9f974cf33a01116d81ccb74df8d3c9 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 13 May 2022 13:15:54 +0200 Subject: [PATCH 123/136] Module lint: Allow no input or output in YAML --- nf_core/modules/lint/meta_yml.py | 38 ++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 3c59c774d4..8d85cdf6c7 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -20,7 +20,7 @@ def meta_yml(module_lint_object, module): ``meta.yml`` and the ``main.nf``. """ - required_keys = ["name", "input", "output"] + required_keys = ["name"] required_keys_lists = ["input", "output"] try: with open(module.meta_yml, "r") as fh: @@ -35,9 +35,9 @@ def meta_yml(module_lint_object, module): all_list_children = True for rk in required_keys: if not rk in meta_yaml.keys(): - module.failed.append(("meta_required_keys", f"`{rk}` not specified", module.meta_yml)) + module.failed.append(("meta_required_keys", f"`{rk}` not specified in YAML", module.meta_yml)) contains_required_keys = False - elif not isinstance(meta_yaml[rk], list) and rk in required_keys_lists: + elif rk in meta_yaml.keys() and not isinstance(meta_yaml[rk], list) and rk in required_keys_lists: module.failed.append(("meta_required_keys", f"`{rk}` is not a list", module.meta_yml)) all_list_children = False if contains_required_keys: @@ -45,24 +45,30 @@ def meta_yml(module_lint_object, module): # Confirm that all input and output channels are specified if contains_required_keys and all_list_children: - meta_input = [list(x.keys())[0] for x in meta_yaml["input"]] - for input in module.inputs: - if input in meta_input: - module.passed.append(("meta_input", f"`{input}` specified", module.meta_yml)) - else: - module.failed.append(("meta_input", f"`{input}` missing in `meta.yml`", module.meta_yml)) + if "input" in meta_yaml: + meta_input = [list(x.keys())[0] for x in meta_yaml["input"]] + for input in module.inputs: + if input in meta_input: + module.passed.append(("meta_input", f"`{input}` specified", module.meta_yml)) + else: + module.failed.append(("meta_input", f"`{input}` missing in `meta.yml`", module.meta_yml)) - meta_output = [list(x.keys())[0] for x in meta_yaml["output"]] - for output in module.outputs: - if output in meta_output: - module.passed.append(("meta_output", "`{output}` specified", module.meta_yml)) - else: - module.failed.append(("meta_output", "`{output}` missing in `meta.yml`", module.meta_yml)) + if "output" in meta_yaml: + meta_output = [list(x.keys())[0] for x in meta_yaml["output"]] + for output in module.outputs: + if output in meta_output: + module.passed.append(("meta_output", "`{output}` specified", module.meta_yml)) + else: + module.failed.append(("meta_output", "`{output}` missing in `meta.yml`", module.meta_yml)) # confirm that the name matches the process name in main.nf if meta_yaml["name"].upper() == module.process_name: module.passed.append(("meta_name", "Correct name specified in `meta.yml`", module.meta_yml)) else: module.failed.append( - ("meta_name", "Conflicting process name between `meta.yml` and `main.nf`", module.meta_yml) + ( + "meta_name", + f"Conflicting process name between meta.yml (`{meta_yaml['name']}`) and main.nf (`{module.process_name}`)", + module.meta_yml, + ) ) From cdbb3f4ee1a455bb96ad2369823d93683c208bf8 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 13 May 2022 13:23:45 +0200 Subject: [PATCH 124/136] Module linting: Only lint script if there was a script. Basically, skip this step if using shell instead. Fixes nf-core/tools#1558 --- nf_core/modules/lint/main_nf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index fb0a5b9fb1..13ae40e0d5 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -99,7 +99,8 @@ def main_nf(module_lint_object, module): check_when_section(module, when_lines) # Check the script definition - check_script_section(module, script_lines) + if len(script_lines): + check_script_section(module, script_lines) # Check whether 'meta' is emitted when given as input if inputs: From 6601b65329a10c18022df8957e0dc978c92b48ea Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 13 May 2022 13:25:23 +0200 Subject: [PATCH 125/136] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5574e40b52..6724d691a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - Allow follow links when generating `test.yml` file with `nf-core modules create-test-yml` ([1570](https://github.com/nf-core/tools/pull/1570)) - Linting now recognised `shell` blocks to avoid error `when: condition has too many lines` ([#1557](https://github.com/nf-core/tools/issues/1557)) - Linting: fix error when using comments after `input` tuple lines ([#1542](https://github.com/nf-core/tools/issues/1542)) +- Linting: Don't lint the `shell` block when `script` is used ([1558](https://github.com/nf-core/tools/pull/1558)) ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] From 88e4695644eff3a1d55374728c9b64a985af02ce Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 13 May 2022 13:44:23 +0200 Subject: [PATCH 126/136] Check that 'template' is used when 'script' is used. Also check that only script or shell used and not both. --- CHANGELOG.md | 1 + nf_core/modules/lint/main_nf.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6724d691a1..3b81346150 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ - Linting now recognised `shell` blocks to avoid error `when: condition has too many lines` ([#1557](https://github.com/nf-core/tools/issues/1557)) - Linting: fix error when using comments after `input` tuple lines ([#1542](https://github.com/nf-core/tools/issues/1542)) - Linting: Don't lint the `shell` block when `script` is used ([1558](https://github.com/nf-core/tools/pull/1558)) +- Linting: Check that `template` is used in `script` blocks ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 13ae40e0d5..464f617a9d 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -56,6 +56,7 @@ def main_nf(module_lint_object, module): state = "module" process_lines = [] script_lines = [] + shell_lines = [] when_lines = [] for l in lines: if re.search("^\s*process\s*\w*\s*{", l) and state == "module": @@ -88,6 +89,8 @@ def main_nf(module_lint_object, module): when_lines.append(l) if state == "script" and not _is_empty(module, l): script_lines.append(l) + if state == "shell" and not _is_empty(module, l): + shell_lines.append(l) # Check the process definitions if check_process_section(module, process_lines): @@ -98,10 +101,21 @@ def main_nf(module_lint_object, module): # Check the when statement check_when_section(module, when_lines) + # Check that we have script or shell, not both + if len(script_lines) and len(shell_lines): + module.failed.append(("main_nf_script_shell", "Script and Shell found, should use only one", module.main_nf)) + # Check the script definition if len(script_lines): check_script_section(module, script_lines) + # Check that shell uses a template + if len(shell_lines): + if any("template" in l for l in shell_lines): + module.passed.append(("main_nf_shell_template", "`template` found in `shell` block", module.main_nf)) + else: + module.failed.append(("main_nf_shell_template", "No `template` found in `shell` block", module.main_nf)) + # Check whether 'meta' is emitted when given as input if inputs: if "meta" in inputs: From db0a52c7e04517e128acaf3c22165ef88a691015 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 13 May 2022 14:20:52 +0200 Subject: [PATCH 127/136] Update nf_core/modules/lint/meta_yml.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/modules/lint/meta_yml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 8d85cdf6c7..dd17d55dc9 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -57,9 +57,9 @@ def meta_yml(module_lint_object, module): meta_output = [list(x.keys())[0] for x in meta_yaml["output"]] for output in module.outputs: if output in meta_output: - module.passed.append(("meta_output", "`{output}` specified", module.meta_yml)) + module.passed.append(("meta_output", f"`{output}` specified", module.meta_yml)) else: - module.failed.append(("meta_output", "`{output}` missing in `meta.yml`", module.meta_yml)) + module.failed.append(("meta_output", f"`{output}` missing in `meta.yml`", module.meta_yml)) # confirm that the name matches the process name in main.nf if meta_yaml["name"].upper() == module.process_name: From 5cd201371e293a9de31e468ae55ab96abf1b17a9 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 13 May 2022 16:05:30 +0200 Subject: [PATCH 128/136] Readme badges update - Remove black backgrounds. Now that GitHub has dark mode, they dissapear into the background. - Add a badge to launch on Tower --- CHANGELOG.md | 1 + nf_core/pipeline-template/README.md | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b81346150..d99f351296 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Add `.prettierignore` file to stop Prettier linting tests from running over test files - Add actions workflow to respond to `@nf-core-bot fix linting` comments on pipeline PRs - Made module template test command match the default used in `nf-core modules create-test-yml` ([#1562](https://github.com/nf-core/tools/issues/1562)) +- Removed black background from Readme badges now that GitHub has a dark mode, added Tower launch badge. ### General diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index bf78682fd6..4287090a03 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -2,17 +2,18 @@ [![GitHub Actions CI Status](https://github.com/{{ name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) [![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22) -[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results) -[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) - -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg?labelColor=000000)](https://www.nextflow.io/) -[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) -[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) -[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) - -[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}) -[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core) -[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) +[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results) +[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8)](https://doi.org/10.5281/zenodo.XXXXXXX) + +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/) +[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?logo=anaconda)](https://docs.conda.io/en/latest/) +[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?logo=docker)](https://www.docker.com/) +[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg)](https://sylabs.io/docs/) +[![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/{{ name }}) + +[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}) +[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?logo=twitter)](https://twitter.com/nf_core) +[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?logo=youtube)](https://www.youtube.com/c/nf-core) ## Introduction From 453a68de77c0cea6cb45431935acb12e5cbcbfef Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 13 May 2022 16:10:59 +0200 Subject: [PATCH 129/136] Update tests --- tests/test_bump_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bump_version.py b/tests/test_bump_version.py index 1697fcfd0a..35bcf9c7c1 100644 --- a/tests/test_bump_version.py +++ b/tests/test_bump_version.py @@ -79,7 +79,7 @@ def test_bump_nextflow_version(datafiles, tmp_path): with open(new_pipeline_obj._fp("README.md")) as fh: readme = fh.read().splitlines() assert ( - "[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A5{}-23aa62.svg?labelColor=000000)](https://www.nextflow.io/)".format( + "[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A5{}-23aa62.svg)](https://www.nextflow.io/)".format( "21.10.3" ) in readme From b795ae21ad5a065cbdaf45aa41278a8cf1656a72 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 13 May 2022 16:15:57 +0200 Subject: [PATCH 130/136] Module linting: Require output --- nf_core/modules/lint/main_nf.py | 6 ++++++ nf_core/modules/lint/meta_yml.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 464f617a9d..31dc6d5495 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -92,6 +92,12 @@ def main_nf(module_lint_object, module): if state == "shell" and not _is_empty(module, l): shell_lines.append(l) + # Check that we have required sections + if not len(outputs): + module.failed.append(("main_nf_script_outputs", "No process 'output' block found", module.main_nf)) + else: + module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf)) + # Check the process definitions if check_process_section(module, process_lines): module.passed.append(("main_nf_container", "Container versions match", module.main_nf)) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index dd17d55dc9..f665f64094 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -20,7 +20,7 @@ def meta_yml(module_lint_object, module): ``meta.yml`` and the ``main.nf``. """ - required_keys = ["name"] + required_keys = ["name", "output"] required_keys_lists = ["input", "output"] try: with open(module.meta_yml, "r") as fh: From 7e960fcb6608b7190f9950bb8dfc62e78f162fb5 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 13 May 2022 14:42:27 +0000 Subject: [PATCH 131/136] Update Gitpod image --- nf_core/gitpod/gitpod.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 0388071d11..793eba21b7 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -26,9 +26,9 @@ RUN conda update -n base -c defaults conda && \ conda config --add channels conda-forge && \ conda install \ openjdk=11.0.13 \ - nextflow=21.10.6 \ + nextflow=22.04.0 \ pytest-workflow=1.6.0 \ - mamba=0.22.1 \ + mamba=0.23.1 \ pip=22.0.4 \ black=22.1.0 \ -n base && \ From df862bb3c254736dfe9f453dd0db4904feade6b5 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 13 May 2022 14:48:24 +0000 Subject: [PATCH 132/136] Remove nextflow self-update --- nf_core/gitpod/gitpod.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 793eba21b7..dce5e2577f 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -32,7 +32,6 @@ RUN conda update -n base -c defaults conda && \ pip=22.0.4 \ black=22.1.0 \ -n base && \ - nextflow self-update && \ conda clean --all -f -y # Install nf-core From 02802b256843fd4d5f3237621bcbd61a03bc1b00 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 13 May 2022 16:59:53 +0200 Subject: [PATCH 133/136] Prepare v2.4 release --- CHANGELOG.md | 33 ++++++++++++++++++--------------- setup.py | 2 +- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88b1999f73..cf5fb16fb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,40 +1,43 @@ # nf-core/tools: Changelog -## v2.4dev +## [v2.4 - Cobolt Koala](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16] ### Template -- Set the default DAG graphic output to HTML to have a default that does not depend on Graphviz being installed on the host system ([#1512](https://github.com/nf-core/tools/pull/1512)). -- Fix bug in pipeline readme logo URL +- Add actions workflow to respond to `@nf-core-bot fix linting` comments on pipeline PRs - Fix Prettier formatting bug in completion email HTML template ([#1509](https://github.com/nf-core/tools/issues/1509)) +- Fix bug in pipeline readme logo URL +- Set the default DAG graphic output to HTML to have a default that does not depend on Graphviz being installed on the host system ([#1512](https://github.com/nf-core/tools/pull/1512)). - Removed retry strategy for AWS tests CI, as Nextflow now handles spot instance retries itself - Add `.prettierignore` file to stop Prettier linting tests from running over test files -- Add actions workflow to respond to `@nf-core-bot fix linting` comments on pipeline PRs - Made module template test command match the default used in `nf-core modules create-test-yml` ([#1562](https://github.com/nf-core/tools/issues/1562)) - Removed black background from Readme badges now that GitHub has a dark mode, added Tower launch badge. ### General -- Bumped the minimum version of `rich` from `v10` to `v10.7.0` -- Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors. - Add actions workflow to respond to `@nf-core-bot fix linting` comments on nf-core/tools PRs -- Linting: Don't allow a `.nf-core.yaml` file, should be `.yml` ([#1515](https://github.com/nf-core/tools/pull/1515)). -- Remove empty JSON schema definition groups to avoid usage errors ([#1419](https://github.com/nf-core/tools/issues/1419)) -- Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520)) - Use [`$XDG_CONFIG_HOME`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) or `~/.config/nf-core` instead of `~/.nfcore` for API cache (the latter can be safely deleted) - Consolidate GitHub API calls into a shared function that uses authentication from the [`gh` GitHub cli tool](https://cli.github.com/) or `GITHUB_AUTH_TOKEN` to avoid rate limiting ([#1499](https://github.com/nf-core/tools/pull/1499)) -- Tweaks to CLI output display of lint results +- Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors. +- Remove empty JSON schema definition groups to avoid usage errors ([#1419](https://github.com/nf-core/tools/issues/1419)) +- Bumped the minimum version of `rich` from `v10` to `v10.7.0` ### Modules -- Escaped test run output before logging it, to avoid a rich ` MarkupError` - Add a new command `nf-core modules mulled` which can generate the name for a multi-tool container image. - Add a new command `nf-core modules test` which runs pytests locally. +- Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520)) - Allow follow links when generating `test.yml` file with `nf-core modules create-test-yml` ([1570](https://github.com/nf-core/tools/pull/1570)) -- Linting now recognised `shell` blocks to avoid error `when: condition has too many lines` ([#1557](https://github.com/nf-core/tools/issues/1557)) -- Linting: fix error when using comments after `input` tuple lines ([#1542](https://github.com/nf-core/tools/issues/1542)) -- Linting: Don't lint the `shell` block when `script` is used ([1558](https://github.com/nf-core/tools/pull/1558)) -- Linting: Check that `template` is used in `script` blocks +- Escaped test run output before logging it, to avoid a rich ` MarkupError` + +### Linting + +- Don't allow a `.nf-core.yaml` file, should be `.yml` ([#1515](https://github.com/nf-core/tools/pull/1515)). +- `shell` blocks now recognised to avoid error `when: condition has too many lines` ([#1557](https://github.com/nf-core/tools/issues/1557)) +- Fixed error when using comments after `input` tuple lines ([#1542](https://github.com/nf-core/tools/issues/1542)) +- Don't lint the `shell` block when `script` is used ([1558](https://github.com/nf-core/tools/pull/1558)) +- Check that `template` is used in `script` blocks +- Tweaks to CLI output display of lint results ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] diff --git a/setup.py b/setup.py index 5fda0ae5be..2622db133b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -version = "2.4dev" +version = "2.4" with open("README.md") as f: readme = f.read() From 6a20d9f09e41fb3fc3290997c2b940e8b3fe3615 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Mon, 16 May 2022 11:44:19 +0200 Subject: [PATCH 134/136] fix: make sniffing more robust by reading full lines fix #1561 --- nf_core/pipeline-template/bin/check_samplesheet.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/bin/check_samplesheet.py b/nf_core/pipeline-template/bin/check_samplesheet.py index d1a9919eec..3652c63c8b 100755 --- a/nf_core/pipeline-template/bin/check_samplesheet.py +++ b/nf_core/pipeline-template/bin/check_samplesheet.py @@ -129,6 +129,16 @@ def validate_unique_samples(self): row[self._sample_col] = f"{sample}_T{seen[sample]}" +def read_head(handle, num_lines=10): + """Read the specified number of lines from the current position in the file.""" + lines = [] + for idx, line in enumerate(handle): + if idx == num_lines: + break + lines.append(line) + return "".join(lines) + + def sniff_format(handle): """ Detect the tabular format. @@ -144,13 +154,13 @@ def sniff_format(handle): https://docs.python.org/3/glossary.html#term-text-file """ - peek = handle.read(2048) + peek = read_head(handle) + handle.seek(0) sniffer = csv.Sniffer() if not sniffer.has_header(peek): logger.critical(f"The given sample sheet does not appear to contain a header.") sys.exit(1) dialect = sniffer.sniff(peek) - handle.seek(0) return dialect From 05cb12d993668a4a233442936f3e66be379b5f23 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 16 May 2022 11:45:00 +0200 Subject: [PATCH 135/136] Fix changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf5fb16fb4..62ce4f9914 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Add `.prettierignore` file to stop Prettier linting tests from running over test files - Made module template test command match the default used in `nf-core modules create-test-yml` ([#1562](https://github.com/nf-core/tools/issues/1562)) - Removed black background from Readme badges now that GitHub has a dark mode, added Tower launch badge. +- Don't save md5sum for `versions.yml` when running `nf-core modules create-test-yml` ([#1511](https://github.com/nf-core/tools/pull/1511)) ### General @@ -50,7 +51,6 @@ Very minor patch release to fix the full size AWS tests and re-run the template - Remove traces of markdownlint in the template ([#1486](https://github.com/nf-core/tools/pull/1486) - Remove accidentally added line in `CHANGELOG.md` in the template ([#1487](https://github.com/nf-core/tools/pull/1487)) - Update linting to check that `.editorconfig` is there and `.yamllint.yml` isn't. -- Don't save md5sum for `versions.yml` when running `nf-core modules create-test-yml` ([#1511](https://github.com/nf-core/tools/pull/1511)) ## [v2.3.1 - Mercury Vulture Formatting](https://github.com/nf-core/tools/releases/tag/2.3.1) - [2022-03-23] From 297906508ec2f293ed34fafb28dc9bd95683b593 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Mon, 16 May 2022 11:47:21 +0200 Subject: [PATCH 136/136] chore: document change --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf5fb16fb4..d50ac75b12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Template +- Read entire lines when sniffing the samplesheet format (fix [#1561](https://github.com/nf-core/tools/issues/1561)) - Add actions workflow to respond to `@nf-core-bot fix linting` comments on pipeline PRs - Fix Prettier formatting bug in completion email HTML template ([#1509](https://github.com/nf-core/tools/issues/1509)) - Fix bug in pipeline readme logo URL