diff --git a/.github/workflows/build_gitpod.yml b/.github/workflows/build_gitpod.yml new file mode 100644 index 0000000000..7cdc5aa4d8 --- /dev/null +++ b/.github/workflows/build_gitpod.yml @@ -0,0 +1,45 @@ +name: nf-core gitpod build and push +# Builds the docker image for gitpod and pushes it to DockerHub + +on: + push: + branches: + - 'master' + - 'dev' + paths: + - 'nf_core/gitpod/gitpod.Dockerfile' + - '.github/workflows/build_gitpod.yml' + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + # Only run for the nf-core repo, for releases and merged PRs + if: ${{ github.repository == 'nf-core/tools' }} + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASS }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v3 + with: + images: nfcore/gitpod + tags: | + type=semver,pattern={{version}} + type=raw,value=latest + + - name: Build and push Docker image + uses: docker/build-push-action@v2 + with: + context: . + file: nf_core/gitpod/gitpod.Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 9b13cadd39..90013bc439 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -47,7 +47,7 @@ jobs: - name: Run nf-core/tools run: | nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" - nextflow run nf-core-testpipeline -profile test,docker + nextflow run nf-core-testpipeline -profile test,docker --outdir ./results - name: Upload log file artifact if: ${{ always() }} diff --git a/.github/workflows/markdown-lint.yml b/.github/workflows/markdown-lint.yml index 193052a437..a34f573fe8 100644 --- a/.github/workflows/markdown-lint.yml +++ b/.github/workflows/markdown-lint.yml @@ -7,9 +7,7 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 - with: - node-version: "10" + - uses: actions/setup-node@v2 - name: Install markdownlint run: npm install -g markdownlint-cli diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml index e0c26f2f4e..3a0e789d5b 100644 --- a/.github/workflows/push_dockerhub_dev.yml +++ b/.github/workflows/push_dockerhub_dev.yml @@ -27,4 +27,3 @@ jobs: run: | echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin docker push nfcore/tools:dev - diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 39aa3acae8..4537d7e5cc 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -3,11 +3,7 @@ name: Python tests # Only run if we changed a Python file on: push: - paths: - - "**.py" pull_request: - paths: - - "**.py" # Uncomment if we need an edge release of Nextflow again # env: NXF_EDGE: 1 @@ -41,7 +37,7 @@ jobs: sudo mv nextflow /usr/local/bin/ - name: Test with pytest - run: python3 -m pytest --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core + run: python3 -m pytest tests/ --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core - uses: codecov/codecov-action@v1 name: Upload code coverage report diff --git a/.github/workflows/tools-api-docs-release.yml b/.github/workflows/tools-api-docs-release.yml index 8c47f2dd0e..c420824722 100644 --- a/.github/workflows/tools-api-docs-release.yml +++ b/.github/workflows/tools-api-docs-release.yml @@ -9,9 +9,9 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: - dir: - - latest - - ${{ github.event.release.tag_name }} + dir: + - latest + - ${{ github.event.release.tag_name }} steps: - name: Check out source-code repository uses: actions/checkout@v2 diff --git a/.github/workflows/yamllint.yml b/.github/workflows/yamllint.yml new file mode 100644 index 0000000000..f0155a3c60 --- /dev/null +++ b/.github/workflows/yamllint.yml @@ -0,0 +1,39 @@ +name: Lint YAML +on: [push, pull_request] +jobs: + YAML: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@master + - name: 'Yamllint' + uses: karancode/yamllint-github-action@master + with: + yamllint_file_or_dir: '.' + yamllint_config_filepath: '.yamllint.yml' + + # If the above check failed, post a comment on the PR explaining the failure + - name: Post PR comment + if: failure() + uses: mshick/add-pr-comment@v1 + with: + message: | + ## YAML linting is failing + + To keep the code consistent with lots of contributors, we run automated code consistency checks. + To fix this CI test, please run: + + * Install `yamllint` + * Install `yamllint` following [this](https://yamllint.readthedocs.io/en/stable/quickstart.html#installing-yamllint) + instructions or alternative install it in your [conda environment](https://anaconda.org/conda-forge/yamllint) + * Fix the markdown errors + * Run the test locally: `yamllint $(find . -type f -name "*.yml" -o -name "*.yaml") -c ./.yamllint.yml` + * Fix any reported errors in your YAML files + + Once you push these changes the test should pass, and you can hide this comment :+1: + + We highly recommend setting up yaml-lint in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + + Thanks again for your contribution! + repo-token: ${{ secrets.GITHUB_TOKEN }} + allow-repeats: false diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 0000000000..804eb092df --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,17 @@ +image: nfcore/gitpod:latest +tasks: + - name: install current state of nf-core/tools + command: python -m pip install --upgrade -r requirements-dev.txt -e . +vscode: + extensions: # based on nf-core.nf-core-extensionpack + - codezombiech.gitignore # Language support for .gitignore files + # - cssho.vscode-svgviewer # SVG viewer + - davidanson.vscode-markdownlint # Markdown/CommonMark linting and style checking for Visual Studio Code + - eamodio.gitlens # Quickly glimpse into whom, why, and when a line or code block was changed + - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files + - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar + - mechatroner.rainbow-csv # Highlight columns in csv files in different colors + # - nextflow.nextflow # Nextflow syntax highlighting + - oderwat.indent-rainbow # Highlight indentation level + - streetsidesoftware.code-spell-checker # Spelling checker for source code + - DavidAnson.vscode-markdownlint # Linter for markdown files diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 0000000000..05d5352339 --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,7 @@ +extends: default +ignore: | + nf_core/*-template/ +rules: + document-start: disable + line-length: disable + truthy: disable diff --git a/CHANGELOG.md b/CHANGELOG.md index 93ec4fa4a6..4f7a5ddd8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,64 @@ # nf-core/tools: Changelog +## [v2.3 - Mercury Vulture](https://github.com/nf-core/tools/releases/tag/2.3) - [2022-03-15] + +### Template + +* Removed mention of `--singularity_pull_docker_container` in pipeline `README.md` +* Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode. +* Add retry strategy for AWS megatests after releasing [nf-core/tower-action v2.2](https://github.com/nf-core/tower-action/releases/tag/v2.2) +* Added `.nf-core.yml` file with `repository_type: pipeline` for modules commands +* Update igenomes path to the `BWAIndex` to fetch the whole `version0.6.0` folder instead of only the `genome.fa` file +* Remove pinned Node version in the GitHub Actions workflows, to fix errors with `markdownlint` +* Bumped `nf-core/tower-action` to `v3` and removed `pipeline` and `revision` from the AWS workflows, which were not needed +* Add yamllint GitHub Action. +* Add `.yamllint.yml` to avoid line length and document start errors ([#1407](https://github.com/nf-core/tools/issues/1407)) +* Add `--publish_dir_mode` back into the pipeline template ([nf-core/rnaseq#752](https://github.com/nf-core/rnaseq/issues/752#issuecomment-1039451607)) +* Add optional loading of of pipeline-specific institutional configs to `nextflow.config` +* Make `--outdir` a mandatory parameter ([nf-core/tools#1415](https://github.com/nf-core/tools/issues/1415)) + +## General + +* Updated `nf-core download` to work with latest DSL2 syntax for containers ([#1379](https://github.com/nf-core/tools/issues/1379)) +* Made `nf-core modules create` detect repository type with explicit `.nf-core.yml` instead of random readme stuff ([#1391](https://github.com/nf-core/tools/pull/1391)) +* Added a Gitpod environment and Dockerfile ([#1384](https://github.com/nf-core/tools/pull/1384)) + * Adds conda, Nextflow, nf-core, pytest-workflow, mamba, and pip to base Gitpod Docker image. + * Adds GH action to build and push Gitpod Docker image. + * Adds Gitpod environment to template. + * Adds Gitpod environment to tools with auto build of nf-core tool. +* Shiny new command-line help formatting ([#1403](https://github.com/nf-core/tools/pull/1403)) +* Call the command line help with `-h` as well as `--help` (was formerly just the latter) ([#1404](https://github.com/nf-core/tools/pull/1404)) +* Add `.yamllint.yml` config file to avoid line length and document start errors in the tools repo itself. +* Switch to `yamllint-github-action`to be able to configure yaml lint exceptions ([#1404](https://github.com/nf-core/tools/issues/1413)) +* Prevent module linting KeyError edge case ([#1321](https://github.com/nf-core/tools/issues/1321)) +* Bump-versions: Don't trim the trailing newline on files, causes editorconfig linting to fail ([#1265](https://github.com/nf-core/tools/issues/1265)) +* Handle exception in `nf-core list` when a broken git repo is found ([#1273](https://github.com/nf-core/tools/issues/1273)) +* Updated URL for pipeline lint test docs ([#1348](https://github.com/nf-core/tools/issues/1348)) +* Updated `nf-core create` to tolerate failures and retry when fetching pipeline logos from the website ([#1369](https://github.com/nf-core/tools/issues/1369)) + +### Modules + +* New command `nf-core modules info` that prints nice documentation about a module to the terminal :sparkles: ([#1427](https://github.com/nf-core/tools/issues/1427)) +* Linting a pipeline now fails instead of warning if a local copy of a module does not match the remote ([#1313](https://github.com/nf-core/tools/issues/1313)) +* Fixed linting bugs where warning was incorrectly generated for: + * `Module does not emit software version` + * `Container versions do not match` + * `input:` / `output:` not being specified in module + * Allow for containers from other biocontainers resource as defined [here](https://github.com/nf-core/modules/blob/cde237e7cec07798e5754b72aeca44efe89fc6db/modules/cat/fastq/main.nf#L7-L8) +* Fixed traceback when using `stageAs` syntax as defined [here](https://github.com/nf-core/modules/blob/cde237e7cec07798e5754b72aeca44efe89fc6db/modules/cat/fastq/main.nf#L11) +* Added `nf-core schema docs` command to output pipline parameter documentation in Markdown format for inclusion in GitHub and other documentation systems ([#741](https://github.com/nf-core/tools/issues/741)) +* Allow conditional process execution from the configuration file ([#1393](https://github.com/nf-core/tools/pull/1393)) +* Add linting for when condition([#1397](https://github.com/nf-core/tools/pull/1397)) +* Added modules ignored table to `nf-core modules bump-versions`. ([#1234](https://github.com/nf-core/tools/issues/1234)) +* Added `--conda-package-version` flag for specifying version of conda package in `nf-core modules create`. ([#1238](https://github.com/nf-core/tools/issues/1238)) +* Add option of writing diffs to file in `nf-core modules update` using either interactive prompts or the new `--diff-file` flag. +* Fixed edge case where module names that were substrings of other modules caused both to be installed ([#1380](https://github.com/nf-core/tools/issues/1380)) +* Tweak handling of empty files when generating the test YAML ([#1376](https://github.com/nf-core/tools/issues/1376)) + * Fail linting if a md5sum for an empty file is found (instead of a warning) + * Don't skip the md5 when generating a test file if an empty file is found (so that linting fails and can be manually checked) +* Linting checks test files for `TODO` statements as well as the main module code ([#1271](https://github.com/nf-core/tools/issues/1271)) +* Handle error if `manifest` isn't set in `nextflow.config` ([#1418](https://github.com/nf-core/tools/issues/1418)) + ## [v2.2 - Lead Liger](https://github.com/nf-core/tools/releases/tag/2.2) - [2021-12-14] ### Template @@ -51,7 +110,6 @@ * Check if README is from modules repo * Update module template to DSL2 v2.0 (remove `functions.nf` from modules template and updating `main.nf` ([#1289](https://github.com/nf-core/tools/pull/)) * Substitute get process/module name custom functions in module `main.nf` using template replacement ([#1284](https://github.com/nf-core/tools/issues/1284)) -* Linting now fails instead of warning if a local copy of a module does not match the remote ([#1313](https://github.com/nf-core/tools/issues/1313)) * Check test YML file for md5sums corresponding to empty files ([#1302](https://github.com/nf-core/tools/issues/1302)) * Exit with an error if empty files are found when generating the test YAML file ([#1302](https://github.com/nf-core/tools/issues/1302)) diff --git a/README.md b/README.md index 65fb66b004..a418150e26 100644 --- a/README.md +++ b/README.md @@ -925,7 +925,7 @@ github.com: git_protocol: ``` -The easiest way to create this configuration file is through *GitHub CLI*: follow +The easiest way to create this configuration file is through _GitHub CLI_: follow its [installation instructions](https://cli.github.com/manual/installation) and then call: @@ -943,7 +943,7 @@ to get more information. The `nf-core modules list` command provides the subcommands `remote` and `local` for listing modules installed in a remote repository and in the local pipeline respectively. Both subcommands come with the `--key ` option for filtering the modules by keywords. -### List remote modules +#### List remote modules To list all modules available on [nf-core/modules](https://github.com/nf-core/modules), you can use `nf-core modules list remote`, which will print all available modules to the terminal. @@ -975,7 +975,7 @@ INFO Modules available from nf-core/modules (master) └────────────────────────────────┘ ``` -### List installed modules +#### List installed modules To list modules installed in a local pipeline directory you can use `nf-core modules list local`. This will list the modules install in the current working directory by default. If you want to specify another directory, use the `--dir ` flag. @@ -1000,6 +1000,54 @@ INFO Modules installed in '.': └─────────────┴─────────────────┴─────────────┴────────────────────────────────────────────────────────┴────────────┘ ``` +## Show information about a module + +For quick help about how a module works, use `nf-core modules info `. +This shows documentation about the module on the command line, similar to what's available on the +[nf-core website](https://nf-co.re/modules). + +```console +$ nf-core modules info fastqc + + ,--./,-. + ___ __ __ __ ___ /,-._.--~\ + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' + + nf-core/tools version 2.3.dev0 - https://nf-co.re + + +╭─ Module: fastqc ───────────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: nf-core/modules │ +│ 🔧 Tools: fastqc │ +│ 📖 Description: Run FastQC on sequenced reads │ +╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + ╷ ╷ + 📥 Inputs │Description │Pattern +╺━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━╸ + meta (map) │Groovy Map containing sample information e.g. [ id:'test', single_end:false ] │ +╶──────────────┼──────────────────────────────────────────────────────────────────────────────────┼───────╴ + reads (file)│List of input FastQ files of size 1 and 2 for single-end and paired-end data, │ + │respectively. │ + ╵ ╵ + ╷ ╷ + 📤 Outputs │Description │ Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━╸ + meta (map) │Groovy Map containing sample information e.g. [ id:'test', │ + │single_end:false ] │ +╶─────────────────┼───────────────────────────────────────────────────────────────────────┼───────────────╴ + html (file) │FastQC report │*_{fastqc.html} +╶─────────────────┼───────────────────────────────────────────────────────────────────────┼───────────────╴ + zip (file) │FastQC report archive │ *_{fastqc.zip} +╶─────────────────┼───────────────────────────────────────────────────────────────────────┼───────────────╴ + versions (file)│File containing software versions │ versions.yml + ╵ ╵ + + 💻 Installation command: nf-core modules install fastqc + +``` + ### Install modules in a pipeline You can install modules from [nf-core/modules](https://github.com/nf-core/modules) in your pipeline using `nf-core modules install`. @@ -1055,6 +1103,7 @@ There are five additional flags that you can use with this command: * `--prompt`: Select the module version using a cli prompt. * `--sha `: Install the module at a specific commit from the `nf-core/modules` repository. * `--diff`: Show the diff between the installed files and the new version before installing. +* `--diff-file `: Specify where the diffs between the local and remote versions of a module should be written * `--all`: Use this flag to run the command on all modules in the pipeline. If you don't want to update certain modules or want to update them to specific versions, you can make use of the `.nf-core.yml` configuration file. For example, you can prevent the `star/align` module installed from `nf-core/modules` from being updated by adding the following to the `.nf-core.yml` file: @@ -1116,11 +1165,15 @@ This command creates a new nf-core module from the nf-core module template. This ensures that your module follows the nf-core guidelines. The template contains extensive `TODO` messages to walk you through the changes you need to make to the template. -You can create a new module using `nf-core modules create`. This will create the new module in the current working directory. To specify another directory, use `--dir `. +You can create a new module using `nf-core modules create`. -If writing a module for the shared [nf-core/modules](https://github.com/nf-core/modules) repository, the `` argument should be the path to the clone of your fork of the modules repository. +This command can be used both when writing a module for the shared [nf-core/modules](https://github.com/nf-core/modules) repository, +and also when creating local modules for a pipeline. -Alternatively, if writing a more niche module that does not make sense to share, `` should be the path to your pipeline. +Which type of repository you are working in is detected by the `repository_type` flag in a `.nf-core.yml` file in the root directory, +set to either `pipeline` or `modules`. +The command will automatically look through parent directories for this file to set the root path, so that you can run the command in a subdirectory. +It will start in the current working directory, or whatever is specified with `--dir `. The `nf-core modules create` command will prompt you with the relevant questions in order to create all of the necessary module files. @@ -1146,7 +1199,7 @@ INFO Provide an appropriate resource label for the process, taken from the n ? Process resource label: process_high INFO Where applicable all sample-specific information e.g. 'id', 'single_end', 'read_group' MUST be provided as an input via a Groovy Map called 'meta'. This information may not be required in some instances, for example indexing reference genome files. -Will the module require a meta map of sample information? (yes/no) [y/n] (y): y +Will the module require a meta map of sample information? [y/n] (y): y INFO Created / edited following files: ./software/star/align/main.nf ./software/star/align/meta.yml diff --git a/nf_core/__main__.py b/nf_core/__main__.py index a9500509a4..7ce4a18a06 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1,18 +1,14 @@ #!/usr/bin/env python """ nf-core: Helper tools for use with nf-core Nextflow pipelines. """ -from nf_core.lint.pipeline_todos import pipeline_todos -from nf_core.modules.bump_versions import ModuleVersionBumper -from click.types import File from rich import print -from rich.prompt import Confirm -import click import logging import os import re import rich.console import rich.logging import rich.traceback +import rich_click as click import sys import nf_core @@ -32,6 +28,35 @@ # Submodules should all traverse back to this log = logging.getLogger() +# Set up nicer formatting of click cli help messages +click.rich_click.MAX_WIDTH = 100 +click.rich_click.USE_RICH_MARKUP = True +click.rich_click.COMMAND_GROUPS = { + "nf-core": [ + { + "name": "Commands for users", + "commands": ["list", "launch", "download", "licences"], + }, + { + "name": "Commands for developers", + "commands": ["create", "lint", "modules", "schema", "bump-version", "sync"], + }, + ], + "nf-core modules": [ + { + "name": "For pipelines", + "commands": ["list", "info", "install", "update", "remove"], + }, + { + "name": "Developing new modules", + "commands": ["create", "create-test-yml", "lint", "bump-versions"], + }, + ], +} +click.rich_click.OPTION_GROUPS = { + "nf-core modules list local": [{"options": ["--dir", "--json", "--help"]}], +} + def run_nf_core(): # Set up rich stderr console @@ -46,7 +71,10 @@ def run_nf_core(): stderr.print("[blue] |\ | |__ __ / ` / \ |__) |__ [yellow] } {", highlight=False) stderr.print("[blue] | \| | \__, \__/ | \ |___ [green]\`-._,-`-,", highlight=False) stderr.print("[green] `._,._,'\n", highlight=False) - stderr.print("[grey39] nf-core/tools version {}".format(nf_core.__version__), highlight=False) + stderr.print( + f"[grey39] nf-core/tools version {nf_core.__version__} - [link=https://nf-co.re]https://nf-co.re[/]", + highlight=False, + ) try: is_outdated, current_vers, remote_vers = nf_core.utils.check_if_outdated() if is_outdated: @@ -56,63 +84,22 @@ def run_nf_core(): ) except Exception as e: log.debug("Could not check latest version: {}".format(e)) - stderr.print("\n\n") + stderr.print("\n") # Lanch the click cli nf_core_cli() -# Customise the order of subcommands for --help -# https://stackoverflow.com/a/47984810/713980 -class CustomHelpOrder(click.Group): - def __init__(self, *args, **kwargs): - self.help_priorities = {} - super(CustomHelpOrder, self).__init__(*args, **kwargs) - - def get_help(self, ctx): - self.list_commands = self.list_commands_for_help - return super(CustomHelpOrder, self).get_help(ctx) - - def list_commands_for_help(self, ctx): - """reorder the list of commands when listing the help""" - commands = super(CustomHelpOrder, self).list_commands(ctx) - return (c[1] for c in sorted((self.help_priorities.get(command, 1000), command) for command in commands)) - - def command(self, *args, **kwargs): - """Behaves the same as `click.Group.command()` except capture - a priority for listing command names in help. - """ - help_priority = kwargs.pop("help_priority", 1000) - help_priorities = self.help_priorities - - def decorator(f): - cmd = super(CustomHelpOrder, self).command(*args, **kwargs)(f) - help_priorities[cmd.name] = help_priority - return cmd - - return decorator - - def group(self, *args, **kwargs): - """Behaves the same as `click.Group.group()` except capture - a priority for listing command names in help. - """ - help_priority = kwargs.pop("help_priority", 1000) - help_priorities = self.help_priorities - - def decorator(f): - cmd = super(CustomHelpOrder, self).command(*args, **kwargs)(f) - help_priorities[cmd.name] = help_priority - return cmd - - return decorator - - -@click.group(cls=CustomHelpOrder) +@click.group(context_settings=dict(help_option_names=["-h", "--help"])) @click.version_option(nf_core.__version__) @click.option("-v", "--verbose", is_flag=True, default=False, help="Print verbose output to the console.") @click.option("-l", "--log-file", help="Save a verbose log to a file.", metavar="") def nf_core_cli(verbose, log_file): + """ + nf-core/tools provides a set of helper tools for use with nf-core Nextflow pipelines. + It is designed for both end-users running pipelines and also developers creating new pipelines. + """ # Set the base logger to output DEBUG log.setLevel(logging.DEBUG) @@ -135,7 +122,7 @@ def nf_core_cli(verbose, log_file): # nf-core list -@nf_core_cli.command(help_priority=1) +@nf_core_cli.command() @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option( "-s", @@ -157,7 +144,7 @@ def list(keywords, sort, json, show_archived): # nf-core launch -@nf_core_cli.command(help_priority=2) +@nf_core_cli.command() @click.argument("pipeline", required=False, metavar="") @click.option("-r", "--revision", help="Release/branch/SHA of the project to run (if remote)") @click.option("-i", "--id", help="ID for web-gui launch parameter set") @@ -178,10 +165,10 @@ def list(keywords, sort, json, show_archived): "-a", "--save-all", is_flag=True, default=False, help="Save all parameters, even if unchanged from default" ) @click.option( - "-h", "--show-hidden", is_flag=True, default=False, help="Show hidden params which don't normally need changing" + "-x", "--show-hidden", is_flag=True, default=False, help="Show hidden params which don't normally need changing" ) @click.option( - "--url", type=str, default="https://nf-co.re/launch", help="Customise the builder URL (for development work)" + "-u", "--url", type=str, default="https://nf-co.re/launch", help="Customise the builder URL (for development work)" ) def launch(pipeline, id, revision, command_only, params_in, params_out, save_all, show_hidden, url): """ @@ -205,9 +192,7 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all # nf-core download - - -@nf_core_cli.command(help_priority=3) +@nf_core_cli.command() @click.argument("pipeline", required=False, metavar="") @click.option("-r", "--revision", type=str, help="Pipeline release") @click.option("-o", "--outdir", type=str, help="Output directory") @@ -237,7 +222,7 @@ def download(pipeline, revision, outdir, compress, force, container, singularity # nf-core licences -@nf_core_cli.command(help_priority=4) +@nf_core_cli.command() @click.argument("pipeline", required=True, metavar="") @click.option("--json", is_flag=True, default=False, help="Print output in JSON") def licences(pipeline, json): @@ -257,28 +242,27 @@ def licences(pipeline, json): sys.exit(1) -# nf-core create def validate_wf_name_prompt(ctx, opts, value): """Force the workflow name to meet the nf-core requirements""" if not re.match(r"^[a-z]+$", value): - click.echo("Invalid workflow name: must be lowercase without punctuation.") + log.error("[red]Invalid workflow name: must be lowercase without punctuation.") value = click.prompt(opts.prompt) return validate_wf_name_prompt(ctx, opts, value) return value -@nf_core_cli.command(help_priority=5) +# nf-core create +@nf_core_cli.command() @click.option( "-n", "--name", prompt="Workflow Name", - required=True, callback=validate_wf_name_prompt, type=str, help="The name of your new pipeline", ) -@click.option("-d", "--description", prompt=True, required=True, type=str, help="A short description of your pipeline") -@click.option("-a", "--author", prompt=True, required=True, type=str, help="Name of the main author(s)") +@click.option("-d", "--description", prompt=True, type=str, help="A short description of your pipeline") +@click.option("-a", "--author", prompt=True, type=str, help="Name of the main author(s)") @click.option("--version", type=str, default="1.0dev", help="The initial version number to use") @click.option("--no-git", is_flag=True, default=False, help="Do not initialise pipeline as new git repository") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output directory if it already exists") @@ -294,8 +278,15 @@ def create(name, description, author, version, no_git, force, outdir): create_obj.init_pipeline() -@nf_core_cli.command(help_priority=6) -@click.option("-d", "--dir", type=click.Path(exists=True), default=".", help="Pipeline directory. Defaults to CWD") +# nf-core lint +@nf_core_cli.command() +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help="Pipeline directory [dim]\[default: current working directory][/]", +) @click.option( "--release", is_flag=True, @@ -318,9 +309,9 @@ def lint(dir, release, fix, key, show_passed, fail_ignored, markdown, json): Runs a large number of automated tests to ensure that the supplied pipeline meets the nf-core guidelines. Documentation of all lint tests can be found - on the nf-core website: https://nf-co.re/tools-docs/ + on the nf-core website: [link=https://nf-co.re/tools-docs/]https://nf-co.re/tools-docs/[/] - You can ignore tests using a file called .nf-core-lint.yaml (if you have a good reason!). + You can ignore tests using a file called [blue].nf-core-lint.yaml[/] [i](if you have a good reason!)[/]. See the documentation for details. """ @@ -346,8 +337,8 @@ def lint(dir, release, fix, key, show_passed, fail_ignored, markdown, json): sys.exit(1) -## nf-core module subcommands -@nf_core_cli.group(cls=CustomHelpOrder, help_priority=7) +# nf-core modules subcommands +@nf_core_cli.group() @click.option( "-g", "--github-repository", @@ -359,7 +350,7 @@ def lint(dir, release, fix, key, show_passed, fail_ignored, markdown, json): @click.pass_context def modules(ctx, github_repository, branch): """ - Tools to manage Nextflow DSL2 modules as hosted on nf-core/modules. + Commands to manage Nextflow DSL2 modules (tool wrappers). """ # ensure that ctx.obj exists and is a dict (in case `cli()` is called # by means other than the `if` block below) @@ -373,22 +364,24 @@ def modules(ctx, github_repository, branch): sys.exit(1) -@modules.group(cls=CustomHelpOrder, help_priority=1) +# nf-core modules list subcommands +@modules.group() @click.pass_context def list(ctx): """ - List modules in a local pipeline or remote repo e.g nf-core/modules. + List modules in a local pipeline or remote repository. """ pass -@list.command(help_priority=1) +# nf-core modules list remote +@list.command() @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") def remote(ctx, keywords, json): """ - List all modules in a remote GitHub repo e.g. nf-core/modules + List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ try: module_list = nf_core.modules.ModuleList(None, remote=True) @@ -399,14 +392,21 @@ def remote(ctx, keywords, json): sys.exit(1) -@list.command(help_priority=2) +# nf-core modules list local +@list.command() @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") -@click.option("-d", "--dir", type=click.Path(exists=True), default=".", help="Pipeline directory. Defaults to CWD") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help="Pipeline directory. [dim]\[default: Current working directory][/]", +) def local(ctx, keywords, json, dir): """ - List all modules installed locally in a pipeline + List modules installed locally in a pipeline """ try: module_list = nf_core.modules.ModuleList(dir, remote=False) @@ -417,10 +417,17 @@ def local(ctx, keywords, json, dir): sys.exit(1) -@modules.command(help_priority=2) +# nf-core modules install +@modules.command() @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") -@click.option("-d", "--dir", type=click.Path(exists=True), default=".", help="Pipeline directory. Defaults to CWD") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help="Pipeline directory. [dim]\[default: current working directory][/]", +) @click.option("-p", "--prompt", is_flag=True, default=False, help="Prompt for the version of the module") @click.option("-f", "--force", is_flag=True, default=False, help="Force reinstallation of module if it already exists") @click.option("-s", "--sha", type=str, metavar="", help="Install module at commit SHA") @@ -441,18 +448,30 @@ def install(ctx, tool, dir, prompt, force, sha): sys.exit(1) -@modules.command(help_priority=3) +# nf-core modules update +@modules.command() @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") -@click.option("-d", "--dir", type=click.Path(exists=True), default=".", help="Pipeline directory. Defaults to CWD") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help="Pipeline directory. [dim]\[default: current working directory][/]", +) @click.option("-f", "--force", is_flag=True, default=False, help="Force update of module") @click.option("-p", "--prompt", is_flag=True, default=False, help="Prompt for the version of the module") @click.option("-s", "--sha", type=str, metavar="", help="Install module at commit SHA") @click.option("-a", "--all", is_flag=True, default=False, help="Update all modules installed in pipeline") @click.option( - "-c", "--diff", is_flag=True, default=False, help="Show differences between module versions before updating" + "-x", + "--save-diff", + type=str, + metavar="", + default=None, + help="Save diffs to a file instead of updating in place", ) -def update(ctx, tool, dir, force, prompt, sha, all, diff): +def update(ctx, tool, dir, force, prompt, sha, all, save_diff): """ Update DSL2 modules within a pipeline. @@ -460,7 +479,7 @@ def update(ctx, tool, dir, force, prompt, sha, all, diff): """ try: module_install = nf_core.modules.ModuleUpdate( - dir, force=force, prompt=prompt, sha=sha, update_all=all, diff=diff + dir, force=force, prompt=prompt, sha=sha, update_all=all, save_diff_fn=save_diff ) module_install.modules_repo = ctx.obj["modules_repo_obj"] exit_status = module_install.update(tool) @@ -471,10 +490,17 @@ def update(ctx, tool, dir, force, prompt, sha, all, diff): sys.exit(1) -@modules.command(help_priority=4) +# nf-core modules remove +@modules.command() @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") -@click.option("-d", "--dir", type=click.Path(exists=True), default=".", help="Pipeline directory. Defaults to CWD") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help="Pipeline directory. [dim]\[default: current working directory][/]", +) def remove(ctx, dir, tool): """ Remove a module from a pipeline. @@ -488,7 +514,8 @@ def remove(ctx, dir, tool): sys.exit(1) -@modules.command("create", help_priority=5) +# nf-core modules create +@modules.command("create") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @@ -498,7 +525,8 @@ def remove(ctx, dir, tool): @click.option("-n", "--no-meta", is_flag=True, default=False, help="Don't use meta map for sample information") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") @click.option("-c", "--conda-name", type=str, default=None, help="Name of the conda package to use") -def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name): +@click.option("-p", "--conda-package-version", type=str, default=None, help="Version of conda package to use") +def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version): """ Create a new DSL2 module from the nf-core template. @@ -519,14 +547,17 @@ def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_nam # Run function try: - module_create = nf_core.modules.ModuleCreate(dir, tool, author, label, has_meta, force, conda_name) + module_create = nf_core.modules.ModuleCreate( + dir, tool, author, label, has_meta, force, conda_name, conda_package_version + ) module_create.create() except UserWarning as e: log.critical(e) sys.exit(1) -@modules.command("create-test-yml", help_priority=6) +# nf-core modules create-test-yml +@modules.command("create-test-yml") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option("-t", "--run-tests", is_flag=True, default=False, help="Run the test workflows") @@ -548,7 +579,8 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): sys.exit(1) -@modules.command(help_priority=7) +# nf-core modules lint +@modules.command() @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @@ -580,7 +612,40 @@ def lint(ctx, tool, dir, key, all, local, passed): sys.exit(1) -@modules.command(help_priority=8) +# nf-core modules info +@modules.command() +@click.pass_context +@click.argument("tool", type=str, required=False, metavar=" or ") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help="Pipeline directory. [dim]\[default: Current working directory][/]", +) +def info(ctx, tool, dir): + """ + Show developer usage information about a given module. + + Parses information from a module's [i]meta.yml[/] and renders help + on the command line. A handy equivalent to searching the + [link=https://nf-co.re/modules]nf-core website[/]. + + If run from a pipeline and a local copy of the module is found, the command + will print this usage info. + If not, usage from the remote modules repo will be shown. + """ + try: + module_info = nf_core.modules.ModuleInfo(dir, tool) + module_info.modules_repo = ctx.obj["modules_repo_obj"] + print(module_info.get_module_info()) + except UserWarning as e: + log.error(e) + sys.exit(1) + + +# nf-core modules bump-versions +@modules.command() @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @@ -592,7 +657,7 @@ def bump_versions(ctx, tool, dir, all, show_all): the nf-core/modules repo. """ try: - version_bumper = ModuleVersionBumper(pipeline_dir=dir) + version_bumper = nf_core.modules.bump_versions.ModuleVersionBumper(pipeline_dir=dir) version_bumper.bump_versions(module=tool, all_modules=all, show_uptodate=show_all) except nf_core.modules.module_utils.ModuleException as e: log.error(e) @@ -602,8 +667,8 @@ def bump_versions(ctx, tool, dir, all, show_all): sys.exit(1) -## nf-core schema subcommands -@nf_core_cli.group(cls=CustomHelpOrder, help_priority=7) +# nf-core schema subcommands +@nf_core_cli.group() def schema(): """ Suite of tools for developers to manage pipeline schema. @@ -614,7 +679,8 @@ def schema(): pass -@schema.command(help_priority=1) +# nf-core schema validate +@schema.command() @click.argument("pipeline", required=True, metavar="") @click.argument("params", type=click.Path(exists=True), required=True, metavar="") def validate(pipeline, params): @@ -642,8 +708,15 @@ def validate(pipeline, params): sys.exit(1) -@schema.command(help_priority=2) -@click.option("-d", "--dir", type=click.Path(exists=True), default=".", help="Pipeline directory. Defaults to CWD") +# nf-core schema build +@schema.command() +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help="Pipeline directory. [dim]\[default: current working directory][/]", +) @click.option("--no-prompts", is_flag=True, help="Do not confirm changes, just update parameters and exit") @click.option("--web-only", is_flag=True, help="Skip building using Nextflow config, just launch the web tool") @click.option( @@ -673,7 +746,8 @@ def build(dir, no_prompts, web_only, url): sys.exit(1) -@schema.command(help_priority=3) +# nf-core schema lint +@schema.command() @click.argument("schema_path", type=click.Path(exists=True), required=True, metavar="") def lint(schema_path): """ @@ -698,9 +772,53 @@ def lint(schema_path): sys.exit(1) -@nf_core_cli.command("bump-version", help_priority=9) +@schema.command() +@click.argument("schema_path", type=click.Path(exists=True), required=False, metavar="") +@click.option("-o", "--output", type=str, metavar="", help="Output filename. Defaults to standard out.") +@click.option( + "-x", "--format", type=click.Choice(["markdown", "html"]), default="markdown", help="Format to output docs in." +) +@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") +@click.option( + "-c", + "--columns", + type=str, + metavar="", + help="CSV list of columns to include in the parameter tables (parameter,description,type,default,required,hidden)", + default="parameter,description,type,default,required,hidden", +) +def docs(schema_path, output, format, force, columns): + """ + Outputs parameter documentation for a pipeline schema. + """ + schema_obj = nf_core.schema.PipelineSchema() + try: + # Assume we're in a pipeline dir root if schema path not set + if schema_path is None: + schema_path = "nextflow_schema.json" + assert os.path.exists( + schema_path + ), "Could not find 'nextflow_schema.json' in current directory. Please specify a path." + schema_obj.get_schema_path(schema_path) + schema_obj.load_schema() + docs = schema_obj.print_documentation(output, format, force, columns.split(",")) + if not output: + print(docs) + except AssertionError as e: + log.error(e) + sys.exit(1) + + +# nf-core bump-version +@nf_core_cli.command("bump-version") @click.argument("new_version", required=True, metavar="") -@click.option("-d", "--dir", type=click.Path(exists=True), default=".", help="Pipeline directory. Defaults to CWD") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help="Pipeline directory. [dim]\[default: current working directory][/]", +) @click.option( "-n", "--nextflow", is_flag=True, default=False, help="Bump required nextflow version instead of pipeline version" ) @@ -735,24 +853,31 @@ def bump_version(new_version, dir, nextflow): sys.exit(1) -@nf_core_cli.command("sync", help_priority=10) -@click.option("-d", "--dir", type=click.Path(exists=True), default=".", help="Pipeline directory. Defaults to CWD") -@click.option("-b", "--from-branch", type=str, help="The git branch to use to fetch workflow vars.") +# nf-core sync +@nf_core_cli.command("sync") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help="Pipeline directory. [dim]\[default: current working directory][/]", +) +@click.option("-b", "--from-branch", type=str, help="The git branch to use to fetch workflow variables.") @click.option("-p", "--pull-request", is_flag=True, default=False, help="Make a GitHub pull-request with the changes.") @click.option("-g", "--github-repository", type=str, help="GitHub PR: target repository.") @click.option("-u", "--username", type=str, help="GitHub PR: auth username.") def sync(dir, from_branch, pull_request, github_repository, username): """ - Sync a pipeline TEMPLATE branch with the nf-core template. + Sync a pipeline [cyan i]TEMPLATE[/] branch with the nf-core template. To keep nf-core pipelines up to date with improvements in the main template, we use a method of synchronisation that uses a special - git branch called TEMPLATE. + git branch called [cyan i]TEMPLATE[/]. - This command updates the TEMPLATE branch with the latest version of + This command updates the [cyan i]TEMPLATE[/] branch with the latest version of the nf-core template, so that these updates can be synchronised with the pipeline. It is run automatically for all pipelines when ever a - new release of nf-core/tools (and the included template) is made. + new release of [link=https://github.com/nf-core/tools]nf-core/tools[/link] (and the included template) is made. """ # Check if pipeline directory contains necessary files try: @@ -769,5 +894,6 @@ def sync(dir, from_branch, pull_request, github_repository, username): sys.exit(1) +# Main script is being run - launch the CLI if __name__ == "__main__": run_nf_core() diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 94bc17b66c..420fd24e7f 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -158,7 +158,7 @@ def update_file_version(filename, pipeline_obj, patterns): newcontent.append(line) if found_match: - content = "\n".join(newcontent) + content = "\n".join(newcontent) + "\n" else: log.error("Could not find version number in {}: '{}'".format(filename, pattern)) diff --git a/nf_core/create.py b/nf_core/create.py index 02dd50a1bc..31404a241d 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -4,13 +4,16 @@ """ from genericpath import exists import git +import imghdr import jinja2 import logging import os import pathlib +import random import requests import shutil import sys +import time import nf_core @@ -149,18 +152,52 @@ def make_pipeline_logo(self): log.debug(f"Fetching logo from {logo_url}") email_logo_path = f"{self.outdir}/assets/{self.name_noslash}_logo_light.png" - os.makedirs(os.path.dirname(email_logo_path), exist_ok=True) - log.debug(f"Writing logo to '{email_logo_path}'") - r = requests.get(f"{logo_url}&w=400") - with open(email_logo_path, "wb") as fh: - fh.write(r.content) + self.download_pipeline_logo(f"{logo_url}&w=400", email_logo_path) for theme in ["dark", "light"]: + readme_logo_url = f"{logo_url}?w=600&theme={theme}" readme_logo_path = f"{self.outdir}/docs/images/{self.name_noslash}_logo_{theme}.png" - log.debug(f"Writing logo to '{readme_logo_path}'") - os.makedirs(os.path.dirname(readme_logo_path), exist_ok=True) - r = requests.get(f"{logo_url}?w=600&theme={theme}") - with open(readme_logo_path, "wb") as fh: + self.download_pipeline_logo(readme_logo_url, readme_logo_path) + + def download_pipeline_logo(self, url, img_fn): + """Attempt to download a logo from the website. Retry if it fails.""" + os.makedirs(os.path.dirname(img_fn), exist_ok=True) + attempt = 0 + max_attempts = 10 + retry_delay = 0 # x up to 10 each time, so first delay will be 1-100 seconds + while attempt < max_attempts: + # If retrying, wait a while + if retry_delay > 0: + log.info(f"Waiting {retry_delay} seconds before next image fetch attempt") + time.sleep(retry_delay) + + attempt += 1 + # Use a random number to avoid the template sync hitting the website simultaneously for all pipelines + retry_delay = random.randint(1, 100) * attempt + log.debug(f"Fetching logo '{img_fn}' (attempt {attempt})") + try: + # Try to fetch the logo from the website + r = requests.get(url, timeout=180) + if r.status_code != 200: + raise UserWarning(f"Got status code {r.status_code}") + # Check that the returned image looks right + + except (ConnectionError, UserWarning) as e: + # Something went wrong - try again + log.warning(e) + log.error(f"Connection error - retrying") + continue + + # Write the new logo to the file + with open(img_fn, "wb") as fh: fh.write(r.content) + # Check that the file looks valid + image_type = imghdr.what(img_fn) + if image_type != "png": + log.error(f"Logo from the website didn't look like an image: '{image_type}'") + continue + + # Got this far, presumably it's good - break the retry loop + break def git_init_pipeline(self): """Initialises the new pipeline as a Git repository and submits first commit.""" diff --git a/nf_core/download.py b/nf_core/download.py index 7f03459cf6..f45e452526 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -416,10 +416,30 @@ def find_container_images(self): """Find container image names for workflow. Starts by using `nextflow config` to pull out any process.container - declarations. This works for DSL1. + declarations. This works for DSL1. It should return a simple string with resolved logic. Second, we look for DSL2 containers. These can't be found with `nextflow config` at the time of writing, so we scrape the pipeline files. + This returns raw source code that will likely need to be cleaned. + + If multiple containers are found, prioritise any prefixed with http for direct download. + + Example syntax: + + Early DSL2: + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0" + } else { + container "quay.io/biocontainers/fastqc:0.11.9--0" + } + + Later DSL2: + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : + 'quay.io/biocontainers/fastqc:0.11.9--0' }" + + DSL1 / Special case DSL2: + container "nfcore/cellranger:6.0.2" """ log.debug("Fetching container names for workflow") @@ -439,35 +459,36 @@ def find_container_images(self): if file.endswith(".nf"): with open(os.path.join(subdir, file), "r") as fh: # Look for any lines with `container = "xxx"` - matches = [] - for line in fh: - match = re.match(r"\s*container\s+[\"']([^\"']+)[\"']", line) - if match: - matches.append(match.group(1)) - - # If we have matches, save the first one that starts with http - for m in matches: - if m.startswith("http"): - containers_raw.append(m.strip('"').strip("'")) - break - # If we get here then we didn't call break - just save the first match - else: - if len(matches) > 0: - containers_raw.append(matches[0].strip('"').strip("'")) + this_container = None + contents = fh.read() + matches = re.findall(r"container\s*\"([^\"]*)\"", contents, re.S) + if matches: + for match in matches: + # Look for a http download URL. + # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/3809435/713980 + url_regex = r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)" + url_match = re.search(url_regex, match, re.S) + if url_match: + this_container = url_match.group(0) + break # Prioritise http, exit loop as soon as we find it + + # No https download, is the entire container string a docker URI? + else: + # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/39672069/713980 + docker_regex = r"^(?:(?=[^:\/]{1,253})(?!-)[a-zA-Z0-9-]{1,63}(? 1 else "")) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile new file mode 100644 index 0000000000..2171bd01fc --- /dev/null +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -0,0 +1,32 @@ +FROM gitpod/workspace-base + +USER root + +# Install Conda +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \ + rm Miniconda3-latest-Linux-x86_64.sh + +ENV PATH="/opt/conda/bin:$PATH" + +RUN chown -R gitpod:gitpod /opt/conda + +USER gitpod + +# Install nextflow, nf-core, Mamba, and pytest-workflow +RUN conda update -n base -c defaults conda && \ + conda config --add channels defaults && \ + conda config --add channels bioconda && \ + conda config --add channels conda-forge && \ + conda install \ + openjdk=11.0.13 \ + nextflow=21.10.6 \ + nf-core=2.2 \ + pytest-workflow=1.6.0 \ + mamba=0.22.1 \ + pip=22.0.4 \ + black=22.1.0 \ + yamllint=1.26.3 \ + -n base && \ + nextflow self-update && \ + conda clean --all -f -y diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 4838726928..14f3667e2c 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -12,15 +12,14 @@ import git import json import logging -import os import re import rich import rich.progress -import yaml import nf_core.utils import nf_core.lint_utils import nf_core.modules.lint +from nf_core import __version__ from nf_core.lint_utils import console log = logging.getLogger(__name__) @@ -331,7 +330,14 @@ def format_result(test_results, table): string for the terminal with appropriate ASCII colours. """ for eid, msg in test_results: - table.add_row(Markdown("[{0}](https://nf-co.re/tools-docs/lint_tests/{0}.html): {1}".format(eid, msg))) + tools_version = __version__ + if "dev" in __version__: + tools_version = "latest" + table.add_row( + Markdown( + f"[{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}.html): {msg}" + ) + ) return table def _s(some_list): diff --git a/nf_core/lint/pipeline_todos.py b/nf_core/lint/pipeline_todos.py index fba499c74e..d0d491b3af 100644 --- a/nf_core/lint/pipeline_todos.py +++ b/nf_core/lint/pipeline_todos.py @@ -8,7 +8,7 @@ log = logging.getLogger(__name__) -def pipeline_todos(self): +def pipeline_todos(self, root_dir=None): """Check for nf-core *TODO* lines. The nf-core workflow template contains a number of comment lines to help developers @@ -34,15 +34,19 @@ def pipeline_todos(self): """ passed = [] warned = [] - failed = [] file_paths = [] + # Pipelines don't provide a path, so use the workflow path. + # Modules run this function twice and provide a string path + if root_dir is None: + root_dir = self.wf_path + ignore = [".git"] - if os.path.isfile(os.path.join(self.wf_path, ".gitignore")): - with io.open(os.path.join(self.wf_path, ".gitignore"), "rt", encoding="latin1") as fh: + if os.path.isfile(os.path.join(root_dir, ".gitignore")): + with io.open(os.path.join(root_dir, ".gitignore"), "rt", encoding="latin1") as fh: for l in fh: ignore.append(os.path.basename(l.strip().rstrip("/"))) - for root, dirs, files in os.walk(self.wf_path, topdown=True): + for root, dirs, files in os.walk(root_dir, topdown=True): # Ignore files for i_base in ignore: i = os.path.join(root, i_base) @@ -65,6 +69,10 @@ def pipeline_todos(self): file_paths.append(os.path.join(root, fname)) except FileNotFoundError: log.debug(f"Could not open file {fname} in pipeline_todos lint test") + + if len(warned) == 0: + passed.append("No TODO strings found") + # HACK file paths are returned to allow usage of this function in modules/lint.py # Needs to be refactored! - return {"passed": passed, "warned": warned, "failed": failed, "file_paths": file_paths} + return {"passed": passed, "warned": warned, "file_paths": file_paths} diff --git a/nf_core/list.py b/nf_core/list.py index 5a94253d00..19b0d81e51 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -374,13 +374,13 @@ def get_local_nf_workflow_details(self): self.active_tag = str(tag) # I'm not sure that we need this any more, it predated the self.branch catch above for detacted HEAD - except TypeError as e: + except (TypeError, git.InvalidGitRepositoryError) as e: log.error( - "Could not fetch status of local Nextflow copy of {}:".format(self.full_name) - + "\n {}".format(str(e)) - + "\n\nIt's probably a good idea to delete this local copy and pull again:".format(self.local_path) - + "\n rm -rf {}".format(self.local_path) - + "\n nextflow pull {}".format(self.full_name) + f"Could not fetch status of local Nextflow copy of '{self.full_name}':" + f"\n [red]{type(e).__name__}:[/] {str(e)}" + "\n\nThis git repository looks broken. It's probably a good idea to delete this local copy and pull again:" + f"\n [magenta]rm -rf {self.local_path}" + f"\n [magenta]nextflow pull {self.full_name}", ) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index d152e970b3..8a61ea32d1 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -5,7 +5,7 @@ // TODO nf-core: A module file SHOULD only define input and output files as command-line parameters. // All other parameters MUST be provided using the "task.ext" directive, see here: // https://www.nextflow.io/docs/latest/process.html#ext -// where "task.ext" is a string. +// where "task.ext" is a string. // Any parameters that need to be evaluated in the context of a particular sample // e.g. single-end/paired-end data MUST also be defined and evaluated appropriately. // TODO nf-core: Software that can be piped together SHOULD be added to separate module files @@ -18,7 +18,7 @@ process {{ tool_name_underscore|upper }} { tag {{ '"$meta.id"' if has_meta else "'$bam'" }} label '{{ process_label }}' - + // TODO nf-core: List required Conda package(s). // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. @@ -43,6 +43,9 @@ process {{ tool_name_underscore|upper }} { // TODO nf-core: List additional required output channels/values here path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: def args = task.ext.args ?: '' {% if has_meta -%} diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index dbce4bd915..f833d52564 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -8,3 +8,4 @@ from .install import ModuleInstall from .update import ModuleUpdate from .remove import ModuleRemove +from .info import ModuleInfo diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 4d24faa52b..7e28556e29 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -15,8 +15,6 @@ from rich.markdown import Markdown import rich from nf_core.utils import rich_force_colors -import sys -import yaml import nf_core.utils import nf_core.modules.module_utils @@ -52,10 +50,11 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): self.up_to_date = [] self.updated = [] self.failed = [] + self.ignored = [] self.show_up_to_date = show_uptodate # Verify that this is not a pipeline - repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) if not repo_type == "modules": raise nf_core.modules.module_utils.ModuleException( "This command only works on the nf-core/modules repository, not on pipelines!" @@ -64,7 +63,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): # Get list of all modules _, nfcore_modules = nf_core.modules.module_utils.get_installed_modules(self.dir) - # Load the .nf-core-tools.config + # Load the .nf-core.yml config self.tools_config = nf_core.utils.load_tools_config(self.dir) # Prompt for module or all @@ -132,7 +131,7 @@ def bump_module_version(self, module: NFCoreModule): if module.module_name in self.bump_versions_config: config_version = self.bump_versions_config[module.module_name] if not config_version: - self.up_to_date.append((f"Omitting module due to config: {module.module_name}", module.module_name)) + self.ignored.append((f"Omitting module due to config.", module.module_name)) return False # check for correct version and newer versions @@ -333,3 +332,16 @@ def format_result(module_updates, table): table.add_column("Update message") table = format_result(self.failed, table) console.print(table) + + # Table of modules ignored due to `.nf-core.yml` + if len(self.ignored) > 0: + console.print( + rich.panel.Panel( + r"[!] {} Module update{} ignored".format(len(self.ignored), _s(self.ignored)), style="grey58" + ) + ) + table = Table(style="grey58", box=rich.box.ROUNDED) + table.add_column("Module name", width=max_mod_name_len) + table.add_column("Update message") + table = format_result(self.ignored, table) + console.print(table) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index a9c071c4ce..f44eeb9e14 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -19,13 +19,23 @@ import yaml import nf_core.utils +import nf_core.modules.module_utils log = logging.getLogger(__name__) class ModuleCreate(object): def __init__( - self, directory=".", tool="", author=None, process_label=None, has_meta=None, force=False, conda_name=None + self, + directory=".", + tool="", + author=None, + process_label=None, + has_meta=None, + force=False, + conda_name=None, + conda_version=None, + repo_type=None, ): self.directory = directory self.tool = tool @@ -35,8 +45,9 @@ def __init__( self.force_overwrite = force self.subtool = None self.tool_conda_name = conda_name + self.tool_conda_version = conda_version self.tool_licence = None - self.repo_type = None + self.repo_type = repo_type self.tool_licence = "" self.tool_description = "" self.tool_doc_url = "" @@ -75,9 +86,12 @@ def create(self): # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules try: - self.repo_type = self.get_repo_type(self.directory) + self.directory, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.directory, self.repo_type) except LookupError as e: raise UserWarning(e) + log.info(f"Repository type: [blue]{self.repo_type}") + if self.directory != ".": + log.info(f"Base directory: '{self.directory}'") log.info( "[yellow]Press enter to use default values [cyan bold](shown in brackets)[/] [yellow]or type your own responses. " @@ -131,9 +145,14 @@ def create(self): anaconda_response = nf_core.utils.anaconda_package(self.tool_conda_name, ["bioconda"]) else: anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"]) - version = anaconda_response.get("latest_version") - if not version: - version = str(max([parse_version(v) for v in anaconda_response["versions"]])) + + if not self.tool_conda_version: + version = anaconda_response.get("latest_version") + if not version: + version = str(max([parse_version(v) for v in anaconda_response["versions"]])) + else: + version = self.tool_conda_version + self.tool_licence = nf_core.utils.parse_anaconda_licence(anaconda_response, version) self.tool_description = anaconda_response.get("summary", "") self.tool_doc_url = anaconda_response.get("doc_url", "") @@ -217,7 +236,7 @@ def create(self): ) while self.has_meta is None: self.has_meta = rich.prompt.Confirm.ask( - "[violet]Will the module require a meta map of sample information? (yes/no)", default=True + "[violet]Will the module require a meta map of sample information?", default=True ) # Create module template with cokiecutter @@ -272,29 +291,6 @@ def render_template(self): template_stat = os.stat(os.path.join(os.path.dirname(nf_core.__file__), "module-template", template_fn)) os.chmod(dest_fn, template_stat.st_mode) - def get_repo_type(self, directory): - """ - Determine whether this is a pipeline repository or a clone of - nf-core/modules - """ - # Verify that the pipeline dir exists - if dir is None or not os.path.exists(directory): - raise UserWarning(f"Could not find directory: {directory}") - - readme = os.path.join(directory, "README.md") - # Determine repository type - if os.path.exists(readme): - with open(readme) as fh: - if fh.readline().rstrip().startswith("# ![nf-core/modules]"): - return "modules" - else: - return "pipeline" - else: - raise UserWarning( - f"This directory does not look like a clone of nf-core/modules or an nf-core pipeline: '{directory}'" - " Please point to a valid directory." - ) - def get_module_dirs(self): """Given a directory and a tool/subtool, set the file paths and check if they already exist diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py new file mode 100644 index 0000000000..88a178546f --- /dev/null +++ b/nf_core/modules/info.py @@ -0,0 +1,194 @@ +import base64 +import logging +import os +import requests +import yaml + +from rich import box +from rich.text import Text +from rich.console import Group +from rich.markdown import Markdown +from rich.panel import Panel +from rich.table import Table + +from .modules_command import ModuleCommand +from .module_utils import get_repo_type, get_installed_modules, get_module_git_log, module_exist_in_repo +from .modules_repo import ModulesRepo + +log = logging.getLogger(__name__) + + +class ModuleInfo(ModuleCommand): + def __init__(self, pipeline_dir, tool): + + self.module = tool + self.meta = None + self.local_path = None + self.remote_location = None + + # Quietly check if this is a pipeline or not + if pipeline_dir: + try: + pipeline_dir, repo_type = get_repo_type(pipeline_dir, use_prompt=False) + log.debug(f"Found {repo_type} repo: {pipeline_dir}") + except UserWarning as e: + log.debug(f"Only showing remote info: {e}") + pipeline_dir = None + + super().__init__(pipeline_dir) + + def get_module_info(self): + """Given the name of a module, parse meta.yml and print usage help.""" + + # Running with a local install, try to find the local meta + if self.dir: + self.meta = self.get_local_yaml() + + # Either failed locally or in remote mode + if not self.meta: + self.meta = self.get_remote_yaml() + + # Could not find the meta + if self.meta == False: + raise UserWarning(f"Could not find module '{self.module}'") + + return self.generate_module_info_help() + + def get_local_yaml(self): + """Attempt to get the meta.yml file from a locally installed module. + + Returns: + dict or bool: Parsed meta.yml found, False otherwise + """ + + # Get installed modules + self.get_pipeline_modules() + + # Try to find and load the meta.yml file + module_base_path = f"{self.dir}/modules/" + if self.repo_type == "modules": + module_base_path = f"{self.dir}/" + for dir, mods in self.module_names.items(): + for mod in mods: + if mod == self.module: + mod_dir = os.path.join(module_base_path, dir, mod) + meta_fn = os.path.join(mod_dir, "meta.yml") + if os.path.exists(meta_fn): + log.debug(f"Found local file: {meta_fn}") + with open(meta_fn, "r") as fh: + self.local_path = mod_dir + return yaml.safe_load(fh) + + log.debug(f"Module '{self.module}' meta.yml not found locally") + return False + + def get_remote_yaml(self): + """Attempt to get the meta.yml file from a remote repo. + + Returns: + dict or bool: Parsed meta.yml found, False otherwise + """ + # Fetch the remote repo information + self.modules_repo.get_modules_file_tree() + + # Check if our requested module is there + if self.module not in self.modules_repo.modules_avail_module_names: + return False + + # Get the remote path + meta_url = None + for file_dict in self.modules_repo.modules_file_tree: + if file_dict.get("path") == f"modules/{self.module}/meta.yml": + meta_url = file_dict.get("url") + + if not meta_url: + return False + + # Download and parse + log.debug(f"Attempting to fetch {meta_url}") + response = requests.get(meta_url) + result = response.json() + file_contents = base64.b64decode(result["content"]) + self.remote_location = self.modules_repo.name + return yaml.safe_load(file_contents) + + def generate_module_info_help(self): + """Take the parsed meta.yml and generate rich help. + + Returns: + rich renderable + """ + + renderables = [] + + # Intro panel + intro_text = Text() + if self.local_path: + intro_text.append(Text.from_markup(f"Location: [blue]{self.local_path}\n")) + elif self.remote_location: + intro_text.append( + Text.from_markup( + f":globe_with_meridians: Repository: [link=https://github.com/{self.remote_location}]{self.remote_location}[/]\n" + ) + ) + + if self.meta.get("tools"): + tools_strings = [] + for tool in self.meta["tools"]: + for tool_name, tool_meta in tool.items(): + tools_strings.append(f"[link={tool_meta['homepage']}]{tool_name}") + intro_text.append(Text.from_markup(f":wrench: Tools: {', '.join(tools_strings)}\n", style="dim")) + + if self.meta.get("description"): + intro_text.append(Text.from_markup(f":book: Description: {self.meta['description']}", style="dim")) + + renderables.append( + Panel( + intro_text, + title=f"[bold]Module: [green]{self.module}\n", + title_align="left", + ) + ) + + # Inputs + if self.meta.get("input"): + inputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) + inputs_table.add_column(":inbox_tray: Inputs") + inputs_table.add_column("Description") + inputs_table.add_column("Pattern", justify="right", style="green") + for input in self.meta["input"]: + for key, info in input.items(): + inputs_table.add_row( + f"[orange1 on black] {key} [/][dim i] ({info['type']})", + Markdown(info["description"] if info["description"] else ""), + info.get("pattern", ""), + ) + + renderables.append(inputs_table) + + # Outputs + if self.meta.get("output"): + outputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) + outputs_table.add_column(":outbox_tray: Outputs") + outputs_table.add_column("Description") + outputs_table.add_column("Pattern", justify="right", style="green") + for output in self.meta["output"]: + for key, info in output.items(): + outputs_table.add_row( + f"[orange1 on black] {key} [/][dim i] ({info['type']})", + Markdown(info["description"] if info["description"] else ""), + info.get("pattern", ""), + ) + + renderables.append(outputs_table) + + # Installation command + if self.remote_location: + cmd_base = "nf-core modules" + if self.remote_location != "nf-core/modules": + cmd_base = f"nf-core modules --github-repository {self.remote_location}" + renderables.append( + Text.from_markup(f"\n :computer: Installation command: [magenta]{cmd_base} install {self.module}\n") + ) + + return Group(*renderables) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index c5c33cc2a8..673e1d7f4e 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -6,8 +6,7 @@ import nf_core.modules.module_utils from .modules_command import ModuleCommand -from .module_utils import get_installed_modules, get_module_git_log, module_exist_in_repo -from .modules_repo import ModulesRepo +from .module_utils import get_module_git_log, module_exist_in_repo log = logging.getLogger(__name__) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index af3367765f..e81eea7f85 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -71,7 +71,7 @@ class ModuleLint(ModuleCommand): def __init__(self, dir): self.dir = dir try: - self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) except LookupError as e: raise UserWarning(e) @@ -93,10 +93,12 @@ def __init__(self, dir): if self.repo_type == "pipeline": # Add as first test to load git_sha before module_changes self.lint_tests.insert(0, "module_version") + # Only check if modules have been changed in pipelines + self.lint_tests.append("module_changes") @staticmethod def _get_all_lint_tests(): - return ["main_nf", "meta_yml", "module_changes", "module_todos", "module_deprecations"] + return ["main_nf", "meta_yml", "module_todos", "module_deprecations"] def lint(self, module=None, key=(), all_modules=False, print_results=True, show_passed=False, local=False): """ diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 065910d3b8..80fe8fbf9e 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -43,16 +43,20 @@ def main_nf(module_lint_object, module): state = "module" process_lines = [] script_lines = [] + when_lines = [] for l in lines: if re.search("^\s*process\s*\w*\s*{", l) and state == "module": state = "process" - if re.search("input\s*:", l) and state == "process": + if re.search("input\s*:", l) and state in ["process"]: state = "input" continue - if re.search("output\s*:", l) and state == "input": + if re.search("output\s*:", l) and state in ["input", "process"]: state = "output" continue - if re.search("script\s*:", l) and state == "output": + if re.search("when\s*:", l) and state in ["input", "output", "process"]: + state = "when" + continue + if re.search("script\s*:", l) and state in ["input", "output", "when", "process"]: state = "script" continue @@ -64,6 +68,8 @@ def main_nf(module_lint_object, module): if state == "output" and not _is_empty(module, l): outputs += _parse_output(module, l) outputs = list(set(outputs)) # remove duplicate 'meta's + if state == "when" and not _is_empty(module, l): + when_lines.append(l) if state == "script" and not _is_empty(module, l): script_lines.append(l) @@ -73,22 +79,32 @@ def main_nf(module_lint_object, module): else: module.warned.append(("main_nf_container", "Container versions do not match", module.main_nf)) + # Check the when statement + check_when_section(module, when_lines) + # Check the script definition check_script_section(module, script_lines) # Check whether 'meta' is emitted when given as input - if "meta" in inputs: - module.has_meta = True - if "meta" in outputs: - module.passed.append(("main_nf_meta_output", "'meta' map emitted in output channel(s)", module.main_nf)) - else: - module.failed.append(("main_nf_meta_output", "'meta' map not emitted in output channel(s)", module.main_nf)) + if inputs: + if "meta" in inputs: + module.has_meta = True + if outputs: + if "meta" in outputs: + module.passed.append( + ("main_nf_meta_output", "'meta' map emitted in output channel(s)", module.main_nf) + ) + else: + module.failed.append( + ("main_nf_meta_output", "'meta' map not emitted in output channel(s)", module.main_nf) + ) # Check that a software version is emitted - if "version" in outputs: - module.passed.append(("main_nf_version_emitted", "Module emits software version", module.main_nf)) - else: - module.warned.append(("main_nf_version_emitted", "Module does not emit software version", module.main_nf)) + if outputs: + if "versions" in outputs: + module.passed.append(("main_nf_version_emitted", "Module emits software version", module.main_nf)) + else: + module.warned.append(("main_nf_version_emitted", "Module does not emit software version", module.main_nf)) return inputs, outputs @@ -114,6 +130,28 @@ def check_script_section(self, lines): self.failed.append(("main_nf_meta_prefix", "'prefix' unspecified in script section", self.main_nf)) +def check_when_section(self, lines): + """ + Lint the when: section + Checks whether the line is modified from 'task.ext.when == null || task.ext.when' + """ + if len(lines) == 0: + self.failed.append(("when_exist", "when: condition has been removed", self.main_nf)) + return + elif len(lines) > 1: + self.failed.append(("when_exist", "when: condition has too many lines", self.main_nf)) + return + else: + self.passed.append(("when_exist", "when: condition is present", self.main_nf)) + + # Check the condition hasn't been changed. + if lines[0].strip() != "task.ext.when == null || task.ext.when": + self.failed.append(("when_condition", "when: condition has been altered", self.main_nf)) + return + else: + self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf)) + + def check_process_section(self, lines): """ Lint the section of a module between the process definition @@ -139,7 +177,7 @@ def check_process_section(self, lines): if all([x.upper() for x in self.process_name]): self.passed.append(("process_capitals", "Process name is in capital letters", self.main_nf)) else: - self.failed.append(("process_capitals", "Process name is not in captial letters", self.main_nf)) + self.failed.append(("process_capitals", "Process name is not in capital letters", self.main_nf)) # Check that process labels are correct correct_process_labels = ["process_low", "process_medium", "process_high", "process_long"] @@ -160,12 +198,23 @@ def check_process_section(self, lines): self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf)) for l in lines: + l = l.strip() + l = l.replace('"', "") + l = l.replace("'", "") if re.search("bioconda::", l): bioconda_packages = [b for b in l.split() if "bioconda::" in b] - if re.search("org/singularity", l): - singularity_tag = l.split("/")[-1].replace('"', "").replace("'", "").split("--")[-1].strip() - if re.search("biocontainers", l): - docker_tag = l.split("/")[-1].replace('"', "").replace("'", "").split("--")[-1].strip() + if l.startswith("https://containers") or l.startswith("https://depot"): + lspl = l.lstrip("https://").split(":") + if len(lspl) == 2: + # e.g. 'https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' : + singularity_tag = "_".join(lspl[0].split("/")[-1].strip().rstrip(".img").split("_")[1:]) + else: + # e.g. 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : + singularity_tag = lspl[-2].strip() + if l.startswith("biocontainers/") or l.startswith("quay.io/"): + # e.g. 'quay.io/biocontainers/krona:2.7.1--pl526_5' }" + # e.g. 'biocontainers/biocontainers:v1.2.0_cv1' }" + docker_tag = l.split(":")[-1].strip("}").strip() # Check that all bioconda packages have build numbers # Also check for newer versions @@ -203,16 +252,18 @@ def check_process_section(self, lines): def _parse_input(self, line): input = [] - # more than one input + line = line.strip() if "tuple" in line: + # If more than one elements in channel should work with both of: + # e.g. tuple val(meta), path(reads) + # e.g. tuple val(meta), path(reads, stageAs: "input*/*") line = line.replace("tuple", "") line = line.replace(" ", "") - line = line.split(",") - - for elem in line: - elem = elem.split("(")[1] - elem = elem.replace(")", "").strip() - input.append(elem) + for idx, elem in enumerate(line.split(")")): + if elem: + elem = elem.split("(")[1] + elem = elem.split(",")[0].strip() + input.append(elem) else: if "(" in line: input.append(line.split("(")[1].replace(")", "")) @@ -225,9 +276,9 @@ def _parse_output(self, line): output = [] if "meta" in line: output.append("meta") - if not "emit" in line: + if not "emit:" in line: self.failed.append(("missing_emit", f"Missing emit statement: {line.strip()}", self.main_nf)) - if "emit" in line: + else: output.append(line.split("emit:")[1].strip()) return output diff --git a/nf_core/modules/lint/module_changes.py b/nf_core/modules/lint/module_changes.py index 44c2501424..4e466a2c3c 100644 --- a/nf_core/modules/lint/module_changes.py +++ b/nf_core/modules/lint/module_changes.py @@ -15,6 +15,8 @@ def module_changes(module_lint_object, module): and compares them to the local copies If the module has a 'git_sha', the file content is checked against this sha + + Only runs when linting a pipeline, not the modules repository """ files_to_check = ["main.nf", "meta.yml"] @@ -49,7 +51,7 @@ def module_changes(module_lint_object, module): remote_copy = r.content.decode("utf-8") if local_copy != remote_copy: - module.warned.append( + module.failed.append( ( "check_local_copy", "Local copy of module does not match remote", diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 7bef0112d1..a8470d0d18 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -55,7 +55,7 @@ def module_tests(module_lint_object, module): # Look for md5sums of empty files for tfile in test.get("files", []): if tfile.get("md5sum") == "d41d8cd98f00b204e9800998ecf8427e": - module.warned.append( + module.failed.append( ( "test_yml_md5sum", "md5sum for empty file found: d41d8cd98f00b204e9800998ecf8427e", @@ -63,7 +63,7 @@ def module_tests(module_lint_object, module): ) ) if tfile.get("md5sum") == "7029066c27ac6f5ef18d660d5741979a": - module.warned.append( + module.failed.append( ( "test_yml_md5sum", "md5sum for compressed empty file found: 7029066c27ac6f5ef18d660d5741979a", diff --git a/nf_core/modules/lint/module_todos.py b/nf_core/modules/lint/module_todos.py index 57c73ebbcd..94b10299a8 100644 --- a/nf_core/modules/lint/module_todos.py +++ b/nf_core/modules/lint/module_todos.py @@ -11,7 +11,17 @@ def module_todos(module_lint_object, module): Slight modification of the "nf_core.lint.pipeline_todos" function to make it work for a single module """ - module.wf_path = module.module_dir - results = pipeline_todos(module) - for i, warning in enumerate(results["warned"]): - module.warned.append(("module_todo", warning, results["file_paths"][i])) + + # Main module directory + mod_results = pipeline_todos(None, root_dir=module.module_dir) + for i, warning in enumerate(mod_results["warned"]): + module.warned.append(("module_todo", warning, mod_results["file_paths"][i])) + for i, passed in enumerate(mod_results["passed"]): + module.passed.append(("module_todo", passed, module.module_dir)) + + # Module tests directory + test_results = pipeline_todos(None, root_dir=module.test_dir) + for i, warning in enumerate(test_results["warned"]): + module.warned.append(("module_todo", warning, test_results["file_paths"][i])) + for i, passed in enumerate(test_results["passed"]): + module.passed.append(("module_todo", passed, module.test_dir)) diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index dd9afc9582..fcd5a421f6 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -126,11 +126,14 @@ def create_modules_json(pipeline_dir): pipeline_dir (str): The directory where the `modules.json` should be created """ pipeline_config = nf_core.utils.fetch_wf_config(pipeline_dir) - pipeline_name = pipeline_config["manifest.name"] - pipeline_url = pipeline_config["manifest.homePage"] + pipeline_name = pipeline_config.get("manifest.name", "") + pipeline_url = pipeline_config.get("manifest.homePage", "") modules_json = {"name": pipeline_name.strip("'"), "homePage": pipeline_url.strip("'"), "repos": dict()} modules_dir = f"{pipeline_dir}/modules" + if not os.path.exists(modules_dir): + raise UserWarning(f"Can't find a ./modules directory. Is this a DSL2 pipeline?") + # Extract all modules repos in the pipeline directory repo_names = [ f"{user_name}/{repo_name}" @@ -337,24 +340,64 @@ def get_installed_modules(dir, repo_type="modules"): return local_modules, nfcore_modules -def get_repo_type(dir): +def get_repo_type(dir, repo_type=None, use_prompt=True): """ Determine whether this is a pipeline repository or a clone of nf-core/modules """ # Verify that the pipeline dir exists if dir is None or not os.path.exists(dir): - raise LookupError("Could not find directory: {}".format(dir)) + raise UserWarning(f"Could not find directory: {dir}") + + # Try to find the root directory + base_dir = os.path.abspath(dir) + config_path_yml = os.path.join(base_dir, ".nf-core.yml") + config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") + while ( + not os.path.exists(config_path_yml) + and not os.path.exists(config_path_yaml) + and base_dir != os.path.dirname(base_dir) + ): + base_dir = os.path.dirname(base_dir) + config_path_yml = os.path.join(base_dir, ".nf-core.yml") + config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") + # Reset dir if we found the config file (will be an absolute path) + if os.path.exists(config_path_yml) or os.path.exists(config_path_yaml): + dir = base_dir + + # Figure out the repository type from the .nf-core.yml config file if we can + tools_config = nf_core.utils.load_tools_config(dir) + repo_type = tools_config.get("repository_type", None) + + # If not set, prompt the user + if not repo_type and use_prompt: + log.warning(f"Can't find a '.nf-core.yml' file that defines 'repository_type'") + repo_type = questionary.select( + "Is this repository an nf-core pipeline or a fork of nf-core/modules?", + choices=[ + {"name": "Pipeline", "value": "pipeline"}, + {"name": "nf-core/modules", "value": "modules"}, + ], + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() - # Determine repository type - if os.path.exists(os.path.join(dir, "README.md")): - with open(os.path.join(dir, "README.md")) as fh: - if fh.readline().rstrip().startswith("# ![nf-core/modules]"): - return "modules" - else: - return "pipeline" - else: - raise LookupError("Could not determine repository type of '{}'".format(dir)) + # Save the choice in the config file + log.info("To avoid this prompt in the future, add the 'repository_type' key to a root '.nf-core.yml' file.") + if rich.prompt.Confirm.ask("[bold][blue]?[/] Would you like me to add this config now?", default=True): + with open(os.path.join(dir, ".nf-core.yml"), "a+") as fh: + fh.write(f"repository_type: {repo_type}\n") + log.info("Config added to '.nf-core.yml'") + + # Not set and not allowed to ask + elif not repo_type: + raise UserWarning("Repository type could not be established") + + # Check if it's a valid answer + if not repo_type in ["pipeline", "modules"]: + raise UserWarning(f"Invalid repository type: '{repo_type}'") + + # It was set on the command line, return what we were given + return [dir, repo_type] def verify_pipeline_dir(dir): diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 866bffabb9..a60291fcfa 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -29,7 +29,7 @@ def __init__(self, dir): self.module_names = [] try: if self.dir: - self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) else: self.repo_type = None except LookupError as e: @@ -207,7 +207,7 @@ def modules_json_up_to_date(self): log.info(f"Recomputing commit SHA for module {format_missing[0]} which was missing from 'modules.json'") else: log.info( - f"Recomputing commit SHAs for modules which which were were missing from 'modules.json': {', '.join(format_missing)}" + f"Recomputing commit SHAs for modules which were missing from 'modules.json': {', '.join(format_missing)}" ) failed_to_find_commit_sha = [] for repo, modules in missing_from_modules_json.items(): @@ -284,18 +284,21 @@ def load_modules_json(self): modules_json = None return modules_json - def update_modules_json(self, modules_json, repo_name, module_name, module_version): + def update_modules_json(self, modules_json, repo_name, module_name, module_version, write_file=True): """Updates the 'module.json' file with new module info""" if repo_name not in modules_json["repos"]: modules_json["repos"][repo_name] = dict() modules_json["repos"][repo_name][module_name] = {"git_sha": module_version} - self.dump_modules_json(modules_json) - - def dump_modules_json(self, modules_json): - modules_json_path = os.path.join(self.dir, "modules.json") # Sort the 'modules.json' repo entries modules_json["repos"] = nf_core.utils.sort_dictionary(modules_json["repos"]) + if write_file: + self.dump_modules_json(modules_json) + else: + return modules_json + def dump_modules_json(self, modules_json): + """Build filename for modules.json and write to file.""" + modules_json_path = os.path.join(self.dir, "modules.json") with open(modules_json_path, "w") as fh: json.dump(modules_json, fh, indent=4) diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index a59e3262aa..a625a0db06 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -128,7 +128,7 @@ def get_module_file_urls(self, module, commit=""): """ results = {} for f in self.modules_file_tree: - if not f["path"].startswith("modules/{}".format(module)): + if not f["path"].startswith("modules/{}/".format(module)): continue if f["type"] != "blob": continue diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index d917a81053..d210cf350e 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -228,16 +228,18 @@ def create_test_file_dict(self, results_dir, is_repeat=False): for root, dir, file in os.walk(results_dir): for elem in file: elem = os.path.join(root, elem) - test_file = {"path": elem} + test_file = {"path": elem} # add the key here so that it comes first in the dict # Check that this isn't an empty file if self.check_if_empty_file(elem): if not is_repeat: - self.errors.append(f"Empty file, skipping md5sum: '{os.path.basename(elem)}'") - else: - elem_md5 = self._md5(elem) - test_file["md5sum"] = elem_md5 + self.errors.append(f"Empty file found! '{os.path.basename(elem)}'") + # Add the md5 anyway, linting should fail later and can be manually removed if needed. + # Originally we skipped this if empty, but then it's too easy to miss the warning. + # Equally, if a file is legitimately empty we don't want to prevent this from working. + elem_md5 = self._md5(elem) + test_file["md5sum"] = elem_md5 # Switch out the results directory path with the expected 'output' directory - elem = elem.replace(results_dir, "output") + test_file["path"] = elem.replace(results_dir, "output") test_files.append(test_file) test_files = sorted(test_files, key=operator.itemgetter("path")) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index dc69c5b15a..910af7ebaa 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -1,9 +1,13 @@ +import copy +import difflib +import enum +import json +import logging import os -import shutil import questionary -import logging +import shutil import tempfile -import difflib +from questionary import question from rich.console import Console from rich.syntax import Syntax @@ -18,13 +22,14 @@ class ModuleUpdate(ModuleCommand): - def __init__(self, pipeline_dir, force=False, prompt=False, sha=None, update_all=False, diff=False): + def __init__(self, pipeline_dir, force=False, prompt=False, sha=None, update_all=False, save_diff_fn=None): super().__init__(pipeline_dir) self.force = force self.prompt = prompt self.sha = sha self.update_all = update_all - self.diff = diff + self.show_diff = False + self.save_diff_fn = save_diff_fn def update(self, module): if self.repo_type == "modules": @@ -157,6 +162,7 @@ def update(self, module): repos_mods_shas[repo_name].append((module, custom_sha)) else: skipped_repos.append(repo_name) + if skipped_repos: skipped_str = "', '".join(skipped_repos) log.info(f"Skipping modules in repositor{'y' if len(skipped_repos) == 1 else 'ies'}: '{skipped_str}'") @@ -177,12 +183,49 @@ def update(self, module): # Load 'modules.json' modules_json = self.load_modules_json() + old_modules_json = copy.deepcopy(modules_json) # Deep copy to avoid mutability if not modules_json: return False + # Ask if we should show the diffs (unless a filename was already given on the command line) + if not self.save_diff_fn: + diff_type = questionary.select( + "Do you want to view diffs of the proposed changes?", + choices=[ + {"name": "No previews, just update everything", "value": 0}, + {"name": "Preview diff in terminal, choose whether to update files", "value": 1}, + {"name": "Just write diffs to a patch file", "value": 2}, + ], + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + self.show_diff = diff_type == 1 + self.save_diff_fn = diff_type == 2 + + # Set up file to save diff + if self.save_diff_fn: # True or a string + # From questionary - no filename yet + if self.save_diff_fn is True: + self.save_diff_fn = questionary.text( + "Enter the filename: ", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + # Check if filename already exists (questionary or cli) + while os.path.exists(self.save_diff_fn): + if questionary.confirm(f"'{self.save_diff_fn}' exists. Remove file?").unsafe_ask(): + os.remove(self.save_diff_fn) + break + self.save_diff_fn = questionary.text( + f"Enter a new filename: ", + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + exit_value = True for modules_repo, module, sha in repos_mods_shas: - dry_run = self.diff + + # Are we updating the files in place or not? + dry_run = self.show_diff or self.save_diff_fn + + # Check if the module we've been asked to update actually exists if not module_exist_in_repo(module, modules_repo): warn_msg = f"Module '{module}' not found in remote '{modules_repo.name}' ({modules_repo.branch})" if self.update_all: @@ -252,61 +295,160 @@ def update(self, module): continue if dry_run: - console = Console(force_terminal=nf_core.utils.rich_force_colors()) - files = os.listdir(os.path.join(*install_folder, module)) + + class DiffEnum(enum.Enum): + """ + Enumeration for keeping track of + the diff status of a pair of files + """ + + UNCHANGED = enum.auto() + CHANGED = enum.auto() + CREATED = enum.auto() + REMOVED = enum.auto() + + diffs = {} + + # Get all unique filenames in the two folders. + # `dict.fromkeys()` is used instead of `set()` to preserve order + files = dict.fromkeys(os.listdir(os.path.join(*install_folder, module))) + files.update(dict.fromkeys(os.listdir(module_dir))) + files = list(files) + temp_folder = os.path.join(*install_folder, module) - log.info( - f"Changes in module '{module}' between ({current_entry['git_sha'] if current_entry is not None else '?'}) and ({version if version is not None else 'latest'})" - ) + # Loop through all the module files and compute their diffs if needed for file in files: temp_path = os.path.join(temp_folder, file) curr_path = os.path.join(module_dir, file) - if os.path.exists(temp_path) and os.path.exists(curr_path): + if os.path.exists(temp_path) and os.path.exists(curr_path) and os.path.isfile(temp_path): with open(temp_path, "r") as fh: new_lines = fh.readlines() with open(curr_path, "r") as fh: old_lines = fh.readlines() + if new_lines == old_lines: # The files are identical - log.info(f"'{os.path.join(module, file)}' is unchanged") + diffs[file] = (DiffEnum.UNCHANGED, ()) else: - log.info(f"Changes in '{os.path.join(module, file)}':") # Compute the diff diff = difflib.unified_diff( old_lines, new_lines, - fromfile=f"{os.path.join(module, file)} (installed)", - tofile=f"{os.path.join(module, file)} (new)", + fromfile=os.path.join(module_dir, file), + tofile=os.path.join(module_dir, file), ) - - # Pretty print the diff using the pygments diff lexer - console.print(Syntax("".join(diff), "diff", theme="ansi_light")) + diffs[file] = (DiffEnum.CHANGED, diff) elif os.path.exists(temp_path): - # The file was created between the commits - log.info(f"Created file '{file}'") + # The file was created + diffs[file] = (DiffEnum.CREATED, ()) elif os.path.exists(curr_path): - # The file was removed between the commits - log.info(f"Removed file '{file}'") - - # Ask the user if they want to install the module - dry_run = not questionary.confirm("Update module?", default=False).unsafe_ask() - if not dry_run: - # The new module files are already installed - # we just need to clear the directory and move the - # new files from the temporary directory - self.clear_module_dir(module, module_dir) - os.mkdir(module_dir) - for file in files: - path = os.path.join(temp_folder, file) - if os.path.exists(path): - shutil.move(path, os.path.join(module_dir, file)) - log.info(f"Updating '{modules_repo.name}/{module}'") - log.debug(f"Updating module '{module}' to {version} from {modules_repo.name}") + # The file was removed + diffs[file] = (DiffEnum.REMOVED, ()) + + if self.save_diff_fn: + log.info(f"Writing diff of '{module}' to '{self.save_diff_fn}'") + with open(self.save_diff_fn, "a") as fh: + fh.write( + f"Changes in module '{module}' between ({current_entry['git_sha'] if current_entry is not None else '?'}) and ({version if version is not None else 'latest'})\n" + ) + + for file, d in diffs.items(): + diff_status, diff = d + if diff_status == DiffEnum.UNCHANGED: + # The files are identical + fh.write(f"'{os.path.join(module_dir, file)}' is unchanged\n") + + elif diff_status == DiffEnum.CREATED: + # The file was created between the commits + fh.write(f"'{os.path.join(module_dir, file)}' was created\n") + + elif diff_status == DiffEnum.REMOVED: + # The file was removed between the commits + fh.write(f"'{os.path.join(module_dir, file)}' was removed\n") + + else: + # The file has changed + fh.write(f"Changes in '{os.path.join(module_dir, file)}':\n") + # Write the diff lines to the file + for line in diff: + fh.write(line) + fh.write("\n") + + fh.write("*" * 60 + "\n") + elif self.show_diff: + console = Console(force_terminal=nf_core.utils.rich_force_colors()) + log.info( + f"Changes in module '{module}' between ({current_entry['git_sha'] if current_entry is not None else '?'}) and ({version if version is not None else 'latest'})" + ) + + for file, d in diffs.items(): + diff_status, diff = d + if diff_status == DiffEnum.UNCHANGED: + # The files are identical + log.info(f"'{os.path.join(module, file)}' is unchanged") + elif diff_status == DiffEnum.CREATED: + # The file was created between the commits + log.info(f"'{os.path.join(module, file)}' was created") + elif diff_status == DiffEnum.REMOVED: + # The file was removed between the commits + log.info(f"'{os.path.join(module, file)}' was removed") + else: + # The file has changed + log.info(f"Changes in '{os.path.join(module, file)}':") + # Pretty print the diff using the pygments diff lexer + console.print(Syntax("".join(diff), "diff", theme="ansi_light")) + # Ask the user if they want to install the module + dry_run = not questionary.confirm( + f"Update module '{module}'?", default=False, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + if not dry_run: + # The new module files are already installed. + # We just need to clear the directory and move the + # new files from the temporary directory + self.clear_module_dir(module, module_dir) + os.makedirs(module_dir) + for file in files: + path = os.path.join(temp_folder, file) + if os.path.exists(path): + shutil.move(path, os.path.join(module_dir, file)) + log.info(f"Updating '{modules_repo.name}/{module}'") + log.debug(f"Updating module '{module}' to {version} from {modules_repo.name}") + + # Update modules.json with newly installed module if not dry_run: - # Update module.json with newly installed module self.update_modules_json(modules_json, modules_repo.name, module, version) + + # Don't save to a file, just iteratively update the variable + else: + modules_json = self.update_modules_json( + modules_json, modules_repo.name, module, version, write_file=False + ) + + if self.save_diff_fn: + # Compare the new modules.json and build a diff + modules_json_diff = difflib.unified_diff( + json.dumps(old_modules_json, indent=4).splitlines(keepends=True), + json.dumps(modules_json, indent=4).splitlines(keepends=True), + fromfile=os.path.join(self.dir, "modules.json"), + tofile=os.path.join(self.dir, "modules.json"), + ) + + # Save diff for modules.json to file + with open(self.save_diff_fn, "a") as fh: + fh.write(f"Changes in './modules.json'\n") + for line in modules_json_diff: + fh.write(line) + fh.write("*" * 60 + "\n") + + log.info("Updates complete :sparkles:") + + if self.save_diff_fn: + log.info( + f"[bold magenta italic] TIP! [/] If you are happy with the changes in '{self.save_diff_fn}', you can apply them by running the command :point_right: [bold magenta italic]git apply {self.save_diff_fn}" + ) + return exit_value diff --git a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml index e904de573f..a9ab8e4513 100644 --- a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,4 +1,3 @@ - name: Bug report description: Report something that is broken or incorrect labels: bug diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index 230705bca7..a6eeaaccef 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -19,7 +19,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/{{ name }}/tree/master/.github/CONTRIBUTING.md) - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). -- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker`). +- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker` --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. - [ ] `CHANGELOG.md` is updated. diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 8e0ab65b23..d2ba27e3a2 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: nf-core/tower-action@v2 + uses: nf-core/tower-action@v3 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters @@ -31,4 +31,6 @@ jobs: "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ github.sha }}{% endraw %}" } profiles: test_full,aws_tower - pre_run_script: 'export NXF_VER=21.10.3' + nextflow_config: | + process.errorStrategy = 'retry' + process.maxRetries = 3 diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index eb14cced64..b5ceb32297 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: nf-core/tower-action@v2 + uses: nf-core/tower-action@v3 {% raw %} with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} @@ -25,4 +25,6 @@ jobs: "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-test-${{ github.sha }}{% endraw %}" } profiles: test,aws_tower - pre_run_script: 'export NXF_VER=21.10.3' + nextflow_config: | + process.errorStrategy = 'retry' + process.maxRetries = 3 diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index d8a7240f73..94edf32438 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -14,7 +14,7 @@ env: jobs: test: - name: Run workflow tests + name: Run pipeline with test data # Only run on push if this is the nf-core dev branch (merged PRs) if: {% raw %}${{{% endraw %} github.event_name != 'push' || (github.event_name == 'push' && github.repository == '{{ name }}') {% raw %}}}{% endraw %} runs-on: ubuntu-latest @@ -49,4 +49,4 @@ jobs: # For example: adding multiple test runs with different parameters # Remember that you can parallelise this by using strategy.matrix run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 15c49c860d..76449396ee 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -12,9 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 - with: - node-version: '10' + - uses: actions/setup-node@v2 - name: Install markdownlint run: npm install -g markdownlint-cli - name: Run Markdownlint @@ -51,9 +49,7 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 - with: - node-version: '10' + - uses: actions/setup-node@v2 - name: Install editorconfig-checker run: npm install -g editorconfig-checker @@ -64,14 +60,13 @@ jobs: YAML: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - uses: actions/setup-node@v1 + - name: Checkout + uses: actions/checkout@master + - name: 'Yamllint' + uses: karancode/yamllint-github-action@master with: - node-version: '10' - - name: Install yaml-lint - run: npm install -g yaml-lint - - name: Run yaml-lint - run: yamllint $(find ${GITHUB_WORKSPACE} -type f -name "*.yml" -o -name "*.yaml") + yamllint_file_or_dir: '.' + yamllint_config_filepath: '.yamllint.yml' # If the above check failed, post a comment on the PR explaining the failure - name: Post PR comment @@ -84,10 +79,11 @@ jobs: To keep the code consistent with lots of contributors, we run automated code consistency checks. To fix this CI test, please run: - * Install `yaml-lint` - * [Install `npm`](https://www.npmjs.com/get-npm) then [install `yaml-lint`](https://www.npmjs.com/package/yaml-lint) (`npm install -g yaml-lint`) + * Install `yamllint` + * Install `yamllint` following [this](https://yamllint.readthedocs.io/en/stable/quickstart.html#installing-yamllint) + instructions or alternative install it in your [conda environment](https://anaconda.org/conda-forge/yamllint) * Fix the markdown errors - * Run the test locally: `yamllint $(find . -type f -name "*.yml" -o -name "*.yaml")` + * Run the test locally: `yamllint $(find . -type f -name "*.yml" -o -name "*.yaml") -c ./.yamllint.yml` * Fix any reported errors in your YAML files Once you push these changes the test should pass, and you can hide this comment :+1: diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml new file mode 100644 index 0000000000..b7d4cee18b --- /dev/null +++ b/nf_core/pipeline-template/.gitpod.yml @@ -0,0 +1,14 @@ +image: nfcore/gitpod:latest + +vscode: + extensions: # based on nf-core.nf-core-extensionpack + - codezombiech.gitignore # Language support for .gitignore files + # - cssho.vscode-svgviewer # SVG viewer + - davidanson.vscode-markdownlint # Markdown/CommonMark linting and style checking for Visual Studio Code + - eamodio.gitlens # Quickly glimpse into whom, why, and when a line or code block was changed + - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files + - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar + - mechatroner.rainbow-csv # Highlight columns in csv files in different colors + # - nextflow.nextflow # Nextflow syntax highlighting + - oderwat.indent-rainbow # Highlight indentation level + - streetsidesoftware.code-spell-checker # Spelling checker for source code diff --git a/nf_core/pipeline-template/.nf-core.yml b/nf_core/pipeline-template/.nf-core.yml new file mode 100644 index 0000000000..3805dc81c1 --- /dev/null +++ b/nf_core/pipeline-template/.nf-core.yml @@ -0,0 +1 @@ +repository_type: pipeline diff --git a/nf_core/pipeline-template/.yamllint.yml b/nf_core/pipeline-template/.yamllint.yml new file mode 100644 index 0000000000..d466deec92 --- /dev/null +++ b/nf_core/pipeline-template/.yamllint.yml @@ -0,0 +1,6 @@ +extends: default + +rules: + document-start: disable + line-length: disable + truthy: disable diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index aa1e007ada..e0049a72e9 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -40,14 +40,14 @@ On release, automated continuous integration tests run the pipeline on a full-si 3. Download the pipeline and test it on a minimal dataset with a single command: ```console - nextflow run {{ name }} -profile test,YOURPROFILE + nextflow run {{ name }} -profile test,YOURPROFILE --outdir ``` Note that some form of configuration will be needed so that Nextflow knows how to fetch the required software. This is usually done in the form of a config profile (`YOURPROFILE` in the example command above). You can chain multiple config profiles in a comma-separated string. > * The pipeline comes with config profiles called `docker`, `singularity`, `podman`, `shifter`, `charliecloud` and `conda` which instruct the pipeline to use the named tool for software management. For example, `-profile test,docker`. > * Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile ` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment. - > * If you are using `singularity` and are persistently observing issues downloading Singularity images directly due to timeout or network issues, then you can use the `--singularity_pull_docker_container` parameter to pull and convert the Docker image instead. Alternatively, you can use the [`nf-core download`](https://nf-co.re/tools/#downloading-pipelines-for-offline-use) command to download images first, before running the pipeline. Setting the [`NXF_SINGULARITY_CACHEDIR` or `singularity.cacheDir`](https://www.nextflow.io/docs/latest/singularity.html?#singularity-docker-hub) Nextflow options enables you to store and re-use the images from a central location for future pipeline runs. + > * If you are using `singularity`, please use the [`nf-core download`](https://nf-co.re/tools/#downloading-pipelines-for-offline-use) command to download images first, before running the pipeline. Setting the [`NXF_SINGULARITY_CACHEDIR` or `singularity.cacheDir`](https://www.nextflow.io/docs/latest/singularity.html?#singularity-docker-hub) Nextflow options enables you to store and re-use the images from a central location for future pipeline runs. > * If you are using `conda`, it is highly recommended to use the [`NXF_CONDA_CACHEDIR` or `conda.cacheDir`](https://www.nextflow.io/docs/latest/conda.html) settings to store the environments in a central location for future pipeline runs. 4. Start running your own analysis! @@ -55,7 +55,7 @@ On release, automated continuous integration tests run the pipeline on a full-si ```console - nextflow run {{ name }} -profile --input samplesheet.csv --genome GRCh37 + nextflow run {{ name }} --input samplesheet.csv --outdir --genome GRCh37 -profile ``` ## Documentation diff --git a/nf_core/pipeline-template/bin/check_samplesheet.py b/nf_core/pipeline-template/bin/check_samplesheet.py index abe2cad944..5473b624c6 100755 --- a/nf_core/pipeline-template/bin/check_samplesheet.py +++ b/nf_core/pipeline-template/bin/check_samplesheet.py @@ -1,145 +1,249 @@ #!/usr/bin/env python -# TODO nf-core: Update the script to check the samplesheet -# This script is based on the example at: https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv -import os -import sys -import errno +"""Provide a command line tool to validate and transform tabular samplesheets.""" + + import argparse +import csv +import logging +import sys +from collections import Counter +from pathlib import Path -def parse_args(args=None): - Description = "Reformat {{ name }} samplesheet file and check its contents." - Epilog = "Example usage: python check_samplesheet.py " +logger = logging.getLogger() - parser = argparse.ArgumentParser(description=Description, epilog=Epilog) - parser.add_argument("FILE_IN", help="Input samplesheet file.") - parser.add_argument("FILE_OUT", help="Output file.") - return parser.parse_args(args) +class RowChecker: + """ + Define a service that can validate and transform each given row. -def make_dir(path): - if len(path) > 0: - try: - os.makedirs(path) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise exception + Attributes: + modified (list): A list of dicts, where each dict corresponds to a previously + validated and transformed row. The order of rows is maintained. + """ -def print_error(error, context="Line", context_str=""): - error_str = "ERROR: Please check samplesheet -> {}".format(error) - if context != "" and context_str != "": - error_str = "ERROR: Please check samplesheet -> {}\n{}: '{}'".format( - error, context.strip(), context_str.strip() + VALID_FORMATS = ( + ".fq.gz", + ".fastq.gz", + ) + + def __init__( + self, + sample_col="sample", + first_col="fastq_1", + second_col="fastq_2", + single_col="single_end", + **kwargs, + ): + """ + Initialize the row checker with the expected column names. + + Args: + sample_col (str): The name of the column that contains the sample name + (default "sample"). + first_col (str): The name of the column that contains the first (or only) + FASTQ file path (default "fastq_1"). + second_col (str): The name of the column that contains the second (if any) + FASTQ file path (default "fastq_2"). + single_col (str): The name of the new column that will be inserted and + records whether the sample contains single- or paired-end sequencing + reads (default "single_end"). + + """ + super().__init__(**kwargs) + self._sample_col = sample_col + self._first_col = first_col + self._second_col = second_col + self._single_col = single_col + self._seen = set() + self.modified = [] + + def validate_and_transform(self, row): + """ + Perform all validations on the given row and insert the read pairing status. + + Args: + row (dict): A mapping from column headers (keys) to elements of that row + (values). + + """ + self._validate_sample(row) + self._validate_first(row) + self._validate_second(row) + self._validate_pair(row) + self._seen.add((row[self._sample_col], row[self._first_col])) + self.modified.append(row) + + def _validate_sample(self, row): + """Assert that the sample name exists and convert spaces to underscores.""" + assert len(row[self._sample_col]) > 0, "Sample input is required." + # Sanitize samples slightly. + row[self._sample_col] = row[self._sample_col].replace(" ", "_") + + def _validate_first(self, row): + """Assert that the first FASTQ entry is non-empty and has the right format.""" + assert len(row[self._first_col]) > 0, "At least the first FASTQ file is required." + self._validate_fastq_format(row[self._first_col]) + + def _validate_second(self, row): + """Assert that the second FASTQ entry has the right format if it exists.""" + if len(row[self._second_col]) > 0: + self._validate_fastq_format(row[self._second_col]) + + def _validate_pair(self, row): + """Assert that read pairs have the same file extension. Report pair status.""" + if row[self._first_col] and row[self._second_col]: + row[self._single_col] = False + assert ( + Path(row[self._first_col]).suffixes == Path(row[self._second_col]).suffixes + ), "FASTQ pairs must have the same file extensions." + else: + row[self._single_col] = True + + def _validate_fastq_format(self, filename): + """Assert that a given filename has one of the expected FASTQ extensions.""" + assert any(filename.endswith(extension) for extension in self.VALID_FORMATS), ( + f"The FASTQ file has an unrecognized extension: {filename}\n" + f"It should be one of: {', '.join(self.VALID_FORMATS)}" ) - print(error_str) - sys.exit(1) + def validate_unique_samples(self): + """ + Assert that the combination of sample name and FASTQ filename is unique. + + In addition to the validation, also rename the sample if more than one sample, + FASTQ file combination exists. + + """ + assert len(self._seen) == len(self.modified), "The pair of sample name and FASTQ must be unique." + if len({pair[0] for pair in self._seen}) < len(self._seen): + counts = Counter(pair[0] for pair in self._seen) + seen = Counter() + for row in self.modified: + sample = row[self._sample_col] + seen[sample] += 1 + if counts[sample] > 1: + row[self._sample_col] = f"{sample}_T{seen[sample]}" + + +def sniff_format(handle): + """ + Detect the tabular format. + + Args: + handle (text file): A handle to a `text file`_ object. The read position is + expected to be at the beginning (index 0). + + Returns: + csv.Dialect: The detected tabular format. + + .. _text file: + https://docs.python.org/3/glossary.html#term-text-file -# TODO nf-core: Update the check_samplesheet function -def check_samplesheet(file_in, file_out): """ - This function checks that the samplesheet follows the following structure: + peek = handle.read(2048) + sniffer = csv.Sniffer() + if not sniffer.has_header(peek): + logger.critical(f"The given sample sheet does not appear to contain a header.") + sys.exit(1) + dialect = sniffer.sniff(peek) + handle.seek(0) + return dialect - sample,fastq_1,fastq_2 - SAMPLE_PE,SAMPLE_PE_RUN1_1.fastq.gz,SAMPLE_PE_RUN1_2.fastq.gz - SAMPLE_PE,SAMPLE_PE_RUN2_1.fastq.gz,SAMPLE_PE_RUN2_2.fastq.gz - SAMPLE_SE,SAMPLE_SE_RUN1_1.fastq.gz, - For an example see: - https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv +def check_samplesheet(file_in, file_out): """ + Check that the tabular samplesheet has the structure expected by nf-core pipelines. - sample_mapping_dict = {} - with open(file_in, "r") as fin: + Validate the general shape of the table, expected columns, and each row. Also add + an additional column which records whether one or two FASTQ reads were found. - ## Check header - MIN_COLS = 2 - # TODO nf-core: Update the column names for the input samplesheet - HEADER = ["sample", "fastq_1", "fastq_2"] - header = [x.strip('"') for x in fin.readline().strip().split(",")] - if header[: len(HEADER)] != HEADER: - print("ERROR: Please check samplesheet header -> {} != {}".format(",".join(header), ",".join(HEADER))) - sys.exit(1) + Args: + file_in (pathlib.Path): The given tabular samplesheet. The format can be either + CSV, TSV, or any other format automatically recognized by ``csv.Sniffer``. + file_out (pathlib.Path): Where the validated and transformed samplesheet should + be created; always in CSV format. - ## Check sample entries - for line in fin: - lspl = [x.strip().strip('"') for x in line.strip().split(",")] - - # Check valid number of columns per row - if len(lspl) < len(HEADER): - print_error( - "Invalid number of columns (minimum = {})!".format(len(HEADER)), - "Line", - line, - ) - num_cols = len([x for x in lspl if x]) - if num_cols < MIN_COLS: - print_error( - "Invalid number of populated columns (minimum = {})!".format(MIN_COLS), - "Line", - line, - ) - - ## Check sample name entries - sample, fastq_1, fastq_2 = lspl[: len(HEADER)] - sample = sample.replace(" ", "_") - if not sample: - print_error("Sample entry has not been specified!", "Line", line) - - ## Check FastQ file extension - for fastq in [fastq_1, fastq_2]: - if fastq: - if fastq.find(" ") != -1: - print_error("FastQ file contains spaces!", "Line", line) - if not fastq.endswith(".fastq.gz") and not fastq.endswith(".fq.gz"): - print_error( - "FastQ file does not have extension '.fastq.gz' or '.fq.gz'!", - "Line", - line, - ) - - ## Auto-detect paired-end/single-end - sample_info = [] ## [single_end, fastq_1, fastq_2] - if sample and fastq_1 and fastq_2: ## Paired-end short reads - sample_info = ["0", fastq_1, fastq_2] - elif sample and fastq_1 and not fastq_2: ## Single-end short reads - sample_info = ["1", fastq_1, fastq_2] - else: - print_error("Invalid combination of columns provided!", "Line", line) - - ## Create sample mapping dictionary = { sample: [ single_end, fastq_1, fastq_2 ] } - if sample not in sample_mapping_dict: - sample_mapping_dict[sample] = [sample_info] - else: - if sample_info in sample_mapping_dict[sample]: - print_error("Samplesheet contains duplicate rows!", "Line", line) - else: - sample_mapping_dict[sample].append(sample_info) - - ## Write validated samplesheet with appropriate columns - if len(sample_mapping_dict) > 0: - out_dir = os.path.dirname(file_out) - make_dir(out_dir) - with open(file_out, "w") as fout: - fout.write(",".join(["sample", "single_end", "fastq_1", "fastq_2"]) + "\n") - for sample in sorted(sample_mapping_dict.keys()): - - ## Check that multiple runs of the same sample are of the same datatype - if not all(x[0] == sample_mapping_dict[sample][0][0] for x in sample_mapping_dict[sample]): - print_error("Multiple runs of a sample must be of the same datatype!", "Sample: {}".format(sample)) - - for idx, val in enumerate(sample_mapping_dict[sample]): - fout.write(",".join(["{}_T{}".format(sample, idx + 1)] + val) + "\n") - else: - print_error("No entries to process!", "Samplesheet: {}".format(file_in)) - - -def main(args=None): - args = parse_args(args) - check_samplesheet(args.FILE_IN, args.FILE_OUT) + Example: + This function checks that the samplesheet follows the following structure, + see also the `viral recon samplesheet`_:: + + sample,fastq_1,fastq_2 + SAMPLE_PE,SAMPLE_PE_RUN1_1.fastq.gz,SAMPLE_PE_RUN1_2.fastq.gz + SAMPLE_PE,SAMPLE_PE_RUN2_1.fastq.gz,SAMPLE_PE_RUN2_2.fastq.gz + SAMPLE_SE,SAMPLE_SE_RUN1_1.fastq.gz, + + .. _viral recon samplesheet: + https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv + + """ + required_columns = {"sample", "fastq_1", "fastq_2"} + # See https://docs.python.org/3.9/library/csv.html#id3 to read up on `newline=""`. + with file_in.open(newline="") as in_handle: + reader = csv.DictReader(in_handle, dialect=sniff_format(in_handle)) + # Validate the existence of the expected header columns. + if not required_columns.issubset(reader.fieldnames): + logger.critical(f"The sample sheet **must** contain the column headers: {', '.join(required_columns)}.") + sys.exit(1) + # Validate each row. + checker = RowChecker() + for i, row in enumerate(reader): + try: + checker.validate_and_transform(row) + except AssertionError as error: + logger.critical(f"{str(error)} On line {i + 2}.") + sys.exit(1) + checker.validate_unique_samples() + header = list(reader.fieldnames) + header.insert(1, "single_end") + # See https://docs.python.org/3.9/library/csv.html#id3 to read up on `newline=""`. + with file_out.open(mode="w", newline="") as out_handle: + writer = csv.DictWriter(out_handle, header, delimiter=",") + writer.writeheader() + for row in checker.modified: + writer.writerow(row) + + +def parse_args(argv=None): + """Define and immediately parse command line arguments.""" + parser = argparse.ArgumentParser( + description="Validate and transform a tabular samplesheet.", + epilog="Example: python check_samplesheet.py samplesheet.csv samplesheet.valid.csv", + ) + parser.add_argument( + "file_in", + metavar="FILE_IN", + type=Path, + help="Tabular input samplesheet in CSV or TSV format.", + ) + parser.add_argument( + "file_out", + metavar="FILE_OUT", + type=Path, + help="Transformed output samplesheet in CSV format.", + ) + parser.add_argument( + "-l", + "--log-level", + help="The desired log level (default WARNING).", + choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"), + default="WARNING", + ) + return parser.parse_args(argv) + + +def main(argv=None): + """Coordinate argument parsing and program execution.""" + args = parse_args(argv) + logging.basicConfig(level=args.log_level, format="[%(levelname)s] %(message)s") + if not args.file_in.is_file(): + logger.error(f"The given input file {args.file_in} was not found!") + sys.exit(2) + args.file_out.parent.mkdir(parents=True, exist_ok=True) + check_samplesheet(args.file_in, args.file_out) if __name__ == "__main__": diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index e0557b9e91..c56ce5a4c9 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -1,7 +1,7 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ {{ name }} Nextflow base config file -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A 'blank slate' config file, appropriate for general use on most high performance compute environments. Assumes that all software is installed and available on the PATH. Runs in `local` mode - all jobs will be run on the logged in environment. diff --git a/nf_core/pipeline-template/conf/igenomes.config b/nf_core/pipeline-template/conf/igenomes.config index 855948def1..7a1b3ac6d3 100644 --- a/nf_core/pipeline-template/conf/igenomes.config +++ b/nf_core/pipeline-template/conf/igenomes.config @@ -1,7 +1,7 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Nextflow config file for iGenomes paths -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Defines reference genomes using iGenome paths. Can be used by any config that customises the base path using: $params.igenomes_base / --igenomes_base @@ -13,7 +13,7 @@ params { genomes { 'GRCh37' { fasta = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/BismarkIndex/" @@ -26,7 +26,7 @@ params { } 'GRCh38' { fasta = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/BismarkIndex/" @@ -38,7 +38,7 @@ params { } 'GRCm38' { fasta = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BismarkIndex/" @@ -51,7 +51,7 @@ params { } 'TAIR10' { fasta = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/BismarkIndex/" @@ -62,7 +62,7 @@ params { } 'EB2' { fasta = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/BismarkIndex/" @@ -72,7 +72,7 @@ params { } 'UMD3.1' { fasta = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/BismarkIndex/" @@ -83,7 +83,7 @@ params { } 'WBcel235' { fasta = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/BismarkIndex/" @@ -94,7 +94,7 @@ params { } 'CanFam3.1' { fasta = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/BismarkIndex/" @@ -105,7 +105,7 @@ params { } 'GRCz10' { fasta = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/BismarkIndex/" @@ -115,7 +115,7 @@ params { } 'BDGP6' { fasta = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/BismarkIndex/" @@ -126,7 +126,7 @@ params { } 'EquCab2' { fasta = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/BismarkIndex/" @@ -137,7 +137,7 @@ params { } 'EB1' { fasta = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/BismarkIndex/" @@ -147,7 +147,7 @@ params { } 'Galgal4' { fasta = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/BismarkIndex/" @@ -157,7 +157,7 @@ params { } 'Gm01' { fasta = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/BismarkIndex/" @@ -167,7 +167,7 @@ params { } 'Mmul_1' { fasta = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/BismarkIndex/" @@ -178,7 +178,7 @@ params { } 'IRGSP-1.0' { fasta = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/BismarkIndex/" @@ -188,7 +188,7 @@ params { } 'CHIMP2.1.4' { fasta = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/BismarkIndex/" @@ -199,7 +199,7 @@ params { } 'Rnor_5.0' { fasta = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_5.0/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_5.0/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_5.0/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_5.0/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_5.0/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_5.0/Sequence/BismarkIndex/" @@ -209,7 +209,7 @@ params { } 'Rnor_6.0' { fasta = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/BismarkIndex/" @@ -219,7 +219,7 @@ params { } 'R64-1-1' { fasta = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/BismarkIndex/" @@ -230,7 +230,7 @@ params { } 'EF2' { fasta = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/BismarkIndex/" @@ -242,7 +242,7 @@ params { } 'Sbi1' { fasta = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/BismarkIndex/" @@ -252,7 +252,7 @@ params { } 'Sscrofa10.2' { fasta = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/BismarkIndex/" @@ -263,7 +263,7 @@ params { } 'AGPv3' { fasta = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/BismarkIndex/" @@ -273,7 +273,7 @@ params { } 'hg38' { fasta = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/BismarkIndex/" @@ -285,7 +285,7 @@ params { } 'hg19' { fasta = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/BismarkIndex/" @@ -298,7 +298,7 @@ params { } 'mm10' { fasta = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/BismarkIndex/" @@ -311,7 +311,7 @@ params { } 'bosTau8' { fasta = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/BismarkIndex/" @@ -321,7 +321,7 @@ params { } 'ce10' { fasta = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/BismarkIndex/" @@ -333,7 +333,7 @@ params { } 'canFam3' { fasta = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/BismarkIndex/" @@ -344,7 +344,7 @@ params { } 'danRer10' { fasta = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/BismarkIndex/" @@ -355,7 +355,7 @@ params { } 'dm6' { fasta = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/BismarkIndex/" @@ -366,7 +366,7 @@ params { } 'equCab2' { fasta = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/BismarkIndex/" @@ -377,7 +377,7 @@ params { } 'galGal4' { fasta = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/BismarkIndex/" @@ -388,7 +388,7 @@ params { } 'panTro4' { fasta = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/BismarkIndex/" @@ -399,7 +399,7 @@ params { } 'rn6' { fasta = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/BismarkIndex/" @@ -409,7 +409,7 @@ params { } 'sacCer3' { fasta = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/BismarkIndex/" @@ -419,7 +419,7 @@ params { } 'susScr3' { fasta = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/BismarkIndex/" diff --git a/nf_core/pipeline-template/conf/modules.config b/nf_core/pipeline-template/conf/modules.config index a0506a4db4..da58a5d881 100644 --- a/nf_core/pipeline-template/conf/modules.config +++ b/nf_core/pipeline-template/conf/modules.config @@ -1,12 +1,12 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Config file for defining DSL2 per module options and publishing paths -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Available keys to override module options: - ext.args = Additional arguments appended to command in module. - ext.args2 = Second set of arguments appended to command in module (multi-tool modules). - ext.args3 = Third set of arguments appended to command in module (multi-tool modules). - ext.prefix = File name prefix for output files. + ext.args = Additional arguments appended to command in module. + ext.args2 = Second set of arguments appended to command in module (multi-tool modules). + ext.args3 = Third set of arguments appended to command in module (multi-tool modules). + ext.prefix = File name prefix for output files. ---------------------------------------------------------------------------------------- */ @@ -14,14 +14,14 @@ process { publishDir = [ path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, - mode: 'copy', + mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] withName: SAMPLESHEET_CHECK { publishDir = [ path: { "${params.outdir}/pipeline_info" }, - mode: 'copy', + mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } @@ -33,7 +33,7 @@ process { withName: CUSTOM_DUMPSOFTWAREVERSIONS { publishDir = [ path: { "${params.outdir}/pipeline_info" }, - mode: 'copy', + mode: params.publish_dir_mode, pattern: '*_versions.yml' ] } diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index eb2a725c87..1e2ea2e6bb 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -1,11 +1,11 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Nextflow config file for running minimal tests -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Defines input files and everything required to run a fast and simple pipeline test. Use as follows: - nextflow run {{ name }} -profile test, + nextflow run {{ name }} -profile test, --outdir ---------------------------------------------------------------------------------------- */ diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index 0fa82b7f90..87e4c96289 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -1,11 +1,11 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Nextflow config file for running full-size tests -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Defines input files and everything required to run a full size pipeline test. Use as follows: - nextflow run {{ name }} -profile test_full, + nextflow run {{ name }} -profile test_full, --outdir ---------------------------------------------------------------------------------------- */ diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 485af3af44..8d84911d67 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -57,7 +57,7 @@ An [example samplesheet](../assets/samplesheet.csv) has been provided with the p The typical command for running the pipeline is as follows: ```console -nextflow run {{ name }} --input samplesheet.csv --genome GRCh37 -profile docker +nextflow run {{ name }} --input samplesheet.csv --outdir --genome GRCh37 -profile docker ``` This will launch the pipeline with the `docker` configuration profile. See below for more information about profiles. @@ -141,11 +141,11 @@ Whilst the default requirements set within the pipeline will hopefully work for For example, if the nf-core/rnaseq pipeline is failing after multiple re-submissions of the `STAR_ALIGN` process due to an exit code of `137` this would indicate that there is an out of memory issue: ```console -[62/149eb0] NOTE: Process `RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) -- Execution is retried (1) -Error executing process > 'RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)' +[62/149eb0] NOTE: Process `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) -- Execution is retried (1) +Error executing process > 'NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)' Caused by: - Process `RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) + Process `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) Command executed: STAR \ @@ -169,17 +169,24 @@ Work dir: Tip: you can replicate the issue by changing to the process work dir and entering the command `bash .command.run` ``` -To bypass this error you would need to find exactly which resources are set by the `STAR_ALIGN` process. The quickest way is to search for `process STAR_ALIGN` in the [nf-core/rnaseq Github repo](https://github.com/nf-core/rnaseq/search?q=process+STAR_ALIGN). We have standardised the structure of Nextflow DSL2 pipelines such that all module files will be present in the `modules/` directory and so based on the search results the file we want is `modules/nf-core/software/star/align/main.nf`. If you click on the link to that file you will notice that there is a `label` directive at the top of the module that is set to [`label process_high`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/modules/nf-core/software/star/align/main.nf#L9). The [Nextflow `label`](https://www.nextflow.io/docs/latest/process.html#label) directive allows us to organise workflow processes in separate groups which can be referenced in a configuration file to select and configure subset of processes having similar computing requirements. The default values for the `process_high` label are set in the pipeline's [`base.config`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L33-L37) which in this case is defined as 72GB. Providing you haven't set any other standard nf-core parameters to __cap__ the [maximum resources](https://nf-co.re/usage/configuration#max-resources) used by the pipeline then we can try and bypass the `STAR_ALIGN` process failure by creating a custom config file that sets at least 72GB of memory, in this case increased to 100GB. The custom config below can then be provided to the pipeline via the [`-c`](#-c) parameter as highlighted in previous sections. +To bypass this error you would need to find exactly which resources are set by the `STAR_ALIGN` process. The quickest way is to search for `process STAR_ALIGN` in the [nf-core/rnaseq Github repo](https://github.com/nf-core/rnaseq/search?q=process+STAR_ALIGN). +We have standardised the structure of Nextflow DSL2 pipelines such that all module files will be present in the `modules/` directory and so, based on the search results, the file we want is `modules/nf-core/software/star/align/main.nf`. +If you click on the link to that file you will notice that there is a `label` directive at the top of the module that is set to [`label process_high`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/modules/nf-core/software/star/align/main.nf#L9). +The [Nextflow `label`](https://www.nextflow.io/docs/latest/process.html#label) directive allows us to organise workflow processes in separate groups which can be referenced in a configuration file to select and configure subset of processes having similar computing requirements. +The default values for the `process_high` label are set in the pipeline's [`base.config`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L33-L37) which in this case is defined as 72GB. +Providing you haven't set any other standard nf-core parameters to **cap** the [maximum resources](https://nf-co.re/usage/configuration#max-resources) used by the pipeline then we can try and bypass the `STAR_ALIGN` process failure by creating a custom config file that sets at least 72GB of memory, in this case increased to 100GB. +The custom config below can then be provided to the pipeline via the [`-c`](#-c) parameter as highlighted in previous sections. ```nextflow process { - withName: STAR_ALIGN { + withName: 'NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN' { memory = 100.GB } } ``` -> **NB:** We specify just the process name i.e. `STAR_ALIGN` in the config file and not the full task name string that is printed to screen in the error message or on the terminal whilst the pipeline is running i.e. `RNASEQ:ALIGN_STAR:STAR_ALIGN`. You may get a warning suggesting that the process selector isn't recognised but you can ignore that if the process name has been specified correctly. This is something that needs to be fixed upstream in core Nextflow. +> **NB:** We specify the full process name i.e. `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN` in the config file because this takes priority over the short name (`STAR_ALIGN`) and allows existing configuration using the full process name to be correctly overridden. +> If you get a warning suggesting that the process selector isn't recognised check that the process name has been specified correctly. ### Updating containers diff --git a/nf_core/pipeline-template/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/lib/NfcoreSchema.groovy index 40ab65f205..b3d092f809 100755 --- a/nf_core/pipeline-template/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/lib/NfcoreSchema.groovy @@ -27,7 +27,7 @@ class NfcoreSchema { /* groovylint-disable-next-line UnusedPrivateMethodParameter */ public static void validateParameters(workflow, params, log, schema_filename='nextflow_schema.json') { def has_error = false - //=====================================================================// + //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// // Check for nextflow core params and unexpected params def json = new File(getSchemaPath(workflow, schema_filename=schema_filename)).text def Map schemaParams = (Map) new JsonSlurper().parseText(json).get('definitions') @@ -135,7 +135,7 @@ class NfcoreSchema { } } - //=====================================================================// + //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// // Validate parameters against the schema InputStream input_stream = new File(getSchemaPath(workflow, schema_filename=schema_filename)).newInputStream() JSONObject raw_schema = new JSONObject(new JSONTokener(input_stream)) diff --git a/nf_core/pipeline-template/lib/Utils.groovy b/nf_core/pipeline-template/lib/Utils.groovy index 1b88aec0ea..28567bd70d 100755 --- a/nf_core/pipeline-template/lib/Utils.groovy +++ b/nf_core/pipeline-template/lib/Utils.groovy @@ -29,12 +29,12 @@ class Utils { conda_check_failed |= !(channels.indexOf('bioconda') < channels.indexOf('defaults')) if (conda_check_failed) { - log.warn "=============================================================================\n" + + log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " There is a problem with your Conda configuration!\n\n" + " You will need to set-up the conda-forge and bioconda channels correctly.\n" + " Please refer to https://bioconda.github.io/user/install.html#set-up-channels\n" + " NB: The order of the channels matters!\n" + - "===================================================================================" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" } } } diff --git a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy index ce022c3a43..0b442225ce 100755 --- a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy +++ b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy @@ -48,11 +48,11 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { // private static void genomeExistsError(params, log) { if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { - log.error "=============================================================================\n" + + log.error "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " Genome '${params.genome}' not found in any config files provided to the pipeline.\n" + " Currently, the available genome keys are:\n" + " ${params.genomes.keySet().join(", ")}\n" + - "===================================================================================" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" System.exit(1) } } diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index dca79e2684..104784f8ea 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -1,8 +1,8 @@ #!/usr/bin/env nextflow /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ {{ name }} -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/{{ name }} Website: https://nf-co.re/{{ short_name }} Slack : https://nfcore.slack.com/channels/{{ short_name }} @@ -12,25 +12,25 @@ nextflow.enable.dsl = 2 /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ GENOME PARAMETER VALUES -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ params.fasta = WorkflowMain.getGenomeAttribute(params, 'fasta') /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ VALIDATE & PRINT PARAMETER SUMMARY -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ WorkflowMain.initialise(workflow, params, log) /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ NAMED WORKFLOW FOR PIPELINE -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ include { {{ short_name|upper }} } from './workflows/{{ short_name }}' @@ -43,9 +43,9 @@ workflow NFCORE_{{ short_name|upper }} { } /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RUN ALL WORKFLOWS -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ // @@ -57,7 +57,7 @@ workflow { } /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ THE END -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index d5e3946ca8..001c3261aa 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -1,7 +1,7 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ {{ name }} Nextflow config file -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Default config options for all compute environments ---------------------------------------------------------------------------------------- */ @@ -24,8 +24,9 @@ params { max_multiqc_email_size = '25.MB' // Boilerplate options - outdir = './results' + outdir = null tracedir = "${params.outdir}/pipeline_info" + publish_dir_mode = 'copy' email = null email_on_fail = null plaintext_email = false @@ -62,6 +63,15 @@ try { System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config") } +// Load {{ name }} custom profiles from different institutions. +// Warning: Uncomment only if a pipeline-specific instititutional config already exists on nf-core/configs! +// try { +// includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" +// } catch (Exception e) { +// System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") +// } + + profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { @@ -121,7 +131,7 @@ if (!params.igenomes_ignore) { } // Export these variables to prevent local Python/R libraries from conflicting with those in the container -// The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. +// The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. // See https://apeltzer.github.io/post/03-julia-lang-nextflow/ for details on that. Once we have a common agreement on where to keep Julia packages, this is adjustable. env { diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 9ccc78f362..598f8a84b8 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -11,7 +11,8 @@ "fa_icon": "fas fa-terminal", "description": "Define where the pipeline should find input data and save output data.", "required": [ - "input" + "input", + "outdir" ], "properties": { "input": { @@ -26,8 +27,8 @@ }, "outdir": { "type": "string", - "description": "Path to the output directory where the results will be saved.", - "default": "./results", + "format": "directory-path", + "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", "fa_icon": "fas fa-folder-open" }, "email": { @@ -178,6 +179,22 @@ "fa_icon": "fas fa-question-circle", "hidden": true }, + "publish_dir_mode": { + "type": "string", + "default": "copy", + "description": "Method used to save pipeline results to output directory.", + "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", + "fa_icon": "fas fa-copy", + "enum": [ + "symlink", + "rellink", + "link", + "copy", + "copyNoFollow", + "move" + ], + "hidden": true + }, "email_on_fail": { "type": "string", "description": "Email address for completion summary, only when pipeline fails.", diff --git a/nf_core/pipeline-template/subworkflows/local/input_check.nf b/nf_core/pipeline-template/subworkflows/local/input_check.nf index cddcbb3ce0..0aecf87fb7 100644 --- a/nf_core/pipeline-template/subworkflows/local/input_check.nf +++ b/nf_core/pipeline-template/subworkflows/local/input_check.nf @@ -12,7 +12,7 @@ workflow INPUT_CHECK { SAMPLESHEET_CHECK ( samplesheet ) .csv .splitCsv ( header:true, sep:',' ) - .map { create_fastq_channels(it) } + .map { create_fastq_channel(it) } .set { reads } emit: @@ -21,22 +21,24 @@ workflow INPUT_CHECK { } // Function to get list of [ meta, [ fastq_1, fastq_2 ] ] -def create_fastq_channels(LinkedHashMap row) { +def create_fastq_channel(LinkedHashMap row) { + // create meta map def meta = [:] - meta.id = row.sample - meta.single_end = row.single_end.toBoolean() + meta.id = row.sample + meta.single_end = row.single_end.toBoolean() - def array = [] + // add path(s) of the fastq file(s) to the meta map + def fastq_meta = [] if (!file(row.fastq_1).exists()) { exit 1, "ERROR: Please check input samplesheet -> Read 1 FastQ file does not exist!\n${row.fastq_1}" } if (meta.single_end) { - array = [ meta, [ file(row.fastq_1) ] ] + fastq_meta = [ meta, [ file(row.fastq_1) ] ] } else { if (!file(row.fastq_2).exists()) { exit 1, "ERROR: Please check input samplesheet -> Read 2 FastQ file does not exist!\n${row.fastq_2}" } - array = [ meta, [ file(row.fastq_1), file(row.fastq_2) ] ] + fastq_meta = [ meta, [ file(row.fastq_1), file(row.fastq_2) ] ] } - return array + return fastq_meta } diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 460a3cf20a..6ac463f8e8 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -1,7 +1,7 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ VALIDATE INPUTS -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ def summary_params = NfcoreSchema.paramsSummaryMap(workflow, params) @@ -18,18 +18,18 @@ for (param in checkPathParamList) { if (param) { file(param, checkIfExists: true if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input samplesheet not specified!' } /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONFIG FILES -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ ch_multiqc_config = file("$projectDir/assets/multiqc_config.yaml", checkIfExists: true) ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config) : Channel.empty() /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT LOCAL MODULES/SUBWORKFLOWS -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ // @@ -38,9 +38,9 @@ ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multi include { INPUT_CHECK } from '../subworkflows/local/input_check' /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT NF-CORE MODULES/SUBWORKFLOWS -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ // @@ -51,9 +51,9 @@ include { MULTIQC } from '../modules/nf-core/modules/multiqc include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/modules/custom/dumpsoftwareversions/main' /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RUN MAIN WORKFLOW -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ // Info required for completion email and summary @@ -104,9 +104,9 @@ workflow {{ short_name|upper }} { } /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ COMPLETION EMAIL AND SUMMARY -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ workflow.onComplete { @@ -117,7 +117,7 @@ workflow.onComplete { } /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ THE END -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/nf_core/schema.py b/nf_core/schema.py index c3825b4ba6..e26a46ba61 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -4,20 +4,16 @@ from __future__ import print_function from rich.prompt import Confirm +import copy import copy import jinja2 import json import jsonschema import logging +import markdown import os -import requests -import requests_cache -import sys -import time import webbrowser import yaml -import copy -import re import nf_core.list, nf_core.utils @@ -416,6 +412,90 @@ def validate_schema_title_description(self, schema=None): desc_attr, self.schema["description"] ) + def print_documentation( + self, + output_fn=None, + format="markdown", + force=False, + columns=["parameter", "description", "type,", "default", "required", "hidden"], + ): + """ + Prints documentation for the schema. + """ + output = self.schema_to_markdown(columns) + if format == "html": + output = self.markdown_to_html(output) + + # Print to file + if output_fn: + if os.path.exists(output_fn) and not force: + log.error(f"File '{output_fn}' exists! Please delete first, or use '--force'") + return + with open(output_fn, "w") as file: + file.write(output) + log.info(f"Documentation written to '{output_fn}'") + + # Return as a string + return output + + def schema_to_markdown(self, columns): + """ + Creates documentation for the schema in Markdown format. + """ + out = f"# {self.schema['title']}\n\n" + out += f"{self.schema['description']}\n" + # Grouped parameters + for definition in self.schema.get("definitions", {}).values(): + out += f"\n## {definition.get('title', {})}\n\n" + out += f"{definition.get('description', '')}\n\n" + out += "".join([f"| {column.title()} " for column in columns]) + out += "|\n" + out += "".join([f"|-----------" for columns in columns]) + out += "|\n" + for p_key, param in definition.get("properties", {}).items(): + for column in columns: + if column == "parameter": + out += f"| `{p_key}` " + elif column == "description": + out += f"| {param.get('description', '')} " + if param.get("help_text", "") != "": + out += f"
Help{param['help_text']}
" + elif column == "type": + out += f"| `{param.get('type', '')}` " + else: + out += f"| {param.get(column, '')} " + out += "|\n" + + # Top-level ungrouped parameters + if len(self.schema.get("properties", {})) > 0: + out += f"\n## Other parameters\n\n" + out += "".join([f"| {column.title()} " for column in columns]) + out += "|\n" + out += "".join([f"|-----------" for columns in columns]) + out += "|\n" + + for p_key, param in self.schema.get("properties", {}).items(): + for column in columns: + if column == "parameter": + out += f"| `{p_key}` " + elif column == "description": + out += f"| {param.get('description', '')} " + if param.get("help_text", "") != "": + out += f"
Help{param['help_text']}
" + elif column == "type": + out += f"| `{param.get('type', '')}` " + else: + out += f"| {param.get(column, '')} " + out += "|\n" + + return out + + def markdown_to_html(self, markdown_str): + """ + Convert markdown to html + """ + return markdown.markdown(markdown_str, extensions=["tables"]) + def make_skeleton_schema(self): """Make a new pipeline schema from the template""" self.schema_from_scratch = True diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000..02a8acac0f --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +filterwarnings = + ignore::pytest.PytestRemovedIn8Warning:_pytest.nodes:140 diff --git a/requirements.txt b/requirements.txt index 33da40c47e..2c1d25763d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,15 @@ click GitPython jinja2 -jsonschema +jsonschema>=3.0 +markdown>=3.3 packaging prompt_toolkit>=3.0.3 -pyyaml pytest-workflow +pyyaml questionary>=1.8.0 -requests_cache requests +requests_cache +rich-click>=1.0.0 rich>=10.0.0 tabulate diff --git a/setup.py b/setup.py index 8b910389fb..b4acd45832 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -version = "2.2" +version = "2.3" with open("README.md") as f: readme = f.read() diff --git a/tests/test_launch.py b/tests/test_launch.py index 8029213f06..6cc4d0371a 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -77,8 +77,8 @@ def test_make_pipeline_schema(self, tmp_path): assert len(self.launcher.schema_obj.schema["definitions"]["input_output_options"]["properties"]) > 2 assert self.launcher.schema_obj.schema["definitions"]["input_output_options"]["properties"]["outdir"] == { "type": "string", - "description": "Path to the output directory where the results will be saved.", - "default": "./results", + "format": "directory-path", + "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", "fa_icon": "fas fa-folder-open", } @@ -87,7 +87,7 @@ def test_get_pipeline_defaults(self): self.launcher.get_pipeline_schema() self.launcher.set_schema_inputs() assert len(self.launcher.schema_obj.input_params) > 0 - assert self.launcher.schema_obj.input_params["outdir"] == "./results" + assert self.launcher.schema_obj.input_params["validate_params"] == True @with_temporary_file def test_get_pipeline_defaults_input_params(self, tmp_file): diff --git a/tests/test_modules.py b/tests/test_modules.py index fb59537c62..cfa3408e69 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -19,8 +19,8 @@ def create_modules_repo_dummy(tmp_dir): os.makedirs(os.path.join(root_dir, "tests", "config")) with open(os.path.join(root_dir, "tests", "config", "pytest_modules.yml"), "w") as fh: fh.writelines(["test:", "\n - modules/test/**", "\n - tests/modules/test/**"]) - with open(os.path.join(root_dir, "README.md"), "w") as fh: - fh.writelines(["# ![nf-core/modules](docs/images/nfcore-modules_logo.png)", "\n"]) + with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: + fh.writelines(["repository_type: modules", "\n"]) # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_medium", False, False) @@ -47,15 +47,10 @@ def setUp(self): self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) self.mods_install_alt = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=True, force=True) - # TODO Remove comments once external repository to have same structure as nf-core/modules - # self.mods_install_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master") - # Set up remove objects print("Setting up remove objects") self.mods_remove = nf_core.modules.ModuleRemove(self.pipeline_dir) self.mods_remove_alt = nf_core.modules.ModuleRemove(self.pipeline_dir) - # TODO Remove comments once external repository to have same structure as nf-core/modules - # self.mods_remove_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master") # Set up the nf-core/modules repo dummy self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) diff --git a/tests/test_schema.py b/tests/test_schema.py index 3a060a516c..acda087690 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -95,6 +95,18 @@ def test_load_schema(self): self.schema_obj.schema_filename = self.template_schema self.schema_obj.load_schema() + def test_schema_docs(self): + """Try to generate Markdown docs for a schema from a file""" + self.schema_obj.schema_filename = self.template_schema + self.schema_obj.load_schema() + docs = self.schema_obj.print_documentation() + print(docs) + assert self.schema_obj.schema["title"] in docs + assert self.schema_obj.schema["description"] in docs + for definition in self.schema_obj.schema.get("definitions", {}).values(): + assert definition["title"] in docs + assert definition["description"] in docs + @with_temporary_file def test_save_schema(self, tmp_file): """Try to save a schema""" @@ -132,7 +144,7 @@ def test_validate_params_pass(self): # Load the template schema self.schema_obj.schema_filename = self.template_schema self.schema_obj.load_schema() - self.schema_obj.input_params = {"input": "fubar.csv"} + self.schema_obj.input_params = {"input": "fubar.csv", "outdir": "results/"} assert self.schema_obj.validate_params() def test_validate_params_fail(self): diff --git a/tests/test_sync.py b/tests/test_sync.py index 727db70104..68c96e2929 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -79,7 +79,7 @@ def test_get_wf_config_missing_required_config(self): raise UserWarning("Should have hit an exception") except nf_core.sync.SyncException as e: # Check that we did actually get some config back - assert psync.wf_config["params.outdir"] == "'./results'" + assert psync.wf_config["params.validate_params"] == "true" # Check that we raised because of the missing fake config var assert e.args[0] == "Workflow config variable `fakethisdoesnotexist` not found!"