From 7feb31fa5da5c936fe569347b02b7ece3e455799 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 22 Nov 2018 09:58:44 +0100 Subject: [PATCH 01/38] Added bioconda badge to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5885dc1924..e9c81ee752 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ # [nf-core/tools](https://github.com/nf-core/tools) [![Build Status](https://travis-ci.org/nf-core/tools.svg?branch=master)](https://travis-ci.org/nf-core/tools) [![codecov](https://codecov.io/gh/nf-core/tools/branch/master/graph/badge.svg)](https://codecov.io/gh/nf-core/tools) +[![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg?style=flat-square)](http://bioconda.github.io/recipes/nf-core/README.html) A python package with helper tools for the nf-core community. From b9d9c5866d3630d535fd3d863a034ed1555f2bd1 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Sat, 24 Nov 2018 10:43:17 +0000 Subject: [PATCH 02/38] Moved custom Institutional configs to nf-core configs on github --- CHANGELOG.md | 8 ++++ .../nextflow.config | 42 +++++++------------ 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17b65d5018..bb4194c289 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # nf-core/tools: Changelog +## v1.3dev +* Institutional custom config profiles moved to github `nf-core/configs` + * These will now be maintained centrally as opposed to being shipped with the pipelines in `conf/` + * Load `base.config` by default for all profiles + * Removed profiles named `standard` and `none` + * Added parameter `--igenomesIgnore` so `igenomes.config` isnt loaded if parameter clashes are observed + * Added parameter `--custom_config_version` for custom config version control. Can use this parameter to provide commit id for reproducibility. Defaults to `master` + ##[v1.3](https://github.com/nf-core/tools/releases/tag/1.3) - 2018-11-21 * `nf-core create` command line interface updated * Interactive prompts for required arguments if not given diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index ac3bdf090e..0fbaa34ec5 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -32,46 +32,34 @@ params { clusterOptions = false awsqueue = false awsregion = 'eu-west-1' + igenomesIgnore = false + custom_config_version = 'master' } +// Load base.config by default for all pipelines +includeConfig 'conf/base.config' + +// Load nf-core custom profiles from different Institutions +includeConfig "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}/nfcore_custom.config" + profiles { - awsbatch { - includeConfig 'conf/base.config' - includeConfig 'conf/awsbatch.config' - includeConfig 'conf/igenomes.config' - } - binac { - includeConfig 'conf/base.config' - includeConfig 'conf/binac.config' - } - cfc { - includeConfig 'conf/base.config' - includeConfig 'conf/cfc.config' - } + awsbatch { includeConfig 'conf/awsbatch.config' } conda { process.conda = "$baseDir/environment.yml" } debug { process.beforeScript = 'echo $HOSTNAME' } docker { docker.enabled = true process.container = params.container } - none { - // Don't load any config (for use with custom home configs) - } singularity { singularity.enabled = true process.container = {"shub://${params.container.replace('nfcore', 'nf-core')}"} } - standard { - includeConfig 'conf/base.config' - } - test { - includeConfig 'conf/base.config' - includeConfig 'conf/test.config' - } - uzh { - includeConfig 'conf/base.config' - includeConfig 'conf/uzh.config' - } + test { includeConfig 'conf/test.config' } +} + +// Load igenomes.config if required +if(!params.igenomesIgnore){ + includeConfig 'conf/igenomes.config' } // Capture exit codes from upstream processes when piping From fc49103a5ad090510e4db7b38d20b9d3ab845bde Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 26 Nov 2018 12:09:21 +0000 Subject: [PATCH 03/38] Fixed a couple of typos --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb4194c289..5b8d4c7b26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,14 @@ # nf-core/tools: Changelog -## v1.3dev +## v1.4dev * Institutional custom config profiles moved to github `nf-core/configs` * These will now be maintained centrally as opposed to being shipped with the pipelines in `conf/` * Load `base.config` by default for all profiles * Removed profiles named `standard` and `none` - * Added parameter `--igenomesIgnore` so `igenomes.config` isnt loaded if parameter clashes are observed + * Added parameter `--igenomesIgnore` so `igenomes.config` is not loaded if parameter clashes are observed * Added parameter `--custom_config_version` for custom config version control. Can use this parameter to provide commit id for reproducibility. Defaults to `master` -##[v1.3](https://github.com/nf-core/tools/releases/tag/1.3) - 2018-11-21 +## [v1.3](https://github.com/nf-core/tools/releases/tag/1.3) - 2018-11-21 * `nf-core create` command line interface updated * Interactive prompts for required arguments if not given * New flag for workflow author From 2dab27f800009b38e552c4d809218eb13be849d7 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 26 Nov 2018 12:30:25 +0000 Subject: [PATCH 04/38] Referenced in README.md but missing from template --- .../docs/configuration/local.md | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/local.md diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/local.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/local.md new file mode 100644 index 0000000000..bc8575ff60 --- /dev/null +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/local.md @@ -0,0 +1,46 @@ +# {{ cookiecutter.name }}: Local Configuration + +If running the pipeline in a local environment, we highly recommend using either Docker or Singularity. + +## Docker +Docker is a great way to run `{{ cookiecutter.name }}`, as it manages all software installations and allows the pipeline to be run in an identical software environment across a range of systems. + +Nextflow has [excellent integration](https://www.nextflow.io/docs/latest/docker.html) with Docker, and beyond installing the two tools, not much else is required. The `{{ cookiecutter.name }}` profile comes with a configuration profile for docker, making it very easy to use. This also comes with the required presets to use the AWS iGenomes resource, meaning that if using common reference genomes you just specify the reference ID and it will be automatically downloaded from AWS S3. + +First, install docker on your system: [Docker Installation Instructions](https://docs.docker.com/engine/installation/) + +Then, simply run the analysis pipeline: +```bash +nextflow run {{ cookiecutter.name }} -profile docker --genome '' --design '' +``` + +Nextflow will recognise `{{ cookiecutter.name }}` and download the pipeline from GitHub. The `-profile docker` configuration lists the [{{ cookiecutter.name }}](https://hub.docker.com/r/{{ cookiecutter.name_docker }}/) image that we have created and is hosted at dockerhub, and this is downloaded. + +For more information about how to work with reference genomes, see [`docs/configuration/reference_genomes.md`](reference_genomes.md). + +### Pipeline versions +The public docker images are tagged with the same version numbers as the code, which you can use to ensure reproducibility. When running the pipeline, specify the pipeline version with `-r`, for example `-r 1.0`. This uses pipeline code and docker image from this tagged version. + + +## Singularity image +Many HPC environments are not able to run Docker due to security issues. [Singularity](http://singularity.lbl.gov/) is a tool designed to run on such HPC systems which is very similar to Docker. Even better, it can use create images directly from dockerhub. + +To use the singularity image for a single run, use `-with-singularity`. This will download the docker container from dockerhub and create a singularity image for you dynamically. + +If you intend to run the pipeline offline, nextflow will not be able to automatically download the singularity image for you. Instead, you'll have to do this yourself manually first, transfer the image file and then point to that. + +First, pull the image file where you have an internet connection: + +> NB: The "tag" at the end of this command corresponds to the pipeline version. +> Here, we're pulling the docker image for version 1.0 of the {{ cookiecutter.name }} pipeline +> Make sure that this tag corresponds to the version of the pipeline that you're using + +```bash +singularity pull --name {{ cookiecutter.name_noslash }}-1.0.img docker://{{ cookiecutter.name }}:1.0 +``` + +Then transfer this file and run the pipeline with this path: + +```bash +nextflow run /path/to/{{ cookiecutter.name_noslash }} -with-singularity /path/to/{{ cookiecutter.name_noslash }}-1.0.img +``` From aec3a91df6945546fde5352e63ab828706e2b69f Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 26 Nov 2018 12:37:31 +0000 Subject: [PATCH 05/38] Custom configs have now been moved to nf-core/configs on github --- .../conf/binac.config | 22 ------------------- .../conf/cfc.config | 21 ------------------ .../conf/uzh.config | 19 ---------------- 3 files changed, 62 deletions(-) delete mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/binac.config delete mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/cfc.config delete mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/uzh.config diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/binac.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/binac.config deleted file mode 100644 index be69d9e8c4..0000000000 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/binac.config +++ /dev/null @@ -1,22 +0,0 @@ -/* - * ---------------------------------------------------------------------------- - * Nextflow config file for use with Singularity on BINAC cluster in Tuebingen - * ---------------------------------------------------------------------------- - * Defines basic usage limits and singularity image id. - */ - -singularity { - enabled = true -} - -process { - beforeScript = 'module load devel/singularity/3.0.1' - executor = 'pbs' - queue = 'short' -} - -params { - max_memory = 128.GB - max_cpus = 28 - max_time = 48.h -} diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/cfc.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/cfc.config deleted file mode 100644 index 6285925b8f..0000000000 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/cfc.config +++ /dev/null @@ -1,21 +0,0 @@ -/* - * ------------------------------------------------------------- - * Nextflow config file for use with Singularity on CFC at QBIC - * ------------------------------------------------------------- - * Defines basic usage limits and singularity image id. - */ - -singularity { - enabled = true -} - -process { - beforeScript = 'module load qbic/singularity_slurm/3.0.1' - executor = 'slurm' -} - -params { - max_memory = 60.GB - max_cpus = 24 - max_time = 140.h -} diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/uzh.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/uzh.config deleted file mode 100644 index 68cd7dd587..0000000000 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/uzh.config +++ /dev/null @@ -1,19 +0,0 @@ -/* - * -------------------------------------------------------------------------------- - * Nextflow config file for use with Singularity on University of Zurich Cluster - * -------------------------------------------------------------------------------- - */ - -singularity { - enabled = true -} - -process { - executor = 'slurm' -} - -params { - max_memory = 1800.GB - max_cpus = 112 - max_time = 168.h -} From 0bce4cb8f057c9653b9846b2cbbcf6d8f77338c6 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 26 Nov 2018 12:37:59 +0000 Subject: [PATCH 06/38] Added details for adding local.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b8d4c7b26..b04f98153e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ * Removed profiles named `standard` and `none` * Added parameter `--igenomesIgnore` so `igenomes.config` is not loaded if parameter clashes are observed * Added parameter `--custom_config_version` for custom config version control. Can use this parameter to provide commit id for reproducibility. Defaults to `master` + * Deleted custom configs from template in `conf/` directory i.e. `uzh.config`, `binac.config` and `cfc.config` +* Added `local.md` to cookiecutter template in `docs/configuration/`. This was referenced in `README.md` but not present. ## [v1.3](https://github.com/nf-core/tools/releases/tag/1.3) - 2018-11-21 * `nf-core create` command line interface updated From 309ec507a9ed4a3897cc30c41c59e7832dfa5bb3 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 27 Nov 2018 13:20:53 +0100 Subject: [PATCH 07/38] Fix tests The openjdk version definened in the example was not the latest anymore, so it suddenly appeared in the linting warning list and caused the linting to fail. --- tests/test_lint.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_lint.py b/tests/test_lint.py index ddeac79ac6..700cfc99ad 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -38,7 +38,7 @@ def pf(wd, path): pf(WD, 'lint_examples/license_incomplete_example')] # The maximum sum of passed tests currently possible -MAX_PASS_CHECKS = 62 +MAX_PASS_CHECKS = 61 # The additional tests passed for releases ADD_PASS_RELEASE = 1 @@ -58,7 +58,7 @@ def test_call_lint_pipeline_pass(self): This should not result in any exception for the minimal working example""" lint_obj = nf_core.lint.run_linting(PATH_WORKING_EXAMPLE, False) - expectations = {"failed": 0, "warned": 2, "passed": MAX_PASS_CHECKS} + expectations = {"failed": 0, "warned": 3, "passed": MAX_PASS_CHECKS} self.assess_lint_status(lint_obj, **expectations) @pytest.mark.xfail(raises=AssertionError) @@ -73,7 +73,7 @@ def test_call_lint_pipeline_release(self): """Test the main execution function of PipelineLint when running with --release""" lint_obj = nf_core.lint.PipelineLint(PATH_WORKING_EXAMPLE) lint_obj.lint_pipeline(release=True) - expectations = {"failed": 0, "warned": 2, "passed": MAX_PASS_CHECKS + ADD_PASS_RELEASE} + expectations = {"failed": 0, "warned": 3, "passed": MAX_PASS_CHECKS + ADD_PASS_RELEASE} self.assess_lint_status(lint_obj, **expectations) def test_failing_dockerfile_example(self): @@ -264,7 +264,7 @@ def test_conda_env_pass(self): lint_obj.pipeline_name = 'tools' lint_obj.config['manifest.version'] = '0.4' lint_obj.check_conda_env_yaml() - expectations = {"failed": 0, "warned": 2, "passed": 5} + expectations = {"failed": 0, "warned": 3, "passed": 4} self.assess_lint_status(lint_obj, **expectations) def test_conda_env_fail(self): From 9b245f58e00e349326c4ec223071c660239e1dc7 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 27 Nov 2018 12:48:49 +0100 Subject: [PATCH 08/38] Attempt at making PRs fail if the CHANGELOG.md is not updated. --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b724febf84..cb12e9cd94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,11 @@ python: before_install: # PRs to master are only ok if coming from dev branch - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ])' + # Check that the changelog has been updated if this is a PR + - echo $TRAVIS_BRANCH + - echo $(git diff --name-only HEAD...$TRAVIS_BRANCH) + - echo $(git diff --name-only HEAD...$TRAVIS_BRANCH | grep CHANGELOG.md | wc -l) + - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $(git diff --name-only HEAD...$TRAVIS_BRANCH | grep CHANGELOG.md | wc -l) -eq 1 ]' install: # Install Nextflow - mkdir /tmp/nextflow @@ -41,5 +46,5 @@ deploy: python: '3.6' # Sync pipelines with possible nf-core template changes -after_deploy: +after_deploy: - ./bin/sync From c9ca1caea23fe20da06e8ea626c6bfbce882107b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 27 Nov 2018 13:42:58 +0100 Subject: [PATCH 09/38] Fetch remotes first See https://github.com/travis-ci/travis-ci/issues/6069#issuecomment-319710346 --- .travis.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index cb12e9cd94..d1d06156dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,10 +10,12 @@ before_install: # PRs to master are only ok if coming from dev branch - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ])' # Check that the changelog has been updated if this is a PR + - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + - git fetch - echo $TRAVIS_BRANCH - - echo $(git diff --name-only HEAD...$TRAVIS_BRANCH) - - echo $(git diff --name-only HEAD...$TRAVIS_BRANCH | grep CHANGELOG.md | wc -l) - - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $(git diff --name-only HEAD...$TRAVIS_BRANCH | grep CHANGELOG.md | wc -l) -eq 1 ]' + - git diff --name-only HEAD origin/${TRAVIS_BRANCH} + - git diff --name-only HEAD origin/${TRAVIS_BRANCH} | grep CHANGELOG.md | wc -l + - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $(git diff --name-only HEAD origin/${TRAVIS_BRANCH} | grep CHANGELOG.md | wc -l) -eq 1 ]' install: # Install Nextflow - mkdir /tmp/nextflow From 155dff5eaaa030b860ef06ef34e58f13a1f96a63 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 27 Nov 2018 13:54:34 +0100 Subject: [PATCH 10/38] Clean up debugging statements --- .travis.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index d1d06156dd..96aec3adfc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,11 +10,7 @@ before_install: # PRs to master are only ok if coming from dev branch - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ])' # Check that the changelog has been updated if this is a PR - - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - - git fetch - - echo $TRAVIS_BRANCH - - git diff --name-only HEAD origin/${TRAVIS_BRANCH} - - git diff --name-only HEAD origin/${TRAVIS_BRANCH} | grep CHANGELOG.md | wc -l + - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $(git diff --name-only HEAD origin/${TRAVIS_BRANCH} | grep CHANGELOG.md | wc -l) -eq 1 ]' install: # Install Nextflow From 6411bedf928e9c67936302b29575459f347b3d92 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 27 Nov 2018 13:55:33 +0100 Subject: [PATCH 11/38] Update the changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 400557a040..464e891c0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * Added parameter `--custom_config_version` for custom config version control. Can use this parameter to provide commit id for reproducibility. Defaults to `master` * Deleted custom configs from template in `conf/` directory i.e. `uzh.config`, `binac.config` and `cfc.config` * Added `local.md` to cookiecutter template in `docs/configuration/`. This was referenced in `README.md` but not present. +* Make Travis CI tests fail on pull requests if the `CHANGELOG.md` file hasn't been updated ## [v1.3](https://github.com/nf-core/tools/releases/tag/1.3) - 2018-11-21 From 7293adacac4496974719f34939f2d2b6aea10c25 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 28 Nov 2018 16:13:48 +0000 Subject: [PATCH 12/38] Changed listing and added link to local installation --- .../{{cookiecutter.name_noslash}}/docs/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md index ea4a6da4b9..9996db35ad 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md @@ -3,9 +3,10 @@ The {{ cookiecutter.name }} documentation is split into the following files: 1. [Installation](installation.md) -2. [Running the pipeline](usage.md) -3. Pipeline configuration +2. Pipeline configuration + * [Local installation](configuration/local.md) * [Adding your own system](configuration/adding_your_own.md) * [Reference genomes](configuration/reference_genomes.md) +3. [Running the pipeline](usage.md) 4. [Output and how to interpret the results](output.md) 5. [Troubleshooting](troubleshooting.md) From fd5a1cccb184c321d84b5e3df9d894d811bc1acd Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 28 Nov 2018 16:15:51 +0000 Subject: [PATCH 13/38] Added bullet for reference genomes in line with docs README.md --- .../pipeline-template/{{cookiecutter.name_noslash}}/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md index b27e9cc4ff..d746645b30 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md @@ -20,6 +20,7 @@ The {{ cookiecutter.name }} pipeline comes with documentation about the pipeline 2. Pipeline configuration * [Local installation](docs/configuration/local.md) * [Adding your own system](docs/configuration/adding_your_own.md) + * [Reference genomes](docs/configuration/reference_genomes.md) 3. [Running the pipeline](docs/usage.md) 4. [Output and how to interpret the results](docs/output.md) 5. [Troubleshooting](docs/troubleshooting.md) From 10013f2b96236622f558fa911d73c998f9b7ebc5 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 28 Nov 2018 16:16:27 +0000 Subject: [PATCH 14/38] Major overhaul, removal and addition of parameters --- .../docs/usage.md | 67 +++++++++---------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 02a5b74572..28e530bc25 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -8,18 +8,18 @@ * [Reproducibility](#reproducibility) * [Main arguments](#main-arguments) * [`-profile`](#-profile-single-dash) - * [`docker`](#docker) * [`awsbatch`](#awsbatch) - * [`standard`](#standard) - * [`binac`](#binac) - * [`cfc`](#cfc) - * [`none`](#none) + * [`conda`](#conda) + * [`docker`](#docker) + * [`singularity`](#singularity) + * [`test`](#test) * [`--reads`](#--reads) * [`--singleEnd`](#--singleend) -* [Reference Genomes](#reference-genomes) +* [Reference genomes](#reference-genomes) * [`--genome`](#--genome) * [`--fasta`](#--fasta) -* [Job Resources](#job-resources) + * [`--igenomesIgnore`](#--igenomesignore) +* [Job resources](#job-resources) * [Automatic resubmission](#automatic-resubmission) * [Custom resource requests](#custom-resource-requests) * [AWS batch specific parameters](#aws-batch-specific-parameters) @@ -32,11 +32,11 @@ * [`-name`](#-name-single-dash) * [`-resume`](#-resume-single-dash) * [`-c`](#-c-single-dash) + * [`--custom_config_version`](#--custom_config_version) * [`--max_memory`](#--max_memory) * [`--max_time`](#--max_time) * [`--max_cpus`](#--max_cpus) - * [`--plaintext_emails`](#--plaintext_emails) - * [`--sampleLevel`](#--sampleLevel) + * [`--plaintext_email`](#--plaintext_email) * [`--multiqc_config`](#--multiqc_config) @@ -48,7 +48,7 @@ It is recommended to limit the Nextflow Java virtual machines memory. We recomme ```bash NXF_OPTS='-Xms1g -Xmx4g' ``` - + ## Running the pipeline The typical command for running the pipeline is as follows: ```bash @@ -81,36 +81,27 @@ First, go to the [{{ cookiecutter.name }} releases page](https://github.com/{{ c This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. -## Main Arguments +## Main arguments ### `-profile` -Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. Note that multiple profiles can be loaded, for example: `-profile standard,docker` - the order of arguments is important! +Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. Note that multiple profiles can be loaded, for example: `-profile standard,docker` - the order of arguments is important! -* `standard` - * The default profile, used if `-profile` is not specified at all. - * Runs locally and expects all software to be installed and available on the `PATH`. +If `-profile` is not specified at all the pipeline will be run locally and expects all software to be installed and available on the `PATH`. + +* `awsbatch` + * A generic configuration profile to be used with AWS Batch. +* `conda` + * A generic configuration profile to be used with [conda](https://conda.io/docs/) + * Pulls most software from [Bioconda](https://bioconda.github.io/) * `docker` * A generic configuration profile to be used with [Docker](http://docker.com/) * Pulls software from dockerhub: [`{{ cookiecutter.name_docker }}`](http://hub.docker.com/r/{{ cookiecutter.name_docker }}/) * `singularity` * A generic configuration profile to be used with [Singularity](http://singularity.lbl.gov/) * Pulls software from singularity-hub -* `conda` - * A generic configuration profile to be used with [conda](https://conda.io/docs/) - * Pulls most software from [Bioconda](https://bioconda.github.io/) -* `binac` - * A profile for the [BinAC](https://www.bwhpc-c5.de/wiki/index.php/Category:BwForCluster_BinAC) cluster - * Pulls images via Singularity from Dockerhub automatically -* `cfc` - * A profile for the Core Facility Cluster at QBiC Tuebingen - * Pulls images via Singularity from Dockerhub automatically -* `awsbatch` - * A generic configuration profile to be used with AWS Batch. * `test` * A profile with a complete configuration for automated testing * Includes links to test data so needs no other parameters -* `none` - * No configuration at all. Useful if you want to build your own config from scratch and want to avoid loading in the default `base` config profile (not recommended). ### `--reads` @@ -138,7 +129,7 @@ By default, the pipeline expects paired-end data. If you have single-end data, y It is not possible to run a mixture of single-end and paired-end files in one run. -## Reference Genomes +## Reference genomes The pipeline config files come bundled with paths to the illumina iGenomes reference index files. If running with docker or AWS, the configuration is set up to use the [AWS-iGenomes](https://ewels.github.io/AWS-iGenomes/) resource. @@ -182,12 +173,17 @@ If you prefer, you can specify the full path to your reference genome when you r --fasta '[path to Fasta reference]' ``` -## Job Resources +### `--igenomesIgnore` +Do not load `igenomes.config` when running the pipeline. You may choose this option if you observe clashes between custom parameters and those supplied in `igenomes.config`. + +## Job resources ### Automatic resubmission Each step in the pipeline has a default set of requirements for number of CPUs, memory and time. For most of the steps in the pipeline, if the job exits with an error code of `143` (exceeded requested resources) it will automatically resubmit with higher requests (2 x original, then 3 x original). If it still fails after three times then the pipeline is stopped. ### Custom resource requests -Wherever process-specific requirements are set in the pipeline, the default value can be changed by creating a custom config file. See the files in [`conf`](../conf) for examples. +Wherever process-specific requirements are set in the pipeline, the default value can be changed by creating a custom config file. See the files hosted at [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf) for examples. + +If you are likely to be running `nf-core` pipelines regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter (see definition below). You can request your custom config file to be added in the `pipelines` chat on the Slack messenger for `nf-core` (may need to be invited first!) or on the `nf-core/Lobby` group on Gitter. ## AWS Batch specific parameters Running the pipeline on AWS Batch requires a couple of specific parameters to be set according to your AWS Batch configuration. Please use the `-awsbatch` profile and then specify all of the following parameters. @@ -229,9 +225,12 @@ Specify the path to a specific config file (this is a core NextFlow command). Note - you can use this to override pipeline defaults. +### `--custom_config_version` +Provide git commit id for custom Institutional configs hosted at `nf-core/configs`. This was implemented for reproducibility purposes. Default is set to `master`. + ### `--max_memory` Use to set a top-limit for the default memory requirement for each process. -Should be a string in the format integer-unit. eg. `--max_memory '8.GB'`` +Should be a string in the format integer-unit. eg. `--max_memory '8.GB'` ### `--max_time` Use to set a top-limit for the default time requirement for each process. @@ -242,7 +241,7 @@ Use to set a top-limit for the default CPU requirement for each process. Should be a string in the format integer-unit. eg. `--max_cpus 1` ### `--plaintext_email` -Set to receive plain-text e-mails instead of HTML formatted. +Set to receive plain-text e-mails instead of HTML formatted. -### `--multiqc_config` +### `--multiqc_config` Specify a path to a custom MultiQC configuration file. From 5fd3cf0b7f4a02bc3cd348eac561945d09b0c7e4 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 28 Nov 2018 16:17:09 +0000 Subject: [PATCH 15/38] Added a bit of info for providing custom config files --- .../docs/configuration/adding_your_own.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md index 7a9d8acee5..99b8314f12 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md @@ -2,11 +2,11 @@ It is entirely possible to run this pipeline on other clusters, though you will need to set up your own config file so that the pipeline knows how to work with your cluster. -> If you think that there are other people using the pipeline who would benefit from your configuration (eg. other common cluster setups), please let us know. We can add a new configuration and profile which can used by specifying `-profile ` when running the pipeline. +> If you think that there are other people using the pipeline who would benefit from your configuration (eg. other common cluster setups), please let us know. We can add a new configuration and profile which can used by specifying `-profile ` when running the pipeline. The config file would then be hosted at `nf-core/configs` and will be pulled automatically before the pipeline is executed. If you are the only person to be running this pipeline, you can create your config file as `~/.nextflow/config` and it will be applied every time you run Nextflow. Alternatively, save the file anywhere and reference it when running the pipeline with `-c path/to/config` (see the [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more). -A basic configuration comes with the pipeline, which runs by default (the `standard` config profile - see [`conf/base.config`](../conf/base.config)). This means that you only need to configure the specifics for your system and overwrite any defaults that you want to change. +A basic configuration comes with the pipeline, which loads the [`conf/base.config`](../../conf/base.config)) by default. This means that you only need to configure the specifics for your system and overwrite any defaults that you want to change. ## Cluster Environment By default, pipeline uses the `local` Nextflow executor - in other words, all jobs are run in the login session. If you're using a simple server, this may be fine. If you're using a compute cluster, this is bad as all jobs will run on the head node. From 34fdb74c72a01efa35ce7f0c31d2646c01d1ad18 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 28 Nov 2018 16:18:03 +0000 Subject: [PATCH 16/38] Bug-fixes and adding links to missing sections --- .../docs/installation.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md index d1e0fc7d89..cfa6651132 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md @@ -12,7 +12,7 @@ To start using the {{ cookiecutter.name }} pipeline, follow the steps below: * [Software deps: Bioconda](#32-software-deps-bioconda) * [Configuration profiles](#33-configuration-profiles) 4. [Reference genomes](#4-reference-genomes) -5. [Appendices](#appendices) +5. [Appendices](#5-appendices) * [Running on UPPMAX](#running-on-uppmax) ## 1) Install NextFlow @@ -100,8 +100,15 @@ This is slower and less reproducible than the above, but is still better than ha The pipeline ships with a conda environment file and nextflow has built-in support for this. To use it first ensure that you have conda installed (we recommend [miniconda](https://conda.io/miniconda.html)), then follow the same pattern as above and use the flag `-profile standard,conda` +#### 3.3) Configuration profiles -## Appendices +See [`docs/configuration/adding_your_own.md`](configuration/adding_your_own.md) + +## 4) Reference genomes + +See [`docs/configuration/reference_genomes.md`](configuration/reference_genomes.md) + +## 5) Appendices #### Running on UPPMAX To run the pipeline on the [Swedish UPPMAX](https://www.uppmax.uu.se/) clusters (`rackham`, `irma`, `bianca` etc), use the command line flag `-profile uppmax`. This tells Nextflow to submit jobs using the SLURM job executor with Singularity for software dependencies. From f5270e3c0b1d41f163af3cbd1b69e00f3d2c727b Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 28 Nov 2018 16:18:37 +0000 Subject: [PATCH 17/38] Added TODO statment if parameters arent relevant --- .../{{cookiecutter.name_noslash}}/docs/troubleshooting.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/troubleshooting.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/troubleshooting.md index 7368d57dae..2ed0587eec 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/troubleshooting.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/troubleshooting.md @@ -1,5 +1,7 @@ # {{ cookiecutter.name }}: Troubleshooting + + ## Input files not found If only no file, only one input file , or only read one and not read two is picked up then something is wrong with your input file declaration From 11b776aa8fc720752573149a2859a890c8b78d83 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 28 Nov 2018 16:22:50 +0000 Subject: [PATCH 18/38] Fixed typo --- .../docs/configuration/adding_your_own.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md index 99b8314f12..f2d1172932 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md @@ -6,7 +6,7 @@ It is entirely possible to run this pipeline on other clusters, though you will If you are the only person to be running this pipeline, you can create your config file as `~/.nextflow/config` and it will be applied every time you run Nextflow. Alternatively, save the file anywhere and reference it when running the pipeline with `-c path/to/config` (see the [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more). -A basic configuration comes with the pipeline, which loads the [`conf/base.config`](../../conf/base.config)) by default. This means that you only need to configure the specifics for your system and overwrite any defaults that you want to change. +A basic configuration comes with the pipeline, which loads the [`conf/base.config`](../../conf/base.config) by default. This means that you only need to configure the specifics for your system and overwrite any defaults that you want to change. ## Cluster Environment By default, pipeline uses the `local` Nextflow executor - in other words, all jobs are run in the login session. If you're using a simple server, this may be fine. If you're using a compute cluster, this is bad as all jobs will run on the head node. From 199ac83e47d3bf394161722ce23f7cc27d7cacb0 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 28 Nov 2018 16:32:14 +0000 Subject: [PATCH 19/38] Added description of changes --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 464e891c0e..2a3dd2ac40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ * Deleted custom configs from template in `conf/` directory i.e. `uzh.config`, `binac.config` and `cfc.config` * Added `local.md` to cookiecutter template in `docs/configuration/`. This was referenced in `README.md` but not present. * Make Travis CI tests fail on pull requests if the `CHANGELOG.md` file hasn't been updated - +* Major overhaul of docs to add/remove parameters, unify linking of files and added description for providing custom contigs where necessary ## [v1.3](https://github.com/nf-core/tools/releases/tag/1.3) - 2018-11-21 * `nf-core create` command line interface updated From c74fc2d30da39631f6bca402d06b76b0036f5cab Mon Sep 17 00:00:00 2001 From: Sven F Date: Wed, 28 Nov 2018 19:00:36 +0000 Subject: [PATCH 20/38] Update nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md Co-Authored-By: drpatelh --- .../docs/configuration/adding_your_own.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md index f2d1172932..b11504716d 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md @@ -2,7 +2,7 @@ It is entirely possible to run this pipeline on other clusters, though you will need to set up your own config file so that the pipeline knows how to work with your cluster. -> If you think that there are other people using the pipeline who would benefit from your configuration (eg. other common cluster setups), please let us know. We can add a new configuration and profile which can used by specifying `-profile ` when running the pipeline. The config file would then be hosted at `nf-core/configs` and will be pulled automatically before the pipeline is executed. +> If you think that there are other people using the pipeline who would benefit from your configuration (eg. other common cluster setups), please let us know. We can add a new configuration and profile which can used by specifying `-profile ` when running the pipeline. The config file will then be hosted at `nf-core/configs` and be pulled automatically before the pipeline is executed. If you are the only person to be running this pipeline, you can create your config file as `~/.nextflow/config` and it will be applied every time you run Nextflow. Alternatively, save the file anywhere and reference it when running the pipeline with `-c path/to/config` (see the [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more). From b94d14807269b4bdc35b9aef084e0e4d10a425f6 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 28 Nov 2018 22:41:37 +0000 Subject: [PATCH 21/38] Removed whitespace --- .../{{cookiecutter.name_noslash}}/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 28e530bc25..3f5d30980f 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -241,7 +241,7 @@ Use to set a top-limit for the default CPU requirement for each process. Should be a string in the format integer-unit. eg. `--max_cpus 1` ### `--plaintext_email` -Set to receive plain-text e-mails instead of HTML formatted. +Set to receive plain-text e-mails instead of HTML formatted. ### `--multiqc_config` Specify a path to a custom MultiQC configuration file. From 4d687f545a6d96b6318d6e00d7fcb3c274299dc9 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 29 Nov 2018 10:11:37 +0000 Subject: [PATCH 22/38] Minor change in description for custom configs --- .../docs/configuration/adding_your_own.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md index b11504716d..ec1beba756 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md @@ -2,7 +2,7 @@ It is entirely possible to run this pipeline on other clusters, though you will need to set up your own config file so that the pipeline knows how to work with your cluster. -> If you think that there are other people using the pipeline who would benefit from your configuration (eg. other common cluster setups), please let us know. We can add a new configuration and profile which can used by specifying `-profile ` when running the pipeline. The config file will then be hosted at `nf-core/configs` and be pulled automatically before the pipeline is executed. +> If you think that there are other people using the pipeline who would benefit from your configuration (eg. other common cluster setups), please let us know. We can add a new configuration and profile which can used by specifying `-profile ` when running the pipeline. The config file will then be hosted at `nf-core/configs` and will be pulled automatically before the pipeline is executed. If you are the only person to be running this pipeline, you can create your config file as `~/.nextflow/config` and it will be applied every time you run Nextflow. Alternatively, save the file anywhere and reference it when running the pipeline with `-c path/to/config` (see the [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more). From 4857789ab2a8777bfb43773bdb8f78cbd683644e Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 29 Nov 2018 10:12:23 +0000 Subject: [PATCH 23/38] More details for custom config and example for --custom_config_version --- .../{{cookiecutter.name_noslash}}/docs/usage.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 3f5d30980f..72d2768902 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -183,7 +183,9 @@ Each step in the pipeline has a default set of requirements for number of CPUs, ### Custom resource requests Wherever process-specific requirements are set in the pipeline, the default value can be changed by creating a custom config file. See the files hosted at [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf) for examples. -If you are likely to be running `nf-core` pipelines regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter (see definition below). You can request your custom config file to be added in the `pipelines` chat on the Slack messenger for `nf-core` (may need to be invited first!) or on the `nf-core/Lobby` group on Gitter. +If you are likely to be running `nf-core` pipelines regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter (see definition below). You can then create a pull request to the `nf-core/configs` repository with the addition of your config file, associated documentation file (see examples in [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs)), and amending [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) to include your custom profile. + +If you have any questions or issues please send us a message on [`Slack`](https://nf-core-invite.herokuapp.com/). ## AWS Batch specific parameters Running the pipeline on AWS Batch requires a couple of specific parameters to be set according to your AWS Batch configuration. Please use the `-awsbatch` profile and then specify all of the following parameters. @@ -228,6 +230,11 @@ Note - you can use this to override pipeline defaults. ### `--custom_config_version` Provide git commit id for custom Institutional configs hosted at `nf-core/configs`. This was implemented for reproducibility purposes. Default is set to `master`. +```bash +## Download and use config file with following git commid id +--custom_config_version d52db660777c4bf36546ddb188ec530c3ada1b96 +``` + ### `--max_memory` Use to set a top-limit for the default memory requirement for each process. Should be a string in the format integer-unit. eg. `--max_memory '8.GB'` From 5a22dc3684fbc68e8ed4c43cfa404994c2aed578 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 3 Dec 2018 14:25:16 +0100 Subject: [PATCH 24/38] Removed unused import statements --- bin/sync | 18 ++++++------------ bin/syncutils/template.py | 6 ++---- nf_core/create.py | 1 - nf_core/download.py | 5 ++--- nf_core/lint.py | 1 - nf_core/utils.py | 1 - 6 files changed, 10 insertions(+), 22 deletions(-) diff --git a/bin/sync b/bin/sync index 020b840112..1a96592fd4 100755 --- a/bin/sync +++ b/bin/sync @@ -1,15 +1,10 @@ #!/usr/bin/env python -from cookiecutter.main import cookiecutter -import git import json import os import requests from requests.auth import HTTPBasicAuth -import shutil import sys -import subprocess -import tempfile import syncutils.template # Set the default nf-core pipeline template branch @@ -53,7 +48,7 @@ def main(): pipelines = res.json().get('remote_workflows') if not pipelines: print("Pipeline information was empty!") - + # TODO: Remove this line, once we go for production pipelines = [ { @@ -82,21 +77,20 @@ def main(): else: print("Created pull-request for pipeline \'{pipeline}\' successfully." .format(pipeline=pipeline["name"])) - + for pipeline, exception in sync_errors: print("Sync for pipeline {name} failed.".format(name=pipeline)) print(exception) for pipeline, return_code, content in pr_errors: - print("Pull-request for pipeline \'{pipeline}\' failed," + print("Pull-request for pipeline \'{pipeline}\' failed," " got return code {return_code}." .format(pipeline=pipeline, return_code=return_code)) print(content) - + if pr_errors or sync_errors: sys.exit(1) - + sys.exit(0) - + if __name__ == "__main__": main() - diff --git a/bin/syncutils/template.py b/bin/syncutils/template.py index d2f5e33b3f..ddada5cfb0 100644 --- a/bin/syncutils/template.py +++ b/bin/syncutils/template.py @@ -3,8 +3,6 @@ import git import os import shutil -import sys -from cookiecutter.main import cookiecutter import nf_core.create @@ -29,7 +27,7 @@ def __init__(self, pipeline, branch=TEMPLATE_BRANCH, repo_url=""): self.templatedir = tempfile.mkdtemp() self.repo = git.Repo.clone_from(self.repo_url, self.tmpdir) assert self.repo - + def sync(self): """Execute the template update. """ @@ -49,7 +47,7 @@ def context_from_nextflow(self, nf_project_dir): # Fetch the config variables from the Nextflow pipeline config = utils.fetch_wf_config(wf_path=nf_project_dir) - + # Checkout again to configured template branch self.repo.git.checkout("origin/{branch}".format(branch=self.branch), b="{branch}".format(branch=self.branch)) diff --git a/nf_core/create.py b/nf_core/create.py index 7ac492bb81..8651e155cb 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -9,7 +9,6 @@ import git import logging import os -import re import shutil import sys import tempfile diff --git a/nf_core/download.py b/nf_core/download.py index 9bd3b772db..b18cd0015a 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -8,7 +8,6 @@ import logging import hashlib import os -import re import requests import requests_cache import subprocess @@ -77,8 +76,8 @@ def download_workflow(self): self.pull_singularity_image(container) def fetch_workflow_details(self, wfs): - """ Fetch details of nf-core workflow to download - + """ Fetch details of nf-core workflow to download + params: - wfs A nf_core.list.Workflows object """ diff --git a/nf_core/lint.py b/nf_core/lint.py index e52105b831..f0d850fca1 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -11,7 +11,6 @@ import os import re import shlex -import subprocess import tempfile import click diff --git a/nf_core/utils.py b/nf_core/utils.py index 221f3379ad..02a2980c8e 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -3,7 +3,6 @@ Common utility functions for the nf-core python package. """ -import logging import os import subprocess From 0429f748119086bc2be04d75b77a5828b55657ec Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 3 Dec 2018 15:34:22 +0100 Subject: [PATCH 25/38] Updated Changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a3dd2ac40..cbf4a6d368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # nf-core/tools: Changelog ## v1.4dev + +#### Template pipeline * Institutional custom config profiles moved to github `nf-core/configs` * These will now be maintained centrally as opposed to being shipped with the pipelines in `conf/` * Load `base.config` by default for all profiles @@ -9,8 +11,11 @@ * Added parameter `--custom_config_version` for custom config version control. Can use this parameter to provide commit id for reproducibility. Defaults to `master` * Deleted custom configs from template in `conf/` directory i.e. `uzh.config`, `binac.config` and `cfc.config` * Added `local.md` to cookiecutter template in `docs/configuration/`. This was referenced in `README.md` but not present. +* Major overhaul of docs to add/remove parameters, unify linking of files and added description for providing custom configs where necessary + +#### Tools helper code * Make Travis CI tests fail on pull requests if the `CHANGELOG.md` file hasn't been updated -* Major overhaul of docs to add/remove parameters, unify linking of files and added description for providing custom contigs where necessary +* Minor bugfixing in Python code (eg. removing unused import statements) ## [v1.3](https://github.com/nf-core/tools/releases/tag/1.3) - 2018-11-21 * `nf-core create` command line interface updated From b49b3b469c28b4ff2c45aacf657ef79f7a1d2a55 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 4 Dec 2018 13:36:26 +0100 Subject: [PATCH 26/38] Requests caching - multi user fix. Move requests cache setup into a utils function Make cache directory and sqlite file world writeable for multi-user installations --- CHANGELOG.md | 1 + nf_core/lint.py | 12 +----------- nf_core/list.py | 13 +++---------- nf_core/utils.py | 23 +++++++++++++++++++++++ 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbf4a6d368..64e39fde04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ #### Tools helper code * Make Travis CI tests fail on pull requests if the `CHANGELOG.md` file hasn't been updated * Minor bugfixing in Python code (eg. removing unused import statements) +* Made the web requests caching work on multi-user installations ## [v1.3](https://github.com/nf-core/tools/releases/tag/1.3) - 2018-11-21 * `nf-core create` command line interface updated diff --git a/nf_core/lint.py b/nf_core/lint.py index f0d850fca1..3e9ee94bec 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -5,30 +5,20 @@ the nf-core community guidelines. """ -import datetime import logging import io import os import re import shlex -import tempfile import click import requests -import requests_cache import yaml import nf_core.utils # Set up local caching for requests to speed up remote queries -cachedir = os.path.join(tempfile.gettempdir(), 'nfcore_cache') -if not os.path.exists(cachedir): - os.mkdir(cachedir) -requests_cache.install_cache( - os.path.join(cachedir, 'nfcore_cache'), - expire_after=datetime.timedelta(hours=1), - backend='sqlite', -) +nf_core.utils.setup_requests_cachedir() # Don't pick up debug logs from the requests package logging.getLogger("requests").setLevel(logging.WARNING) diff --git a/nf_core/list.py b/nf_core/list.py index 0d19df2801..cc08ac2560 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -11,22 +11,15 @@ import re import subprocess import sys -import tempfile import git import requests -import requests_cache import tabulate +import nf_core.utils + # Set up local caching for requests to speed up remote queries -cachedir = os.path.join(tempfile.gettempdir(), 'nfcore_cache') -if not os.path.exists(cachedir): - os.mkdir(cachedir) -requests_cache.install_cache( - os.path.join(cachedir, 'nfcore_cache'), - expire_after=datetime.timedelta(hours=1), - backend='sqlite', -) +nf_core.utils.setup_requests_cachedir() def list_workflows(sort='release', json=False, keywords=[]): """ Main function to list all nf-core workflows """ diff --git a/nf_core/utils.py b/nf_core/utils.py index 02a2980c8e..9070538d85 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -3,8 +3,10 @@ Common utility functions for the nf-core python package. """ +import datetime import os import subprocess +import tempfile def fetch_wf_config(wf_path): """ @@ -24,3 +26,24 @@ def fetch_wf_config(wf_path): k, v = ul.split(' = ', 1) config[k] = v return config + + +def setup_requests_cachedir(): + """ + Set up local caching for requests to speed up remote queries + """ + + # Only import it if we need it + import requests_cache + + cachedir = os.path.join(tempfile.gettempdir(), 'nfcore_cache') + if not os.path.exists(cachedir): + os.mkdir(cachedir) + requests_cache.install_cache( + os.path.join(cachedir, 'nfcore_cache'), + expire_after=datetime.timedelta(hours=1), + backend='sqlite', + ) + # Make world-writeable so that multi-user installations work + os.chmod(cachedir, 0o777) + os.chmod(os.path.join(cachedir, 'nfcore_cache.sqlite'), 0o777) From b31e9c83ad20d333d8fac1cab21e921a3e5c4b9a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 6 Dec 2018 14:37:26 +0100 Subject: [PATCH 27/38] Add Channel handling of input files --- .../{{cookiecutter.name_noslash}}/main.nf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index d99869dc61..21657502bb 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -95,8 +95,8 @@ if( workflow.profile == 'awsbatch') { } // Stage config files -multiqc_config = file(params.multiqc_config) -output_docs = file("$baseDir/docs/output.md") +ch_multiqc_config = Channel.fromPath(file(params.multiqc_config)) +ch_output_docs = Channel.fromPath(file("$baseDir/docs/output.md")) /* * Create a channel for input read files @@ -232,7 +232,7 @@ process multiqc { publishDir "${params.outdir}/MultiQC", mode: 'copy' input: - file multiqc_config + file multiqc_config from ch_multiqc_config // TODO nf-core: Add in log files from your new processes for MultiQC to find! file ('fastqc/*') from fastqc_results.collect().ifEmpty([]) file ('software_versions/*') from software_versions_yaml @@ -260,7 +260,7 @@ process output_documentation { publishDir "${params.outdir}/Documentation", mode: 'copy' input: - file output_docs + file output_docs from ch_output_docs output: file "results_description.html" From e0ee8ba009c8330c3e42b1da432c01789beae38a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 6 Dec 2018 14:40:45 +0100 Subject: [PATCH 28/38] Add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64e39fde04..02c02cdf53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * Added parameter `--igenomesIgnore` so `igenomes.config` is not loaded if parameter clashes are observed * Added parameter `--custom_config_version` for custom config version control. Can use this parameter to provide commit id for reproducibility. Defaults to `master` * Deleted custom configs from template in `conf/` directory i.e. `uzh.config`, `binac.config` and `cfc.config` +* `multiqc_config` and `output_md` are now put into channels instead of using the files directly (see issue [#222](https://github.com/nf-core/tools/issues/222)) * Added `local.md` to cookiecutter template in `docs/configuration/`. This was referenced in `README.md` but not present. * Major overhaul of docs to add/remove parameters, unify linking of files and added description for providing custom configs where necessary From f8eeeb5f22400734a9c35d10ef73f2064445c2f9 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 7 Dec 2018 11:10:40 +0100 Subject: [PATCH 29/38] Mini bump to get PR to buildagain --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02c02cdf53..dfd323ec39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ * Added parameter `--igenomesIgnore` so `igenomes.config` is not loaded if parameter clashes are observed * Added parameter `--custom_config_version` for custom config version control. Can use this parameter to provide commit id for reproducibility. Defaults to `master` * Deleted custom configs from template in `conf/` directory i.e. `uzh.config`, `binac.config` and `cfc.config` -* `multiqc_config` and `output_md` are now put into channels instead of using the files directly (see issue [#222](https://github.com/nf-core/tools/issues/222)) +* `multiqc_config` and `output_md` are now put into channels instead of using the files directly (see issue [#222](https://github.com/nf-core/tools/issues/222)) * Added `local.md` to cookiecutter template in `docs/configuration/`. This was referenced in `README.md` but not present. * Major overhaul of docs to add/remove parameters, unify linking of files and added description for providing custom configs where necessary From 503813bc0c9acb1a281d42118f33f5f14bfb6435 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 7 Dec 2018 14:42:30 +0100 Subject: [PATCH 30/38] Update nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf Co-Authored-By: apeltzer --- nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 21657502bb..2052b75eff 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -96,7 +96,7 @@ if( workflow.profile == 'awsbatch') { // Stage config files ch_multiqc_config = Channel.fromPath(file(params.multiqc_config)) -ch_output_docs = Channel.fromPath(file("$baseDir/docs/output.md")) +ch_output_docs = Channel.fromPath("$baseDir/docs/output.md") /* * Create a channel for input read files From 46828e6877c4f11f4e776616ac2065b4f0b07e29 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 7 Dec 2018 14:42:36 +0100 Subject: [PATCH 31/38] Update nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf Co-Authored-By: apeltzer --- nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 2052b75eff..578cf825bf 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -95,7 +95,7 @@ if( workflow.profile == 'awsbatch') { } // Stage config files -ch_multiqc_config = Channel.fromPath(file(params.multiqc_config)) +ch_multiqc_config = Channel.fromPath(params.multiqc_config) ch_output_docs = Channel.fromPath("$baseDir/docs/output.md") /* From 93ed1e2b61eb45f1e5db02e6ce5b6b2a92e712ab Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 7 Dec 2018 15:03:49 +0100 Subject: [PATCH 32/38] Handle missing nextflow installation. See nf-core/tools#208 --- CHANGELOG.md | 3 ++- bin/syncutils/utils.py | 9 ++++++--- nf_core/list.py | 6 ++++++ nf_core/utils.py | 3 +++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfd323ec39..782c8537df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ * Added parameter `--igenomesIgnore` so `igenomes.config` is not loaded if parameter clashes are observed * Added parameter `--custom_config_version` for custom config version control. Can use this parameter to provide commit id for reproducibility. Defaults to `master` * Deleted custom configs from template in `conf/` directory i.e. `uzh.config`, `binac.config` and `cfc.config` -* `multiqc_config` and `output_md` are now put into channels instead of using the files directly (see issue [#222](https://github.com/nf-core/tools/issues/222)) +* `multiqc_config` and `output_md` are now put into channels instead of using the files directly (see issue [#222](https://github.com/nf-core/tools/issues/222)) * Added `local.md` to cookiecutter template in `docs/configuration/`. This was referenced in `README.md` but not present. * Major overhaul of docs to add/remove parameters, unify linking of files and added description for providing custom configs where necessary @@ -18,6 +18,7 @@ * Make Travis CI tests fail on pull requests if the `CHANGELOG.md` file hasn't been updated * Minor bugfixing in Python code (eg. removing unused import statements) * Made the web requests caching work on multi-user installations +* Handle exception if nextflow isn't installed ## [v1.3](https://github.com/nf-core/tools/releases/tag/1.3) - 2018-11-21 * `nf-core create` command line interface updated diff --git a/bin/syncutils/utils.py b/bin/syncutils/utils.py index 666c23b13a..10b4e056fc 100644 --- a/bin/syncutils/utils.py +++ b/bin/syncutils/utils.py @@ -10,6 +10,9 @@ def fetch_wf_config(wf_path): try: with open(os.devnull, 'w') as devnull: nfconfig_raw = subprocess.check_output(['nextflow', 'config', '-flat', wf_path], stderr=devnull) + except OSError as e: + if e.errno == os.errno.ENOENT: + raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") except subprocess.CalledProcessError as e: raise AssertionError("`nextflow config` returned non-zero error code: %s,\n %s", e.returncode, e.output) else: @@ -20,10 +23,10 @@ def fetch_wf_config(wf_path): return config def create_context(config): - """Consumes a flat Nextflow config file and will create + """Consumes a flat Nextflow config file and will create a context dictionary with information for the nf-core template creation. - Returns: A dictionary with: + Returns: A dictionary with: { 'pipeline_name': '' 'pipeline_short_description': '' @@ -32,7 +35,7 @@ def create_context(config): """ context = {} context["pipeline_name"] = config.get("manifest.name") if config.get("manifest.name") else get_name_from_url(config.get("manifest.homePage")) - context["pipeline_short_description"] = config.get("manifest.description") + context["pipeline_short_description"] = config.get("manifest.description") context["version"] = config.get("manifest.version") if config.get("manifest.version") else config.get("params.version") context["author"] = config.get("manifest.author") if config.get("manifest.author") else "No author provided" return context diff --git a/nf_core/list.py b/nf_core/list.py index cc08ac2560..d36c573f8f 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -74,6 +74,9 @@ def get_local_nf_workflows(self): try: with open(os.devnull, 'w') as devnull: nflist_raw = subprocess.check_output(['nextflow', 'list'], stderr=devnull) + except OSError as e: + if e.errno == os.errno.ENOENT: + raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") except subprocess.CalledProcessError as e: raise AssertionError("`nextflow list` returned non-zero error code: %s,\n %s", e.returncode, e.output) else: @@ -237,6 +240,9 @@ def get_local_nf_workflow_details(self): try: with open(os.devnull, 'w') as devnull: nfinfo_raw = subprocess.check_output(['nextflow', 'info', '-d', self.full_name], stderr=devnull) + except OSError as e: + if e.errno == os.errno.ENOENT: + raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") except subprocess.CalledProcessError as e: raise AssertionError("`nextflow list` returned non-zero error code: %s,\n %s", e.returncode, e.output) else: diff --git a/nf_core/utils.py b/nf_core/utils.py index 9070538d85..ba43be90b3 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -18,6 +18,9 @@ def fetch_wf_config(wf_path): try: with open(os.devnull, 'w') as devnull: nfconfig_raw = subprocess.check_output(['nextflow', 'config', '-flat', wf_path], stderr=devnull) + except OSError as e: + if e.errno == os.errno.ENOENT: + raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") except subprocess.CalledProcessError as e: raise AssertionError("`nextflow config` returned non-zero error code: %s,\n %s", e.returncode, e.output) else: From a4aac1c058e89782f34dca541fefe3d5dd1e8862 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 11 Dec 2018 14:30:27 +0100 Subject: [PATCH 33/38] Travis - Pull :dev docker image for tests. See nf-core/tools#225. --- CHANGELOG.md | 2 ++ docs/lint_errors.md | 2 +- nf_core/lint.py | 4 ++-- .../{{cookiecutter.name_noslash}}/.travis.yml | 6 +++--- tests/lint_examples/minimal_working_example/.travis.yml | 4 ++-- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 782c8537df..d2eaedbab8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,12 +13,14 @@ * `multiqc_config` and `output_md` are now put into channels instead of using the files directly (see issue [#222](https://github.com/nf-core/tools/issues/222)) * Added `local.md` to cookiecutter template in `docs/configuration/`. This was referenced in `README.md` but not present. * Major overhaul of docs to add/remove parameters, unify linking of files and added description for providing custom configs where necessary +* Travis: Pull the `dev` tagged docker image for testing #### Tools helper code * Make Travis CI tests fail on pull requests if the `CHANGELOG.md` file hasn't been updated * Minor bugfixing in Python code (eg. removing unused import statements) * Made the web requests caching work on multi-user installations * Handle exception if nextflow isn't installed +* Linting: Update for Travis: Pull the `dev` tagged docker image for testing ## [v1.3](https://github.com/nf-core/tools/releases/tag/1.3) - 2018-11-21 * `nf-core create` command line interface updated diff --git a/docs/lint_errors.md b/docs/lint_errors.md index 93c18304ef..8a80c1af66 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -107,7 +107,7 @@ nf-core pipelines must have CI testing with Travis or Circle CI. This test fails if the following happens: * `.travis.yml` does not contain the string `nf-core lint ${TRAVIS_BUILD_DIR}` under `script` -* `.travis.yml` does not contain the string `docker pull ` under `before_install` +* `.travis.yml` does not contain the string `docker pull :dev` under `before_install` * Where `` is fetched from `params.container` in the `nextflow.config` file, without the docker tag _(if we have the tag the tests fail when making a release)_ * `.travis.yml` does not test the Nextflow version specified in the pipeline as `manifest.nextflowVersion` * This is expected in the `env` section of the config, eg: diff --git a/nf_core/lint.py b/nf_core/lint.py index 3e9ee94bec..4f479bdf45 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -389,7 +389,7 @@ def check_ci_config(self): # Check that we're pulling the right docker image if self.config.get('params.container', ''): docker_notag = re.sub(r':(?:[\.\d]+|latest)$', '', self.config.get('params.container', '').strip('"\'')) - docker_pull_cmd = 'docker pull {}'.format(docker_notag) + docker_pull_cmd = 'docker pull {}:dev'.format(docker_notag) try: assert(docker_pull_cmd in ciconf.get('before_install', [])) except AssertionError: @@ -398,7 +398,7 @@ def check_ci_config(self): self.passed.append((5, "CI is pulling the correct docker image: {}".format(docker_pull_cmd))) # Check that we tag the docker image properly - docker_tag_cmd = 'docker tag {} {}'.format(docker_notag, self.config.get('params.container', '').strip('"\'')) + docker_tag_cmd = 'docker tag {}:dev {}'.format(docker_notag, self.config.get('params.container', '').strip('"\'')) try: assert(docker_tag_cmd in ciconf.get('before_install')) except AssertionError: diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml index a6874c1f2c..4f94e1e1b5 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.travis.yml @@ -11,9 +11,9 @@ before_install: # PRs to master are only ok if coming from dev branch - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ])' # Pull the docker image first so the test doesn't wait for this - - docker pull {{ cookiecutter.name_docker }} + - docker pull {{ cookiecutter.name_docker }}:dev # Fake the tag locally so that the pipeline runs properly - - docker tag {{ cookiecutter.name_docker }} {{ cookiecutter.name_docker }}:latest + - docker tag {{ cookiecutter.name_docker }}:dev {{ cookiecutter.name_docker }}:latest install: # Install Nextflow @@ -21,7 +21,7 @@ install: - wget -qO- get.nextflow.io | bash - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow # Install nf-core/tools - - pip install --upgrade pip + - pip install --upgrade pip - pip install nf-core # Reset - mkdir ${TRAVIS_BUILD_DIR}/tests && cd ${TRAVIS_BUILD_DIR}/tests diff --git a/tests/lint_examples/minimal_working_example/.travis.yml b/tests/lint_examples/minimal_working_example/.travis.yml index e31e7f8d83..7b04d4d0ad 100644 --- a/tests/lint_examples/minimal_working_example/.travis.yml +++ b/tests/lint_examples/minimal_working_example/.travis.yml @@ -13,9 +13,9 @@ before_install: # PRs to master are only ok if coming from dev branch - '[ $TRAVIS_PULL_REQUEST = "false" ] || [ $TRAVIS_BRANCH != "master" ] || ([ $TRAVIS_PULL_REQUEST_SLUG = $TRAVIS_REPO_SLUG ] && [ $TRAVIS_PULL_REQUEST_BRANCH = "dev" ])' # Pull the docker image first so the test doesn't wait for this - - docker pull nfcore/tools + - docker pull nfcore/tools:dev # Fake the tag locally so that the pipeline runs properly - - docker tag nfcore/tools nfcore/tools:0.4 + - docker tag nfcore/tools:dev nfcore/tools:0.4 install: # Install Nextflow From a916d327cec9a4aa2f73efd2bd112982b8c089c2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 11 Dec 2018 14:43:35 +0100 Subject: [PATCH 34/38] Updated a few docker tag statements that I had missed. Gotta love automated testing... --- README.md | 4 ++-- nf_core/bump_version.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8371b22f9d..7e31e4bffb 100644 --- a/README.md +++ b/README.md @@ -299,8 +299,8 @@ INFO: Updating version in nextflow.config + container = 'nfcore/mypipeline:1.0' INFO: Updating version in .travis.yml - - docker tag nfcore/mypipeline nfcore/mypipeline:latest - + docker tag nfcore/mypipeline nfcore/mypipeline:1.0 + - docker tag nfcore/mypipeline:dev nfcore/mypipeline:latest + + docker tag nfcore/mypipeline:dev nfcore/mypipeline:1.0 INFO: Updating version in Singularity - VERSION 1.0dev diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 6256e18b16..07c4990247 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -39,8 +39,8 @@ def bump_pipeline_version(lint_obj, new_version): update_file_version("nextflow.config", lint_obj, nfconfig_pattern, nfconfig_newstr) # Update travis image tag - nfconfig_pattern = r"docker tag nfcore/{name} nfcore/{name}:(?:{tag}|latest)".format(name=lint_obj.pipeline_name.lower(), tag=current_version.replace('.',r'\.')) - nfconfig_newstr = "docker tag nfcore/{name} nfcore/{name}:{tag}".format(name=lint_obj.pipeline_name.lower(), tag=docker_tag) + nfconfig_pattern = r"docker tag nfcore/{name}:dev nfcore/{name}:(?:{tag}|latest)".format(name=lint_obj.pipeline_name.lower(), tag=current_version.replace('.',r'\.')) + nfconfig_newstr = "docker tag nfcore/{name}:dev nfcore/{name}:{tag}".format(name=lint_obj.pipeline_name.lower(), tag=docker_tag) update_file_version(".travis.yml", lint_obj, nfconfig_pattern, nfconfig_newstr) # Update Singularity version name From 826818f9c1fc1888cdc1450e7078035d29409fe8 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 12 Dec 2018 15:55:47 +0100 Subject: [PATCH 35/38] Bumped version for 1.4 release --- CHANGELOG.md | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2eaedbab8..a40092c739 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## v1.4dev +## [v1.4](https://github.com/nf-core/tools/releases/tag/1.4) - 2018-12-12 #### Template pipeline * Institutional custom config profiles moved to github `nf-core/configs` @@ -8,7 +8,7 @@ * Load `base.config` by default for all profiles * Removed profiles named `standard` and `none` * Added parameter `--igenomesIgnore` so `igenomes.config` is not loaded if parameter clashes are observed - * Added parameter `--custom_config_version` for custom config version control. Can use this parameter to provide commit id for reproducibility. Defaults to `master` + * Added parameter `--custom_config_version` for custom config version control. Can use this parameter to provide commit id for reproducibility. Defaults to `master` * Deleted custom configs from template in `conf/` directory i.e. `uzh.config`, `binac.config` and `cfc.config` * `multiqc_config` and `output_md` are now put into channels instead of using the files directly (see issue [#222](https://github.com/nf-core/tools/issues/222)) * Added `local.md` to cookiecutter template in `docs/configuration/`. This was referenced in `README.md` but not present. diff --git a/setup.py b/setup.py index f1da910e62..6a66950b3b 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages import sys -version = '1.3' +version = '1.4' with open('README.md') as f: readme = f.read() From 141a63c4645636dcb17ae0c0b469cbe1c6eccddc Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 12 Dec 2018 16:18:42 +0100 Subject: [PATCH 36/38] Template docs update. * Removed remaining instances of -profile standard * Removed UPPMAX-specific appendices for installation docs. --- .../docs/installation.md | 26 +++++-------------- .../docs/usage.md | 4 +-- .../{{cookiecutter.name_noslash}}/main.nf | 4 +-- 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md index cfa6651132..e4aea88c25 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md @@ -12,8 +12,6 @@ To start using the {{ cookiecutter.name }} pipeline, follow the steps below: * [Software deps: Bioconda](#32-software-deps-bioconda) * [Configuration profiles](#33-configuration-profiles) 4. [Reference genomes](#4-reference-genomes) -5. [Appendices](#5-appendices) - * [Running on UPPMAX](#running-on-uppmax) ## 1) Install NextFlow Nextflow runs on most POSIX systems (Linux, Mac OSX etc). It can be installed by running the following commands: @@ -61,7 +59,9 @@ If you would like to make changes to the pipeline, it's best to make a fork on G ## 3) Pipeline configuration -By default, the pipeline runs with the `standard` configuration profile. This uses a number of sensible defaults for process requirements and is suitable for running on a simple (if powerful!) basic server. You can see this configuration in [`conf/base.config`](../conf/base.config). +By default, the pipeline loads a basic server configuration [`conf/base.config`](../conf/base.config) +This uses a number of sensible defaults for process requirements and is suitable for running +on a simple (if powerful!) local server. Be warned of two important points about this default configuration: @@ -69,15 +69,16 @@ Be warned of two important points about this default configuration: * All jobs are run in the login session. If you're using a simple server, this may be fine. If you're using a compute cluster, this is bad as all jobs will run on the head node. * See the [nextflow docs](https://www.nextflow.io/docs/latest/executor.html) for information about running with other hardware backends. Most job scheduler systems are natively supported. 2. Nextflow will expect all software to be installed and available on the `PATH` + * It's expected to use an additional config profile for docker, singularity or conda support. See below. #### 3.1) Software deps: Docker First, install docker on your system: [Docker Installation Instructions](https://docs.docker.com/engine/installation/) -Then, running the pipeline with the option `-profile standard,docker` tells Nextflow to enable Docker for this run. An image containing all of the software requirements will be automatically fetched and used from dockerhub (https://hub.docker.com/r/{{ cookiecutter.name_docker }}). +Then, running the pipeline with the option `-profile docker` tells Nextflow to enable Docker for this run. An image containing all of the software requirements will be automatically fetched and used from dockerhub (https://hub.docker.com/r/{{ cookiecutter.name_docker }}). #### 3.1) Software deps: Singularity If you're not able to use Docker then [Singularity](http://singularity.lbl.gov/) is a great alternative. -The process is very similar: running the pipeline with the option `-profile standard,singularity` tells Nextflow to enable singularity for this run. An image containing all of the software requirements will be automatically fetched and used from singularity hub. +The process is very similar: running the pipeline with the option `-profile singularity` tells Nextflow to enable singularity for this run. An image containing all of the software requirements will be automatically fetched and used from singularity hub. If running offline with Singularity, you'll need to download and transfer the Singularity image first: @@ -98,7 +99,7 @@ Remember to pull updated versions of the singularity image if you update the pip If you're not able to use Docker _or_ Singularity, you can instead use conda to manage the software requirements. This is slower and less reproducible than the above, but is still better than having to install all requirements yourself! The pipeline ships with a conda environment file and nextflow has built-in support for this. -To use it first ensure that you have conda installed (we recommend [miniconda](https://conda.io/miniconda.html)), then follow the same pattern as above and use the flag `-profile standard,conda` +To use it first ensure that you have conda installed (we recommend [miniconda](https://conda.io/miniconda.html)), then follow the same pattern as above and use the flag `-profile conda` #### 3.3) Configuration profiles @@ -107,16 +108,3 @@ See [`docs/configuration/adding_your_own.md`](configuration/adding_your_own.md) ## 4) Reference genomes See [`docs/configuration/reference_genomes.md`](configuration/reference_genomes.md) - -## 5) Appendices - -#### Running on UPPMAX -To run the pipeline on the [Swedish UPPMAX](https://www.uppmax.uu.se/) clusters (`rackham`, `irma`, `bianca` etc), use the command line flag `-profile uppmax`. This tells Nextflow to submit jobs using the SLURM job executor with Singularity for software dependencies. - -Note that you will need to specify your UPPMAX project ID when running a pipeline. To do this, use the command line flag `--project `. The pipeline will exit with an error message if you try to run it pipeline with the default UPPMAX config profile without a project. - -**Optional Extra:** To avoid having to specify your project every time you run Nextflow, you can add it to your personal Nextflow config file instead. Add this line to `~/.nextflow/config`: - -```nextflow -params.project = 'project_ID' // eg. b2017123 -``` diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 72d2768902..f387d5c36f 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -52,7 +52,7 @@ NXF_OPTS='-Xms1g -Xmx4g' ## Running the pipeline The typical command for running the pipeline is as follows: ```bash -nextflow run {{ cookiecutter.name }} --reads '*_R{1,2}.fastq.gz' -profile standard,docker +nextflow run {{ cookiecutter.name }} --reads '*_R{1,2}.fastq.gz' -profile docker ``` This will launch the pipeline with the `docker` configuration profile. See below for more information about profiles. @@ -84,7 +84,7 @@ This version number will be logged in reports when you run the pipeline, so that ## Main arguments ### `-profile` -Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. Note that multiple profiles can be loaded, for example: `-profile standard,docker` - the order of arguments is important! +Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. Note that multiple profiles can be loaded, for example: `-profile docker` - the order of arguments is important! If `-profile` is not specified at all the pipeline will be run locally and expects all software to be installed and available on the `PATH`. diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 578cf825bf..18caf75ad2 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -27,13 +27,13 @@ def helpMessage() { The typical command for running the pipeline is as follows: - nextflow run {{ cookiecutter.name }} --reads '*_R{1,2}.fastq.gz' -profile standard,docker + nextflow run {{ cookiecutter.name }} --reads '*_R{1,2}.fastq.gz' -profile docker Mandatory arguments: --reads Path to input data (must be surrounded with quotes) --genome Name of iGenomes reference -profile Configuration profile to use. Can use multiple (comma separated) - Available: standard, conda, docker, singularity, awsbatch, test + Available: conda, docker, singularity, awsbatch, test and more. Options: --singleEnd Specifies that the input is single end reads From 58bdc962a9a65d90bd31f77eefc349237576dbfa Mon Sep 17 00:00:00 2001 From: Sven F Date: Wed, 12 Dec 2018 16:21:28 +0100 Subject: [PATCH 37/38] Update CHANGELOG.md Co-Authored-By: apeltzer --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a40092c739..4a6a92d015 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## [v1.4](https://github.com/nf-core/tools/releases/tag/1.4) - 2018-12-12 +## [v1.4](https://github.com/nf-core/tools/releases/tag/1.4) - 2018-12-12 Tantalum Butterfly #### Template pipeline * Institutional custom config profiles moved to github `nf-core/configs` From 636d050b521681709e05b1b10dd84faf23ff19c1 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 12 Dec 2018 16:23:49 +0100 Subject: [PATCH 38/38] Goddam changelog updates --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a40092c739..e8e8144966 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ * Added `local.md` to cookiecutter template in `docs/configuration/`. This was referenced in `README.md` but not present. * Major overhaul of docs to add/remove parameters, unify linking of files and added description for providing custom configs where necessary * Travis: Pull the `dev` tagged docker image for testing +* Removed UPPMAX-specific documentation from the template. #### Tools helper code * Make Travis CI tests fail on pull requests if the `CHANGELOG.md` file hasn't been updated