From 283e82c080e67d3a9026327f5cab503ed1e4df9f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 13 Mar 2019 23:16:53 +0100 Subject: [PATCH 01/38] v1.6dev version bump --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a855c19df2..5b9a343f34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # nf-core/tools: Changelog +## v1.6dev + +_..nothing yet.._ + ## [v1.5](https://github.com/nf-core/tools/releases/tag/1.5) - 2019-03-13 Iron Shark #### Template pipeline diff --git a/setup.py b/setup.py index ade9ac04b7..721274c390 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages import sys -version = '1.5' +version = '1.6dev' with open('README.md') as f: readme = f.read() From 8f368fcac2b8b09e1042aca09aded3bf8f7fe606 Mon Sep 17 00:00:00 2001 From: Tobias Date: Fri, 15 Mar 2019 23:25:37 +0100 Subject: [PATCH 02/38] add check for tracefiles in s3 buckets --- nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 2cad5a6af9..d4680d41f7 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -90,6 +90,8 @@ if( workflow.profile == 'awsbatch') { // Check workDir/outdir paths to be S3 buckets if running on AWSBatch // related: https://github.com/nextflow-io/nextflow/issues/813 if (!workflow.workDir.startsWith('s3:') || !params.outdir.startsWith('s3:')) exit 1, "Workdir or Outdir not on S3 - specify S3 Buckets for each to run on AWSBatch!" + // Prevent trace files to be stored on S3 since S3 does not support rolling files. + if (workflow.tracedir.startsWith('s3:')) exit 1, "Specify a local tracedir or run without trace! S3 cannot be used for tracefiles." } // Stage config files From 73669f5ee0cc6ab66b39291ca3038ed8e75d8429 Mon Sep 17 00:00:00 2001 From: Tobias Date: Fri, 15 Mar 2019 23:38:00 +0100 Subject: [PATCH 03/38] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b9a343f34..29e64e63f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ _..nothing yet.._ * Completion email now includes MultiQC report if not too big * `params.genome` is now checked if set, to ensure that it's a valid iGenomes key * Together with nf-core/configs, helper function now checks hostname and suggests a valid config profile +* `awsbatch` executor requires the `tracedir` not to be set to an `s3` bucket. #### Tools helper code * New `nf-core launch` command to interactively launch nf-core pipelines from command-line From 65f1399da46fac85bb5462c0d1ed956cefc31d22 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sun, 17 Mar 2019 18:03:58 +0100 Subject: [PATCH 04/38] Some more tweaks from the final rnaseq template merge --- .../{{cookiecutter.name_noslash}}/.gitignore | 1 + .../{{cookiecutter.name_noslash}}/conf/test.config | 2 ++ .../docs/configuration/reference_genomes.md | 3 ++- .../{{cookiecutter.name_noslash}}/docs/installation.md | 4 +++- .../{{cookiecutter.name_noslash}}/docs/output.md | 2 +- .../pipeline-template/{{cookiecutter.name_noslash}}/main.nf | 3 ++- .../{{cookiecutter.name_noslash}}/nextflow.config | 1 - 7 files changed, 11 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore index 46f69e414b..5b54e3e6c2 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.gitignore @@ -4,3 +4,4 @@ data/ results/ .DS_Store tests/test_data +*.pyc diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test.config index 661cb5e72c..3b221bb358 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test.config @@ -8,6 +8,8 @@ */ params { + config_profile_name = 'Test profile' + config_profile_description = 'Minimal test dataset to check pipeline function' // Limit resources so that this can run on Travis max_cpus = 2 max_memory = 6.GB diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/reference_genomes.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/reference_genomes.md index 622a2e052d..eade099839 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/reference_genomes.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/reference_genomes.md @@ -39,7 +39,8 @@ Multiple reference index types are held together with consistent structure for m We have put a copy of iGenomes up onto AWS S3 hosting and this pipeline is configured to use this by default. The hosting fees for AWS iGenomes are currently kindly funded by a grant from Amazon. The pipeline will automatically download the required reference files when you run the pipeline. -For more information about the AWS iGenomes, see https://ewels.github.io/AWS-iGenomes/ + +For more information about AWS iGenomes, see [https://ewels.github.io/AWS-iGenomes/](https://ewels.github.io/AWS-iGenomes/). Downloading the files takes time and bandwidth, so we recommend making a local copy of the iGenomes resource. Once downloaded, you can customise the variable `params.igenomes_base` in your custom configuration file to point to the reference location. diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md index 5475c064c4..76779af573 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md @@ -78,7 +78,9 @@ Be warned of two important points about this default configuration: ### Docker First, install docker on your system: [Docker Installation Instructions](https://docs.docker.com/engine/installation/) -Then, running the pipeline with the option `-profile docker` tells Nextflow to enable Docker for this run. An image containing all of the software requirements will be automatically fetched and used from dockerhub (https://hub.docker.com/r/{{ cookiecutter.name_docker }}). +Then, running the pipeline with the option `-profile docker` tells Nextflow to enable Docker for this run. +An image containing all of the software requirements will be automatically fetched and used from dockerhub +([https://hub.docker.com/r/{{ cookiecutter.name_docker }}](https://hub.docker.com/r/{{ cookiecutter.name_docker }})). ### Singularity If you're not able to use Docker then [Singularity](http://singularity.lbl.gov/) is a great alternative. diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md index 034aa90199..c9b9b9e4e1 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/output.md @@ -38,4 +38,4 @@ The pipeline has special steps which allow the software versions used to be repo * `Project_multiqc_data/` * Directory containing parsed statistics from the different tools used in the pipeline -For more information about how to use MultiQC reports, see http://multiqc.info +For more information about how to use MultiQC reports, see [http://multiqc.info](http://multiqc.info) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 2cad5a6af9..8178ac77ab 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -124,6 +124,7 @@ if(params.readPaths){ // Header log info log.info nfcoreHeader() def summary = [:] +if(workflow.revision) summary['Pipeline Release'] = workflow.revision summary['Run Name'] = custom_runName ?: workflow.runName // TODO nf-core: Report custom parameters here summary['Reads'] = params.reads @@ -358,7 +359,7 @@ workflow.onComplete { c_green = params.monochrome_logs ? '' : "\033[0;32m"; c_red = params.monochrome_logs ? '' : "\033[0;31m"; if(workflow.success){ - log.info "${c_purple}[{{ cookiecutter.name }}]${c_green} Pipeline complete${c_reset}" + log.info "${c_purple}[{{ cookiecutter.name }}]${c_green} Pipeline completed successfully${c_reset}" } else { checkHostname() log.info "${c_purple}[{{ cookiecutter.name }}]${c_red} Pipeline completed with errors${c_reset}" diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index 71896df174..956c38a409 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -24,7 +24,6 @@ params { help = false igenomes_base = "./iGenomes" tracedir = "${params.outdir}/pipeline_info" - clusterOptions = false awsqueue = false awsregion = 'eu-west-1' igenomesIgnore = false From 65a19647a24285fd82036893021eb19fefa15c01 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sun, 17 Mar 2019 18:06:45 +0100 Subject: [PATCH 05/38] Run markdown lint on the compiled pipeline --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 32485e0fce..b3894a6c64 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,10 +26,13 @@ install: - sudo apt-get install npm && npm install -g markdownlint-cli # For documentation linting script: + # Test the main code in this repo - pytest --cov=nf_core . + - markdownlint . -c ${TRAVIS_BUILD_DIR}/.github/markdownlint.yml + # Test the pipeline template code - nf-core create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" - nf-core lint nf-core-testpipeline - - markdownlint . -c ${TRAVIS_BUILD_DIR}/.github/markdownlint.yml + - markdownlint nf-core-testpipeline -c nf-core-testpipeline/.github/markdownlint.yml after_success: - codecov --rcfile=${TRAVIS_BUILD_DIR}/.github/coveragerc From 2a4453a3d60c84aa74ec6eeaf3baad1544b703b5 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sun, 17 Mar 2019 18:08:19 +0100 Subject: [PATCH 06/38] Changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b9a343f34..8d4fb34752 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,9 @@ ## v1.6dev -_..nothing yet.._ +#### Template pipeline +* Fixed markdown linting +* Tools CI testing now runs markdown lint on compiled template pipeline ## [v1.5](https://github.com/nf-core/tools/releases/tag/1.5) - 2019-03-13 Iron Shark From c8b8b5aa340579d3edc9fcf237284d3d4ae4cb2d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sun, 17 Mar 2019 18:10:26 +0100 Subject: [PATCH 07/38] Fixed template multiqc config bug --- .../{{cookiecutter.name_noslash}}/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config index 956c38a409..6f0d53a0bb 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/nextflow.config @@ -16,7 +16,7 @@ params { // Boilerplate options name = false - multiqc_config = "$baseDir/conf/multiqc_config.yaml" + multiqc_config = "$baseDir/assets/multiqc_config.yaml" email = false maxMultiqcEmailFileSize = 25.MB plaintext_email = false From 0ae95be49570e335c09ac41ad61f12e65a48d068 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 18 Mar 2019 15:31:59 +0100 Subject: [PATCH 08/38] Drop Rnaseq from blacklist --- bin/blacklist.json | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/blacklist.json b/bin/blacklist.json index c196795bad..515750fb99 100644 --- a/bin/blacklist.json +++ b/bin/blacklist.json @@ -7,7 +7,6 @@ "exoseq", "nascent", "neutronstar", - "rnaseq", "smrnaseq", "vipr" ] From 2510d6424ac5aa8fb16ffd93ca05ae764f069ff6 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 18 Mar 2019 15:34:46 +0100 Subject: [PATCH 09/38] Add changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29e64e63f0..77196ee232 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## v1.6dev _..nothing yet.._ +#### Syncing Functionality +* Drop [nf-core/rnaseq](https://github.com/nf-core/rnaseq]) from `blacklist.json` to make template sync available ## [v1.5](https://github.com/nf-core/tools/releases/tag/1.5) - 2019-03-13 Iron Shark From 8008042f2635fead596d558cecd233f9af4fb5a4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 19 Mar 2019 15:20:38 +0100 Subject: [PATCH 10/38] Get the latest release with nf-core download --- nf_core/download.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/download.py b/nf_core/download.py index 53b143c8bc..b242a20c9b 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -103,6 +103,8 @@ def fetch_workflow_details(self, wfs): # Find latest release hash if self.release is None and len(wf.releases) > 0: + # Sort list of releases so that most recent is first + wf.releases = sorted(wf.releases, key=lambda k: k['published_at_timestamp'], reverse=True) self.release = wf.releases[0]['tag_name'] self.wf_sha = wf.releases[0]['tag_sha'] logging.debug("No release specified. Using latest release: {}".format(self.release)) From 78b2e2a5478ce2dae5946fd3c7aaa09571bcda0a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 19 Mar 2019 15:21:17 +0100 Subject: [PATCH 11/38] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a88613ecba..52b1e45faf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ #### Tools helper code * Drop [nf-core/rnaseq](https://github.com/nf-core/rnaseq]) from `blacklist.json` to make template sync available +* Fix bug in `nf-core download` so that it now fetches the latest release by default ## [v1.5](https://github.com/nf-core/tools/releases/tag/1.5) - 2019-03-13 Iron Shark From bfa8666548c8444c05f067dd1659ccf337f11c9c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 19 Mar 2019 15:40:30 +0100 Subject: [PATCH 12/38] download datesort release - handle missing publish dates --- nf_core/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/download.py b/nf_core/download.py index b242a20c9b..2fb472b00d 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -104,7 +104,7 @@ def fetch_workflow_details(self, wfs): # Find latest release hash if self.release is None and len(wf.releases) > 0: # Sort list of releases so that most recent is first - wf.releases = sorted(wf.releases, key=lambda k: k['published_at_timestamp'], reverse=True) + wf.releases = sorted(wf.releases, key=lambda k: k.get('published_at_timestamp', 0), reverse=True) self.release = wf.releases[0]['tag_name'] self.wf_sha = wf.releases[0]['tag_sha'] logging.debug("No release specified. Using latest release: {}".format(self.release)) From 006a2431ef64785f4c0bde36f3494b824d461feb Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 20 Mar 2019 16:42:20 +0100 Subject: [PATCH 13/38] download: Make unzipped files executable. Fixes nf-core/tools#289 --- nf_core/download.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/download.py b/nf_core/download.py index 2fb472b00d..176f2ba159 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -170,6 +170,11 @@ def download_wf_files(self): gh_name = '{}-{}'.format(self.wf_name, self.wf_sha).split('/')[-1] os.rename(os.path.join(self.outdir, gh_name), os.path.join(self.outdir, 'workflow')) + # Make downloaded files executable + for dirpath, subdirs, filelist in os.walk(os.path.join(self.outdir, 'workflow')): + for fname in filelist: + os.chmod(os.path.join(dirpath, fname), 0o775) + def find_container_images(self): """ Find container image names for workflow """ From c058158e407fbc4e17f4f3e780d93b163adac7c9 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 20 Mar 2019 16:43:19 +0100 Subject: [PATCH 14/38] Changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52b1e45faf..d8e135fdc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,9 @@ #### Tools helper code * Drop [nf-core/rnaseq](https://github.com/nf-core/rnaseq]) from `blacklist.json` to make template sync available -* Fix bug in `nf-core download` so that it now fetches the latest release by default +* Fix bugs in `nf-core download` + * The _latest_ release is now fetched by default if not specified + * Downloaded pipeline files are now properly executable ## [v1.5](https://github.com/nf-core/tools/releases/tag/1.5) - 2019-03-13 Iron Shark From 30ebd9ed6002e495d9c2d58808779a5f44f25823 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 25 Mar 2019 14:10:46 +0100 Subject: [PATCH 15/38] Markdown: fix list indentation --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e135fdc2..dc43547fa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,8 @@ #### Tools helper code * Drop [nf-core/rnaseq](https://github.com/nf-core/rnaseq]) from `blacklist.json` to make template sync available * Fix bugs in `nf-core download` - * The _latest_ release is now fetched by default if not specified - * Downloaded pipeline files are now properly executable + * The _latest_ release is now fetched by default if not specified + * Downloaded pipeline files are now properly executable ## [v1.5](https://github.com/nf-core/tools/releases/tag/1.5) - 2019-03-13 Iron Shark From d4d766d9c0a32800e15322577b733c649818a736 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sun, 31 Mar 2019 12:02:59 +0200 Subject: [PATCH 16/38] Improve Readme ToC --- CHANGELOG.md | 2 +- Dockerfile | 2 +- README.md | 24 +++++++++++++++++------- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc43547fa0..50736fa5b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ * Drop [nf-core/rnaseq](https://github.com/nf-core/rnaseq]) from `blacklist.json` to make template sync available * Fix bugs in `nf-core download` * The _latest_ release is now fetched by default if not specified - * Downloaded pipeline files are now properly executable + * Downloaded pipeline files are now properly executable. ## [v1.5](https://github.com/nf-core/tools/releases/tag/1.5) - 2019-03-13 Iron Shark diff --git a/Dockerfile b/Dockerfile index 6594e7bd0b..a63e25af58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,5 +3,5 @@ LABEL authors="phil.ewels@scilifelab.se,alexander.peltzer@qbic.uni-tuebingen.de" description="Docker image containing base requirements for the nfcore pipelines" # Install procps so that Nextflow can poll CPU usage -RUN apt-get update && apt-get install -y procps && apt-get clean -y +RUN apt-get update && apt-get install -y procps && apt-get clean -y RUN conda install conda=4.6.7 diff --git a/README.md b/README.md index a593218564..23c8a17021 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,17 @@ A python package with helper tools for the nf-core community. ## Table of contents -* [Installation](#installation) -* [Listing pipelines](#listing-pipelines) (`nf-core list`) -* [Downloading pipelines for offline use](#downloading-pipelines-for-offline-use) (`nf-core download`) -* [Listing software licences](#pipeline-software-licences): List software licences for a given workflow (`nf-core licences`) -* [Creating a new workflow](#creating-a-new-workflow) (`nf-core create`) -* [Checking a pipeline against nf-core guidelines](#linting-a-workflow) (`nf-core lint`) -* [Bumping a pipeline version number](#bumping-a-pipeline-version-number) (`nf-core bump-version`) +* [`nf-core` tools installation](#installation) +* [`nf-core list` - List available pipelines](#listing-pipelines) +* [`nf-core download` - Download pipeline for offline use](#downloading-pipelines-for-offline-use) +* [`nf-core licences` - List software licences in a pipeline](#pipeline-software-licences) +* [`nf-core create` - Create a new workflow from the nf-core template](#creating-a-new-workflow) +* [`nf-core lint` - Check pipeline code against nf-core guidelines](#linting-a-workflow) +* [`nf-core bump-version` - Change a pipeline version number](#bumping-a-pipeline-version-number) + + +The nf-core tools package is written in Python and can be imported and used within other packages. +For documentation of the internal Python functions, please refer to the [Tools Python API docs](https://nf-co.re/tools-docs/). ## Installation @@ -25,6 +29,12 @@ You can install `nf-core/tools` from [PyPI](https://pypi.python.org/pypi/nf-core pip install nf-core ``` +Alternatively, it can be installed using [bioconda](https://bioconda.github.io/recipes/nf-core/README.html): + +```bash +conda install -c bioconda nf-core +``` + If you would like the development version instead, the command is: ```bash From 7eea1f01491811ee24c4c1284ec71f188004deca Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sun, 31 Mar 2019 14:05:40 +0200 Subject: [PATCH 17/38] Fixed list, documented sort, better order of command in --help --- CHANGELOG.md | 6 ++ README.md | 134 +++++++++++++++++++++++++++----- nf_core/list.py | 27 +++++-- scripts/nf-core | 198 ++++++++++++++++++++++++++++-------------------- 4 files changed, 257 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50736fa5b1..a32203aa33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,15 @@ #### Tools helper code * Drop [nf-core/rnaseq](https://github.com/nf-core/rnaseq]) from `blacklist.json` to make template sync available +* Updated main help command to sort the subcommands in a more logical order +* Updated readme to describe the new `nf-core launch` command * Fix bugs in `nf-core download` * The _latest_ release is now fetched by default if not specified * Downloaded pipeline files are now properly executable. +* Fixed bugs in `nf-core list` + * Sorting now works again + * Output is partially coloured (better highlighting out of date pipelines) + * Improved documentation ## [v1.5](https://github.com/nf-core/tools/releases/tag/1.5) - 2019-03-13 Iron Shark diff --git a/README.md b/README.md index 23c8a17021..5d9551da85 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,12 @@ A python package with helper tools for the nf-core community. * [`nf-core` tools installation](#installation) * [`nf-core list` - List available pipelines](#listing-pipelines) +* [`nf-core launch` - Run a pipeline with interactive parameter prompts](#launch-a-pipeline) * [`nf-core download` - Download pipeline for offline use](#downloading-pipelines-for-offline-use) * [`nf-core licences` - List software licences in a pipeline](#pipeline-software-licences) * [`nf-core create` - Create a new workflow from the nf-core template](#creating-a-new-workflow) * [`nf-core lint` - Check pipeline code against nf-core guidelines](#linting-a-workflow) -* [`nf-core bump-version` - Change a pipeline version number](#bumping-a-pipeline-version-number) +* [`nf-core bump-version` - Update nf-core pipeline version number](#bumping-a-pipeline-version-number) The nf-core tools package is written in Python and can be imported and used within other packages. @@ -70,23 +71,27 @@ $ nf-core list `._,._,' -Name Version Published Last Pulled Default local is latest release? ------------------ --------- ----------- ------------- ---------------------------------- -nf-core/hlatyping 1.1.0 5 days ago 9 minutes ago Yes -nf-core/methylseq 1.1 1 week ago 2 months ago No -nf-core/chipseq dev - - No -nf-core/eager dev - - No -nf-core/exoseq dev - - No -nf-core/mag dev - - No -nf-core/rnaseq dev - - No -nf-core/smrnaseq dev - - No -nf-core/vipr dev - - No +Name Version Released Last Pulled Have latest release? +------------------------- --------- ------------ -------------- ---------------------- +nf-core/rnaseq 1.3 4 days ago 27 minutes ago Yes +nf-core/hlatyping 1.1.4 3 weeks ago 1 months ago No +nf-core/eager 2.0.6 3 weeks ago - - +nf-core/mhcquant 1.2.6 3 weeks ago - - +nf-core/rnafusion 1.0 1 months ago - - +nf-core/methylseq 1.3 1 months ago 3 months ago No +nf-core/ampliseq 1.0.0 3 months ago - - +nf-core/deepvariant 1.0 4 months ago - - +nf-core/atacseq dev - 1 months ago No +nf-core/bacass dev - - - +nf-core/bcellmagic dev - - - +nf-core/chipseq dev - 1 months ago No +nf-core/clinvap dev - - - ``` To narrow down the list, supply one or more additional keywords to filter the pipelines based on matches in titles, descriptions and topics: ```txt -nf-core list rna rna-seq +$ nf-core list rna rna-seq ,--./,-. ___ __ __ __ ___ /,-._.--~\ @@ -95,16 +100,109 @@ nf-core list rna rna-seq `._,._,' -Name Version Published Last Pulled Default local is latest release? ----------------- --------- ------------ ------------- ---------------------------------- -nf-core/rnaseq 1.0 20 hours ago - No -nf-core/smrnaseq dev - - No +Name Version Released Last Pulled Have latest release? +----------------- --------- ------------ -------------- ---------------------- +nf-core/rnaseq 1.3 4 days ago 28 minutes ago Yes +nf-core/rnafusion 1.0 1 months ago - - +nf-core/lncpipe dev - - - +nf-core/smrnaseq dev - - - ``` -You can sort the results by latest release (default), name (alphabetical) or number of GitHub stars using the `-s`/`--stars` option. +You can sort the results by latest release (`-s release`, default), +when you last pulled a local copy (`-s pulled`), +alphabetically (`-s name`), +or number of GitHub stars (`-s stars`). + +```txt +$ nf-core list -s stars + + ,--./,-. + ___ __ __ __ ___ /,-._.--~\ + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' + + +Name Stargazers Version Released Last Pulled Have latest release? +------------------------- ------------ --------- ------------ -------------- ---------------------- +nf-core/rnaseq 81 1.3 4 days ago 30 minutes ago Yes +nf-core/methylseq 22 1.3 1 months ago 3 months ago No +nf-core/ampliseq 21 1.0.0 3 months ago - - +nf-core/chipseq 20 dev - 1 months ago No +nf-core/deepvariant 15 1.0 4 months ago - - +nf-core/eager 14 2.0.6 3 weeks ago - - +nf-core/rnafusion 14 1.0 1 months ago - - +nf-core/lncpipe 9 dev - - - +nf-core/exoseq 8 dev - - - +nf-core/mag 8 dev - - - +``` Finally, to return machine-readable JSON output, use the `--json` flag. +## Launch a pipeline +Some nextflow pipelines have a considerable number of command line flags that can be used. +To help with this, the `nf-core launch` command uses an interactive command-line wizard tool to prompt you for +values for running nextflow and the pipeline parameters. + +If the pipeline in question has a `parameters.settings.json` schema, parameters will be grouped and have associated description text and variable typing. + +Nextflow `params` variables are saved in to a JSON file called `nfx-params.json` and used by nextflow with the `-params-file` flag. +This makes it easier to reuse these in the future. + +It is not essential to run the pipeline - the wizard will ask you if you want to launch the command at the end. +If not, you finish with the `params` JSON file and a nextflow command that you can copy and paste. + +``` +$ nf-core launch rnaseq + + ,--./,-. + ___ __ __ __ ___ /,-._.--~\ + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' + + +INFO: Launching nf-core/rnaseq +Main nextflow options + +Config profile to use + -profile [standard]: docker + +Unique name for this nextflow run + -name [None]: test_run + +Work directory for intermediate files + -w [./work]: + +Resume a previous workflow run + -resume [y/N]: + +Release / revision to use + -r [None]: 1.3 + + +Parameter group: Main options +Do you want to change the group's defaults? [y/N]: y + +Input files +Specify the location of your input FastQ files. + --reads ['data/*{1,2}.fastq.gz']: '/path/to/reads_*{R1,R2}.fq.gz' + +[..truncated..] + +Nextflow command: + nextflow run nf-core/rnaseq -profile "docker" -name "test_run" -r "1.3" --params-file "/Users/ewels/testing/nfx-params.json" + + +Do you want to run this command now? [y/N]: y + +INFO: Launching workflow! +N E X T F L O W ~ version 19.01.0 +Launching `nf-core/rnaseq` [evil_engelbart] - revision: 37f260d360 [master] + +[..truncated..] +``` + ## Downloading pipelines for offline use Sometimes you may need to run an nf-core pipeline on a server or HPC system that has no internet connection. In this case you will need to fetch the pipeline files first, then manually transfer them to your system. diff --git a/nf_core/list.py b/nf_core/list.py index 786eb45fd5..07d2149954 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -4,6 +4,7 @@ from __future__ import print_function from collections import OrderedDict +import click import datetime import json import logging @@ -156,20 +157,30 @@ def filtered_workflows(self): def print_summary(self): """Prints a summary of all pipelines.""" + filtered_workflows = self.filtered_workflows() + # Sort by released / dev, then alphabetical - if self.sort_workflows_by == 'release': - self.remote_workflows.sort( + if not self.sort_workflows_by or self.sort_workflows_by == 'release': + filtered_workflows.sort( key=lambda wf: ( (wf.releases[-1].get('published_at_timestamp', 0) if len(wf.releases) > 0 else 0) * -1, wf.full_name.lower() ) ) + # Sort by date pulled + elif self.sort_workflows_by == 'pulled': + def sort_pulled_date(wf): + try: + return wf.local_wf.last_pull * -1 + except: + return 0 + filtered_workflows.sort(key=sort_pulled_date) # Sort by name elif self.sort_workflows_by == 'name': - self.remote_workflows.sort( key=lambda wf: wf.full_name.lower() ) + filtered_workflows.sort( key=lambda wf: wf.full_name.lower() ) # Sort by stars, then name elif self.sort_workflows_by == 'stars': - self.remote_workflows.sort( + filtered_workflows.sort( key=lambda wf: ( wf.stargazers_count * -1, wf.full_name.lower() @@ -178,19 +189,19 @@ def print_summary(self): # Build summary list to print summary = list() - for wf in self.filtered_workflows(): - version = wf.releases[-1]['tag_name'] if len(wf.releases) > 0 else 'dev' + for wf in filtered_workflows: + version = click.style(wf.releases[-1]['tag_name'], fg='blue') if len(wf.releases) > 0 else click.style('dev', fg='yellow') published = wf.releases[-1]['published_at_pretty'] if len(wf.releases) > 0 else '-' pulled = wf.local_wf.last_pull_pretty if wf.local_wf is not None else '-' if wf.local_wf is not None: - is_latest = 'Yes' if wf.local_is_latest else 'No' + is_latest = click.style('Yes', fg='green') if wf.local_is_latest else click.style('No', fg='red') else: is_latest = '-' rowdata = [ wf.full_name, version, published, pulled, is_latest ] if self.sort_workflows_by == 'stars': rowdata.insert(1, wf.stargazers_count) summary.append(rowdata) - t_headers = ['Name', 'Version', 'Published', 'Last Pulled', 'Default local is latest release?'] + t_headers = ['Name', 'Version', 'Released', 'Last Pulled', 'Have latest release?'] if self.sort_workflows_by == 'stars': t_headers.insert(1, 'Stargazers') diff --git a/scripts/nf-core b/scripts/nf-core index 08228b607b..623174b0ef 100755 --- a/scripts/nf-core +++ b/scripts/nf-core @@ -13,7 +13,35 @@ import nf_core.bump_version, nf_core.create, nf_core.download, nf_core.launch, n import logging -@click.group() +# Customise the order of subcommands for --help +# https://stackoverflow.com/a/47984810/713980 +class CustomHelpOrder(click.Group): + def __init__(self, *args, **kwargs): + self.help_priorities = {} + super(CustomHelpOrder, self).__init__(*args, **kwargs) + + def get_help(self, ctx): + self.list_commands = self.list_commands_for_help + return super(CustomHelpOrder, self).get_help(ctx) + + def list_commands_for_help(self, ctx): + """reorder the list of commands when listing the help""" + commands = super(CustomHelpOrder, self).list_commands(ctx) + return (c[1] for c in sorted((self.help_priorities.get(command, 1000), command) for command in commands)) + + def command(self, *args, **kwargs): + """Behaves the same as `click.Group.command()` except capture + a priority for listing command names in help. + """ + help_priority = kwargs.pop('help_priority', 1000) + help_priorities = self.help_priorities + def decorator(f): + cmd = super(CustomHelpOrder, self).command(*args, **kwargs)(f) + help_priorities[cmd.name] = help_priority + return cmd + return decorator + +@click.group(cls=CustomHelpOrder) @click.version_option(nf_core.__version__) @click.option( '-v', '--verbose', @@ -27,28 +55,8 @@ def nf_core_cli(verbose): else: logging.basicConfig(level=logging.INFO, format="\n%(levelname)s: %(message)s") -@nf_core_cli.command() -@click.argument( - 'pipeline_dir', - type = click.Path(exists=True), - required = True, - metavar = "" -) -@click.option( - '--release', - is_flag = True, - default = os.environ.get('TRAVIS_BRANCH') == 'master' and os.environ.get('TRAVIS_REPO_SLUG', '').startswith('nf-core/') and not os.environ.get('TRAVIS_REPO_SLUG', '') == 'nf-core/tools', - help = "Execute additional checks for release-ready workflows." -) -def lint(pipeline_dir, release): - """ Check pipeline against nf-core guidelines """ - - # Run the lint tests! - lint_obj = nf_core.lint.run_linting(pipeline_dir, release) - if len(lint_obj.failed) > 0: - sys.exit(1) - -@nf_core_cli.command() +# nf-core list +@nf_core_cli.command(help_priority=1) @click.argument( 'keywords', required = False, @@ -57,7 +65,7 @@ def lint(pipeline_dir, release): ) @click.option( '-s', '--sort', - type = click.Choice(['release', 'name', 'stars']), + type = click.Choice(['release', 'pulled', 'name', 'stars']), default = 'release', help = "How to sort listed pipelines" ) @@ -67,29 +75,33 @@ def lint(pipeline_dir, release): default = False, help = "Print full output as JSON" ) -def list(sort, json, keywords): +def list(keywords, sort, json): """ List nf-core pipelines with local info """ - nf_core.list.list_workflows(sort, json, keywords) + nf_core.list.list_workflows(keywords, sort, json) -@nf_core_cli.command() +# nf-core launch +@nf_core_cli.command(help_priority=2) @click.argument( 'pipeline', required = True, metavar = "" ) @click.option( - '--json', + '-p', '--params', + type = str, + help = "Local parameter settings file in JSON." +) +@click.option( + '-d', '--direct', is_flag = True, - default = False, - help = "Print output in JSON" + help = "Uses given values from the parameter file directly." ) -def licences(pipeline, json): - """ List software licences for a given workflow """ - lic = nf_core.licences.WorkflowLicences(pipeline) - lic.fetch_conda_licences() - lic.print_licences(as_json=json) +def launch(pipeline, params, direct): + """ Run pipeline, interactive parameter prompts """ + nf_core.launch.launch_pipeline(pipeline, params, direct) -@nf_core_cli.command() +# nf-core download +@nf_core_cli.command(help_priority=3) @click.argument( 'pipeline', required = True, @@ -116,60 +128,26 @@ def download(pipeline, release, singularity, outdir): dl = nf_core.download.DownloadWorkflow(pipeline, release, singularity, outdir) dl.download_workflow() -@nf_core_cli.command() +# nf-core licences +@nf_core_cli.command(help_priority=4) @click.argument( 'pipeline', required = True, metavar = "" ) @click.option( - '-p', '--params', - type = str, - help = "Local parameter settings file in JSON." -) -@click.option( - '-d', '--direct', - is_flag = True, - help = "Uses given values from the parameter file directly." -) -def launch(pipeline, params, direct): - """ Interactively build a nextflow command and launch a pipeline """ - nf_core.launch.launch_pipeline(pipeline, params, direct) - -@nf_core_cli.command('bump-version') -@click.argument( - 'pipeline_dir', - type = click.Path(exists=True), - required = True, - metavar = "" -) -@click.argument( - 'new_version', - required = True, - metavar = "" -) -@click.option( - '-n', '--nextflow', + '--json', is_flag = True, default = False, - help = "Bump required nextflow version instead of pipeline version" + help = "Print output in JSON" ) -def bump_version(pipeline_dir, new_version, nextflow): - """ Update nf-core pipeline version number """ - - # First, lint the pipeline to check everything is in order - logging.info("Running nf-core lint tests") - lint_obj = nf_core.lint.run_linting(pipeline_dir, False) - if len(lint_obj.failed) > 0: - logging.error("Please fix lint errors before bumping versions") - return - - # Bump the pipeline version number - if not nextflow: - nf_core.bump_version.bump_pipeline_version(lint_obj, new_version) - else: - nf_core.bump_version.bump_nextflow_version(lint_obj, new_version) +def licences(pipeline, json): + """ List software licences for a given workflow """ + lic = nf_core.licences.WorkflowLicences(pipeline) + lic.fetch_conda_licences() + lic.print_licences(as_json=json) +# nf-core create def validate_wf_name_prompt(ctx, opts, value): """ Force the workflow name to meet the nf-core requirements """ if not re.match(r'^[a-z]+$', value): @@ -177,8 +155,7 @@ def validate_wf_name_prompt(ctx, opts, value): value = click.prompt(opts.prompt) return validate_wf_name_prompt(ctx, opts, value) return value - -@nf_core_cli.command() +@nf_core_cli.command(help_priority=5) @click.option( '-n', '--name', prompt = 'Workflow Name', @@ -229,6 +206,63 @@ def create(name, description, author, new_version, no_git, force, outdir): create_obj = nf_core.create.PipelineCreate(name, description, author, new_version, no_git, force, outdir) create_obj.init_pipeline() +@nf_core_cli.command(help_priority=6) +@click.argument( + 'pipeline_dir', + type = click.Path(exists=True), + required = True, + metavar = "" +) +@click.option( + '--release', + is_flag = True, + default = os.environ.get('TRAVIS_BRANCH') == 'master' and os.environ.get('TRAVIS_REPO_SLUG', '').startswith('nf-core/') and not os.environ.get('TRAVIS_REPO_SLUG', '') == 'nf-core/tools', + help = "Execute additional checks for release-ready workflows." +) +def lint(pipeline_dir, release): + """ Check pipeline against nf-core guidelines """ + + # Run the lint tests! + lint_obj = nf_core.lint.run_linting(pipeline_dir, release) + if len(lint_obj.failed) > 0: + sys.exit(1) + + +@nf_core_cli.command('bump-version', help_priority=7) +@click.argument( + 'pipeline_dir', + type = click.Path(exists=True), + required = True, + metavar = "" +) +@click.argument( + 'new_version', + required = True, + metavar = "" +) +@click.option( + '-n', '--nextflow', + is_flag = True, + default = False, + help = "Bump required nextflow version instead of pipeline version" +) +def bump_version(pipeline_dir, new_version, nextflow): + """ Update nf-core pipeline version number """ + + # First, lint the pipeline to check everything is in order + logging.info("Running nf-core lint tests") + lint_obj = nf_core.lint.run_linting(pipeline_dir, False) + if len(lint_obj.failed) > 0: + logging.error("Please fix lint errors before bumping versions") + return + + # Bump the pipeline version number + if not nextflow: + nf_core.bump_version.bump_pipeline_version(lint_obj, new_version) + else: + nf_core.bump_version.bump_nextflow_version(lint_obj, new_version) + + if __name__ == '__main__': click.echo(click.style("\n ,--.", fg='green')+click.style("/",fg='black')+click.style(",-.", fg='green')) From 0632a31124d6bb04791bb578531029d6e942144e Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sun, 31 Mar 2019 14:19:44 +0200 Subject: [PATCH 18/38] shell block formatting was not very inspiring --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5d9551da85..9329853636 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ This makes it easier to reuse these in the future. It is not essential to run the pipeline - the wizard will ask you if you want to launch the command at the end. If not, you finish with the `params` JSON file and a nextflow command that you can copy and paste. -``` +```txt $ nf-core launch rnaseq ,--./,-. From 7e94d867fa2f223229e450ac9d8fe04e3974da53 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sun, 31 Mar 2019 14:20:48 +0200 Subject: [PATCH 19/38] Code block console seems best --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 9329853636..6c49df1e26 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ The command `nf-core list` shows all available nf-core pipelines along with thei An example of the output from the command is as follows: -```txt +```console $ nf-core list ,--./,-. @@ -90,7 +90,7 @@ nf-core/clinvap dev - - - To narrow down the list, supply one or more additional keywords to filter the pipelines based on matches in titles, descriptions and topics: -```txt +```console $ nf-core list rna rna-seq ,--./,-. @@ -113,7 +113,7 @@ when you last pulled a local copy (`-s pulled`), alphabetically (`-s name`), or number of GitHub stars (`-s stars`). -```txt +```console $ nf-core list -s stars ,--./,-. @@ -152,7 +152,7 @@ This makes it easier to reuse these in the future. It is not essential to run the pipeline - the wizard will ask you if you want to launch the command at the end. If not, you finish with the `params` JSON file and a nextflow command that you can copy and paste. -```txt +```console $ nf-core launch rnaseq ,--./,-. @@ -211,7 +211,7 @@ To make this process easier and ensure accurate retrieval of correctly versioned By default, the pipeline will just download the pipeline code. If you specify the flag `--singularity`, it will also download any singularity image files that are required. -```txt +```console $ nf-core download methylseq --singularity ,--./,-. @@ -232,7 +232,7 @@ INFO: Downloading 1 singularity container nf-core-methylseq-1.0.simg [762.28MB] [####################################] 780573/780572 ``` -```txt +```console $ tree -L 2 nf-core-methylseq-1.0/ nf-core-methylseq-1.0/ @@ -258,7 +258,7 @@ nf-core-methylseq-1.0/ ## Pipeline software licences Sometimes it's useful to see the software licences of the tools used in a pipeline. You can use the `licences` subcommand to fetch and print the software licence from each conda / PyPI package used in an nf-core pipeline. -```txt +```console $ nf-core licences rnaseq ,--./,-. @@ -299,7 +299,7 @@ With a given pipeline name, description and author, it makes a starter pipeline After creating the files, the command initialises the folder as a git repository and makes an initial commit. This first "vanilla" commit which is identical to the output from the templating tool is important, as it allows us to keep your pipeline in sync with the base template in the future. See the [nf-core syncing docs](http://nf-co.re/sync) for more information. -```txt +```console $ nf-core create ,--./,-. @@ -325,7 +325,7 @@ INFO: Done. Remember to add a remote and push to GitHub: Once you have run the command, create a new empty repository on GitHub under your username (not the `nf-core` organisation, yet). On your computer, add this repository as a git remote and push to it: -```txt +```console git remote add origin https://github.com/ewels/nf-core-nextbigthing.git git push --set-upstream origin master ``` @@ -343,7 +343,7 @@ This is the same test that is used on the automated continuous integration tests For example, the current version looks something like this: -```txt +```console $ cd path/to/my_pipeline $ nf-core lint . @@ -376,7 +376,7 @@ The command uses results from the linting process, so will only work with workfl Usage is `nf-core bump-version `, eg: -```txt +```console $ cd path/to/my_pipeline $ nf-core bump-version . 1.0 From 995911789be9b29a6ec7c00c59a9219967ef6fdd Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 5 Apr 2019 17:07:03 +0200 Subject: [PATCH 20/38] Linting: Handle non-404 non-200 response codes from the anaconda API --- nf_core/lint.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/lint.py b/nf_core/lint.py index 63125982c9..912043f61f 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -698,7 +698,11 @@ def check_anaconda_package(self, dep): dep_json = response.json() self.conda_package_info[dep] = dep_json return + else if response.status_code != 404: + self.warned.append((8, "Anaconda API returned unexpected response code '{}' for: {}\n{}".format(response.status_code, anaconda_api_url, response))) + raise ValueError else: + # We have looped through each channel and had a 404 response code on everything self.failed.append((8, "Could not find Conda dependency using the Anaconda API: {}".format(dep))) raise ValueError From 60f7d176d7f507a1ff6625fad55433dc7e5fa370 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 5 Apr 2019 17:10:23 +0200 Subject: [PATCH 21/38] Been writing too much PHP recently... --- nf_core/lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index 912043f61f..e1094a8bf1 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -698,7 +698,7 @@ def check_anaconda_package(self, dep): dep_json = response.json() self.conda_package_info[dep] = dep_json return - else if response.status_code != 404: + elif response.status_code != 404: self.warned.append((8, "Anaconda API returned unexpected response code '{}' for: {}\n{}".format(response.status_code, anaconda_api_url, response))) raise ValueError else: From 570b9baf54e3a09511965fb50fcccf7ad5b689a9 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 6 Apr 2019 14:48:13 +0200 Subject: [PATCH 22/38] Linting: Don't reverse the conda channel order --- nf_core/lint.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/lint.py b/nf_core/lint.py index e1094a8bf1..cd90406464 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -683,7 +683,7 @@ def check_anaconda_package(self, dep): if '::' in depname: dep_channels = [depname.split('::')[0]] depname = depname.split('::')[1] - for ch in reversed(dep_channels): + for ch in dep_channels: anaconda_api_url = 'https://api.anaconda.org/package/{}/{}'.format(ch, depname) try: response = requests.get(anaconda_api_url, timeout=10) @@ -701,6 +701,8 @@ def check_anaconda_package(self, dep): elif response.status_code != 404: self.warned.append((8, "Anaconda API returned unexpected response code '{}' for: {}\n{}".format(response.status_code, anaconda_api_url, response))) raise ValueError + elif response.status_code == 404: + logging.debug("Could not find {} in conda channel {}".format(dep, ch)) else: # We have looped through each channel and had a 404 response code on everything self.failed.append((8, "Could not find Conda dependency using the Anaconda API: {}".format(dep))) From 9672f5067a14455c9b7c91771ff6ae1c0ea8ce61 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 6 Apr 2019 14:54:45 +0200 Subject: [PATCH 23/38] Changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a32203aa33..ac1318345d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ * Sorting now works again * Output is partially coloured (better highlighting out of date pipelines) * Improved documentation +* Fixed bugs in `nf-core lint` + * The order of conda channels is now correct, avoiding occasional erroneous errors that packages weren't found ([#207](https://github.com/nf-core/tools/issues/207)) ## [v1.5](https://github.com/nf-core/tools/releases/tag/1.5) - 2019-03-13 Iron Shark From 14d9c9eb1f8ea939cd9b98084cc1a5e2b064c0f4 Mon Sep 17 00:00:00 2001 From: Tobias Date: Sat, 6 Apr 2019 21:45:48 +0200 Subject: [PATCH 24/38] cleanup awsbatch sanity checking remove workdir check since the s3 prefix is removed by nextflow --- .../pipeline-template/{{cookiecutter.name_noslash}}/main.nf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index d6c7693f12..5ac45c382f 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -86,10 +86,9 @@ if( !(workflow.runName ==~ /[a-z]+_[a-z]+/) ){ if( workflow.profile == 'awsbatch') { // AWSBatch sanity checking if (!params.awsqueue || !params.awsregion) exit 1, "Specify correct --awsqueue and --awsregion parameters on AWSBatch!" - if (!workflow.workDir.startsWith('s3') || !params.outdir.startsWith('s3')) exit 1, "Specify S3 URLs for workDir and outdir parameters on AWSBatch!" - // Check workDir/outdir paths to be S3 buckets if running on AWSBatch + // Check outdir paths to be S3 buckets if running on AWSBatch // related: https://github.com/nextflow-io/nextflow/issues/813 - if (!workflow.workDir.startsWith('s3:') || !params.outdir.startsWith('s3:')) exit 1, "Workdir or Outdir not on S3 - specify S3 Buckets for each to run on AWSBatch!" + if (!params.outdir.startsWith('s3:')) exit 1, "Outdir not on S3 - specify S3 Bucket to run on AWSBatch!" // Prevent trace files to be stored on S3 since S3 does not support rolling files. if (workflow.tracedir.startsWith('s3:')) exit 1, "Specify a local tracedir or run without trace! S3 cannot be used for tracefiles." } From 20c8424f734ce40cb1c688ed193c8ce5071dce5a Mon Sep 17 00:00:00 2001 From: Tobias Date: Sat, 6 Apr 2019 21:52:35 +0200 Subject: [PATCH 25/38] update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac1318345d..2a3758a74e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## v1.6dev #### Template pipeline +* awsbatch `work-dir` checking moved to nextflow itself. Removed unsatisfiable check in main.nf template. * Fixed markdown linting * Tools CI testing now runs markdown lint on compiled template pipeline From 830532d41db97e05a08eda416e91c004a9c202a4 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 7 Apr 2019 17:54:28 +0200 Subject: [PATCH 26/38] Update centralized docs repository --- CHANGELOG.md | 2 + .../docs/README.md | 10 +- .../docs/configuration/adding_your_own.md | 86 ------------- .../docs/configuration/local.md | 47 ------- .../docs/configuration/reference_genomes.md | 51 -------- .../docs/installation.md | 115 ------------------ .../docs/troubleshooting.md | 30 ----- 7 files changed, 7 insertions(+), 334 deletions(-) delete mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md delete mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/local.md delete mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/reference_genomes.md delete mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md delete mode 100644 nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/troubleshooting.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a3758a74e..7c4b88d0c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ * awsbatch `work-dir` checking moved to nextflow itself. Removed unsatisfiable check in main.nf template. * Fixed markdown linting * Tools CI testing now runs markdown lint on compiled template pipeline +* Migrated large portions of documentation to the [nf-core website](https://github.com/nf-core/nf-co.re/pull/93) + #### Tools helper code * Drop [nf-core/rnaseq](https://github.com/nf-core/rnaseq]) from `blacklist.json` to make template sync available diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md index 9996db35ad..82f460342e 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md @@ -2,11 +2,11 @@ The {{ cookiecutter.name }} documentation is split into the following files: -1. [Installation](installation.md) +1. [Installation](https://nf-co.re/usage/installation) 2. Pipeline configuration - * [Local installation](configuration/local.md) - * [Adding your own system](configuration/adding_your_own.md) - * [Reference genomes](configuration/reference_genomes.md) + * [Local installation](https://nf-co.re/usage/local_installation) + * [Adding your own system config](https://nf-co.re/usage/adding_own_config) + * [Reference genomes](https://nf-co.re/usage/reference_genomeslocal_) 3. [Running the pipeline](usage.md) 4. [Output and how to interpret the results](output.md) -5. [Troubleshooting](troubleshooting.md) +5. [Troubleshooting](https://nf-co.re/usage/troubleshooting) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md deleted file mode 100644 index 9d9062ae94..0000000000 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/adding_your_own.md +++ /dev/null @@ -1,86 +0,0 @@ -# {{ cookiecutter.name }}: Configuration for other clusters - -It is entirely possible to run this pipeline on other clusters, though you will need to set up your own config file so that the pipeline knows how to work with your cluster. - -> If you think that there are other people using the pipeline who would benefit from your configuration (eg. other common cluster setups), please let us know. We can add a new configuration and profile which can used by specifying `-profile ` when running the pipeline. The config file will then be hosted at `nf-core/configs` and will be pulled automatically before the pipeline is executed. - -If you are the only person to be running this pipeline, you can create your config file as `~/.nextflow/config` and it will be applied every time you run Nextflow. Alternatively, save the file anywhere and reference it when running the pipeline with `-c path/to/config` (see the [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html) for more). - -A basic configuration comes with the pipeline, which loads the [`conf/base.config`](../../conf/base.config) by default. This means that you only need to configure the specifics for your system and overwrite any defaults that you want to change. - -## Cluster Environment -By default, pipeline uses the `local` Nextflow executor - in other words, all jobs are run in the login session. If you're using a simple server, this may be fine. If you're using a compute cluster, this is bad as all jobs will run on the head node. - -To specify your cluster environment, add the following line to your config file: - -```nextflow -process.executor = 'YOUR_SYSTEM_TYPE' -``` - -Many different cluster types are supported by Nextflow. For more information, please see the [Nextflow documentation](https://www.nextflow.io/docs/latest/executor.html). - -Note that you may need to specify cluster options, such as a project or queue. To do so, use the `clusterOptions` config option: - -```nextflow -process { - executor = 'SLURM' - clusterOptions = '-A myproject' -} -``` - - -## Software Requirements -To run the pipeline, several software packages are required. How you satisfy these requirements is essentially up to you and depends on your system. If possible, we _highly_ recommend using either Docker or Singularity. - -Please see the [`installation documentation`](../installation.md) for how to run using the below as a one-off. These instructions are about configuring a config file for repeated use. - -### Docker -Docker is a great way to run {{ cookiecutter.name }}, as it manages all software installations and allows the pipeline to be run in an identical software environment across a range of systems. - -Nextflow has [excellent integration](https://www.nextflow.io/docs/latest/docker.html) with Docker, and beyond installing the two tools, not much else is required - nextflow will automatically fetch the [{{ cookiecutter.name_docker }}](https://hub.docker.com/r/{{ cookiecutter.name_docker }}/) image that we have created and is hosted at dockerhub at run time. - -To add docker support to your own config file, add the following: - -```nextflow -docker.enabled = true -process.container = "{{ cookiecutter.name_docker }}" -``` - -Note that the dockerhub organisation name annoyingly can't have a hyphen, so is `nfcore` and not `nf-core`. - - -### Singularity image -Many HPC environments are not able to run Docker due to security issues. -[Singularity](http://singularity.lbl.gov/) is a tool designed to run on such HPC systems which is very similar to Docker. - -To specify singularity usage in your pipeline config file, add the following: - -```nextflow -singularity.enabled = true -process.container = "{{ cookiecutter.name }}" -``` - -If you intend to run the pipeline offline, nextflow will not be able to automatically download the singularity image for you. -Instead, you'll have to do this yourself manually first, transfer the image file and then point to that. - -First, pull the image file where you have an internet connection: - -```bash -singularity pull --name {{ cookiecutter.name_noslash }}.simg {{ cookiecutter.name }} -``` - -Then transfer this file and point the config file to the image: - -```nextflow -singularity.enabled = true -process.container = "/path/to/{{ cookiecutter.name_noslash }}.simg" -``` - - -### Conda -If you're not able to use Docker or Singularity, you can instead use conda to manage the software requirements. -To use conda in your own config file, add the following: - -```nextflow -process.conda = "$baseDir/environment.yml" -``` diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/local.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/local.md deleted file mode 100644 index bfbb0fd5af..0000000000 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/local.md +++ /dev/null @@ -1,47 +0,0 @@ -# {{ cookiecutter.name }}: Local Configuration - -If running the pipeline in a local environment, we highly recommend using either Docker or Singularity. - -## Docker -Docker is a great way to run `{{ cookiecutter.name }}`, as it manages all software installations and allows the pipeline to be run in an identical software environment across a range of systems. - -Nextflow has [excellent integration](https://www.nextflow.io/docs/latest/docker.html) with Docker, and beyond installing the two tools, not much else is required. The `{{ cookiecutter.name }}` profile comes with a configuration profile for docker, making it very easy to use. This also comes with the required presets to use the AWS iGenomes resource, meaning that if using common reference genomes you just specify the reference ID and it will be automatically downloaded from AWS S3. - -First, install docker on your system: [Docker Installation Instructions](https://docs.docker.com/engine/installation/) - -Then, simply run the analysis pipeline: - -```bash -nextflow run {{ cookiecutter.name }} -profile docker --genome '' --design '' -``` - -Nextflow will recognise `{{ cookiecutter.name }}` and download the pipeline from GitHub. The `-profile docker` configuration lists the [{{ cookiecutter.name }}](https://hub.docker.com/r/{{ cookiecutter.name_docker }}/) image that we have created and is hosted at dockerhub, and this is downloaded. - -For more information about how to work with reference genomes, see [`docs/configuration/reference_genomes.md`](reference_genomes.md). - -### Pipeline versions -The public docker images are tagged with the same version numbers as the code, which you can use to ensure reproducibility. When running the pipeline, specify the pipeline version with `-r`, for example `-r 1.0`. This uses pipeline code and docker image from this tagged version. - - -## Singularity image -Many HPC environments are not able to run Docker due to security issues. [Singularity](http://singularity.lbl.gov/) is a tool designed to run on such HPC systems which is very similar to Docker. Even better, it can use create images directly from dockerhub. - -To use the singularity image for a single run, use `-with-singularity`. This will download the docker container from dockerhub and create a singularity image for you dynamically. - -If you intend to run the pipeline offline, nextflow will not be able to automatically download the singularity image for you. Instead, you'll have to do this yourself manually first, transfer the image file and then point to that. - -First, pull the image file where you have an internet connection: - -> NB: The "tag" at the end of this command corresponds to the pipeline version. -> Here, we're pulling the docker image for version 1.0 of the {{ cookiecutter.name }} pipeline -> Make sure that this tag corresponds to the version of the pipeline that you're using - -```bash -singularity pull --name {{ cookiecutter.name_noslash }}-1.0.img docker://{{ cookiecutter.name }}:1.0 -``` - -Then transfer this file and run the pipeline with this path: - -```bash -nextflow run /path/to/{{ cookiecutter.name_noslash }} -with-singularity /path/to/{{ cookiecutter.name_noslash }}-1.0.img -``` diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/reference_genomes.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/reference_genomes.md deleted file mode 100644 index eade099839..0000000000 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/configuration/reference_genomes.md +++ /dev/null @@ -1,51 +0,0 @@ -# {{ cookiecutter.name }}: Reference Genomes Configuration - -The {{ cookiecutter.name }} pipeline needs a reference genome for alignment and annotation. - -These paths can be supplied on the command line at run time (see the [usage docs](../usage.md)), -but for convenience it's often better to save these paths in a nextflow config file. -See below for instructions on how to do this. -Read [Adding your own system](adding_your_own.md) to find out how to set up custom config files. - -## Adding paths to a config file -Specifying long paths every time you run the pipeline is a pain. -To make this easier, the pipeline comes configured to understand reference genome keywords which correspond to preconfigured paths, meaning that you can just specify `--genome ID` when running the pipeline. - -Note that this genome key can also be specified in a config file if you always use the same genome. - -To use this system, add paths to your config file using the following template: - -```nextflow -params { - genomes { - 'YOUR-ID' { - fasta = '/genome.fa' - } - 'OTHER-GENOME' { - // [..] - } - } - // Optional - default genome. Ignored if --genome 'OTHER-GENOME' specified on command line - genome = 'YOUR-ID' -} -``` - -You can add as many genomes as you like as long as they have unique IDs. - -## illumina iGenomes -To make the use of reference genomes easier, illumina has developed a centralised resource called [iGenomes](https://support.illumina.com/sequencing/sequencing_software/igenome.html). -Multiple reference index types are held together with consistent structure for multiple genomes. - -We have put a copy of iGenomes up onto AWS S3 hosting and this pipeline is configured to use this by default. -The hosting fees for AWS iGenomes are currently kindly funded by a grant from Amazon. -The pipeline will automatically download the required reference files when you run the pipeline. - -For more information about AWS iGenomes, see [https://ewels.github.io/AWS-iGenomes/](https://ewels.github.io/AWS-iGenomes/). - -Downloading the files takes time and bandwidth, so we recommend making a local copy of the iGenomes resource. -Once downloaded, you can customise the variable `params.igenomes_base` in your custom configuration file to point to the reference location. -For example: - -```nextflow -params.igenomes_base = '/path/to/data/igenomes/' -``` diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md deleted file mode 100644 index 76779af573..0000000000 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/installation.md +++ /dev/null @@ -1,115 +0,0 @@ -# {{ cookiecutter.name }}: Installation - -To start using the {{ cookiecutter.name }} pipeline, follow the steps below: - - - -* [Install NextFlow](#install-nextflow) -* [Install the pipeline](#install-the-pipeline) - * [Automatic](#automatic) - * [Offline](#offline) - * [Development](#development) -* [Pipeline configuration](#pipeline-configuration) - * [Docker](#docker) - * [Singularity](#singularity) - * [Conda](#conda) - * [Configuration profiles](#configuration-profiles) -* [Reference genomes](#reference-genomes) - - -## Install NextFlow -Nextflow runs on most POSIX systems (Linux, Mac OSX etc). It can be installed by running the following commands: - -```bash -# Make sure that Java v8+ is installed: -java -version - -# Install Nextflow -curl -fsSL get.nextflow.io | bash - -# Add Nextflow binary to your PATH: -mv nextflow ~/bin/ -# OR system-wide installation: -# sudo mv nextflow /usr/local/bin -``` - -See [nextflow.io](https://www.nextflow.io/) for further instructions on how to install and configure Nextflow. - -## Install the pipeline - -### Automatic -This pipeline itself needs no installation - NextFlow will automatically fetch it from GitHub if `{{ cookiecutter.name }}` is specified as the pipeline name. - -### Offline -The above method requires an internet connection so that Nextflow can download the pipeline files. If you're running on a system that has no internet connection, you'll need to download and transfer the pipeline files manually: - -```bash -wget https://github.com/{{ cookiecutter.name }}/archive/master.zip -mkdir -p ~/my-pipelines/nf-core/ -unzip master.zip -d ~/my-pipelines/nf-core/ -cd ~/my_data/ -nextflow run ~/my-pipelines/{{ cookiecutter.name }}-master -``` - -To stop nextflow from looking for updates online, you can tell it to run in offline mode by specifying the following environment variable in your ~/.bashrc file: - -```bash -export NXF_OFFLINE='TRUE' -``` - -### Development - -If you would like to make changes to the pipeline, it's best to make a fork on GitHub and then clone the files. Once cloned you can run the pipeline directly as above. - - -## Pipeline configuration -By default, the pipeline loads a basic server configuration [`conf/base.config`](../conf/base.config) -This uses a number of sensible defaults for process requirements and is suitable for running -on a simple (if powerful!) local server. - -Be warned of two important points about this default configuration: - -1. The default profile uses the `local` executor - * All jobs are run in the login session. If you're using a simple server, this may be fine. If you're using a compute cluster, this is bad as all jobs will run on the head node. - * See the [nextflow docs](https://www.nextflow.io/docs/latest/executor.html) for information about running with other hardware backends. Most job scheduler systems are natively supported. -2. Nextflow will expect all software to be installed and available on the `PATH` - * It's expected to use an additional config profile for docker, singularity or conda support. See below. - -### Docker -First, install docker on your system: [Docker Installation Instructions](https://docs.docker.com/engine/installation/) - -Then, running the pipeline with the option `-profile docker` tells Nextflow to enable Docker for this run. -An image containing all of the software requirements will be automatically fetched and used from dockerhub -([https://hub.docker.com/r/{{ cookiecutter.name_docker }}](https://hub.docker.com/r/{{ cookiecutter.name_docker }})). - -### Singularity -If you're not able to use Docker then [Singularity](http://singularity.lbl.gov/) is a great alternative. -The process is very similar: running the pipeline with the option `-profile singularity` tells Nextflow to enable singularity for this run. An image containing all of the software requirements will be automatically fetched and used from singularity hub. - -If running offline with Singularity, you'll need to download and transfer the Singularity image first: - -```bash -singularity pull --name {{ cookiecutter.name_noslash }}.simg {{ cookiecutter.name }} -``` - -Once transferred, use `-with-singularity` and specify the path to the image file: - -```bash -nextflow run /path/to/{{ cookiecutter.name_noslash }} -with-singularity {{ cookiecutter.name_noslash }}.simg -``` - -Remember to pull updated versions of the singularity image if you update the pipeline. - -### Conda -If you're not able to use Docker _or_ Singularity, you can instead use conda to manage the software requirements. -This is slower and less reproducible than the above, but is still better than having to install all requirements yourself! -The pipeline ships with a conda environment file and nextflow has built-in support for this. -To use it first ensure that you have conda installed (we recommend [miniconda](https://conda.io/miniconda.html)), then follow the same pattern as above and use the flag `-profile conda` - -### Configuration profiles - -See [`docs/configuration/adding_your_own.md`](configuration/adding_your_own.md) - -## Reference genomes - -See [`docs/configuration/reference_genomes.md`](configuration/reference_genomes.md) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/troubleshooting.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/troubleshooting.md deleted file mode 100644 index d8c6066798..0000000000 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/troubleshooting.md +++ /dev/null @@ -1,30 +0,0 @@ -# {{ cookiecutter.name }}: Troubleshooting - - - -## Input files not found - -If only no file, only one input file , or only read one and not read two is picked up then something is wrong with your input file declaration - -1. The path must be enclosed in quotes (`'` or `"`) -2. The path must have at least one `*` wildcard character. This is even if you are only running one paired end sample. -3. When using the pipeline with paired end data, the path must use `{1,2}` or `{R1,R2}` notation to specify read pairs. -4. If you are running Single end data make sure to specify `--singleEnd` - -If the pipeline can't find your files then you will get the following error - -```bash -ERROR ~ Cannot find any reads matching: *{1,2}.fastq.gz -``` - -Note that if your sample name is "messy" then you have to be very particular with your glob specification. A file name like `L1-1-D-2h_S1_L002_R1_001.fastq.gz` can be difficult enough for a human to read. Specifying `*{1,2}*.gz` wont work give you what you want Whilst `*{R1,R2}*.gz` will. - - -## Data organization -The pipeline can't take a list of multiple input files - it takes a glob expression. If your input files are scattered in different paths then we recommend that you generate a directory with symlinked files. If running in paired end mode please make sure that your files are sensibly named so that they can be properly paired. See the previous point. - -## Extra resources and getting help -If you still have an issue with running the pipeline then feel free to contact us. -Have a look at the [pipeline website](https://github.com/{{ cookiecutter.name }}) to find out how. - -If you have problems that are related to Nextflow and not our pipeline then check out the [Nextflow gitter channel](https://gitter.im/nextflow-io/nextflow) or the [google group](https://groups.google.com/forum/#!forum/nextflow). From 0b20556276abbfaabb2a373feb8c91055f7372f5 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 8 Apr 2019 06:38:07 +0100 Subject: [PATCH 27/38] Bugfixes and multiqc updates --- .github/CONTRIBUTING.md | 6 +++--- .../.github/CONTRIBUTING.md | 6 ++++-- .../CODE_OF_CONDUCT.md | 2 +- .../{{cookiecutter.name_noslash}}/README.md | 12 ++++++------ .../assets/email_template.txt | 17 ----------------- .../assets/multiqc_config.yaml | 2 ++ .../bin/scrape_software_versions.py | 14 ++++++++++++-- .../docs/README.md | 2 +- .../{{cookiecutter.name_noslash}}/docs/usage.md | 4 ++-- .../{{cookiecutter.name_noslash}}/main.nf | 11 +++++++++-- 10 files changed, 40 insertions(+), 36 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 7d74b2272d..7c81d57942 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -6,7 +6,7 @@ We try to manage the required tasks for nf-core/tools using GitHub issues, you p However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) -> If you need help using or developing nf-core/tools then the best place to go is the Gitter chatroom where you can ask us questions directly: https://gitter.im/nf-core/Lobby +> If you need help using or developing nf-core/tools then the best place to ask is the nf-core `tools` channel on [Slack](https://nf-core-invite.herokuapp.com/). ## Contribution workflow If you'd like to write some code for nf-core/tools, the standard workflow @@ -24,7 +24,7 @@ If you're not used to this workflow with git, you can start with some [basic doc ## Style guide Google provides an excellent [style guide](https://github.com/google/styleguide/blob/gh-pages/pyguide.md), which -is an best practise extension of [PEP](https://www.python.org/dev/peps/), the Python Enhancement Proposals. Have a look at the +is an best practise extension of [PEP](https://www.python.org/dev/peps/), the Python Enhancement Proposals. Have a look at the [docstring](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings) section, which is in particular important, as nf-core tool's code documentation is generated out of these automatically. @@ -68,4 +68,4 @@ nf-core lint nf-core-testpipeline ``` ## Getting help -For further information/help, please consult the [nf-core/tools documentation](https://github.com/nf-core/tools#documentation) and don't hesitate to get in touch on [Gitter](https://gitter.im/nf-core/Lobby) +For further information/help, please consult the [nf-core/tools documentation](https://github.com/nf-core/tools#documentation) and don't hesitate to get in touch on the nf-core `tools` channel on [Slack](https://nf-core-invite.herokuapp.com/). diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index 6eed32c16c..b5fbadfd4d 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -6,7 +6,9 @@ We try to manage the required tasks for {{ cookiecutter.name }} using GitHub iss However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) -> If you need help using or modifying {{ cookiecutter.name }} then the best place to go is the Gitter chatroom where you can ask us questions directly: https://gitter.im/nf-core/Lobby +> If you need help using or modifying {{ cookiecutter.name }} then the best place to ask is on the pipeline channel on [Slack](https://nf-core-invite.herokuapp.com/). + + ## Contribution workflow If you'd like to write some code for {{ cookiecutter.name }}, the standard workflow @@ -42,4 +44,4 @@ If there are any failures then the automated tests fail. These tests are run both with the latest available version of Nextflow and also the minimum required version that is stated in the pipeline code. ## Getting help -For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on [Gitter](https://gitter.im/nf-core/Lobby) +For further information/help, please consult the [{{ cookiecutter.name }} documentation](https://github.com/{{ cookiecutter.name }}#documentation) and don't hesitate to get in touch on the pipeline channel on [Slack](https://nf-core-invite.herokuapp.com/). diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CODE_OF_CONDUCT.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CODE_OF_CONDUCT.md index 21096193ad..09226d0d8d 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CODE_OF_CONDUCT.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/CODE_OF_CONDUCT.md @@ -34,7 +34,7 @@ This Code of Conduct applies both within project spaces and in public spaces whe ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team on the [Gitter channel](https://gitter.im/nf-core/Lobby). The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team on [Slack](https://nf-core-invite.herokuapp.com/). The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md index 1f8cc81504..db24d0d1c4 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/README.md @@ -9,20 +9,20 @@ [![Docker](https://img.shields.io/docker/automated/{{ cookiecutter.name_docker }}.svg)](https://hub.docker.com/r/{{ cookiecutter.name_docker }}) ## Introduction -The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool to run tasks across multiple compute infrastructures in a very portable manner. It comes with docker / singularity containers making installation trivial and results highly reproducible. +The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool to run tasks across multiple compute infrastructures in a very portable manner. It comes with docker containers making installation trivial and results highly reproducible. ## Documentation The {{ cookiecutter.name }} pipeline comes with documentation about the pipeline, found in the `docs/` directory: -1. [Installation](docs/installation.md) +1. [Installation](https://nf-co.re/usage/installation) 2. Pipeline configuration - * [Local installation](docs/configuration/local.md) - * [Adding your own system](docs/configuration/adding_your_own.md) - * [Reference genomes](docs/configuration/reference_genomes.md) + * [Local installation](https://nf-co.re/usage/local_installation) + * [Adding your own system config](https://nf-co.re/usage/adding_own_config) + * [Reference genomes](https://nf-co.re/usage/reference_genomes) 3. [Running the pipeline](docs/usage.md) 4. [Output and how to interpret the results](docs/output.md) -5. [Troubleshooting](docs/troubleshooting.md) +5. [Troubleshooting](https://nf-co.re/usage/troubleshooting) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/email_template.txt b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/email_template.txt index e4096abdb4..2277f36786 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/email_template.txt +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/email_template.txt @@ -17,23 +17,6 @@ ${errorReport} } %> -<% if (!success){ - out << """#################################################### -## {{ cookiecutter.name }} execution completed unsuccessfully! ## -#################################################### -The exit status of the task that caused the workflow execution to fail was: $exitStatus. -The full error message was: - -${errorReport} -""" -} else { - out << "## {{ cookiecutter.name }} execution completed successfully! ##" -} -%> - - - - The workflow was completed at $dateComplete (duration: $duration) The command used to launch the workflow was as follows: diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/multiqc_config.yaml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/multiqc_config.yaml index 621736a877..96d0c202d8 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/multiqc_config.yaml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/assets/multiqc_config.yaml @@ -5,3 +5,5 @@ report_comment: > report_section_order: {{ cookiecutter.name.lower().replace(' ', '-') }}-software-versions: order: -1000 + +export_plots: true diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/bin/scrape_software_versions.py b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/bin/scrape_software_versions.py index f732bb1328..b5dd4d62f5 100755 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/bin/scrape_software_versions.py +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/bin/scrape_software_versions.py @@ -24,9 +24,14 @@ if match: results[k] = "v{}".format(match.group(1)) +# Remove software set to false in results +for k in results: + if not results[k]: + del(results[k]) + # Dump to YAML print (''' -id: '{{ cookiecutter.name.lower().replace(' ', '-') }}-software-versions' +id: 'software_versions' section_name: '{{ cookiecutter.name }} Software Versions' section_href: 'https://github.com/{{ cookiecutter.name }}' plot_type: 'html' @@ -35,5 +40,10 @@
''') for k,v in results.items(): - print("
{}
{}
".format(k,v)) + print("
{}
{}
".format(k,v)) print ("
") + +# Write out regexes as csv file: +with open('software_versions.csv', 'w') as f: + for k,v in results.items(): + f.write("{}\t{}\n".format(k,v)) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md index 82f460342e..7dd10924f9 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/README.md @@ -6,7 +6,7 @@ The {{ cookiecutter.name }} documentation is split into the following files: 2. Pipeline configuration * [Local installation](https://nf-co.re/usage/local_installation) * [Adding your own system config](https://nf-co.re/usage/adding_own_config) - * [Reference genomes](https://nf-co.re/usage/reference_genomeslocal_) + * [Reference genomes](https://nf-co.re/usage/reference_genomes) 3. [Running the pipeline](usage.md) 4. [Output and how to interpret the results](output.md) 5. [Troubleshooting](https://nf-co.re/usage/troubleshooting) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 98f0c3f19a..36f8fe598c 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -101,7 +101,7 @@ If `-profile` is not specified at all the pipeline will be run locally and expec * Pulls software from dockerhub: [`{{ cookiecutter.name_docker }}`](http://hub.docker.com/r/{{ cookiecutter.name_docker }}/) * `singularity` * A generic configuration profile to be used with [Singularity](http://singularity.lbl.gov/) - * Pulls software from DockerHub + * Pulls software from DockerHub: [`{{ cookiecutter.name_docker }}`](http://hub.docker.com/r/{{ cookiecutter.name_docker }}/) * `test` * A profile with a complete configuration for automated testing * Includes links to test data so needs no other parameters @@ -190,7 +190,7 @@ Wherever process-specific requirements are set in the pipeline, the default valu If you are likely to be running `nf-core` pipelines regularly it may be a good idea to request that your custom config file is uploaded to the `nf-core/configs` git repository. Before you do this please can you test that the config file works with your pipeline of choice using the `-c` parameter (see definition below). You can then create a pull request to the `nf-core/configs` repository with the addition of your config file, associated documentation file (see examples in [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs)), and amending [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) to include your custom profile. -If you have any questions or issues please send us a message on [`Slack`](https://nf-core-invite.herokuapp.com/). +If you have any questions or issues please send us a message on [Slack](https://nf-core-invite.herokuapp.com/). ## AWS Batch specific parameters Running the pipeline on AWS Batch requires a couple of specific parameters to be set according to your AWS Batch configuration. Please use the `-awsbatch` profile and then specify all of the following parameters. diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 5ac45c382f..a5e941f7c0 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -178,9 +178,15 @@ ${summary.collect { k,v -> "
$k
${v ?: ' + if (filename.indexOf(".csv") > 0) filename + else null + } output: file 'software_versions_mqc.yaml' into software_versions_yaml + file "software_versions.csv" script: // TODO nf-core: Get all tools to print their version number here @@ -189,7 +195,7 @@ process get_software_versions { echo $workflow.nextflow.version > v_nextflow.txt fastqc --version > v_fastqc.txt multiqc --version > v_multiqc.txt - scrape_software_versions.py > software_versions_mqc.yaml + scrape_software_versions.py &> software_versions_mqc.yaml """ } @@ -227,12 +233,13 @@ process multiqc { file multiqc_config from ch_multiqc_config // TODO nf-core: Add in log files from your new processes for MultiQC to find! file ('fastqc/*') from fastqc_results.collect().ifEmpty([]) - file ('software_versions/*') from software_versions_yaml + file ('software_versions/*') from software_versions_yaml.collect() file workflow_summary from create_workflow_summary(summary) output: file "*multiqc_report.html" into multiqc_report file "*_data" + file "multiqc_plots" script: rtitle = custom_runName ? "--title \"$custom_runName\"" : '' From 796adc2c60133ef702de0cf8b057334429ab6c5f Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 8 Apr 2019 06:45:23 +0100 Subject: [PATCH 28/38] Update CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c4b88d0c3..4f985c44f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,9 @@ * Fixed markdown linting * Tools CI testing now runs markdown lint on compiled template pipeline * Migrated large portions of documentation to the [nf-core website](https://github.com/nf-core/nf-co.re/pull/93) - +* Removed Gitter references in `.github/` directories for `tools/` and pipeline template. +* Changed `scrape_software_versions.py` to output `.csv` file +* Added `export_plots` parameter to multiqc config #### Tools helper code * Drop [nf-core/rnaseq](https://github.com/nf-core/rnaseq]) from `blacklist.json` to make template sync available From 4807d76146fdfc7b6771eebb26ae879ceced6575 Mon Sep 17 00:00:00 2001 From: maxibor Date: Mon, 8 Apr 2019 15:51:14 +0200 Subject: [PATCH 29/38] add logging of ignored errored processes --- .../{{cookiecutter.name_noslash}}/main.nf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index a5e941f7c0..a3d90900ad 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -366,6 +366,13 @@ workflow.onComplete { c_purple = params.monochrome_logs ? '' : "\033[0;35m"; c_green = params.monochrome_logs ? '' : "\033[0;32m"; c_red = params.monochrome_logs ? '' : "\033[0;31m"; + + if (workflow.stats.ignoredCountFmt > 0 && workflow.success) { + log.info "${c_purple}Warning, pipeline completed, but with errored process(es) ${c_reset}" + log.info "${c_red}Number of ignored errored process(es) : ${workflow.stats.ignoredCountFmt} ${c_reset}" + log.info "${c_green}Number of successfully ran process(es) : ${workflow.stats.succeedCountFmt} ${c_reset}" + } + if(workflow.success){ log.info "${c_purple}[{{ cookiecutter.name }}]${c_green} Pipeline completed successfully${c_reset}" } else { From b8fa3dd4c1a65e42d14eb175c9ec8ba02df90f40 Mon Sep 17 00:00:00 2001 From: maxibor Date: Mon, 8 Apr 2019 15:57:17 +0200 Subject: [PATCH 30/38] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f985c44f2..b2bf65e111 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ * Improved documentation * Fixed bugs in `nf-core lint` * The order of conda channels is now correct, avoiding occasional erroneous errors that packages weren't found ([#207](https://github.com/nf-core/tools/issues/207)) +* Add reporting of ignored errored process + * As a solution for [#103](https://github.com/nf-core/tools/issues/103)) ## [v1.5](https://github.com/nf-core/tools/releases/tag/1.5) - 2019-03-13 Iron Shark From d93cd5bbf92435471dac3ac9d190ebf0fd79b7ad Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 9 Apr 2019 10:31:54 +0200 Subject: [PATCH 31/38] Checks for pipeline object attributes during filtering --- bin/sync | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/bin/sync b/bin/sync index 96ee9efe54..e4f551de07 100755 --- a/bin/sync +++ b/bin/sync @@ -39,6 +39,17 @@ def create_pullrequest(pipeline, origin="dev", template="TEMPLATE", token="", us auth=HTTPBasicAuth(user, token)) +def filter_blacklisted_pipelines_from_list(pipelines, blacklisted_pipelines): + filtered_pipelines = [] + for pipeline in pipelines: + if not pipeline.get('name'): + print("No attribute \'name\' for pipeline found: {}".format(pipeline)) + else: + filtered_pipelines.append(pipeline) if pipeline.get('name') not in blacklisted_pipelines \ + else filtered_pipelines + return filtered_pipelines + + def main(): # Check that the commit event is a GitHub tag event assert os.environ['TRAVIS_TAG'] @@ -62,7 +73,7 @@ def main(): pipeline["name"] for pipeline in pipelines) # Exclude blacklisted pipelines - pipelines = [pipeline for pipeline in pipelines if pipeline['name'] not in blacklist['pipelines']] + pipelines = filter_blacklisted_pipelines_from_list(pipelines, blacklist) # Update the template branch of each pipeline repo for pipeline in pipelines: From 398899e8f508435a6a5ea3a86feca781e756f97f Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 9 Apr 2019 11:09:39 +0200 Subject: [PATCH 32/38] Introduces more descriptive functions --- bin/sync | 95 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 52 insertions(+), 43 deletions(-) diff --git a/bin/sync b/bin/sync index e4f551de07..c3b37528f8 100755 --- a/bin/sync +++ b/bin/sync @@ -19,6 +19,8 @@ GITHUB_PR_URL_TEMPL = "https://api.github.com/repos/nf-core/{pipeline}/pulls" # Current script dir PATH_PARENT_DIR = os.path.dirname(os.path.realpath(__file__)) +sync_errors = [] +pr_errors = [] def create_pullrequest(pipeline, origin="dev", template="TEMPLATE", token="", user="nf-core"): @@ -50,65 +52,72 @@ def filter_blacklisted_pipelines_from_list(pipelines, blacklisted_pipelines): return filtered_pipelines +def fetch_black_listed_pipelines_from_file(file_path): + with open(file_path) as fh: + blacklist = json.load(fh) + return blacklist + + +def fetch_nfcore_workflows_from_website(url): + try: + res = requests.get(url) + pipelines = res.json().get('remote_workflows') + except Exception as e: + print("Could not get remote workflows. Reason was: {}".format(e)) + pipelines = [] + return pipelines + + +def update_template_branch_for_pipeline(pipeline): + try: + syncutils.template.NfcoreTemplate( + pipeline['name'], + branch=DEF_TEMPLATE_BRANCH, + repo_url=GH_BASE_URL.format(token=os.environ["NF_CORE_BOT"], pipeline=pipeline['name']) + ).sync() + except Exception as e: + sync_errors.append((pipeline['name'], e)) + + +def create_pullrequest_if_update_sucessful(pipeline): + name = pipeline.get('name') + for errored_pipeline, _ in sync_errors: + if name == errored_pipeline: + return + response = create_pullrequest(name, token=os.environ["NF_CORE_BOT"]) + if response.status_code != 201: + pr_errors.append((name, response.status_code, response.content)) + else: + print("Created pull-request for pipeline \'{pipeline}\' successfully." + .format(pipeline=name)) + + def main(): - # Check that the commit event is a GitHub tag event assert os.environ['TRAVIS_TAG'] assert os.environ['NF_CORE_BOT'] - # Catch exceptions in lists, and list them at the end - sync_errors = [] - pr_errors = [] - - # Get blacklisted pipelines, that should be excluded from sync - with open(PATH_PARENT_DIR + "/blacklist.json") as fh: - blacklist = json.load(fh) - - # Get nf-core pipelines info - res = requests.get(NF_CORE_PIPELINE_INFO) - pipelines = res.json().get('remote_workflows') - if not pipelines: - print("Pipeline information was empty!") + blacklisted_pipeline_names = fetch_black_listed_pipelines_from_file(PATH_PARENT_DIR + "/blacklist.json") - pipelines_without_template = syncutils.utils.repos_without_template_branch( - pipeline["name"] for pipeline in pipelines) + pipelines = fetch_nfcore_workflows_from_website(NF_CORE_PIPELINE_INFO) - # Exclude blacklisted pipelines - pipelines = filter_blacklisted_pipelines_from_list(pipelines, blacklist) + filtered_pipelines = filter_blacklisted_pipelines_from_list(pipelines, blacklisted_pipeline_names) - # Update the template branch of each pipeline repo - for pipeline in pipelines: + for pipeline in filtered_pipelines: print("Update template branch for pipeline '{pipeline}'... ".format(pipeline=pipeline['name'])) - try: - syncutils.template.NfcoreTemplate( - pipeline['name'], - branch=DEF_TEMPLATE_BRANCH, - repo_url=GH_BASE_URL.format(token=os.environ["NF_CORE_BOT"], pipeline=pipeline['name']) - ).sync() - except Exception as e: - sync_errors.append((pipeline['name'], e)) - - # Create a pull request from each template branch to the origin branch - for pipeline in pipelines: + update_template_branch_for_pipeline(pipeline) print("Trying to open pull request for pipeline {}...".format(pipeline['name'])) - response = create_pullrequest(pipeline['name'], token=os.environ["NF_CORE_BOT"]) - if response.status_code != 201: - pr_errors.append((pipeline['name'], response.status_code, response.content)) - else: - print("Created pull-request for pipeline \'{pipeline}\' successfully." - .format(pipeline=pipeline["name"])) + create_pullrequest_if_update_sucessful(pipeline) for pipeline, exception in sync_errors: - print("Sync for pipeline {name} failed.".format(name=pipeline)) + print("WARNING!!!! Sync for pipeline {name} failed.".format(name=pipeline)) print(exception) for pipeline, return_code, content in pr_errors: - print("Pull-request for pipeline \'{pipeline}\' failed," - " got return code {return_code}." - .format(pipeline=pipeline, return_code=return_code)) + print("WARNING!!!! Pull-request for pipeline \'{pipeline}\' failed," + " got return code {return_code}." + .format(pipeline=pipeline, return_code=return_code)) print(content) - if pr_errors or sync_errors: sys.exit(1) - sys.exit(0) From ea5db7642f017c43ae4d234b60ca24bf304260e0 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 9 Apr 2019 11:16:40 +0200 Subject: [PATCH 33/38] Adds changelog entry for sync --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2bf65e111..bb3f714aab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## v1.6dev +#### Syncing +* Code refactoring to make the script more readable +* No travis build failure anymore on sync errors +* More verbose logging + #### Template pipeline * awsbatch `work-dir` checking moved to nextflow itself. Removed unsatisfiable check in main.nf template. * Fixed markdown linting From 929dcb1f319cb035ca76fcd59e6543fbb8d90dd5 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 9 Apr 2019 13:01:43 +0200 Subject: [PATCH 34/38] Update .github/CONTRIBUTING.md Co-Authored-By: sven1103 --- .github/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 7c81d57942..936e7a876f 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -24,7 +24,7 @@ If you're not used to this workflow with git, you can start with some [basic doc ## Style guide Google provides an excellent [style guide](https://github.com/google/styleguide/blob/gh-pages/pyguide.md), which -is an best practise extension of [PEP](https://www.python.org/dev/peps/), the Python Enhancement Proposals. Have a look at the +is a best practise extension of [PEP](https://www.python.org/dev/peps/), the Python Enhancement Proposals. Have a look at the [docstring](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings) section, which is in particular important, as nf-core tool's code documentation is generated out of these automatically. From 1d9e889c18f0049582d59ae58b5777510fc36d72 Mon Sep 17 00:00:00 2001 From: Sven F Date: Tue, 9 Apr 2019 13:02:10 +0200 Subject: [PATCH 35/38] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6c49df1e26..90ee8b1f12 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ Some nextflow pipelines have a considerable number of command line flags that ca To help with this, the `nf-core launch` command uses an interactive command-line wizard tool to prompt you for values for running nextflow and the pipeline parameters. -If the pipeline in question has a `parameters.settings.json` schema, parameters will be grouped and have associated description text and variable typing. +If the pipeline in question has a `parameters.settings.json` file following the [nf-core parameter JSON schema](https://nf-co.re/parameters.schema.json), parameters will be grouped and have associated description text and variable typing. Nextflow `params` variables are saved in to a JSON file called `nfx-params.json` and used by nextflow with the `-params-file` flag. This makes it easier to reuse these in the future. From 439dad5e2692378f564728ab065103f9f7020c25 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 9 Apr 2019 13:07:06 +0200 Subject: [PATCH 36/38] Update CHANGELOG.md Co-Authored-By: sven1103 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb3f714aab..d6e3847e01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## v1.6dev +## v1.6 #### Syncing * Code refactoring to make the script more readable From 3eb11ec42f9e63e2666b1dce28c33f5ed7a6ef9e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 9 Apr 2019 13:07:18 +0200 Subject: [PATCH 37/38] Update setup.py Co-Authored-By: sven1103 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 721274c390..8d7cd3f396 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages import sys -version = '1.6dev' +version = '1.6' with open('README.md') as f: readme = f.read() From 95624ed47077d4cbb43512ba85514dd601d9449b Mon Sep 17 00:00:00 2001 From: Sven F Date: Tue, 9 Apr 2019 13:18:19 +0200 Subject: [PATCH 38/38] Update multiqc dependency to 1.7 --- .../{{cookiecutter.name_noslash}}/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml index 887be185ec..f5e90f73b3 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/environment.yml @@ -8,4 +8,4 @@ channels: dependencies: # TODO nf-core: Add required software dependencies here - fastqc=0.11.8 - - multiqc=1.6 + - multiqc=1.7