From edb2245655e77d13063fc7d1005b995deb9d3e53 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Mon, 2 Aug 2021 10:38:36 +0200 Subject: [PATCH 001/186] Added new flag and prompt for specifying conda package version --- nf_core/__main__.py | 7 +++++-- nf_core/modules/create.py | 21 +++++++++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 483d709780..2338cbc806 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -493,7 +493,8 @@ def remove(ctx, dir, tool): @click.option("-n", "--no-meta", is_flag=True, default=False, help="Don't use meta map for sample information") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") @click.option("-c", "--conda-name", type=str, default=None, help="Name of the conda package to use") -def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name): +@click.option("-p", "--conda-package-version", type=str, default=None, help="Version of conda package to use") +def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version): """ Create a new DSL2 module from the nf-core template. @@ -514,7 +515,9 @@ def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_nam # Run function try: - module_create = nf_core.modules.ModuleCreate(dir, tool, author, label, has_meta, force, conda_name) + module_create = nf_core.modules.ModuleCreate( + dir, tool, author, label, has_meta, force, conda_name, conda_package_version + ) module_create.create() except UserWarning as e: log.critical(e) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 8473f64b8d..60a11f90ac 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -25,7 +25,15 @@ class ModuleCreate(object): def __init__( - self, directory=".", tool="", author=None, process_label=None, has_meta=None, force=False, conda_name=None + self, + directory=".", + tool="", + author=None, + process_label=None, + has_meta=None, + force=False, + conda_name=None, + conda_version=None, ): self.directory = directory self.tool = tool @@ -35,6 +43,7 @@ def __init__( self.force_overwrite = force self.subtool = None self.tool_conda_name = conda_name + self.tool_conda_version = conda_version self.tool_licence = None self.repo_type = None self.tool_licence = "" @@ -131,9 +140,13 @@ def create(self): anaconda_response = nf_core.utils.anaconda_package(self.tool_conda_name, ["bioconda"]) else: anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"]) - version = anaconda_response.get("latest_version") - if not version: - version = str(max([parse_version(v) for v in anaconda_response["versions"]])) + if not self.tool_conda_version: + version = questionary.select( + "Select bioconda version:", + choices=[str(parse_version(v)) for v in anaconda_response["versions"]], + ).unsafe_ask() + else: + version = self.tool_conda_version self.tool_licence = nf_core.utils.parse_anaconda_licence(anaconda_response, version) self.tool_description = anaconda_response.get("summary", "") self.tool_doc_url = anaconda_response.get("doc_url", "") From 658dce5ace4caecbf4273e21295e7dacf409e509 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Mon, 2 Aug 2021 10:49:01 +0200 Subject: [PATCH 002/186] Update changelog and remove prompt --- CHANGELOG.md | 1 + nf_core/modules/create.py | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9f518293f..d2abc015ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### Modules * Fixed typo in `module_utils.py`. +* Added `--conda-package-version` flag for specifying version of conda package in `nf-core modules create`. ([#1238](https://github.com/nf-core/tools/issues/1238)) ## [v2.1 - Zinc Zebra](https://github.com/nf-core/tools/releases/tag/2.1) - [2021-07-27] diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 60a11f90ac..6fa3e74bbc 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -140,13 +140,14 @@ def create(self): anaconda_response = nf_core.utils.anaconda_package(self.tool_conda_name, ["bioconda"]) else: anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"]) + if not self.tool_conda_version: - version = questionary.select( - "Select bioconda version:", - choices=[str(parse_version(v)) for v in anaconda_response["versions"]], - ).unsafe_ask() + version = anaconda_response.get("latest_version") + if not version: + version = str(max([parse_version(v) for v in anaconda_response["versions"]])) else: version = self.tool_conda_version + self.tool_licence = nf_core.utils.parse_anaconda_licence(anaconda_response, version) self.tool_description = anaconda_response.get("summary", "") self.tool_doc_url = anaconda_response.get("doc_url", "") From 2bc2e6a96d43a15b21d673558fd69b7ed29e19f1 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Mon, 2 Aug 2021 11:57:35 +0200 Subject: [PATCH 003/186] Add modules ignored table --- nf_core/modules/bump_versions.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 4d24faa52b..840ddff2f2 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -52,6 +52,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): self.up_to_date = [] self.updated = [] self.failed = [] + self.ignored = [] self.show_up_to_date = show_uptodate # Verify that this is not a pipeline @@ -132,7 +133,7 @@ def bump_module_version(self, module: NFCoreModule): if module.module_name in self.bump_versions_config: config_version = self.bump_versions_config[module.module_name] if not config_version: - self.up_to_date.append((f"Omitting module due to config: {module.module_name}", module.module_name)) + self.ignored.append((f"Omitting module due to config.", module.module_name)) return False # check for correct version and newer versions @@ -333,3 +334,16 @@ def format_result(module_updates, table): table.add_column("Update message") table = format_result(self.failed, table) console.print(table) + + # Table of modules ignored due to `.nf-core.yml` + if len(self.ignored) > 0: + console.print( + rich.panel.Panel( + r"[!] {} Module update{} ignored".format(len(self.ignored), _s(self.ignored)), style="grey58" + ) + ) + table = Table(style="grey58", box=rich.box.ROUNDED) + table.add_column("Module name", width=max_mod_name_len) + table.add_column("Update message") + table = format_result(self.ignored, table) + console.print(table) From e7eacf9ff653064e354176c41e61afbc134f6eea Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Mon, 2 Aug 2021 11:59:08 +0200 Subject: [PATCH 004/186] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9f518293f..704693bbbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### Modules * Fixed typo in `module_utils.py`. +* Added modules ignored table to `nf-core modules bump-versions`. ([#1234](https://github.com/nf-core/tools/issues/1234)) ## [v2.1 - Zinc Zebra](https://github.com/nf-core/tools/releases/tag/2.1) - [2021-07-27] From acb5a4c23a1594f57d81f785e52d2921f81588e6 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Sun, 8 Aug 2021 23:03:45 +0200 Subject: [PATCH 005/186] Add option of writing diff to file (including prompt) --- nf_core/modules/update.py | 125 +++++++++++++++++++++++++++++--------- 1 file changed, 95 insertions(+), 30 deletions(-) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index dc69c5b15a..afe77557be 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -4,6 +4,7 @@ import logging import tempfile import difflib +import enum from rich.console import Console from rich.syntax import Syntax @@ -180,6 +181,27 @@ def update(self, module): if not modules_json: return False + if self.diff: + # Ask the user whether the diffs should be written to a file or + # or displayed in the terminal directly + diff_file = questionary.select( + "How should the diffs be handled?", + choices=[ + {"name": "Display in terminal", "value": False}, + {"name": "Write to file", "value": True}, + ], + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + if diff_file: + diff_file_name = questionary.text( + "Enter the file name: ", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + while os.path.exists(diff_file_name): + diff_file_name = questionary.text( + f"'{diff_file_name}' already exists. Enter a new file name: ", + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + exit_value = True for modules_repo, module, sha in repos_mods_shas: dry_run = self.diff @@ -252,13 +274,17 @@ def update(self, module): continue if dry_run: - console = Console(force_terminal=nf_core.utils.rich_force_colors()) + + class DiffEnum(enum.Enum): + """Enumeration for keeping track of the diff status""" + + UNCHANGED = enum.auto() + CREATED = enum.auto() + REMOVED = enum.auto() + + diffs = {} files = os.listdir(os.path.join(*install_folder, module)) temp_folder = os.path.join(*install_folder, module) - log.info( - f"Changes in module '{module}' between ({current_entry['git_sha'] if current_entry is not None else '?'}) and ({version if version is not None else 'latest'})" - ) - for file in files: temp_path = os.path.join(temp_folder, file) curr_path = os.path.join(module_dir, file) @@ -268,10 +294,8 @@ def update(self, module): with open(curr_path, "r") as fh: old_lines = fh.readlines() if new_lines == old_lines: - # The files are identical - log.info(f"'{os.path.join(module, file)}' is unchanged") + diffs[file] = DiffEnum.UNCHANGED else: - log.info(f"Changes in '{os.path.join(module, file)}':") # Compute the diff diff = difflib.unified_diff( old_lines, @@ -279,32 +303,73 @@ def update(self, module): fromfile=f"{os.path.join(module, file)} (installed)", tofile=f"{os.path.join(module, file)} (new)", ) + diffs[file] = diff + elif os.path.exists(temp_path): + diff[file] = DiffEnum.CREATED + elif os.path.exists(curr_path): + diffs[file] = DiffEnum.REMOVED + + if diff_file: + log.info(f"Writing diff of '{module}' to file") + with open(diff_file_name, "a") as fh: + fh.write( + f"Changes in module '{module}' between ({current_entry['git_sha'] if current_entry is not None else '?'}) and ({version if version is not None else 'latest'})\n" + ) + for file, diff in diffs.items(): + if diff == DiffEnum.UNCHANGED: + # The files are identical + fh.write(f"'{os.path.join(module, file)}' is unchanged\n") + elif diff == DiffEnum.CREATED: + # The file was created between the commits + fh.write(f"'{os.path.join(module, file)} was created\n") + elif diff == DiffEnum.REMOVED: + # The file was removed between the commits + fh.write(f"'{os.path.join(module, file)} was removed\n") + else: + fh.write(f"Changes in '{os.path.join(module, file)}':\n") + # Pretty print the diff using the pygments diff lexer + for line in diff: + fh.write(line) + fh.write("\n") + + fh.write("*" * 60 + "\n") + else: + console = Console(force_terminal=nf_core.utils.rich_force_colors()) + log.info( + f"Changes in module '{module}' between ({current_entry['git_sha'] if current_entry is not None else '?'}) and ({version if version is not None else 'latest'})" + ) + # Ask the user if they want to install the module + for file, diff in diffs.items(): + if diff == DiffEnum.UNCHANGED: + # The files are identical + log.info(f"'{os.path.join(module, file)}' is unchanged") + elif diff == DiffEnum.CREATED: + # The file was created between the commits + log.info(f"'{os.path.join(module, file)} was created") + elif diff == DiffEnum.REMOVED: + # The file was removed between the commits + log.info(f"'{os.path.join(module, file)} was removed") + else: + log.info(f"Changes in '{os.path.join(module, file)}':") # Pretty print the diff using the pygments diff lexer console.print(Syntax("".join(diff), "diff", theme="ansi_light")) - elif os.path.exists(temp_path): - # The file was created between the commits - log.info(f"Created file '{file}'") - - elif os.path.exists(curr_path): - # The file was removed between the commits - log.info(f"Removed file '{file}'") - - # Ask the user if they want to install the module - dry_run = not questionary.confirm("Update module?", default=False).unsafe_ask() - if not dry_run: - # The new module files are already installed - # we just need to clear the directory and move the - # new files from the temporary directory - self.clear_module_dir(module, module_dir) - os.mkdir(module_dir) - for file in files: - path = os.path.join(temp_folder, file) - if os.path.exists(path): - shutil.move(path, os.path.join(module_dir, file)) - log.info(f"Updating '{modules_repo.name}/{module}'") - log.debug(f"Updating module '{module}' to {version} from {modules_repo.name}") + dry_run = not questionary.confirm( + "Update module?", default=False, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + if not dry_run: + # The new module files are already installed + # we just need to clear the directory and move the + # new files from the temporary directory + self.clear_module_dir(module, module_dir) + os.mkdir(module_dir) + for file in files: + path = os.path.join(temp_folder, file) + if os.path.exists(path): + shutil.move(path, os.path.join(module_dir, file)) + log.info(f"Updating '{modules_repo.name}/{module}'") + log.debug(f"Updating module '{module}' to {version} from {modules_repo.name}") if not dry_run: # Update module.json with newly installed module From ba78e11e3920840a7fccca642b82ef174f32f044 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Mon, 9 Aug 2021 09:12:15 +0200 Subject: [PATCH 006/186] Make code nicer --- nf_core/modules/update.py | 65 ++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index afe77557be..47f4d73a27 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -276,15 +276,27 @@ def update(self, module): if dry_run: class DiffEnum(enum.Enum): - """Enumeration for keeping track of the diff status""" + """ + Enumeration for keeping track of + the diff status of a pair of files + """ UNCHANGED = enum.auto() + CHANGED = enum.auto() CREATED = enum.auto() REMOVED = enum.auto() diffs = {} - files = os.listdir(os.path.join(*install_folder, module)) + + # Get all (unique) file names in the two folders. + # `dict.fromkeys()` is used instead of `set()` to preserve order + files = dict.fromkeys(os.listdir(os.path.join(*install_folder, module))) + files.update(dict.fromkeys(os.listdir(module_dir))) + files = list(files) + temp_folder = os.path.join(*install_folder, module) + + # Loop through all the module files and compute their diffs if needed for file in files: temp_path = os.path.join(temp_folder, file) curr_path = os.path.join(module_dir, file) @@ -294,7 +306,8 @@ class DiffEnum(enum.Enum): with open(curr_path, "r") as fh: old_lines = fh.readlines() if new_lines == old_lines: - diffs[file] = DiffEnum.UNCHANGED + # The files are identical + diffs[file] = (DiffEnum.UNCHANGED, ()) else: # Compute the diff diff = difflib.unified_diff( @@ -303,11 +316,19 @@ class DiffEnum(enum.Enum): fromfile=f"{os.path.join(module, file)} (installed)", tofile=f"{os.path.join(module, file)} (new)", ) - diffs[file] = diff + diffs[file] = (DiffEnum.CHANGED, diff) elif os.path.exists(temp_path): - diff[file] = DiffEnum.CREATED + # The file was created + with open(temp_path, "r") as fh: + new_lines = fh.readlines() + + diffs[file] = (DiffEnum.CREATED, ()) elif os.path.exists(curr_path): - diffs[file] = DiffEnum.REMOVED + # The file was removed + with open(curr_path, "r") as fh: + old_lines = fh.readlines() + + diffs[file] = (DiffEnum.REMOVED, ()) if diff_file: log.info(f"Writing diff of '{module}' to file") @@ -315,19 +336,20 @@ class DiffEnum(enum.Enum): fh.write( f"Changes in module '{module}' between ({current_entry['git_sha'] if current_entry is not None else '?'}) and ({version if version is not None else 'latest'})\n" ) - for file, diff in diffs.items(): - if diff == DiffEnum.UNCHANGED: + for file, d in diffs.items(): + diff_status, diff = d + if diff_status == DiffEnum.UNCHANGED: # The files are identical fh.write(f"'{os.path.join(module, file)}' is unchanged\n") - elif diff == DiffEnum.CREATED: + elif diff_status == DiffEnum.CREATED: # The file was created between the commits - fh.write(f"'{os.path.join(module, file)} was created\n") - elif diff == DiffEnum.REMOVED: + fh.write(f"'{os.path.join(module, file)}' was created\n") + elif diff_status == DiffEnum.REMOVED: # The file was removed between the commits - fh.write(f"'{os.path.join(module, file)} was removed\n") + fh.write(f"'{os.path.join(module, file)}' was removed\n") else: fh.write(f"Changes in '{os.path.join(module, file)}':\n") - # Pretty print the diff using the pygments diff lexer + # Write the diff lines to the file for line in diff: fh.write(line) fh.write("\n") @@ -340,16 +362,17 @@ class DiffEnum(enum.Enum): ) # Ask the user if they want to install the module - for file, diff in diffs.items(): - if diff == DiffEnum.UNCHANGED: + for file, d in diffs.items(): + diff_status, diff = d + if diff_status == DiffEnum.UNCHANGED: # The files are identical log.info(f"'{os.path.join(module, file)}' is unchanged") - elif diff == DiffEnum.CREATED: + elif diff_status == DiffEnum.CREATED: # The file was created between the commits - log.info(f"'{os.path.join(module, file)} was created") - elif diff == DiffEnum.REMOVED: + log.info(f"'{os.path.join(module, file)}' was created:") + elif diff_status == DiffEnum.REMOVED: # The file was removed between the commits - log.info(f"'{os.path.join(module, file)} was removed") + log.info(f"'{os.path.join(module, file)}' was removed:") else: log.info(f"Changes in '{os.path.join(module, file)}':") # Pretty print the diff using the pygments diff lexer @@ -359,8 +382,8 @@ class DiffEnum(enum.Enum): "Update module?", default=False, style=nf_core.utils.nfcore_question_style ).unsafe_ask() if not dry_run: - # The new module files are already installed - # we just need to clear the directory and move the + # The new module files are already installed. + # We just need to clear the directory and move the # new files from the temporary directory self.clear_module_dir(module, module_dir) os.mkdir(module_dir) From a20d1ef28e89fc4c1edeab3a38e9593600866aed Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Mon, 9 Aug 2021 09:21:25 +0200 Subject: [PATCH 007/186] Add option to remove diff file --- nf_core/modules/update.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index 47f4d73a27..c173af0fba 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -5,6 +5,7 @@ import tempfile import difflib import enum +from questionary import question from rich.console import Console from rich.syntax import Syntax @@ -197,8 +198,11 @@ def update(self, module): "Enter the file name: ", style=nf_core.utils.nfcore_question_style ).unsafe_ask() while os.path.exists(diff_file_name): + if questionary.confirm("'{diff_file_name}' exists. Remove file?"): + os.remove(diff_file_name) + break diff_file_name = questionary.text( - f"'{diff_file_name}' already exists. Enter a new file name: ", + f"Enter a new file name: ", style=nf_core.utils.nfcore_question_style, ).unsafe_ask() @@ -305,6 +309,7 @@ class DiffEnum(enum.Enum): new_lines = fh.readlines() with open(curr_path, "r") as fh: old_lines = fh.readlines() + if new_lines == old_lines: # The files are identical diffs[file] = (DiffEnum.UNCHANGED, ()) @@ -317,17 +322,13 @@ class DiffEnum(enum.Enum): tofile=f"{os.path.join(module, file)} (new)", ) diffs[file] = (DiffEnum.CHANGED, diff) + elif os.path.exists(temp_path): # The file was created - with open(temp_path, "r") as fh: - new_lines = fh.readlines() - diffs[file] = (DiffEnum.CREATED, ()) + elif os.path.exists(curr_path): # The file was removed - with open(curr_path, "r") as fh: - old_lines = fh.readlines() - diffs[file] = (DiffEnum.REMOVED, ()) if diff_file: @@ -336,18 +337,23 @@ class DiffEnum(enum.Enum): fh.write( f"Changes in module '{module}' between ({current_entry['git_sha'] if current_entry is not None else '?'}) and ({version if version is not None else 'latest'})\n" ) + for file, d in diffs.items(): diff_status, diff = d if diff_status == DiffEnum.UNCHANGED: # The files are identical fh.write(f"'{os.path.join(module, file)}' is unchanged\n") + elif diff_status == DiffEnum.CREATED: # The file was created between the commits fh.write(f"'{os.path.join(module, file)}' was created\n") + elif diff_status == DiffEnum.REMOVED: # The file was removed between the commits fh.write(f"'{os.path.join(module, file)}' was removed\n") + else: + # The file has changed fh.write(f"Changes in '{os.path.join(module, file)}':\n") # Write the diff lines to the file for line in diff: @@ -361,7 +367,6 @@ class DiffEnum(enum.Enum): f"Changes in module '{module}' between ({current_entry['git_sha'] if current_entry is not None else '?'}) and ({version if version is not None else 'latest'})" ) - # Ask the user if they want to install the module for file, d in diffs.items(): diff_status, diff = d if diff_status == DiffEnum.UNCHANGED: @@ -374,10 +379,12 @@ class DiffEnum(enum.Enum): # The file was removed between the commits log.info(f"'{os.path.join(module, file)}' was removed:") else: + # The file has changed log.info(f"Changes in '{os.path.join(module, file)}':") # Pretty print the diff using the pygments diff lexer console.print(Syntax("".join(diff), "diff", theme="ansi_light")) + # Ask the user if they want to install the module dry_run = not questionary.confirm( "Update module?", default=False, style=nf_core.utils.nfcore_question_style ).unsafe_ask() From 09b85ecb4acfc0ec3b17df9f6713d4a28fbe885f Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Mon, 9 Aug 2021 10:07:44 +0200 Subject: [PATCH 008/186] Add '--diff-file' flag --- nf_core/__main__.py | 11 +++++-- nf_core/modules/update.py | 62 +++++++++++++++++++++++---------------- 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 4d828fd671..abcc8b716f 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -452,7 +452,14 @@ def install(ctx, tool, dir, prompt, force, sha): @click.option( "-c", "--diff", is_flag=True, default=False, help="Show differences between module versions before updating" ) -def update(ctx, tool, dir, force, prompt, sha, all, diff): +@click.option( + "--diff-file", + type=str, + metavar="", + default=None, + help="Specify the file to which the module diffs should be written", +) +def update(ctx, tool, dir, force, prompt, sha, all, diff, diff_file): """ Update DSL2 modules within a pipeline. @@ -460,7 +467,7 @@ def update(ctx, tool, dir, force, prompt, sha, all, diff): """ try: module_install = nf_core.modules.ModuleUpdate( - dir, force=force, prompt=prompt, sha=sha, update_all=all, diff=diff + dir, force=force, prompt=prompt, sha=sha, update_all=all, diff=diff, diff_file=diff_file ) module_install.modules_repo = ctx.obj["modules_repo_obj"] exit_status = module_install.update(tool) diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index c173af0fba..17f74b4e4a 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -20,13 +20,16 @@ class ModuleUpdate(ModuleCommand): - def __init__(self, pipeline_dir, force=False, prompt=False, sha=None, update_all=False, diff=False): + def __init__(self, pipeline_dir, force=False, prompt=False, sha=None, update_all=False, diff=False, diff_file=None): super().__init__(pipeline_dir) self.force = force self.prompt = prompt self.sha = sha self.update_all = update_all self.diff = diff + self.diff_file = diff_file + if self.diff_file is not None: + self.diff = True def update(self, module): if self.repo_type == "modules": @@ -159,6 +162,7 @@ def update(self, module): repos_mods_shas[repo_name].append((module, custom_sha)) else: skipped_repos.append(repo_name) + if skipped_repos: skipped_str = "', '".join(skipped_repos) log.info(f"Skipping modules in repositor{'y' if len(skipped_repos) == 1 else 'ies'}: '{skipped_str}'") @@ -183,28 +187,34 @@ def update(self, module): return False if self.diff: - # Ask the user whether the diffs should be written to a file or - # or displayed in the terminal directly - diff_file = questionary.select( - "How should the diffs be handled?", - choices=[ - {"name": "Display in terminal", "value": False}, - {"name": "Write to file", "value": True}, - ], - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - if diff_file: - diff_file_name = questionary.text( - "Enter the file name: ", style=nf_core.utils.nfcore_question_style + if self.diff_file is None: + # Ask the user whether the diffs should be written to a file or + # or displayed in the terminal directly + self.diff_file = questionary.select( + "How should the diffs be handled?", + choices=[ + {"name": "Display in terminal", "value": False}, + {"name": "Write to file", "value": True}, + ], + style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - while os.path.exists(diff_file_name): - if questionary.confirm("'{diff_file_name}' exists. Remove file?"): - os.remove(diff_file_name) - break - diff_file_name = questionary.text( - f"Enter a new file name: ", - style=nf_core.utils.nfcore_question_style, + if self.diff_file: + self.diff_file = questionary.text( + "Enter the filename: ", style=nf_core.utils.nfcore_question_style ).unsafe_ask() + while os.path.exists(self.diff_file): + if questionary.confirm(f"'{self.diff_file}' exists. Remove file?"): + os.remove(self.diff_file) + break + self.diff_file = questionary.text( + f"Enter a new filename: ", + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + elif os.path.exists(self.diff_file): + # Since we append to the file later, it should be empty to begin with + os.remove( + self.diff_file, + ) exit_value = True for modules_repo, module, sha in repos_mods_shas: @@ -292,7 +302,7 @@ class DiffEnum(enum.Enum): diffs = {} - # Get all (unique) file names in the two folders. + # Get all unique filenames in the two folders. # `dict.fromkeys()` is used instead of `set()` to preserve order files = dict.fromkeys(os.listdir(os.path.join(*install_folder, module))) files.update(dict.fromkeys(os.listdir(module_dir))) @@ -331,9 +341,9 @@ class DiffEnum(enum.Enum): # The file was removed diffs[file] = (DiffEnum.REMOVED, ()) - if diff_file: + if self.diff_file: log.info(f"Writing diff of '{module}' to file") - with open(diff_file_name, "a") as fh: + with open(self.diff_file, "a") as fh: fh.write( f"Changes in module '{module}' between ({current_entry['git_sha'] if current_entry is not None else '?'}) and ({version if version is not None else 'latest'})\n" ) @@ -374,10 +384,10 @@ class DiffEnum(enum.Enum): log.info(f"'{os.path.join(module, file)}' is unchanged") elif diff_status == DiffEnum.CREATED: # The file was created between the commits - log.info(f"'{os.path.join(module, file)}' was created:") + log.info(f"'{os.path.join(module, file)}' was created") elif diff_status == DiffEnum.REMOVED: # The file was removed between the commits - log.info(f"'{os.path.join(module, file)}' was removed:") + log.info(f"'{os.path.join(module, file)}' was removed") else: # The file has changed log.info(f"Changes in '{os.path.join(module, file)}':") From 66c795ae32d5486e44a5b268b6c81c0596b55c46 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Mon, 9 Aug 2021 10:18:02 +0200 Subject: [PATCH 009/186] Update docs and CHANGELOG.md --- CHANGELOG.md | 1 + README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c757b4ae2b..e35507039a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ * Fixed typo in `module_utils.py`. * Added `--diff` flag to `nf-core modules update` which shows the diff between the installed files and the versions +* Add option of writing diffs to file in `nf-core modules update` using either interactive prompts or the new `--diff-file` flag. ## [v2.1 - Zinc Zebra](https://github.com/nf-core/tools/releases/tag/2.1) - [2021-07-27] diff --git a/README.md b/README.md index c874796049..ee463d301f 100644 --- a/README.md +++ b/README.md @@ -1025,6 +1025,7 @@ There are five additional flags that you can use with this command: * `--prompt`: Select the module version using a cli prompt. * `--sha `: Install the module at a specific commit from the `nf-core/modules` repository. * `--diff`: Show the diff between the installed files and the new version before installing. +* `--diff-file `: Specify where the diffs between the local and remote versions of a module should be written * `--all`: Use this flag to run the command on all modules in the pipeline. If you don't want to update certain modules or want to update them to specific versions, you can make use of the `.nf-core.yml` configuration file. For example, you can prevent the `star/align` module installed from `nf-core/modules` from being updated by adding the following to the `.nf-core.yml` file: From 46c186646aaee7b3d0b1c6b51a3876f6ef4de9e2 Mon Sep 17 00:00:00 2001 From: Ramprasad Neethiraj <20065894+ramprasadn@users.noreply.github.com> Date: Thu, 30 Sep 2021 11:26:35 +0200 Subject: [PATCH 010/186] Update input check subworkflow --- .../subworkflows/local/input_check.nf | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/input_check.nf b/nf_core/pipeline-template/subworkflows/local/input_check.nf index b664bc8caf..1e3e761a45 100644 --- a/nf_core/pipeline-template/subworkflows/local/input_check.nf +++ b/nf_core/pipeline-template/subworkflows/local/input_check.nf @@ -13,30 +13,33 @@ workflow INPUT_CHECK { main: SAMPLESHEET_CHECK ( samplesheet ) .splitCsv ( header:true, sep:',' ) - .map { create_fastq_channels(it) } - .set { reads } + .set { sheet } + + reads = sheet.map { create_fastq_channel(it) } emit: reads // channel: [ val(meta), [ reads ] ] } // Function to get list of [ meta, [ fastq_1, fastq_2 ] ] -def create_fastq_channels(LinkedHashMap row) { +def create_fastq_channel(LinkedHashMap row) { + // create meta map def meta = [:] meta.id = row.sample meta.single_end = row.single_end.toBoolean() - def array = [] + // add path(s) of the fastq file(s) to the meta map + def fastq_meta = [] if (!file(row.fastq_1).exists()) { exit 1, "ERROR: Please check input samplesheet -> Read 1 FastQ file does not exist!\n${row.fastq_1}" } if (meta.single_end) { - array = [ meta, [ file(row.fastq_1) ] ] + fastq_meta = [ meta, [ file(row.fastq_1) ] ] } else { if (!file(row.fastq_2).exists()) { exit 1, "ERROR: Please check input samplesheet -> Read 2 FastQ file does not exist!\n${row.fastq_2}" } - array = [ meta, [ file(row.fastq_1), file(row.fastq_2) ] ] + fastq_meta = [ meta, [ file(row.fastq_1), file(row.fastq_2) ] ] } - return array + return fastq_meta } From 612ec86bd43cead167862fc6fe554bc550128fd4 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 26 Nov 2021 11:47:37 +0000 Subject: [PATCH 011/186] Update input_check.nf --- nf_core/pipeline-template/subworkflows/local/input_check.nf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/input_check.nf b/nf_core/pipeline-template/subworkflows/local/input_check.nf index 1e3e761a45..f0268c69fe 100644 --- a/nf_core/pipeline-template/subworkflows/local/input_check.nf +++ b/nf_core/pipeline-template/subworkflows/local/input_check.nf @@ -13,9 +13,8 @@ workflow INPUT_CHECK { main: SAMPLESHEET_CHECK ( samplesheet ) .splitCsv ( header:true, sep:',' ) - .set { sheet } - - reads = sheet.map { create_fastq_channel(it) } + .map { create_fastq_channel(it) } + .set { reads } emit: reads // channel: [ val(meta), [ reads ] ] From 3c1d0fcf62821965b3326dc09ed5dc955f19ee04 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 26 Nov 2021 11:47:52 +0000 Subject: [PATCH 012/186] Update input_check.nf --- nf_core/pipeline-template/subworkflows/local/input_check.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/input_check.nf b/nf_core/pipeline-template/subworkflows/local/input_check.nf index f0268c69fe..b5c8c28118 100644 --- a/nf_core/pipeline-template/subworkflows/local/input_check.nf +++ b/nf_core/pipeline-template/subworkflows/local/input_check.nf @@ -24,8 +24,8 @@ workflow INPUT_CHECK { def create_fastq_channel(LinkedHashMap row) { // create meta map def meta = [:] - meta.id = row.sample - meta.single_end = row.single_end.toBoolean() + meta.id = row.sample + meta.single_end = row.single_end.toBoolean() // add path(s) of the fastq file(s) to the meta map def fastq_meta = [] From faa75fc219946037f991eb2099612d540758f9d1 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 14 Dec 2021 16:49:40 +0000 Subject: [PATCH 013/186] Bump versions to 2.3dev --- CHANGELOG.md | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93ec4fa4a6..3a9522cea7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # nf-core/tools: Changelog +## v2.3dev + +### Template + +### General + +### Modules + ## [v2.2 - Lead Liger](https://github.com/nf-core/tools/releases/tag/2.2) - [2021-12-14] ### Template diff --git a/setup.py b/setup.py index 8b910389fb..f6b47282a4 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -version = "2.2" +version = "2.3dev" with open("README.md") as f: readme = f.read() From 85a9b20b0898d7ab67be23acaa02f711d5bf8e31 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 16 Dec 2021 08:13:28 +0100 Subject: [PATCH 014/186] Pin jsonschema minimum 3.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 33da40c47e..b6e20629d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ click GitPython jinja2 -jsonschema +jsonschema>=3.0 packaging prompt_toolkit>=3.0.3 pyyaml From 9268338b90c96544cd56ea4e288d19e6591cff7d Mon Sep 17 00:00:00 2001 From: Simon Pearce <24893913+SPPearce@users.noreply.github.com> Date: Thu, 16 Dec 2021 17:13:46 +0000 Subject: [PATCH 015/186] Update test_yml_builder.py Moved the elem.replace line to before the test_file variable is created, so it now does something. --- nf_core/modules/test_yml_builder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index d917a81053..e5c4e030cc 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -228,6 +228,8 @@ def create_test_file_dict(self, results_dir, is_repeat=False): for root, dir, file in os.walk(results_dir): for elem in file: elem = os.path.join(root, elem) + # Switch out the results directory path with the expected 'output' directory + elem = elem.replace(results_dir, "output") test_file = {"path": elem} # Check that this isn't an empty file if self.check_if_empty_file(elem): @@ -236,8 +238,7 @@ def create_test_file_dict(self, results_dir, is_repeat=False): else: elem_md5 = self._md5(elem) test_file["md5sum"] = elem_md5 - # Switch out the results directory path with the expected 'output' directory - elem = elem.replace(results_dir, "output") + test_files.append(test_file) test_files = sorted(test_files, key=operator.itemgetter("path")) From 91581c4489977f64e4146a38cff8386838b56aa7 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 16 Dec 2021 19:36:54 +0100 Subject: [PATCH 016/186] Test yml builder: Switch output directory after we check the file --- nf_core/modules/test_yml_builder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index e5c4e030cc..c843bae558 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -228,9 +228,7 @@ def create_test_file_dict(self, results_dir, is_repeat=False): for root, dir, file in os.walk(results_dir): for elem in file: elem = os.path.join(root, elem) - # Switch out the results directory path with the expected 'output' directory - elem = elem.replace(results_dir, "output") - test_file = {"path": elem} + test_file = {} # Check that this isn't an empty file if self.check_if_empty_file(elem): if not is_repeat: @@ -238,7 +236,9 @@ def create_test_file_dict(self, results_dir, is_repeat=False): else: elem_md5 = self._md5(elem) test_file["md5sum"] = elem_md5 - + # Switch out the results directory path with the expected 'output' directory + elem = elem.replace(results_dir, "output") + test_file["path"] = elem test_files.append(test_file) test_files = sorted(test_files, key=operator.itemgetter("path")) From faa79e7562e6e02de856aa877f2f584237bd6ce1 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 16 Dec 2021 19:45:35 +0100 Subject: [PATCH 017/186] Test file yml: refactor to keep path as first key in dict --- nf_core/modules/test_yml_builder.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index c843bae558..5bfe2365c7 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -228,7 +228,7 @@ def create_test_file_dict(self, results_dir, is_repeat=False): for root, dir, file in os.walk(results_dir): for elem in file: elem = os.path.join(root, elem) - test_file = {} + test_file = {"path": elem} # add the key here so that it comes first in the dict # Check that this isn't an empty file if self.check_if_empty_file(elem): if not is_repeat: @@ -237,8 +237,7 @@ def create_test_file_dict(self, results_dir, is_repeat=False): elem_md5 = self._md5(elem) test_file["md5sum"] = elem_md5 # Switch out the results directory path with the expected 'output' directory - elem = elem.replace(results_dir, "output") - test_file["path"] = elem + test_file["path"] = elem.replace(results_dir, "output") test_files.append(test_file) test_files = sorted(test_files, key=operator.itemgetter("path")) From ce80987764be1316e81c55bedf37f73b39af9875 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 16 Dec 2021 19:53:37 +0100 Subject: [PATCH 018/186] Lint: Modules check fail, for pipelines only --- nf_core/modules/lint/__init__.py | 4 +++- nf_core/modules/lint/module_changes.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index af3367765f..892cacf8de 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -93,10 +93,12 @@ def __init__(self, dir): if self.repo_type == "pipeline": # Add as first test to load git_sha before module_changes self.lint_tests.insert(0, "module_version") + # Only check if modules have been changed in pipelines + self.lint_tests.append("module_changes") @staticmethod def _get_all_lint_tests(): - return ["main_nf", "meta_yml", "module_changes", "module_todos", "module_deprecations"] + return ["main_nf", "meta_yml", "module_todos", "module_deprecations"] def lint(self, module=None, key=(), all_modules=False, print_results=True, show_passed=False, local=False): """ diff --git a/nf_core/modules/lint/module_changes.py b/nf_core/modules/lint/module_changes.py index 44c2501424..4e466a2c3c 100644 --- a/nf_core/modules/lint/module_changes.py +++ b/nf_core/modules/lint/module_changes.py @@ -15,6 +15,8 @@ def module_changes(module_lint_object, module): and compares them to the local copies If the module has a 'git_sha', the file content is checked against this sha + + Only runs when linting a pipeline, not the modules repository """ files_to_check = ["main.nf", "meta.yml"] @@ -49,7 +51,7 @@ def module_changes(module_lint_object, module): remote_copy = r.content.decode("utf-8") if local_copy != remote_copy: - module.warned.append( + module.failed.append( ( "check_local_copy", "Local copy of module does not match remote", From 8e7acae497b618d31cd488b7df5d0993893969fe Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 16 Dec 2021 19:56:15 +0100 Subject: [PATCH 019/186] Changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a9522cea7..4927241768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### Modules +* Linting a pipeline now fails instead of warning if a local copy of a module does not match the remote ([#1313](https://github.com/nf-core/tools/issues/1313)) + ## [v2.2 - Lead Liger](https://github.com/nf-core/tools/releases/tag/2.2) - [2021-12-14] ### Template @@ -59,7 +61,6 @@ * Check if README is from modules repo * Update module template to DSL2 v2.0 (remove `functions.nf` from modules template and updating `main.nf` ([#1289](https://github.com/nf-core/tools/pull/)) * Substitute get process/module name custom functions in module `main.nf` using template replacement ([#1284](https://github.com/nf-core/tools/issues/1284)) -* Linting now fails instead of warning if a local copy of a module does not match the remote ([#1313](https://github.com/nf-core/tools/issues/1313)) * Check test YML file for md5sums corresponding to empty files ([#1302](https://github.com/nf-core/tools/issues/1302)) * Exit with an error if empty files are found when generating the test YAML file ([#1302](https://github.com/nf-core/tools/issues/1302)) From 638382797df420377a3c6f237c37756d24f684c5 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 17 Dec 2021 07:42:23 +0000 Subject: [PATCH 020/186] Bump version to 2.2.1 --- CHANGELOG.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a9522cea7..f1f925ea5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## v2.3dev +## [v2.2.1 - Lead Ligerito](https://github.com/nf-core/tools/releases/tag/2.2.1) - [2021-12-17] ### Template diff --git a/setup.py b/setup.py index f6b47282a4..8336daf6aa 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -version = "2.3dev" +version = "2.2.1" with open("README.md") as f: readme = f.read() From 7758e2179dfd4bcaaeb6b7118663a80c9ac4489d Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 17 Dec 2021 07:44:14 +0000 Subject: [PATCH 021/186] Remove mention of --singularity_pull_docker_container --- nf_core/pipeline-template/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index aa1e007ada..9d56da69e2 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -47,7 +47,7 @@ On release, automated continuous integration tests run the pipeline on a full-si > * The pipeline comes with config profiles called `docker`, `singularity`, `podman`, `shifter`, `charliecloud` and `conda` which instruct the pipeline to use the named tool for software management. For example, `-profile test,docker`. > * Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile ` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment. - > * If you are using `singularity` and are persistently observing issues downloading Singularity images directly due to timeout or network issues, then you can use the `--singularity_pull_docker_container` parameter to pull and convert the Docker image instead. Alternatively, you can use the [`nf-core download`](https://nf-co.re/tools/#downloading-pipelines-for-offline-use) command to download images first, before running the pipeline. Setting the [`NXF_SINGULARITY_CACHEDIR` or `singularity.cacheDir`](https://www.nextflow.io/docs/latest/singularity.html?#singularity-docker-hub) Nextflow options enables you to store and re-use the images from a central location for future pipeline runs. + > * If you are using `singularity`, please use the [`nf-core download`](https://nf-co.re/tools/#downloading-pipelines-for-offline-use) command to download images first, before running the pipeline. Setting the [`NXF_SINGULARITY_CACHEDIR` or `singularity.cacheDir`](https://www.nextflow.io/docs/latest/singularity.html?#singularity-docker-hub) Nextflow options enables you to store and re-use the images from a central location for future pipeline runs. > * If you are using `conda`, it is highly recommended to use the [`NXF_CONDA_CACHEDIR` or `conda.cacheDir`](https://www.nextflow.io/docs/latest/conda.html) settings to store the environments in a central location for future pipeline runs. 4. Start running your own analysis! From e4dd247d6bf7ca892bd6d0ecb600dd5fdf316a0f Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 17 Dec 2021 08:21:10 +0000 Subject: [PATCH 022/186] Fix Module does not emit software version bug --- nf_core/modules/lint/main_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 065910d3b8..cfabf244be 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -85,7 +85,7 @@ def main_nf(module_lint_object, module): module.failed.append(("main_nf_meta_output", "'meta' map not emitted in output channel(s)", module.main_nf)) # Check that a software version is emitted - if "version" in outputs: + if "versions" in outputs: module.passed.append(("main_nf_version_emitted", "Module emits software version", module.main_nf)) else: module.warned.append(("main_nf_version_emitted", "Module does not emit software version", module.main_nf)) From 90a58bb11b804677f8f0d1cf109cd8e56c8f8cb2 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 17 Dec 2021 08:39:42 +0000 Subject: [PATCH 023/186] Update CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1f925ea5d..3dbe4f2ea2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,12 @@ ### Template -### General +* Removed mention of `--singularity_pull_docker_container` in pipeline `README.md` ### Modules +* Fixed a linting bug where warning was incorrectly generated for `Module does not emit software version` + ## [v2.2 - Lead Liger](https://github.com/nf-core/tools/releases/tag/2.2) - [2021-12-14] ### Template From 041cf47a8490dd1316788bf764d89e7fcdafcbdd Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 17 Dec 2021 08:48:14 +0000 Subject: [PATCH 024/186] Fix container versions do not match bug --- nf_core/modules/lint/main_nf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index cfabf244be..10a22aad96 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -163,9 +163,9 @@ def check_process_section(self, lines): if re.search("bioconda::", l): bioconda_packages = [b for b in l.split() if "bioconda::" in b] if re.search("org/singularity", l): - singularity_tag = l.split("/")[-1].replace('"', "").replace("'", "").split("--")[-1].strip() + singularity_tag = l.split()[0].split("/")[-1].split("/")[-1].replace('"', "").replace("'", "").split(':')[-1] if re.search("biocontainers", l): - docker_tag = l.split("/")[-1].replace('"', "").replace("'", "").split("--")[-1].strip() + docker_tag = l.split()[0].split("/")[-1].split("/")[-1].replace('"', "").replace("'", "").split(':')[-1] # Check that all bioconda packages have build numbers # Also check for newer versions From 3e9f8b1c218023debda2af92058b7f809d91ced6 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 17 Dec 2021 08:49:31 +0000 Subject: [PATCH 025/186] Update CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dbe4f2ea2..ee2137077d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,9 @@ ### Modules -* Fixed a linting bug where warning was incorrectly generated for `Module does not emit software version` +* Fixed linting bugs where warning was incorrectly generated for: + * `Module does not emit software version` + * `Container versions do not match` ## [v2.2 - Lead Liger](https://github.com/nf-core/tools/releases/tag/2.2) - [2021-12-14] From b6539bf8a03a867f7b04323863ad24d2a4b3addf Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 17 Dec 2021 21:49:41 +0000 Subject: [PATCH 026/186] Black --- nf_core/modules/lint/main_nf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 10a22aad96..5b37b8bbda 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -163,9 +163,11 @@ def check_process_section(self, lines): if re.search("bioconda::", l): bioconda_packages = [b for b in l.split() if "bioconda::" in b] if re.search("org/singularity", l): - singularity_tag = l.split()[0].split("/")[-1].split("/")[-1].replace('"', "").replace("'", "").split(':')[-1] + singularity_tag = ( + l.split()[0].split("/")[-1].split("/")[-1].replace('"', "").replace("'", "").split(":")[-1] + ) if re.search("biocontainers", l): - docker_tag = l.split()[0].split("/")[-1].split("/")[-1].replace('"', "").replace("'", "").split(':')[-1] + docker_tag = l.split()[0].split("/")[-1].split("/")[-1].replace('"', "").replace("'", "").split(":")[-1] # Check that all bioconda packages have build numbers # Also check for newer versions From d072a62b8ae5306b09b6df2266e9fb6cbf6f1eb8 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Sat, 18 Dec 2021 09:50:55 +0100 Subject: [PATCH 027/186] Replace = with~ in all places in pipeline template to stop vscode conflict not resolved issues --- nf_core/pipeline-template/conf/base.config | 4 +-- .../pipeline-template/conf/igenomes.config | 4 +-- nf_core/pipeline-template/conf/modules.config | 4 +-- nf_core/pipeline-template/conf/test.config | 4 +-- .../pipeline-template/conf/test_full.config | 4 +-- nf_core/pipeline-template/lib/Utils.groovy | 4 +-- .../lib/WorkflowPipeline.groovy | 4 +-- nf_core/pipeline-template/main.nf | 24 ++++++++-------- nf_core/pipeline-template/nextflow.config | 6 ++-- .../pipeline-template/workflows/pipeline.nf | 28 +++++++++---------- 10 files changed, 43 insertions(+), 43 deletions(-) diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index e0557b9e91..c56ce5a4c9 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -1,7 +1,7 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ {{ name }} Nextflow base config file -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A 'blank slate' config file, appropriate for general use on most high performance compute environments. Assumes that all software is installed and available on the PATH. Runs in `local` mode - all jobs will be run on the logged in environment. diff --git a/nf_core/pipeline-template/conf/igenomes.config b/nf_core/pipeline-template/conf/igenomes.config index 855948def1..6ff960c81a 100644 --- a/nf_core/pipeline-template/conf/igenomes.config +++ b/nf_core/pipeline-template/conf/igenomes.config @@ -1,7 +1,7 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Nextflow config file for iGenomes paths -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Defines reference genomes using iGenome paths. Can be used by any config that customises the base path using: $params.igenomes_base / --igenomes_base diff --git a/nf_core/pipeline-template/conf/modules.config b/nf_core/pipeline-template/conf/modules.config index a0506a4db4..f51e020fe0 100644 --- a/nf_core/pipeline-template/conf/modules.config +++ b/nf_core/pipeline-template/conf/modules.config @@ -1,7 +1,7 @@ /* -======================================================================================== +######################################################################################## Config file for defining DSL2 per module options and publishing paths -======================================================================================== +######################################################################################## Available keys to override module options: ext.args = Additional arguments appended to command in module. ext.args2 = Second set of arguments appended to command in module (multi-tool modules). diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index eb2a725c87..9dc33f2d14 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -1,7 +1,7 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Nextflow config file for running minimal tests -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Defines input files and everything required to run a fast and simple pipeline test. Use as follows: diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index 0fa82b7f90..ad37132e98 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -1,7 +1,7 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Nextflow config file for running full-size tests -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Defines input files and everything required to run a full size pipeline test. Use as follows: diff --git a/nf_core/pipeline-template/lib/Utils.groovy b/nf_core/pipeline-template/lib/Utils.groovy index 1b88aec0ea..28567bd70d 100755 --- a/nf_core/pipeline-template/lib/Utils.groovy +++ b/nf_core/pipeline-template/lib/Utils.groovy @@ -29,12 +29,12 @@ class Utils { conda_check_failed |= !(channels.indexOf('bioconda') < channels.indexOf('defaults')) if (conda_check_failed) { - log.warn "=============================================================================\n" + + log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " There is a problem with your Conda configuration!\n\n" + " You will need to set-up the conda-forge and bioconda channels correctly.\n" + " Please refer to https://bioconda.github.io/user/install.html#set-up-channels\n" + " NB: The order of the channels matters!\n" + - "===================================================================================" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" } } } diff --git a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy index ce022c3a43..0b442225ce 100755 --- a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy +++ b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy @@ -48,11 +48,11 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { // private static void genomeExistsError(params, log) { if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { - log.error "=============================================================================\n" + + log.error "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " Genome '${params.genome}' not found in any config files provided to the pipeline.\n" + " Currently, the available genome keys are:\n" + " ${params.genomes.keySet().join(", ")}\n" + - "===================================================================================" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" System.exit(1) } } diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index dca79e2684..104784f8ea 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -1,8 +1,8 @@ #!/usr/bin/env nextflow /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ {{ name }} -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/{{ name }} Website: https://nf-co.re/{{ short_name }} Slack : https://nfcore.slack.com/channels/{{ short_name }} @@ -12,25 +12,25 @@ nextflow.enable.dsl = 2 /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ GENOME PARAMETER VALUES -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ params.fasta = WorkflowMain.getGenomeAttribute(params, 'fasta') /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ VALIDATE & PRINT PARAMETER SUMMARY -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ WorkflowMain.initialise(workflow, params, log) /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ NAMED WORKFLOW FOR PIPELINE -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ include { {{ short_name|upper }} } from './workflows/{{ short_name }}' @@ -43,9 +43,9 @@ workflow NFCORE_{{ short_name|upper }} { } /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RUN ALL WORKFLOWS -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ // @@ -57,7 +57,7 @@ workflow { } /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ THE END -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index d5e3946ca8..2ec838b8d6 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -1,7 +1,7 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ {{ name }} Nextflow config file -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Default config options for all compute environments ---------------------------------------------------------------------------------------- */ @@ -121,7 +121,7 @@ if (!params.igenomes_ignore) { } // Export these variables to prevent local Python/R libraries from conflicting with those in the container -// The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. +// The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. // See https://apeltzer.github.io/post/03-julia-lang-nextflow/ for details on that. Once we have a common agreement on where to keep Julia packages, this is adjustable. env { diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 460a3cf20a..6ac463f8e8 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -1,7 +1,7 @@ /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ VALIDATE INPUTS -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ def summary_params = NfcoreSchema.paramsSummaryMap(workflow, params) @@ -18,18 +18,18 @@ for (param in checkPathParamList) { if (param) { file(param, checkIfExists: true if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input samplesheet not specified!' } /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONFIG FILES -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ ch_multiqc_config = file("$projectDir/assets/multiqc_config.yaml", checkIfExists: true) ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config) : Channel.empty() /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT LOCAL MODULES/SUBWORKFLOWS -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ // @@ -38,9 +38,9 @@ ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multi include { INPUT_CHECK } from '../subworkflows/local/input_check' /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT NF-CORE MODULES/SUBWORKFLOWS -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ // @@ -51,9 +51,9 @@ include { MULTIQC } from '../modules/nf-core/modules/multiqc include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/modules/custom/dumpsoftwareversions/main' /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ RUN MAIN WORKFLOW -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ // Info required for completion email and summary @@ -104,9 +104,9 @@ workflow {{ short_name|upper }} { } /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ COMPLETION EMAIL AND SUMMARY -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ workflow.onComplete { @@ -117,7 +117,7 @@ workflow.onComplete { } /* -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ THE END -======================================================================================== +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ From 266898bfd14c2f0688ab34b6f57d40c3ad568f1b Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Sat, 18 Dec 2021 09:53:19 +0100 Subject: [PATCH 028/186] Update changelog and straggler --- CHANGELOG.md | 2 ++ nf_core/pipeline-template/lib/NfcoreSchema.groovy | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4927241768..548996555d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +* Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode + ### General ### Modules diff --git a/nf_core/pipeline-template/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/lib/NfcoreSchema.groovy index 40ab65f205..b3d092f809 100755 --- a/nf_core/pipeline-template/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/lib/NfcoreSchema.groovy @@ -27,7 +27,7 @@ class NfcoreSchema { /* groovylint-disable-next-line UnusedPrivateMethodParameter */ public static void validateParameters(workflow, params, log, schema_filename='nextflow_schema.json') { def has_error = false - //=====================================================================// + //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// // Check for nextflow core params and unexpected params def json = new File(getSchemaPath(workflow, schema_filename=schema_filename)).text def Map schemaParams = (Map) new JsonSlurper().parseText(json).get('definitions') @@ -135,7 +135,7 @@ class NfcoreSchema { } } - //=====================================================================// + //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// // Validate parameters against the schema InputStream input_stream = new File(getSchemaPath(workflow, schema_filename=schema_filename)).newInputStream() JSONObject raw_schema = new JSONObject(new JSONTokener(input_stream)) From 092c96f258e2ca93381676d3b87c1407192a9071 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Sat, 18 Dec 2021 09:56:51 +0100 Subject: [PATCH 029/186] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 548996555d..62b3bae133 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Template -* Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode +* Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode. ### General From 04be6746a70a7c57b70a02b7b3ba6770eabfb4c7 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Sat, 18 Dec 2021 12:44:47 +0100 Subject: [PATCH 030/186] Missed one... --- nf_core/pipeline-template/conf/modules.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/conf/modules.config b/nf_core/pipeline-template/conf/modules.config index f51e020fe0..c59bfd8dae 100644 --- a/nf_core/pipeline-template/conf/modules.config +++ b/nf_core/pipeline-template/conf/modules.config @@ -1,7 +1,7 @@ /* -######################################################################################## +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Config file for defining DSL2 per module options and publishing paths -######################################################################################## +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Available keys to override module options: ext.args = Additional arguments appended to command in module. ext.args2 = Second set of arguments appended to command in module (multi-tool modules). From 024ef4949c255ac65d4d72b7d0f514e8c63f4a5c Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 23 Dec 2021 21:11:25 +0000 Subject: [PATCH 031/186] Fix bug when using stageAs in input in cat/fastq --- nf_core/modules/lint/main_nf.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 5b37b8bbda..76b8e08ede 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -209,12 +209,11 @@ def _parse_input(self, line): if "tuple" in line: line = line.replace("tuple", "") line = line.replace(" ", "") - line = line.split(",") - - for elem in line: - elem = elem.split("(")[1] - elem = elem.replace(")", "").strip() - input.append(elem) + for idx, elem in enumerate(line.split(')')): + if elem: + elem = elem.split('(')[1] + elem = elem.split(',')[0].strip() + input.append(elem) else: if "(" in line: input.append(line.split("(")[1].replace(")", "")) From 951164f2e97eb054c1168b8f9cf175fb61279e57 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 23 Dec 2021 21:19:34 +0000 Subject: [PATCH 032/186] Fix bug when using stageAs in input in cat/fastq --- nf_core/modules/lint/main_nf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 76b8e08ede..61ac6615b0 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -205,6 +205,7 @@ def check_process_section(self, lines): def _parse_input(self, line): input = [] + line = line.strip() # more than one input if "tuple" in line: line = line.replace("tuple", "") From 8bc3e16271777122d10d6e102cfd130175cf8013 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 23 Dec 2021 21:49:37 +0000 Subject: [PATCH 033/186] Fix bug in container regex --- nf_core/modules/lint/main_nf.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 61ac6615b0..1ac0e48770 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -163,11 +163,9 @@ def check_process_section(self, lines): if re.search("bioconda::", l): bioconda_packages = [b for b in l.split() if "bioconda::" in b] if re.search("org/singularity", l): - singularity_tag = ( - l.split()[0].split("/")[-1].split("/")[-1].replace('"', "").replace("'", "").split(":")[-1] - ) + singularity_tag = l.split(':')[-2].strip().replace('"', "").replace("'", "") if re.search("biocontainers", l): - docker_tag = l.split()[0].split("/")[-1].split("/")[-1].replace('"', "").replace("'", "").split(":")[-1] + docker_tag = l.split(':')[-1].split('}')[0].strip().replace('"', "").replace("'", "") # Check that all bioconda packages have build numbers # Also check for newer versions From 750c47f4b57972e10cba8e5454b7c184a46b4aeb Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 23 Dec 2021 22:54:27 +0000 Subject: [PATCH 034/186] Make Container versions do not match lint more flexible --- nf_core/modules/lint/main_nf.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 1ac0e48770..b8491af36e 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -160,12 +160,23 @@ def check_process_section(self, lines): self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf)) for l in lines: + l = l.strip() + l = l.replace('"','') + l = l.replace("'",'') if re.search("bioconda::", l): bioconda_packages = [b for b in l.split() if "bioconda::" in b] - if re.search("org/singularity", l): - singularity_tag = l.split(':')[-2].strip().replace('"', "").replace("'", "") - if re.search("biocontainers", l): - docker_tag = l.split(':')[-1].split('}')[0].strip().replace('"', "").replace("'", "") + if l.startswith("https://containers") or l.startswith('https://depot'): + lspl = l.lstrip('https://').split(':') + if len(lspl) == 2: + ## e.g. 'https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' : + singularity_tag = '_'.join(lspl[0].split('/')[-1].strip().rstrip('.img').split('_')[1:]) + else: + ## e.g. 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : + singularity_tag = lspl[-2].strip() + if l.startswith("biocontainers/") or l.startswith('quay.io/'): + ## e.g. 'quay.io/biocontainers/krona:2.7.1--pl526_5' }" + ## e.g. 'biocontainers/biocontainers:v1.2.0_cv1' }" + docker_tag = l.split(':')[-1].strip('}').strip() # Check that all bioconda packages have build numbers # Also check for newer versions From 934e57d5a09c10a83679edd5a59980379b6d66ae Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 23 Dec 2021 23:18:37 +0000 Subject: [PATCH 035/186] Allow for input: or output: not being defined in module --- nf_core/modules/lint/main_nf.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index b8491af36e..be4e7d4166 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -46,13 +46,13 @@ def main_nf(module_lint_object, module): for l in lines: if re.search("^\s*process\s*\w*\s*{", l) and state == "module": state = "process" - if re.search("input\s*:", l) and state == "process": + if re.search("input\s*:", l) and state in ["process"]: state = "input" continue - if re.search("output\s*:", l) and state == "input": + if re.search("output\s*:", l) and state in ["input", "process"]: state = "output" continue - if re.search("script\s*:", l) and state == "output": + if re.search("script\s*:", l) and state in ["input", "output", "process"]: state = "script" continue @@ -77,18 +77,21 @@ def main_nf(module_lint_object, module): check_script_section(module, script_lines) # Check whether 'meta' is emitted when given as input - if "meta" in inputs: - module.has_meta = True - if "meta" in outputs: - module.passed.append(("main_nf_meta_output", "'meta' map emitted in output channel(s)", module.main_nf)) - else: - module.failed.append(("main_nf_meta_output", "'meta' map not emitted in output channel(s)", module.main_nf)) + if inputs: + if "meta" in inputs: + module.has_meta = True + if outputs: + if "meta" in outputs: + module.passed.append(("main_nf_meta_output", "'meta' map emitted in output channel(s)", module.main_nf)) + else: + module.failed.append(("main_nf_meta_output", "'meta' map not emitted in output channel(s)", module.main_nf)) # Check that a software version is emitted - if "versions" in outputs: - module.passed.append(("main_nf_version_emitted", "Module emits software version", module.main_nf)) - else: - module.warned.append(("main_nf_version_emitted", "Module does not emit software version", module.main_nf)) + if outputs: + if "versions" in outputs: + module.passed.append(("main_nf_version_emitted", "Module emits software version", module.main_nf)) + else: + module.warned.append(("main_nf_version_emitted", "Module does not emit software version", module.main_nf)) return inputs, outputs From cde9eefe538061b9619a860a3de7a1f3891f0e0b Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 23 Dec 2021 23:42:13 +0000 Subject: [PATCH 036/186] Update CHANGELOG --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4136b6c116..98cd864357 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ * Fixed linting bugs where warning was incorrectly generated for: * `Module does not emit software version` * `Container versions do not match` + * `input:` / `output:` not being specified in module + * Allow for containers from other biocontainers resource as defined [here](https://github.com/nf-core/modules/blob/cde237e7cec07798e5754b72aeca44efe89fc6db/modules/cat/fastq/main.nf#L7-L8) +* Fixed traceback when using `stageAs` syntax as defined [here](https://github.com/nf-core/modules/blob/cde237e7cec07798e5754b72aeca44efe89fc6db/modules/cat/fastq/main.nf#L11) ## [v2.2 - Lead Liger](https://github.com/nf-core/tools/releases/tag/2.2) - [2021-12-14] From f1c1cd69a9e38c726fda234e796ad8324c949d1c Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 23 Dec 2021 23:44:56 +0000 Subject: [PATCH 037/186] Revert bump version --- CHANGELOG.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98cd864357..8d50565d51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## [v2.2.1 - Lead Ligerito](https://github.com/nf-core/tools/releases/tag/2.2.1) - [2021-12-17] +## v2.3dev ### Template diff --git a/setup.py b/setup.py index 8336daf6aa..f6b47282a4 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -version = "2.2.1" +version = "2.3dev" with open("README.md") as f: readme = f.read() From 8ca73d64094a822c4c08fecf098e535a23499815 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 23 Dec 2021 23:47:00 +0000 Subject: [PATCH 038/186] Black --- nf_core/modules/lint/main_nf.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index be4e7d4166..c11384ebaf 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -82,9 +82,13 @@ def main_nf(module_lint_object, module): module.has_meta = True if outputs: if "meta" in outputs: - module.passed.append(("main_nf_meta_output", "'meta' map emitted in output channel(s)", module.main_nf)) + module.passed.append( + ("main_nf_meta_output", "'meta' map emitted in output channel(s)", module.main_nf) + ) else: - module.failed.append(("main_nf_meta_output", "'meta' map not emitted in output channel(s)", module.main_nf)) + module.failed.append( + ("main_nf_meta_output", "'meta' map not emitted in output channel(s)", module.main_nf) + ) # Check that a software version is emitted if outputs: @@ -164,22 +168,22 @@ def check_process_section(self, lines): for l in lines: l = l.strip() - l = l.replace('"','') - l = l.replace("'",'') + l = l.replace('"', "") + l = l.replace("'", "") if re.search("bioconda::", l): bioconda_packages = [b for b in l.split() if "bioconda::" in b] - if l.startswith("https://containers") or l.startswith('https://depot'): - lspl = l.lstrip('https://').split(':') + if l.startswith("https://containers") or l.startswith("https://depot"): + lspl = l.lstrip("https://").split(":") if len(lspl) == 2: ## e.g. 'https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' : - singularity_tag = '_'.join(lspl[0].split('/')[-1].strip().rstrip('.img').split('_')[1:]) + singularity_tag = "_".join(lspl[0].split("/")[-1].strip().rstrip(".img").split("_")[1:]) else: ## e.g. 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : singularity_tag = lspl[-2].strip() - if l.startswith("biocontainers/") or l.startswith('quay.io/'): + if l.startswith("biocontainers/") or l.startswith("quay.io/"): ## e.g. 'quay.io/biocontainers/krona:2.7.1--pl526_5' }" ## e.g. 'biocontainers/biocontainers:v1.2.0_cv1' }" - docker_tag = l.split(':')[-1].strip('}').strip() + docker_tag = l.split(":")[-1].strip("}").strip() # Check that all bioconda packages have build numbers # Also check for newer versions @@ -222,10 +226,10 @@ def _parse_input(self, line): if "tuple" in line: line = line.replace("tuple", "") line = line.replace(" ", "") - for idx, elem in enumerate(line.split(')')): + for idx, elem in enumerate(line.split(")")): if elem: - elem = elem.split('(')[1] - elem = elem.split(',')[0].strip() + elem = elem.split("(")[1] + elem = elem.split(",")[0].strip() input.append(elem) else: if "(" in line: From 0bfa000e0bd0f8f2502c429ca6b8c064b9b0d0cb Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 30 Dec 2021 00:11:42 +0000 Subject: [PATCH 039/186] Add examples for input tuple parsing --- nf_core/modules/lint/main_nf.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index c11384ebaf..a3a35d84fc 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -175,14 +175,14 @@ def check_process_section(self, lines): if l.startswith("https://containers") or l.startswith("https://depot"): lspl = l.lstrip("https://").split(":") if len(lspl) == 2: - ## e.g. 'https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' : + # e.g. 'https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' : singularity_tag = "_".join(lspl[0].split("/")[-1].strip().rstrip(".img").split("_")[1:]) else: - ## e.g. 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : + # e.g. 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : singularity_tag = lspl[-2].strip() if l.startswith("biocontainers/") or l.startswith("quay.io/"): - ## e.g. 'quay.io/biocontainers/krona:2.7.1--pl526_5' }" - ## e.g. 'biocontainers/biocontainers:v1.2.0_cv1' }" + # e.g. 'quay.io/biocontainers/krona:2.7.1--pl526_5' }" + # e.g. 'biocontainers/biocontainers:v1.2.0_cv1' }" docker_tag = l.split(":")[-1].strip("}").strip() # Check that all bioconda packages have build numbers @@ -222,8 +222,10 @@ def check_process_section(self, lines): def _parse_input(self, line): input = [] line = line.strip() - # more than one input if "tuple" in line: + # If more than one elements in channel should work with both of: + # e.g. tuple val(meta), path(reads) + # e.g. tuple val(meta), path(reads, stageAs: "input*/*") line = line.replace("tuple", "") line = line.replace(" ", "") for idx, elem in enumerate(line.split(")")): From 9add8e62b1c0e48b0f819295acbb762778ff369d Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 30 Dec 2021 00:12:52 +0000 Subject: [PATCH 040/186] Black --- nf_core/modules/lint/main_nf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index a3a35d84fc..8e0fcc108c 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -224,8 +224,8 @@ def _parse_input(self, line): line = line.strip() if "tuple" in line: # If more than one elements in channel should work with both of: - # e.g. tuple val(meta), path(reads) - # e.g. tuple val(meta), path(reads, stageAs: "input*/*") + # e.g. tuple val(meta), path(reads) + # e.g. tuple val(meta), path(reads, stageAs: "input*/*") line = line.replace("tuple", "") line = line.replace(" ", "") for idx, elem in enumerate(line.split(")")): From d716aadb5b0b9a1dc3ea44294a1c431cd508017a Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Tue, 11 Jan 2022 14:37:57 -0500 Subject: [PATCH 041/186] implemented Markdown-formatted parameter doc generation --- nf_core/__main__.py | 20 ++++++++++++++++++++ nf_core/schema.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index a9500509a4..7795df3833 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -698,6 +698,26 @@ def lint(schema_path): sys.exit(1) +@schema.command(help_priority=4) +@click.argument("schema_path", type=click.Path(exists=True), required=True, metavar="") +@click.option("--markdown", type=str, metavar="", help="File to write documentation to in Markdown format") +@click.option("--columns", type=str, metavar="", help="Columns to include in the parameter tables", default="parameter,description,type,default,required,hidden") +def docs(schema_path, markdown, columns): + """ + Outputs parameter documentation for a pipeline schema. + """ + schema_obj = nf_core.schema.PipelineSchema() + try: + schema_obj.get_schema_path(schema_path) + schema_obj.load_schema() + try: + schema_obj.print_documentation(markdown, columns) + except AssertionError as e: + log.warning(e) + except AssertionError as e: + sys.exit(1) + + @nf_core_cli.command("bump-version", help_priority=9) @click.argument("new_version", required=True, metavar="") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", help="Pipeline directory. Defaults to CWD") diff --git a/nf_core/schema.py b/nf_core/schema.py index c3825b4ba6..6e2165ff57 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -416,6 +416,41 @@ def validate_schema_title_description(self, schema=None): desc_attr, self.schema["description"] ) + def print_documentation(self, markdown, columns): + """ + Prints documentation for the schema. + """ + if (markdown): + with open(markdown, 'w') as file: + self.print_documentation_markdown(file, columns) + print(f"Documentation written to {markdown}") + else: + self.print_documentation_markdown(sys.stdout, columns) + + def print_documentation_markdown(self, file, columns_csv): + """ + Prints documentation for the schema in Markdown format to the given file-like object. + """ + columns = columns_csv.split(',') + out = lambda s: print(s, end='', file=file) + out(f"# {self.schema['title']}\n") + out(f"{self.schema['description']}\n") + for d_key, definition in self.schema.get("definitions", {}).items(): + out(f"\n## {definition.get('title', {})}") + out(f"\n{definition.get('description', '')}") + out('\n\n') + out(''.join([f"| {column.title()} " for column in columns])) + out('\n') + out(''.join([f"|-----------" for columns in columns])) + out('\n') + for p_key, param in definition.get("properties", {}).items(): + for column in columns: + if (column == "parameter"): + out(f"| {p_key} ") + else: + out(f"| {param.get(column, '')} ") + out('\n') + def make_skeleton_schema(self): """Make a new pipeline schema from the template""" self.schema_from_scratch = True From b8e4be6d2a01bc53a971e2fcf8367f66287fb297 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Tue, 11 Jan 2022 14:58:35 -0500 Subject: [PATCH 042/186] added unit test for schema docs command --- nf_core/__main__.py | 2 +- nf_core/schema.py | 2 +- tests/test_schema.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 7795df3833..234d4a52d9 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -701,7 +701,7 @@ def lint(schema_path): @schema.command(help_priority=4) @click.argument("schema_path", type=click.Path(exists=True), required=True, metavar="") @click.option("--markdown", type=str, metavar="", help="File to write documentation to in Markdown format") -@click.option("--columns", type=str, metavar="", help="Columns to include in the parameter tables", default="parameter,description,type,default,required,hidden") +@click.option("--columns", type=str, metavar="", help="CSV list of columns to include in the parameter tables (parameter,description,type,default,required,hidden)", default="parameter,description,type,default,required,hidden") def docs(schema_path, markdown, columns): """ Outputs parameter documentation for a pipeline schema. diff --git a/nf_core/schema.py b/nf_core/schema.py index 6e2165ff57..7f974204e4 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -427,7 +427,7 @@ def print_documentation(self, markdown, columns): else: self.print_documentation_markdown(sys.stdout, columns) - def print_documentation_markdown(self, file, columns_csv): + def print_documentation_markdown(self, file, columns_csv='parameter,description,type,default,required,hidden'): """ Prints documentation for the schema in Markdown format to the given file-like object. """ diff --git a/tests/test_schema.py b/tests/test_schema.py index 3a060a516c..147eda110e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -95,6 +95,20 @@ def test_load_schema(self): self.schema_obj.schema_filename = self.template_schema self.schema_obj.load_schema() + def test_schema_docs(self): + """Try to generate Markdown docs for a schema from a file""" + self.schema_obj.schema_filename = self.template_schema + self.schema_obj.load_schema() + import io + f = io.StringIO('') + self.schema_obj.print_documentation_markdown(f) + docs = f.getvalue() + assert self.schema_obj.schema['title'] in docs + assert self.schema_obj.schema['description'] in docs + for d_key, definition in self.schema_obj.schema.get("definitions", {}).items(): + assert definition['title'] in docs + assert definition['description'] in docs + @with_temporary_file def test_save_schema(self, tmp_file): """Try to save a schema""" From 926e57948dfeb5b301852cedb9b0f9289c41dfea Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Tue, 11 Jan 2022 15:09:47 -0500 Subject: [PATCH 043/186] updated changelog with new schema docs command --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 561356b222..c3d04301c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ * `input:` / `output:` not being specified in module * Allow for containers from other biocontainers resource as defined [here](https://github.com/nf-core/modules/blob/cde237e7cec07798e5754b72aeca44efe89fc6db/modules/cat/fastq/main.nf#L7-L8) * Fixed traceback when using `stageAs` syntax as defined [here](https://github.com/nf-core/modules/blob/cde237e7cec07798e5754b72aeca44efe89fc6db/modules/cat/fastq/main.nf#L11) +* Added `nf-core schema docs` command to output pipline parameter documentation in Markdown format for inclusion in GitHub and other documentation systems ([#741](https://github.com/nf-core/tools/issues/741)) ## [v2.2 - Lead Liger](https://github.com/nf-core/tools/releases/tag/2.2) - [2021-12-14] From 4d37b26983080b326e8d2e013687fbf3ff84f7e0 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Tue, 11 Jan 2022 15:11:38 -0500 Subject: [PATCH 044/186] code formatted with black --- nf_core/__main__.py | 10 ++++++++-- nf_core/schema.py | 26 +++++++++++++------------- tests/test_schema.py | 13 +++++++------ 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 234d4a52d9..f04af53bb8 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -701,10 +701,16 @@ def lint(schema_path): @schema.command(help_priority=4) @click.argument("schema_path", type=click.Path(exists=True), required=True, metavar="") @click.option("--markdown", type=str, metavar="", help="File to write documentation to in Markdown format") -@click.option("--columns", type=str, metavar="", help="CSV list of columns to include in the parameter tables (parameter,description,type,default,required,hidden)", default="parameter,description,type,default,required,hidden") +@click.option( + "--columns", + type=str, + metavar="", + help="CSV list of columns to include in the parameter tables (parameter,description,type,default,required,hidden)", + default="parameter,description,type,default,required,hidden", +) def docs(schema_path, markdown, columns): """ - Outputs parameter documentation for a pipeline schema. + Outputs parameter documentation for a pipeline schema. """ schema_obj = nf_core.schema.PipelineSchema() try: diff --git a/nf_core/schema.py b/nf_core/schema.py index 7f974204e4..7434e94d9d 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -420,36 +420,36 @@ def print_documentation(self, markdown, columns): """ Prints documentation for the schema. """ - if (markdown): - with open(markdown, 'w') as file: + if markdown: + with open(markdown, "w") as file: self.print_documentation_markdown(file, columns) print(f"Documentation written to {markdown}") else: self.print_documentation_markdown(sys.stdout, columns) - - def print_documentation_markdown(self, file, columns_csv='parameter,description,type,default,required,hidden'): + + def print_documentation_markdown(self, file, columns_csv="parameter,description,type,default,required,hidden"): """ Prints documentation for the schema in Markdown format to the given file-like object. """ - columns = columns_csv.split(',') - out = lambda s: print(s, end='', file=file) + columns = columns_csv.split(",") + out = lambda s: print(s, end="", file=file) out(f"# {self.schema['title']}\n") out(f"{self.schema['description']}\n") for d_key, definition in self.schema.get("definitions", {}).items(): out(f"\n## {definition.get('title', {})}") out(f"\n{definition.get('description', '')}") - out('\n\n') - out(''.join([f"| {column.title()} " for column in columns])) - out('\n') - out(''.join([f"|-----------" for columns in columns])) - out('\n') + out("\n\n") + out("".join([f"| {column.title()} " for column in columns])) + out("\n") + out("".join([f"|-----------" for columns in columns])) + out("\n") for p_key, param in definition.get("properties", {}).items(): for column in columns: - if (column == "parameter"): + if column == "parameter": out(f"| {p_key} ") else: out(f"| {param.get(column, '')} ") - out('\n') + out("\n") def make_skeleton_schema(self): """Make a new pipeline schema from the template""" diff --git a/tests/test_schema.py b/tests/test_schema.py index 147eda110e..500a967c6d 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -100,15 +100,16 @@ def test_schema_docs(self): self.schema_obj.schema_filename = self.template_schema self.schema_obj.load_schema() import io - f = io.StringIO('') + + f = io.StringIO("") self.schema_obj.print_documentation_markdown(f) docs = f.getvalue() - assert self.schema_obj.schema['title'] in docs - assert self.schema_obj.schema['description'] in docs + assert self.schema_obj.schema["title"] in docs + assert self.schema_obj.schema["description"] in docs for d_key, definition in self.schema_obj.schema.get("definitions", {}).items(): - assert definition['title'] in docs - assert definition['description'] in docs - + assert definition["title"] in docs + assert definition["description"] in docs + @with_temporary_file def test_save_schema(self, tmp_file): """Try to save a schema""" From d3139391bf932658d90a65d0297dc42453bc4680 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Tue, 11 Jan 2022 15:34:15 -0500 Subject: [PATCH 045/186] modifications to pass markdownlint and improve formatting of schema docs --- nf_core/schema.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 7434e94d9d..a7f8178733 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -433,22 +433,24 @@ def print_documentation_markdown(self, file, columns_csv="parameter,description, """ columns = columns_csv.split(",") out = lambda s: print(s, end="", file=file) - out(f"# {self.schema['title']}\n") + out(f"# {self.schema['title']}\n\n") out(f"{self.schema['description']}\n") for d_key, definition in self.schema.get("definitions", {}).items(): out(f"\n## {definition.get('title', {})}") - out(f"\n{definition.get('description', '')}") + out(f"\n\n{definition.get('description', '')}") out("\n\n") - out("".join([f"| {column.title()} " for column in columns])) + out("".join([f"|{column.title()}" for column in columns])) out("\n") out("".join([f"|-----------" for columns in columns])) out("\n") for p_key, param in definition.get("properties", {}).items(): for column in columns: if column == "parameter": - out(f"| {p_key} ") + out(f"|`{p_key}`") + elif column == "type": + out(f"|`{param.get('type', '')}`") else: - out(f"| {param.get(column, '')} ") + out(f"|{param.get(column, '')}") out("\n") def make_skeleton_schema(self): From 70d84f83e1d1b31b900ac897d022b1161c50b61f Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Wed, 12 Jan 2022 09:37:21 +0000 Subject: [PATCH 046/186] Add retry strategy for AWS megatests after releasing nf-core/tower-action v2.2 --- CHANGELOG.md | 1 + nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 4 +++- nf_core/pipeline-template/.github/workflows/awstest.yml | 4 +++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 561356b222..7d0a950331 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * Removed mention of `--singularity_pull_docker_container` in pipeline `README.md` * Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode. +* Add retry strategy for AWS megatests after releasing [nf-core/tower-action v2.2](https://github.com/nf-core/tower-action/releases/tag/v2.2) ### Modules diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 8e0ab65b23..64e3d0a82f 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -31,4 +31,6 @@ jobs: "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ github.sha }}{% endraw %}" } profiles: test_full,aws_tower - pre_run_script: 'export NXF_VER=21.10.3' + nextflow_config: | + process.errorStrategy = 'retry' + process.maxRetries = 3 diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index eb14cced64..a4b0ec461a 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -25,4 +25,6 @@ jobs: "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-test-${{ github.sha }}{% endraw %}" } profiles: test,aws_tower - pre_run_script: 'export NXF_VER=21.10.3' + nextflow_config: | + process.errorStrategy = 'retry' + process.maxRetries = 3 From ac263725582054268b80f6541b87e26b726c9d1e Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Wed, 19 Jan 2022 11:37:33 +0100 Subject: [PATCH 047/186] Add Gitpod Dockerfile --- .gitpod.Dockerfile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .gitpod.Dockerfile diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile new file mode 100644 index 0000000000..53595fd73d --- /dev/null +++ b/.gitpod.Dockerfile @@ -0,0 +1,19 @@ +FROM gitpod/workspace-full + +USER root + +# Install Conda +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \ + rm Miniconda3-latest-Linux-x86_64.sh + +ENV PATH="/opt/conda/bin:$PATH" + +RUN chown -R gitpod:gitpod /opt/conda + +USER gitpod + +# Install nextflow, nf-core, Mamba, and pytest-workflow +RUN conda install nextflow nf-core pytest-workflow mamba -n base -c conda-forge -c bioconda && \ + nextflow self-update && \ + conda clean --all -f -y From 0eee68474cc2808db85942b2267763791154a519 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Wed, 19 Jan 2022 11:49:30 +0100 Subject: [PATCH 048/186] Add Gitpod yaml --- .gitpod.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .gitpod.yml diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 0000000000..1e9df7794e --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,18 @@ +image: + file: .gitpod.Dockerfile +github: + prebuilds: + # enable for the default branch (defaults to true) + master: true + # enable for all branches in this repo (defaults to false) + branches: false + # enable for pull requests coming from this repo (defaults to true) + pullRequests: true + # enable for pull requests coming from forks (defaults to false) + pullRequestsFromForks: false + # add a check to pull requests (defaults to true) + addCheck: prevent-merge-on-error + # add a "Review in Gitpod" button as a comment to pull requests (defaults to false) + addComment: false + # add a "Review in Gitpod" button to the pull request's description (defaults to false) + addBadge: false \ No newline at end of file From 585af68801e77898d6581d25aa3041d8e7173b4c Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Wed, 19 Jan 2022 15:00:47 +0100 Subject: [PATCH 049/186] Rename Dockerfile --- .gitpod.Dockerfile => nf_core/gitpod/gitpod.Dockerfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .gitpod.Dockerfile => nf_core/gitpod/gitpod.Dockerfile (100%) diff --git a/.gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile similarity index 100% rename from .gitpod.Dockerfile rename to nf_core/gitpod/gitpod.Dockerfile From ae1693dd263b5b6584c4b0b8c09f629d54fe48da Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Wed, 19 Jan 2022 15:02:41 +0100 Subject: [PATCH 050/186] Add gitpod to pipeline template --- nf_core/pipeline-template/.gitpod.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 nf_core/pipeline-template/.gitpod.yml diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml new file mode 100644 index 0000000000..ef0cf2c6ff --- /dev/null +++ b/nf_core/pipeline-template/.gitpod.yml @@ -0,0 +1 @@ +image: nf-core/gitpod From e487f416a0c862b6dd81f167b585d8efdab89640 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Wed, 19 Jan 2022 15:05:02 +0100 Subject: [PATCH 051/186] Add GH action to build Gitpod --- .github/workflows/build_gitpod.yml | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/build_gitpod.yml diff --git a/.github/workflows/build_gitpod.yml b/.github/workflows/build_gitpod.yml new file mode 100644 index 0000000000..c51b3643af --- /dev/null +++ b/.github/workflows/build_gitpod.yml @@ -0,0 +1,39 @@ +name: nf-core gitpod build and push +# Builds the docker image for gitpod and pushes it to DockerHub + +on: + merge: + branches: + - 'master' + paths: + - 'nf_core/gitpod/gitpod.Dockerfile' + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + # Only run for the nf-core repo, for releases and merged PRs + if: ${{ github.repository == 'nf-core/tools' }} + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: nfcore/gitpod + + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From 7407f0dfd3b50f061e6b2591dc046ecc4f3f5ef2 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Wed, 19 Jan 2022 15:05:13 +0100 Subject: [PATCH 052/186] Restore Dockerfile for testing purposes --- .gitpod.Dockerfile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .gitpod.Dockerfile diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile new file mode 100644 index 0000000000..53595fd73d --- /dev/null +++ b/.gitpod.Dockerfile @@ -0,0 +1,19 @@ +FROM gitpod/workspace-full + +USER root + +# Install Conda +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \ + rm Miniconda3-latest-Linux-x86_64.sh + +ENV PATH="/opt/conda/bin:$PATH" + +RUN chown -R gitpod:gitpod /opt/conda + +USER gitpod + +# Install nextflow, nf-core, Mamba, and pytest-workflow +RUN conda install nextflow nf-core pytest-workflow mamba -n base -c conda-forge -c bioconda && \ + nextflow self-update && \ + conda clean --all -f -y From 932c1993df9b98cb1e6b0fbc44709978a87e06bd Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Wed, 19 Jan 2022 15:10:22 +0100 Subject: [PATCH 053/186] Fix GH action --- .github/workflows/build_gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_gitpod.yml b/.github/workflows/build_gitpod.yml index c51b3643af..ff79193166 100644 --- a/.github/workflows/build_gitpod.yml +++ b/.github/workflows/build_gitpod.yml @@ -2,7 +2,7 @@ name: nf-core gitpod build and push # Builds the docker image for gitpod and pushes it to DockerHub on: - merge: + push: branches: - 'master' paths: From ec079a89448741349eb28da4d97509718e0329dd Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 19 Jan 2022 15:13:27 +0100 Subject: [PATCH 054/186] First attempt to make download work with new DSL2 syntax. Hopefully closes #1379. Needs testing. --- CHANGELOG.md | 4 +++ nf_core/download.py | 77 ++++++++++++++++++++++++++++----------------- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d0a950331..5c2b650d45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ * Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode. * Add retry strategy for AWS megatests after releasing [nf-core/tower-action v2.2](https://github.com/nf-core/tower-action/releases/tag/v2.2) +## General + +* Updated `nf-core download` to work with latest DSL2 syntax for containers ([#1379](https://github.com/nf-core/tools/issues/1379)) + ### Modules * Linting a pipeline now fails instead of warning if a local copy of a module does not match the remote ([#1313](https://github.com/nf-core/tools/issues/1313)) diff --git a/nf_core/download.py b/nf_core/download.py index 7f03459cf6..5ce75a8afb 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -416,10 +416,30 @@ def find_container_images(self): """Find container image names for workflow. Starts by using `nextflow config` to pull out any process.container - declarations. This works for DSL1. + declarations. This works for DSL1. It should return a simple string with resolved logic. Second, we look for DSL2 containers. These can't be found with `nextflow config` at the time of writing, so we scrape the pipeline files. + This returns raw source code that will likely need to be cleaned. + + If multiple containers are found, prioritise any prefixed with http for direct download. + + Example syntax: + + Early DSL2: + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0" + } else { + container "quay.io/biocontainers/fastqc:0.11.9--0" + } + + Later DSL2: + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : + 'quay.io/biocontainers/fastqc:0.11.9--0' }" + + DSL1 / Special case DSL2: + container "nfcore/cellranger:6.0.2" """ log.debug("Fetching container names for workflow") @@ -439,35 +459,36 @@ def find_container_images(self): if file.endswith(".nf"): with open(os.path.join(subdir, file), "r") as fh: # Look for any lines with `container = "xxx"` - matches = [] - for line in fh: - match = re.match(r"\s*container\s+[\"']([^\"']+)[\"']", line) - if match: - matches.append(match.group(1)) - - # If we have matches, save the first one that starts with http - for m in matches: - if m.startswith("http"): - containers_raw.append(m.strip('"').strip("'")) - break - # If we get here then we didn't call break - just save the first match - else: - if len(matches) > 0: - containers_raw.append(matches[0].strip('"').strip("'")) + this_container = None + contents = file.read() + matches = re.findall(r"container\s*\"([^\"]*)\"", contents, re.S) + if matches: + for match in matches: + # Look for a http download URL. + # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/3809435/713980 + url_regex = r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)" + url_match = re.match(url_regex, match.group(1)) + if url_match: + this_container = url_match.group(0) + break # Prioritise http, exit loop as soon as we find it + + # No https download, is it a simple docker URI? + else: + # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/39672069/713980 + docker_regex = r"^(?:(?=[^:\/]{1,253})(?!-)[a-zA-Z0-9-]{1,63}(? 1 else "")) From f3cfa775af4a3ceda7da12309eb3a8bc874db571 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 19 Jan 2022 15:31:37 +0100 Subject: [PATCH 055/186] Basic testing, make sure the code actually runs --- nf_core/download.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 5ce75a8afb..f45e452526 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -460,23 +460,23 @@ def find_container_images(self): with open(os.path.join(subdir, file), "r") as fh: # Look for any lines with `container = "xxx"` this_container = None - contents = file.read() + contents = fh.read() matches = re.findall(r"container\s*\"([^\"]*)\"", contents, re.S) if matches: for match in matches: # Look for a http download URL. # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/3809435/713980 url_regex = r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)" - url_match = re.match(url_regex, match.group(1)) + url_match = re.search(url_regex, match, re.S) if url_match: this_container = url_match.group(0) break # Prioritise http, exit loop as soon as we find it - # No https download, is it a simple docker URI? + # No https download, is the entire container string a docker URI? else: # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/39672069/713980 docker_regex = r"^(?:(?=[^:\/]{1,253})(?!-)[a-zA-Z0-9-]{1,63}(? Date: Thu, 20 Jan 2022 10:53:33 +0100 Subject: [PATCH 056/186] Update nf_core/pipeline-template/.gitpod.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipeline-template/.gitpod.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml index ef0cf2c6ff..6cee4e130a 100644 --- a/nf_core/pipeline-template/.gitpod.yml +++ b/nf_core/pipeline-template/.gitpod.yml @@ -1 +1,15 @@ image: nf-core/gitpod + +vscode: + extensions: # based on nf-core.nf-core-extensionpack + - codezombiech.gitignore + # - cssho.vscode-svgviewer + - davidanson.vscode-markdownlint + - eamodio.gitlens + - EditorConfig.EditorConfig + - Gruntfuggly.todo-tree + - mechatroner.rainbow-csv + # - nextflow.nextflow + - oderwat.indent-rainbow + - redhat.vscode-yaml + - streetsidesoftware.code-spell-checker From a7d33d90b858026e7945d906e6cc82391767d4ed Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 20 Jan 2022 11:23:22 +0100 Subject: [PATCH 057/186] Update .gitpod.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- .gitpod.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitpod.yml b/.gitpod.yml index 1e9df7794e..97a5d5a36b 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,9 @@ image: file: .gitpod.Dockerfile +task: + - name: install current state of nf-core/tools + init: | + pip install --upgrade -r requirements-dev.txt -e . github: prebuilds: # enable for the default branch (defaults to true) From 1f918f9717c3e301c928cb172cf877caa7cf53bd Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 20 Jan 2022 11:20:46 +0100 Subject: [PATCH 058/186] Add extension pack explanations --- nf_core/pipeline-template/.gitpod.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml index 6cee4e130a..a2fa269521 100644 --- a/nf_core/pipeline-template/.gitpod.yml +++ b/nf_core/pipeline-template/.gitpod.yml @@ -2,14 +2,14 @@ image: nf-core/gitpod vscode: extensions: # based on nf-core.nf-core-extensionpack - - codezombiech.gitignore - # - cssho.vscode-svgviewer - - davidanson.vscode-markdownlint - - eamodio.gitlens - - EditorConfig.EditorConfig - - Gruntfuggly.todo-tree - - mechatroner.rainbow-csv - # - nextflow.nextflow - - oderwat.indent-rainbow - - redhat.vscode-yaml - - streetsidesoftware.code-spell-checker + - codezombiech.gitignore # Language support for .gitignore files + # - cssho.vscode-svgviewer # SVG viewer + - davidanson.vscode-markdownlint # Markdown/CommonMark linting and style checking for Visual Studio Code + - eamodio.gitlens # Quickly glimpse into whom, why, and when a line or code block was changed + - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files + - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar + - mechatroner.rainbow-csv # Highlight columns in csv files in different colors + # - nextflow.nextflow # Nextflow syntax highlighting + - oderwat.indent-rainbow # Highlight indentation level + - redhat.vscode-yaml # YAML Language support + - streetsidesoftware.code-spell-checker # Spelling checker for source code From e60533af42721d04fe444d7ed1a0c2b46a8ac7d3 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 20 Jan 2022 14:22:12 +0100 Subject: [PATCH 059/186] Update base Gitpod yaml. --- .gitpod.Dockerfile | 19 ------------------- .gitpod.yml | 32 ++++++++++++++------------------ 2 files changed, 14 insertions(+), 37 deletions(-) delete mode 100644 .gitpod.Dockerfile diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile deleted file mode 100644 index 53595fd73d..0000000000 --- a/.gitpod.Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM gitpod/workspace-full - -USER root - -# Install Conda -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ - bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \ - rm Miniconda3-latest-Linux-x86_64.sh - -ENV PATH="/opt/conda/bin:$PATH" - -RUN chown -R gitpod:gitpod /opt/conda - -USER gitpod - -# Install nextflow, nf-core, Mamba, and pytest-workflow -RUN conda install nextflow nf-core pytest-workflow mamba -n base -c conda-forge -c bioconda && \ - nextflow self-update && \ - conda clean --all -f -y diff --git a/.gitpod.yml b/.gitpod.yml index 97a5d5a36b..cd9a238867 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,22 +1,18 @@ -image: - file: .gitpod.Dockerfile +image: nf-core/gitpod task: - name: install current state of nf-core/tools init: | pip install --upgrade -r requirements-dev.txt -e . -github: - prebuilds: - # enable for the default branch (defaults to true) - master: true - # enable for all branches in this repo (defaults to false) - branches: false - # enable for pull requests coming from this repo (defaults to true) - pullRequests: true - # enable for pull requests coming from forks (defaults to false) - pullRequestsFromForks: false - # add a check to pull requests (defaults to true) - addCheck: prevent-merge-on-error - # add a "Review in Gitpod" button as a comment to pull requests (defaults to false) - addComment: false - # add a "Review in Gitpod" button to the pull request's description (defaults to false) - addBadge: false \ No newline at end of file +vscode: + extensions: # based on nf-core.nf-core-extensionpack + - codezombiech.gitignore # Language support for .gitignore files + # - cssho.vscode-svgviewer # SVG viewer + - davidanson.vscode-markdownlint # Markdown/CommonMark linting and style checking for Visual Studio Code + - eamodio.gitlens # Quickly glimpse into whom, why, and when a line or code block was changed + - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files + - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar + - mechatroner.rainbow-csv # Highlight columns in csv files in different colors + # - nextflow.nextflow # Nextflow syntax highlighting + - oderwat.indent-rainbow # Highlight indentation level + - redhat.vscode-yaml # YAML Language support + - streetsidesoftware.code-spell-checker # Spelling checker for source code From 9227e86211924e391a4b8f901e597119ac96a5ff Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 20 Jan 2022 15:29:57 +0100 Subject: [PATCH 060/186] Update .github/workflows/build_gitpod.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- .github/workflows/build_gitpod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_gitpod.yml b/.github/workflows/build_gitpod.yml index ff79193166..bae98942fc 100644 --- a/.github/workflows/build_gitpod.yml +++ b/.github/workflows/build_gitpod.yml @@ -21,8 +21,8 @@ jobs: - name: Log in to Docker Hub uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASS }} - name: Extract metadata (tags, labels) for Docker id: meta From 6c9dd05044d3a0794de049d929b4906664f6c06a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 21 Jan 2022 10:31:47 +0100 Subject: [PATCH 061/186] Minor code refactor, add '--force' --- nf_core/__main__.py | 10 ++++--- nf_core/schema.py | 65 ++++++++++++++++++++++++--------------------- 2 files changed, 41 insertions(+), 34 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index f04af53bb8..75c77423b1 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -700,15 +700,19 @@ def lint(schema_path): @schema.command(help_priority=4) @click.argument("schema_path", type=click.Path(exists=True), required=True, metavar="") -@click.option("--markdown", type=str, metavar="", help="File to write documentation to in Markdown format") @click.option( + "-m", "--markdown", type=str, metavar="", help="File to write documentation to in Markdown format" +) +@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") +@click.option( + "-c", "--columns", type=str, metavar="", help="CSV list of columns to include in the parameter tables (parameter,description,type,default,required,hidden)", default="parameter,description,type,default,required,hidden", ) -def docs(schema_path, markdown, columns): +def docs(schema_path, markdown, force, columns): """ Outputs parameter documentation for a pipeline schema. """ @@ -717,7 +721,7 @@ def docs(schema_path, markdown, columns): schema_obj.get_schema_path(schema_path) schema_obj.load_schema() try: - schema_obj.print_documentation(markdown, columns) + schema_obj.print_documentation(markdown, force, columns.split(",")) except AssertionError as e: log.warning(e) except AssertionError as e: diff --git a/nf_core/schema.py b/nf_core/schema.py index a7f8178733..2f375cded8 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -10,14 +10,10 @@ import jsonschema import logging import os -import requests -import requests_cache import sys -import time import webbrowser import yaml import copy -import re import nf_core.list, nf_core.utils @@ -416,42 +412,49 @@ def validate_schema_title_description(self, schema=None): desc_attr, self.schema["description"] ) - def print_documentation(self, markdown, columns): + def print_documentation( + self, + markdown_fn=None, + force=False, + columns=["parameter", "description", "type,", "efault", "required", "hidden"], + ): """ Prints documentation for the schema. """ - if markdown: - with open(markdown, "w") as file: - self.print_documentation_markdown(file, columns) - print(f"Documentation written to {markdown}") + if markdown_fn: + if os.path.exists(markdown_fn) and not force: + log.error(f"Markdown file '{markdown_fn}' exists! Please delete first, or use '--force'") + return + with open(markdown_fn, "w") as file: + file.write(self.schema_to_markdown(columns)) + log.info(f"Documentation written to '{markdown_fn}'") else: - self.print_documentation_markdown(sys.stdout, columns) - - def print_documentation_markdown(self, file, columns_csv="parameter,description,type,default,required,hidden"): - """ - Prints documentation for the schema in Markdown format to the given file-like object. - """ - columns = columns_csv.split(",") - out = lambda s: print(s, end="", file=file) - out(f"# {self.schema['title']}\n\n") - out(f"{self.schema['description']}\n") - for d_key, definition in self.schema.get("definitions", {}).items(): - out(f"\n## {definition.get('title', {})}") - out(f"\n\n{definition.get('description', '')}") - out("\n\n") - out("".join([f"|{column.title()}" for column in columns])) - out("\n") - out("".join([f"|-----------" for columns in columns])) - out("\n") + print(self.schema_to_markdown(columns)) + + def schema_to_markdown(self, columns): + """ + Creates documentation for the schema in Markdown format. + """ + out = f"# {self.schema['title']}\n\n" + out += f"{self.schema['description']}\n" + for definition in self.schema.get("definitions", {}).values(): + out += f"\n## {definition.get('title', {})}" + out += f"\n\n{definition.get('description', '')}" + out += "\n\n" + out += "".join([f"|{column.title()}" for column in columns]) + out += "\n" + out += "".join([f"|-----------" for columns in columns]) + out += "\n" for p_key, param in definition.get("properties", {}).items(): for column in columns: if column == "parameter": - out(f"|`{p_key}`") + out += f"|`{p_key}`" elif column == "type": - out(f"|`{param.get('type', '')}`") + out += f"|`{param.get('type', '')}`" else: - out(f"|{param.get(column, '')}") - out("\n") + out += f"|{param.get(column, '')}" + out += "\n" + return out def make_skeleton_schema(self): """Make a new pipeline schema from the template""" From 5aea4428505e1a894afb9b8e860a3235c325731c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 21 Jan 2022 10:33:01 +0100 Subject: [PATCH 062/186] Remove unused try/except --- nf_core/__main__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 75c77423b1..f581f9d1ca 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -720,11 +720,9 @@ def docs(schema_path, markdown, force, columns): try: schema_obj.get_schema_path(schema_path) schema_obj.load_schema() - try: - schema_obj.print_documentation(markdown, force, columns.split(",")) - except AssertionError as e: - log.warning(e) + schema_obj.print_documentation(markdown, force, columns.split(",")) except AssertionError as e: + log.error(e) sys.exit(1) From 1f28973968a59aae8e7068d8642ab54bf31c331b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 21 Jan 2022 11:01:56 +0100 Subject: [PATCH 063/186] Add option to generate HTML instead of markdown --- nf_core/__main__.py | 7 +++--- nf_core/schema.py | 53 ++++++++++++++++++++++++++++----------------- requirements.txt | 1 + 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index f581f9d1ca..74d73480e0 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -700,8 +700,9 @@ def lint(schema_path): @schema.command(help_priority=4) @click.argument("schema_path", type=click.Path(exists=True), required=True, metavar="") +@click.option("-o", "--output", type=str, metavar="", help="Output filename. Defaults to standard out.") @click.option( - "-m", "--markdown", type=str, metavar="", help="File to write documentation to in Markdown format" + "-x", "--format", type=click.Choice(["markdown", "html"]), default="markdown", help="Format to output docs in." ) @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") @click.option( @@ -712,7 +713,7 @@ def lint(schema_path): help="CSV list of columns to include in the parameter tables (parameter,description,type,default,required,hidden)", default="parameter,description,type,default,required,hidden", ) -def docs(schema_path, markdown, force, columns): +def docs(schema_path, output, format, force, columns): """ Outputs parameter documentation for a pipeline schema. """ @@ -720,7 +721,7 @@ def docs(schema_path, markdown, force, columns): try: schema_obj.get_schema_path(schema_path) schema_obj.load_schema() - schema_obj.print_documentation(markdown, force, columns.split(",")) + schema_obj.print_documentation(output, format, force, columns.split(",")) except AssertionError as e: log.error(e) sys.exit(1) diff --git a/nf_core/schema.py b/nf_core/schema.py index 2f375cded8..29314cac3c 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -4,16 +4,16 @@ from __future__ import print_function from rich.prompt import Confirm +import copy import copy import jinja2 import json import jsonschema import logging +import markdown import os -import sys import webbrowser import yaml -import copy import nf_core.list, nf_core.utils @@ -414,22 +414,30 @@ def validate_schema_title_description(self, schema=None): def print_documentation( self, - markdown_fn=None, + output_fn=None, + format="markdown", force=False, columns=["parameter", "description", "type,", "efault", "required", "hidden"], ): """ Prints documentation for the schema. """ - if markdown_fn: - if os.path.exists(markdown_fn) and not force: - log.error(f"Markdown file '{markdown_fn}' exists! Please delete first, or use '--force'") + output = self.schema_to_markdown(columns) + if format == "html": + output = self.markdown_to_html(output) + + # Print to file + if output_fn: + if os.path.exists(output_fn) and not force: + log.error(f"File '{output_fn}' exists! Please delete first, or use '--force'") return - with open(markdown_fn, "w") as file: - file.write(self.schema_to_markdown(columns)) - log.info(f"Documentation written to '{markdown_fn}'") + with open(output_fn, "w") as file: + file.write(output) + log.info(f"Documentation written to '{output_fn}'") + + # Print to stdout else: - print(self.schema_to_markdown(columns)) + print(output) def schema_to_markdown(self, columns): """ @@ -438,24 +446,29 @@ def schema_to_markdown(self, columns): out = f"# {self.schema['title']}\n\n" out += f"{self.schema['description']}\n" for definition in self.schema.get("definitions", {}).values(): - out += f"\n## {definition.get('title', {})}" - out += f"\n\n{definition.get('description', '')}" - out += "\n\n" - out += "".join([f"|{column.title()}" for column in columns]) - out += "\n" + out += f"\n## {definition.get('title', {})}\n\n" + out += f"{definition.get('description', '')}\n\n" + out += "".join([f"| {column.title()} " for column in columns]) + out += "|\n" out += "".join([f"|-----------" for columns in columns]) - out += "\n" + out += "|\n" for p_key, param in definition.get("properties", {}).items(): for column in columns: if column == "parameter": - out += f"|`{p_key}`" + out += f"| `{p_key}` " elif column == "type": - out += f"|`{param.get('type', '')}`" + out += f"| `{param.get('type', '')}` " else: - out += f"|{param.get(column, '')}" - out += "\n" + out += f"| {param.get(column, '')} " + out += "|\n" return out + def markdown_to_html(self, markdown_str): + """ + Convert markdown to html + """ + return markdown.markdown(markdown_str, extensions=["tables"]) + def make_skeleton_schema(self): """Make a new pipeline schema from the template""" self.schema_from_scratch = True diff --git a/requirements.txt b/requirements.txt index b6e20629d4..8bf74567ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ click GitPython jinja2 jsonschema>=3.0 +markdown>=3.3 packaging prompt_toolkit>=3.0.3 pyyaml From 4c29edae5f5d3ef1ffdf518b8c6d57bb8d08e93b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 21 Jan 2022 11:13:44 +0100 Subject: [PATCH 064/186] Add help_text in a
tag --- nf_core/schema.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 29314cac3c..ab0df5d8ed 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -417,7 +417,7 @@ def print_documentation( output_fn=None, format="markdown", force=False, - columns=["parameter", "description", "type,", "efault", "required", "hidden"], + columns=["parameter", "description", "type,", "default", "required", "hidden"], ): """ Prints documentation for the schema. @@ -456,6 +456,10 @@ def schema_to_markdown(self, columns): for column in columns: if column == "parameter": out += f"| `{p_key}` " + elif column == "description": + out += f"| {param.get('description', '')} " + if param.get("help_text", "") != "": + out += f"
Help{param['help_text']}
" elif column == "type": out += f"| `{param.get('type', '')}` " else: From 4de4730b5642ee2b33030e4c5ba596604b45c121 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 21 Jan 2022 13:04:22 +0100 Subject: [PATCH 065/186] Add conda update to Dockerfile --- nf_core/gitpod/gitpod.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 53595fd73d..f0f76e3394 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -14,6 +14,7 @@ RUN chown -R gitpod:gitpod /opt/conda USER gitpod # Install nextflow, nf-core, Mamba, and pytest-workflow -RUN conda install nextflow nf-core pytest-workflow mamba -n base -c conda-forge -c bioconda && \ +RUN conda update -n base -c defaults conda && \ + conda install nextflow nf-core pytest-workflow mamba -n base -c conda-forge -c bioconda && \ nextflow self-update && \ conda clean --all -f -y From 2d601491a52476908992b1b573abf8282c0c8d31 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 21 Jan 2022 13:05:15 +0100 Subject: [PATCH 066/186] Add file to build --- .github/workflows/build_gitpod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_gitpod.yml b/.github/workflows/build_gitpod.yml index bae98942fc..3c6b210b0c 100644 --- a/.github/workflows/build_gitpod.yml +++ b/.github/workflows/build_gitpod.yml @@ -34,6 +34,7 @@ jobs: uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc with: context: . + file: nf_core/gitpod/gitpod.Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From b2230a72b66cf052fcf294d8e7d6215a6259c44a Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 21 Jan 2022 13:15:09 +0100 Subject: [PATCH 067/186] Change SHA commit to major versions --- .github/workflows/build_gitpod.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_gitpod.yml b/.github/workflows/build_gitpod.yml index 3c6b210b0c..ef73c88557 100644 --- a/.github/workflows/build_gitpod.yml +++ b/.github/workflows/build_gitpod.yml @@ -19,19 +19,19 @@ jobs: uses: actions/checkout@v2 - name: Log in to Docker Hub - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASS }} - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + uses: docker/metadata-action@v3 with: images: nfcore/gitpod - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + uses: docker/build-push-action@v2 with: context: . file: nf_core/gitpod/gitpod.Dockerfile From fa9cb246fa21e2a45daa697133c1831f0df018b1 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 21 Jan 2022 13:18:45 +0100 Subject: [PATCH 068/186] Fix image name --- .gitpod.yml | 2 +- nf_core/pipeline-template/.gitpod.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index cd9a238867..f60c0c5bbc 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nf-core/gitpod +image: nfcore/gitpod task: - name: install current state of nf-core/tools init: | diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml index a2fa269521..179170568a 100644 --- a/nf_core/pipeline-template/.gitpod.yml +++ b/nf_core/pipeline-template/.gitpod.yml @@ -1,4 +1,4 @@ -image: nf-core/gitpod +image: nfcore/gitpod vscode: extensions: # based on nf-core.nf-core-extensionpack From 12665d03e1424e7ee6713258549099b3d515ffd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Fri, 21 Jan 2022 13:28:48 +0100 Subject: [PATCH 069/186] fix typo in .gitpod.yml --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index f60c0c5bbc..1cb402fc3d 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,5 +1,5 @@ image: nfcore/gitpod -task: +tasks: - name: install current state of nf-core/tools init: | pip install --upgrade -r requirements-dev.txt -e . From 9d915bee0083f527fcab0199d6646bcc2d532f06 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 21 Jan 2022 16:31:09 +0100 Subject: [PATCH 070/186] Add Build path for nf-core tools dev --- .github/workflows/build_gitpod.yml | 3 +++ .gitpod.yml | 8 +++++--- nf_core/pipeline-template/.gitpod.yml | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_gitpod.yml b/.github/workflows/build_gitpod.yml index ef73c88557..c9858db1d4 100644 --- a/.github/workflows/build_gitpod.yml +++ b/.github/workflows/build_gitpod.yml @@ -29,6 +29,9 @@ jobs: uses: docker/metadata-action@v3 with: images: nfcore/gitpod + tags: | + type=semver,pattern={{version}} + type=raw,value=latest - name: Build and push Docker image uses: docker/build-push-action@v2 diff --git a/.gitpod.yml b/.gitpod.yml index 1cb402fc3d..79a7031ae6 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,8 +1,10 @@ -image: nfcore/gitpod +image: nfcore/gitpod:latest tasks: - name: install current state of nf-core/tools - init: | - pip install --upgrade -r requirements-dev.txt -e . + command: | + NFCORE_BUILD_PATH=$HOME/nf-core_build + pip install --upgrade -r requirements-dev.txt -e . -t $NFCORE_BUILD_PATH + PATH="$NFCORE_BUILD_PATH:$PATH" vscode: extensions: # based on nf-core.nf-core-extensionpack - codezombiech.gitignore # Language support for .gitignore files diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml index 179170568a..b6c480db07 100644 --- a/nf_core/pipeline-template/.gitpod.yml +++ b/nf_core/pipeline-template/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod +image: nfcore/gitpod:latest vscode: extensions: # based on nf-core.nf-core-extensionpack From 2d01159bc38accca38be31a72c56cb70b76d055a Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 21 Jan 2022 21:23:55 +0100 Subject: [PATCH 071/186] Add GA action to workflow check --- .github/workflows/build_gitpod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_gitpod.yml b/.github/workflows/build_gitpod.yml index c9858db1d4..da926a310e 100644 --- a/.github/workflows/build_gitpod.yml +++ b/.github/workflows/build_gitpod.yml @@ -7,6 +7,7 @@ on: - 'master' paths: - 'nf_core/gitpod/gitpod.Dockerfile' + - '.github/workflows/build_gitpod.yml' jobs: push_to_registry: From f14c2f8f6a04e73e6b483e4806d27b946f038e4c Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Mon, 31 Jan 2022 09:56:31 +0100 Subject: [PATCH 072/186] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- .gitpod.yml | 1 - nf_core/pipeline-template/.gitpod.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 79a7031ae6..ea900790b9 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -16,5 +16,4 @@ vscode: - mechatroner.rainbow-csv # Highlight columns in csv files in different colors # - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - - redhat.vscode-yaml # YAML Language support - streetsidesoftware.code-spell-checker # Spelling checker for source code diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml index b6c480db07..b7d4cee18b 100644 --- a/nf_core/pipeline-template/.gitpod.yml +++ b/nf_core/pipeline-template/.gitpod.yml @@ -11,5 +11,4 @@ vscode: - mechatroner.rainbow-csv # Highlight columns in csv files in different colors # - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - - redhat.vscode-yaml # YAML Language support - streetsidesoftware.code-spell-checker # Spelling checker for source code From bc04c64dd0f39bf23cf31a2f62a56f619d6ad263 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Mon, 31 Jan 2022 10:00:21 +0100 Subject: [PATCH 073/186] Add conda channel defaults --- nf_core/gitpod/gitpod.Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index f0f76e3394..598665ce4d 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -17,4 +17,7 @@ USER gitpod RUN conda update -n base -c defaults conda && \ conda install nextflow nf-core pytest-workflow mamba -n base -c conda-forge -c bioconda && \ nextflow self-update && \ + conda config --add channels defaults && \ + conda config --add channels bioconda && \ + conda config --add channels conda-forge && \ conda clean --all -f -y From 70fc9c4698ceb87924e88e2f12a92b484ce2a661 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Mon, 31 Jan 2022 11:09:13 +0100 Subject: [PATCH 074/186] Add pip to conda environment --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 598665ce4d..d5a17cd33c 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -15,7 +15,7 @@ USER gitpod # Install nextflow, nf-core, Mamba, and pytest-workflow RUN conda update -n base -c defaults conda && \ - conda install nextflow nf-core pytest-workflow mamba -n base -c conda-forge -c bioconda && \ + conda install nextflow nf-core pytest-workflow mamba pip -n base -c conda-forge -c bioconda && \ nextflow self-update && \ conda config --add channels defaults && \ conda config --add channels bioconda && \ From 9e51064fe12b296e7473bd834ef30392d0969fe3 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Mon, 31 Jan 2022 11:10:06 +0100 Subject: [PATCH 075/186] Update pip install command --- .gitpod.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index ea900790b9..86ed680600 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -2,9 +2,7 @@ image: nfcore/gitpod:latest tasks: - name: install current state of nf-core/tools command: | - NFCORE_BUILD_PATH=$HOME/nf-core_build - pip install --upgrade -r requirements-dev.txt -e . -t $NFCORE_BUILD_PATH - PATH="$NFCORE_BUILD_PATH:$PATH" + python -m pip install --upgrade -r requirements-dev.txt -e . vscode: extensions: # based on nf-core.nf-core-extensionpack - codezombiech.gitignore # Language support for .gitignore files From e36d7291ad2f5dced6d5037b06e5dd44b506c11d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 13:33:57 +0100 Subject: [PATCH 076/186] modules list local - handle DSL1 pipelines --- nf_core/modules/module_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index dd9afc9582..4e3a0c5ffe 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -131,6 +131,9 @@ def create_modules_json(pipeline_dir): modules_json = {"name": pipeline_name.strip("'"), "homePage": pipeline_url.strip("'"), "repos": dict()} modules_dir = f"{pipeline_dir}/modules" + if not os.path.exists(modules_dir): + raise UserWarning(f"Can't find a ./modules directory. Is this a DSL2 pipeline?") + # Extract all modules repos in the pipeline directory repo_names = [ f"{user_name}/{repo_name}" From c8e48be1602e12d9443e2197527660bdbb54f5bf Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 14:00:34 +0100 Subject: [PATCH 077/186] Rewrite modules create repo type check. * Use .nf-core.yml file if we can find it, with 'repository_type' key * Traverse parent directories to look for this file, reset target directory if found * Add flag to specify if no .nf-core.yml file available --- nf_core/__main__.py | 5 ++- nf_core/modules/bump_versions.py | 2 +- nf_core/modules/create.py | 57 ++++++++++++++++++-------- nf_core/pipeline-template/.nf-core.yml | 1 + 4 files changed, 44 insertions(+), 21 deletions(-) create mode 100644 nf_core/pipeline-template/.nf-core.yml diff --git a/nf_core/__main__.py b/nf_core/__main__.py index a9500509a4..398495e000 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -498,7 +498,8 @@ def remove(ctx, dir, tool): @click.option("-n", "--no-meta", is_flag=True, default=False, help="Don't use meta map for sample information") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") @click.option("-c", "--conda-name", type=str, default=None, help="Name of the conda package to use") -def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name): +@click.option("-r", "--repo-type", type=click.Choice(["pipeline", "modules"]), default=None, help="Type of repository") +def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name, repo_type): """ Create a new DSL2 module from the nf-core template. @@ -519,7 +520,7 @@ def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_nam # Run function try: - module_create = nf_core.modules.ModuleCreate(dir, tool, author, label, has_meta, force, conda_name) + module_create = nf_core.modules.ModuleCreate(dir, tool, author, label, has_meta, force, conda_name, repo_type) module_create.create() except UserWarning as e: log.critical(e) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 4d24faa52b..f7e0a595c3 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -64,7 +64,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): # Get list of all modules _, nfcore_modules = nf_core.modules.module_utils.get_installed_modules(self.dir) - # Load the .nf-core-tools.config + # Load the .nf-core.yml config self.tools_config = nf_core.utils.load_tools_config(self.dir) # Prompt for module or all diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index a9c071c4ce..8bbbf41440 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -25,7 +25,15 @@ class ModuleCreate(object): def __init__( - self, directory=".", tool="", author=None, process_label=None, has_meta=None, force=False, conda_name=None + self, + directory=".", + tool="", + author=None, + process_label=None, + has_meta=None, + force=False, + conda_name=None, + repo_type=None, ): self.directory = directory self.tool = tool @@ -36,7 +44,7 @@ def __init__( self.subtool = None self.tool_conda_name = conda_name self.tool_licence = None - self.repo_type = None + self.repo_type = repo_type self.tool_licence = "" self.tool_description = "" self.tool_doc_url = "" @@ -75,9 +83,12 @@ def create(self): # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules try: - self.repo_type = self.get_repo_type(self.directory) + self.get_repo_type() except LookupError as e: raise UserWarning(e) + log.info(f"Repository type: [blue]{self.repo_type}") + if self.directory != ".": + log.info(f"Base directory: '{self.directory}'") log.info( "[yellow]Press enter to use default values [cyan bold](shown in brackets)[/] [yellow]or type your own responses. " @@ -272,27 +283,37 @@ def render_template(self): template_stat = os.stat(os.path.join(os.path.dirname(nf_core.__file__), "module-template", template_fn)) os.chmod(dest_fn, template_stat.st_mode) - def get_repo_type(self, directory): + def get_repo_type(self): """ Determine whether this is a pipeline repository or a clone of nf-core/modules """ # Verify that the pipeline dir exists - if dir is None or not os.path.exists(directory): - raise UserWarning(f"Could not find directory: {directory}") - - readme = os.path.join(directory, "README.md") - # Determine repository type - if os.path.exists(readme): - with open(readme) as fh: - if fh.readline().rstrip().startswith("# ![nf-core/modules]"): - return "modules" - else: - return "pipeline" - else: + if dir is None or not os.path.exists(self.directory): + raise UserWarning(f"Could not find directory: {self.directory}") + + # Try to find the root directory + base_dir = os.path.abspath(self.directory) + config_path = os.path.join(base_dir, ".nf-core.yml") + while not os.path.exists(config_path) and base_dir != os.path.dirname(base_dir): + base_dir = os.path.dirname(base_dir) + config_path = os.path.join(base_dir, ".nf-core.yml") + # Reset self.directory if we found the config file (will be an absolute path) + if os.path.exists(config_path): + self.directory = base_dir + + # Figure out the repository type from the .nf-core.yml config file if we can + if os.path.exists(config_path): + tools_config = nf_core.utils.load_tools_config(self.directory) + if tools_config.get("repository_type") in ["pipeline", "modules"]: + self.repo_type = tools_config["repository_type"] + return + + # Could be set on the command line - throw an error if not + if not self.repo_type: raise UserWarning( - f"This directory does not look like a clone of nf-core/modules or an nf-core pipeline: '{directory}'" - " Please point to a valid directory." + f"Can't find a '.nf-core.yml' file with 'repository_type' set to 'pipeline' or 'modules': '{self.directory}'" + "\nPlease use the '--repo-type' flag or create '.nf-core.yml'" ) def get_module_dirs(self): diff --git a/nf_core/pipeline-template/.nf-core.yml b/nf_core/pipeline-template/.nf-core.yml new file mode 100644 index 0000000000..3805dc81c1 --- /dev/null +++ b/nf_core/pipeline-template/.nf-core.yml @@ -0,0 +1 @@ +repository_type: pipeline From 42795ec61f7f68a8c144c0e46e09aa3f82ad28d5 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 14:38:15 +0100 Subject: [PATCH 078/186] .yml or .yaml, check cli flag doesn't conflict with .nf-core.yml --- nf_core/modules/create.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 8bbbf41440..72b106844f 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -294,20 +294,29 @@ def get_repo_type(self): # Try to find the root directory base_dir = os.path.abspath(self.directory) - config_path = os.path.join(base_dir, ".nf-core.yml") - while not os.path.exists(config_path) and base_dir != os.path.dirname(base_dir): + config_path_yml = os.path.join(base_dir, ".nf-core.yml") + config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") + while ( + not os.path.exists(config_path_yml) + and not os.path.exists(config_path_yaml) + and base_dir != os.path.dirname(base_dir) + ): base_dir = os.path.dirname(base_dir) - config_path = os.path.join(base_dir, ".nf-core.yml") + config_path_yml = os.path.join(base_dir, ".nf-core.yml") + config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") # Reset self.directory if we found the config file (will be an absolute path) - if os.path.exists(config_path): + if os.path.exists(config_path_yml) or os.path.exists(config_path_yaml): self.directory = base_dir # Figure out the repository type from the .nf-core.yml config file if we can - if os.path.exists(config_path): - tools_config = nf_core.utils.load_tools_config(self.directory) - if tools_config.get("repository_type") in ["pipeline", "modules"]: - self.repo_type = tools_config["repository_type"] - return + tools_config = nf_core.utils.load_tools_config(self.directory) + if tools_config.get("repository_type") in ["pipeline", "modules"]: + if self.repo_type is not None and self.repo_type != tools_config["repository_type"]: + raise UserWarning( + f"'--repo-type {self.repo_type}' conflicts with [i][red]repository_type[/]: [blue]pipeline[/][/] in '{os.path.relpath(config_path_yml)}'" + ) + self.repo_type = tools_config["repository_type"] + return # Could be set on the command line - throw an error if not if not self.repo_type: From f527df628cfd8ee8b1081392bfb74d321d8abac5 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 14:45:22 +0100 Subject: [PATCH 079/186] Changelog and docs --- CHANGELOG.md | 2 ++ README.md | 14 ++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c2b650d45..86ae6db51c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,12 @@ * Removed mention of `--singularity_pull_docker_container` in pipeline `README.md` * Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode. * Add retry strategy for AWS megatests after releasing [nf-core/tower-action v2.2](https://github.com/nf-core/tower-action/releases/tag/v2.2) +* Added `.nf-core.yml` file with `repository_type: pipeline` for modules commands ## General * Updated `nf-core download` to work with latest DSL2 syntax for containers ([#1379](https://github.com/nf-core/tools/issues/1379)) +* Made `nf-core modules create` detect repository type with explicit `.nf-core.yml` or `--repo-type`, instead of random readme stuff ([#1391](https://github.com/nf-core/tools/pull/1391)) ### Modules diff --git a/README.md b/README.md index 65fb66b004..404f990eba 100644 --- a/README.md +++ b/README.md @@ -925,7 +925,7 @@ github.com: git_protocol: ``` -The easiest way to create this configuration file is through *GitHub CLI*: follow +The easiest way to create this configuration file is through _GitHub CLI_: follow its [installation instructions](https://cli.github.com/manual/installation) and then call: @@ -1116,11 +1116,17 @@ This command creates a new nf-core module from the nf-core module template. This ensures that your module follows the nf-core guidelines. The template contains extensive `TODO` messages to walk you through the changes you need to make to the template. -You can create a new module using `nf-core modules create`. This will create the new module in the current working directory. To specify another directory, use `--dir `. +You can create a new module using `nf-core modules create`. -If writing a module for the shared [nf-core/modules](https://github.com/nf-core/modules) repository, the `` argument should be the path to the clone of your fork of the modules repository. +This command can be used both when writing a module for the shared [nf-core/modules](https://github.com/nf-core/modules) repository, +and also when creating local modules for a pipeline. -Alternatively, if writing a more niche module that does not make sense to share, `` should be the path to your pipeline. +Which type of repository you are working in is detected by the `repository_type` flag in a `.nf-core.yml` file in the root directory, +set to either `pipeline` or `modules`. +The command will automatically look through parent directories for this file to set the root path, so that you can run the command in a subdirectory. +It will start in the current working directory, or whatever is specified with `--dir `. + +If you do not have a `.nf-core.yml` file you can specify the repository type with the `--repo-type` flag. The `nf-core modules create` command will prompt you with the relevant questions in order to create all of the necessary module files. From 5141c558d13d7410045ca05d81347ea4cf85adf4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 14:54:49 +0100 Subject: [PATCH 080/186] Fix tests, hopefully --- tests/test_modules.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index fb59537c62..ce95c7543b 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -19,8 +19,8 @@ def create_modules_repo_dummy(tmp_dir): os.makedirs(os.path.join(root_dir, "tests", "config")) with open(os.path.join(root_dir, "tests", "config", "pytest_modules.yml"), "w") as fh: fh.writelines(["test:", "\n - modules/test/**", "\n - tests/modules/test/**"]) - with open(os.path.join(root_dir, "README.md"), "w") as fh: - fh.writelines(["# ![nf-core/modules](docs/images/nfcore-modules_logo.png)", "\n"]) + with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: + fh.writelines(["repository_type: modules", "\n"]) # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_medium", False, False) @@ -47,15 +47,10 @@ def setUp(self): self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) self.mods_install_alt = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=True, force=True) - # TODO Remove comments once external repository to have same structure as nf-core/modules - # self.mods_install_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master") - # Set up remove objects print("Setting up remove objects") self.mods_remove = nf_core.modules.ModuleRemove(self.pipeline_dir) - self.mods_remove_alt = nf_core.modules.ModuleRemove(self.pipeline_dir) - # TODO Remove comments once external repository to have same structure as nf-core/modules - # self.mods_remove_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master") + self.mods_remove_alt = nf_core.modules.ModuleRemove(self.pipeline_dir # Set up the nf-core/modules repo dummy self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) From fe45705d8e658ac811862c06f626a46265c17512 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 14:56:46 +0100 Subject: [PATCH 081/186] Fix syntax error typo --- tests/test_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index ce95c7543b..cfa3408e69 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -50,7 +50,7 @@ def setUp(self): # Set up remove objects print("Setting up remove objects") self.mods_remove = nf_core.modules.ModuleRemove(self.pipeline_dir) - self.mods_remove_alt = nf_core.modules.ModuleRemove(self.pipeline_dir + self.mods_remove_alt = nf_core.modules.ModuleRemove(self.pipeline_dir) # Set up the nf-core/modules repo dummy self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) From 1d1f5e8d3131d41425ab8ef4e0532f00a53f4c46 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Tue, 1 Feb 2022 15:13:11 +0100 Subject: [PATCH 082/186] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93ec4fa4a6..841aacdeba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,11 @@ * Remove base `Dockerfile` used for DSL1 pipeline container builds * Run tests with Python 3.10 * [#1363](https://github.com/nf-core/tools/pull/1363) Fix tools CI workflow nextflow versions. +* [#1384](https://github.com/nf-core/tools/pull/1384) Adds Gitpod environment and Dockerfile. + * Adds conda, Nextflow, nf-core, pytest-workflow, mamba, and pip to base Gitpod Docker image. + * Adds GH action to build and push Gitpod Docker image. + * Adds Gitpod environment to template. + * Adds Gitpod environment to tools with auto build of nf-core tool. ### Modules From 29d681cbfcb8f1f87fedbadb722c4e2d907bf22b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 17:08:17 +0100 Subject: [PATCH 083/186] Found duplicated method, standardised + updated --- nf_core/modules/bump_versions.py | 4 +-- nf_core/modules/create.py | 45 ++-------------------------- nf_core/modules/lint/__init__.py | 2 +- nf_core/modules/module_utils.py | 47 +++++++++++++++++++++++------- nf_core/modules/modules_command.py | 2 +- 5 files changed, 41 insertions(+), 59 deletions(-) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index f7e0a595c3..d8b0450a69 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -15,8 +15,6 @@ from rich.markdown import Markdown import rich from nf_core.utils import rich_force_colors -import sys -import yaml import nf_core.utils import nf_core.modules.module_utils @@ -55,7 +53,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): self.show_up_to_date = show_uptodate # Verify that this is not a pipeline - repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) if not repo_type == "modules": raise nf_core.modules.module_utils.ModuleException( "This command only works on the nf-core/modules repository, not on pipelines!" diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 72b106844f..c4371853ed 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -19,6 +19,7 @@ import yaml import nf_core.utils +import nf_core.modules.module_utils log = logging.getLogger(__name__) @@ -83,7 +84,7 @@ def create(self): # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules try: - self.get_repo_type() + self.directory, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.directory, self.repo_type) except LookupError as e: raise UserWarning(e) log.info(f"Repository type: [blue]{self.repo_type}") @@ -283,48 +284,6 @@ def render_template(self): template_stat = os.stat(os.path.join(os.path.dirname(nf_core.__file__), "module-template", template_fn)) os.chmod(dest_fn, template_stat.st_mode) - def get_repo_type(self): - """ - Determine whether this is a pipeline repository or a clone of - nf-core/modules - """ - # Verify that the pipeline dir exists - if dir is None or not os.path.exists(self.directory): - raise UserWarning(f"Could not find directory: {self.directory}") - - # Try to find the root directory - base_dir = os.path.abspath(self.directory) - config_path_yml = os.path.join(base_dir, ".nf-core.yml") - config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") - while ( - not os.path.exists(config_path_yml) - and not os.path.exists(config_path_yaml) - and base_dir != os.path.dirname(base_dir) - ): - base_dir = os.path.dirname(base_dir) - config_path_yml = os.path.join(base_dir, ".nf-core.yml") - config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") - # Reset self.directory if we found the config file (will be an absolute path) - if os.path.exists(config_path_yml) or os.path.exists(config_path_yaml): - self.directory = base_dir - - # Figure out the repository type from the .nf-core.yml config file if we can - tools_config = nf_core.utils.load_tools_config(self.directory) - if tools_config.get("repository_type") in ["pipeline", "modules"]: - if self.repo_type is not None and self.repo_type != tools_config["repository_type"]: - raise UserWarning( - f"'--repo-type {self.repo_type}' conflicts with [i][red]repository_type[/]: [blue]pipeline[/][/] in '{os.path.relpath(config_path_yml)}'" - ) - self.repo_type = tools_config["repository_type"] - return - - # Could be set on the command line - throw an error if not - if not self.repo_type: - raise UserWarning( - f"Can't find a '.nf-core.yml' file with 'repository_type' set to 'pipeline' or 'modules': '{self.directory}'" - "\nPlease use the '--repo-type' flag or create '.nf-core.yml'" - ) - def get_module_dirs(self): """Given a directory and a tool/subtool, set the file paths and check if they already exist diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 892cacf8de..af432ec269 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -71,7 +71,7 @@ class ModuleLint(ModuleCommand): def __init__(self, dir): self.dir = dir try: - self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir, self.repo_type) except LookupError as e: raise UserWarning(e) diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index dd9afc9582..a7c600a996 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -337,24 +337,49 @@ def get_installed_modules(dir, repo_type="modules"): return local_modules, nfcore_modules -def get_repo_type(dir): +def get_repo_type(dir, repo_type=None): """ Determine whether this is a pipeline repository or a clone of nf-core/modules """ # Verify that the pipeline dir exists if dir is None or not os.path.exists(dir): - raise LookupError("Could not find directory: {}".format(dir)) + raise UserWarning(f"Could not find directory: {dir}") + + # Try to find the root directory + base_dir = os.path.abspath(dir) + config_path_yml = os.path.join(base_dir, ".nf-core.yml") + config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") + while ( + not os.path.exists(config_path_yml) + and not os.path.exists(config_path_yaml) + and base_dir != os.path.dirname(base_dir) + ): + base_dir = os.path.dirname(base_dir) + config_path_yml = os.path.join(base_dir, ".nf-core.yml") + config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") + # Reset dir if we found the config file (will be an absolute path) + if os.path.exists(config_path_yml) or os.path.exists(config_path_yaml): + dir = base_dir + + # Figure out the repository type from the .nf-core.yml config file if we can + tools_config = nf_core.utils.load_tools_config(dir) + if tools_config.get("repository_type") in ["pipeline", "modules"]: + if repo_type is not None and repo_type != tools_config["repository_type"]: + raise UserWarning( + f"'--repo-type {repo_type}' conflicts with [i][red]repository_type[/]: [blue]pipeline[/][/] in '{os.path.relpath(config_path_yml)}'" + ) + return [dir, tools_config["repository_type"]] - # Determine repository type - if os.path.exists(os.path.join(dir, "README.md")): - with open(os.path.join(dir, "README.md")) as fh: - if fh.readline().rstrip().startswith("# ![nf-core/modules]"): - return "modules" - else: - return "pipeline" - else: - raise LookupError("Could not determine repository type of '{}'".format(dir)) + # Could be set on the command line - throw an error if not + if not repo_type: + raise UserWarning( + f"Can't find a '.nf-core.yml' file with 'repository_type' set to 'pipeline' or 'modules': '{dir}'" + "\nPlease use the '--repo-type' flag or create '.nf-core.yml'" + ) + + # It was set on the command line, return what we were given + return [dir, repo_type] def verify_pipeline_dir(dir): diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 866bffabb9..58198740b6 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -29,7 +29,7 @@ def __init__(self, dir): self.module_names = [] try: if self.dir: - self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) else: self.repo_type = None except LookupError as e: From c5697402d4e16541fd3a76e312ee36a3e07e95e2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 1 Feb 2022 17:12:17 +0100 Subject: [PATCH 084/186] Modules lint: don't supply repo type, find it --- nf_core/modules/lint/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index af432ec269..e81eea7f85 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -71,7 +71,7 @@ class ModuleLint(ModuleCommand): def __init__(self, dir): self.dir = dir try: - self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir, self.repo_type) + self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) except LookupError as e: raise UserWarning(e) From fb834036b0a1f30f32ba73cf31601db3c706c186 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 3 Feb 2022 10:58:46 +0100 Subject: [PATCH 085/186] Allow conditional process execution from configuration --- nf_core/module-template/modules/main.nf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index d152e970b3..8a61ea32d1 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -5,7 +5,7 @@ // TODO nf-core: A module file SHOULD only define input and output files as command-line parameters. // All other parameters MUST be provided using the "task.ext" directive, see here: // https://www.nextflow.io/docs/latest/process.html#ext -// where "task.ext" is a string. +// where "task.ext" is a string. // Any parameters that need to be evaluated in the context of a particular sample // e.g. single-end/paired-end data MUST also be defined and evaluated appropriately. // TODO nf-core: Software that can be piped together SHOULD be added to separate module files @@ -18,7 +18,7 @@ process {{ tool_name_underscore|upper }} { tag {{ '"$meta.id"' if has_meta else "'$bam'" }} label '{{ process_label }}' - + // TODO nf-core: List required Conda package(s). // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. @@ -43,6 +43,9 @@ process {{ tool_name_underscore|upper }} { // TODO nf-core: List additional required output channels/values here path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: def args = task.ext.args ?: '' {% if has_meta -%} From 632f7a614a0c30efabd5026729c8b9c997865ba7 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 3 Feb 2022 12:02:02 +0100 Subject: [PATCH 086/186] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b3e816059..a721316a3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ * `input:` / `output:` not being specified in module * Allow for containers from other biocontainers resource as defined [here](https://github.com/nf-core/modules/blob/cde237e7cec07798e5754b72aeca44efe89fc6db/modules/cat/fastq/main.nf#L7-L8) * Fixed traceback when using `stageAs` syntax as defined [here](https://github.com/nf-core/modules/blob/cde237e7cec07798e5754b72aeca44efe89fc6db/modules/cat/fastq/main.nf#L11) +* Allow conditional process execution from the configuration file ([#1393](https://github.com/nf-core/tools/pull/1393)) ## [v2.2 - Lead Liger](https://github.com/nf-core/tools/releases/tag/2.2) - [2021-12-14] From 56ac0772243b5b005ec4d73b98f43c3e3c775968 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 3 Feb 2022 12:03:15 +0100 Subject: [PATCH 087/186] Changelog message fix --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b3e816059..cb49aa93e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ ## General * Updated `nf-core download` to work with latest DSL2 syntax for containers ([#1379](https://github.com/nf-core/tools/issues/1379)) +* [#1384](https://github.com/nf-core/tools/pull/1384) Adds Gitpod environment and Dockerfile. + * Adds conda, Nextflow, nf-core, pytest-workflow, mamba, and pip to base Gitpod Docker image. + * Adds GH action to build and push Gitpod Docker image. + * Adds Gitpod environment to template. + * Adds Gitpod environment to tools with auto build of nf-core tool. ### Modules @@ -63,11 +68,6 @@ * Remove base `Dockerfile` used for DSL1 pipeline container builds * Run tests with Python 3.10 * [#1363](https://github.com/nf-core/tools/pull/1363) Fix tools CI workflow nextflow versions. -* [#1384](https://github.com/nf-core/tools/pull/1384) Adds Gitpod environment and Dockerfile. - * Adds conda, Nextflow, nf-core, pytest-workflow, mamba, and pip to base Gitpod Docker image. - * Adds GH action to build and push Gitpod Docker image. - * Adds Gitpod environment to template. - * Adds Gitpod environment to tools with auto build of nf-core tool. ### Modules From 190c0b9b76a2757ab9817cd0fdef4bb8ab071c2a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 3 Feb 2022 16:14:44 +0100 Subject: [PATCH 088/186] Bumped `nf-core/tower-action` to `v3` Also removed `pipeline` and `revision` from the AWS workflows, which were not needed. See https://github.com/nf-core/tower-action/pull/11 for context --- CHANGELOG.md | 1 + nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 4 +--- nf_core/pipeline-template/.github/workflows/awstest.yml | 4 +--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb49aa93e7..4c73240acc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * Removed mention of `--singularity_pull_docker_container` in pipeline `README.md` * Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode. * Add retry strategy for AWS megatests after releasing [nf-core/tower-action v2.2](https://github.com/nf-core/tower-action/releases/tag/v2.2) +* Bumped `nf-core/tower-action` to `v3` and removed `pipeline` and `revision` from the AWS workflows, which were not needed ## General diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 64e3d0a82f..4e3c168cc1 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: nf-core/tower-action@v2 + uses: nf-core/tower-action@v3 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters @@ -23,8 +23,6 @@ jobs: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} - pipeline: ${{ github.repository }} - revision: ${{ github.sha }} workdir: s3://${{ secrets.AWS_S3_BUCKET }}{% endraw %}/work/{{ short_name }}/{% raw %}work-${{ github.sha }}{% endraw %} parameters: | { diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index a4b0ec461a..e980264d9d 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -11,14 +11,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: nf-core/tower-action@v2 + uses: nf-core/tower-action@v3 {% raw %} with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} - pipeline: ${{ github.repository }} - revision: ${{ github.sha }} workdir: s3://${{ secrets.AWS_S3_BUCKET }}{% endraw %}/work/{{ short_name }}/{% raw %}work-${{ github.sha }}{% endraw %} parameters: | { From 931006f9856070ea719e65f0db32ae2cef849ace Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Fri, 4 Feb 2022 10:44:44 +0100 Subject: [PATCH 089/186] Update igenomes path to BWAIndex --- .../pipeline-template/conf/igenomes.config | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/nf_core/pipeline-template/conf/igenomes.config b/nf_core/pipeline-template/conf/igenomes.config index 6ff960c81a..7a1b3ac6d3 100644 --- a/nf_core/pipeline-template/conf/igenomes.config +++ b/nf_core/pipeline-template/conf/igenomes.config @@ -13,7 +13,7 @@ params { genomes { 'GRCh37' { fasta = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Homo_sapiens/Ensembl/GRCh37/Sequence/BismarkIndex/" @@ -26,7 +26,7 @@ params { } 'GRCh38' { fasta = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Homo_sapiens/NCBI/GRCh38/Sequence/BismarkIndex/" @@ -38,7 +38,7 @@ params { } 'GRCm38' { fasta = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BismarkIndex/" @@ -51,7 +51,7 @@ params { } 'TAIR10' { fasta = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Arabidopsis_thaliana/Ensembl/TAIR10/Sequence/BismarkIndex/" @@ -62,7 +62,7 @@ params { } 'EB2' { fasta = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Bacillus_subtilis_168/Ensembl/EB2/Sequence/BismarkIndex/" @@ -72,7 +72,7 @@ params { } 'UMD3.1' { fasta = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Bos_taurus/Ensembl/UMD3.1/Sequence/BismarkIndex/" @@ -83,7 +83,7 @@ params { } 'WBcel235' { fasta = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Caenorhabditis_elegans/Ensembl/WBcel235/Sequence/BismarkIndex/" @@ -94,7 +94,7 @@ params { } 'CanFam3.1' { fasta = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Canis_familiaris/Ensembl/CanFam3.1/Sequence/BismarkIndex/" @@ -105,7 +105,7 @@ params { } 'GRCz10' { fasta = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Danio_rerio/Ensembl/GRCz10/Sequence/BismarkIndex/" @@ -115,7 +115,7 @@ params { } 'BDGP6' { fasta = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Drosophila_melanogaster/Ensembl/BDGP6/Sequence/BismarkIndex/" @@ -126,7 +126,7 @@ params { } 'EquCab2' { fasta = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Equus_caballus/Ensembl/EquCab2/Sequence/BismarkIndex/" @@ -137,7 +137,7 @@ params { } 'EB1' { fasta = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Escherichia_coli_K_12_DH10B/Ensembl/EB1/Sequence/BismarkIndex/" @@ -147,7 +147,7 @@ params { } 'Galgal4' { fasta = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Gallus_gallus/Ensembl/Galgal4/Sequence/BismarkIndex/" @@ -157,7 +157,7 @@ params { } 'Gm01' { fasta = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Glycine_max/Ensembl/Gm01/Sequence/BismarkIndex/" @@ -167,7 +167,7 @@ params { } 'Mmul_1' { fasta = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Macaca_mulatta/Ensembl/Mmul_1/Sequence/BismarkIndex/" @@ -178,7 +178,7 @@ params { } 'IRGSP-1.0' { fasta = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Oryza_sativa_japonica/Ensembl/IRGSP-1.0/Sequence/BismarkIndex/" @@ -188,7 +188,7 @@ params { } 'CHIMP2.1.4' { fasta = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Pan_troglodytes/Ensembl/CHIMP2.1.4/Sequence/BismarkIndex/" @@ -199,7 +199,7 @@ params { } 'Rnor_5.0' { fasta = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_5.0/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_5.0/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_5.0/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_5.0/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_5.0/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_5.0/Sequence/BismarkIndex/" @@ -209,7 +209,7 @@ params { } 'Rnor_6.0' { fasta = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Rattus_norvegicus/Ensembl/Rnor_6.0/Sequence/BismarkIndex/" @@ -219,7 +219,7 @@ params { } 'R64-1-1' { fasta = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Saccharomyces_cerevisiae/Ensembl/R64-1-1/Sequence/BismarkIndex/" @@ -230,7 +230,7 @@ params { } 'EF2' { fasta = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Schizosaccharomyces_pombe/Ensembl/EF2/Sequence/BismarkIndex/" @@ -242,7 +242,7 @@ params { } 'Sbi1' { fasta = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Sorghum_bicolor/Ensembl/Sbi1/Sequence/BismarkIndex/" @@ -252,7 +252,7 @@ params { } 'Sscrofa10.2' { fasta = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Sus_scrofa/Ensembl/Sscrofa10.2/Sequence/BismarkIndex/" @@ -263,7 +263,7 @@ params { } 'AGPv3' { fasta = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Zea_mays/Ensembl/AGPv3/Sequence/BismarkIndex/" @@ -273,7 +273,7 @@ params { } 'hg38' { fasta = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Homo_sapiens/UCSC/hg38/Sequence/BismarkIndex/" @@ -285,7 +285,7 @@ params { } 'hg19' { fasta = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Homo_sapiens/UCSC/hg19/Sequence/BismarkIndex/" @@ -298,7 +298,7 @@ params { } 'mm10' { fasta = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Mus_musculus/UCSC/mm10/Sequence/BismarkIndex/" @@ -311,7 +311,7 @@ params { } 'bosTau8' { fasta = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Bos_taurus/UCSC/bosTau8/Sequence/BismarkIndex/" @@ -321,7 +321,7 @@ params { } 'ce10' { fasta = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Caenorhabditis_elegans/UCSC/ce10/Sequence/BismarkIndex/" @@ -333,7 +333,7 @@ params { } 'canFam3' { fasta = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Canis_familiaris/UCSC/canFam3/Sequence/BismarkIndex/" @@ -344,7 +344,7 @@ params { } 'danRer10' { fasta = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Danio_rerio/UCSC/danRer10/Sequence/BismarkIndex/" @@ -355,7 +355,7 @@ params { } 'dm6' { fasta = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Drosophila_melanogaster/UCSC/dm6/Sequence/BismarkIndex/" @@ -366,7 +366,7 @@ params { } 'equCab2' { fasta = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Equus_caballus/UCSC/equCab2/Sequence/BismarkIndex/" @@ -377,7 +377,7 @@ params { } 'galGal4' { fasta = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Gallus_gallus/UCSC/galGal4/Sequence/BismarkIndex/" @@ -388,7 +388,7 @@ params { } 'panTro4' { fasta = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Pan_troglodytes/UCSC/panTro4/Sequence/BismarkIndex/" @@ -399,7 +399,7 @@ params { } 'rn6' { fasta = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Rattus_norvegicus/UCSC/rn6/Sequence/BismarkIndex/" @@ -409,7 +409,7 @@ params { } 'sacCer3' { fasta = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Saccharomyces_cerevisiae/UCSC/sacCer3/Sequence/BismarkIndex/" @@ -419,7 +419,7 @@ params { } 'susScr3' { fasta = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/WholeGenomeFasta/genome.fa" - bwa = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/BWAIndex/genome.fa" + bwa = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/BWAIndex/version0.6.0/" bowtie2 = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/Bowtie2Index/" star = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/STARIndex/" bismark = "${params.igenomes_base}/Sus_scrofa/UCSC/susScr3/Sequence/BismarkIndex/" From 63b3eb2ce701b4a95ffdd0817944c34c688112d3 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Fri, 4 Feb 2022 11:14:26 +0100 Subject: [PATCH 090/186] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 937a4a87dc..5ec2966903 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * Removed mention of `--singularity_pull_docker_container` in pipeline `README.md` * Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode. * Add retry strategy for AWS megatests after releasing [nf-core/tower-action v2.2](https://github.com/nf-core/tower-action/releases/tag/v2.2) +* Update igenomes path to the `BWAIndex` to fetch the whole `version0.6.0` folder instead of only the `genome.fa` file ## General From ca878e58ce066e67c82a1ba5e1ea645159d5f494 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 4 Feb 2022 12:45:18 +0000 Subject: [PATCH 091/186] Add module linting for when condition --- nf_core/modules/lint/main_nf.py | 35 +++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 8e0fcc108c..f3228fe714 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -43,6 +43,7 @@ def main_nf(module_lint_object, module): state = "module" process_lines = [] script_lines = [] + when_lines = [] for l in lines: if re.search("^\s*process\s*\w*\s*{", l) and state == "module": state = "process" @@ -52,7 +53,10 @@ def main_nf(module_lint_object, module): if re.search("output\s*:", l) and state in ["input", "process"]: state = "output" continue - if re.search("script\s*:", l) and state in ["input", "output", "process"]: + if re.search("when\s*:", l) and state in ["input", "output", "process"]: + state = "when" + continue + if re.search("script\s*:", l) and state in ["input", "output", "when", "process"]: state = "script" continue @@ -64,6 +68,8 @@ def main_nf(module_lint_object, module): if state == "output" and not _is_empty(module, l): outputs += _parse_output(module, l) outputs = list(set(outputs)) # remove duplicate 'meta's + if state == "when" and not _is_empty(module, l): + when_lines.append(l) if state == "script" and not _is_empty(module, l): script_lines.append(l) @@ -73,6 +79,9 @@ def main_nf(module_lint_object, module): else: module.warned.append(("main_nf_container", "Container versions do not match", module.main_nf)) + # Check the when statement + check_when_section(module, when_lines) + # Check the script definition check_script_section(module, script_lines) @@ -121,6 +130,28 @@ def check_script_section(self, lines): self.failed.append(("main_nf_meta_prefix", "'prefix' unspecified in script section", self.main_nf)) +def check_when_section(self, lines): + """ + Lint the when section + Checks whether the line is modified from 'task.ext.when == null || task.ext.when' + """ + if len(lines) == 0: + self.failed.append(("when_exist", "When condition has been removed", self.main_nf)) + return + elif len(lines) > 1: + self.failed.append(("when_exist", "When condition has too many lines", self.main_nf)) + return + else: + self.passed.append(("when_exist", "When condition is present", self.main_nf)) + + # Check the condition hasn't been changed. + if lines[0].strip() != "task.ext.when == null || task.ext.when": + self.failed.append(("when_condition", "When condition has been altered", self.main_nf)) + return + else: + self.passed.append(("when_condition", "When condition is unchanged", self.main_nf)) + + def check_process_section(self, lines): """ Lint the section of a module between the process definition @@ -146,7 +177,7 @@ def check_process_section(self, lines): if all([x.upper() for x in self.process_name]): self.passed.append(("process_capitals", "Process name is in capital letters", self.main_nf)) else: - self.failed.append(("process_capitals", "Process name is not in captial letters", self.main_nf)) + self.failed.append(("process_capitals", "Process name is not in capitial letters", self.main_nf)) # Check that process labels are correct correct_process_labels = ["process_low", "process_medium", "process_high", "process_long"] From be163e5c62b811b931c1a3f9d9d1b603523af969 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 4 Feb 2022 12:48:25 +0000 Subject: [PATCH 092/186] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ec2966903..9037560e9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ * Allow for containers from other biocontainers resource as defined [here](https://github.com/nf-core/modules/blob/cde237e7cec07798e5754b72aeca44efe89fc6db/modules/cat/fastq/main.nf#L7-L8) * Fixed traceback when using `stageAs` syntax as defined [here](https://github.com/nf-core/modules/blob/cde237e7cec07798e5754b72aeca44efe89fc6db/modules/cat/fastq/main.nf#L11) * Allow conditional process execution from the configuration file ([#1393](https://github.com/nf-core/tools/pull/1393)) +* Add linting for when condition([#1397](https://github.com/nf-core/tools/pull/1397)) ## [v2.2 - Lead Liger](https://github.com/nf-core/tools/releases/tag/2.2) - [2021-12-14] From e4dfe4403cb0b5ac70f72915f3ba0cde0eaffd59 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 4 Feb 2022 13:18:27 +0000 Subject: [PATCH 093/186] Fix spelling --- nf_core/modules/lint/main_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index f3228fe714..084d053562 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -177,7 +177,7 @@ def check_process_section(self, lines): if all([x.upper() for x in self.process_name]): self.passed.append(("process_capitals", "Process name is in capital letters", self.main_nf)) else: - self.failed.append(("process_capitals", "Process name is not in capitial letters", self.main_nf)) + self.failed.append(("process_capitals", "Process name is not in capital letters", self.main_nf)) # Check that process labels are correct correct_process_labels = ["process_low", "process_medium", "process_high", "process_long"] From 4bb44dcfabb68b3b4066bbd83587f4eaeb24f191 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 7 Feb 2022 20:02:48 +0100 Subject: [PATCH 094/186] Bump node-version to 16 Fixes markdown linting --- .github/workflows/markdown-lint.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/markdown-lint.yml b/.github/workflows/markdown-lint.yml index 193052a437..18cea737b6 100644 --- a/.github/workflows/markdown-lint.yml +++ b/.github/workflows/markdown-lint.yml @@ -9,7 +9,7 @@ jobs: - uses: actions/setup-node@v1 with: - node-version: "10" + node-version: "16" - name: Install markdownlint run: npm install -g markdownlint-cli diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 15c49c860d..2577db5307 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: - node-version: '10' + node-version: '16' - name: Install markdownlint run: npm install -g markdownlint-cli - name: Run Markdownlint @@ -53,7 +53,7 @@ jobs: - uses: actions/setup-node@v1 with: - node-version: '10' + node-version: '16' - name: Install editorconfig-checker run: npm install -g editorconfig-checker @@ -67,7 +67,7 @@ jobs: - uses: actions/checkout@v1 - uses: actions/setup-node@v1 with: - node-version: '10' + node-version: '16' - name: Install yaml-lint run: npm install -g yaml-lint - name: Run yaml-lint From eac043137d957a7350b71832ffbaa8de0f489e41 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 7 Feb 2022 20:05:00 +0100 Subject: [PATCH 095/186] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9037560e9c..f10d433082 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode. * Add retry strategy for AWS megatests after releasing [nf-core/tower-action v2.2](https://github.com/nf-core/tower-action/releases/tag/v2.2) * Update igenomes path to the `BWAIndex` to fetch the whole `version0.6.0` folder instead of only the `genome.fa` file +* Bump Node version to 16 in the GitHub Actions workflows, to fix errors with `markdownlint` ## General From e80474fea6a65b2efff8a9d53837628bf153bd8c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 7 Feb 2022 20:07:37 +0100 Subject: [PATCH 096/186] Fix markdownlint, add some line breaks --- README.md | 2 +- nf_core/pipeline-template/docs/usage.md | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 65fb66b004..f9ae7f2253 100644 --- a/README.md +++ b/README.md @@ -925,7 +925,7 @@ github.com: git_protocol: ``` -The easiest way to create this configuration file is through *GitHub CLI*: follow +The easiest way to create this configuration file is through _GitHub CLI_: follow its [installation instructions](https://cli.github.com/manual/installation) and then call: diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 485af3af44..34af20b733 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -1,6 +1,6 @@ # {{ name }}: Usage -## :warning: Please read this documentation on the nf-core website: [https://nf-co.re/{{ short_name }}/usage](https://nf-co.re/{{ short_name }}/usage) +## :warning: Please read this documentation on the nf-core website: [{{ short_name }}/usage]({{ short_name }}/usage) > _Documentation of pipeline parameters is generated automatically from the pipeline schema and can no longer be found in markdown files._ @@ -83,7 +83,7 @@ nextflow pull {{ name }} It is a good idea to specify a pipeline version when running the pipeline on your data. This ensures that a specific version of the pipeline code and software are used when you run your pipeline. If you keep using the same tag, you'll be running the same version of the pipeline, even if there have been changes to the code since. -First, go to the [{{ name }} releases page](https://github.com/{{ name }}/releases) and find the latest version number - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. +First, go to the [{{ name }} releases page]({{ name }}/releases) and find the latest version number - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. @@ -169,7 +169,13 @@ Work dir: Tip: you can replicate the issue by changing to the process work dir and entering the command `bash .command.run` ``` -To bypass this error you would need to find exactly which resources are set by the `STAR_ALIGN` process. The quickest way is to search for `process STAR_ALIGN` in the [nf-core/rnaseq Github repo](https://github.com/nf-core/rnaseq/search?q=process+STAR_ALIGN). We have standardised the structure of Nextflow DSL2 pipelines such that all module files will be present in the `modules/` directory and so based on the search results the file we want is `modules/nf-core/software/star/align/main.nf`. If you click on the link to that file you will notice that there is a `label` directive at the top of the module that is set to [`label process_high`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/modules/nf-core/software/star/align/main.nf#L9). The [Nextflow `label`](https://www.nextflow.io/docs/latest/process.html#label) directive allows us to organise workflow processes in separate groups which can be referenced in a configuration file to select and configure subset of processes having similar computing requirements. The default values for the `process_high` label are set in the pipeline's [`base.config`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L33-L37) which in this case is defined as 72GB. Providing you haven't set any other standard nf-core parameters to __cap__ the [maximum resources](https://nf-co.re/usage/configuration#max-resources) used by the pipeline then we can try and bypass the `STAR_ALIGN` process failure by creating a custom config file that sets at least 72GB of memory, in this case increased to 100GB. The custom config below can then be provided to the pipeline via the [`-c`](#-c) parameter as highlighted in previous sections. +To bypass this error you would need to find exactly which resources are set by the `STAR_ALIGN` process. The quickest way is to search for `process STAR_ALIGN` in the [nf-core/rnaseq Github repo](https://github.com/nf-core/rnaseq/search?q=process+STAR_ALIGN). +We have standardised the structure of Nextflow DSL2 pipelines such that all module files will be present in the `modules/` directory and so based on the search results the file we want is `modules/nf-core/software/star/align/main.nf`. +If you click on the link to that file you will notice that there is a `label` directive at the top of the module that is set to [`label process_high`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/modules/nf-core/software/star/align/main.nf#L9). +The [Nextflow `label`](https://www.nextflow.io/docs/latest/process.html#label) directive allows us to organise workflow processes in separate groups which can be referenced in a configuration file to select and configure subset of processes having similar computing requirements. +The default values for the `process_high` label are set in the pipeline's [`base.config`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L33-L37) which in this case is defined as 72GB. +Providing you haven't set any other standard nf-core parameters to **cap** the [maximum resources](https://nf-co.re/usage/configuration#max-resources) used by the pipeline then we can try and bypass the `STAR_ALIGN` process failure by creating a custom config file that sets at least 72GB of memory, in this case increased to 100GB. +The custom config below can then be provided to the pipeline via the [`-c`](#-c) parameter as highlighted in previous sections. ```nextflow process { @@ -179,7 +185,8 @@ process { } ``` -> **NB:** We specify just the process name i.e. `STAR_ALIGN` in the config file and not the full task name string that is printed to screen in the error message or on the terminal whilst the pipeline is running i.e. `RNASEQ:ALIGN_STAR:STAR_ALIGN`. You may get a warning suggesting that the process selector isn't recognised but you can ignore that if the process name has been specified correctly. This is something that needs to be fixed upstream in core Nextflow. +> **NB:** We specify just the process name i.e. `STAR_ALIGN` in the config file and not the full task name string that is printed to screen in the error message or on the terminal whilst the pipeline is running i.e. `RNASEQ:ALIGN_STAR:STAR_ALIGN`. +> You may get a warning suggesting that the process selector isn't recognised but you can ignore that if the process name has been specified correctly. This is something that needs to be fixed upstream in core Nextflow. ### Updating containers From 45172e2833b1ea01ffd64ef35cf0c27ac1bd7010 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 8 Feb 2022 17:10:43 +0100 Subject: [PATCH 097/186] Strip pinned node version in GitHub actions Also revert auto-markdown fixing in template markdown files. --- .github/workflows/markdown-lint.yml | 4 +--- CHANGELOG.md | 2 +- .../pipeline-template/.github/workflows/linting.yml | 12 +++--------- nf_core/pipeline-template/docs/usage.md | 4 ++-- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/.github/workflows/markdown-lint.yml b/.github/workflows/markdown-lint.yml index 18cea737b6..a34f573fe8 100644 --- a/.github/workflows/markdown-lint.yml +++ b/.github/workflows/markdown-lint.yml @@ -7,9 +7,7 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 - with: - node-version: "16" + - uses: actions/setup-node@v2 - name: Install markdownlint run: npm install -g markdownlint-cli diff --git a/CHANGELOG.md b/CHANGELOG.md index f10d433082..e58365bd8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ * Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode. * Add retry strategy for AWS megatests after releasing [nf-core/tower-action v2.2](https://github.com/nf-core/tower-action/releases/tag/v2.2) * Update igenomes path to the `BWAIndex` to fetch the whole `version0.6.0` folder instead of only the `genome.fa` file -* Bump Node version to 16 in the GitHub Actions workflows, to fix errors with `markdownlint` +* Remove pinned Node version in the GitHub Actions workflows, to fix errors with `markdownlint` ## General diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 2577db5307..4b49a53ca6 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -12,9 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 - with: - node-version: '16' + - uses: actions/setup-node@v2 - name: Install markdownlint run: npm install -g markdownlint-cli - name: Run Markdownlint @@ -51,9 +49,7 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 - with: - node-version: '16' + - uses: actions/setup-node@v2 - name: Install editorconfig-checker run: npm install -g editorconfig-checker @@ -65,9 +61,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - uses: actions/setup-node@v1 - with: - node-version: '16' + - uses: actions/setup-node@v2 - name: Install yaml-lint run: npm install -g yaml-lint - name: Run yaml-lint diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 34af20b733..16af4ea6da 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -1,6 +1,6 @@ # {{ name }}: Usage -## :warning: Please read this documentation on the nf-core website: [{{ short_name }}/usage]({{ short_name }}/usage) +## :warning: Please read this documentation on the nf-core website: [https://nf-co.re/{{ short_name }}/usage](https://nf-co.re/{{ short_name }}/usage) > _Documentation of pipeline parameters is generated automatically from the pipeline schema and can no longer be found in markdown files._ @@ -83,7 +83,7 @@ nextflow pull {{ name }} It is a good idea to specify a pipeline version when running the pipeline on your data. This ensures that a specific version of the pipeline code and software are used when you run your pipeline. If you keep using the same tag, you'll be running the same version of the pipeline, even if there have been changes to the code since. -First, go to the [{{ name }} releases page]({{ name }}/releases) and find the latest version number - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. +First, go to the [{{ name }} releases page](https://github.com/{{ name }}/releases) and find the latest version number - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. From 53f689c656f7e2bfa298b44fbf117acc81c8f6d9 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 9 Feb 2022 12:23:28 +0100 Subject: [PATCH 098/186] Update main_nf.py --- nf_core/modules/lint/main_nf.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 084d053562..9f27f94f2c 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -132,24 +132,24 @@ def check_script_section(self, lines): def check_when_section(self, lines): """ - Lint the when section + Lint the when: section Checks whether the line is modified from 'task.ext.when == null || task.ext.when' """ if len(lines) == 0: - self.failed.append(("when_exist", "When condition has been removed", self.main_nf)) + self.failed.append(("when_exist", "when: condition has been removed", self.main_nf)) return elif len(lines) > 1: - self.failed.append(("when_exist", "When condition has too many lines", self.main_nf)) + self.failed.append(("when_exist", "when: condition has too many lines", self.main_nf)) return else: - self.passed.append(("when_exist", "When condition is present", self.main_nf)) + self.passed.append(("when_exist", "when: condition is present", self.main_nf)) # Check the condition hasn't been changed. if lines[0].strip() != "task.ext.when == null || task.ext.when": - self.failed.append(("when_condition", "When condition has been altered", self.main_nf)) + self.failed.append(("when_condition", "when: condition has been altered", self.main_nf)) return else: - self.passed.append(("when_condition", "When condition is unchanged", self.main_nf)) + self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf)) def check_process_section(self, lines): From 7bf040f0a6d5406afcdf5530c5d830ea66d8525d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Wed, 9 Feb 2022 12:35:48 +0100 Subject: [PATCH 099/186] switch to gitpod/workspace-python to reduce image size --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index d5a17cd33c..69ac3aef86 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,4 +1,4 @@ -FROM gitpod/workspace-full +FROM gitpod/workspace-python USER root From d631253f22bf776217910d4fcba03f8c8b2243d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Wed, 9 Feb 2022 12:38:52 +0100 Subject: [PATCH 100/186] format fixing for @ewels --- .gitpod.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 86ed680600..1e83175b0e 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,8 +1,7 @@ image: nfcore/gitpod:latest tasks: - name: install current state of nf-core/tools - command: | - python -m pip install --upgrade -r requirements-dev.txt -e . + command: python -m pip install --upgrade -r requirements-dev.txt -e . vscode: extensions: # based on nf-core.nf-core-extensionpack - codezombiech.gitignore # Language support for .gitignore files From e28e8152c1e8729348c8cf2e9611fbfbd415bbb9 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 9 Feb 2022 15:32:17 +0100 Subject: [PATCH 101/186] First stab at nicely formatted cli outut using Rich. Took code from rich-cli written by @willmcgugan and played around with until it worked for us. --- nf_core/__main__.py | 149 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 122 insertions(+), 27 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index a9500509a4..1bd95d7926 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1,11 +1,6 @@ #!/usr/bin/env python """ nf-core: Helper tools for use with nf-core Nextflow pipelines. """ -from nf_core.lint.pipeline_todos import pipeline_todos -from nf_core.modules.bump_versions import ModuleVersionBumper -from click.types import File -from rich import print -from rich.prompt import Confirm import click import logging import os @@ -13,6 +8,12 @@ import rich.console import rich.logging import rich.traceback +from rich import print +from rich.text import Text +from rich.highlighter import RegexHighlighter +from rich.panel import Panel +from rich.table import Table +from rich.theme import Theme import sys import nf_core @@ -46,7 +47,10 @@ def run_nf_core(): stderr.print("[blue] |\ | |__ __ / ` / \ |__) |__ [yellow] } {", highlight=False) stderr.print("[blue] | \| | \__, \__/ | \ |___ [green]\`-._,-`-,", highlight=False) stderr.print("[green] `._,._,'\n", highlight=False) - stderr.print("[grey39] nf-core/tools version {}".format(nf_core.__version__), highlight=False) + stderr.print( + f"[grey39] nf-core/tools version {nf_core.__version__} - [link=https://nf-co.re]https://nf-co.re[/]", + highlight=False, + ) try: is_outdated, current_vers, remote_vers = nf_core.utils.check_if_outdated() if is_outdated: @@ -62,6 +66,86 @@ def run_nf_core(): nf_core_cli() +def rich_format_help(obj, ctx, formatter): + """ + Print nicely formatted help text using rich + + This code was shamelessly stolen from rich-cli, the + original author was @willmcgugan - thanks Will! + + I've modified it a little to work with click groups + and to spit out output in a style that fits well with our tool. + + If this the rich-click plugin gets made, we can probably strip + this out and just use that instead. + + Original source: + https://github.com/Textualize/rich-cli/blob/8a2767c7a340715fc6fbf4930ace717b9b2fc5e5/src/rich_cli/__main__.py#L162-L236 + """ + + class OptionHighlighter(RegexHighlighter): + highlights = [ + r"(?P\-\w)(?!\S)", + r"(?P