From d47a2cca52079a6a77342793ea9dcf46fc68f53f Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:52:52 +0100 Subject: [PATCH 1/9] fix(1929): more type-gating --- nf_core/components/components_command.py | 4 + nf_core/components/create.py | 96 +++++++------------ nf_core/components/nfcore_component.py | 29 +++--- nf_core/modules/lint/module_tests.py | 2 + nf_core/pipelines/lint/__init__.py | 3 + .../lint/local_component_structure.py | 31 ++++++ .../subworkflows/lint/subworkflow_tests.py | 3 +- tests/modules/test_create.py | 4 +- 8 files changed, 98 insertions(+), 74 deletions(-) create mode 100644 nf_core/pipelines/lint/local_component_structure.py diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index f25fb33a6f..f04bb7da6b 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -71,6 +71,10 @@ def get_local_components(self) -> List[str]: """ local_component_dir = Path(self.directory, self.component_type, "local") return [ + str(Path(directory).relative_to(local_component_dir)) + for directory, _, files in os.walk(local_component_dir) + if "main.nf" in files + ] + [ str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" ] diff --git a/nf_core/components/create.py b/nf_core/components/create.py index c781905618..0a2c6aaff7 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -75,11 +75,11 @@ def create(self) -> bool: e.g bam_sort or bam_sort_samtools, respectively. If is a pipeline, this function creates a file called: - '/modules/local/tool.nf' + '/modules/local/tool/main.nf' OR - '/modules/local/tool_subtool.nf' + '/modules/local/tool/subtool/main.nf' OR for subworkflows - '/subworkflows/local/subworkflow_name.nf' + '/subworkflows/local/subworkflow_name/main.nf' If is a clone of nf-core/modules, it creates or modifies the following files: @@ -355,70 +355,46 @@ def _get_component_dirs(self) -> Dict[str, Path]: """ file_paths = {} if self.repo_type == "pipeline": - local_component_dir = Path(self.directory, self.component_type, "local") - # Check whether component file already exists - component_file = local_component_dir / f"{self.component_name}.nf" - if component_file.exists() and not self.force_overwrite: - raise UserWarning( - f"{self.component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite" - ) - - if self.component_type == "modules": - # If a subtool, check if there is a module called the base tool name already - if self.subtool and (local_component_dir / f"{self.component}.nf").exists(): - raise UserWarning( - f"Module '{self.component}' exists already, cannot make subtool '{self.component_name}'" - ) - - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{local_component_dir}/{self.component}_*.nf") - if not self.subtool and tool_glob: - raise UserWarning( - f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" - ) - - # Set file paths - file_paths["main.nf"] = component_file + component_dir = Path(self.directory, self.component_type, "local", self.component_dir) elif self.repo_type == "modules": component_dir = Path(self.directory, self.component_type, self.org, self.component_dir) - # Check if module/subworkflow directories exist already - if component_dir.exists() and not self.force_overwrite and not self.migrate_pytest: - raise UserWarning( - f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" - ) + else: + raise ValueError("`repo_type` not set correctly") - if self.component_type == "modules": - # If a subtool, check if there is a module called the base tool name already - parent_tool_main_nf = Path( - self.directory, - self.component_type, - self.org, - self.component, - "main.nf", + # Check if module/subworkflow directories exist already + if component_dir.exists() and not self.force_overwrite and not self.migrate_pytest: + raise UserWarning( + f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" + ) + + if self.component_type == "modules": + # If a subtool, check if there is a module called the base tool name already + parent_tool_main_nf = Path( + self.directory, + self.component_type, + self.org, + self.component, + "main.nf", + ) + if self.subtool and parent_tool_main_nf.exists() and not self.migrate_pytest: + raise UserWarning( + f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" ) - if self.subtool and parent_tool_main_nf.exists() and not self.migrate_pytest: - raise UserWarning( - f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" - ) - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob( - f"{Path(self.directory, self.component_type, self.org, self.component)}/*/main.nf" + # If no subtool, check that there isn't already a tool/subtool + tool_glob = glob.glob(f"{Path(self.directory, self.component_type, self.org, self.component)}/*/main.nf") + if not self.subtool and tool_glob and not self.migrate_pytest: + raise UserWarning( + f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" ) - if not self.subtool and tool_glob and not self.migrate_pytest: - raise UserWarning( - f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" - ) - # Set file paths - # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure - file_paths["main.nf"] = component_dir / "main.nf" - file_paths["meta.yml"] = component_dir / "meta.yml" - if self.component_type == "modules": - file_paths["environment.yml"] = component_dir / "environment.yml" - file_paths["tests/main.nf.test.j2"] = component_dir / "tests" / "main.nf.test" - else: - raise ValueError("`repo_type` not set correctly") + # Set file paths + # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure + file_paths["main.nf"] = component_dir / "main.nf" + file_paths["meta.yml"] = component_dir / "meta.yml" + if self.component_type == "modules": + file_paths["environment.yml"] = component_dir / "environment.yml" + file_paths["tests/main.nf.test.j2"] = component_dir / "tests" / "main.nf.test" return file_paths diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 37e43a536e..65c7a4acf4 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -75,8 +75,8 @@ def __init__( repo_dir = self.component_dir.parts[:name_index][-1] self.org = repo_dir - self.nftest_testdir = Path(self.component_dir, "tests") - self.nftest_main_nf = Path(self.nftest_testdir, "main.nf.test") + self.nftest_testdir: Optional[Path] = Path(self.component_dir, "tests") + self.nftest_main_nf: Optional[Path] = Path(self.nftest_testdir, "main.nf.test") if self.repo_type == "pipeline": patch_fn = f"{self.component_name.replace('/', '-')}.diff" @@ -86,15 +86,22 @@ def __init__( self.patch_path = patch_path else: # The main file is just the local module - self.main_nf = self.component_dir - self.component_name = self.component_dir.stem - # These attributes are only used by nf-core modules - # so just initialize them to None - self.meta_yml = None - self.environment_yml = None - self.test_dir = None - self.test_yml = None - self.test_main_nf = None + if self.component_dir.is_dir(): + self.main_nf = Path(self.component_dir, "main.nf") + self.component_name = self.component_dir.stem + # These attributes are only required by nf-core modules + # so just set them to None if they don't exist + self.meta_yml = p if (p := Path(self.component_dir, "meta.yml")).exists() else None + self.environment_yml = p if (p := Path(self.component_dir, "environment.yml")).exists() else None + self.nftest_testdir = p if (p := Path(self.component_dir, "tests")).exists() else None + if self.nftest_testdir is not None: + self.nftest_main_nf = p if (p := Path(self.nftest_testdir, "main.nf.test")).exists() else None + else: + self.main_nf = self.component_dir + self.meta_yml = None + self.environment_yml = None + self.nftest_testdir = None + self.nftest_main_nf = None def __repr__(self) -> str: return f"" diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 6722c12129..8f5fc91425 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -22,6 +22,8 @@ def module_tests(_, module: NFCoreComponent): and contains a ``main.nf.test`` and a ``main.nf.test.snap`` """ + if module.nftest_testdir is None or module.nftest_main_nf is None: + raise ValueError() repo_dir = module.component_dir.parts[: module.component_dir.parts.index(module.component_name.split("/")[0])][-1] test_dir = Path(module.base_dir, "tests", "modules", repo_dir, module.component_name) pytest_main_nf = Path(test_dir, "main.nf") diff --git a/nf_core/pipelines/lint/__init__.py b/nf_core/pipelines/lint/__init__.py index 8cc7c37cb2..6b3acaf2f5 100644 --- a/nf_core/pipelines/lint/__init__.py +++ b/nf_core/pipelines/lint/__init__.py @@ -38,6 +38,7 @@ from .files_exist import files_exist from .files_unchanged import files_unchanged from .included_configs import included_configs +from .local_component_structure import local_component_structure from .merge_markers import merge_markers from .modules_json import modules_json from .modules_structure import modules_structure @@ -89,6 +90,7 @@ class PipelineLint(nf_core.utils.Pipeline): merge_markers = merge_markers modules_json = modules_json modules_structure = modules_structure + local_component_structure = local_component_structure multiqc_config = multiqc_config nextflow_config = nextflow_config nfcore_yml = nfcore_yml @@ -151,6 +153,7 @@ def _get_all_lint_tests(release_mode): "modules_json", "multiqc_config", "modules_structure", + "local_component_structure", "base_config", "modules_config", "nfcore_yml", diff --git a/nf_core/pipelines/lint/local_component_structure.py b/nf_core/pipelines/lint/local_component_structure.py new file mode 100644 index 0000000000..e5e1a00dee --- /dev/null +++ b/nf_core/pipelines/lint/local_component_structure.py @@ -0,0 +1,31 @@ +import logging +from pathlib import Path + +log = logging.getLogger(__name__) + + +def local_component_structure(self): + """ + Check that the local modules and subworkflows directories in a pipeline have the correct format: + + .. code-block:: bash + + modules/local/TOOL/SUBTOOL + + Prior to nf-core/tools release 3.0.3 the directory structure allowed top-level `*.nf` files: + + .. code-block:: bash + + modules/local/modules/TOOL_SUBTOOL.nf + """ + warned = [] + for nf_file in Path(self.wf_path, "modules", "local").glob("*.nf"): + warned.append(f"{nf_file.name} in modules/local should be moved to a TOOL/SUBTOOL/main.nf structure") + for nf_file in Path(self.wf_path, "subworkflows", "local").glob("*.nf"): + warned.append(f"{nf_file.name} in subworkflows/local should be moved to a TOOL/SUBTOOL/main.nf structure") + + # If there are modules installed in the wrong location + passed = [] + if len(warned) == 0: + passed = ["modules directory structure is correct 'modules/nf-core/TOOL/SUBTOOL'"] + return {"passed": passed, "warned": warned, "failed": [], "ignored": []} diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 7ca825f04f..5242c28eb0 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -23,7 +23,8 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): Additionally, checks that all included components in test ``main.nf`` are specified in ``test.yml`` """ - + if subworkflow.nftest_testdir is None or subworkflow.nftest_main_nf is None: + raise ValueError() repo_dir = subworkflow.component_dir.parts[ : subworkflow.component_dir.parts.index(subworkflow.component_name.split("/")[0]) ][-1] diff --git a/tests/modules/test_create.py b/tests/modules/test_create.py index 219f869997..4ae9f9b5c4 100644 --- a/tests/modules/test_create.py +++ b/tests/modules/test_create.py @@ -31,7 +31,7 @@ def test_modules_create_succeed(self): ) with requests_cache.disabled(): module_create.create() - assert os.path.exists(os.path.join(self.pipeline_dir, "modules", "local", "trimgalore.nf")) + assert os.path.exists(os.path.join(self.pipeline_dir, "modules", "local", "trimgalore/main.nf")) def test_modules_create_fail_exists(self): """Fail at creating the same module twice""" @@ -46,7 +46,7 @@ def test_modules_create_fail_exists(self): with pytest.raises(UserWarning) as excinfo: with requests_cache.disabled(): module_create.create() - assert "Module file exists already" in str(excinfo.value) + assert "module directory exists:" in str(excinfo.value) def test_modules_create_nfcore_modules(self): """Create a module in nf-core/modules clone""" From e508ced096eb16e7400f5f8fbc7d93ebb3bd6563 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 28 Oct 2024 15:54:44 +0000 Subject: [PATCH 2/9] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2bf6fcae8..227e6748d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ - Update GitHub Actions ([#3237](https://github.com/nf-core/tools/pull/3237)) - add `--dir/-d` option to schema commands ([#3247](https://github.com/nf-core/tools/pull/3247)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.1 ([#3250](https://github.com/nf-core/tools/pull/3250)) +- fix(1929): more type-gating ([#3256](https://github.com/nf-core/tools/pull/3256)) ## [v3.0.2 - Titanium Tapir Patch](https://github.com/nf-core/tools/releases/tag/3.0.2) - [2024-10-11] From ff68ba39bd08b66a1d896b99d72e475f3a73f6b9 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:03:36 +0100 Subject: [PATCH 3/9] fix(1929): update subworkflow tests and add lint test markdown --- .../_src/pipeline_lint_tests/local_component_structure.md | 5 +++++ tests/subworkflows/test_create.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 docs/api/_src/pipeline_lint_tests/local_component_structure.md diff --git a/docs/api/_src/pipeline_lint_tests/local_component_structure.md b/docs/api/_src/pipeline_lint_tests/local_component_structure.md new file mode 100644 index 0000000000..1884d862be --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/local_component_structure.md @@ -0,0 +1,5 @@ +# modules_structure + +```{eval-rst} +.. automethod:: nf_core.pipelines.lint.PipelineLint.local_component_structure +``` diff --git a/tests/subworkflows/test_create.py b/tests/subworkflows/test_create.py index 48cb482260..704a23772e 100644 --- a/tests/subworkflows/test_create.py +++ b/tests/subworkflows/test_create.py @@ -19,7 +19,7 @@ def test_subworkflows_create_succeed(self): self.pipeline_dir, "test_subworkflow_local", "@author", True ) subworkflow_create.create() - assert Path(self.pipeline_dir, "subworkflows", "local", "test_subworkflow_local.nf").exists() + assert Path(self.pipeline_dir, "subworkflows", "local", "test_subworkflow_local/main.nf").exists() def test_subworkflows_create_fail_exists(self): """Fail at creating the same subworkflow twice""" @@ -29,7 +29,7 @@ def test_subworkflows_create_fail_exists(self): subworkflow_create.create() with pytest.raises(UserWarning) as excinfo: subworkflow_create.create() - assert "Subworkflow file exists already" in str(excinfo.value) + assert "subworkflow directory exists" in str(excinfo.value) def test_subworkflows_create_nfcore_modules(self): """Create a subworkflow in nf-core/modules clone""" From 2308e3bb567d2797d185a330caef07d9f4e7aca2 Mon Sep 17 00:00:00 2001 From: awgymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:47:50 +0100 Subject: [PATCH 4/9] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: JĂșlia Mir Pedrol --- nf_core/pipelines/lint/local_component_structure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/lint/local_component_structure.py b/nf_core/pipelines/lint/local_component_structure.py index e5e1a00dee..61c1d45e29 100644 --- a/nf_core/pipelines/lint/local_component_structure.py +++ b/nf_core/pipelines/lint/local_component_structure.py @@ -12,7 +12,7 @@ def local_component_structure(self): modules/local/TOOL/SUBTOOL - Prior to nf-core/tools release 3.0.3 the directory structure allowed top-level `*.nf` files: + Prior to nf-core/tools release 3.1.0 the directory structure allowed top-level `*.nf` files: .. code-block:: bash @@ -22,7 +22,7 @@ def local_component_structure(self): for nf_file in Path(self.wf_path, "modules", "local").glob("*.nf"): warned.append(f"{nf_file.name} in modules/local should be moved to a TOOL/SUBTOOL/main.nf structure") for nf_file in Path(self.wf_path, "subworkflows", "local").glob("*.nf"): - warned.append(f"{nf_file.name} in subworkflows/local should be moved to a TOOL/SUBTOOL/main.nf structure") + warned.append(f"{nf_file.name} in subworkflows/local should be moved to a SUBWORKFLOW_NAME/main.nf structure") # If there are modules installed in the wrong location passed = [] From 31cffdd533b948364221db70c527747b0a756682 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:17:00 +0100 Subject: [PATCH 5/9] feat(1929): allow missing files when linting local modules --- CHANGELOG.md | 4 +++- nf_core/components/lint/__init__.py | 4 ++++ nf_core/modules/lint/__init__.py | 18 +++++++++++++++ nf_core/modules/lint/environment_yml.py | 11 +++++++++- nf_core/modules/lint/meta_yml.py | 12 ++++++---- nf_core/modules/lint/module_tests.py | 29 ++++++++++++++++++++++--- 6 files changed, 69 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 227e6748d1..8ec829a0f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,9 +18,12 @@ ### Modules - add a panel around diff previews when updating ([#3246](https://github.com/nf-core/tools/pull/3246)) +- Modules created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) ### Subworkflows +- Subworkflows created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) + ### General - Include .nf-core.yml in `nf-core pipelines bump-version` ([#3220](https://github.com/nf-core/tools/pull/3220)) @@ -33,7 +36,6 @@ - Update GitHub Actions ([#3237](https://github.com/nf-core/tools/pull/3237)) - add `--dir/-d` option to schema commands ([#3247](https://github.com/nf-core/tools/pull/3247)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.1 ([#3250](https://github.com/nf-core/tools/pull/3250)) -- fix(1929): more type-gating ([#3256](https://github.com/nf-core/tools/pull/3256)) ## [v3.0.2 - Titanium Tapir Patch](https://github.com/nf-core/tools/releases/tag/3.0.2) - [2024-10-11] diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index fcc3b414d8..e434cace27 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -162,6 +162,10 @@ def _set_registry(self, registry) -> None: self.registry = registry log.debug(f"Registry set to {self.registry}") + @property + def local_module_exclude_tests(self): + return ["module_version", "module_changes", "modules_patch"] + @staticmethod def get_all_module_lint_tests(is_pipeline): if is_pipeline: diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 49012cff40..1ce9902a31 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -234,13 +234,31 @@ def lint_module( # TODO: consider unifying modules and subworkflows lint_module() function and add it to the ComponentLint class # Only check the main script in case of a local module if local: + mod.get_inputs_from_main_nf() + mod.get_outputs_from_main_nf() + # Update meta.yml file if requested + if self.fix and mod.meta_yml is not None: + self.update_meta_yml_file(mod) + + for test_name in self.lint_tests: + if test_name in self.local_module_exclude_tests: + continue + if test_name == "main_nf": + getattr(self, test_name)(mod, fix_version, self.registry, progress_bar) + elif test_name in ["meta_yml", "environment_yml"]: + # Allow files to be missing for local + getattr(self, test_name)(mod, allow_missing=True) + """ self.main_nf(mod, fix_version, self.registry, progress_bar) + """ + self.passed += [LintResult(mod, *m) for m in mod.passed] warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)] if not self.fail_warned: self.warned += warned else: self.failed += warned + self.failed += [LintResult(mod, *m) for m in mod.failed] # Otherwise run all the lint tests else: diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index 4488b0befa..6571f07ae8 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) -def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None: +def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent, allow_missing: bool = False) -> None: """ Lint an ``environment.yml`` file. @@ -23,6 +23,15 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) env_yml = None # load the environment.yml file if module.environment_yml is None: + if allow_missing: + module.warned.append( + ( + "environment_yml_exists", + "Module's `environment.yml` does not exist", + Path(module.component_dir, "environment.yml"), + ), + ) + return raise LintExceptionError("Module does not have an `environment.yml` file") try: with open(module.environment_yml) as fh: diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 4ad728d10b..9ffbfb040e 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -13,7 +13,7 @@ log = logging.getLogger(__name__) -def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None: +def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent, allow_missing: bool = False) -> None: """ Lint a ``meta.yml`` file @@ -42,7 +42,13 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None module (NFCoreComponent): The module to lint """ - + if module.meta_yml is None: + if allow_missing: + module.warned.append( + ("meta_yml_exists", "Module `meta.yml` does not exist", Path(module.component_dir, "meta.yml")) + ) + return + raise LintExceptionError("Module does not have a `meta.yml` file") # Check if we have a patch file, get original file in that case meta_yaml = read_meta_yml(module_lint_object, module) if module.is_patched and module_lint_object.modules_repo.repo_path is not None: @@ -56,8 +62,6 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None if lines is not None: yaml = ruamel.yaml.YAML() meta_yaml = yaml.safe_load("".join(lines)) - if module.meta_yml is None: - raise LintExceptionError("Module does not have a `meta.yml` file") if meta_yaml is None: module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml)) return diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 8f5fc91425..6826b2e743 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -9,12 +9,13 @@ import yaml +from nf_core.components.lint import LintExceptionError from nf_core.components.nfcore_component import NFCoreComponent log = logging.getLogger(__name__) -def module_tests(_, module: NFCoreComponent): +def module_tests(_, module: NFCoreComponent, allow_missing: bool = False): """ Lint the tests of a module in ``nf-core/modules`` @@ -22,8 +23,30 @@ def module_tests(_, module: NFCoreComponent): and contains a ``main.nf.test`` and a ``main.nf.test.snap`` """ - if module.nftest_testdir is None or module.nftest_main_nf is None: - raise ValueError() + if module.nftest_testdir is None: + if allow_missing: + module.warned.append( + ( + "test_dir_exists", + "nf-test directory is missing", + Path(module.component_dir, "tests"), + ) + ) + return + raise LintExceptionError("Module does not have a `tests` dir") + + if module.nftest_main_nf is None: + if allow_missing: + module.warned.append( + ( + "test_main_nf_exists", + "test `main.nf.test` does not exist", + Path(module.component_dir, "tests", "main.nf.test"), + ) + ) + return + raise LintExceptionError("Module does not have a `tests` dir") + repo_dir = module.component_dir.parts[: module.component_dir.parts.index(module.component_name.split("/")[0])][-1] test_dir = Path(module.base_dir, "tests", "modules", repo_dir, module.component_name) pytest_main_nf = Path(test_dir, "main.nf") From dbc8c981e96b7287c5276cf4403cde6581f204a3 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:35:33 +0100 Subject: [PATCH 6/9] feat(1929): local linting option now *only* lints local modules. Add tests --- nf_core/modules/lint/__init__.py | 7 ++- nf_core/subworkflows/lint/__init__.py | 3 +- nf_core/subworkflows/lint/meta_yml.py | 24 ++++++++--- .../subworkflows/lint/subworkflow_tests.py | 29 +++++++++++-- tests/modules/test_lint.py | 43 ++++++++++++++++++- tests/subworkflows/test_lint.py | 39 ++++++++++++++++- 6 files changed, 129 insertions(+), 16 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 1ce9902a31..af057b67fc 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -170,7 +170,7 @@ def lint( self.lint_modules(local_modules, registry=registry, local=True, fix_version=fix_version) # Lint nf-core modules - if len(remote_modules) > 0: + if not local and len(remote_modules) > 0: self.lint_modules(remote_modules, registry=registry, local=False, fix_version=fix_version) if print_results: @@ -248,9 +248,8 @@ def lint_module( elif test_name in ["meta_yml", "environment_yml"]: # Allow files to be missing for local getattr(self, test_name)(mod, allow_missing=True) - """ - self.main_nf(mod, fix_version, self.registry, progress_bar) - """ + else: + getattr(self, test_name)(mod) self.passed += [LintResult(mod, *m) for m in mod.passed] warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)] diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index cedae62f11..ee46f3f3b2 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -152,7 +152,7 @@ def lint( self.lint_subworkflows(local_subworkflows, registry=registry, local=True) # Lint nf-core subworkflows - if len(remote_subworkflows) > 0: + if not local and len(remote_subworkflows) > 0: self.lint_subworkflows(remote_subworkflows, registry=registry, local=False) if print_results: @@ -208,6 +208,7 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): # Only check the main script in case of a local subworkflow if local: self.main_nf(swf) + self.meta_yml(swf, allow_missing=True) self.passed += [LintResult(swf, *s) for s in swf.passed] warned = [LintResult(swf, *m) for m in (swf.warned + swf.failed)] if not self.fail_warned: diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index be282bc453..262c80f948 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -6,11 +6,12 @@ import yaml import nf_core.components.components_utils +from nf_core.components.lint import LintExceptionError log = logging.getLogger(__name__) -def meta_yml(subworkflow_lint_object, subworkflow): +def meta_yml(subworkflow_lint_object, subworkflow, allow_missing: bool = False): """ Lint a ``meta.yml`` file @@ -28,6 +29,18 @@ def meta_yml(subworkflow_lint_object, subworkflow): """ # Read the meta.yml file + if subworkflow.meta_yml is None: + if allow_missing: + subworkflow.warned.append( + ( + "meta_yml_exists", + "Subworkflow `meta.yml` does not exist", + Path(subworkflow.component_dir, "meta.yml"), + ) + ) + return + raise LintExceptionError("Subworkflow does not have a `meta.yml` file") + try: with open(subworkflow.meta_yml) as fh: meta_yaml = yaml.safe_load(fh) @@ -49,7 +62,7 @@ def meta_yml(subworkflow_lint_object, subworkflow): if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: - hint = f"\nCheck that the child entries of {e.path[0]+'.'+e.path[2]} are indented correctly." + hint = f"\nCheck that the child entries of {e.path[0]}.{e.path[2]} are indented correctly." subworkflow.failed.append( ( "meta_yml_valid", @@ -96,10 +109,9 @@ def meta_yml(subworkflow_lint_object, subworkflow): ) # confirm that all included components in ``main.nf`` are specified in ``meta.yml`` - included_components = nf_core.components.components_utils.get_components_to_install(subworkflow.component_dir) - included_components = ( - included_components[0] + included_components[1] - ) # join included modules and included subworkflows in a single list + included_components_ = nf_core.components.components_utils.get_components_to_install(subworkflow.component_dir) + included_components = included_components_[0] + included_components_[1] + # join included modules and included subworkflows in a single list if "components" in meta_yaml: meta_components = [x for x in meta_yaml["components"]] for component in set(included_components): diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 5242c28eb0..d216f9bedc 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -9,12 +9,13 @@ import yaml +from nf_core.components.lint import LintExceptionError from nf_core.components.nfcore_component import NFCoreComponent log = logging.getLogger(__name__) -def subworkflow_tests(_, subworkflow: NFCoreComponent): +def subworkflow_tests(_, subworkflow: NFCoreComponent, allow_missing: bool = False): """ Lint the tests of a subworkflow in ``nf-core/modules`` @@ -23,8 +24,30 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): Additionally, checks that all included components in test ``main.nf`` are specified in ``test.yml`` """ - if subworkflow.nftest_testdir is None or subworkflow.nftest_main_nf is None: - raise ValueError() + if subworkflow.nftest_testdir is None: + if allow_missing: + subworkflow.warned.append( + ( + "test_dir_exists", + "nf-test directory is missing", + Path(subworkflow.component_dir, "tests"), + ) + ) + return + raise LintExceptionError("Module does not have a `tests` dir") + + if subworkflow.nftest_main_nf is None: + if allow_missing: + subworkflow.warned.append( + ( + "test_main_nf_exists", + "test `main.nf.test` does not exist", + Path(subworkflow.component_dir, "tests", "main.nf.test"), + ) + ) + return + raise LintExceptionError("Subworkflow does not have a `tests` dir") + repo_dir = subworkflow.component_dir.parts[ : subworkflow.component_dir.parts.index(subworkflow.component_name.split("/")[0]) ][-1] diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 5372807987..37f992a7c9 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -1,4 +1,5 @@ import json +import shutil from pathlib import Path from typing import Union @@ -158,7 +159,7 @@ ] -class TestModulesCreate(TestModules): +class TestModulesLint(TestModules): def _setup_patch(self, pipeline_dir: Union[str, Path], modify_module: bool): install_obj = nf_core.modules.install.ModuleInstall( pipeline_dir, @@ -760,6 +761,46 @@ def test_modules_empty_file_in_stub_snapshot(self): with open(snap_file, "w") as fh: fh.write(content) + def test_modules_lint_local(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore") + local = Path(self.pipeline_dir, "modules", "local", "trimgalore") + shutil.move(installed, local) + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True, all_modules=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + + def test_modules_lint_local_missing_files(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore") + local = Path(self.pipeline_dir, "modules", "local", "trimgalore") + shutil.move(installed, local) + Path(self.pipeline_dir, "modules", "local", "trimgalore", "environment.yml").unlink() + Path(self.pipeline_dir, "modules", "local", "trimgalore", "meta.yml").unlink() + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True, all_modules=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + warnings = [x.message for x in module_lint.warned] + assert "Module's `environment.yml` does not exist" in warnings + assert "Module `meta.yml` does not exist" in warnings + + def test_modules_lint_local_old_format(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore", "main.nf") + Path(self.pipeline_dir, "modules", "local").mkdir() + local = Path(self.pipeline_dir, "modules", "local", "trimgalore.nf") + shutil.copy(installed, local) + self.mods_remove.remove("trimgalore", force=True) + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True, all_modules=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + # A skeleton object with the passed/warned/failed list attrs # Use this in place of a ModuleLint object to test behaviour of diff --git a/tests/subworkflows/test_lint.py b/tests/subworkflows/test_lint.py index d94b55b3d3..49c671c088 100644 --- a/tests/subworkflows/test_lint.py +++ b/tests/subworkflows/test_lint.py @@ -31,7 +31,6 @@ def test_subworkflows_lint_new_subworkflow(self): subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.nfcore_modules) subworkflow_lint.lint(print_results=True, all_subworkflows=True) assert len(subworkflow_lint.failed) == 0 - assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -397,3 +396,41 @@ def test_subworkflows_empty_file_in_stub_snapshot(self): # reset the file with open(snap_file, "w") as fh: fh.write(content) + + def test_subworkflows_lint_local(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2") + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2") + shutil.move(installed, local) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + + def test_subworkflows_lint_local_missing_files(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2") + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2") + shutil.move(installed, local) + Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2", "meta.yml").unlink() + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + warnings = [x.message for x in subworkflow_lint.warned] + assert "Subworkflow `meta.yml` does not exist" in warnings + + def test_subworkflows_lint_local_old_format(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2", "main.nf") + Path(self.pipeline_dir, "subworkflows", "local").mkdir(exist_ok=True) + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2.nf") + shutil.copy(installed, local) + self.subworkflow_remove.remove("fastq_align_bowtie2", force=True) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 From 17546b65fff55176a96e9106a87e46552fed8327 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:46:36 +0100 Subject: [PATCH 7/9] feat(1929): don't ask for a module/subworkflow name to lint if linting local --- nf_core/modules/lint/__init__.py | 3 +-- nf_core/subworkflows/lint/__init__.py | 2 +- tests/modules/test_lint.py | 6 +++--- tests/subworkflows/test_lint.py | 6 +++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index af057b67fc..5f64953116 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -117,7 +117,7 @@ def lint( """ # TODO: consider unifying modules and subworkflows lint() function and add it to the ComponentLint class # Prompt for module or all - if module is None and not all_modules and len(self.all_remote_components) > 0: + if module is None and not (local or all_modules) and len(self.all_remote_components) > 0: questions = [ { "type": "list", @@ -257,7 +257,6 @@ def lint_module( self.warned += warned else: self.failed += warned - self.failed += [LintResult(mod, *m) for m in mod.failed] # Otherwise run all the lint tests else: diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index ee46f3f3b2..8538cc3bf8 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -99,7 +99,7 @@ def lint( """ # TODO: consider unifying modules and subworkflows lint() function and add it to the ComponentLint class # Prompt for subworkflow or all - if subworkflow is None and not all_subworkflows: + if subworkflow is None and not (local or all_subworkflows): questions = [ { "type": "list", diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 37f992a7c9..38597b44a0 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -767,7 +767,7 @@ def test_modules_lint_local(self): local = Path(self.pipeline_dir, "modules", "local", "trimgalore") shutil.move(installed, local) module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) - module_lint.lint(print_results=False, local=True, all_modules=True) + module_lint.lint(print_results=False, local=True) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 @@ -780,7 +780,7 @@ def test_modules_lint_local_missing_files(self): Path(self.pipeline_dir, "modules", "local", "trimgalore", "environment.yml").unlink() Path(self.pipeline_dir, "modules", "local", "trimgalore", "meta.yml").unlink() module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) - module_lint.lint(print_results=False, local=True, all_modules=True) + module_lint.lint(print_results=False, local=True) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 @@ -796,7 +796,7 @@ def test_modules_lint_local_old_format(self): shutil.copy(installed, local) self.mods_remove.remove("trimgalore", force=True) module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) - module_lint.lint(print_results=False, local=True, all_modules=True) + module_lint.lint(print_results=False, local=True) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 diff --git a/tests/subworkflows/test_lint.py b/tests/subworkflows/test_lint.py index 49c671c088..8f6ff353a1 100644 --- a/tests/subworkflows/test_lint.py +++ b/tests/subworkflows/test_lint.py @@ -403,7 +403,7 @@ def test_subworkflows_lint_local(self): local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2") shutil.move(installed, local) subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) - subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + subworkflow_lint.lint(print_results=False, local=True) assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -415,7 +415,7 @@ def test_subworkflows_lint_local_missing_files(self): shutil.move(installed, local) Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2", "meta.yml").unlink() subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) - subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + subworkflow_lint.lint(print_results=False, local=True) assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -430,7 +430,7 @@ def test_subworkflows_lint_local_old_format(self): shutil.copy(installed, local) self.subworkflow_remove.remove("fastq_align_bowtie2", force=True) subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) - subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + subworkflow_lint.lint(print_results=False, local=True) assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 From e959d457096befe7e124af19435bbafc06e8636c Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:59:47 +0100 Subject: [PATCH 8/9] feat(1929): add test for new pipeline lint check of local component dirs --- .../lint/local_component_structure.py | 23 +++++++++------ .../lint/test_local_component_structure.py | 28 +++++++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 tests/pipelines/lint/test_local_component_structure.py diff --git a/nf_core/pipelines/lint/local_component_structure.py b/nf_core/pipelines/lint/local_component_structure.py index 61c1d45e29..71f02ba545 100644 --- a/nf_core/pipelines/lint/local_component_structure.py +++ b/nf_core/pipelines/lint/local_component_structure.py @@ -18,14 +18,21 @@ def local_component_structure(self): modules/local/modules/TOOL_SUBTOOL.nf """ - warned = [] + warned_mods = [] for nf_file in Path(self.wf_path, "modules", "local").glob("*.nf"): - warned.append(f"{nf_file.name} in modules/local should be moved to a TOOL/SUBTOOL/main.nf structure") - for nf_file in Path(self.wf_path, "subworkflows", "local").glob("*.nf"): - warned.append(f"{nf_file.name} in subworkflows/local should be moved to a SUBWORKFLOW_NAME/main.nf structure") - + warned_mods.append(f"{nf_file.name} in modules/local should be moved to a TOOL/SUBTOOL/main.nf structure") # If there are modules installed in the wrong location passed = [] - if len(warned) == 0: - passed = ["modules directory structure is correct 'modules/nf-core/TOOL/SUBTOOL'"] - return {"passed": passed, "warned": warned, "failed": [], "ignored": []} + if len(warned_mods) == 0: + passed = ["local modules directory structure is correct 'modules/local/TOOL/SUBTOOL'"] + + warned_swfs = [] + for nf_file in Path(self.wf_path, "subworkflows", "local").glob("*.nf"): + warned_swfs.append( + f"{nf_file.name} in subworkflows/local should be moved to a SUBWORKFLOW_NAME/main.nf structure" + ) + + if len(warned_swfs) == 0: + passed = ["local subworkflows directory structure is correct 'subworkflows/local/TOOL/SUBTOOL'"] + + return {"passed": passed, "warned": warned_mods + warned_swfs, "failed": [], "ignored": []} diff --git a/tests/pipelines/lint/test_local_component_structure.py b/tests/pipelines/lint/test_local_component_structure.py new file mode 100644 index 0000000000..93dc3174a3 --- /dev/null +++ b/tests/pipelines/lint/test_local_component_structure.py @@ -0,0 +1,28 @@ +from pathlib import Path + +import nf_core.pipelines.lint + +from ..test_lint import TestLint + + +class TestLintLocalComponentStructure(TestLint): + def setUp(self) -> None: + super().setUp() + self.new_pipeline = self._make_pipeline_copy() + + def test_local_component_structure(self): + local_modules = Path(self.new_pipeline, "modules", "local") + local_swf = Path(self.new_pipeline, "subworkflows", "local") + local_modules.mkdir(parents=True, exist_ok=True) + local_swf.mkdir(parents=True, exist_ok=True) + + (local_modules / "dummy_module.nf").touch() + (local_swf / "dummy_subworkflow.nf").touch() + + lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) + lint_obj._load() + + results = lint_obj.local_component_structure() + assert len(results.get("warned", [])) == 2 + assert len(results.get("failed", [])) == 0 + assert len(results.get("ignored", [])) == 0 From e7d58cb17767715b85e41523106087def6de53c1 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 30 Oct 2024 10:04:31 +0100 Subject: [PATCH 9/9] feat(1929): run todo linting on local subworkflows --- nf_core/subworkflows/lint/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index 8538cc3bf8..c1be26d0d5 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -209,6 +209,7 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): if local: self.main_nf(swf) self.meta_yml(swf, allow_missing=True) + self.subworkflow_todos(swf) self.passed += [LintResult(swf, *s) for s in swf.passed] warned = [LintResult(swf, *m) for m in (swf.warned + swf.failed)] if not self.fail_warned: