Merge pull request #2897 from nf-core/remove-old-customdumpsoftwareve…

…rsions

Remove old references to CUSTOMDUMPSOFTWAREVERSIONS

CHANGELOG.md

@@ -48,6 +48,7 @@
 - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 ([#2894](https://github.com/nf-core/tools/pull/2894))
 - Update GitHub Actions ([#2902](https://github.com/nf-core/tools/pull/2902))
 - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.5 ([#2903](https://github.com/nf-core/tools/pull/2903))
+- Remove old references to CUSTOMDUMPSOFTWAREVERSIONS and add linting checks ([#2897](https://github.com/nf-core/tools/pull/2897))
 
 ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29]
 

docs/api/_src/pipeline_lint_tests/base_config.md

@@ -0,0 +1,5 @@
+# base_config
+
+```{eval-rst}
+.. automethod:: nf_core.lint.PipelineLint.base_config
+```

docs/api/_src/pipeline_lint_tests/modules_config.md

@@ -0,0 +1,5 @@
+# modules_config
+
+```{eval-rst}
+.. automethod:: nf_core.lint.PipelineLint.modules_config
+```

nf_core/lint/__init__.py

@@ -58,6 +58,7 @@ class PipelineLint(nf_core.utils.Pipeline):
     from .actions_schema_validation import (  # type: ignore[misc]
         actions_schema_validation,
     )
+    from .configs import base_config, modules_config  # type: ignore[misc]
     from .files_exist import files_exist  # type: ignore[misc]
     from .files_unchanged import files_unchanged  # type: ignore[misc]
     from .merge_markers import merge_markers  # type: ignore[misc]
@@ -124,6 +125,8 @@ def _get_all_lint_tests(release_mode):
             "modules_json",
             "multiqc_config",
             "modules_structure",
+            "base_config",
+            "modules_config",
             "nfcore_yml",
         ] + (["version_consistency"] if release_mode else [])
 

nf_core/lint/configs.py

@@ -0,0 +1,101 @@
+import logging
+import re
+from pathlib import Path
+from typing import Dict, List
+
+from nf_core.lint_utils import ignore_file
+
+log = logging.getLogger(__name__)
+
+
+class LintConfig:
+    def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]):
+        self.wf_path = wf_path
+        self.lint_config = lint_config
+
+    def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]:
+        """Lint a file and add the result to the passed or failed list."""
+
+        passed: List[str] = []
+        failed: List[str] = []
+        ignored: List[str] = []
+        ignore_configs: List[str] = []
+
+        fn = Path(self.wf_path, file_path)
+
+        passed, failed, ignored, ignore_configs = ignore_file(lint_name, file_path, Path(self.wf_path))
+
+        error_message = f"`{file_path}` not found"
+        # check for partial match in failed or ignored
+        if not any(f.startswith(error_message) for f in (failed + ignored)):
+            try:
+                with open(fn) as fh:
+                    config = fh.read()
+            except Exception as e:
+                return {"failed": [f"Could not parse file: {fn}, {e}"]}
+
+            # find sections with a withName: prefix
+            sections = re.findall(r"withName:\s*['\"]?(\w+)['\"]?", config)
+            log.debug(f"found sections: {sections}")
+
+            # find all .nf files in the workflow directory
+            nf_files = list(Path(self.wf_path).rglob("*.nf"))
+            log.debug(f"found nf_files: {[str(f) for f in nf_files]}")
+
+            # check if withName sections are present in config, but not in workflow files
+            for section in sections:
+                if section not in ignore_configs or section.lower() not in ignore_configs:
+                    if not any(section in nf_file.read_text() for nf_file in nf_files):
+                        failed.append(
+                            f"`{file_path}` contains `withName:{section}`, but the corresponding process is not present in any of the Nextflow scripts."
+                        )
+                    else:
+                        passed.append(f"`{section}` found in `{file_path}` and Nextflow scripts.")
+                else:
+                    ignored.append(f"``{section}` is ignored.")
+
+        return {"passed": passed, "failed": failed, "ignored": ignored}
+
+
+def modules_config(self) -> Dict[str, List[str]]:
+    """Make sure the conf/modules.config file follows the nf-core template, especially removed sections.
+
+    .. note:: You can choose to ignore this lint tests by editing the file called
+        ``.nf-core.yml`` in the root of your pipeline and setting the test to false:
+
+        .. code-block:: yaml
+
+            lint:
+                modules_config: False
+
+        To disable this test only for specific modules, you can specify a list of module names.
+
+        .. code-block:: yaml
+
+            lint:
+                modules_config:
+                    - fastqc
+
+    """
+
+    result = LintConfig(self.wf_path, self.lint_config).lint_file("modules_config", Path("conf", "modules.config"))
+
+    return result
+
+
+def base_config(self) -> Dict[str, List[str]]:
+    """Make sure the conf/base.config file follows the nf-core template, especially removed sections.
+
+    .. note:: You can choose to ignore this lint tests by editing the file called
+        ``.nf-core.yml`` in the root of your pipeline and setting the test to false:
+
+        .. code-block:: yaml
+
+            lint:
+                base_config: False
+
+    """
+
+    result = LintConfig(self.wf_path, self.lint_config).lint_file("base_config", Path("conf", "base.config"))
+
+    return result

nf_core/lint/multiqc_config.py

@@ -1,8 +1,10 @@
-import os
+from pathlib import Path
 from typing import Dict, List
 
 import yaml
 
+from nf_core.lint_utils import ignore_file
+
 
 def multiqc_config(self) -> Dict[str, List[str]]:
     """Make sure basic multiQC plugins are installed and plots are exported
@@ -21,100 +23,108 @@ def multiqc_config(self) -> Dict[str, List[str]]:
                 order: -1001
         export_plots: true
 
-    """
-
-    passed: List[str] = []
-    failed: List[str] = []
-
-    # Remove field that should be ignored according to the linting config
-    ignore_configs = self.lint_config.get("multiqc_config", [])
+    .. note:: You can choose to ignore this lint tests by editing the file called
+        ``.nf-core.yml`` in the root of your pipeline and setting the test to false:
 
-    fn = os.path.join(self.wf_path, "assets", "multiqc_config.yml")
+        .. code-block:: yaml
 
-    # Return a failed status if we can't find the file
-    if not os.path.isfile(fn):
-        return {"ignored": ["'assets/multiqc_config.yml' not found"]}
+            lint:
+                multiqc_config: False
 
-    try:
-        with open(fn) as fh:
-            mqc_yml = yaml.safe_load(fh)
-    except Exception as e:
-        return {"failed": [f"Could not parse yaml file: {fn}, {e}"]}
-
-    # check if requried sections are present
-    required_sections = ["report_section_order", "export_plots", "report_comment"]
-    for section in required_sections:
-        if section not in mqc_yml and section not in ignore_configs:
-            failed.append(f"'assets/multiqc_config.yml' does not contain `{section}`")
-            return {"passed": passed, "failed": failed}
-        else:
-            passed.append(f"'assets/multiqc_config.yml' contains `{section}`")
-
-    try:
-        orders = {}
-        summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary"
-        min_plugins = ["software_versions", summary_plugin_name]
-        for plugin in min_plugins:
-            if plugin not in mqc_yml["report_section_order"]:
-                raise AssertionError(f"Section {plugin} missing in report_section_order")
-            if "order" not in mqc_yml["report_section_order"][plugin]:
-                raise AssertionError(f"Section {plugin} 'order' missing. Must be < 0")
-            plugin_order = mqc_yml["report_section_order"][plugin]["order"]
-            if plugin_order >= 0:
-                raise AssertionError(f"Section {plugin} 'order' must be < 0")
-
-        for plugin in mqc_yml["report_section_order"]:
-            if "order" in mqc_yml["report_section_order"][plugin]:
-                orders[plugin] = mqc_yml["report_section_order"][plugin]["order"]
-
-        if orders[summary_plugin_name] != min(orders.values()):
-            raise AssertionError(f"Section {summary_plugin_name} should have the lowest order")
-        orders.pop(summary_plugin_name)
-        if orders["software_versions"] != min(orders.values()):
-            raise AssertionError("Section software_versions should have the second lowest order")
-    except (AssertionError, KeyError, TypeError) as e:
-        failed.append(f"'assets/multiqc_config.yml' does not meet requirements: {e}")
-    else:
-        passed.append("'assets/multiqc_config.yml' follows the ordering scheme of the minimally required plugins.")
-
-    if "report_comment" not in ignore_configs:
-        # Check that the minimum plugins exist and are coming first in the summary
-        version = self.nf_config.get("manifest.version", "").strip(" '\"")
-        if "dev" in version:
-            version = "dev"
-            report_comments = (
-                f'This report has been generated by the <a href="https://github.com/nf-core/{self.pipeline_name}/tree/dev" target="_blank">nf-core/{self.pipeline_name}</a>'
-                f" analysis pipeline. For information about how to interpret these results, please see the "
-                f'<a href="https://nf-co.re/{self.pipeline_name}/dev/docs/output" target="_blank">documentation</a>.'
-            )
+    """
 
+    passed: List[str] = []
+    failed: List[str] = []
+    ignored: List[str] = []
+
+    fn = Path(self.wf_path, "assets", "multiqc_config.yml")
+    file_path = fn.relative_to(self.wf_path)
+    passed, failed, ignored, ignore_configs = ignore_file("multiqc_config", file_path, self.wf_path)
+
+    # skip other tests if the file is not found
+    error_message = f"`{file_path}` not found"
+    # check for partial match in failed or ignored
+    if not any(f.startswith(error_message) for f in (failed + ignored)):
+        try:
+            with open(fn) as fh:
+                mqc_yml = yaml.safe_load(fh)
+        except Exception as e:
+            return {"failed": [f"Could not parse yaml file: {fn}, {e}"]}
+
+        # check if required sections are present
+        required_sections = ["report_section_order", "export_plots", "report_comment"]
+        for section in required_sections:
+            if section not in mqc_yml and section not in ignore_configs:
+                failed.append(f"`assets/multiqc_config.yml` does not contain `{section}`")
+                return {"passed": passed, "failed": failed}
+            else:
+                passed.append(f"`assets/multiqc_config.yml` contains `{section}`")
+
+        try:
+            orders = {}
+            summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary"
+            min_plugins = ["software_versions", summary_plugin_name]
+            for plugin in min_plugins:
+                if plugin not in mqc_yml["report_section_order"]:
+                    raise AssertionError(f"Section {plugin} missing in report_section_order")
+                if "order" not in mqc_yml["report_section_order"][plugin]:
+                    raise AssertionError(f"Section {plugin} 'order' missing. Must be < 0")
+                plugin_order = mqc_yml["report_section_order"][plugin]["order"]
+                if plugin_order >= 0:
+                    raise AssertionError(f"Section {plugin} 'order' must be < 0")
+
+            for plugin in mqc_yml["report_section_order"]:
+                if "order" in mqc_yml["report_section_order"][plugin]:
+                    orders[plugin] = mqc_yml["report_section_order"][plugin]["order"]
+
+            if orders[summary_plugin_name] != min(orders.values()):
+                raise AssertionError(f"Section {summary_plugin_name} should have the lowest order")
+            orders.pop(summary_plugin_name)
+            if orders["software_versions"] != min(orders.values()):
+                raise AssertionError("Section software_versions should have the second lowest order")
+        except (AssertionError, KeyError, TypeError) as e:
+            failed.append(f"`assets/multiqc_config.yml` does not meet requirements: {e}")
         else:
-            report_comments = (
-                f'This report has been generated by the <a href="https://github.com/nf-core/{self.pipeline_name}/releases/tag/{version}" target="_blank">nf-core/{self.pipeline_name}</a>'
-                f" analysis pipeline. For information about how to interpret these results, please see the "
-                f'<a href="https://nf-co.re/{self.pipeline_name}/{version}/docs/output" target="_blank">documentation</a>.'
-            )
-
-        if mqc_yml["report_comment"].strip() != report_comments:
-            # find where the report_comment is wrong and give it as a hint
-            hint = report_comments
-            failed.append(
-                f"'assets/multiqc_config.yml' does not contain a matching 'report_comment'.  \n"
-                f"The expected comment is:  \n"
-                f"```{hint}```  \n"
-                f"The current comment is:  \n"
-                f"```{ mqc_yml['report_comment'].strip()}```"
-            )
+            passed.append("`assets/multiqc_config.yml` follows the ordering scheme of the minimally required plugins.")
+
+        if "report_comment" not in ignore_configs:
+            # Check that the minimum plugins exist and are coming first in the summary
+            version = self.nf_config.get("manifest.version", "").strip(" '\"")
+            if "dev" in version:
+                version = "dev"
+                report_comments = (
+                    f'This report has been generated by the <a href="https://github.com/nf-core/{self.pipeline_name}/tree/dev" target="_blank">nf-core/{self.pipeline_name}</a>'
+                    f" analysis pipeline. For information about how to interpret these results, please see the "
+                    f'<a href="https://nf-co.re/{self.pipeline_name}/dev/docs/output" target="_blank">documentation</a>.'
+                )
+
+            else:
+                report_comments = (
+                    f'This report has been generated by the <a href="https://github.com/nf-core/{self.pipeline_name}/releases/tag/{version}" target="_blank">nf-core/{self.pipeline_name}</a>'
+                    f" analysis pipeline. For information about how to interpret these results, please see the "
+                    f'<a href="https://nf-co.re/{self.pipeline_name}/{version}/docs/output" target="_blank">documentation</a>.'
+                )
+
+            if mqc_yml["report_comment"].strip() != report_comments:
+                # find where the report_comment is wrong and give it as a hint
+                hint = report_comments
+                failed.append(
+                    f"`assets/multiqc_config.yml` does not contain a matching 'report_comment'.  \n"
+                    f"The expected comment is:  \n"
+                    f"```{hint}```  \n"
+                    f"The current comment is:  \n"
+                    f"```{ mqc_yml['report_comment'].strip()}```"
+                )
+            else:
+                passed.append("`assets/multiqc_config.yml` contains a matching 'report_comment'.")
+
+        # Check that export_plots is activated
+        try:
+            if not mqc_yml["export_plots"]:
+                raise AssertionError()
+        except (AssertionError, KeyError, TypeError):
+            failed.append("`assets/multiqc_config.yml` does not contain 'export_plots: true'.")
         else:
-            passed.append("'assets/multiqc_config.yml' contains a matching 'report_comment'.")
-
-    # Check that export_plots is activated
-    try:
-        if not mqc_yml["export_plots"]:
-            raise AssertionError()
-    except (AssertionError, KeyError, TypeError):
-        failed.append("'assets/multiqc_config.yml' does not contain 'export_plots: true'.")
-    else:
-        passed.append("'assets/multiqc_config.yml' contains 'export_plots: true'.")
-
-    return {"passed": passed, "failed": failed}
+            passed.append("`assets/multiqc_config.yml` contains 'export_plots: true'.")
+
+    return {"passed": passed, "failed": failed, "ignored": ignored}

nf_core/lint_utils.py

@@ -2,6 +2,7 @@
 import logging
 import subprocess
 from pathlib import Path
+from typing import List
 
 import rich
 from rich.console import Console
@@ -101,3 +102,30 @@ def dump_json_with_prettier(file_name, file_content):
     with open(file_name, "w") as fh:
         json.dump(file_content, fh, indent=4)
     run_prettier_on_file(file_name)
+
+
+def ignore_file(lint_name: str, file_path: Path, dir_path: Path) -> List[List[str]]:
+    """Ignore a file and add the result to the ignored list. Return the passed, failed, ignored and ignore_configs lists."""
+
+    passed: List[str] = []
+    failed: List[str] = []
+    ignored: List[str] = []
+    _, lint_conf = nf_core.utils.load_tools_config(dir_path)
+    lint_conf = lint_conf.get("lint", {})
+    ignore_entry: List[str] | bool = lint_conf.get(lint_name, [])
+    full_path = dir_path / file_path
+    # Return a failed status if we can't find the file
+    if not full_path.is_file():
+        if isinstance(ignore_entry, bool) and not ignore_entry:
+            ignored.append(f"`{file_path}` not found, but it is ignored.")
+            ignore_entry = []
+        else:
+            failed.append(f"`{file_path}` not found.")
+    else:
+        passed.append(f"`{file_path}` found and not ignored.")
+
+    # we handled the only case where ignore_entry should be a bool, convert it to a list, to make downstream code easier
+    if isinstance(ignore_entry, bool):
+        ignore_entry = []
+
+    return [passed, failed, ignored, ignore_entry]

nf_core/pipeline-template/conf/base.config

@@ -59,7 +59,4 @@ process {
         errorStrategy = 'retry'
         maxRetries    = 2
     }
-    withName:CUSTOM_DUMPSOFTWAREVERSIONS {
-        cache = false
-    }
 }