Merge branch 'dev' into run-prettier-on-sync

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.2
+    rev: v0.8.6
     hooks:
       - id: ruff # linter
         args: [--fix, --exit-non-zero-on-fix] # sort imports and fix
@@ -19,7 +19,7 @@ repos:
         alias: ec
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.13.0"
+    rev: "v1.14.1"
     hooks:
       - id: mypy
         additional_dependencies:

CHANGELOG.md

@@ -8,11 +8,14 @@
 
 ### Modules
 
+- Fix bump-versions: only append module name if it is a dir and contains main.nf ([#3384](https://github.com/nf-core/tools/pull/3384))
+
 ### Subworkflows
 
 ### General
 
-- run pre-comit when rendering template for pipelines sync
+- Parameters schema validation: allow oneOf, anyOf and allOf with `required` ([#3386](https://github.com/nf-core/tools/pull/3386))
+- Run pre-comit when rendering template for pipelines sync ([#3371](https://github.com/nf-core/tools/pull/3371))
 
 ### Version updates
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12-slim@sha256:2b0079146a74e23bf4ae8f6a28e1b484c6292f6fb904cbb51825b4a19812fcd8
+FROM python:3.12-slim@sha256:10f3aaab98db50cba827d3b33a91f39dc9ec2d02ca9b85cbc5008220d07b17f3
 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \
     description="Docker image containing requirements for nf-core/tools"
 

nf_core/modules/modules_utils.py

@@ -65,19 +65,20 @@ def get_installed_modules(directory: Path, repo_type="modules") -> Tuple[List[st
             local_modules = sorted([x for x in local_modules if x.endswith(".nf")])
 
     # Get nf-core modules
-    if os.path.exists(nfcore_modules_dir):
-        for m in sorted([m for m in os.listdir(nfcore_modules_dir) if not m == "lib"]):
-            if not os.path.isdir(os.path.join(nfcore_modules_dir, m)):
+    if nfcore_modules_dir.exists():
+        for m in sorted([m for m in nfcore_modules_dir.iterdir() if not m == "lib"]):
+            if not m.is_dir():
                 raise ModuleExceptionError(
                     f"File found in '{nfcore_modules_dir}': '{m}'! This directory should only contain module directories."
                 )
-            m_content = os.listdir(os.path.join(nfcore_modules_dir, m))
+            m_content = [d.name for d in m.iterdir()]
             # Not a module, but contains sub-modules
             if "main.nf" not in m_content:
                 for tool in m_content:
-                    nfcore_modules_names.append(os.path.join(m, tool))
+                    if (m / tool).is_dir() and "main.nf" in [d.name for d in (m / tool).iterdir()]:
+                        nfcore_modules_names.append(str(Path(m.name, tool)))
             else:
-                nfcore_modules_names.append(m)
+                nfcore_modules_names.append(m.name)
 
     # Make full (relative) file paths and create NFCoreComponent objects
     if local_modules_dir:

nf_core/pipeline-template/.github/workflows/ci.yml

@@ -46,6 +46,8 @@ jobs:
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          fetch-depth: 0
 
       - name: Set up Nextflow
         uses: nf-core/setup-nextflow@v2

nf_core/pipelines/download.py

@@ -1536,7 +1536,7 @@ def singularity_pull_image(
 
         progress.remove_task(task)
 
-    def compress_download(self) -> None:
+    def compress_download(self):
         """Take the downloaded files and make a compressed .tar.gz archive."""
         log.debug(f"Creating archive: {self.output_filename}")
 

nf_core/pipelines/schema.py

@@ -327,11 +327,32 @@ def validate_default_params(self):
             schema_no_required = copy.deepcopy(self.schema)
             if "required" in schema_no_required:
                 schema_no_required.pop("required")
+            for keyword in ["allOf", "anyOf", "oneOf"]:
+                if keyword in schema_no_required:
+                    for i, kw_content in enumerate(schema_no_required[keyword]):
+                        if "required" in kw_content:
+                            schema_no_required[keyword][i].pop("required")
+                    schema_no_required[keyword] = [
+                        kw_content for kw_content in schema_no_required[keyword] if kw_content
+                    ]
+                    if not schema_no_required[keyword]:
+                        schema_no_required.pop(keyword)
             for group_key, group in schema_no_required.get(self.defs_notation, {}).items():
                 if "required" in group:
                     schema_no_required[self.defs_notation][group_key].pop("required")
+                for keyword in ["allOf", "anyOf", "oneOf"]:
+                    if keyword in group:
+                        for i, kw_content in enumerate(group[keyword]):
+                            if "required" in kw_content:
+                                schema_no_required[self.defs_notation][group_key][keyword][i].pop("required")
+                        schema_no_required[self.defs_notation][group_key][keyword] = [
+                            kw_content for kw_content in group[keyword] if kw_content
+                        ]
+                        if not group[keyword]:
+                            schema_no_required[self.defs_notation][group_key].pop(keyword)
             jsonschema.validate(self.schema_defaults, schema_no_required)
         except jsonschema.exceptions.ValidationError as e:
+            log.debug(f"Complete error message:\n{e}")
             raise AssertionError(f"Default parameters are invalid: {e.message}")
         for param, default in self.schema_defaults.items():
             if default in ("null", "", None, "None") or default is False:

setup.py

@@ -2,7 +2,7 @@
 
 from setuptools import find_packages, setup
 
-version = "3.1.2"
+version = "3.1.2dev"
 
 with open("README.md") as f:
     readme = f.read()

tests/modules/test_modules_utils.py

@@ -0,0 +1,20 @@
+import nf_core.modules.modules_utils
+
+from ..test_modules import TestModules
+
+
+class TestModulesUtils(TestModules):
+    def test_get_installed_modules(self):
+        """Test getting installed modules"""
+        _, nfcore_modules = nf_core.modules.modules_utils.get_installed_modules(self.nfcore_modules)
+        assert len(nfcore_modules) == 1
+        assert nfcore_modules[0].component_name == "bpipe/test"
+
+    def test_get_installed_modules_with_files(self):
+        """Test getting installed modules. When a module contains a file in its directory, it shouldn't be picked up as a tool/subtool"""
+        # Create a file in the module directory
+        with open(self.nfcore_modules / "modules" / "nf-core" / "bpipe" / "test_file.txt", "w") as fh:
+            fh.write("test")
+
+        _, nfcore_modules = nf_core.modules.modules_utils.get_installed_modules(self.nfcore_modules)
+        assert len(nfcore_modules) == 1

tests/pipelines/test_schema.py

@@ -285,6 +285,89 @@ def test_remove_schema_notfound_configs_childschema(self):
         assert len(params_removed) == 1
         assert "foo" in params_removed
 
+    def test_validate_defaults(self):
+        """Test validating default values"""
+        self.schema_obj.schema = {
+            "properties": {"foo": {"type": "string"}, "bar": {"type": "string"}},
+            "required": ["foo"],
+        }
+        self.schema_obj.schema_defaults = {"foo": "foo", "bar": "bar"}
+        self.schema_obj.no_prompts = True
+        try:
+            self.schema_obj.validate_default_params()
+        except AssertionError:
+            self.fail("Error validating schema defaults")
+
+    def test_validate_defaults_required(self):
+        """Test validating default values when required params don't have a default"""
+        self.schema_obj.schema = {
+            "properties": {"foo": {"type": "string"}, "bar": {"type": "string"}},
+            "required": ["foo"],
+        }
+        self.schema_obj.schema_defaults = {}
+        self.schema_obj.no_prompts = True
+        try:
+            self.schema_obj.validate_default_params()
+        except AssertionError:
+            self.fail("Error validating schema defaults")
+
+    def test_validate_defaults_required_inside_group(self):
+        """Test validating default values when required params don't have a default, inside a group"""
+        self.schema_obj.schema = {
+            "$defs": {
+                "subSchemaId": {
+                    "properties": {"foo": {"type": "string"}, "bar": {"type": "string"}},
+                    "required": ["foo"],
+                },
+            }
+        }
+        self.schema_obj.schema_defaults = {}
+        self.schema_obj.no_prompts = True
+        try:
+            self.schema_obj.validate_default_params()
+        except AssertionError:
+            self.fail("Error validating schema defaults")
+
+    def test_validate_defaults_required_inside_group_with_anyof(self):
+        """Test validating default values when required params don't have a default, inside a group with anyOf"""
+        self.schema_obj.schema = {
+            "$defs": {
+                "subSchemaId": {
+                    "anyOf": [{"required": ["foo"]}, {"required": ["bar"]}],
+                    "properties": {"foo": {"type": "string"}, "bar": {"type": "string"}},
+                },
+            }
+        }
+        self.schema_obj.schema_defaults = {}
+        self.schema_obj.no_prompts = True
+        try:
+            self.schema_obj.validate_default_params()
+        except AssertionError:
+            self.fail("Error validating schema defaults")
+
+    def test_validate_defaults_required_with_anyof(self):
+        """Test validating default values when required params don't have a default, with anyOf"""
+        self.schema_obj.schema = {
+            "properties": {"foo": {"type": "string"}, "bar": {"type": "string"}, "baz": {"type": "string"}},
+            "anyOf": [{"required": ["foo"]}, {"required": ["bar"]}],
+        }
+        self.schema_obj.schema_defaults = {"baz": "baz"}
+        self.schema_obj.no_prompts = True
+        try:
+            self.schema_obj.validate_default_params()
+        except AssertionError:
+            self.fail("Error validating schema defaults")
+
+    def test_validate_defaults_error(self):
+        """Test validating default raises an exception when a default is not valid"""
+        self.schema_obj.schema = {
+            "properties": {"foo": {"type": "string"}},
+        }
+        self.schema_obj.schema_defaults = {"foo": 1}
+        self.schema_obj.no_prompts = True
+        with self.assertRaises(AssertionError):
+            self.schema_obj.validate_default_params()
+
     def test_add_schema_found_configs(self):
         """Try adding a new parameter to the schema from the config"""
         self.schema_obj.pipeline_params = {"foo": "bar"}