Fix usage of cache path and implement parallized testing

.github/actions/create-lint-wf/action.yml

@@ -105,10 +105,10 @@ runs:
 
     - name: nf-core modules list remote
       shell: bash
-      run: nf-core --log-file log.txt modules list remote
+      run: nf-core --log-file log.txt modules list remote --dir nf-core-testpipeline/
       working-directory: create-lint-wf
 
     - name: nf-core modules list remote gitlab
       shell: bash
-      run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote
+      run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote --dir nf-core-testpipeline/
       working-directory: create-lint-wf

.github/workflows/pytest.yml

@@ -72,13 +72,19 @@ jobs:
       tests: ${{ steps.list_tests.outputs.tests }}
 
   test:
-    name: Run ${{matrix.test}} with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }}
+    name: Run ${{matrix.tests.test}} with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }}
     needs: [setup, list_tests]
     if: ${{ needs.setup.outputs.run-tests }}
     # run on self-hosted runners for test_components.py (because of the gitlab branch), based on the input if it is dispatched manually, on github if it is a rerun or on self-hosted by default
-    runs-on: ${{ matrix.test == 'test_components.py' && 'self-hosted' || (github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted') }}
+    runs-on: ${{ matrix.tests.test == 'test_components.py' && 'self-hosted' || (github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted') }}
     strategy:
-      matrix: ${{ fromJson(needs.list_tests.outputs.tests) }}
+      matrix:
+        tests: ${{ fromJson(needs.list_tests.outputs.tests) }}
+        xdist: [""]
+        include:
+          - tests: "test_components.py"
+            xdist: "-n auto --maxfail 1"
+
       fail-fast: false # run all tests even if one fails
     steps:
       - name: go to subdirectory and change nextflow workdir
@@ -111,7 +117,7 @@ jobs:
           sudo apt install -y git
 
       - name: Set up Singularity
-        if: ${{ matrix.test == 'test_download.py'}}
+        if: ${{ matrix.tests.test == 'test_download.py'}}
         uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7
         with:
           singularity-version: 3.8.3
@@ -132,7 +138,7 @@ jobs:
 
       - name: Test with pytest
         run: |
-          python3 -m pytest tests/${{matrix.test}} --color=yes --cov --durations=0 && exit_code=0|| exit_code=$?
+          python3 -m pytest tests/${{matrix.tests.test}} --color=yes --cov --durations=0 ${{matrix.xdist}} && exit_code=0|| exit_code=$?
           # don't fail if no tests were collected, e.g. for test_licence.py
           if [ "${exit_code}" -eq 5 ]; then
             echo "No tests were collected"
@@ -144,7 +150,7 @@ jobs:
 
       - name: remove slashes from test name
         run: |
-          test=$(echo ${{ matrix.test }} | sed 's/\//__/g')
+          test=$(echo ${{ matrix.tests.test }} | sed 's/\//__/g')
           echo "test=${test}" >> $GITHUB_ENV
 
       - name: Store snapshot report

CHANGELOG.md

@@ -40,6 +40,7 @@
 - Update GitHub Actions ([#3237](https://github.com/nf-core/tools/pull/3237))
 - add `--dir/-d` option to schema commands ([#3247](https://github.com/nf-core/tools/pull/3247))
 - Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.1 ([#3250](https://github.com/nf-core/tools/pull/3250))
+- Fix usage of cache path and implement parallized testing ([#3291](https://github.com/nf-core/tools/pull/3291))
 - handle new schema structure in `nf-core pipelines create-params-file` ([#3276](https://github.com/nf-core/tools/pull/3276))
 - Update Gitpod image to use Miniforge instead of Miniconda([#3274](https://github.com/nf-core/tools/pull/3274))
 - Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.3 ([#3275](https://github.com/nf-core/tools/pull/3275))

nf_core/__main__.py

@@ -115,6 +115,10 @@
     "nf-core modules list local": [{"options": ["--dir", "--json", "--help"]}],
 }
 
+click.rich_click.OPTION_GROUPS = {
+    "nf-core modules list remote": [{"options": ["--dir", "--json", "--help"]}],
+}
+
 # Set up rich stderr console
 stderr = rich.console.Console(stderr=True, force_terminal=rich_force_colors())
 stdout = rich.console.Console(force_terminal=rich_force_colors())
@@ -874,11 +878,19 @@ def modules_list(ctx):
 @click.pass_context
 @click.argument("keywords", required=False, nargs=-1, metavar="<filter keywords>")
 @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout")
-def command_modules_list_remote(ctx, keywords, json):
+@click.option(
+    "-d",
+    "--dir",
+    "directory",
+    type=click.Path(exists=True),
+    default=".",
+    help=r"Pipeline directory. [dim]\[default: Current working directory][/]",
+)
+def command_modules_list_remote(ctx, keywords, json, directory):  # pylint: disable=redefined-builtin
     """
     List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/].
     """
-    modules_list_remote(ctx, keywords, json)
+    modules_list_remote(ctx, keywords, json, directory)
 
 
 # nf-core modules list local
@@ -1432,11 +1444,19 @@ def subworkflows_list(ctx):
 @click.pass_context
 @click.argument("keywords", required=False, nargs=-1, metavar="<filter keywords>")
 @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout")
-def command_subworkflows_list_remote(ctx, keywords, json):
+@click.option(
+    "-d",
+    "--dir",
+    "directory",
+    type=click.Path(exists=True),
+    default=".",
+    help=r"Pipeline directory. [dim]\[default: Current working directory][/]",
+)
+def command_subworkflows_list_remote(ctx, keywords, json, directory):  # pylint: disable=redefined-builtin
     """
     List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/].
     """
-    subworkflows_list_remote(ctx, keywords, json)
+    subworkflows_list_remote(ctx, keywords, json, directory)
 
 
 # nf-core subworkflows list local

nf_core/commands_modules.py

@@ -9,19 +9,19 @@
 stdout = rich.console.Console(force_terminal=rich_force_colors())
 
 
-def modules_list_remote(ctx, keywords, json):
+def modules_list_remote(ctx, keywords, json, directory):  # pylint: disable=redefined-builtin
     """
     List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/].
     """
     from nf_core.modules.list import ModuleList
 
     try:
         module_list = ModuleList(
-            ".",
-            True,
-            ctx.obj["modules_repo_url"],
-            ctx.obj["modules_repo_branch"],
-            ctx.obj["modules_repo_no_pull"],
+            directory=directory,
+            remote=True,
+            remote_url=ctx.obj["modules_repo_url"],
+            branch=ctx.obj["modules_repo_branch"],
+            no_pull=ctx.obj["modules_repo_no_pull"],
         )
         stdout.print(module_list.list_components(keywords, json))
     except (UserWarning, LookupError) as e:
@@ -37,11 +37,11 @@ def modules_list_local(ctx, keywords, json, directory):  # pylint: disable=redef
 
     try:
         module_list = ModuleList(
-            directory,
-            False,
-            ctx.obj["modules_repo_url"],
-            ctx.obj["modules_repo_branch"],
-            ctx.obj["modules_repo_no_pull"],
+            directory=directory,
+            remote=False,
+            remote_url=ctx.obj["modules_repo_url"],
+            branch=ctx.obj["modules_repo_branch"],
+            no_pull=ctx.obj["modules_repo_no_pull"],
         )
         stdout.print(module_list.list_components(keywords, json))
     except (UserWarning, LookupError) as e:

nf_core/commands_subworkflows.py

@@ -63,19 +63,19 @@ def subworkflows_test(ctx, subworkflow, directory, no_prompts, update, once, pro
         sys.exit(1)
 
 
-def subworkflows_list_remote(ctx, keywords, json):
+def subworkflows_list_remote(ctx, keywords, json, directory):
     """
     List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/].
     """
     from nf_core.subworkflows import SubworkflowList
 
     try:
         subworkflow_list = SubworkflowList(
-            ".",
-            True,
-            ctx.obj["modules_repo_url"],
-            ctx.obj["modules_repo_branch"],
-            ctx.obj["modules_repo_no_pull"],
+            directory=directory,
+            remote=True,
+            modules_repo_url=ctx.obj["modules_repo_url"],
+            modules_repo_branch=ctx.obj["modules_repo_branch"],
+            modules_repo_no_pull=ctx.obj["modules_repo_no_pull"],
         )
 
         stdout.print(subworkflow_list.list_components(keywords, json))
@@ -92,11 +92,11 @@ def subworkflows_list_local(ctx, keywords, json, directory):  # pylint: disable=
 
     try:
         subworkflow_list = SubworkflowList(
-            directory,
-            False,
-            ctx.obj["modules_repo_url"],
-            ctx.obj["modules_repo_branch"],
-            ctx.obj["modules_repo_no_pull"],
+            directory=directory,
+            remote=False,
+            modules_repo_url=ctx.obj["modules_repo_url"],
+            modules_repo_branch=ctx.obj["modules_repo_branch"],
+            modules_repo_no_pull=ctx.obj["modules_repo_no_pull"],
         )
         stdout.print(subworkflow_list.list_components(keywords, json))
     except (UserWarning, LookupError) as e:

nf_core/components/components_command.py

@@ -6,6 +6,7 @@
 from typing import Dict, List, Optional, Union
 
 import nf_core.utils
+from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.modules_repo import ModulesRepo
 
@@ -33,7 +34,9 @@ def __init__(
         Initialise the ComponentClass object
         """
         self.component_type: str = component_type
-        self.directory: Path = Path(directory)
+        self.directory: Path = Path(directory).absolute()
+        if remote_url is None:
+            remote_url = NF_CORE_MODULES_REMOTE
         self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress)
         self.hide_progress: bool = hide_progress
         self.no_prompts: bool = no_prompts
@@ -60,10 +63,10 @@ def _configure_repo_and_paths(self, nf_dir_req: bool = True) -> None:
         except FileNotFoundError:
             raise
 
-        self.default_modules_path = Path("modules", self.org)
-        self.default_tests_path = Path("tests", "modules", self.org)
-        self.default_subworkflows_path = Path("subworkflows", self.org)
-        self.default_subworkflows_tests_path = Path("tests", "subworkflows", self.org)
+        self.default_modules_path = Path("modules", self.org).absolute()
+        self.default_tests_path = Path("tests", "modules", self.org).absolute()
+        self.default_subworkflows_path = Path("subworkflows", self.org).absolute()
+        self.default_subworkflows_tests_path = Path("tests", "subworkflows", self.org).absolute()
 
     def get_local_components(self) -> List[str]:
         """
@@ -212,6 +215,8 @@ def check_modules_structure(self) -> None:
                 self.modules_repo.setup_local_repo(
                     self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress
                 )
+                if self.modules_repo.repo_path is None:
+                    raise ValueError("Modules repo path is None")
                 # Move wrong modules to the right directory
                 for module in wrong_location_modules:
                     modules_dir = Path("modules").resolve()

nf_core/components/components_test.py

@@ -122,9 +122,11 @@ def check_inputs(self) -> None:
                     ).unsafe_ask()
                 except LookupError:
                     raise
-
-        self.component_dir = Path(self.component_type, self.modules_repo.repo_path, *self.component_name.split("/"))
-
+        if self.modules_repo.repo_path is None:
+            raise ValueError("modules_repo.repo_path is None")
+        self.component_dir = Path(
+            self.component_type, self.modules_repo.repo_path, *self.component_name.split("/")
+        ).absolute()
         # First, sanity check that the module directory exists
         if not Path(self.directory, self.component_dir).is_dir():
             raise UserWarning(
@@ -194,7 +196,6 @@ def generate_snapshot(self) -> bool:
         self.update = False  # reset self.update to False to test if the new snapshot is stable
         tag = f"subworkflows/{self.component_name}" if self.component_type == "subworkflows" else self.component_name
         profile = self.profile if self.profile else os.environ["PROFILE"]
-
         result = nf_core.utils.run_cmd(
             "nf-test",
             f"test --tag {tag} --profile {profile} {verbose} {update}",
@@ -208,14 +209,16 @@ def generate_snapshot(self) -> bool:
             obsolete_snapshots = compiled_pattern.search(nftest_out.decode())
             if obsolete_snapshots:
                 self.obsolete_snapshots = True
-
             # check if nf-test was successful
             if "Assertion failed:" in nftest_out.decode():
                 return False
             elif "no valid tests found." in nftest_out.decode():
                 log.error("Test file 'main.nf.test' not found")
                 self.errors.append("Test file 'main.nf.test' not found")
                 return False
+            elif "No tests to execute" in nftest_out.decode():
+                log.debug("No tests to execute")  # no tests to execute is not an error anymore in nf-test v0.9.2
+                return True
             else:
                 log.debug("nf-test successful")
                 return True

nf_core/components/create.py

@@ -132,7 +132,7 @@ def create(self) -> bool:
 
         if self.subtool:
             self.component_name = f"{self.component}/{self.subtool}"
-            self.component_dir = Path(self.component, self.subtool)
+            self.component_dir = Path(self.component, self.subtool).absolute()
 
         self.component_name_underscore = self.component_name.replace("/", "_")
 

nf_core/components/list.py

@@ -1,6 +1,5 @@
 import json
 import logging
-from pathlib import Path
 from typing import Dict, List, Optional, Union, cast
 
 import rich.table
@@ -13,17 +12,9 @@
 
 
 class ComponentList(ComponentCommand):
-    def __init__(
-        self,
-        component_type: str,
-        pipeline_dir: Union[str, Path] = ".",
-        remote: bool = True,
-        remote_url: Optional[str] = None,
-        branch: Optional[str] = None,
-        no_pull: bool = False,
-    ) -> None:
+    def __init__(self, component_type: str, remote: bool = True, **kwargs) -> None:
         self.remote = remote
-        super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull)
+        super().__init__(component_type, **kwargs)
 
     def _configure_repo_and_paths(self, nf_dir_req: bool = True) -> None:
         """

nf_core/components/update.py

@@ -414,7 +414,7 @@ def get_single_component_info(self, component):
                 config_entry = self.update_config[self.modules_repo.remote_url][install_dir].get(component)
         if config_entry is not None and config_entry is not True:
             if config_entry is False:
-                log.warn(
+                log.warning(
                     f"{self.component_type[:-1].title()}'s update entry in '.nf-core.yml' for '{component}' is set to False"
                 )
                 return (self.modules_repo, None, None, None)

nf_core/modules/bump_versions.py

@@ -19,7 +19,6 @@
 from rich.table import Table
 
 import nf_core.modules.modules_utils
-import nf_core.utils
 from nf_core.components.components_command import ComponentCommand
 from nf_core.components.nfcore_component import NFCoreComponent
 from nf_core.utils import NFCoreYamlConfig, custom_yaml_dumper, rich_force_colors
@@ -29,14 +28,8 @@
 
 
 class ModuleVersionBumper(ComponentCommand):
-    def __init__(
-        self,
-        pipeline_dir: Union[str, Path],
-        remote_url: Optional[str] = None,
-        branch: Optional[str] = None,
-        no_pull: bool = False,
-    ):
-        super().__init__("modules", pipeline_dir, remote_url, branch, no_pull)
+    def __init__(self, **kwargs) -> None:
+        super().__init__("modules", **kwargs)
 
         self.up_to_date: List[Tuple[str, str]] = []
         self.updated: List[Tuple[str, str]] = []