From 2348c699bcebcb2df2cfc96b93c679b719ab48ff Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Mon, 11 Nov 2024 17:03:04 +0100
Subject: [PATCH 01/22] rename cache paths from `nfcore` to `nf-core`

---
 nf_core/utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/nf_core/utils.py b/nf_core/utils.py
index 16125aed33..b86a7653f3 100644
--- a/nf_core/utils.py
+++ b/nf_core/utils.py
@@ -67,10 +67,10 @@
 )
 
 NFCORE_CACHE_DIR = Path(
-    os.environ.get("XDG_CACHE_HOME", Path(os.getenv("HOME") or "", ".cache")),
-    "nfcore",
+    os.getenv("XDG_CACHE_HOME", Path(os.getenv("HOME") or "", ".cache")),
+    "nf-core",
 )
-NFCORE_DIR = Path(os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME") or "", ".config")), "nfcore")
+NFCORE_DIR = Path(os.getenv("XDG_CONFIG_HOME", Path(os.getenv("HOME") or "", ".config")), "nf-core")
 
 
 def fetch_remote_version(source_url):

From e6a6dbf4090ed8474dd9bfddb81fec979a882582 Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Thu, 21 Nov 2024 13:49:29 +0100
Subject: [PATCH 02/22] use always absolute paths, set default value for
 remote_url

---
 nf_core/components/components_command.py | 15 ++++++++-----
 nf_core/components/components_test.py    | 13 ++++++-----
 nf_core/components/create.py             |  2 +-
 nf_core/modules/modules_repo.py          | 28 +++++++++++-------------
 nf_core/synced_repo.py                   | 16 +++++++-------
 nf_core/utils.py                         |  5 ++++-
 6 files changed, 44 insertions(+), 35 deletions(-)

diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py
index f25fb33a6f..10937d1271 100644
--- a/nf_core/components/components_command.py
+++ b/nf_core/components/components_command.py
@@ -6,6 +6,7 @@
 from typing import Dict, List, Optional, Union
 
 import nf_core.utils
+from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.modules_repo import ModulesRepo
 
@@ -33,7 +34,9 @@ def __init__(
         Initialise the ComponentClass object
         """
         self.component_type: str = component_type
-        self.directory: Path = Path(directory)
+        self.directory: Path = Path(directory).absolute()
+        if remote_url is None:
+            remote_url = NF_CORE_MODULES_REMOTE
         self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress)
         self.hide_progress: bool = hide_progress
         self.no_prompts: bool = no_prompts
@@ -60,10 +63,10 @@ def _configure_repo_and_paths(self, nf_dir_req: bool = True) -> None:
         except FileNotFoundError:
             raise
 
-        self.default_modules_path = Path("modules", self.org)
-        self.default_tests_path = Path("tests", "modules", self.org)
-        self.default_subworkflows_path = Path("subworkflows", self.org)
-        self.default_subworkflows_tests_path = Path("tests", "subworkflows", self.org)
+        self.default_modules_path = Path("modules", self.org).absolute()
+        self.default_tests_path = Path("tests", "modules", self.org).absolute()
+        self.default_subworkflows_path = Path("subworkflows", self.org).absolute()
+        self.default_subworkflows_tests_path = Path("tests", "subworkflows", self.org).absolute()
 
     def get_local_components(self) -> List[str]:
         """
@@ -212,6 +215,8 @@ def check_modules_structure(self) -> None:
                 self.modules_repo.setup_local_repo(
                     self.modules_repo.remote_url, self.modules_repo.branch, self.hide_progress
                 )
+                if self.modules_repo.repo_path is None:
+                    raise ValueError("Modules repo path is None")
                 # Move wrong modules to the right directory
                 for module in wrong_location_modules:
                     modules_dir = Path("modules").resolve()
diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py
index 57c0034ba4..76e49a029b 100644
--- a/nf_core/components/components_test.py
+++ b/nf_core/components/components_test.py
@@ -122,9 +122,11 @@ def check_inputs(self) -> None:
                     ).unsafe_ask()
                 except LookupError:
                     raise
-
-        self.component_dir = Path(self.component_type, self.modules_repo.repo_path, *self.component_name.split("/"))
-
+        if self.modules_repo.repo_path is None:
+            raise ValueError("modules_repo.repo_path is None")
+        self.component_dir = Path(
+            self.component_type, self.modules_repo.repo_path, *self.component_name.split("/")
+        ).absolute()
         # First, sanity check that the module directory exists
         if not Path(self.directory, self.component_dir).is_dir():
             raise UserWarning(
@@ -194,7 +196,6 @@ def generate_snapshot(self) -> bool:
         self.update = False  # reset self.update to False to test if the new snapshot is stable
         tag = f"subworkflows/{self.component_name}" if self.component_type == "subworkflows" else self.component_name
         profile = self.profile if self.profile else os.environ["PROFILE"]
-
         result = nf_core.utils.run_cmd(
             "nf-test",
             f"test --tag {tag} --profile {profile} {verbose} {update}",
@@ -208,7 +209,6 @@ def generate_snapshot(self) -> bool:
             obsolete_snapshots = compiled_pattern.search(nftest_out.decode())
             if obsolete_snapshots:
                 self.obsolete_snapshots = True
-
             # check if nf-test was successful
             if "Assertion failed:" in nftest_out.decode():
                 return False
@@ -216,6 +216,9 @@ def generate_snapshot(self) -> bool:
                 log.error("Test file 'main.nf.test' not found")
                 self.errors.append("Test file 'main.nf.test' not found")
                 return False
+            elif "No tests to execute" in nftest_out.decode():
+                log.debug("No tests to execute")  # no tests to execute is not an error anymore in nf-test v0.9.2
+                return True
             else:
                 log.debug("nf-test successful")
                 return True
diff --git a/nf_core/components/create.py b/nf_core/components/create.py
index c781905618..e1ec330248 100644
--- a/nf_core/components/create.py
+++ b/nf_core/components/create.py
@@ -132,7 +132,7 @@ def create(self) -> bool:
 
         if self.subtool:
             self.component_name = f"{self.component}/{self.subtool}"
-            self.component_dir = Path(self.component, self.subtool)
+            self.component_dir = Path(self.component, self.subtool).absolute()
 
         self.component_name_underscore = self.component_name.replace("/", "_")
 
diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py
index 357fc49cc5..641399d804 100644
--- a/nf_core/modules/modules_repo.py
+++ b/nf_core/modules/modules_repo.py
@@ -47,16 +47,11 @@ def __init__(
 
         # This allows us to set this one time and then keep track of the user's choice
         ModulesRepo.no_pull_global |= no_pull
-
-        # Check if the remote seems to be well formed
-        if remote_url is None:
-            remote_url = NF_CORE_MODULES_REMOTE
-
-        self.remote_url = remote_url
+        self.remote_url = remote_url or NF_CORE_MODULES_REMOTE
 
         self.fullname = nf_core.modules.modules_utils.repo_full_name_from_remote(self.remote_url)
 
-        self.setup_local_repo(remote_url, branch, hide_progress)
+        self.setup_local_repo(self.remote_url, branch, hide_progress)
 
         config_fn, repo_config = load_tools_config(self.local_repo_dir)
         if config_fn is None or repo_config is None:
@@ -83,7 +78,9 @@ def gitless_repo(self):
             gitless_repo_url = gitless_repo_url[:-4]
         return gitless_repo_url
 
-    def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False):
+    def setup_local_repo(
+        self, remote_url: str, branch: Optional[str] = None, hide_progress: bool = True, in_cache: bool = False
+    ) -> None:
         """
         Sets up the local git repository. If the repository has been cloned previously, it
         returns a git.Repo object of that clone. Otherwise it tries to clone the repository from
@@ -95,8 +92,9 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False):
         Sets self.repo
         """
         self.local_repo_dir = Path(NFCORE_DIR if not in_cache else NFCORE_CACHE_DIR, self.fullname)
+
         try:
-            if not os.path.exists(self.local_repo_dir):
+            if not self.local_repo_dir.exists():
                 try:
                     pbar = rich.progress.Progress(
                         "[bold blue]{task.description}",
@@ -107,13 +105,13 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False):
                     )
                     with pbar:
                         self.repo = git.Repo.clone_from(
-                            remote,
+                            remote_url,
                             self.local_repo_dir,
-                            progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Cloning"),
+                            progress=RemoteProgressbar(pbar, self.fullname, remote_url, "Cloning"),
                         )
                     ModulesRepo.update_local_repo_status(self.fullname, True)
                 except GitCommandError:
-                    raise LookupError(f"Failed to clone from the remote: `{remote}`")
+                    raise LookupError(f"Failed to clone from the remote: `{remote_url}`")
                 # Verify that the requested branch exists by checking it out
                 self.setup_branch(branch)
             else:
@@ -132,7 +130,7 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False):
                     )
                     with pbar:
                         self.repo.remotes.origin.fetch(
-                            progress=RemoteProgressbar(pbar, self.fullname, self.remote_url, "Pulling")
+                            progress=RemoteProgressbar(pbar, self.fullname, remote_url, "Pulling")
                         )
                     ModulesRepo.update_local_repo_status(self.fullname, True)
 
@@ -143,13 +141,13 @@ def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False):
                 # Now merge the changes
                 tracking_branch = self.repo.active_branch.tracking_branch()
                 if tracking_branch is None:
-                    raise LookupError(f"There is no remote tracking branch '{self.branch}' in '{self.remote_url}'")
+                    raise LookupError(f"There is no remote tracking branch '{self.branch}' in '{remote_url}'")
                 self.repo.git.merge(tracking_branch.name)
         except (GitCommandError, InvalidGitRepositoryError) as e:
             log.error(f"[red]Could not set up local cache of modules repository:[/]\n{e}\n")
             if rich.prompt.Confirm.ask(f"[violet]Delete local cache '{self.local_repo_dir}' and try again?"):
                 log.info(f"Removing '{self.local_repo_dir}'")
                 shutil.rmtree(self.local_repo_dir)
-                self.setup_local_repo(remote, branch, hide_progress)
+                self.setup_local_repo(remote_url, branch, hide_progress)
             else:
                 raise LookupError("Exiting due to error with local modules git repo")
diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py
index e2a76ccaeb..ca0a3d0d60 100644
--- a/nf_core/synced_repo.py
+++ b/nf_core/synced_repo.py
@@ -175,7 +175,7 @@ def verify_sha(self, prompt, sha):
 
         return True
 
-    def setup_branch(self, branch):
+    def setup_branch(self, branch: Optional[str] = None) -> None:
         """
         Verify that we have a branch and otherwise use the default one.
         The branch is then checked out to verify that it exists in the repo.
@@ -195,7 +195,7 @@ def setup_branch(self, branch):
         # Verify that the branch exists by checking it out
         self.branch_exists()
 
-    def get_default_branch(self):
+    def get_default_branch(self) -> str:
         """
         Gets the default branch for the repo (the branch origin/HEAD is pointing to)
         """
@@ -203,16 +203,16 @@ def get_default_branch(self):
         _, branch = origin_head.ref.name.split("/")
         return branch
 
-    def branch_exists(self):
+    def branch_exists(self) -> None:
         """
         Verifies that the branch exists in the repository by trying to check it out
         """
         try:
             self.checkout_branch()
-        except GitCommandError:
-            raise LookupError(f"Branch '{self.branch}' not found in '{self.remote_url}'")
+        except GitCommandError as e:
+            raise LookupError(e.stderr)
 
-    def verify_branch(self):
+    def verify_branch(self) -> None:
         """
         Verifies the active branch conforms to the correct directory structure
         """
@@ -225,7 +225,7 @@ def verify_branch(self):
                 )
             raise LookupError(err_str)
 
-    def checkout_branch(self):
+    def checkout_branch(self) -> None:
         """
         Checks out the specified branch of the repository
         """
@@ -242,7 +242,7 @@ def checkout_branch(self):
             else:
                 raise e
 
-    def checkout(self, commit):
+    def checkout(self, commit: str) -> None:
         """
         Checks out the repository at the requested commit
 
diff --git a/nf_core/utils.py b/nf_core/utils.py
index b86a7653f3..bbce707a55 100644
--- a/nf_core/utils.py
+++ b/nf_core/utils.py
@@ -305,6 +305,7 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict:
             try:
                 k, v = ul.split(" = ", 1)
                 config[k] = v.strip("'\"")
+                log.debug(f"Added config key:value pair: {k}={v}")
             except ValueError:
                 log.debug(f"Couldn't find key=value config pair:\n  {ul}")
 
@@ -1360,8 +1361,10 @@ def set_wd(path: Path) -> Generator[None, None, None]:
         Path to the working directory to be used inside this context.
     """
     start_wd = Path().absolute()
-    os.chdir(Path(path).resolve())
     try:
+        os.chdir(Path(path).resolve())
         yield
+    except Exception as e:
+        log.error(f"Error setting working directory to {path}: {e}")
     finally:
         os.chdir(start_wd)

From 0e5b8493f0141e40150c6a26f2ebcf5a6e6b2dbd Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Thu, 21 Nov 2024 13:52:13 +0100
Subject: [PATCH 03/22] simplify initialization of sub-commands

---
 nf_core/components/list.py       | 13 ++----------
 nf_core/modules/bump_versions.py | 11 ++---------
 nf_core/modules/lint/__init__.py | 34 +++++++-------------------------
 nf_core/modules/list.py          | 13 ++----------
 nf_core/subworkflows/list.py     | 13 ++----------
 nf_core/subworkflows/update.py   | 32 ++----------------------------
 6 files changed, 17 insertions(+), 99 deletions(-)

diff --git a/nf_core/components/list.py b/nf_core/components/list.py
index 4c20e60864..7c4ba048d4 100644
--- a/nf_core/components/list.py
+++ b/nf_core/components/list.py
@@ -1,6 +1,5 @@
 import json
 import logging
-from pathlib import Path
 from typing import Dict, List, Optional, Union, cast
 
 import rich.table
@@ -13,17 +12,9 @@
 
 
 class ComponentList(ComponentCommand):
-    def __init__(
-        self,
-        component_type: str,
-        pipeline_dir: Union[str, Path] = ".",
-        remote: bool = True,
-        remote_url: Optional[str] = None,
-        branch: Optional[str] = None,
-        no_pull: bool = False,
-    ) -> None:
+    def __init__(self, component_type: str, remote: bool = True, **kwargs) -> None:
         self.remote = remote
-        super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull)
+        super().__init__(component_type, **kwargs)
 
     def _configure_repo_and_paths(self, nf_dir_req: bool = True) -> None:
         """
diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py
index d98eac7cd6..0bfb8657c4 100644
--- a/nf_core/modules/bump_versions.py
+++ b/nf_core/modules/bump_versions.py
@@ -19,7 +19,6 @@
 from rich.table import Table
 
 import nf_core.modules.modules_utils
-import nf_core.utils
 from nf_core.components.components_command import ComponentCommand
 from nf_core.components.nfcore_component import NFCoreComponent
 from nf_core.utils import NFCoreYamlConfig, custom_yaml_dumper, rich_force_colors
@@ -29,14 +28,8 @@
 
 
 class ModuleVersionBumper(ComponentCommand):
-    def __init__(
-        self,
-        pipeline_dir: Union[str, Path],
-        remote_url: Optional[str] = None,
-        branch: Optional[str] = None,
-        no_pull: bool = False,
-    ):
-        super().__init__("modules", pipeline_dir, remote_url, branch, no_pull)
+    def __init__(self, **kwargs) -> None:
+        super().__init__("modules", **kwargs)
 
         self.up_to_date: List[Tuple[str, str]] = []
         self.updated: List[Tuple[str, str]] = []
diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py
index 49012cff40..5e1d7c1a74 100644
--- a/nf_core/modules/lint/__init__.py
+++ b/nf_core/modules/lint/__init__.py
@@ -20,7 +20,7 @@
 import nf_core.components.nfcore_component
 import nf_core.modules.modules_utils
 import nf_core.utils
-from nf_core.components.components_utils import get_biotools_id
+from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE, get_biotools_id
 from nf_core.components.lint import ComponentLint, LintExceptionError, LintResult
 from nf_core.components.nfcore_component import NFCoreComponent
 from nf_core.pipelines.lint_utils import console, run_prettier_on_file
@@ -58,28 +58,8 @@ class ModuleLint(ComponentLint):
     module_todos = module_todos
     module_version = module_version
 
-    def __init__(
-        self,
-        directory: Union[str, Path],
-        fail_warned: bool = False,
-        fix: bool = False,
-        remote_url: Optional[str] = None,
-        branch: Optional[str] = None,
-        no_pull: bool = False,
-        registry: Optional[str] = None,
-        hide_progress: bool = False,
-    ):
-        super().__init__(
-            component_type="modules",
-            directory=directory,
-            fail_warned=fail_warned,
-            fix=fix,
-            remote_url=remote_url,
-            branch=branch,
-            no_pull=no_pull,
-            registry=registry,
-            hide_progress=hide_progress,
-        )
+    def __init__(self, directory, **kwargs) -> None:
+        super().__init__("modules", directory=directory, **kwargs)
 
     def lint(
         self,
@@ -312,11 +292,11 @@ def update_meta_yml_file(self, mod):
                             for x, meta_ch_element in enumerate(meta_element):
                                 if element_name in meta_ch_element.keys():
                                     # Copy current features of that input element form meta.yml
-                                    for feature in meta_element[x][element_name].keys():
+                                    for feature in meta_ch_element[x][element_name].keys():
                                         if feature not in element[element_name].keys():
-                                            corrected_meta_yml["input"][i][j][element_name][feature] = meta_element[x][
-                                                element_name
-                                            ][feature]
+                                            corrected_meta_yml["input"][i][j][element_name][feature] = meta_ch_element[
+                                                x
+                                            ][element_name][feature]
                                     break
 
         if "output" in meta_yml and correct_outputs != meta_outputs:
diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py
index 68da570f67..85ba2dfb40 100644
--- a/nf_core/modules/list.py
+++ b/nf_core/modules/list.py
@@ -1,6 +1,4 @@
 import logging
-from pathlib import Path
-from typing import Optional, Union
 
 from nf_core.components.list import ComponentList
 
@@ -8,12 +6,5 @@
 
 
 class ModuleList(ComponentList):
-    def __init__(
-        self,
-        pipeline_dir: Union[str, Path] = ".",
-        remote: bool = True,
-        remote_url: Optional[str] = None,
-        branch: Optional[str] = None,
-        no_pull: bool = False,
-    ):
-        super().__init__("modules", pipeline_dir, remote, remote_url, branch, no_pull)
+    def __init__(self, **kwargs) -> None:
+        super().__init__("modules", **kwargs)
diff --git a/nf_core/subworkflows/list.py b/nf_core/subworkflows/list.py
index 9e84d6cbe0..5668fe7528 100644
--- a/nf_core/subworkflows/list.py
+++ b/nf_core/subworkflows/list.py
@@ -1,6 +1,4 @@
 import logging
-from pathlib import Path
-from typing import Optional, Union
 
 from nf_core.components.list import ComponentList
 
@@ -8,12 +6,5 @@
 
 
 class SubworkflowList(ComponentList):
-    def __init__(
-        self,
-        pipeline_dir: Union[str, Path] = ".",
-        remote: bool = True,
-        remote_url: Optional[str] = None,
-        branch: Optional[str] = None,
-        no_pull: bool = False,
-    ) -> None:
-        super().__init__("subworkflows", pipeline_dir, remote, remote_url, branch, no_pull)
+    def __init__(self, **kwargs) -> None:
+        super().__init__("subworkflows", **kwargs)
diff --git a/nf_core/subworkflows/update.py b/nf_core/subworkflows/update.py
index 9b6bf16928..70f00cc748 100644
--- a/nf_core/subworkflows/update.py
+++ b/nf_core/subworkflows/update.py
@@ -2,33 +2,5 @@
 
 
 class SubworkflowUpdate(ComponentUpdate):
-    def __init__(
-        self,
-        pipeline_dir,
-        force=False,
-        prompt=False,
-        sha=None,
-        update_all=False,
-        show_diff=None,
-        save_diff_fn=None,
-        update_deps=False,
-        remote_url=None,
-        branch=None,
-        no_pull=False,
-        limit_output=False,
-    ):
-        super().__init__(
-            pipeline_dir,
-            "subworkflows",
-            force,
-            prompt,
-            sha,
-            update_all,
-            show_diff,
-            save_diff_fn,
-            update_deps,
-            remote_url,
-            branch,
-            no_pull,
-            limit_output,
-        )
+    def __init__(self, pipeline_dir, **kwargs) -> None:
+        super().__init__(pipeline_dir, "subworkflows", **kwargs)

From f5b14f1bbc7b5d0c5e32cdc460c56872d5370339 Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Thu, 21 Nov 2024 13:53:00 +0100
Subject: [PATCH 04/22] setup temporary module cache to allow parallelized
 testing

---
 tests/components/generate_snapshot.py |  2 +-
 tests/test_components.py              | 12 ++++++------
 tests/test_modules.py                 |  5 +++++
 tests/test_pipelines.py               |  4 ----
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py
index a5a8eaba39..aebe97b3c9 100644
--- a/tests/components/generate_snapshot.py
+++ b/tests/components/generate_snapshot.py
@@ -120,7 +120,7 @@ def test_test_not_found(self):
             remote_url=GITLAB_URL,
             branch=GITLAB_NFTEST_BRANCH,
         )
-        test_file = Path("modules", "nf-core-test", "fastp", "tests", "main.nf.test")
+        test_file = Path(snap_generator.default_modules_path, snap_generator.component_name, "tests", "main.nf.test")
         test_file.rename(test_file.parent / "main.nf.test.bak")
         with pytest.raises(UserWarning) as e:
             snap_generator.run()
diff --git a/tests/test_components.py b/tests/test_components.py
index eaf999c3c3..43d11827e3 100644
--- a/tests/test_components.py
+++ b/tests/test_components.py
@@ -8,6 +8,8 @@
 
 from git.repo import Repo
 
+# needs to be run before .utils import otherwise NFCORE_DIR is not set to a temp dir
+os.environ["XDG_CONFIG_HOME"] = str(Path(tempfile.mkdtemp()))
 from .utils import GITLAB_NFTEST_BRANCH, GITLAB_URL
 
 
@@ -16,9 +18,8 @@ class TestComponents(unittest.TestCase):
 
     def setUp(self):
         """Clone a testing version the nf-core/modules repo"""
-        self.tmp_dir = Path(tempfile.mkdtemp())
-        self.nfcore_modules = Path(self.tmp_dir, "modules-test")
-
+        self.nfcore_modules = Path(tempfile.mkdtemp(), "modules-test")
+        self.nfcore_modules.mkdir(parents=True, exist_ok=True)
         Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_NFTEST_BRANCH)
 
         # Set $PROFILE environment variable to docker - tests will run with Docker
@@ -28,9 +29,8 @@ def setUp(self):
     def tearDown(self):
         """Clean up temporary files and folders"""
 
-        # Clean up temporary files
-        if self.tmp_dir.is_dir():
-            shutil.rmtree(self.tmp_dir)
+        if self.nfcore_modules.exists():
+            shutil.rmtree(self.nfcore_modules)
 
     ############################################
     # Test of the individual components commands. #
diff --git a/tests/test_modules.py b/tests/test_modules.py
index d0692236e8..c4e14d5063 100644
--- a/tests/test_modules.py
+++ b/tests/test_modules.py
@@ -1,6 +1,8 @@
 """Tests covering the modules commands"""
 
 import json
+import os
+import tempfile
 import unittest
 from pathlib import Path
 
@@ -19,6 +21,8 @@
 from nf_core.pipelines.lint_utils import run_prettier_on_file
 from nf_core.utils import NFCoreYamlConfig
 
+# needs to be run before .utils import otherwise NFCORE_DIR is not set to a temp dir
+os.environ["XDG_CONFIG_HOME"] = str(Path(tempfile.mkdtemp()))
 from .utils import (
     GITLAB_BRANCH_TEST_BRANCH,
     GITLAB_BRANCH_TEST_OLD_SHA,
@@ -35,6 +39,7 @@
 
 def create_modules_repo_dummy(tmp_dir):
     """Create a dummy copy of the nf-core/modules repo"""
+
     yaml = ruamel.yaml.YAML()
     yaml.preserve_quotes = True
     yaml.indent(mapping=2, sequence=2, offset=0)
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 656ccbef55..df159091d9 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -13,10 +13,6 @@ def setUp(self) -> None:
         self.pipeline_obj = Pipeline(self.pipeline_dir)
         self.pipeline_obj._load()
 
-    def tearDown(self) -> None:
-        """Remove the test pipeline directory"""
-        shutil.rmtree(self.tmp_dir)
-
     def _make_pipeline_copy(self):
         """Make a copy of the test pipeline that can be edited
 

From f30110bdf386afe6a179f123fda23a183782a3ec Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Thu, 21 Nov 2024 13:56:29 +0100
Subject: [PATCH 05/22] add pytest-xdist to parallelize pytests

---
 .github/workflows/pytest.yml | 2 +-
 requirements-dev.txt         | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
index 90bf384b78..f313d8f8dc 100644
--- a/.github/workflows/pytest.yml
+++ b/.github/workflows/pytest.yml
@@ -132,7 +132,7 @@ jobs:
 
       - name: Test with pytest
         run: |
-          python3 -m pytest tests/${{matrix.test}} --color=yes --cov --durations=0 && exit_code=0|| exit_code=$?
+          python3 -m pytest tests/${{matrix.test}} --color=yes --cov --durations=0 -n auto && exit_code=0|| exit_code=$?
           # don't fail if no tests were collected, e.g. for test_licence.py
           if [ "${exit_code}" -eq 5 ]; then
             echo "No tests were collected"
diff --git a/requirements-dev.txt b/requirements-dev.txt
index aab9b1e5d7..1909c40637 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -18,5 +18,7 @@ typing_extensions >=4.0.0
 pytest-asyncio
 pytest-textual-snapshot==1.0.0
 pytest-workflow>=2.0.0
+pytest-xdist
 pytest>=8.0.0
 ruff
+

From 656aa6e05441a217ff89dca0519310ebca11c572 Mon Sep 17 00:00:00 2001
From: nf-core-bot <core@nf-co.re>
Date: Thu, 21 Nov 2024 12:58:51 +0000
Subject: [PATCH 06/22] [automated] Update CHANGELOG.md

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 61d8dbf518..61859224a9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -35,6 +35,7 @@
 - Update GitHub Actions ([#3237](https://github.com/nf-core/tools/pull/3237))
 - add `--dir/-d` option to schema commands ([#3247](https://github.com/nf-core/tools/pull/3247))
 - Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.1 ([#3250](https://github.com/nf-core/tools/pull/3250))
+- Fix usage of cache path and implement parallized testing ([#3291](https://github.com/nf-core/tools/pull/3291))
 
 ## [v3.0.2 - Titanium Tapir Patch](https://github.com/nf-core/tools/releases/tag/3.0.2) - [2024-10-11]
 

From db836f13d4cb69d7ebcac1ad4f3700bbdeab507a Mon Sep 17 00:00:00 2001
From: mirpedrol <mirp.julia@gmail.com>
Date: Thu, 21 Nov 2024 16:10:42 +0100
Subject: [PATCH 07/22] give keyword arguments to ModuleList

---
 nf_core/commands_modules.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py
index 33b1f75160..bf046a60a6 100644
--- a/nf_core/commands_modules.py
+++ b/nf_core/commands_modules.py
@@ -17,11 +17,11 @@ def modules_list_remote(ctx, keywords, json):
 
     try:
         module_list = ModuleList(
-            ".",
-            True,
-            ctx.obj["modules_repo_url"],
-            ctx.obj["modules_repo_branch"],
-            ctx.obj["modules_repo_no_pull"],
+            directory=".",
+            remote=True,
+            remote_url=ctx.obj["modules_repo_url"],
+            branch=ctx.obj["modules_repo_branch"],
+            no_pull=ctx.obj["modules_repo_no_pull"],
         )
         stdout.print(module_list.list_components(keywords, json))
     except (UserWarning, LookupError) as e:
@@ -37,11 +37,11 @@ def modules_list_local(ctx, keywords, json, directory):  # pylint: disable=redef
 
     try:
         module_list = ModuleList(
-            directory,
-            False,
-            ctx.obj["modules_repo_url"],
-            ctx.obj["modules_repo_branch"],
-            ctx.obj["modules_repo_no_pull"],
+            directory=directory,
+            remote=False,
+            remote_url=ctx.obj["modules_repo_url"],
+            branch=ctx.obj["modules_repo_branch"],
+            no_pull=ctx.obj["modules_repo_no_pull"],
         )
         stdout.print(module_list.list_components(keywords, json))
     except (UserWarning, LookupError) as e:

From 4e30db1085aa11e8cf33d341b968f76f9b7d9d34 Mon Sep 17 00:00:00 2001
From: mirpedrol <mirp.julia@gmail.com>
Date: Thu, 21 Nov 2024 16:35:42 +0100
Subject: [PATCH 08/22] test modules list remote from directory

---
 .github/actions/create-lint-wf/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml
index 6ad6b7b157..95424e8b78 100644
--- a/.github/actions/create-lint-wf/action.yml
+++ b/.github/actions/create-lint-wf/action.yml
@@ -105,7 +105,7 @@ runs:
 
     - name: nf-core modules list remote
       shell: bash
-      run: nf-core --log-file log.txt modules list remote
+      run: nf-core --log-file log.txt modules list remote --dir nf-core-testpipeline/
       working-directory: create-lint-wf
 
     - name: nf-core modules list remote gitlab

From 697f97abcc32f3e06acdae8dc8ee22201d238247 Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Wed, 27 Nov 2024 11:58:46 +0100
Subject: [PATCH 09/22] resolve asyncio warning

---
 pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pyproject.toml b/pyproject.toml
index 775f04c9a1..7a7144debf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -17,6 +17,7 @@ norecursedirs = [
     "nf_core",
     "docs",
 ]
+asyncio_default_fixture_loop_scope = "loop"
 
 [tool.ruff]
 line-length = 120

From b12f0fe71c3f9fb7ae7133008412194b4a4165c0 Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Wed, 27 Nov 2024 12:06:36 +0100
Subject: [PATCH 10/22] handle existing request cache more gracefully

---
 nf_core/utils.py | 37 ++++++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/nf_core/utils.py b/nf_core/utils.py
index bbce707a55..5fca190f56 100644
--- a/nf_core/utils.py
+++ b/nf_core/utils.py
@@ -379,15 +379,34 @@ def setup_requests_cachedir() -> Dict[str, Union[Path, datetime.timedelta, str]]
     Also returns the config dict so that we can use the same setup with a Session.
     """
     pyversion: str = ".".join(str(v) for v in sys.version_info[0:3])
-    cachedir: Path = setup_nfcore_cachedir(f"cache_{pyversion}")
-    config: Dict[str, Union[Path, datetime.timedelta, str]] = {
-        "cache_name": Path(cachedir, "github_info"),
-        "expire_after": datetime.timedelta(hours=1),
-        "backend": "sqlite",
-    }
-
-    logging.getLogger("requests_cache").setLevel(logging.WARNING)
-    return config
+    try:
+        # Create cache directory with exist_ok=True
+        cachedir: Path = setup_nfcore_cachedir(f"cache_{pyversion}")
+        cachedir.mkdir(parents=True, exist_ok=True)
+
+        # Ensure the github_info subdirectory exists
+        github_cache = cachedir / "github_info"
+        github_cache.mkdir(parents=True, exist_ok=True)
+
+        config: Dict[str, Union[Path, datetime.timedelta, str]] = {
+            "cache_name": github_cache,
+            "expire_after": datetime.timedelta(hours=1),
+            "backend": "sqlite",
+        }
+
+        logging.getLogger("requests_cache").setLevel(logging.WARNING)
+        return config
+    except PermissionError:
+        # Fallback to temporary directory if we can't create in home
+        import tempfile
+
+        tmp_dir = Path(tempfile.gettempdir()) / "nf-core"
+        tmp_dir.mkdir(parents=True, exist_ok=True)
+        return {
+            "cache_name": tmp_dir / "github_info",
+            "expire_after": datetime.timedelta(hours=1),
+            "backend": "sqlite",
+        }
 
 
 def setup_nfcore_cachedir(cache_fn: Union[str, Path]) -> Path:

From 7523b415a9c190e68248f81d6769f1f9b03946db Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Wed, 27 Nov 2024 12:27:40 +0100
Subject: [PATCH 11/22] log.warn -> log.warning

---
 nf_core/components/update.py           | 2 +-
 nf_core/pipelines/create/githubrepo.py | 2 +-
 nf_core/utils.py                       | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/nf_core/components/update.py b/nf_core/components/update.py
index 901a7f02fe..f71b567438 100644
--- a/nf_core/components/update.py
+++ b/nf_core/components/update.py
@@ -414,7 +414,7 @@ def get_single_component_info(self, component):
                 config_entry = self.update_config[self.modules_repo.remote_url][install_dir].get(component)
         if config_entry is not None and config_entry is not True:
             if config_entry is False:
-                log.warn(
+                log.warning(
                     f"{self.component_type[:-1].title()}'s update entry in '.nf-core.yml' for '{component}' is set to False"
                 )
                 return (self.modules_repo, None, None, None)
diff --git a/nf_core/pipelines/create/githubrepo.py b/nf_core/pipelines/create/githubrepo.py
index b37dfb6170..b688eee68f 100644
--- a/nf_core/pipelines/create/githubrepo.py
+++ b/nf_core/pipelines/create/githubrepo.py
@@ -150,7 +150,7 @@ def on_button_pressed(self, event: Button.Pressed) -> None:
                         f"Repo will be created in the GitHub organisation account '{github_variables['repo_org']}'"
                     )
                 except UnknownObjectException:
-                    log.warn(f"Provided organisation '{github_variables['repo_org']}' not found. ")
+                    log.warning(f"Provided organisation '{github_variables['repo_org']}' not found. ")
 
             # Create the repo
             try:
diff --git a/nf_core/utils.py b/nf_core/utils.py
index 5fca190f56..e17baaf489 100644
--- a/nf_core/utils.py
+++ b/nf_core/utils.py
@@ -418,7 +418,7 @@ def setup_nfcore_cachedir(cache_fn: Union[str, Path]) -> Path:
         if not Path(cachedir).exists():
             Path(cachedir).mkdir(parents=True)
     except PermissionError:
-        log.warn(f"Could not create cache directory: {cachedir}")
+        log.warning(f"Could not create cache directory: {cachedir}")
 
     return cachedir
 

From 0e193cbff6b4c09846d1bf80322c3d8fe651e56d Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Wed, 27 Nov 2024 12:30:03 +0100
Subject: [PATCH 12/22] avoid race condition when using parallel tests

---
 nf_core/utils.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/nf_core/utils.py b/nf_core/utils.py
index e17baaf489..5085bb5ee5 100644
--- a/nf_core/utils.py
+++ b/nf_core/utils.py
@@ -361,11 +361,9 @@ def run_cmd(executable: str, cmd: str) -> Union[Tuple[bytes, bytes], None]:
 
 def setup_nfcore_dir() -> bool:
     """Creates a directory for files that need to be kept between sessions
-
     Currently only used for keeping local copies of modules repos
     """
-    if not NFCORE_DIR.exists():
-        NFCORE_DIR.mkdir(parents=True)
+    NFCORE_DIR.mkdir(parents=True, exist_ok=True)
     return True
 
 
@@ -411,15 +409,12 @@ def setup_requests_cachedir() -> Dict[str, Union[Path, datetime.timedelta, str]]
 
 def setup_nfcore_cachedir(cache_fn: Union[str, Path]) -> Path:
     """Sets up local caching for caching files between sessions."""
-
     cachedir = Path(NFCORE_CACHE_DIR, cache_fn)
-
     try:
-        if not Path(cachedir).exists():
-            Path(cachedir).mkdir(parents=True)
+        # Create directory with parents=True and exist_ok=True to handle race conditions
+        Path(cachedir).mkdir(parents=True, exist_ok=True)
     except PermissionError:
         log.warning(f"Could not create cache directory: {cachedir}")
-
     return cachedir
 
 

From b58ea07844f7debdb6f20edbbe7a69c25e840c66 Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Wed, 27 Nov 2024 12:35:00 +0100
Subject: [PATCH 13/22] avoid conflicting nextflow caches

---
 tests/test_pipelines.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 5f3549761d..7c30f3a79e 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -1,3 +1,4 @@
+import os
 import shutil
 from unittest import TestCase
 
@@ -15,6 +16,9 @@ def setUp(self) -> None:
         self.pipeline_obj = Pipeline(self.pipeline_dir)
         self.pipeline_obj._load()
 
+        # setup nextflow cache dir
+        os.environ["NXF_HOME"] = str(self.tmp_dir / "nextflow_cache")
+
     def _make_pipeline_copy(self):
         """Make a copy of the test pipeline that can be edited
 

From 770d9c7eccd08bbb1186280b8525e5bd09f960cb Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Wed, 27 Nov 2024 12:45:53 +0100
Subject: [PATCH 14/22] no need for `--dir` in remote

---
 .github/actions/create-lint-wf/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml
index 95424e8b78..6ad6b7b157 100644
--- a/.github/actions/create-lint-wf/action.yml
+++ b/.github/actions/create-lint-wf/action.yml
@@ -105,7 +105,7 @@ runs:
 
     - name: nf-core modules list remote
       shell: bash
-      run: nf-core --log-file log.txt modules list remote --dir nf-core-testpipeline/
+      run: nf-core --log-file log.txt modules list remote
       working-directory: create-lint-wf
 
     - name: nf-core modules list remote gitlab

From 62c725e323bc2a367fd2064ddb544d257d6fbd68 Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Wed, 27 Nov 2024 12:56:26 +0100
Subject: [PATCH 15/22] add `--dir` option to `components list remote`

---
 .github/actions/create-lint-wf/action.yml |  2 +-
 nf_core/__main__.py                       | 28 +++++++++++++++++++----
 nf_core/commands_modules.py               |  4 ++--
 nf_core/commands_subworkflows.py          | 22 +++++++++---------
 4 files changed, 38 insertions(+), 18 deletions(-)

diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml
index 6ad6b7b157..95424e8b78 100644
--- a/.github/actions/create-lint-wf/action.yml
+++ b/.github/actions/create-lint-wf/action.yml
@@ -105,7 +105,7 @@ runs:
 
     - name: nf-core modules list remote
       shell: bash
-      run: nf-core --log-file log.txt modules list remote
+      run: nf-core --log-file log.txt modules list remote --dir nf-core-testpipeline/
       working-directory: create-lint-wf
 
     - name: nf-core modules list remote gitlab
diff --git a/nf_core/__main__.py b/nf_core/__main__.py
index 81d088e13d..7a3161ad62 100644
--- a/nf_core/__main__.py
+++ b/nf_core/__main__.py
@@ -115,6 +115,10 @@
     "nf-core modules list local": [{"options": ["--dir", "--json", "--help"]}],
 }
 
+click.rich_click.OPTION_GROUPS = {
+    "nf-core modules list remote": [{"options": ["--dir", "--json", "--help"]}],
+}
+
 # Set up rich stderr console
 stderr = rich.console.Console(stderr=True, force_terminal=rich_force_colors())
 stdout = rich.console.Console(force_terminal=rich_force_colors())
@@ -874,11 +878,19 @@ def modules_list(ctx):
 @click.pass_context
 @click.argument("keywords", required=False, nargs=-1, metavar="<filter keywords>")
 @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout")
-def command_modules_list_remote(ctx, keywords, json):
+@click.option(
+    "-d",
+    "--dir",
+    "directory",
+    type=click.Path(exists=True),
+    default=".",
+    help=r"Pipeline directory. [dim]\[default: Current working directory][/]",
+)
+def command_modules_list_remote(ctx, keywords, json, directory):  # pylint: disable=redefined-builtin
     """
     List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/].
     """
-    modules_list_remote(ctx, keywords, json)
+    modules_list_remote(ctx, keywords, json, directory)
 
 
 # nf-core modules list local
@@ -1432,11 +1444,19 @@ def subworkflows_list(ctx):
 @click.pass_context
 @click.argument("keywords", required=False, nargs=-1, metavar="<filter keywords>")
 @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout")
-def command_subworkflows_list_remote(ctx, keywords, json):
+@click.option(
+    "-d",
+    "--dir",
+    "directory",
+    type=click.Path(exists=True),
+    default=".",
+    help=r"Pipeline directory. [dim]\[default: Current working directory][/]",
+)
+def command_subworkflows_list_remote(ctx, keywords, json, directory):  # pylint: disable=redefined-builtin
     """
     List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/].
     """
-    subworkflows_list_remote(ctx, keywords, json)
+    subworkflows_list_remote(ctx, keywords, json, directory)
 
 
 # nf-core subworkflows list local
diff --git a/nf_core/commands_modules.py b/nf_core/commands_modules.py
index bf046a60a6..7deb565e8d 100644
--- a/nf_core/commands_modules.py
+++ b/nf_core/commands_modules.py
@@ -9,7 +9,7 @@
 stdout = rich.console.Console(force_terminal=rich_force_colors())
 
 
-def modules_list_remote(ctx, keywords, json):
+def modules_list_remote(ctx, keywords, json, directory):  # pylint: disable=redefined-builtin
     """
     List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/].
     """
@@ -17,7 +17,7 @@ def modules_list_remote(ctx, keywords, json):
 
     try:
         module_list = ModuleList(
-            directory=".",
+            directory=directory,
             remote=True,
             remote_url=ctx.obj["modules_repo_url"],
             branch=ctx.obj["modules_repo_branch"],
diff --git a/nf_core/commands_subworkflows.py b/nf_core/commands_subworkflows.py
index 8e90a8116b..b1fdddac75 100644
--- a/nf_core/commands_subworkflows.py
+++ b/nf_core/commands_subworkflows.py
@@ -63,7 +63,7 @@ def subworkflows_test(ctx, subworkflow, directory, no_prompts, update, once, pro
         sys.exit(1)
 
 
-def subworkflows_list_remote(ctx, keywords, json):
+def subworkflows_list_remote(ctx, keywords, json, directory):
     """
     List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/].
     """
@@ -71,11 +71,11 @@ def subworkflows_list_remote(ctx, keywords, json):
 
     try:
         subworkflow_list = SubworkflowList(
-            ".",
-            True,
-            ctx.obj["modules_repo_url"],
-            ctx.obj["modules_repo_branch"],
-            ctx.obj["modules_repo_no_pull"],
+            directory=directory,
+            remote=True,
+            modules_repo_url=ctx.obj["modules_repo_url"],
+            modules_repo_branch=ctx.obj["modules_repo_branch"],
+            modules_repo_no_pull=ctx.obj["modules_repo_no_pull"],
         )
 
         stdout.print(subworkflow_list.list_components(keywords, json))
@@ -92,11 +92,11 @@ def subworkflows_list_local(ctx, keywords, json, directory):  # pylint: disable=
 
     try:
         subworkflow_list = SubworkflowList(
-            directory,
-            False,
-            ctx.obj["modules_repo_url"],
-            ctx.obj["modules_repo_branch"],
-            ctx.obj["modules_repo_no_pull"],
+            directory=directory,
+            remote=False,
+            modules_repo_url=ctx.obj["modules_repo_url"],
+            modules_repo_branch=ctx.obj["modules_repo_branch"],
+            modules_repo_no_pull=ctx.obj["modules_repo_no_pull"],
         )
         stdout.print(subworkflow_list.list_components(keywords, json))
     except (UserWarning, LookupError) as e:

From be12d8b5cac3db3ee8f4771727646081ee7fa8e2 Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Wed, 27 Nov 2024 13:06:46 +0100
Subject: [PATCH 16/22] fix one more remote listing

---
 .github/actions/create-lint-wf/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml
index 95424e8b78..fdd2fbb4c0 100644
--- a/.github/actions/create-lint-wf/action.yml
+++ b/.github/actions/create-lint-wf/action.yml
@@ -110,5 +110,5 @@ runs:
 
     - name: nf-core modules list remote gitlab
       shell: bash
-      run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote
+      run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote --dir nf-core-testpipeline/
       working-directory: create-lint-wf

From 30404195083fd929c152652cfc9ff8bc09a9f1fd Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Wed, 27 Nov 2024 13:12:52 +0100
Subject: [PATCH 17/22] try a different approach to set the NXF_HOME

---
 tests/test_pipelines.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 7c30f3a79e..aac8c0230c 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -1,9 +1,12 @@
 import os
 import shutil
+import tempfile
 from unittest import TestCase
 
 import pytest
 
+# needs to be run before .utils import otherwise NFCORE_DIR is not set to a temp dir
+os.environ["NXF_HOME"] = tempfile.mkdtemp()
 from nf_core.utils import Pipeline
 
 from .utils import create_tmp_pipeline
@@ -16,9 +19,6 @@ def setUp(self) -> None:
         self.pipeline_obj = Pipeline(self.pipeline_dir)
         self.pipeline_obj._load()
 
-        # setup nextflow cache dir
-        os.environ["NXF_HOME"] = str(self.tmp_dir / "nextflow_cache")
-
     def _make_pipeline_copy(self):
         """Make a copy of the test pipeline that can be edited
 

From 1b4ece897475124d101798815bc375b470996358 Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Wed, 27 Nov 2024 15:25:22 +0100
Subject: [PATCH 18/22] fix tests

---
 tests/test_pipelines.py | 1 +
 tests/test_utils.py     | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index aac8c0230c..e2c1d4d768 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -7,6 +7,7 @@
 
 # needs to be run before .utils import otherwise NFCORE_DIR is not set to a temp dir
 os.environ["NXF_HOME"] = tempfile.mkdtemp()
+os.environ["XDG_CONFIG_HOME"] = tempfile.mkdtemp()
 from nf_core.utils import Pipeline
 
 from .utils import create_tmp_pipeline
diff --git a/tests/test_utils.py b/tests/test_utils.py
index b13c8eb37d..6a8fa4e85f 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -202,6 +202,6 @@ def test_set_wd(self):
     def test_set_wd_revert_on_raise(self):
         wd_before_context = Path().resolve()
         with pytest.raises(Exception):
-            with nf_core.utils.set_wd(self.tmp_dir):
+            with nf_core.utils.set_wd(self.tmp_dir / "non_existent_dir"):
                 raise Exception
         assert wd_before_context == Path().resolve()

From 01df2cf2fd27c75faf0d13d65ea3dd3ee6c35f5c Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Wed, 27 Nov 2024 15:30:34 +0100
Subject: [PATCH 19/22] fix cache location

---
 tests/test_components.py | 2 +-
 tests/test_modules.py    | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/test_components.py b/tests/test_components.py
index 43d11827e3..f8d75671c0 100644
--- a/tests/test_components.py
+++ b/tests/test_components.py
@@ -9,7 +9,7 @@
 from git.repo import Repo
 
 # needs to be run before .utils import otherwise NFCORE_DIR is not set to a temp dir
-os.environ["XDG_CONFIG_HOME"] = str(Path(tempfile.mkdtemp()))
+os.environ["XDG_CONFIG_HOME"] = tempfile.mkdtemp()
 from .utils import GITLAB_NFTEST_BRANCH, GITLAB_URL
 
 
diff --git a/tests/test_modules.py b/tests/test_modules.py
index c4e14d5063..195574332b 100644
--- a/tests/test_modules.py
+++ b/tests/test_modules.py
@@ -19,10 +19,11 @@
 import nf_core.pipelines.create.create
 from nf_core import __version__
 from nf_core.pipelines.lint_utils import run_prettier_on_file
-from nf_core.utils import NFCoreYamlConfig
 
 # needs to be run before .utils import otherwise NFCORE_DIR is not set to a temp dir
-os.environ["XDG_CONFIG_HOME"] = str(Path(tempfile.mkdtemp()))
+os.environ["XDG_CONFIG_HOME"] = tempfile.mkdtemp()
+from nf_core.utils import NFCoreYamlConfig
+
 from .utils import (
     GITLAB_BRANCH_TEST_BRANCH,
     GITLAB_BRANCH_TEST_OLD_SHA,

From 8c6a549f02ffe4c3985ef995cde8789526128a11 Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Mon, 2 Dec 2024 13:26:43 +0100
Subject: [PATCH 20/22] fix test_components test

---
 tests/components/generate_snapshot.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py
index aebe97b3c9..336ac8aec5 100644
--- a/tests/components/generate_snapshot.py
+++ b/tests/components/generate_snapshot.py
@@ -110,7 +110,7 @@ def test_update_snapshot_module(self):
         assert snap_content["Single-End"]["timestamp"] != original_timestamp
 
 
-def test_test_not_found(self):
+def test_test_not_found(self, capsys):
     """Generate the snapshot for a module in nf-core/modules clone which doesn't contain tests"""
     with set_wd(self.nfcore_modules):
         snap_generator = ComponentsTest(
@@ -122,9 +122,11 @@ def test_test_not_found(self):
         )
         test_file = Path(snap_generator.default_modules_path, snap_generator.component_name, "tests", "main.nf.test")
         test_file.rename(test_file.parent / "main.nf.test.bak")
-        with pytest.raises(UserWarning) as e:
-            snap_generator.run()
-        assert "Test file 'main.nf.test' not found" in str(e.value)
+
+        captured = capsys.readouterr()
+        assert snap_generator.run()
+        assert "No tests to execute" in captured.out
+
         Path(test_file.parent / "main.nf.test.bak").rename(test_file)
 
 

From ee3366c097fad239e5037ba1b41b4d58db2ed727 Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Mon, 2 Dec 2024 15:03:02 +0100
Subject: [PATCH 21/22] fix no test found test

---
 tests/components/generate_snapshot.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py
index 336ac8aec5..55d5870ce0 100644
--- a/tests/components/generate_snapshot.py
+++ b/tests/components/generate_snapshot.py
@@ -1,6 +1,8 @@
 """Test generate a snapshot"""
 
 import json
+import sys
+from io import StringIO
 from pathlib import Path
 from unittest.mock import MagicMock
 
@@ -110,7 +112,7 @@ def test_update_snapshot_module(self):
         assert snap_content["Single-End"]["timestamp"] != original_timestamp
 
 
-def test_test_not_found(self, capsys):
+def test_test_not_found(self):
     """Generate the snapshot for a module in nf-core/modules clone which doesn't contain tests"""
     with set_wd(self.nfcore_modules):
         snap_generator = ComponentsTest(
@@ -123,9 +125,15 @@ def test_test_not_found(self, capsys):
         test_file = Path(snap_generator.default_modules_path, snap_generator.component_name, "tests", "main.nf.test")
         test_file.rename(test_file.parent / "main.nf.test.bak")
 
-        captured = capsys.readouterr()
-        assert snap_generator.run()
-        assert "No tests to execute" in captured.out
+        captured_output = StringIO()
+        sys.stdout = captured_output
+
+        try:
+            assert snap_generator.run()
+            output = captured_output.getvalue()
+            assert "No tests to execute" in output
+        finally:
+            sys.stdout = sys.__stdout__  # Restore stdout
 
         Path(test_file.parent / "main.nf.test.bak").rename(test_file)
 

From 90ae1aecbb423e614f48a19d82c04561fcbf811e Mon Sep 17 00:00:00 2001
From: mashehu <mashehu3@gmail.com>
Date: Mon, 2 Dec 2024 15:03:28 +0100
Subject: [PATCH 22/22] run only test_components in parallel for now

---
 .github/workflows/pytest.yml | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
index 7992f9aaae..ae9b42f70c 100644
--- a/.github/workflows/pytest.yml
+++ b/.github/workflows/pytest.yml
@@ -72,13 +72,19 @@ jobs:
       tests: ${{ steps.list_tests.outputs.tests }}
 
   test:
-    name: Run ${{matrix.test}} with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }}
+    name: Run ${{matrix.tests.test}} with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }}
     needs: [setup, list_tests]
     if: ${{ needs.setup.outputs.run-tests }}
     # run on self-hosted runners for test_components.py (because of the gitlab branch), based on the input if it is dispatched manually, on github if it is a rerun or on self-hosted by default
-    runs-on: ${{ matrix.test == 'test_components.py' && 'self-hosted' || (github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted') }}
+    runs-on: ${{ matrix.tests.test == 'test_components.py' && 'self-hosted' || (github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted') }}
     strategy:
-      matrix: ${{ fromJson(needs.list_tests.outputs.tests) }}
+      matrix:
+        tests: ${{ fromJson(needs.list_tests.outputs.tests) }}
+        xdist: [""]
+        include:
+          - tests: "test_components.py"
+            xdist: "-n auto --maxfail 1"
+
       fail-fast: false # run all tests even if one fails
     steps:
       - name: go to subdirectory and change nextflow workdir
@@ -111,7 +117,7 @@ jobs:
           sudo apt install -y git
 
       - name: Set up Singularity
-        if: ${{ matrix.test == 'test_download.py'}}
+        if: ${{ matrix.tests.test == 'test_download.py'}}
         uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7
         with:
           singularity-version: 3.8.3
@@ -132,7 +138,7 @@ jobs:
 
       - name: Test with pytest
         run: |
-          python3 -m pytest tests/${{matrix.test}} --color=yes --cov --durations=0 -n auto && exit_code=0|| exit_code=$?
+          python3 -m pytest tests/${{matrix.tests.test}} --color=yes --cov --durations=0 ${{matrix.xdist}} && exit_code=0|| exit_code=$?
           # don't fail if no tests were collected, e.g. for test_licence.py
           if [ "${exit_code}" -eq 5 ]; then
             echo "No tests were collected"
@@ -144,7 +150,7 @@ jobs:
 
       - name: remove slashes from test name
         run: |
-          test=$(echo ${{ matrix.test }} | sed 's/\//__/g')
+          test=$(echo ${{ matrix.tests.test }} | sed 's/\//__/g')
           echo "test=${test}" >> $GITHUB_ENV
 
       - name: Store snapshot report