From 2f0969e1c5a3f0452bd6b24d11cda5398eca115a Mon Sep 17 00:00:00 2001 From: fellen31 Date: Thu, 4 Apr 2024 07:49:21 +0200 Subject: [PATCH] allow download commits --- nf_core/download.py | 16 +++++++++--- nf_core/launch.py | 2 +- nf_core/utils.py | 9 +++++-- tests/test_download.py | 56 +++++++++++++++++++++++++++++++++++++++--- tests/test_utils.py | 18 ++++++++------ 5 files changed, 82 insertions(+), 19 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 9d4decc424..c2a25cff26 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -144,6 +144,7 @@ def __init__( self.wf_revisions = {} self.wf_branches = {} + self.wf_commits = {} self.wf_sha = {} self.wf_download_url = {} self.nf_config = {} @@ -160,7 +161,7 @@ def download_workflow(self): # Get workflow details try: self.prompt_pipeline_name() - self.pipeline, self.wf_revisions, self.wf_branches = nf_core.utils.get_repo_releases_branches( + self.pipeline, self.wf_revisions, self.wf_branches, self.wf_commits = nf_core.utils.get_repo_releases_branches_commits( self.pipeline, self.wfs ) self.prompt_revision() @@ -354,13 +355,19 @@ def prompt_revision(self): raise AssertionError(f"No revisions of {self.pipeline} available for download.") def get_revision_hash(self): - """Find specified revision / branch hash""" + """Find specified revision / branch / commit hash""" for revision in self.revision: # revision is a list of strings, but may be of length 1 + # Branch if revision in self.wf_branches.keys(): self.wf_sha = {**self.wf_sha, revision: self.wf_branches[revision]} - + # Commit - full or short hash + elif revision in self.wf_commits or any(full_hash[:7] == revision for full_hash in self.wf_commits): + for full_hash in self.wf_commits: + if full_hash[:7] == revision or full_hash == revision: + self.wf_sha = {**self.wf_sha, revision: full_hash} + break # Revision else: for r in self.wf_revisions: @@ -368,7 +375,7 @@ def get_revision_hash(self): self.wf_sha = {**self.wf_sha, revision: r["tag_sha"]} break - # Can't find the revisions or branch - throw an error + # Can't find the revisions or branch - throw an error else: log.info( "Available {} revisions: '{}'".format( @@ -616,6 +623,7 @@ def download_wf_files(self, revision, wf_sha, download_url): # Rename the internal directory name to be more friendly gh_name = f"{self.pipeline}-{wf_sha if bool(wf_sha) else ''}".split("/")[-1] + os.rename( os.path.join(self.outdir, gh_name), os.path.join(self.outdir, revision_dirname), diff --git a/nf_core/launch.py b/nf_core/launch.py index bc0cd58aec..1706f867c8 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -207,7 +207,7 @@ def get_pipeline_schema(self): if not self.pipeline_revision: try: - self.pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches( + self.pipeline, wf_releases, wf_branches, _ = nf_core.utils.get_repo_releases_branches_commits( self.pipeline, self.wfs ) except AssertionError as e: diff --git a/nf_core/utils.py b/nf_core/utils.py index 6af2f25165..1c855e6459 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -940,7 +940,7 @@ def validate(self, value): return True -def get_repo_releases_branches(pipeline, wfs): +def get_repo_releases_branches_commits(pipeline, wfs): """Fetches details of a nf-core workflow to download. Args: @@ -956,6 +956,7 @@ def get_repo_releases_branches(pipeline, wfs): wf_releases = [] wf_branches = {} + wf_commits = [] # Repo is a nf-core pipeline for wf in wfs.remote_workflows: @@ -1009,8 +1010,12 @@ def get_repo_releases_branches(pipeline, wfs): ): wf_branches[branch["name"]] = branch["commit"]["sha"] + # Get commit information from github api + commit_response = gh_api.safe_get(f"https://api.github.com/repos/{pipeline}/commits?sha={branch['name']}") + for commit in commit_response.json(): + wf_commits.append(commit["sha"]) # Return pipeline again in case we added the nf-core/ prefix - return pipeline, wf_releases, wf_branches + return pipeline, wf_releases, wf_branches, wf_commits CONFIG_PATHS = [".nf-core.yml", ".nf-core.yaml"] diff --git a/tests/test_download.py b/tests/test_download.py index 14a96be26f..25f992f00c 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -32,7 +32,8 @@ def test_get_release_hash_release(self): download_obj.pipeline, download_obj.wf_revisions, download_obj.wf_branches, - ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) + _ + ) = nf_core.utils.get_repo_releases_branches_commits(pipeline, wfs) download_obj.get_revision_hash() assert download_obj.wf_sha[download_obj.revision[0]] == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" assert download_obj.outdir == "nf-core-methylseq_1.6" @@ -51,7 +52,8 @@ def test_get_release_hash_branch(self): download_obj.pipeline, download_obj.wf_revisions, download_obj.wf_branches, - ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) + _ + ) = nf_core.utils.get_repo_releases_branches_commits(pipeline, wfs) download_obj.get_revision_hash() assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7" assert download_obj.outdir == "nf-core-exoseq_dev" @@ -60,6 +62,50 @@ def test_get_release_hash_branch(self): == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" ) + def test_get_release_hash_long_commit(self): + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + # Exoseq pipeline is archived, so `dev` branch should be stable + pipeline = "exoseq" + + download_obj = DownloadWorkflow(pipeline="exoseq", revision="819cbac792b76cf66c840b567ed0ee9a2f620db7") + ( + download_obj.pipeline, + download_obj.wf_revisions, + download_obj.wf_branches, + download_obj.wf_commits, + ) = nf_core.utils.get_repo_releases_branches_commits(pipeline, wfs) + download_obj.get_revision_hash() + print(download_obj) + assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7" + assert download_obj.outdir == "nf-core-exoseq_819cbac792b76cf66c840b567ed0ee9a2f620db7" + assert ( + download_obj.wf_download_url[download_obj.revision[0]] + == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" + ) + + def test_get_release_hash_short_commit(self): + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + # Exoseq pipeline is archived, so `dev` branch should be stable + pipeline = "exoseq" + + download_obj = DownloadWorkflow(pipeline="exoseq", revision="819cbac") + ( + download_obj.pipeline, + download_obj.wf_revisions, + download_obj.wf_branches, + download_obj.wf_commits, + ) = nf_core.utils.get_repo_releases_branches_commits(pipeline, wfs) + download_obj.get_revision_hash() + print(download_obj) + assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7" + assert download_obj.outdir == "nf-core-exoseq_819cbac" + assert ( + download_obj.wf_download_url[download_obj.revision[0]] + == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" + ) + def test_get_release_hash_non_existent_release(self): wfs = nf_core.list.Workflows() wfs.get_remote_workflows() @@ -69,7 +115,8 @@ def test_get_release_hash_non_existent_release(self): download_obj.pipeline, download_obj.wf_revisions, download_obj.wf_branches, - ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) + _ + ) = nf_core.utils.get_repo_releases_branches_commits(pipeline, wfs) with pytest.raises(AssertionError): download_obj.get_revision_hash() @@ -588,7 +635,8 @@ def test_download_workflow_for_platform(self, tmp_dir, _): download_obj.pipeline, download_obj.wf_revisions, download_obj.wf_branches, - ) = nf_core.utils.get_repo_releases_branches(download_obj.pipeline, wfs) + _ + ) = nf_core.utils.get_repo_releases_branches_commits(download_obj.pipeline, wfs) download_obj.get_revision_hash() diff --git a/tests/test_utils.py b/tests/test_utils.py index 145060450f..28a19a748b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -160,39 +160,41 @@ def test_pip_erroneous_package(self): with pytest.raises(ValueError): nf_core.utils.pip_package("not_a_package=1.0") - def test_get_repo_releases_branches_nf_core(self): + def test_get_repo_releases_branches_commits_nf_core(self): wfs = nf_core.list.Workflows() wfs.get_remote_workflows() - pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches("methylseq", wfs) + pipeline, wf_releases, wf_branches, wf_commits = nf_core.utils.get_repo_releases_branches_commits("methylseq", wfs) for r in wf_releases: if r.get("tag_name") == "1.6": break else: raise AssertionError("Release 1.6 not found") assert "dev" in wf_branches.keys() + assert "54f823e102ef3d04077cc091a5ae435519f9923a" in wf_commits - def test_get_repo_releases_branches_not_nf_core(self): + def test_get_repo_releases_branches_commits_not_nf_core(self): wfs = nf_core.list.Workflows() wfs.get_remote_workflows() - pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches("MultiQC/MultiQC", wfs) + pipeline, wf_releases, wf_branches, wf_commits = nf_core.utils.get_repo_releases_branches_commits("MultiQC/MultiQC", wfs) for r in wf_releases: if r.get("tag_name") == "v1.10": break else: raise AssertionError("MultiQC release v1.10 not found") assert "main" in wf_branches.keys() + assert "6764be5a6874f6601765e70316b404c01f6407fc" in wf_commits - def test_get_repo_releases_branches_not_exists(self): + def test_get_repo_releases_branches_commits_not_exists(self): wfs = nf_core.list.Workflows() wfs.get_remote_workflows() with pytest.raises(AssertionError): - nf_core.utils.get_repo_releases_branches("made_up_pipeline", wfs) + nf_core.utils.get_repo_releases_branches_commits("made_up_pipeline", wfs) - def test_get_repo_releases_branches_not_exists_slash(self): + def test_get_repo_releases_branches_commits_not_exists_slash(self): wfs = nf_core.list.Workflows() wfs.get_remote_workflows() with pytest.raises(AssertionError): - nf_core.utils.get_repo_releases_branches("made-up/pipeline", wfs) + nf_core.utils.get_repo_releases_branches_commits("made-up/pipeline", wfs) def test_validate_file_md5():