From bf3224c14d1c44796e322f6a46da5fa88d5a527a Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 20 Dec 2024 22:35:15 +0000 Subject: [PATCH 1/9] Allow downloading a specific commit ID --- nf_core/pipelines/download.py | 15 +++++++++++---- nf_core/utils.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/nf_core/pipelines/download.py b/nf_core/pipelines/download.py index 11adebce2c..e46c43ad6b 100644 --- a/nf_core/pipelines/download.py +++ b/nf_core/pipelines/download.py @@ -374,22 +374,27 @@ def prompt_revision(self) -> None: raise AssertionError(f"No revisions of {self.pipeline} available for download.") def get_revision_hash(self): - """Find specified revision / branch hash""" + """Find specified revision / branch / commit hash""" for revision in self.revision: # revision is a list of strings, but may be of length 1 # Branch if revision in self.wf_branches.keys(): self.wf_sha = {**self.wf_sha, revision: self.wf_branches[revision]} - # Revision else: + # Revision for r in self.wf_revisions: if r["tag_name"] == revision: self.wf_sha = {**self.wf_sha, revision: r["tag_sha"]} break - # Can't find the revisions or branch - throw an error else: + # Commit - full or short hash + if commit_id := nf_core.utils.get_repo_commit(self.pipeline, revision): + self.wf_sha = {**self.wf_sha, revision: commit_id} + continue + + # Can't find the revisions or branch - throw an error log.info( "Available {} revisions: '{}'".format( self.pipeline, @@ -397,7 +402,9 @@ def get_revision_hash(self): ) ) log.info("Available {} branches: '{}'".format(self.pipeline, "', '".join(self.wf_branches.keys()))) - raise AssertionError(f"Not able to find revision / branch '{revision}' for {self.pipeline}") + raise AssertionError( + f"Not able to find revision / branch / commit '{revision}' for {self.pipeline}" + ) # Set the outdir if not self.outdir: diff --git a/nf_core/utils.py b/nf_core/utils.py index e2b61329cc..6734ab50c0 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1096,6 +1096,16 @@ def get_repo_releases_branches(pipeline, wfs): return pipeline, wf_releases, wf_branches +def get_repo_commit(pipeline, commit_id): + commit_response = gh_api.get( + f"https://api.github.com/repos/{pipeline}/commits/{commit_id}", headers={"Accept": "application/vnd.github.sha"} + ) + if commit_response.status_code == 200: + return commit_response.text + else: + return None + + CONFIG_PATHS = [".nf-core.yml", ".nf-core.yaml"] DEPRECATED_CONFIG_PATHS = [".nf-core-lint.yml", ".nf-core-lint.yaml"] From 9dcb4a0a3adcf89264efda6b9e43592672dd860b Mon Sep 17 00:00:00 2001 From: fellen31 Date: Thu, 4 Apr 2024 07:49:21 +0200 Subject: [PATCH 2/9] Added tests --- tests/pipelines/test_download.py | 42 ++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/pipelines/test_download.py b/tests/pipelines/test_download.py index d1e2c41a68..c68d2ecc58 100644 --- a/tests/pipelines/test_download.py +++ b/tests/pipelines/test_download.py @@ -81,6 +81,48 @@ def test_get_release_hash_branch(self): == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" ) + def test_get_release_hash_long_commit(self): + wfs = nf_core.pipelines.list.Workflows() + wfs.get_remote_workflows() + # Exoseq pipeline is archived, so `dev` branch should be stable + pipeline = "exoseq" + + download_obj = DownloadWorkflow(pipeline="exoseq", revision="819cbac792b76cf66c840b567ed0ee9a2f620db7") + ( + download_obj.pipeline, + download_obj.wf_revisions, + download_obj.wf_branches, + ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) + download_obj.get_revision_hash() + print(download_obj) + assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7" + assert download_obj.outdir == "nf-core-exoseq_819cbac792b76cf66c840b567ed0ee9a2f620db7" + assert ( + download_obj.wf_download_url[download_obj.revision[0]] + == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" + ) + + def test_get_release_hash_short_commit(self): + wfs = nf_core.pipelines.list.Workflows() + wfs.get_remote_workflows() + # Exoseq pipeline is archived, so `dev` branch should be stable + pipeline = "exoseq" + + download_obj = DownloadWorkflow(pipeline="exoseq", revision="819cbac") + ( + download_obj.pipeline, + download_obj.wf_revisions, + download_obj.wf_branches, + ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) + download_obj.get_revision_hash() + print(download_obj) + assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7" + assert download_obj.outdir == "nf-core-exoseq_819cbac" + assert ( + download_obj.wf_download_url[download_obj.revision[0]] + == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" + ) + def test_get_release_hash_non_existent_release(self): wfs = nf_core.pipelines.list.Workflows() wfs.get_remote_workflows() From 7b931f1fac9dec6d42a75cb0fa97c9263e7eae91 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 20 Dec 2024 23:04:45 +0000 Subject: [PATCH 3/9] Added more tests --- tests/test_utils.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index b13c8eb37d..0418c605cc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -169,6 +169,26 @@ def test_get_repo_releases_branches_not_exists_slash(self): with pytest.raises(AssertionError): nf_core.utils.get_repo_releases_branches("made-up/pipeline", wfs) + def test_get_repo_commit(self): + # The input can be a commit in standard long/short form, but also any length as long as it can be uniquely resolved + assert ( + nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3b95aaf01d98391a62a10a3990c0a4de395") + == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" + ) + assert ( + nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3b95aaf01d") + == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" + ) + assert ( + nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3b") == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" + ) + assert ( + nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3") == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" + ) + assert nf_core.utils.get_repo_commit("nf-core/methylseq", "xyz") is None + assert nf_core.utils.get_repo_commit("made_up_pipeline", "") is None + assert nf_core.utils.get_repo_commit("made-up/pipeline", "") is None + def test_validate_file_md5(self): # MD5(test) = d8e8fca2dc0f896fd7cb4cb0031ba249 test_file = TEST_DATA_DIR / "test.txt" From c3563374e11d628c50c0cb9d1b3272d02c58b745 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 20 Dec 2024 23:29:09 +0000 Subject: [PATCH 4/9] Updated the changelog --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 926535077d..95b7d3783b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # nf-core/tools: Changelog +## v3.2.0dev + +### Template + +### Download + +- Allow `nf-core pipelines download -r` to download commits ([#3374](https://github.com/nf-core/tools/pull/3374)) + +### Linting + +### Modules + +### Subworkflows + +### General + +### Version updates + ## [v3.1.1 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.1) - [2024-12-20] ### Template From b0b9559594f3cd2d55670c2b838f2b14a6ac4731 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 20 Dec 2024 23:34:44 +0000 Subject: [PATCH 5/9] Added some documentation --- nf_core/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nf_core/utils.py b/nf_core/utils.py index 6734ab50c0..2ac0943c59 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1097,6 +1097,16 @@ def get_repo_releases_branches(pipeline, wfs): def get_repo_commit(pipeline, commit_id): + """Check if the repo contains the requested commit_id, and expand it to long form if necessary. + + Args: + pipeline (str): GitHub repo username/repo + commit_id: The requested commit ID (SHA). It can be in standard long/short form, or any length. + + Returns: + commit_id: String or None + """ + commit_response = gh_api.get( f"https://api.github.com/repos/{pipeline}/commits/{commit_id}", headers={"Accept": "application/vnd.github.sha"} ) From a7aee9c9c9164df1cd9b0d5f685ca88f82e8a766 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 13 Jan 2025 16:18:17 +0000 Subject: [PATCH 6/9] Factored out the commit ID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- tests/pipelines/test_download.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/pipelines/test_download.py b/tests/pipelines/test_download.py index c68d2ecc58..1046634f36 100644 --- a/tests/pipelines/test_download.py +++ b/tests/pipelines/test_download.py @@ -86,8 +86,9 @@ def test_get_release_hash_long_commit(self): wfs.get_remote_workflows() # Exoseq pipeline is archived, so `dev` branch should be stable pipeline = "exoseq" + revision = "819cbac792b76cf66c840b567ed0ee9a2f620db7" - download_obj = DownloadWorkflow(pipeline="exoseq", revision="819cbac792b76cf66c840b567ed0ee9a2f620db7") + download_obj = DownloadWorkflow(pipeline=pipeline, revision=revision) ( download_obj.pipeline, download_obj.wf_revisions, @@ -95,11 +96,11 @@ def test_get_release_hash_long_commit(self): ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) download_obj.get_revision_hash() print(download_obj) - assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7" - assert download_obj.outdir == "nf-core-exoseq_819cbac792b76cf66c840b567ed0ee9a2f620db7" + assert download_obj.wf_sha[download_obj.revision[0]] == revision + assert download_obj.outdir == f"nf-core-exoseq_{revision}" assert ( download_obj.wf_download_url[download_obj.revision[0]] - == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" + == f"https://github.com/nf-core/exoseq/archive/{revision}.zip" ) def test_get_release_hash_short_commit(self): From 1844f5587f19ef87db4b2c78d63f3112568537aa Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 13 Jan 2025 16:18:33 +0000 Subject: [PATCH 7/9] Not needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- tests/pipelines/test_download.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pipelines/test_download.py b/tests/pipelines/test_download.py index 1046634f36..608f524215 100644 --- a/tests/pipelines/test_download.py +++ b/tests/pipelines/test_download.py @@ -95,7 +95,6 @@ def test_get_release_hash_long_commit(self): download_obj.wf_branches, ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) download_obj.get_revision_hash() - print(download_obj) assert download_obj.wf_sha[download_obj.revision[0]] == revision assert download_obj.outdir == f"nf-core-exoseq_{revision}" assert ( From 5ae7d53a91e64e16b6dc103248a983d3fb2a2ffe Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 13 Jan 2025 16:22:29 +0000 Subject: [PATCH 8/9] Factored out the commit ID --- tests/test_utils.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0418c605cc..b7761253a3 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -171,20 +171,11 @@ def test_get_repo_releases_branches_not_exists_slash(self): def test_get_repo_commit(self): # The input can be a commit in standard long/short form, but also any length as long as it can be uniquely resolved - assert ( - nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3b95aaf01d98391a62a10a3990c0a4de395") - == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" - ) - assert ( - nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3b95aaf01d") - == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" - ) - assert ( - nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3b") == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" - ) - assert ( - nf_core.utils.get_repo_commit("nf-core/methylseq", "b3e5e3") == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" - ) + revision = "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" + assert nf_core.utils.get_repo_commit("nf-core/methylseq", revision) == revision + assert nf_core.utils.get_repo_commit("nf-core/methylseq", revision[:16]) == revision + assert nf_core.utils.get_repo_commit("nf-core/methylseq", revision[:7]) == revision + assert nf_core.utils.get_repo_commit("nf-core/methylseq", revision[:6]) == revision assert nf_core.utils.get_repo_commit("nf-core/methylseq", "xyz") is None assert nf_core.utils.get_repo_commit("made_up_pipeline", "") is None assert nf_core.utils.get_repo_commit("made-up/pipeline", "") is None From 62c3c2bc14a907a166f6cbff23b5ba1349b5f9cc Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 13 Jan 2025 16:24:41 +0000 Subject: [PATCH 9/9] Factored out the commit ID --- tests/pipelines/test_download.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/pipelines/test_download.py b/tests/pipelines/test_download.py index 608f524215..6db7392107 100644 --- a/tests/pipelines/test_download.py +++ b/tests/pipelines/test_download.py @@ -107,8 +107,10 @@ def test_get_release_hash_short_commit(self): wfs.get_remote_workflows() # Exoseq pipeline is archived, so `dev` branch should be stable pipeline = "exoseq" + revision = "819cbac792b76cf66c840b567ed0ee9a2f620db7" + short_rev = revision[:7] - download_obj = DownloadWorkflow(pipeline="exoseq", revision="819cbac") + download_obj = DownloadWorkflow(pipeline="exoseq", revision=short_rev) ( download_obj.pipeline, download_obj.wf_revisions, @@ -116,11 +118,11 @@ def test_get_release_hash_short_commit(self): ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) download_obj.get_revision_hash() print(download_obj) - assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7" - assert download_obj.outdir == "nf-core-exoseq_819cbac" + assert download_obj.wf_sha[download_obj.revision[0]] == revision + assert download_obj.outdir == f"nf-core-exoseq_{short_rev}" assert ( download_obj.wf_download_url[download_obj.revision[0]] - == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" + == f"https://github.com/nf-core/exoseq/archive/{revision}.zip" ) def test_get_release_hash_non_existent_release(self):