Merge pull request #3374 from nf-core/download_commits

Allow `nf-core pipelines download -r` to download commits

CHANGELOG.md

@@ -4,6 +4,10 @@
 
 ### Template
 
+### Download
+
+- Allow `nf-core pipelines download -r` to download commits ([#3374](https://github.com/nf-core/tools/pull/3374))
+
 ### Linting
 
 ### Modules

nf_core/pipelines/download.py

@@ -374,30 +374,37 @@ def prompt_revision(self) -> None:
                     raise AssertionError(f"No revisions of {self.pipeline} available for download.")
 
     def get_revision_hash(self):
-        """Find specified revision / branch hash"""
+        """Find specified revision / branch / commit hash"""
 
         for revision in self.revision:  # revision is a list of strings, but may be of length 1
             # Branch
             if revision in self.wf_branches.keys():
                 self.wf_sha = {**self.wf_sha, revision: self.wf_branches[revision]}
 
-            # Revision
             else:
+                # Revision
                 for r in self.wf_revisions:
                     if r["tag_name"] == revision:
                         self.wf_sha = {**self.wf_sha, revision: r["tag_sha"]}
                         break
 
-                # Can't find the revisions or branch - throw an error
                 else:
+                    # Commit - full or short hash
+                    if commit_id := nf_core.utils.get_repo_commit(self.pipeline, revision):
+                        self.wf_sha = {**self.wf_sha, revision: commit_id}
+                        continue
+
+                    # Can't find the revisions or branch - throw an error
                     log.info(
                         "Available {} revisions: '{}'".format(
                             self.pipeline,
                             "', '".join([r["tag_name"] for r in self.wf_revisions]),
                         )
                     )
                     log.info("Available {} branches: '{}'".format(self.pipeline, "', '".join(self.wf_branches.keys())))
-                    raise AssertionError(f"Not able to find revision / branch '{revision}' for {self.pipeline}")
+                    raise AssertionError(
+                        f"Not able to find revision / branch / commit '{revision}' for {self.pipeline}"
+                    )
 
         # Set the outdir
         if not self.outdir:

nf_core/utils.py

@@ -1096,6 +1096,26 @@ def get_repo_releases_branches(pipeline, wfs):
     return pipeline, wf_releases, wf_branches
 
 
+def get_repo_commit(pipeline, commit_id):
+    """Check if the repo contains the requested commit_id, and expand it to long form if necessary.
+
+    Args:
+        pipeline (str): GitHub repo username/repo
+        commit_id: The requested commit ID (SHA). It can be in standard long/short form, or any length.
+
+    Returns:
+        commit_id: String or None
+    """
+
+    commit_response = gh_api.get(
+        f"https://api.github.com/repos/{pipeline}/commits/{commit_id}", headers={"Accept": "application/vnd.github.sha"}
+    )
+    if commit_response.status_code == 200:
+        return commit_response.text
+    else:
+        return None
+
+
 CONFIG_PATHS = [".nf-core.yml", ".nf-core.yaml"]
 DEPRECATED_CONFIG_PATHS = [".nf-core-lint.yml", ".nf-core-lint.yaml"]
 

tests/pipelines/test_download.py

@@ -81,6 +81,50 @@ def test_get_release_hash_branch(self):
             == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip"
         )
 
+    def test_get_release_hash_long_commit(self):
+        wfs = nf_core.pipelines.list.Workflows()
+        wfs.get_remote_workflows()
+        # Exoseq pipeline is archived, so `dev` branch should be stable
+        pipeline = "exoseq"
+        revision = "819cbac792b76cf66c840b567ed0ee9a2f620db7"
+
+        download_obj = DownloadWorkflow(pipeline=pipeline, revision=revision)
+        (
+            download_obj.pipeline,
+            download_obj.wf_revisions,
+            download_obj.wf_branches,
+        ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs)
+        download_obj.get_revision_hash()
+        assert download_obj.wf_sha[download_obj.revision[0]] == revision
+        assert download_obj.outdir == f"nf-core-exoseq_{revision}"
+        assert (
+            download_obj.wf_download_url[download_obj.revision[0]]
+            == f"https://github.com/nf-core/exoseq/archive/{revision}.zip"
+        )
+
+    def test_get_release_hash_short_commit(self):
+        wfs = nf_core.pipelines.list.Workflows()
+        wfs.get_remote_workflows()
+        # Exoseq pipeline is archived, so `dev` branch should be stable
+        pipeline = "exoseq"
+        revision = "819cbac792b76cf66c840b567ed0ee9a2f620db7"
+        short_rev = revision[:7]
+
+        download_obj = DownloadWorkflow(pipeline="exoseq", revision=short_rev)
+        (
+            download_obj.pipeline,
+            download_obj.wf_revisions,
+            download_obj.wf_branches,
+        ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs)
+        download_obj.get_revision_hash()
+        print(download_obj)
+        assert download_obj.wf_sha[download_obj.revision[0]] == revision
+        assert download_obj.outdir == f"nf-core-exoseq_{short_rev}"
+        assert (
+            download_obj.wf_download_url[download_obj.revision[0]]
+            == f"https://github.com/nf-core/exoseq/archive/{revision}.zip"
+        )
+
     def test_get_release_hash_non_existent_release(self):
         wfs = nf_core.pipelines.list.Workflows()
         wfs.get_remote_workflows()

tests/test_utils.py

@@ -169,6 +169,17 @@ def test_get_repo_releases_branches_not_exists_slash(self):
         with pytest.raises(AssertionError):
             nf_core.utils.get_repo_releases_branches("made-up/pipeline", wfs)
 
+    def test_get_repo_commit(self):
+        # The input can be a commit in standard long/short form, but also any length as long as it can be uniquely resolved
+        revision = "b3e5e3b95aaf01d98391a62a10a3990c0a4de395"
+        assert nf_core.utils.get_repo_commit("nf-core/methylseq", revision) == revision
+        assert nf_core.utils.get_repo_commit("nf-core/methylseq", revision[:16]) == revision
+        assert nf_core.utils.get_repo_commit("nf-core/methylseq", revision[:7]) == revision
+        assert nf_core.utils.get_repo_commit("nf-core/methylseq", revision[:6]) == revision
+        assert nf_core.utils.get_repo_commit("nf-core/methylseq", "xyz") is None
+        assert nf_core.utils.get_repo_commit("made_up_pipeline", "") is None
+        assert nf_core.utils.get_repo_commit("made-up/pipeline", "") is None
+
     def test_validate_file_md5(self):
         # MD5(test) = d8e8fca2dc0f896fd7cb4cb0031ba249
         test_file = TEST_DATA_DIR / "test.txt"