Skip to content

Commit

Permalink
allow download commits
Browse files Browse the repository at this point in the history
  • Loading branch information
@fellen31
fellen31 committed Apr 4, 2024
1 parent 20c1ea5 commit 2f0969e
Show file tree
Hide file tree
Showing 5 changed files with 82 additions and 19 deletions.
16 changes: 12 additions & 4 deletions nf_core/download.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ def __init__(

self.wf_revisions = {}
self.wf_branches = {}
self.wf_commits = {}
self.wf_sha = {}
self.wf_download_url = {}
self.nf_config = {}
Expand All @@ -160,7 +161,7 @@ def download_workflow(self):
# Get workflow details
try:
self.prompt_pipeline_name()
self.pipeline, self.wf_revisions, self.wf_branches = nf_core.utils.get_repo_releases_branches(
self.pipeline, self.wf_revisions, self.wf_branches, self.wf_commits = nf_core.utils.get_repo_releases_branches_commits(
self.pipeline, self.wfs
)
self.prompt_revision()
Expand Down Expand Up @@ -354,21 +355,27 @@ def prompt_revision(self):
raise AssertionError(f"No revisions of {self.pipeline} available for download.")

def get_revision_hash(self):
"""Find specified revision / branch hash"""
"""Find specified revision / branch / commit hash"""

for revision in self.revision: # revision is a list of strings, but may be of length 1

# Branch
if revision in self.wf_branches.keys():
self.wf_sha = {**self.wf_sha, revision: self.wf_branches[revision]}

# Commit - full or short hash
elif revision in self.wf_commits or any(full_hash[:7] == revision for full_hash in self.wf_commits):
for full_hash in self.wf_commits:
if full_hash[:7] == revision or full_hash == revision:
self.wf_sha = {**self.wf_sha, revision: full_hash}
break
# Revision
else:
for r in self.wf_revisions:
if r["tag_name"] == revision:
self.wf_sha = {**self.wf_sha, revision: r["tag_sha"]}
break

# Can't find the revisions or branch - throw an error
# Can't find the revisions or branch - throw an error
else:
log.info(
"Available {} revisions: '{}'".format(
Expand Down Expand Up @@ -616,6 +623,7 @@ def download_wf_files(self, revision, wf_sha, download_url):

# Rename the internal directory name to be more friendly
gh_name = f"{self.pipeline}-{wf_sha if bool(wf_sha) else ''}".split("/")[-1]

os.rename(
os.path.join(self.outdir, gh_name),
os.path.join(self.outdir, revision_dirname),
Expand Down
2 changes: 1 addition & 1 deletion nf_core/launch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ def get_pipeline_schema(self):

if not self.pipeline_revision:
try:
self.pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches(
self.pipeline, wf_releases, wf_branches, _ = nf_core.utils.get_repo_releases_branches_commits(
self.pipeline, self.wfs
)
except AssertionError as e:
Expand Down
9 changes: 7 additions & 2 deletions nf_core/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -940,7 +940,7 @@ def validate(self, value):
return True


def get_repo_releases_branches(pipeline, wfs):
def get_repo_releases_branches_commits(pipeline, wfs):
"""Fetches details of a nf-core workflow to download.
Args:
Expand All @@ -956,6 +956,7 @@ def get_repo_releases_branches(pipeline, wfs):

wf_releases = []
wf_branches = {}
wf_commits = []

# Repo is a nf-core pipeline
for wf in wfs.remote_workflows:
Expand Down Expand Up @@ -1009,8 +1010,12 @@ def get_repo_releases_branches(pipeline, wfs):
):
wf_branches[branch["name"]] = branch["commit"]["sha"]

# Get commit information from github api
commit_response = gh_api.safe_get(f"https://api.github.com/repos/{pipeline}/commits?sha={branch['name']}")
for commit in commit_response.json():
wf_commits.append(commit["sha"])
# Return pipeline again in case we added the nf-core/ prefix
return pipeline, wf_releases, wf_branches
return pipeline, wf_releases, wf_branches, wf_commits


CONFIG_PATHS = [".nf-core.yml", ".nf-core.yaml"]
Expand Down
56 changes: 52 additions & 4 deletions tests/test_download.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ def test_get_release_hash_release(self):
download_obj.pipeline,
download_obj.wf_revisions,
download_obj.wf_branches,
) = nf_core.utils.get_repo_releases_branches(pipeline, wfs)
_
) = nf_core.utils.get_repo_releases_branches_commits(pipeline, wfs)
download_obj.get_revision_hash()
assert download_obj.wf_sha[download_obj.revision[0]] == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395"
assert download_obj.outdir == "nf-core-methylseq_1.6"
Expand All @@ -51,7 +52,8 @@ def test_get_release_hash_branch(self):
download_obj.pipeline,
download_obj.wf_revisions,
download_obj.wf_branches,
) = nf_core.utils.get_repo_releases_branches(pipeline, wfs)
_
) = nf_core.utils.get_repo_releases_branches_commits(pipeline, wfs)
download_obj.get_revision_hash()
assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7"
assert download_obj.outdir == "nf-core-exoseq_dev"
Expand All @@ -60,6 +62,50 @@ def test_get_release_hash_branch(self):
== "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip"
)

def test_get_release_hash_long_commit(self):
wfs = nf_core.list.Workflows()
wfs.get_remote_workflows()
# Exoseq pipeline is archived, so `dev` branch should be stable
pipeline = "exoseq"

download_obj = DownloadWorkflow(pipeline="exoseq", revision="819cbac792b76cf66c840b567ed0ee9a2f620db7")
(
download_obj.pipeline,
download_obj.wf_revisions,
download_obj.wf_branches,
download_obj.wf_commits,
) = nf_core.utils.get_repo_releases_branches_commits(pipeline, wfs)
download_obj.get_revision_hash()
print(download_obj)
assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7"
assert download_obj.outdir == "nf-core-exoseq_819cbac792b76cf66c840b567ed0ee9a2f620db7"
assert (
download_obj.wf_download_url[download_obj.revision[0]]
== "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip"
)

def test_get_release_hash_short_commit(self):
wfs = nf_core.list.Workflows()
wfs.get_remote_workflows()
# Exoseq pipeline is archived, so `dev` branch should be stable
pipeline = "exoseq"

download_obj = DownloadWorkflow(pipeline="exoseq", revision="819cbac")
(
download_obj.pipeline,
download_obj.wf_revisions,
download_obj.wf_branches,
download_obj.wf_commits,
) = nf_core.utils.get_repo_releases_branches_commits(pipeline, wfs)
download_obj.get_revision_hash()
print(download_obj)
assert download_obj.wf_sha[download_obj.revision[0]] == "819cbac792b76cf66c840b567ed0ee9a2f620db7"
assert download_obj.outdir == "nf-core-exoseq_819cbac"
assert (
download_obj.wf_download_url[download_obj.revision[0]]
== "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip"
)

def test_get_release_hash_non_existent_release(self):
wfs = nf_core.list.Workflows()
wfs.get_remote_workflows()
Expand All @@ -69,7 +115,8 @@ def test_get_release_hash_non_existent_release(self):
download_obj.pipeline,
download_obj.wf_revisions,
download_obj.wf_branches,
) = nf_core.utils.get_repo_releases_branches(pipeline, wfs)
_
) = nf_core.utils.get_repo_releases_branches_commits(pipeline, wfs)
with pytest.raises(AssertionError):
download_obj.get_revision_hash()

Expand Down Expand Up @@ -588,7 +635,8 @@ def test_download_workflow_for_platform(self, tmp_dir, _):
download_obj.pipeline,
download_obj.wf_revisions,
download_obj.wf_branches,
) = nf_core.utils.get_repo_releases_branches(download_obj.pipeline, wfs)
_
) = nf_core.utils.get_repo_releases_branches_commits(download_obj.pipeline, wfs)

download_obj.get_revision_hash()

Expand Down
18 changes: 10 additions & 8 deletions tests/test_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,39 +160,41 @@ def test_pip_erroneous_package(self):
with pytest.raises(ValueError):
nf_core.utils.pip_package("not_a_package=1.0")

def test_get_repo_releases_branches_nf_core(self):
def test_get_repo_releases_branches_commits_nf_core(self):
wfs = nf_core.list.Workflows()
wfs.get_remote_workflows()
pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches("methylseq", wfs)
pipeline, wf_releases, wf_branches, wf_commits = nf_core.utils.get_repo_releases_branches_commits("methylseq", wfs)
for r in wf_releases:
if r.get("tag_name") == "1.6":
break
else:
raise AssertionError("Release 1.6 not found")
assert "dev" in wf_branches.keys()
assert "54f823e102ef3d04077cc091a5ae435519f9923a" in wf_commits

def test_get_repo_releases_branches_not_nf_core(self):
def test_get_repo_releases_branches_commits_not_nf_core(self):
wfs = nf_core.list.Workflows()
wfs.get_remote_workflows()
pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches("MultiQC/MultiQC", wfs)
pipeline, wf_releases, wf_branches, wf_commits = nf_core.utils.get_repo_releases_branches_commits("MultiQC/MultiQC", wfs)
for r in wf_releases:
if r.get("tag_name") == "v1.10":
break
else:
raise AssertionError("MultiQC release v1.10 not found")
assert "main" in wf_branches.keys()
assert "6764be5a6874f6601765e70316b404c01f6407fc" in wf_commits

def test_get_repo_releases_branches_not_exists(self):
def test_get_repo_releases_branches_commits_not_exists(self):
wfs = nf_core.list.Workflows()
wfs.get_remote_workflows()
with pytest.raises(AssertionError):
nf_core.utils.get_repo_releases_branches("made_up_pipeline", wfs)
nf_core.utils.get_repo_releases_branches_commits("made_up_pipeline", wfs)

def test_get_repo_releases_branches_not_exists_slash(self):
def test_get_repo_releases_branches_commits_not_exists_slash(self):
wfs = nf_core.list.Workflows()
wfs.get_remote_workflows()
with pytest.raises(AssertionError):
nf_core.utils.get_repo_releases_branches("made-up/pipeline", wfs)
nf_core.utils.get_repo_releases_branches_commits("made-up/pipeline", wfs)


def test_validate_file_md5():
Expand Down

0 comments on commit 2f0969e

Please sign in to comment.