-
Notifications
You must be signed in to change notification settings - Fork 192
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #140 from nf-core/dev
Version 1.1 merge Master > Dev
- Loading branch information
Showing
56 changed files
with
2,308 additions
and
110 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,3 @@ | ||
| include LICENSE | ||
| include README.md | ||
| recursive-include nf_core * |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| #!/usr/bin/env python | ||
|
|
||
| from cookiecutter.main import cookiecutter | ||
| import git | ||
| import json | ||
| import os | ||
| import requests | ||
| from requests.auth import HTTPBasicAuth | ||
| import shutil | ||
| import sys | ||
| import subprocess | ||
| import tempfile | ||
| import syncutils.template | ||
|
|
||
| # Set the default nf-core pipeline template branch | ||
| DEF_TEMPLATE_BRANCH = "TEMPLATE" | ||
| # The GitHub base url or the nf-core project | ||
| GH_BASE_URL = "https://{token}@github.com/nf-core/{pipeline}" | ||
| # The JSON file is updated on every push event on the nf-core GitHub project | ||
| NF_CORE_PIPELINE_INFO = "http://nf-co.re/pipelines.json" | ||
| # The API endpoint for creating pull requests | ||
| GITHUB_PR_URL_TEMPL = "https://api.github.com/repos/nf-core/{pipeline}/pulls" | ||
|
|
||
|
|
||
| def create_pullrequest(pipeline, origin="dev", template="TEMPLATE", token="", user="nf-core"): | ||
| """Create a pull request to a base branch (default: dev), | ||
| from a head branch (default: TEMPLATE) | ||
| Returns: An instance of class requests.Response | ||
| """ | ||
| content = {} | ||
| content['title'] = "Important pipeline nf-core update! (version {tag})".format(tag=os.environ['TRAVIS_TAG']) | ||
| content['body'] = "Some important changes have been made in the nf-core pipelines templates.\n" \ | ||
| "Please make sure to merge this in ASAP and make a new minor release of your pipeline.\n\n" \ | ||
| "Follow the link [nf-core/tools](https://github.com/nf-core/tools/releases/tag/{})".format(os.environ['TRAVIS_TAG']) | ||
| content['head'] = "{}".format(template) | ||
| content['base'] = origin | ||
| return requests.post(url=GITHUB_PR_URL_TEMPL.format(pipeline=pipeline), | ||
| data=json.dumps(content), | ||
| auth=HTTPBasicAuth(user, token)) | ||
|
|
||
| def main(): | ||
| # Check that the commit event is a GitHub tag event | ||
| assert os.environ['TRAVIS_TAG'] | ||
| assert os.environ['NF_CORE_BOT'] | ||
|
|
||
| # Catch exceptions in lists, and list them at the end | ||
| sync_errors = [] | ||
| pr_errors = [] | ||
|
|
||
| # Get nf-core pipelines info | ||
| res = requests.get(NF_CORE_PIPELINE_INFO) | ||
| pipelines = json.loads(res.content).get('remote_workflows') | ||
| if not pipelines: | ||
| print("Pipeline information was empty!") | ||
|
|
||
| # TODO: Remove this line, once we go for production | ||
| pipelines = [{"name":"hlatyping"}] # just for testing | ||
|
|
||
| # Update the template branch of each pipeline repo | ||
| for pipeline in pipelines: | ||
| print("Update template branch for pipeline '{pipeline}'... ".format(pipeline=pipeline['name'])) | ||
| try: | ||
| syncutils.template.NfcoreTemplate( | ||
| pipeline['name'], | ||
| branch=DEF_TEMPLATE_BRANCH, | ||
| repo_url=GH_BASE_URL.format(token=os.environ["NF_CORE_BOT"], pipeline=pipeline['name']) | ||
| ).sync() | ||
| except Exception as e: | ||
| sync_errors.append((pipeline['name'], e)) | ||
|
|
||
| # Create a pull request from each template branch to the origin branch | ||
| for pipeline in pipelines: | ||
| print("Trying to open pull request for pipeline {}...".format(pipeline['name'])) | ||
| response = create_pullrequest(pipeline['name'], os.environ["NF_CORE_BOT"]) | ||
| if response.status_code != 201: | ||
| pr_errors.append((pipeline['name'], response.status_code, response.content)) | ||
| else: | ||
| print("Created pull-request for pipeline \'{pipeline}\' successfully." | ||
| .format(pipeline=pipeline["name"])) | ||
|
|
||
| for pipeline, exception in sync_errors: | ||
| print("Sync for pipeline {name} failed.".format(pipeline)) | ||
| print(exception) | ||
|
|
||
| for pipeline, return_code, content in pr_errors: | ||
| print("Pull-request for pipeline \'{pipeline}\' failed," | ||
| " got return code {return_code}." | ||
| .format(pipeline=pipeline["name"], return_code=return_code)) | ||
| print(content) | ||
|
|
||
| if pr_errors or sync_errors: sys.exit(1) | ||
|
|
||
| sys.exit(0) | ||
|
|
||
| if __name__ == "__main__": | ||
| main() | ||
|
|
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| import tempfile | ||
| import utils | ||
| import git | ||
| import os | ||
| import shutil | ||
| import sys | ||
| from cookiecutter.main import cookiecutter | ||
|
|
||
| import nf_core.create | ||
|
|
||
|
|
||
| class NfcoreTemplate: | ||
| """Updates the template content of an nf-core pipeline in | ||
| its `TEMPLATE` branch. | ||
| Args: - pipeline: The pipeline name | ||
| - branch: The template branch name, default=`TEMPLATE` | ||
| - token: GitHub auth token | ||
| """ | ||
| def __init__(self, pipeline, branch='master', repo_url=""): | ||
| """Basic constructor | ||
| """ | ||
| self.pipeline = pipeline | ||
| self.repo_url = repo_url | ||
| self.branch = branch | ||
| self.tmpdir = tempfile.mkdtemp() | ||
| self.templatedir = tempfile.mkdtemp() | ||
| self.repo = git.Repo.clone_from(self.repo_url, self.tmpdir) | ||
| assert self.repo | ||
|
|
||
| def sync(self): | ||
| """Execute the template update. | ||
| """ | ||
| context = self.context_from_nextflow(nf_project_dir=self.tmpdir) | ||
| self.update_child_template(self.templatedir, self.tmpdir, context=context) | ||
| self.commit_changes() | ||
| self.push_changes() | ||
|
|
||
| def context_from_nextflow(self, nf_project_dir): | ||
| """Fetch a Nextflow pipeline's config settings. | ||
| Returns: A cookiecutter-readable context (Python dictionary) | ||
| """ | ||
| # Check if we are on "master" (main pipeline code) | ||
| if self.repo.active_branch is not "master": | ||
| self.repo.git.checkout("origin/master", b="master") | ||
|
|
||
| # Fetch the config variables from the Nextflow pipeline | ||
| config = utils.fetch_wf_config(wf_path=nf_project_dir) | ||
|
|
||
| # Checkout again to configured template branch | ||
| self.repo.git.checkout("origin/{branch}".format(branch=self.branch), | ||
| b="{branch}".format(branch=self.branch)) | ||
|
|
||
| return utils.create_context(config) | ||
|
|
||
|
|
||
| def update_child_template(self, templatedir, target_dir, context=None): | ||
| """Apply the changes of the cookiecutter template | ||
| to the pipelines template branch. | ||
| """ | ||
| # Clear the pipeline's template branch content | ||
| for f in os.listdir(self.tmpdir): | ||
| if f == ".git": continue | ||
| try: | ||
| shutil.rmtree(os.path.join(target_dir, f)) | ||
| except: | ||
| os.remove(os.path.join(target_dir, f)) | ||
|
|
||
| # Create the new template structure | ||
| nf_core.create.PipelineCreate( | ||
| name=context.get('pipeline_name'), | ||
| description=context.get('pipeline_short_description'), | ||
| new_version=context.get('version'), | ||
| no_git=True, | ||
| force=True, | ||
| outdir=templatedir | ||
| ) | ||
|
|
||
| def commit_changes(self): | ||
| """Commits the changes of the new template to the current branch. | ||
| """ | ||
| self.repo.git.add(A=True) | ||
| self.repo.index.commit("Update nf-core pipeline template.") | ||
|
|
||
| def push_changes(self): | ||
| self.repo.git.push() | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| import os | ||
| import subprocess | ||
|
|
||
| def fetch_wf_config(wf_path): | ||
| """ | ||
| Use nextflow to retrieve the nf configuration variables from a workflow | ||
| """ | ||
| config = dict() | ||
| # Call `nextflow config` and pipe stderr to /dev/null | ||
| try: | ||
| with open(os.devnull, 'w') as devnull: | ||
| nfconfig_raw = subprocess.check_output(['nextflow', 'config', '-flat', wf_path], stderr=devnull) | ||
| except subprocess.CalledProcessError as e: | ||
| raise AssertionError("`nextflow config` returned non-zero error code: %s,\n %s", e.returncode, e.output) | ||
| else: | ||
| for l in nfconfig_raw.splitlines(): | ||
| ul = l.decode() | ||
| k, v = ul.split(' = ', 1) | ||
| config[k] = v.replace("\'", "").replace("\"", "") | ||
| return config | ||
|
|
||
| def create_context(config): | ||
| """Consumes a flat Nextflow config file and will create | ||
| a context dictionary with information for the nf-core template creation. | ||
| Returns: A dictionary with: | ||
| { | ||
| 'pipeline_name': '<parsed_name>' | ||
| 'pipeline_short_description': '<parsed_description>' | ||
| 'version': '<parsed_version>' | ||
| } | ||
| """ | ||
| context = {} | ||
| context["pipeline_name"] = config.get("manifest.name") if config.get("manifest.name") else get_name_from_url(config.get("manifest.homePage")) | ||
| context["pipeline_short_description"] = config.get("manifest.description") | ||
| context["version"] = config.get("manifest.version") if config.get("manifest.version") else config.get("params.version") | ||
| return context | ||
|
|
||
| def get_name_from_url(url): | ||
| return url.split("/")[-1] if url else "" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.