diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index b0dd5cc25a..fabf219aea 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -24,9 +24,7 @@ jobs: message: | Hi @${{ github.event.pull_request.user.login }}, - It looks like this pull-request is has been made against the ${{github.event.pull_request.head.repo.full_name}} `master` branch. - The `master` branch on nf-core repositories should always contain code from the latest release. - Beacuse of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.head.repo.full_name}} `dev` branch. + It looks like this pull-request has been made against the ${{github.event.pull_request.head.repo.full_name}} `master` branch. The `master` branch on nf-core repositories should always contain code from the latest release. Beacuse of this, PRs to `master` are only allowed if they come from the ${{github.event.pull_request.head.repo.full_name}} `dev` branch. You do not need to close this PR, you can change the target branch to `dev` by clicking the _"Edit"_ button at the top of this page. diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 256a33570d..764649e5cc 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -4,13 +4,34 @@ on: types: [published] jobs: - sync-all: - name: Sync all pipelines + get-pipelines: runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - id: set-matrix + run: | + curl -O https://nf-co.re/pipeline_names.json + echo "::set-output name=matrix::$(cat pipeline_names.json)" + sync: + needs: get-pipelines + runs-on: ubuntu-latest + strategy: + matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} + fail-fast: false steps: + - uses: actions/checkout@v2 - name: Check out source-code repository + name: Check out nf-core/tools + + - uses: actions/checkout@v2 + name: Check out nf-core/${{ matrix.pipeline }} + with: + repository: nf-core/${{ matrix.pipeline }} + ref: dev + token: ${{ secrets.nf_core_bot_auth_token }} + path: nf-core/${{ matrix.pipeline }} - name: Set up Python 3.8 uses: actions/setup-python@v1 @@ -32,14 +53,20 @@ jobs: - name: Run synchronisation if: github.repository == 'nf-core/tools' env: - AUTH_TOKEN: ${{ secrets.nf_core_bot_auth_token }} + GITHUB_AUTH_TOKEN: ${{ secrets.nf_core_bot_auth_token }} run: | git config --global user.email "core@nf-co.re" git config --global user.name "nf-core-bot" - nf-core --log-file sync_log.txt sync --all --username nf-core-bot --auth-token $AUTH_TOKEN + nf-core --log-file sync_log_${{ matrix.pipeline }}.txt sync nf-core/${{ matrix.pipeline }} \ + --from-branch dev \ + --pull-request \ + --username nf-core-bot \ + --repository nf-core/${{ matrix.pipeline }} + - name: Upload sync log file artifact + if: ${{ always() }} uses: actions/upload-artifact@v2 with: - name: sync-log-file - path: sync_log.txt + name: sync_log_${{ matrix.pipeline }} + path: sync_log_${{ matrix.pipeline }}.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index f4d8d69258..d959b2d246 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,32 @@ # nf-core/tools: Changelog -## [v1.10.1](https://github.com/nf-core/tools/releases/tag/1.10.1) - [2020-07-30] +## [v1.10.2 - Copper Camel _(brought back from the dead)_](https://github.com/nf-core/tools/releases/tag/1.10.2) - [2020-07-31] + +Second patch release to address some small errors discovered in the pipeline template. +Apologies for the inconvenience. + +* Fix syntax error in `/push_dockerhub.yml` GitHub Action workflow +* Change `params.readPaths` -> `params.input_paths` in `test_full.config` +* Check results when posting the lint results as a GitHub comment + * This feature is unfortunately not possible when making PRs from forks outside of the nf-core organisation for now. +* More major refactoring of the automated pipeline sync + * New GitHub Actions matrix parallelisation of sync jobs across pipelines [[#673](https://github.com/nf-core/tools/issues/673)] + * Removed the `--all` behaviour from `nf-core sync` as we no longer need it + * Sync now uses a new list of pipelines on the website which does not include archived pipelines [[#712](https://github.com/nf-core/tools/issues/712)] + * When making a PR it checks if a PR already exists - if so it updates it [[#710](https://github.com/nf-core/tools/issues/710)] + * More tests and code refactoring for more stable code. Hopefully fixes 404 error [[#711](https://github.com/nf-core/tools/issues/711)] + +## [v1.10.1 - Copper Camel _(patch)_](https://github.com/nf-core/tools/releases/tag/1.10.1) - [2020-07-30] Patch release to fix the automatic template synchronisation, which failed in the v1.10 release. * Improved logging: `nf-core --log-file log.txt` now saves a verbose log to disk. -* GitHub actions sync now uploads verbose log as an artifact. -* Sync - fixed several minor bugs, improved logging. +* nf-core/tools GitHub Actions pipeline sync now uploads verbose log as an artifact. +* Sync - fixed several minor bugs, made logging less verbose. * Python Rich library updated to `>=4.2.1` +* Hopefully fix git config for pipeline sync so that commit comes from @nf-core-bot +* Fix sync auto-PR text indentation so that it doesn't all show as code +* Added explicit flag `--show-passed` for `nf-core lint` instead of taking logging verbosity ## [v1.10 - Copper Camel](https://github.com/nf-core/tools/releases/tag/1.10) - [2020-07-30] diff --git a/docs/lint_errors.md b/docs/lint_errors.md index d4aa8b8390..39794dc29c 100644 --- a/docs/lint_errors.md +++ b/docs/lint_errors.md @@ -177,7 +177,7 @@ This test will fail if the following requirements are not met in these files: 2. `linting.yml`: Specifies the commands to lint the pipeline repository using `nf-core lint` and `markdownlint` * Must be turned on for `push` and `pull_request`. - * Must have the command `nf-core lint ${GITHUB_WORKSPACE}`. + * Must have the command `nf-core -l lint_log.txt lint ${GITHUB_WORKSPACE}`. * Must have the command `markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml`. 3. `branch.yml`: Ensures that pull requests to the protected `master` branch are coming from the correct branch when a PR is opened against the _nf-core_ repository. diff --git a/nf_core/__main__.py b/nf_core/__main__.py index a52b18a101..be9bdfaac9 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -550,14 +550,12 @@ def bump_version(pipeline_dir, new_version, nextflow): @nf_core_cli.command("sync", help_priority=10) -@click.argument("pipeline_dir", type=click.Path(exists=True), nargs=-1, metavar="") +@click.argument("pipeline_dir", required=True, type=click.Path(exists=True), metavar="") @click.option("-b", "--from-branch", type=str, help="The git branch to use to fetch workflow vars.") @click.option("-p", "--pull-request", is_flag=True, default=False, help="Make a GitHub pull-request with the changes.") -@click.option("-u", "--username", type=str, help="GitHub username for the PR.") -@click.option("-r", "--repository", type=str, help="GitHub repository name for the PR.") -@click.option("-a", "--auth-token", type=str, help="GitHub API personal access token.") -@click.option("--all", is_flag=True, default=False, help="Sync template for all nf-core pipelines.") -def sync(pipeline_dir, from_branch, pull_request, username, repository, auth_token, all): +@click.option("-r", "--repository", type=str, help="GitHub PR: target repository.") +@click.option("-u", "--username", type=str, help="GitHub PR: auth username.") +def sync(pipeline_dir, from_branch, pull_request, repository, username): """ Sync a pipeline TEMPLATE branch with the nf-core template. @@ -571,24 +569,13 @@ def sync(pipeline_dir, from_branch, pull_request, username, repository, auth_tok new release of nf-core/tools (and the included template) is made. """ - # Pull and sync all nf-core pipelines - if all: - nf_core.sync.sync_all_pipelines(username, auth_token) - else: - # Manually check for the required parameter - if not pipeline_dir or len(pipeline_dir) != 1: - log.error("Either use --all or specify one ") - sys.exit(1) - else: - pipeline_dir = pipeline_dir[0] - - # Sync the given pipeline dir - sync_obj = nf_core.sync.PipelineSync(pipeline_dir, from_branch, pull_request) - try: - sync_obj.sync() - except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e: - log.error(e) - sys.exit(1) + # Sync the given pipeline dir + sync_obj = nf_core.sync.PipelineSync(pipeline_dir, from_branch, pull_request, repository, username) + try: + sync_obj.sync() + except (nf_core.sync.SyncException, nf_core.sync.PullRequestException) as e: + log.error(e) + sys.exit(1) if __name__ == "__main__": diff --git a/nf_core/lint.py b/nf_core/lint.py index 34f1ac5c2f..fcc47ae3d6 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -771,7 +771,7 @@ def check_actions_lint(self): self.passed.append((5, "Continuous integration runs Markdown lint Tests: `{}`".format(fn))) # Check that the nf-core linting runs - nfcore_lint_cmd = "nf-core lint ${GITHUB_WORKSPACE}" + nfcore_lint_cmd = "nf-core -l lint_log.txt lint ${GITHUB_WORKSPACE}" try: steps = lintwf["jobs"]["nf-core"]["steps"] assert any([nfcore_lint_cmd in step["run"] for step in steps if "run" in step.keys()]) @@ -1440,39 +1440,55 @@ def github_comment(self): """ If we are running in a GitHub PR, try to post results as a comment """ - if os.environ.get("GITHUB_TOKEN", "") != "" and os.environ.get("GITHUB_COMMENTS_URL", "") != "": - try: - headers = {"Authorization": "token {}".format(os.environ["GITHUB_TOKEN"])} - # Get existing comments - GET - get_r = requests.get(url=os.environ["GITHUB_COMMENTS_URL"], headers=headers) - if get_r.status_code == 200: - - # Look for an existing comment to update - update_url = False - for comment in get_r.json(): - if comment["user"]["login"] == "github-actions[bot]" and comment["body"].startswith( - "\n#### `nf-core lint` overall result" - ): - # Update existing comment - PATCH - log.info("Updating GitHub comment") - update_r = requests.patch( - url=comment["url"], - data=json.dumps({"body": self.get_results_md().replace("Posted", "**Updated**")}), - headers=headers, - ) - return - - # Create new comment - POST - if len(self.warned) > 0 or len(self.failed) > 0: - log.info("Posting GitHub comment") - post_r = requests.post( - url=os.environ["GITHUB_COMMENTS_URL"], - data=json.dumps({"body": self.get_results_md()}), + if os.environ.get("GITHUB_TOKEN", "") == "": + log.debug("Environment variable GITHUB_TOKEN not found") + return + if os.environ.get("GITHUB_COMMENTS_URL", "") == "": + log.debug("Environment variable GITHUB_COMMENTS_URL not found") + return + try: + headers = {"Authorization": "token {}".format(os.environ["GITHUB_TOKEN"])} + # Get existing comments - GET + get_r = requests.get(url=os.environ["GITHUB_COMMENTS_URL"], headers=headers) + if get_r.status_code == 200: + + # Look for an existing comment to update + update_url = False + for comment in get_r.json(): + if comment["user"]["login"] == "github-actions[bot]" and comment["body"].startswith( + "\n#### `nf-core lint` overall result" + ): + # Update existing comment - PATCH + log.info("Updating GitHub comment") + update_r = requests.patch( + url=comment["url"], + data=json.dumps({"body": self.get_results_md().replace("Posted", "**Updated**")}), headers=headers, ) + return + + # Create new comment - POST + if len(self.warned) > 0 or len(self.failed) > 0: + r = requests.post( + url=os.environ["GITHUB_COMMENTS_URL"], + data=json.dumps({"body": self.get_results_md()}), + headers=headers, + ) + try: + r_json = json.loads(r.content) + response_pp = json.dumps(r_json, indent=4) + except: + r_json = r.content + response_pp = r.content + + if r.status_code == 201: + log.info("Posted GitHub comment: {}".format(r_json["html_url"])) + log.debug(response_pp) + else: + log.warn("Could not post GitHub comment: '{}'\n{}".format(r.status_code, response_pp)) - except Exception as e: - log.warning("Could not post GitHub comment: {}\n{}".format(os.environ["GITHUB_COMMENTS_URL"], e)) + except Exception as e: + log.warning("Could not post GitHub comment: {}\n{}".format(os.environ["GITHUB_COMMENTS_URL"], e)) def _wrap_quotes(self, files): if not isinstance(files, list): diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md index bd292ca180..3836aa7637 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/CONTRIBUTING.md @@ -46,7 +46,7 @@ These tests are run both with the latest available version of `Nextflow` and als ## Patch -: warning: Only in the unlikely and regretful event of a release happening with a bug. +:warning: Only in the unlikely and regretful event of a release happening with a bug. * On your own fork, make a new branch `patch` based on `upstream/master`. * Fix the bug, and bump version (X.Y.Z+1). diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/awsfulltest.yml index 9da1209356..07b164e6bf 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/awsfulltest.yml @@ -21,7 +21,7 @@ jobs: run: conda install -c conda-forge awscli - name: Start AWS batch job # TODO nf-core: You can customise AWS full pipeline tests as required - # Add full size test data (but still relatively small datasets for few samples) + # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters # Then specify `-profile test_full` instead of `-profile test` on the AWS batch command {% raw %}env: diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml index 94ec0a87ca..a561ad54f0 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/branch.yml @@ -17,6 +17,7 @@ jobs: {% raw %} # If the above check failed, post a comment on the PR explaining the failure + # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets - name: Post PR comment if: failure() uses: mshick/add-pr-comment@v1 diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml index d10a057a9b..3298bafb73 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/linting.yml @@ -57,5 +57,12 @@ jobs: GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} - run: nf-core lint ${GITHUB_WORKSPACE} + run: nf-core -l lint_log.txt lint ${GITHUB_WORKSPACE} + + - name: Upload linting log file artifact + if: ${{ always() }} + uses: actions/upload-artifact@v2 + with: + name: linting-log-file + path: lint_log.txt {% endraw %} diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/push_dockerhub.yml b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/push_dockerhub.yml index 65079085c5..0035185f96 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/push_dockerhub.yml +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/.github/workflows/push_dockerhub.yml @@ -8,32 +8,33 @@ on: release: types: [published] -push_dockerhub: - name: Push new Docker image to Docker Hub - runs-on: ubuntu-latest - # Only run for the nf-core repo, for releases and merged PRs - if: {% raw %}${{{% endraw %} github.repository == '{{ cookiecutter.name }}' {% raw %}}}{% endraw %} - env: - DOCKERHUB_USERNAME: {% raw %}${{ secrets.DOCKERHUB_USERNAME }}{% endraw %} - DOCKERHUB_PASS: {% raw %}${{ secrets.DOCKERHUB_PASS }}{% endraw %} - steps: - - name: Check out pipeline code - uses: actions/checkout@v2 +jobs: + push_dockerhub: + name: Push new Docker image to Docker Hub + runs-on: ubuntu-latest + # Only run for the nf-core repo, for releases and merged PRs + if: {% raw %}${{{% endraw %} github.repository == '{{ cookiecutter.name }}' {% raw %}}}{% endraw %} + env: + DOCKERHUB_USERNAME: {% raw %}${{ secrets.DOCKERHUB_USERNAME }}{% endraw %} + DOCKERHUB_PASS: {% raw %}${{ secrets.DOCKERHUB_PASS }}{% endraw %} + steps: + - name: Check out pipeline code + uses: actions/checkout@v2 - - name: Build new docker image - run: docker build --no-cache . -t {{ cookiecutter.name_docker }}:latest + - name: Build new docker image + run: docker build --no-cache . -t {{ cookiecutter.name_docker }}:latest - - name: Push Docker image to DockerHub (dev) - if: {% raw %}${{ github.event_name == 'push' }}{% endraw %} - run: | - echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - docker tag {{ cookiecutter.name_docker }}:latest {{ cookiecutter.name_docker }}:dev - docker push {{ cookiecutter.name_docker }}:dev + - name: Push Docker image to DockerHub (dev) + if: {% raw %}${{ github.event_name == 'push' }}{% endraw %} + run: | + echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin + docker tag {{ cookiecutter.name_docker }}:latest {{ cookiecutter.name_docker }}:dev + docker push {{ cookiecutter.name_docker }}:dev - - name: Push Docker image to DockerHub (release) - if: {% raw %}${{ github.event_name == 'release' }}{% endraw %} - run: | - echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - docker push {{ cookiecutter.name_docker }}:latest - docker tag {{ cookiecutter.name_docker }}:latest {{ cookiecutter.name_docker }}:{% raw %}${{ github.event.release.tag_name }}{% endraw %} - docker push {{ cookiecutter.name_docker }}:{% raw %}${{ github.event.release.tag_name }}{% endraw %} + - name: Push Docker image to DockerHub (release) + if: {% raw %}${{ github.event_name == 'release' }}{% endraw %} + run: | + echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin + docker push {{ cookiecutter.name_docker }}:latest + docker tag {{ cookiecutter.name_docker }}:latest {{ cookiecutter.name_docker }}:{% raw %}${{ github.event.release.tag_name }}{% endraw %} + docker push {{ cookiecutter.name_docker }}:{% raw %}${{ github.event.release.tag_name }}{% endraw %} diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test_full.config b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test_full.config index 170c6bd4fc..d9abb981eb 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test_full.config +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/conf/test_full.config @@ -15,7 +15,7 @@ params { // TODO nf-core: Specify the paths to your full test data ( on nf-core/test-datasets or directly in repositories, e.g. SRA) // TODO nf-core: Give any required params for the test so that command line flags are not needed single_end = false - readPaths = [ + input_paths = [ ['Testdata', ['https://github.com/nf-core/test-datasets/raw/exoseq/testdata/Testdata_R1.tiny.fastq.gz', 'https://github.com/nf-core/test-datasets/raw/exoseq/testdata/Testdata_R2.tiny.fastq.gz']], ['SRR389222', ['https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', 'https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub2.fastq.gz']] ] diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md index 3517a308c9..b0111a0afd 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/docs/usage.md @@ -80,7 +80,7 @@ You can also supply a run name to resume a specific run: `-resume [run-name]`. U ### `-c` -Specify the path to a specific config file (this is a core NextFlow command). See the [nf-core website documentation](https://nf-co.re/usage/configuration) for more information. +Specify the path to a specific config file (this is a core Nextflow command). See the [nf-core website documentation](https://nf-co.re/usage/configuration) for more information. #### Custom resource requests diff --git a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf index 715d15aaea..b30a6727aa 100644 --- a/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf +++ b/nf_core/pipeline-template/{{cookiecutter.name_noslash}}/main.nf @@ -127,7 +127,7 @@ def summary = [:] if (workflow.revision) summary['Pipeline Release'] = workflow.revision summary['Run Name'] = custom_runName ?: workflow.runName // TODO nf-core: Report custom parameters here -summary['Reads'] = params.input +summary['Input'] = params.input summary['Fasta Ref'] = params.fasta summary['Data Type'] = params.single_end ? 'Single-End' : 'Paired-End' summary['Max Resources'] = "$params.max_memory memory, $params.max_cpus cpus, $params.max_time time per job" diff --git a/nf_core/sync.py b/nf_core/sync.py index 7e455b054f..0f48eb5161 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -44,7 +44,6 @@ class PipelineSync(object): make_pr (bool): Set this to `True` to create a GitHub pull-request with the changes gh_username (str): GitHub username gh_repo (str): GitHub repository name - gh_auth_token (str): Authorisation token used to make PR with GitHub API Attributes: pipeline_dir (str): Path to target pipeline directory @@ -55,11 +54,10 @@ class PipelineSync(object): required_config_vars (list): List of nextflow variables required to make template pipeline gh_username (str): GitHub username gh_repo (str): GitHub repository name - gh_auth_token (str): Authorisation token used to make PR with GitHub API """ def __init__( - self, pipeline_dir, from_branch=None, make_pr=False, gh_username=None, gh_repo=None, gh_auth_token=None, + self, pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None, ): """ Initialise syncing object """ @@ -73,18 +71,16 @@ def __init__( self.gh_username = gh_username self.gh_repo = gh_repo - self.gh_auth_token = gh_auth_token def sync(self): """ Find workflow attributes, create a new template pipeline on TEMPLATE """ - config_log_msg = "Pipeline directory: {}".format(self.pipeline_dir) + log.info("Pipeline directory: {}".format(self.pipeline_dir)) if self.from_branch: - config_log_msg += "\n Using branch `{}` to fetch workflow variables".format(self.from_branch) + log.info("Using branch `{}` to fetch workflow variables".format(self.from_branch)) if self.make_pr: - config_log_msg += "\n Will attempt to automatically create a pull request on GitHub.com" - log.info(config_log_msg) + log.info("Will attempt to automatically create a pull request") self.inspect_sync_dir() self.get_wf_config() @@ -125,7 +121,7 @@ def inspect_sync_dir(self): # get current branch so we can switch back later self.original_branch = self.repo.active_branch.name - log.debug("Original pipeline repository branch is '{}'".format(self.original_branch)) + log.info("Original pipeline repository branch is '{}'".format(self.original_branch)) # Check to see if there are uncommitted changes on current branch if self.repo.is_dirty(untracked_files=True): @@ -140,7 +136,7 @@ def get_wf_config(self): # Try to check out target branch (eg. `origin/dev`) try: if self.from_branch and self.repo.active_branch.name != self.from_branch: - log.debug("Checking out workflow branch '{}'".format(self.from_branch)) + log.info("Checking out workflow branch '{}'".format(self.from_branch)) self.repo.git.checkout(self.from_branch) except git.exc.GitCommandError: raise SyncException("Branch `{}` not found!".format(self.from_branch)) @@ -152,26 +148,6 @@ def get_wf_config(self): except git.exc.GitCommandError as e: log.error("Could not find active repo branch: ".format(e)) - # Figure out the GitHub username and repo name from the 'origin' remote if we can - try: - origin_url = self.repo.remotes.origin.url.rstrip(".git") - gh_origin_match = re.search(r"github\.com[:\/]([^\/]+)/([^\/]+)$", origin_url) - if gh_origin_match: - self.gh_username = gh_origin_match.group(1) - self.gh_repo = gh_origin_match.group(2) - else: - raise AttributeError - except AttributeError as e: - log.debug( - "Could not find repository URL for remote called 'origin' from remote: {}".format(self.repo.remotes) - ) - else: - log.debug( - "Found username and repo from remote: {}, {} - {}".format( - self.gh_username, self.gh_repo, self.repo.remotes.origin.url - ) - ) - # Fetch workflow variables log.debug("Fetching workflow config variables") self.wf_config = nf_core.utils.fetch_wf_config(self.pipeline_dir) @@ -201,7 +177,7 @@ def delete_template_branch_files(self): Delete all files in the TEMPLATE branch """ # Delete everything - log.debug("Deleting all files in TEMPLATE branch") + log.info("Deleting all files in TEMPLATE branch") for the_file in os.listdir(self.pipeline_dir): if the_file == ".git": continue @@ -219,7 +195,7 @@ def make_template_pipeline(self): """ Delete all files and make a fresh template using the workflow variables """ - log.debug("Making a new template pipeline using pipeline variables") + log.info("Making a new template pipeline using pipeline variables") # Only show error messages from pipeline creation logging.getLogger("nf_core.create").setLevel(logging.ERROR) @@ -246,7 +222,7 @@ def commit_template_changes(self): self.repo.git.add(A=True) self.repo.index.commit("Template update for nf-core/tools version {}".format(nf_core.__version__)) self.made_changes = True - log.debug("Committed changes to TEMPLATE branch") + log.info("Committed changes to TEMPLATE branch") except Exception as e: raise SyncException("Could not commit changes to TEMPLATE:\n{}".format(e)) return True @@ -256,7 +232,7 @@ def push_template_branch(self): and try to make a PR. If we don't have the auth token, try to figure out a URL for the PR and print this to the console. """ - log.debug("Pushing TEMPLATE branch to remote: '{}'".format(os.path.basename(self.pipeline_dir))) + log.info("Pushing TEMPLATE branch to remote: '{}'".format(os.path.basename(self.pipeline_dir))) try: self.repo.git.push() except git.exc.GitCommandError as e: @@ -277,17 +253,18 @@ def make_pull_request(self): # If we've been asked to make a PR, check that we have the credentials try: - assert self.gh_auth_token is not None + assert os.environ.get("GITHUB_AUTH_TOKEN", "") != "" except AssertionError: - log.info( - "Make a PR at the following URL:\n https://github.com/{}/{}/compare/{}...TEMPLATE".format( - self.gh_username, self.gh_repo, self.original_branch + raise PullRequestException( + "Environment variable GITHUB_AUTH_TOKEN not set - cannot make PR\n" + "Make a PR at the following URL:\n https://github.com/{}/compare/{}...TEMPLATE".format( + self.gh_repo, self.original_branch ) ) - raise PullRequestException("No GitHub authentication token set - cannot make PR") - log.debug("Submitting a pull request via the GitHub API") + log.info("Submitting a pull request via the GitHub API") + pr_title = "Important! Template update for nf-core/tools v{}".format(nf_core.__version__) pr_body_text = ( "A new release of the main template in nf-core/tools has just been released. " "This automated pull-request attempts to apply the relevant updates to this pipeline.\n\n" @@ -299,17 +276,90 @@ def make_pull_request(self): "please see the [nf-core/tools v{tag} release page](https://github.com/nf-core/tools/releases/tag/{tag})." ).format(tag=nf_core.__version__) + # Try to update an existing pull-request + if self.update_existing_pull_request(pr_title, pr_body_text) is False: + # None found - make a new pull-request + self.submit_pull_request(pr_title, pr_body_text) + + def update_existing_pull_request(self, pr_title, pr_body_text): + """ + List existing pull-requests between TEMPLATE and self.from_branch + + If one is found, attempt to update it with a new title and body text + If none are found, return False + """ + assert os.environ.get("GITHUB_AUTH_TOKEN", "") != "" + # Look for existing pull-requests + list_prs_url = "https://api.github.com/repos/{}/pulls?head=nf-core:TEMPLATE&base={}".format( + self.gh_repo, self.from_branch + ) + r = requests.get( + url=list_prs_url, auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ.get("GITHUB_AUTH_TOKEN")), + ) + try: + r_json = json.loads(r.content) + r_pp = json.dumps(r_json, indent=4) + except: + r_json = r.content + r_pp = r.content + + # PR worked + if r.status_code == 200: + log.debug("GitHub API listing existing PRs:\n{}".format(r_pp)) + + # No open PRs + if len(r_json) == 0: + log.info("No open PRs found between TEMPLATE and {}".format(self.from_branch)) + return False + + # Update existing PR + pr_update_api_url = r_json[0]["url"] + pr_content = {"title": pr_title, "body": pr_body_text} + + r = requests.patch( + url=pr_update_api_url, + data=json.dumps(pr_content), + auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ.get("GITHUB_AUTH_TOKEN")), + ) + try: + r_json = json.loads(r.content) + r_pp = json.dumps(r_json, indent=4) + except: + r_json = r.content + r_pp = r.content + + # PR update worked + if r.status_code == 200: + log.debug("GitHub API PR-update worked:\n{}".format(r_pp)) + log.info("Updated GitHub PR: {}".format(r_json["html_url"])) + return True + # Something went wrong + else: + log.warn("Could not update PR ('{}'):\n{}\n{}".format(r.status_code, pr_update_api_url, r_pp)) + return False + + # Something went wrong + else: + log.warn("Could not list open PRs ('{}')\n{}\n{}".format(r.status_code, list_prs_url, r_pp)) + return False + + def submit_pull_request(self, pr_title, pr_body_text): + """ + Create a new pull-request on GitHub + """ + assert os.environ.get("GITHUB_AUTH_TOKEN", "") != "" pr_content = { - "title": "Important! Template update for nf-core/tools v{}".format(nf_core.__version__), + "title": pr_title, "body": pr_body_text, "maintainer_can_modify": True, "head": "TEMPLATE", "base": self.from_branch, } + r = requests.post( - url="https://api.github.com/repos/{}/{}/pulls".format(self.gh_username, self.gh_repo), + url="https://api.github.com/repos/{}/pulls".format(self.gh_repo), data=json.dumps(pr_content), - auth=requests.auth.HTTPBasicAuth(self.gh_username, self.gh_auth_token), + auth=requests.auth.HTTPBasicAuth(self.gh_username, os.environ.get("GITHUB_AUTH_TOKEN")), ) try: self.gh_pr_returned_data = json.loads(r.content) @@ -318,90 +368,23 @@ def make_pull_request(self): self.gh_pr_returned_data = r.content returned_data_prettyprint = r.content - if r.status_code != 201: + # PR worked + if r.status_code == 201: + log.debug("GitHub API PR worked:\n{}".format(returned_data_prettyprint)) + log.info("GitHub PR created: {}".format(self.gh_pr_returned_data["html_url"])) + + # Something went wrong + else: raise PullRequestException( "GitHub API returned code {}: \n{}".format(r.status_code, returned_data_prettyprint) ) - else: - log.debug("GitHub API PR worked:\n{}".format(returned_data_prettyprint)) - log.info("GitHub PR created: {}".format(self.gh_pr_returned_data["html_url"])) def reset_target_dir(self): """ Reset the target pipeline directory. Check out the original branch. """ - log.debug("Checking out original branch: '{}'".format(self.original_branch)) + log.info("Checking out original branch: '{}'".format(self.original_branch)) try: self.repo.git.checkout(self.original_branch) except git.exc.GitCommandError as e: raise SyncException("Could not reset to original branch `{}`:\n{}".format(self.from_branch, e)) - - -def sync_all_pipelines(gh_username=None, gh_auth_token=None): - """Sync all nf-core pipelines - """ - - # Get remote workflows - wfs = nf_core.list.Workflows() - wfs.get_remote_workflows() - - successful_syncs = [] - failed_syncs = [] - - # Set up a working directory - tmpdir = tempfile.mkdtemp() - - # Let's do some updating! - for wf in wfs.remote_workflows: - - log.info("-" * 30) - log.info("Syncing {}".format(wf.full_name)) - - # Make a local working directory - wf_local_path = os.path.join(tmpdir, wf.name) - os.mkdir(wf_local_path) - log.debug("Sync working directory: {}".format(wf_local_path)) - - # Clone the repo - wf_remote_url = "https://{}@github.com/nf-core/{}".format(gh_auth_token, wf.name) - repo = git.Repo.clone_from(wf_remote_url, wf_local_path) - assert repo - - # Only show error messages from pipeline creation - logging.getLogger("nf_core.create").setLevel(logging.ERROR) - - # Sync the repo - log.debug("Running template sync") - sync_obj = nf_core.sync.PipelineSync( - pipeline_dir=wf_local_path, - from_branch="dev", - make_pr=True, - gh_username=gh_username, - gh_auth_token=gh_auth_token, - ) - try: - sync_obj.sync() - except (SyncException, PullRequestException) as e: - log.error("Sync failed for {}:\n{}".format(wf.full_name, e)) - failed_syncs.append(wf.name) - except Exception as e: - log.error("Something went wrong when syncing {}:\n{}".format(wf.full_name, e)) - failed_syncs.append(wf.name) - else: - log.info( - "[green]Sync successful for {0}:[/] [blue][link={1}]{1}[/link]".format( - wf.full_name, sync_obj.gh_pr_returned_data.get("html_url") - ) - ) - successful_syncs.append(wf.name) - - # Clean up - log.debug("Removing work directory: {}".format(wf_local_path)) - shutil.rmtree(wf_local_path) - - if len(successful_syncs) > 0: - log.info("[green]Finished. Successfully synchronised {} pipelines".format(len(successful_syncs))) - - if len(failed_syncs) > 0: - failed_list = "\n - ".join(failed_syncs) - log.error("[red]Errors whilst synchronising {} pipelines:\n - {}".format(len(failed_syncs), failed_list)) diff --git a/setup.py b/setup.py index 778d1a6d03..cbb355615e 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages import sys -version = "1.10.1" +version = "1.10.2" with open("README.md") as f: readme = f.read() diff --git a/tests/lint_examples/minimalworkingexample/.github/workflows/linting.yml b/tests/lint_examples/minimalworkingexample/.github/workflows/linting.yml index 05ae346910..9e261424e0 100644 --- a/tests/lint_examples/minimalworkingexample/.github/workflows/linting.yml +++ b/tests/lint_examples/minimalworkingexample/.github/workflows/linting.yml @@ -41,5 +41,4 @@ jobs: run: | pip install nf-core - name: Run nf-core lint - run: | - nf-core lint ${GITHUB_WORKSPACE} + run: nf-core -l lint_log.txt lint ${GITHUB_WORKSPACE} diff --git a/tests/test_sync.py b/tests/test_sync.py index 01b56ac358..c7726cfb7d 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -30,6 +30,7 @@ def test_inspect_sync_dir_notgit(self): psync = nf_core.sync.PipelineSync(tempfile.mkdtemp()) try: psync.inspect_sync_dir() + raise UserWarning("Should have hit an exception") except nf_core.sync.SyncException as e: assert "does not appear to be a git repository" in e.args[0] @@ -42,6 +43,7 @@ def test_inspect_sync_dir_dirty(self): psync = nf_core.sync.PipelineSync(self.pipeline_dir) try: psync.inspect_sync_dir() + raise UserWarning("Should have hit an exception") except nf_core.sync.SyncException as e: os.remove(test_fn) assert e.args[0].startswith("Uncommitted changes found in pipeline directory!") @@ -56,24 +58,10 @@ def test_get_wf_config_no_branch(self): try: psync.inspect_sync_dir() psync.get_wf_config() + raise UserWarning("Should have hit an exception") except nf_core.sync.SyncException as e: assert e.args[0] == "Branch `foo` not found!" - def test_get_wf_config_fetch_origin(self): - """ - Try getting the GitHub username and repo from the git origin - - Also checks the fetched config variables, should pass - """ - # Try to sync, check we halt with the right error - psync = nf_core.sync.PipelineSync(self.pipeline_dir) - psync.inspect_sync_dir() - # Add a remote to the git repo - psync.repo.create_remote("origin", "https://github.com/nf-core/demo.git") - psync.get_wf_config() - assert psync.gh_username == "nf-core" - assert psync.gh_repo == "demo" - def test_get_wf_config_missing_required_config(self): """ Try getting a workflow config, then make it miss a required config option """ # Try to sync, check we halt with the right error @@ -82,6 +70,7 @@ def test_get_wf_config_missing_required_config(self): try: psync.inspect_sync_dir() psync.get_wf_config() + raise UserWarning("Should have hit an exception") except nf_core.sync.SyncException as e: # Check that we did actually get some config back assert psync.wf_config["params.outdir"] == "'./results'" @@ -163,6 +152,7 @@ def test_push_template_branch_error(self): # Try to push changes try: psync.push_template_branch() + raise UserWarning("Should have hit an exception") except nf_core.sync.PullRequestException as e: assert e.args[0].startswith("Could not push TEMPLATE branch") @@ -173,6 +163,7 @@ def test_make_pull_request_missing_username(self): psync.gh_repo = None try: psync.make_pull_request() + raise UserWarning("Should have hit an exception") except nf_core.sync.PullRequestException as e: assert e.args[0] == "Could not find GitHub username and repo name" @@ -180,14 +171,19 @@ def test_make_pull_request_missing_auth(self): """ Try making a PR without any auth """ psync = nf_core.sync.PipelineSync(self.pipeline_dir) psync.gh_username = "foo" - psync.gh_repo = "bar" - psync.gh_auth_token = None + psync.gh_repo = "foo/bar" + if "GITHUB_AUTH_TOKEN" in os.environ: + del os.environ["GITHUB_AUTH_TOKEN"] try: psync.make_pull_request() + raise UserWarning("Should have hit an exception") except nf_core.sync.PullRequestException as e: - assert e.args[0] == "No GitHub authentication token set - cannot make PR" + assert e.args[0] == ( + "Environment variable GITHUB_AUTH_TOKEN not set - cannot make PR\n" + "Make a PR at the following URL:\n https://github.com/foo/bar/compare/None...TEMPLATE" + ) - def mocked_requests_post(**kwargs): + def mocked_requests_get(**kwargs): """ Helper function to emulate POST requests responses from the web """ class MockResponse: @@ -195,31 +191,76 @@ def __init__(self, data, status_code): self.status_code = status_code self.content = json.dumps(data) - if kwargs["url"] == "https://api.github.com/repos/bad/response/pulls": - return MockResponse({}, 404) + url_template = "https://api.github.com/repos/{}/response/pulls?head=nf-core:TEMPLATE&base=None" + if kwargs["url"] == url_template.format("no_existing_pr"): + response_data = [] + return MockResponse(response_data, 200) + + if kwargs["url"] == url_template.format("existing_pr"): + response_data = [{"url": "url_to_update_pr"}] + return MockResponse(response_data, 200) + + return MockResponse({"get_url": kwargs["url"]}, 404) + + def mocked_requests_patch(**kwargs): + """ Helper function to emulate POST requests responses from the web """ + + class MockResponse: + def __init__(self, data, status_code): + self.status_code = status_code + self.content = json.dumps(data) + + if kwargs["url"] == "url_to_update_pr": + response_data = {"html_url": "great_success"} + return MockResponse(response_data, 200) + + return MockResponse({"patch_url": kwargs["url"]}, 404) - if kwargs["url"] == "https://api.github.com/repos/good/response/pulls": + def mocked_requests_post(**kwargs): + """ Helper function to emulate POST requests responses from the web """ + + class MockResponse: + def __init__(self, data, status_code): + self.status_code = status_code + self.content = json.dumps(data) + + if kwargs["url"] == "https://api.github.com/repos/no_existing_pr/response/pulls": response_data = {"html_url": "great_success"} return MockResponse(response_data, 201) + return MockResponse({"post_url": kwargs["url"]}, 404) + + @mock.patch("requests.get", side_effect=mocked_requests_get) @mock.patch("requests.post", side_effect=mocked_requests_post) - def test_make_pull_request_bad_response(self, mock_post): - """ Try making a PR without any auth """ + def test_make_pull_request_success(self, mock_get, mock_post): + """ Try making a PR - successful response """ psync = nf_core.sync.PipelineSync(self.pipeline_dir) - psync.gh_username = "bad" - psync.gh_repo = "response" - psync.gh_auth_token = "test" + psync.gh_username = "no_existing_pr" + psync.gh_repo = "no_existing_pr/response" + os.environ["GITHUB_AUTH_TOKEN"] = "test" + psync.make_pull_request() + assert psync.gh_pr_returned_data["html_url"] == "great_success" + + @mock.patch("requests.get", side_effect=mocked_requests_get) + @mock.patch("requests.post", side_effect=mocked_requests_post) + def test_make_pull_request_bad_response(self, mock_get, mock_post): + """ Try making a PR and getting a 404 error """ + psync = nf_core.sync.PipelineSync(self.pipeline_dir) + psync.gh_username = "bad_url" + psync.gh_repo = "bad_url/response" + os.environ["GITHUB_AUTH_TOKEN"] = "test" try: psync.make_pull_request() + raise UserWarning("Should have hit an exception") except nf_core.sync.PullRequestException as e: assert e.args[0].startswith("GitHub API returned code 404:") - @mock.patch("requests.post", side_effect=mocked_requests_post) - def test_make_pull_request_bad_response(self, mock_post): - """ Try making a PR without any auth """ + @mock.patch("requests.get", side_effect=mocked_requests_get) + @mock.patch("requests.patch", side_effect=mocked_requests_patch) + def test_update_existing_pull_request(self, mock_get, mock_patch): + """ Try discovering a PR and updating it """ psync = nf_core.sync.PipelineSync(self.pipeline_dir) - psync.gh_username = "good" - psync.gh_repo = "response" - psync.gh_auth_token = "test" - psync.make_pull_request() - assert psync.gh_pr_returned_data["html_url"] == "great_success" + psync.gh_username = "existing_pr" + psync.gh_repo = "existing_pr/response" + os.environ["GITHUB_AUTH_TOKEN"] = "test" + assert psync.update_existing_pull_request("title", "body") is True