From 3dd369b1966d611a20849e330e0386464d13afac Mon Sep 17 00:00:00 2001 From: Christian Simon Date: Fri, 2 Aug 2024 15:54:09 +0100 Subject: [PATCH 01/15] Add workflow --- .github/workflows/bench.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/bench.yaml diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml new file mode 100644 index 000000000..7d3bf04de --- /dev/null +++ b/.github/workflows/bench.yaml @@ -0,0 +1,24 @@ +name: Benchmark +on: [push, pull_request] +jobs: + test: + name: Bench + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.22 + uses: actions/setup-go@v5 + with: + go-version: 1.22 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + with: + fetch-depth: 1024 + + # TODO: Pin release + # TODO: Remove git-base + # TODO: Provide output + - name: Run Benchmark + run: go run github.com/grafana/pyrobench@65a9bc2 -v compare --git-base main~1 From f5e8eb8aad90b25a7486625495949e5a729d24f2 Mon Sep 17 00:00:00 2001 From: Christian Simon Date: Fri, 2 Aug 2024 15:56:14 +0100 Subject: [PATCH 02/15] Pin specific commit --- .github/workflows/bench.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 7d3bf04de..a09457282 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -21,4 +21,4 @@ jobs: # TODO: Remove git-base # TODO: Provide output - name: Run Benchmark - run: go run github.com/grafana/pyrobench@65a9bc2 -v compare --git-base main~1 + run: go run github.com/grafana/pyrobench@65a9bc2 -v compare --git-base 5eca05b1e4cf63df5d007ec6b13ce7394a0fffe3 From bf35d9befa37b0e7c0c9f8780ee207d52c9d6cb1 Mon Sep 17 00:00:00 2001 From: Bryan Huhta Date: Fri, 2 Aug 2024 10:32:16 -0700 Subject: [PATCH 03/15] Add step to list comments --- .github/workflows/bench.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index a09457282..f5b287dbf 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -22,3 +22,17 @@ jobs: # TODO: Provide output - name: Run Benchmark run: go run github.com/grafana/pyrobench@65a9bc2 -v compare --git-base 5eca05b1e4cf63df5d007ec6b13ce7394a0fffe3 + + - uses: actions/github-script@v7 + id: create-comment + with: + # TODO(bryan): Only run on pull_request, not push + script: | + github.rest.issues.listComments({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + }); + + - name: Get result + run: echo "${{steps.create-comment.outputs.result}}" From 550c80bc352a07e01ef79b1e89a4c2575365282b Mon Sep 17 00:00:00 2001 From: Bryan Huhta Date: Fri, 2 Aug 2024 10:40:33 -0700 Subject: [PATCH 04/15] remove running benchmarks for now --- .github/workflows/bench.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index f5b287dbf..fad29fba0 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -17,11 +17,12 @@ jobs: with: fetch-depth: 1024 - # TODO: Pin release - # TODO: Remove git-base - # TODO: Provide output - - name: Run Benchmark - run: go run github.com/grafana/pyrobench@65a9bc2 -v compare --git-base 5eca05b1e4cf63df5d007ec6b13ce7394a0fffe3 + # Commenting out this step for now while we test the PR comment portion + # # TODO: Pin release + # # TODO: Remove git-base + # # TODO: Provide output + # - name: Run Benchmark + # run: go run github.com/grafana/pyrobench@65a9bc2 -v compare --git-base 5eca05b1e4cf63df5d007ec6b13ce7394a0fffe3 - uses: actions/github-script@v7 id: create-comment From 6f0a03f31d4c201e7f9da0a9c5b0cf2c57af8a76 Mon Sep 17 00:00:00 2001 From: Bryan Huhta Date: Fri, 2 Aug 2024 10:43:59 -0700 Subject: [PATCH 05/15] Create a comment --- .github/workflows/bench.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index fad29fba0..a5f0d2cd6 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -29,6 +29,13 @@ jobs: with: # TODO(bryan): Only run on pull_request, not push script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: 'This is a test comment', + }); + github.rest.issues.listComments({ issue_number: context.issue.number, owner: context.repo.owner, From 00177c73d3d7be83c4be24b1fbbaa3ba4abe2df3 Mon Sep 17 00:00:00 2001 From: Bryan Huhta Date: Fri, 2 Aug 2024 10:45:46 -0700 Subject: [PATCH 06/15] debug context --- .github/workflows/bench.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index a5f0d2cd6..0a496560c 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -29,6 +29,8 @@ jobs: with: # TODO(bryan): Only run on pull_request, not push script: | + console.log(context); + github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, From ab51ea17c8e8034d2b19c517ea743a4b8814bce5 Mon Sep 17 00:00:00 2001 From: Bryan Huhta Date: Fri, 2 Aug 2024 11:20:46 -0700 Subject: [PATCH 07/15] Testing more options --- .github/workflows/bench.yaml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 0a496560c..14cad2a9d 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -26,19 +26,10 @@ jobs: - uses: actions/github-script@v7 id: create-comment + if: ${{ github.event_name == 'pull_request' }} with: - # TODO(bryan): Only run on pull_request, not push script: | - console.log(context); - - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: 'This is a test comment', - }); - - github.rest.issues.listComments({ + return github.rest.issues.listComments({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, From 529e1141cb43ea07767c8a3b3cfec08a3b8bb69f Mon Sep 17 00:00:00 2001 From: Bryan Huhta Date: Fri, 2 Aug 2024 11:21:42 -0700 Subject: [PATCH 08/15] Use await --- .github/workflows/bench.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 14cad2a9d..cc1ddefda 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -29,7 +29,7 @@ jobs: if: ${{ github.event_name == 'pull_request' }} with: script: | - return github.rest.issues.listComments({ + return await github.rest.issues.listComments({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, From 211464bd1b1b9745451e95a3b8cce444f62f860e Mon Sep 17 00:00:00 2001 From: Bryan Huhta Date: Fri, 2 Aug 2024 11:28:08 -0700 Subject: [PATCH 09/15] escape json string --- .github/workflows/bench.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index cc1ddefda..2446a3797 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -36,4 +36,4 @@ jobs: }); - name: Get result - run: echo "${{steps.create-comment.outputs.result}}" + run: echo '${{steps.create-comment.outputs.result}}' From 0ff57189e09b36df444b67cfabea5d30ed12e54f Mon Sep 17 00:00:00 2001 From: Bryan Huhta Date: Fri, 2 Aug 2024 12:22:02 -0700 Subject: [PATCH 10/15] Experiment with job outputs --- .github/workflows/bench.yaml | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 2446a3797..2e84ab04a 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -4,8 +4,10 @@ jobs: test: name: Bench runs-on: ubuntu-latest - steps: + outputs: + flame_graph_ids: ${{ steps.run_benchmarks.outputs.flame_graph_ids }} + steps: - name: Set up Go 1.22 uses: actions/setup-go@v5 with: @@ -17,12 +19,14 @@ jobs: with: fetch-depth: 1024 - # Commenting out this step for now while we test the PR comment portion - # # TODO: Pin release - # # TODO: Remove git-base - # # TODO: Provide output - # - name: Run Benchmark - # run: go run github.com/grafana/pyrobench@65a9bc2 -v compare --git-base 5eca05b1e4cf63df5d007ec6b13ce7394a0fffe3 + # TODO: Pin release + # TODO: Remove git-base + # TODO: Provide output + - name: Run Benchmark + id: run_benchmarks + run: | + IDS=$(go run github.com/grafana/pyrobench@65a9bc2 -v compare --git-base 5eca05b1e4cf63df5d007ec6b13ce7394a0fffe3) + echo 'flame_graph_ids="${IDS}"' >> "$GITHUB_OUTPUT" - uses: actions/github-script@v7 id: create-comment @@ -36,4 +40,9 @@ jobs: }); - name: Get result - run: echo '${{steps.create-comment.outputs.result}}' + if: ${{ github.event_name == 'pull_request' }} + needs: + - run_benchmarks + run: | + echo '${{steps.create-comment.outputs.result}}' + echo '${{needs.run_benchmarks.outputs.flame_graph_ids}}' From e2fbbc22db926ac7a37b22267f924936a8a01728 Mon Sep 17 00:00:00 2001 From: Bryan Huhta Date: Fri, 2 Aug 2024 13:00:29 -0700 Subject: [PATCH 11/15] Experiment with job outputs v2 --- .github/workflows/bench.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 2e84ab04a..7c3e07bc9 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -4,8 +4,6 @@ jobs: test: name: Bench runs-on: ubuntu-latest - outputs: - flame_graph_ids: ${{ steps.run_benchmarks.outputs.flame_graph_ids }} steps: - name: Set up Go 1.22 @@ -41,8 +39,6 @@ jobs: - name: Get result if: ${{ github.event_name == 'pull_request' }} - needs: - - run_benchmarks run: | echo '${{steps.create-comment.outputs.result}}' - echo '${{needs.run_benchmarks.outputs.flame_graph_ids}}' + echo '${{steps.run_benchmarks.outputs.flame_graph_ids}}' From fac205c872fb486038d1e756dd06c6bb6e3c89b7 Mon Sep 17 00:00:00 2001 From: Bryan Huhta Date: Fri, 2 Aug 2024 13:08:22 -0700 Subject: [PATCH 12/15] Use newer commit --- .github/workflows/bench.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 7c3e07bc9..d0ad56782 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -23,7 +23,7 @@ jobs: - name: Run Benchmark id: run_benchmarks run: | - IDS=$(go run github.com/grafana/pyrobench@65a9bc2 -v compare --git-base 5eca05b1e4cf63df5d007ec6b13ce7394a0fffe3) + IDS=$(go run github.com/grafana/pyrobench@ce823ac -v compare --git-base 5eca05b1e4cf63df5d007ec6b13ce7394a0fffe3) echo 'flame_graph_ids="${IDS}"' >> "$GITHUB_OUTPUT" - uses: actions/github-script@v7 From 2c211aff4113f8f1feb8bc2eab8eabedd824b1f3 Mon Sep 17 00:00:00 2001 From: Bryan Huhta Date: Fri, 2 Aug 2024 19:35:13 -0700 Subject: [PATCH 13/15] Use test report script --- .github/workflows/bench.yaml | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index d0ad56782..8ca481efb 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -21,24 +21,17 @@ jobs: # TODO: Remove git-base # TODO: Provide output - name: Run Benchmark - id: run_benchmarks run: | - IDS=$(go run github.com/grafana/pyrobench@ce823ac -v compare --git-base 5eca05b1e4cf63df5d007ec6b13ce7394a0fffe3) - echo 'flame_graph_ids="${IDS}"' >> "$GITHUB_OUTPUT" + go run github.com/grafana/pyrobench@3fa67b1 -v compare --git-base 15dd5805938793bb952dd7772c8caab77ba8ff1a >> results.txt + + - name: Download test report + run: curl -sL https://raw.githubusercontent.com/grafana/hackathon-2024-08-pyrobench/main/js/report.js -o report.js - uses: actions/github-script@v7 id: create-comment if: ${{ github.event_name == 'pull_request' }} with: + github-token: ${{ secrets.PAT }} script: | - return await github.rest.issues.listComments({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - }); - - - name: Get result - if: ${{ github.event_name == 'pull_request' }} - run: | - echo '${{steps.create-comment.outputs.result}}' - echo '${{steps.run_benchmarks.outputs.flame_graph_ids}}' + const report = require('./report.js'); + report({ github, context }); From f6188907f304c77020e046194c81fbf3dd5869aa Mon Sep 17 00:00:00 2001 From: Bryan Huhta Date: Fri, 2 Aug 2024 19:35:31 -0700 Subject: [PATCH 14/15] Don't use PAT --- .github/workflows/bench.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 8ca481efb..50d9d35cd 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -31,7 +31,6 @@ jobs: id: create-comment if: ${{ github.event_name == 'pull_request' }} with: - github-token: ${{ secrets.PAT }} script: | const report = require('./report.js'); report({ github, context }); From b6b466fae7b9e06562507b1c8191cd87f46ad7ec Mon Sep 17 00:00:00 2001 From: Bryan Huhta Date: Fri, 2 Aug 2024 19:39:05 -0700 Subject: [PATCH 15/15] Fix base --- .github/workflows/bench.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yaml b/.github/workflows/bench.yaml index 50d9d35cd..4063f969e 100644 --- a/.github/workflows/bench.yaml +++ b/.github/workflows/bench.yaml @@ -22,7 +22,7 @@ jobs: # TODO: Provide output - name: Run Benchmark run: | - go run github.com/grafana/pyrobench@3fa67b1 -v compare --git-base 15dd5805938793bb952dd7772c8caab77ba8ff1a >> results.txt + go run github.com/grafana/pyrobench@3fa67b1 -v compare --git-base 5eca05b1e4cf63df5d007ec6b13ce7394a0fffe3 >> results.txt - name: Download test report run: curl -sL https://raw.githubusercontent.com/grafana/hackathon-2024-08-pyrobench/main/js/report.js -o report.js