Speed up playwright runs by sharding using tenbin

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
element-hq · Jan 9, 2025 · b394f5e · b394f5e
1 parent 9826a88
commit b394f5e
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 2 deletions.
diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml
@@ -164,13 +164,25 @@ jobs:
               if: matrix.project == 'WebKit' && steps.playwright-cache.outputs.cache-hit == 'true'
               run: yarn playwright install-deps webkit
 
+            # Restore test-results.json file, which records the execution time of each test file.
+            # splitTests function uses this file for sharding.
+            - name: Restore playwright-results.json
+              uses: actions/cache/restore@v4
+              with:
+                  path: playwright-results.json
+                  key: playwright-results-${{ github.ref_name }}
+                  restore-keys: |
+                      playwright-results-${{ github.base_ref }}
+                      playwright-results-develop
+
             # We skip tests tagged with @mergequeue when running on PRs, but run them in MQ and everywhere else
             - name: Run Playwright tests
               run: |
                   yarn playwright test \
-                      --shard "${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}" \
                       --project="${{ matrix.project }}" \
                       ${{ (github.event_name == 'pull_request' && matrix.runAllTests == false ) && '--grep-invert @mergequeue' || '' }}
+              env:
+                  SHARD: "${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}"
 
             - name: Upload blob report to GitHub Actions Artifacts
               if: always()
@@ -212,10 +224,17 @@ jobs:
 
             - name: Merge into HTML Report
               if: inputs.skip != true
-              run: yarn playwright merge-reports --reporter=html,./playwright/flaky-reporter.ts,./playwright/stale-screenshot-reporter.ts ./all-blob-reports
+              run: yarn playwright merge-reports --reporter=html,json,./playwright/flaky-reporter.ts,./playwright/stale-screenshot-reporter.ts ./all-blob-reports
               env:
                   # Only pass creds to the flaky-reporter on main branch runs
                   GITHUB_TOKEN: ${{ github.ref_name == 'develop' && secrets.ELEMENT_BOT_TOKEN || '' }}
+                  PLAYWRIGHT_JSON_OUTPUT_NAME: playwright-results.json
+
+            - name: Cache playwright-results.json
+              uses: actions/cache/save@v4
+              with:
+                  path: playwright-results.json
+                  key: playwright-results-${{ github.ref_name }}
 
             # Upload the HTML report even if one of our reporters fails, this can happen when stale screenshots are detected
             - name: Upload HTML report

diff --git a/package.json b/package.json
@@ -182,6 +182,7 @@
         "@sentry/webpack-plugin": "^2.7.1",
         "@stylistic/eslint-plugin": "^2.9.0",
         "@svgr/webpack": "^8.0.0",
+        "@tenbin/playwright": "^0.5.0",
         "@testcontainers/postgresql": "^10.16.0",
         "@testing-library/dom": "^10.4.0",
         "@testing-library/jest-dom": "^6.4.8",

diff --git a/playwright.config.ts b/playwright.config.ts
@@ -7,6 +7,7 @@ Please see LICENSE files in the repository root for full details.
 */
 
 import { defineConfig, devices } from "@playwright/test";
+import { splitTests } from "@tenbin/playwright/dist/index";
 
 const baseURL = process.env["BASE_URL"] ?? "http://localhost:8080";
 
@@ -70,4 +71,11 @@ export default defineConfig({
     snapshotDir: "playwright/snapshots",
     snapshotPathTemplate: "{snapshotDir}/{testFilePath}/{arg}-{platform}{ext}",
     forbidOnly: !!process.env.CI,
+    testMatch: process.env.CI
+        ? splitTests({
+              shard: process.env.SHARD ?? "1/1",
+              pattern: ["playwright/e2e/**/*.spec.ts"],
+              reportFile: "playwright-results.json",
+          })
+        : undefined,
 });
diff --git a/yarn.lock b/yarn.lock
@@ -2717,6 +2717,20 @@
     "@svgr/plugin-jsx" "8.1.0"
     "@svgr/plugin-svgo" "8.1.0"
 
+"@tenbin/core@0.5.0":
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/@tenbin/core/-/core-0.5.0.tgz#394f74d90e618e9c15a1f376871a5247b06e35d7"
+  integrity sha512-QawJTsn1svoLDJipr0dDl/L3npkiMmA1pO5OfveoV6/hajQSrEI0QJtxg/kXMyi0P+EUJ55KL0BVsDkh78OvZA==
+
+"@tenbin/playwright@^0.5.0":
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/@tenbin/playwright/-/playwright-0.5.0.tgz#8203d3419144fb812430663cc6362658df462abd"
+  integrity sha512-ApHQnl7a4avRa89CmDA+3C01MIxavV4HPtEmgXMvc8zg5AbFlmb+bu2VPQMtqfm4u4UgY0J/oO8SLjBABF5VCw==
+  dependencies:
+    "@tenbin/core" "0.5.0"
+    glob "^11.0.0"
+    picocolors "^1.1.0"
+
 "@testcontainers/postgresql@^10.16.0":
   version "10.16.0"
   resolved "https://registry.yarnpkg.com/@testcontainers/postgresql/-/postgresql-10.16.0.tgz#0437a9b426d64ea958e745a0e2ae19462b786f81"