Run Playwright tests on Firefox & "Safari" nightly

.github/workflows/end-to-end-tests.yaml

@@ -3,6 +3,9 @@
 # as an artifact and run end-to-end tests.
 name: End to End Tests
 on:
+    # CRON to run all Projects at 6am UTC
+    schedule:
+        - cron: "0 6 * * *"
     pull_request: {}
     merge_group:
         types: [checks_requested]
@@ -32,6 +35,8 @@ concurrency:
 env:
     # fetchdep.sh needs to know our PR number
     PR_NUMBER: ${{ github.event.pull_request.number }}
+    # Use 6 runners in the default case, but 4 when running on a schedule where we run all 5 projects (20 runners total)
+    NUM_RUNNERS: ${{ github.event_name == 'schedule' && 4 || 6 }}
 
 permissions: {} # No permissions required
 
@@ -40,6 +45,9 @@ jobs:
         name: "Build Element-Web"
         runs-on: ubuntu-24.04
         if: inputs.skip != true
+        outputs:
+            num-runners: ${{ env.NUM_RUNNERS }}
+            runners-matrix: ${{ steps.runner-vars.outputs.matrix }}
         steps:
             - name: Checkout code
               uses: actions/checkout@v4
@@ -79,8 +87,17 @@ jobs:
                   path: webapp
                   retention-days: 1
 
+            - name: Calculate runner variables
+              id: runner-vars
+              uses: actions/github-script@v7
+              with:
+                  script: |
+                      const numRunners = parseInt(process.env.NUM_RUNNERS, 10);
+                      const matrix = Array.from({ length: numRunners }, (_, i) => i + 1);
+                      core.setOutput("matrix", JSON.stringify(matrix));
+
     playwright:
-        name: "Run Tests ${{ matrix.runner }}/${{ strategy.job-total }}"
+        name: "Run Tests [${{ matrix.project }}] ${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}"
         needs: build
         if: inputs.skip != true
         runs-on: ubuntu-24.04
@@ -92,7 +109,19 @@ jobs:
             fail-fast: false
             matrix:
                 # Run multiple instances in parallel to speed up the tests
-                runner: [1, 2, 3, 4, 5, 6]
+                runner: ${{ fromJSON(needs.build.outputs.runners-matrix) }}
+                project:
+                    - Chrome
+                    - Firefox
+                    - WebKit
+                isCron:
+                    - ${{ github.event_name == 'schedule' }}
+                # Skip the Firefox & Safari runs unless this was a cron trigger
+                exclude:
+                    - isCron: false
+                      project: Firefox
+                    - isCron: false
+                      project: WebKit
         steps:
             - uses: actions/checkout@v4
               with:
@@ -124,24 +153,30 @@ jobs:
               with:
                   path: |
                       ~/.cache/ms-playwright
-                  key: ${{ runner.os }}-playwright-${{ steps.playwright.outputs.version }}-chromium
+                  key: ${{ runner.os }}-playwright-${{ steps.playwright.outputs.version }}
 
-            - name: Install Playwright browser
+            - name: Install Playwright browsers
               if: steps.playwright-cache.outputs.cache-hit != 'true'
-              run: yarn playwright install --with-deps --no-shell chromium
+              run: yarn playwright install --with-deps --no-shell
+
+            - name: Install system dependencies for WebKit
+              # Some WebKit dependencies seem to lay outside the cache and will need to be installed separately
+              if: matrix.project == 'WebKit' && steps.playwright-cache.outputs.cache-hit == 'true'
+              run: yarn playwright install-deps webkit
 
             # We skip tests tagged with @mergequeue when running on PRs, but run them in MQ and everywhere else
             - name: Run Playwright tests
               run: |
                   yarn playwright test \
-                      --shard "${{ matrix.runner }}/${{ strategy.job-total }}" \
+                      --shard "${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}" \
+                      --project="${{ matrix.project }}" \
                       ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '' }}
 
             - name: Upload blob report to GitHub Actions Artifacts
               if: always()
               uses: actions/upload-artifact@v4
               with:
-                  name: all-blob-reports-${{ matrix.runner }}
+                  name: all-blob-reports-${{ matrix.project }}-${{ matrix.runner }}
                   path: blob-report
                   retention-days: 1
 

docs/playwright.md

@@ -53,15 +53,11 @@ yarn run test:playwright:open --headed --debug
 
 See more command line options at <https://playwright.dev/docs/test-cli>.
 
-### Running with Rust cryptography
+## Projects
 
-`matrix-js-sdk` is currently in the
-[process](https://github.com/vector-im/element-web/issues/21972) of being
-updated to replace its end-to-end encryption implementation to use the [Matrix
-Rust SDK](https://github.com/matrix-org/matrix-rust-sdk). This is not currently
-enabled by default, but it is possible to have Playwright configure Element to use
-the Rust crypto implementation by passing `--project="Rust Crypto"` or using
-the top left options in open mode.
+By default, Playwright will run all "Projects", this means tests will run against Chrome, Firefox and "Safari" (Webkit).
+We only run tests against Chrome in pull request CI, but all projects in the merge queue.
+Some tests are excluded from running on certain browsers due to incompatibilities in the test harness.
 
 ## How the Tests Work
 
@@ -224,3 +220,14 @@ We use test tags to categorise tests for running subsets more efficiently.
 
 - `@mergequeue`: Tests that are slow or flaky and cover areas of the app we update seldom, should not be run on every PR commit but will be run in the Merge Queue.
 - `@screenshot`: Tests that use `toMatchScreenshot` to speed up a run of `test:playwright:screenshots`. A test with this tag must not also have the `@mergequeue` tag as this would cause false positives in the stale screenshot detection.
+- `@no-$project`: Tests which are unsupported in $Project. These tests will be skipped when running in $Project.
+
+Anything testing Matrix media will need to have `@no-firefox` and `@no-webkit` as those rely on the service worker which
+has to be disabled in Playwright on Firefox & Webkit to retain routing functionality.
+Anything testing VoIP/microphone will need to have `@no-webkit` as fake microphone functionality is not available
+there at this time.
+
+## Colima
+
+If you are running under Colima, you may need to set the environment variable `TMPDIR` to `/tmp/colima` or a path
+within `$HOME` to allow bind mounting temporary directories into the Docker containers.

package.json

@@ -64,7 +64,7 @@
         "test:playwright:open": "yarn test:playwright --ui",
         "test:playwright:screenshots": "yarn test:playwright:screenshots:build && yarn test:playwright:screenshots:run",
         "test:playwright:screenshots:build": "docker build playwright -t element-web-playwright",
-        "test:playwright:screenshots:run": "docker run --rm --network host -e BASE_URL -e CI -v $(pwd):/work/ -v $(node -e 'console.log(require(`path`).dirname(require.resolve(`matrix-js-sdk/package.json`)))'):/work/node_modules/matrix-js-sdk -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/:/tmp/ -it element-web-playwright --grep @screenshot",
+        "test:playwright:screenshots:run": "docker run --rm --network host -e BASE_URL -e CI -v $(pwd):/work/ -v $(node -e 'console.log(require(`path`).dirname(require.resolve(`matrix-js-sdk/package.json`)))'):/work/node_modules/matrix-js-sdk -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/:/tmp/ -it element-web-playwright --grep @screenshot --project=Chrome",
         "coverage": "yarn test --coverage",
         "analyse:unused-exports": "ts-node ./scripts/analyse_unused_exports.ts",
         "analyse:webpack-bundles": "webpack-bundle-analyzer webpack-stats.json webapp",

playwright.config.ts

@@ -11,16 +11,49 @@ import { defineConfig, devices } from "@playwright/test";
 const baseURL = process.env["BASE_URL"] ?? "http://localhost:8080";
 
 export default defineConfig({
-    projects: [{ name: "Chrome", use: { ...devices["Desktop Chrome"], channel: "chromium" } }],
+    projects: [
+        {
+            name: "Chrome",
+            use: {
+                ...devices["Desktop Chrome"],
+                channel: "chromium",
+                permissions: ["clipboard-write", "clipboard-read", "microphone"],
+                launchOptions: {
+                    args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"],
+                },
+            },
+        },
+        {
+            name: "Firefox",
+            use: {
+                ...devices["Desktop Firefox"],
+                launchOptions: {
+                    firefoxUserPrefs: {
+                        "permissions.default.microphone": 1,
+                    },
+                },
+                // This is needed to work around an issue between Playwright routes, Firefox, and Service workers
+                // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120
+                serviceWorkers: "block",
+            },
+            ignoreSnapshots: true,
+        },
+        {
+            name: "WebKit",
+            use: {
+                ...devices["Desktop Safari"],
+                // Seemingly WebKit has the same issue as Firefox in Playwright routes not working
+                // https://playwright.dev/docs/network#missing-network-events-and-service-workers
+                serviceWorkers: "block",
+            },
+            ignoreSnapshots: true,
+        },
+    ],
     use: {
         viewport: { width: 1280, height: 720 },
         ignoreHTTPSErrors: true,
         video: "retain-on-failure",
         baseURL,
-        permissions: ["clipboard-write", "clipboard-read", "microphone"],
-        launchOptions: {
-            args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"],
-        },
         trace: "on-first-retry",
     },
     webServer: {

playwright/e2e/audio-player/audio-player.spec.ts

@@ -13,7 +13,7 @@ import { SettingLevel } from "../../../src/settings/SettingLevel";
 import { Layout } from "../../../src/settings/enums/Layout";
 import { ElementAppPage } from "../../pages/ElementAppPage";
 
-test.describe("Audio player", () => {
+test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => {
     test.use({
         displayName: "Hanako",
     });