From 2f2fbbe56a3e3909da8daade9bb79322a3462182 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 17 Dec 2024 15:25:39 +0000 Subject: [PATCH 01/20] Run Playwright tests on Firefox Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/workflows/end-to-end-tests.yaml | 6 +++--- playwright.config.ts | 21 +++++++++++++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 5a75040866b..e1b9ef0a66d 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -124,11 +124,11 @@ jobs: with: path: | ~/.cache/ms-playwright - key: ${{ runner.os }}-playwright-${{ steps.playwright.outputs.version }}-chromium + key: ${{ runner.os }}-playwright-${{ steps.playwright.outputs.version }} - - name: Install Playwright browser + - name: Install Playwright browsers if: steps.playwright-cache.outputs.cache-hit != 'true' - run: yarn playwright install --with-deps --no-shell chromium + run: yarn playwright install --with-deps --no-shell # We skip tests tagged with @mergequeue when running on PRs, but run them in MQ and everywhere else - name: Run Playwright tests diff --git a/playwright.config.ts b/playwright.config.ts index 06c1b05322d..adf4a44adab 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -11,16 +11,29 @@ import { defineConfig, devices } from "@playwright/test"; const baseURL = process.env["BASE_URL"] ?? "http://localhost:8080"; export default defineConfig({ - projects: [{ name: "Chrome", use: { ...devices["Desktop Chrome"], channel: "chromium" } }], + projects: [ + { + name: "Chrome", + use: { + ...devices["Desktop Chrome"], + channel: "chromium", + launchOptions: { + args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"], + }, + }, + }, + { + name: "Firefox", + use: { ...devices["Desktop Firefox"], channel: "firefox", screenshot: "off" }, + // grep: /@firefox/, + }, + ], use: { viewport: { width: 1280, height: 720 }, ignoreHTTPSErrors: true, video: "retain-on-failure", baseURL, permissions: ["clipboard-write", "clipboard-read", "microphone"], - launchOptions: { - args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"], - }, trace: "on-first-retry", }, webServer: { From f006e3aefec56d1de33082778130122789baf9a9 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 17 Dec 2024 16:02:04 +0000 Subject: [PATCH 02/20] Update playwright.config.ts --- playwright.config.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index adf4a44adab..280776d8b1b 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -17,6 +17,7 @@ export default defineConfig({ use: { ...devices["Desktop Chrome"], channel: "chromium", + permissions: ["clipboard-write", "clipboard-read", "microphone"], launchOptions: { args: ["--use-fake-ui-for-media-stream", "--use-fake-device-for-media-stream", "--mute-audio"], }, @@ -24,8 +25,17 @@ export default defineConfig({ }, { name: "Firefox", - use: { ...devices["Desktop Firefox"], channel: "firefox", screenshot: "off" }, - // grep: /@firefox/, + use: { + ...devices["Desktop Firefox"], + channel: "firefox", + screenshot: "off", + launchOptions: { + firefoxUserPrefs: { + "permissions.default.microphone": 1, + }, + }, + }, + // grepInvert: /@no-firefox/, }, ], use: { @@ -33,7 +43,6 @@ export default defineConfig({ ignoreHTTPSErrors: true, video: "retain-on-failure", baseURL, - permissions: ["clipboard-write", "clipboard-read", "microphone"], trace: "on-first-retry", }, webServer: { From 929f16c074c66da4533bece23888c14107dd4c35 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 17 Dec 2024 16:31:41 +0000 Subject: [PATCH 03/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/workflows/end-to-end-tests.yaml | 2 +- playwright.config.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index e1b9ef0a66d..49b7608c876 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -135,7 +135,7 @@ jobs: run: | yarn playwright test \ --shard "${{ matrix.runner }}/${{ strategy.job-total }}" \ - ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '' }} + ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '--project Chrome' }} - name: Upload blob report to GitHub Actions Artifacts if: always() diff --git a/playwright.config.ts b/playwright.config.ts index 280776d8b1b..b0cb3f9e75e 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -35,7 +35,7 @@ export default defineConfig({ }, }, }, - // grepInvert: /@no-firefox/, + grepInvert: /@no-firefox/, }, ], use: { From ad6d8f9ea31d85bf1c44ace51669fd18c234143c Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 17 Dec 2024 17:10:30 +0000 Subject: [PATCH 04/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/workflows/end-to-end-tests.yaml | 2 +- playwright.config.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 49b7608c876..e1b9ef0a66d 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -135,7 +135,7 @@ jobs: run: | yarn playwright test \ --shard "${{ matrix.runner }}/${{ strategy.job-total }}" \ - ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '--project Chrome' }} + ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '' }} - name: Upload blob report to GitHub Actions Artifacts if: always() diff --git a/playwright.config.ts b/playwright.config.ts index b0cb3f9e75e..1ac9ac303a4 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -28,13 +28,13 @@ export default defineConfig({ use: { ...devices["Desktop Firefox"], channel: "firefox", - screenshot: "off", launchOptions: { firefoxUserPrefs: { "permissions.default.microphone": 1, }, }, }, + ignoreSnapshots: true, grepInvert: /@no-firefox/, }, ], From 15cf95f7a6737e9fc2116c649da41dfd5566ba6f Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 17 Dec 2024 19:04:47 +0000 Subject: [PATCH 05/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/crypto/dehydration.spec.ts | 3 ++ .../get-openid-token.spec.ts | 3 ++ .../e2e/integration-manager/kick.spec.ts | 3 ++ .../integration-manager/read_events.spec.ts | 3 ++ .../integration-manager/send_event.spec.ts | 3 ++ playwright/e2e/location/location.spec.ts | 3 +- playwright/e2e/login/soft_logout.spec.ts | 3 ++ playwright/e2e/oidc/index.ts | 3 ++ playwright/e2e/register/register.spec.ts | 7 +++- .../account-user-settings-tab.spec.ts | 3 ++ .../preferences-user-settings-tab.spec.ts | 2 +- .../threadsActivityCentre.spec.ts | 36 ++++++++++--------- playwright/e2e/timeline/timeline.spec.ts | 1 - playwright/e2e/update/update.spec.ts | 3 ++ playwright/plugins/docker/index.ts | 9 +++-- 15 files changed, 63 insertions(+), 22 deletions(-) diff --git a/playwright/e2e/crypto/dehydration.spec.ts b/playwright/e2e/crypto/dehydration.spec.ts index 590ab774b50..304294f4632 100644 --- a/playwright/e2e/crypto/dehydration.spec.ts +++ b/playwright/e2e/crypto/dehydration.spec.ts @@ -47,6 +47,9 @@ test.describe("Dehydration", () => { test.use({ displayName: NAME, + // This is needed to work around an issue between Playwright, Firefox, and Service workers + // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 + serviceWorkers: "block", }); test("Create dehydrated device", async ({ page, user, app }, workerInfo) => { diff --git a/playwright/e2e/integration-manager/get-openid-token.spec.ts b/playwright/e2e/integration-manager/get-openid-token.spec.ts index 48d52d35fa4..5c1e18cf0c0 100644 --- a/playwright/e2e/integration-manager/get-openid-token.spec.ts +++ b/playwright/e2e/integration-manager/get-openid-token.spec.ts @@ -62,6 +62,9 @@ test.describe("Integration Manager: Get OpenID Token", () => { }); await use({ roomId }); }, + // This is needed to work around an issue between Playwright, Firefox, and Service workers + // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 + serviceWorkers: "block", }); let integrationManagerUrl: string; diff --git a/playwright/e2e/integration-manager/kick.spec.ts b/playwright/e2e/integration-manager/kick.spec.ts index 59c2703a18d..e0894eadb9b 100644 --- a/playwright/e2e/integration-manager/kick.spec.ts +++ b/playwright/e2e/integration-manager/kick.spec.ts @@ -106,6 +106,9 @@ test.describe("Integration Manager: Kick", () => { displayName: BOT_DISPLAY_NAME, autoAcceptInvites: true, }, + // This is needed to work around an issue between Playwright, Firefox, and Service workers + // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 + serviceWorkers: "block", }); let integrationManagerUrl: string; diff --git a/playwright/e2e/integration-manager/read_events.spec.ts b/playwright/e2e/integration-manager/read_events.spec.ts index 791d5bd725b..433594704fc 100644 --- a/playwright/e2e/integration-manager/read_events.spec.ts +++ b/playwright/e2e/integration-manager/read_events.spec.ts @@ -77,6 +77,9 @@ test.describe("Integration Manager: Read Events", () => { }); await use({ roomId }); }, + // This is needed to work around an issue between Playwright, Firefox, and Service workers + // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 + serviceWorkers: "block", }); let integrationManagerUrl: string; diff --git a/playwright/e2e/integration-manager/send_event.spec.ts b/playwright/e2e/integration-manager/send_event.spec.ts index 363719d8f14..75e0bd1120e 100644 --- a/playwright/e2e/integration-manager/send_event.spec.ts +++ b/playwright/e2e/integration-manager/send_event.spec.ts @@ -83,6 +83,9 @@ test.describe("Integration Manager: Send Event", () => { }); await use({ roomId }); }, + // This is needed to work around an issue between Playwright, Firefox, and Service workers + // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 + serviceWorkers: "block", }); let integrationManagerUrl: string; diff --git a/playwright/e2e/location/location.spec.ts b/playwright/e2e/location/location.spec.ts index e0c23d6c22f..2277c16d4fc 100644 --- a/playwright/e2e/location/location.spec.ts +++ b/playwright/e2e/location/location.spec.ts @@ -10,7 +10,8 @@ import { Locator, Page } from "@playwright/test"; import { test, expect } from "../../element-web-test"; -test.describe("Location sharing", () => { +// Firefox headless lacks WebGL support https://bugzilla.mozilla.org/show_bug.cgi?id=1375585 +test.describe("Location sharing", { tag: "@no-firefox" }, () => { const selectLocationShareTypeOption = (page: Page, shareType: string): Locator => { return page.getByTestId(`share-location-option-${shareType}`); }; diff --git a/playwright/e2e/login/soft_logout.spec.ts b/playwright/e2e/login/soft_logout.spec.ts index ca0c11132a3..5402414e5c6 100644 --- a/playwright/e2e/login/soft_logout.spec.ts +++ b/playwright/e2e/login/soft_logout.spec.ts @@ -21,6 +21,9 @@ test.describe("Soft logout", () => { template: "default", oAuthServerPort: oAuthServer.port, }), + // This is needed to work around an issue between Playwright, Firefox, and Service workers + // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 + serviceWorkers: "block", }); test.describe("with password user", () => { diff --git a/playwright/e2e/oidc/index.ts b/playwright/e2e/oidc/index.ts index 9403406d80c..c2e0541a063 100644 --- a/playwright/e2e/oidc/index.ts +++ b/playwright/e2e/oidc/index.ts @@ -61,6 +61,9 @@ export const test = base.extend<{ default_server_config: wellKnown, }); }, + // This is needed to work around an issue between Playwright, Firefox, and Service workers + // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 + serviceWorkers: "block", }); export { expect }; diff --git a/playwright/e2e/register/register.spec.ts b/playwright/e2e/register/register.spec.ts index 19608ee174d..38647f7bd73 100644 --- a/playwright/e2e/register/register.spec.ts +++ b/playwright/e2e/register/register.spec.ts @@ -9,7 +9,12 @@ Please see LICENSE files in the repository root for full details. import { test, expect } from "../../element-web-test"; test.describe("Registration", () => { - test.use({ startHomeserverOpts: "consent" }); + test.use({ + startHomeserverOpts: "consent", + // This is needed to work around an issue between Playwright, Firefox, and Service workers + // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 + serviceWorkers: "block", + }); test.beforeEach(async ({ page }) => { await page.goto("/#/register"); diff --git a/playwright/e2e/settings/account-user-settings-tab.spec.ts b/playwright/e2e/settings/account-user-settings-tab.spec.ts index 7390ccfd8d1..93b9481c95c 100644 --- a/playwright/e2e/settings/account-user-settings-tab.spec.ts +++ b/playwright/e2e/settings/account-user-settings-tab.spec.ts @@ -21,6 +21,9 @@ test.describe("Account user settings tab", () => { const locator = await app.settings.openUserSettings("Account"); await use(locator); }, + // This is needed to work around an issue between Playwright, Firefox, and Service workers + // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 + serviceWorkers: "block", }); test("should be rendered properly", { tag: "@screenshot" }, async ({ uut, user }) => { diff --git a/playwright/e2e/settings/preferences-user-settings-tab.spec.ts b/playwright/e2e/settings/preferences-user-settings-tab.spec.ts index 8dc2570b426..2961623d76d 100644 --- a/playwright/e2e/settings/preferences-user-settings-tab.spec.ts +++ b/playwright/e2e/settings/preferences-user-settings-tab.spec.ts @@ -31,7 +31,7 @@ test.describe("Preferences user settings tab", () => { await expect(tab).toMatchScreenshot("Preferences-user-settings-tab-should-be-rendered-properly-1.png"); }); - test("should be able to change the app language", async ({ uut, user }) => { + test("should be able to change the app language", { tag: "@no-firefox" }, async ({ uut, user }) => { // Check language and region setting dropdown const languageInput = uut.getByRole("button", { name: "Language Dropdown" }); await languageInput.scrollIntoViewIfNeeded(); diff --git a/playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts b/playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts index ecf458c0600..4d6626c907a 100644 --- a/playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts +++ b/playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts @@ -64,22 +64,26 @@ test.describe("Threads Activity Centre", () => { await util.assertHighlightIndicator(); }); - test("should show the rooms with unread threads", { tag: "@screenshot" }, async ({ room1, room2, util, msg }) => { - await util.goTo(room2); - await util.populateThreads(room1, room2, msg); - // The indicator should be shown - await util.assertHighlightIndicator(); - - // Verify that we have the expected rooms in the TAC - await util.openTac(); - await util.assertRoomsInTac([ - { room: room2.name, notificationLevel: "highlight" }, - { room: room1.name, notificationLevel: "notification" }, - ]); - - // Verify that we don't have a visual regression - await expect(util.getTacPanel()).toMatchScreenshot("tac-panel-mix-unread.png"); - }); + test( + "should show the rooms with unread threads", + { tag: ["@screenshot", "@no-firefox"] }, + async ({ room1, room2, util, msg }) => { + await util.goTo(room2); + await util.populateThreads(room1, room2, msg); + // The indicator should be shown + await util.assertHighlightIndicator(); + + // Verify that we have the expected rooms in the TAC + await util.openTac(); + await util.assertRoomsInTac([ + { room: room2.name, notificationLevel: "highlight" }, + { room: room1.name, notificationLevel: "notification" }, + ]); + + // Verify that we don't have a visual regression + await expect(util.getTacPanel()).toMatchScreenshot("tac-panel-mix-unread.png"); + }, + ); test("should update with a thread is read", { tag: "@screenshot" }, async ({ room1, room2, util, msg }) => { await util.goTo(room2); diff --git a/playwright/e2e/timeline/timeline.spec.ts b/playwright/e2e/timeline/timeline.spec.ts index 7aaabb9759d..c532a53ae0f 100644 --- a/playwright/e2e/timeline/timeline.spec.ts +++ b/playwright/e2e/timeline/timeline.spec.ts @@ -914,7 +914,6 @@ test.describe("Timeline", () => { }); test("can reply with a voice message", async ({ page, app, room, context }) => { - await context.grantPermissions(["microphone"]); await viewRoomSendMessageAndSetupReply(page, app, room.roomId); const composerOptions = await app.openMessageComposerOptions(); diff --git a/playwright/e2e/update/update.spec.ts b/playwright/e2e/update/update.spec.ts index d073a334333..d6300692add 100644 --- a/playwright/e2e/update/update.spec.ts +++ b/playwright/e2e/update/update.spec.ts @@ -13,6 +13,9 @@ test.describe("Update", () => { test.use({ displayName: "Ursa", + // This is needed to work around an issue between Playwright, Firefox, and Service workers + // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 + serviceWorkers: "block", }); test.beforeEach(async ({ context }) => { diff --git a/playwright/plugins/docker/index.ts b/playwright/plugins/docker/index.ts index 895a7d0f123..533daf49ccb 100644 --- a/playwright/plugins/docker/index.ts +++ b/playwright/plugins/docker/index.ts @@ -10,6 +10,7 @@ import * as os from "os"; import * as crypto from "crypto"; import * as childProcess from "child_process"; import * as fse from "fs-extra"; +import { runtime } from "webpack"; /** * @param cmd - command to execute @@ -140,8 +141,12 @@ export class Docker { * Detects whether the docker command is actually podman. * To do this, it looks for "podman" in the output of "docker --help". */ + static _isPodman?: boolean; static async isPodman(): Promise { - const { stdout } = await exec("docker", ["--help"], true); - return stdout.toLowerCase().includes("podman"); + if (Docker._isPodman === undefined) { + const { stdout } = await exec("docker", ["--help"], true); + Docker._isPodman = stdout.toLowerCase().includes("podman"); + } + return Docker._isPodman; } } From d3067d083c0936dd724e8b770220293787148859 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 17 Dec 2024 19:11:24 +0000 Subject: [PATCH 06/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright.config.ts | 3 +++ playwright/e2e/crypto/dehydration.spec.ts | 3 --- playwright/e2e/integration-manager/get-openid-token.spec.ts | 3 --- playwright/e2e/integration-manager/kick.spec.ts | 3 --- playwright/e2e/integration-manager/read_events.spec.ts | 3 --- playwright/e2e/integration-manager/send_event.spec.ts | 3 --- playwright/e2e/login/soft_logout.spec.ts | 3 --- playwright/e2e/oidc/index.ts | 3 --- playwright/e2e/register/register.spec.ts | 3 --- playwright/e2e/settings/account-user-settings-tab.spec.ts | 3 --- playwright/e2e/update/update.spec.ts | 3 --- playwright/plugins/docker/index.ts | 1 - 12 files changed, 3 insertions(+), 31 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index 1ac9ac303a4..d35788b6ae1 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -33,6 +33,9 @@ export default defineConfig({ "permissions.default.microphone": 1, }, }, + // This is needed to work around an issue between Playwright, Firefox, and Service workers + // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 + serviceWorkers: "block", }, ignoreSnapshots: true, grepInvert: /@no-firefox/, diff --git a/playwright/e2e/crypto/dehydration.spec.ts b/playwright/e2e/crypto/dehydration.spec.ts index 304294f4632..590ab774b50 100644 --- a/playwright/e2e/crypto/dehydration.spec.ts +++ b/playwright/e2e/crypto/dehydration.spec.ts @@ -47,9 +47,6 @@ test.describe("Dehydration", () => { test.use({ displayName: NAME, - // This is needed to work around an issue between Playwright, Firefox, and Service workers - // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 - serviceWorkers: "block", }); test("Create dehydrated device", async ({ page, user, app }, workerInfo) => { diff --git a/playwright/e2e/integration-manager/get-openid-token.spec.ts b/playwright/e2e/integration-manager/get-openid-token.spec.ts index 5c1e18cf0c0..48d52d35fa4 100644 --- a/playwright/e2e/integration-manager/get-openid-token.spec.ts +++ b/playwright/e2e/integration-manager/get-openid-token.spec.ts @@ -62,9 +62,6 @@ test.describe("Integration Manager: Get OpenID Token", () => { }); await use({ roomId }); }, - // This is needed to work around an issue between Playwright, Firefox, and Service workers - // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 - serviceWorkers: "block", }); let integrationManagerUrl: string; diff --git a/playwright/e2e/integration-manager/kick.spec.ts b/playwright/e2e/integration-manager/kick.spec.ts index e0894eadb9b..59c2703a18d 100644 --- a/playwright/e2e/integration-manager/kick.spec.ts +++ b/playwright/e2e/integration-manager/kick.spec.ts @@ -106,9 +106,6 @@ test.describe("Integration Manager: Kick", () => { displayName: BOT_DISPLAY_NAME, autoAcceptInvites: true, }, - // This is needed to work around an issue between Playwright, Firefox, and Service workers - // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 - serviceWorkers: "block", }); let integrationManagerUrl: string; diff --git a/playwright/e2e/integration-manager/read_events.spec.ts b/playwright/e2e/integration-manager/read_events.spec.ts index 433594704fc..791d5bd725b 100644 --- a/playwright/e2e/integration-manager/read_events.spec.ts +++ b/playwright/e2e/integration-manager/read_events.spec.ts @@ -77,9 +77,6 @@ test.describe("Integration Manager: Read Events", () => { }); await use({ roomId }); }, - // This is needed to work around an issue between Playwright, Firefox, and Service workers - // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 - serviceWorkers: "block", }); let integrationManagerUrl: string; diff --git a/playwright/e2e/integration-manager/send_event.spec.ts b/playwright/e2e/integration-manager/send_event.spec.ts index 75e0bd1120e..363719d8f14 100644 --- a/playwright/e2e/integration-manager/send_event.spec.ts +++ b/playwright/e2e/integration-manager/send_event.spec.ts @@ -83,9 +83,6 @@ test.describe("Integration Manager: Send Event", () => { }); await use({ roomId }); }, - // This is needed to work around an issue between Playwright, Firefox, and Service workers - // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 - serviceWorkers: "block", }); let integrationManagerUrl: string; diff --git a/playwright/e2e/login/soft_logout.spec.ts b/playwright/e2e/login/soft_logout.spec.ts index 5402414e5c6..ca0c11132a3 100644 --- a/playwright/e2e/login/soft_logout.spec.ts +++ b/playwright/e2e/login/soft_logout.spec.ts @@ -21,9 +21,6 @@ test.describe("Soft logout", () => { template: "default", oAuthServerPort: oAuthServer.port, }), - // This is needed to work around an issue between Playwright, Firefox, and Service workers - // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 - serviceWorkers: "block", }); test.describe("with password user", () => { diff --git a/playwright/e2e/oidc/index.ts b/playwright/e2e/oidc/index.ts index c2e0541a063..9403406d80c 100644 --- a/playwright/e2e/oidc/index.ts +++ b/playwright/e2e/oidc/index.ts @@ -61,9 +61,6 @@ export const test = base.extend<{ default_server_config: wellKnown, }); }, - // This is needed to work around an issue between Playwright, Firefox, and Service workers - // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 - serviceWorkers: "block", }); export { expect }; diff --git a/playwright/e2e/register/register.spec.ts b/playwright/e2e/register/register.spec.ts index 38647f7bd73..c1274362668 100644 --- a/playwright/e2e/register/register.spec.ts +++ b/playwright/e2e/register/register.spec.ts @@ -11,9 +11,6 @@ import { test, expect } from "../../element-web-test"; test.describe("Registration", () => { test.use({ startHomeserverOpts: "consent", - // This is needed to work around an issue between Playwright, Firefox, and Service workers - // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 - serviceWorkers: "block", }); test.beforeEach(async ({ page }) => { diff --git a/playwright/e2e/settings/account-user-settings-tab.spec.ts b/playwright/e2e/settings/account-user-settings-tab.spec.ts index 93b9481c95c..7390ccfd8d1 100644 --- a/playwright/e2e/settings/account-user-settings-tab.spec.ts +++ b/playwright/e2e/settings/account-user-settings-tab.spec.ts @@ -21,9 +21,6 @@ test.describe("Account user settings tab", () => { const locator = await app.settings.openUserSettings("Account"); await use(locator); }, - // This is needed to work around an issue between Playwright, Firefox, and Service workers - // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 - serviceWorkers: "block", }); test("should be rendered properly", { tag: "@screenshot" }, async ({ uut, user }) => { diff --git a/playwright/e2e/update/update.spec.ts b/playwright/e2e/update/update.spec.ts index d6300692add..d073a334333 100644 --- a/playwright/e2e/update/update.spec.ts +++ b/playwright/e2e/update/update.spec.ts @@ -13,9 +13,6 @@ test.describe("Update", () => { test.use({ displayName: "Ursa", - // This is needed to work around an issue between Playwright, Firefox, and Service workers - // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 - serviceWorkers: "block", }); test.beforeEach(async ({ context }) => { diff --git a/playwright/plugins/docker/index.ts b/playwright/plugins/docker/index.ts index 533daf49ccb..6cc13860be3 100644 --- a/playwright/plugins/docker/index.ts +++ b/playwright/plugins/docker/index.ts @@ -10,7 +10,6 @@ import * as os from "os"; import * as crypto from "crypto"; import * as childProcess from "child_process"; import * as fse from "fs-extra"; -import { runtime } from "webpack"; /** * @param cmd - command to execute From 8ffce1a8675ed2f110a760cf443ed9c7911fa904 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 17 Dec 2024 20:14:53 +0000 Subject: [PATCH 07/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright.config.ts | 14 ++++++- .../e2e/audio-player/audio-player.spec.ts | 2 +- playwright/e2e/right-panel/file-panel.spec.ts | 2 +- .../threadsActivityCentre.spec.ts | 38 +++++++++---------- playwright/e2e/threads/threads.spec.ts | 4 +- playwright/e2e/timeline/timeline.spec.ts | 4 +- playwright/e2e/widgets/stickers.spec.ts | 2 +- 7 files changed, 36 insertions(+), 30 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index d35788b6ae1..b5aef11daec 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -27,19 +27,29 @@ export default defineConfig({ name: "Firefox", use: { ...devices["Desktop Firefox"], - channel: "firefox", launchOptions: { firefoxUserPrefs: { "permissions.default.microphone": 1, }, }, - // This is needed to work around an issue between Playwright, Firefox, and Service workers + // This is needed to work around an issue between Playwright routes, Firefox, and Service workers // https://github.com/microsoft/playwright/issues/33561#issuecomment-2471642120 serviceWorkers: "block", }, ignoreSnapshots: true, grepInvert: /@no-firefox/, }, + { + name: "Safari", + use: { + ...devices["Desktop Safari"], + // Seemingly Safari has the same issue as Firefox in Playwright routes not working + // https://playwright.dev/docs/network#missing-network-events-and-service-workers + serviceWorkers: "block", + }, + ignoreSnapshots: true, + grepInvert: /@no-webkit/, + }, ], use: { viewport: { width: 1280, height: 720 }, diff --git a/playwright/e2e/audio-player/audio-player.spec.ts b/playwright/e2e/audio-player/audio-player.spec.ts index 2bb9ab0be45..a54508b0714 100644 --- a/playwright/e2e/audio-player/audio-player.spec.ts +++ b/playwright/e2e/audio-player/audio-player.spec.ts @@ -13,7 +13,7 @@ import { SettingLevel } from "../../../src/settings/SettingLevel"; import { Layout } from "../../../src/settings/enums/Layout"; import { ElementAppPage } from "../../pages/ElementAppPage"; -test.describe("Audio player", () => { +test.describe("Audio player", { tag: "@no-firefox" }, () => { test.use({ displayName: "Hanako", }); diff --git a/playwright/e2e/right-panel/file-panel.spec.ts b/playwright/e2e/right-panel/file-panel.spec.ts index c535bcdfbb6..24e5f172b62 100644 --- a/playwright/e2e/right-panel/file-panel.spec.ts +++ b/playwright/e2e/right-panel/file-panel.spec.ts @@ -39,7 +39,7 @@ test.describe("FilePanel", () => { await expect(page.locator(".mx_FilePanel")).toBeVisible(); }); - test.describe("render", () => { + test.describe("render", { tag: "@no-firefox" }, () => { test("should render empty state", { tag: "@screenshot" }, async ({ page }) => { // Wait until the information about the empty state is rendered await expect(page.locator(".mx_EmptyState")).toBeVisible(); diff --git a/playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts b/playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts index 4d6626c907a..965047e75ef 100644 --- a/playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts +++ b/playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts @@ -9,7 +9,7 @@ import { expect, test } from "."; import { CommandOrControl } from "../../utils"; -test.describe("Threads Activity Centre", () => { +test.describe("Threads Activity Centre", { tag: "@no-firefox" }, () => { test.use({ displayName: "Alice", botCreateOpts: { displayName: "Other User" }, @@ -64,26 +64,22 @@ test.describe("Threads Activity Centre", () => { await util.assertHighlightIndicator(); }); - test( - "should show the rooms with unread threads", - { tag: ["@screenshot", "@no-firefox"] }, - async ({ room1, room2, util, msg }) => { - await util.goTo(room2); - await util.populateThreads(room1, room2, msg); - // The indicator should be shown - await util.assertHighlightIndicator(); - - // Verify that we have the expected rooms in the TAC - await util.openTac(); - await util.assertRoomsInTac([ - { room: room2.name, notificationLevel: "highlight" }, - { room: room1.name, notificationLevel: "notification" }, - ]); - - // Verify that we don't have a visual regression - await expect(util.getTacPanel()).toMatchScreenshot("tac-panel-mix-unread.png"); - }, - ); + test("should show the rooms with unread threads", { tag: "@screenshot" }, async ({ room1, room2, util, msg }) => { + await util.goTo(room2); + await util.populateThreads(room1, room2, msg); + // The indicator should be shown + await util.assertHighlightIndicator(); + + // Verify that we have the expected rooms in the TAC + await util.openTac(); + await util.assertRoomsInTac([ + { room: room2.name, notificationLevel: "highlight" }, + { room: room1.name, notificationLevel: "notification" }, + ]); + + // Verify that we don't have a visual regression + await expect(util.getTacPanel()).toMatchScreenshot("tac-panel-mix-unread.png"); + }); test("should update with a thread is read", { tag: "@screenshot" }, async ({ room1, room2, util, msg }) => { await util.goTo(room2); diff --git a/playwright/e2e/threads/threads.spec.ts b/playwright/e2e/threads/threads.spec.ts index 06ec57653c7..b4b5b9841f4 100644 --- a/playwright/e2e/threads/threads.spec.ts +++ b/playwright/e2e/threads/threads.spec.ts @@ -324,7 +324,7 @@ test.describe("Threads", () => { }); }); - test("can send voice messages", async ({ page, app, user }) => { + test("can send voice messages", { tag: "@no-firefox" }, async ({ page, app, user }) => { // Increase right-panel size, so that voice messages fit await page.evaluate(() => { window.localStorage.setItem("mx_rhs_size", "600"); @@ -353,7 +353,7 @@ test.describe("Threads", () => { test( "should send location and reply to the location on ThreadView", - { tag: "@screenshot" }, + { tag: ["@screenshot", "@no-firefox"] }, async ({ page, app, bot }) => { const roomId = await app.client.createRoom({}); await app.client.inviteUser(roomId, bot.credentials.userId); diff --git a/playwright/e2e/timeline/timeline.spec.ts b/playwright/e2e/timeline/timeline.spec.ts index c532a53ae0f..4183032bb34 100644 --- a/playwright/e2e/timeline/timeline.spec.ts +++ b/playwright/e2e/timeline/timeline.spec.ts @@ -90,7 +90,7 @@ test.describe("Timeline", () => { let oldAvatarUrl: string; let newAvatarUrl: string; - test.describe("useOnlyCurrentProfiles", () => { + test.describe("useOnlyCurrentProfiles", { tag: "@no-firefox" }, () => { test.beforeEach(async ({ app, user }) => { ({ content_uri: oldAvatarUrl } = await app.client.uploadContent(OLD_AVATAR, { type: "image/png" })); await app.client.setAvatarUrl(oldAvatarUrl); @@ -876,7 +876,7 @@ test.describe("Timeline", () => { }); }); - test.describe("message sending", () => { + test.describe("message sending", { tag: "@no-firefox" }, () => { const MESSAGE = "Hello world"; const reply = "Reply"; const viewRoomSendMessageAndSetupReply = async (page: Page, app: ElementAppPage, roomId: string) => { diff --git a/playwright/e2e/widgets/stickers.spec.ts b/playwright/e2e/widgets/stickers.spec.ts index 318f7129616..aeadedb1bc5 100644 --- a/playwright/e2e/widgets/stickers.spec.ts +++ b/playwright/e2e/widgets/stickers.spec.ts @@ -127,7 +127,7 @@ async function setWidgetAccountData( }); } -test.describe("Stickers", () => { +test.describe("Stickers", { tag: "@no-firefox" }, () => { test.use({ displayName: "Sally", room: async ({ app }, use) => { From c4c69f94c503625ea045c2333cf197f03b9aef99 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 17 Dec 2024 21:02:01 +0000 Subject: [PATCH 08/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/workflows/end-to-end-tests.yaml | 5 +++++ playwright/e2e/oidc/oidc-aware.spec.ts | 2 +- playwright/e2e/oidc/oidc-native.spec.ts | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index e1b9ef0a66d..6ba62d5d4ec 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -130,6 +130,11 @@ jobs: if: steps.playwright-cache.outputs.cache-hit != 'true' run: yarn playwright install --with-deps --no-shell + - name: Install system dependencies for WebKit + # Some WebKit dependencies seem to lay outside the cache and will need to be installed separately + if: github.event_name != 'pull_request' && steps.playwright-cache.outputs.cache-hit == 'true' + run: yarn playwright install-deps webkit + # We skip tests tagged with @mergequeue when running on PRs, but run them in MQ and everywhere else - name: Run Playwright tests run: | diff --git a/playwright/e2e/oidc/oidc-aware.spec.ts b/playwright/e2e/oidc/oidc-aware.spec.ts index a2f1e62714c..0f78d1f1649 100644 --- a/playwright/e2e/oidc/oidc-aware.spec.ts +++ b/playwright/e2e/oidc/oidc-aware.spec.ts @@ -9,7 +9,7 @@ Please see LICENSE files in the repository root for full details. import { test, expect, registerAccountMas } from "."; import { isDendrite } from "../../plugins/homeserver/dendrite"; -test.describe("OIDC Aware", () => { +test.describe("OIDC Aware", { tag: "@no-firefox" }, () => { test.skip(isDendrite, "does not yet support MAS"); test.slow(); // trace recording takes a while here diff --git a/playwright/e2e/oidc/oidc-native.spec.ts b/playwright/e2e/oidc/oidc-native.spec.ts index 3309826b634..a93af31c40f 100644 --- a/playwright/e2e/oidc/oidc-native.spec.ts +++ b/playwright/e2e/oidc/oidc-native.spec.ts @@ -10,7 +10,7 @@ import { test, expect, registerAccountMas } from "."; import { isDendrite } from "../../plugins/homeserver/dendrite"; import { ElementAppPage } from "../../pages/ElementAppPage.ts"; -test.describe("OIDC Native", () => { +test.describe("OIDC Native", { tag: "@no-firefox" }, () => { test.skip(isDendrite, "does not yet support MAS"); test.slow(); // trace recording takes a while here From c0a46e12ce651ca6394fb96c434d45ef1cc685ea Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 17 Dec 2024 21:37:49 +0000 Subject: [PATCH 09/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/workflows/end-to-end-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 6ba62d5d4ec..6d7761af975 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -132,7 +132,7 @@ jobs: - name: Install system dependencies for WebKit # Some WebKit dependencies seem to lay outside the cache and will need to be installed separately - if: github.event_name != 'pull_request' && steps.playwright-cache.outputs.cache-hit == 'true' + if: steps.playwright-cache.outputs.cache-hit == 'true' run: yarn playwright install-deps webkit # We skip tests tagged with @mergequeue when running on PRs, but run them in MQ and everywhere else From 1955033c408464a02b0d06ee988f623c03fd34f7 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 18 Dec 2024 10:17:22 +0000 Subject: [PATCH 10/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/audio-player/audio-player.spec.ts | 2 +- playwright/e2e/oidc/oidc-aware.spec.ts | 2 +- playwright/e2e/oidc/oidc-native.spec.ts | 2 +- playwright/e2e/right-panel/file-panel.spec.ts | 2 +- playwright/e2e/settings/preferences-user-settings-tab.spec.ts | 2 +- playwright/e2e/threads/threads.spec.ts | 2 +- playwright/e2e/timeline/timeline.spec.ts | 4 ++-- playwright/e2e/widgets/stickers.spec.ts | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/playwright/e2e/audio-player/audio-player.spec.ts b/playwright/e2e/audio-player/audio-player.spec.ts index a54508b0714..2749d7eb1d5 100644 --- a/playwright/e2e/audio-player/audio-player.spec.ts +++ b/playwright/e2e/audio-player/audio-player.spec.ts @@ -13,7 +13,7 @@ import { SettingLevel } from "../../../src/settings/SettingLevel"; import { Layout } from "../../../src/settings/enums/Layout"; import { ElementAppPage } from "../../pages/ElementAppPage"; -test.describe("Audio player", { tag: "@no-firefox" }, () => { +test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.use({ displayName: "Hanako", }); diff --git a/playwright/e2e/oidc/oidc-aware.spec.ts b/playwright/e2e/oidc/oidc-aware.spec.ts index 0f78d1f1649..7b155f27a4d 100644 --- a/playwright/e2e/oidc/oidc-aware.spec.ts +++ b/playwright/e2e/oidc/oidc-aware.spec.ts @@ -9,7 +9,7 @@ Please see LICENSE files in the repository root for full details. import { test, expect, registerAccountMas } from "."; import { isDendrite } from "../../plugins/homeserver/dendrite"; -test.describe("OIDC Aware", { tag: "@no-firefox" }, () => { +test.describe("OIDC Aware", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.skip(isDendrite, "does not yet support MAS"); test.slow(); // trace recording takes a while here diff --git a/playwright/e2e/oidc/oidc-native.spec.ts b/playwright/e2e/oidc/oidc-native.spec.ts index a93af31c40f..2ae5cf83e60 100644 --- a/playwright/e2e/oidc/oidc-native.spec.ts +++ b/playwright/e2e/oidc/oidc-native.spec.ts @@ -10,7 +10,7 @@ import { test, expect, registerAccountMas } from "."; import { isDendrite } from "../../plugins/homeserver/dendrite"; import { ElementAppPage } from "../../pages/ElementAppPage.ts"; -test.describe("OIDC Native", { tag: "@no-firefox" }, () => { +test.describe("OIDC Native", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.skip(isDendrite, "does not yet support MAS"); test.slow(); // trace recording takes a while here diff --git a/playwright/e2e/right-panel/file-panel.spec.ts b/playwright/e2e/right-panel/file-panel.spec.ts index 24e5f172b62..c5a106d8412 100644 --- a/playwright/e2e/right-panel/file-panel.spec.ts +++ b/playwright/e2e/right-panel/file-panel.spec.ts @@ -39,7 +39,7 @@ test.describe("FilePanel", () => { await expect(page.locator(".mx_FilePanel")).toBeVisible(); }); - test.describe("render", { tag: "@no-firefox" }, () => { + test.describe("render", { tag: ["@no-firefox", "@no-webkit"] }, () => { test("should render empty state", { tag: "@screenshot" }, async ({ page }) => { // Wait until the information about the empty state is rendered await expect(page.locator(".mx_EmptyState")).toBeVisible(); diff --git a/playwright/e2e/settings/preferences-user-settings-tab.spec.ts b/playwright/e2e/settings/preferences-user-settings-tab.spec.ts index 2961623d76d..fb2dae4eb07 100644 --- a/playwright/e2e/settings/preferences-user-settings-tab.spec.ts +++ b/playwright/e2e/settings/preferences-user-settings-tab.spec.ts @@ -31,7 +31,7 @@ test.describe("Preferences user settings tab", () => { await expect(tab).toMatchScreenshot("Preferences-user-settings-tab-should-be-rendered-properly-1.png"); }); - test("should be able to change the app language", { tag: "@no-firefox" }, async ({ uut, user }) => { + test("should be able to change the app language", { tag: ["@no-firefox", "@no-webkit"] }, async ({ uut, user }) => { // Check language and region setting dropdown const languageInput = uut.getByRole("button", { name: "Language Dropdown" }); await languageInput.scrollIntoViewIfNeeded(); diff --git a/playwright/e2e/threads/threads.spec.ts b/playwright/e2e/threads/threads.spec.ts index b4b5b9841f4..b6d72da3587 100644 --- a/playwright/e2e/threads/threads.spec.ts +++ b/playwright/e2e/threads/threads.spec.ts @@ -324,7 +324,7 @@ test.describe("Threads", () => { }); }); - test("can send voice messages", { tag: "@no-firefox" }, async ({ page, app, user }) => { + test("can send voice messages", { tag: ["@no-firefox", "@no-webkit"] }, async ({ page, app, user }) => { // Increase right-panel size, so that voice messages fit await page.evaluate(() => { window.localStorage.setItem("mx_rhs_size", "600"); diff --git a/playwright/e2e/timeline/timeline.spec.ts b/playwright/e2e/timeline/timeline.spec.ts index 4183032bb34..4761876de45 100644 --- a/playwright/e2e/timeline/timeline.spec.ts +++ b/playwright/e2e/timeline/timeline.spec.ts @@ -90,7 +90,7 @@ test.describe("Timeline", () => { let oldAvatarUrl: string; let newAvatarUrl: string; - test.describe("useOnlyCurrentProfiles", { tag: "@no-firefox" }, () => { + test.describe("useOnlyCurrentProfiles", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.beforeEach(async ({ app, user }) => { ({ content_uri: oldAvatarUrl } = await app.client.uploadContent(OLD_AVATAR, { type: "image/png" })); await app.client.setAvatarUrl(oldAvatarUrl); @@ -876,7 +876,7 @@ test.describe("Timeline", () => { }); }); - test.describe("message sending", { tag: "@no-firefox" }, () => { + test.describe("message sending", { tag: ["@no-firefox", "@no-webkit"] }, () => { const MESSAGE = "Hello world"; const reply = "Reply"; const viewRoomSendMessageAndSetupReply = async (page: Page, app: ElementAppPage, roomId: string) => { diff --git a/playwright/e2e/widgets/stickers.spec.ts b/playwright/e2e/widgets/stickers.spec.ts index aeadedb1bc5..71cd0804209 100644 --- a/playwright/e2e/widgets/stickers.spec.ts +++ b/playwright/e2e/widgets/stickers.spec.ts @@ -127,7 +127,7 @@ async function setWidgetAccountData( }); } -test.describe("Stickers", { tag: "@no-firefox" }, () => { +test.describe("Stickers", { tag: ["@no-firefox", "@no-webkit"] }, () => { test.use({ displayName: "Sally", room: async ({ app }, use) => { From 932d80079fb37def8745a5a2f8a444271331c9bb Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 18 Dec 2024 15:16:29 +0000 Subject: [PATCH 11/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/crypto/migration.spec.ts | 2 +- .../e2e/room-directory/room-directory.spec.ts | 68 ++++++++++-------- .../general-room-settings-tab.spec.ts | 2 +- playwright/e2e/spaces/spaces.spec.ts | 72 ++++++++++--------- 4 files changed, 78 insertions(+), 66 deletions(-) diff --git a/playwright/e2e/crypto/migration.spec.ts b/playwright/e2e/crypto/migration.spec.ts index 048b39f06a5..2a084ac4766 100644 --- a/playwright/e2e/crypto/migration.spec.ts +++ b/playwright/e2e/crypto/migration.spec.ts @@ -25,7 +25,7 @@ const test = base.extend({ }, }); -test.describe("migration", function () { +test.describe("migration", { tag: "@no-webkit" }, function () { test.use({ displayName: "Alice" }); test("Should support migration from legacy crypto", async ({ context, user, page }, workerInfo) => { diff --git a/playwright/e2e/room-directory/room-directory.spec.ts b/playwright/e2e/room-directory/room-directory.spec.ts index f299a929bb7..b3d2cf0ee99 100644 --- a/playwright/e2e/room-directory/room-directory.spec.ts +++ b/playwright/e2e/room-directory/room-directory.spec.ts @@ -15,37 +15,43 @@ test.describe("Room Directory", () => { botCreateOpts: { displayName: "Paul" }, }); - test("should allow admin to add alias & publish room to directory", async ({ page, app, user, bot }) => { - const roomId = await app.client.createRoom({ - name: "Gaming", - preset: "public_chat" as Preset, - }); - - await app.viewRoomByName("Gaming"); - await app.settings.openRoomSettings(); - - // First add a local address `gaming` - const localAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Local Addresses" }); - await localAddresses.getByRole("textbox").fill("gaming"); - await localAddresses.getByRole("button", { name: "Add" }).click(); - await expect(localAddresses.getByText("#gaming:localhost")).toHaveClass("mx_EditableItem_item"); - - // Publish into the public rooms directory - const publishedAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Published Addresses" }); - await expect(publishedAddresses.locator("#canonicalAlias")).toHaveValue("#gaming:localhost"); - const checkbox = publishedAddresses - .locator(".mx_SettingsFlag", { hasText: "Publish this room to the public in localhost's room directory?" }) - .getByRole("switch"); - await checkbox.check(); - await expect(checkbox).toBeChecked(); - - await app.closeDialog(); - - const resp = await bot.publicRooms({}); - expect(resp.total_room_count_estimate).toEqual(1); - expect(resp.chunk).toHaveLength(1); - expect(resp.chunk[0].room_id).toEqual(roomId); - }); + test( + "should allow admin to add alias & publish room to directory", + { tag: "@no-webkit" }, + async ({ page, app, user, bot }) => { + const roomId = await app.client.createRoom({ + name: "Gaming", + preset: "public_chat" as Preset, + }); + + await app.viewRoomByName("Gaming"); + await app.settings.openRoomSettings(); + + // First add a local address `gaming` + const localAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Local Addresses" }); + await localAddresses.getByRole("textbox").fill("gaming"); + await localAddresses.getByRole("button", { name: "Add" }).click(); + await expect(localAddresses.getByText("#gaming:localhost")).toHaveClass("mx_EditableItem_item"); + + // Publish into the public rooms directory + const publishedAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Published Addresses" }); + await expect(publishedAddresses.locator("#canonicalAlias")).toHaveValue("#gaming:localhost"); + const checkbox = publishedAddresses + .locator(".mx_SettingsFlag", { + hasText: "Publish this room to the public in localhost's room directory?", + }) + .getByRole("switch"); + await checkbox.check(); + await expect(checkbox).toBeChecked(); + + await app.closeDialog(); + + const resp = await bot.publicRooms({}); + expect(resp.total_room_count_estimate).toEqual(1); + expect(resp.chunk).toHaveLength(1); + expect(resp.chunk[0].room_id).toEqual(roomId); + }, + ); test( "should allow finding published rooms in directory", diff --git a/playwright/e2e/settings/general-room-settings-tab.spec.ts b/playwright/e2e/settings/general-room-settings-tab.spec.ts index 828ba5285bb..7102a258bca 100644 --- a/playwright/e2e/settings/general-room-settings-tab.spec.ts +++ b/playwright/e2e/settings/general-room-settings-tab.spec.ts @@ -36,7 +36,7 @@ test.describe("General room settings tab", () => { await expect(settings.getByText("Show more")).toBeVisible(); }); - test("long address should not cause dialog to overflow", async ({ page, app }) => { + test("long address should not cause dialog to overflow", { tag: "@no-webkit" }, async ({ page, app }) => { const settings = await app.settings.openRoomSettings("General"); // 1. Set the room-address to be a really long string const longString = "abcasdhjasjhdaj1jh1asdhasjdhajsdhjavhjksd".repeat(4); diff --git a/playwright/e2e/spaces/spaces.spec.ts b/playwright/e2e/spaces/spaces.spec.ts index 233cdee3b4b..374fc6b0687 100644 --- a/playwright/e2e/spaces/spaces.spec.ts +++ b/playwright/e2e/spaces/spaces.spec.ts @@ -55,38 +55,44 @@ test.describe("Spaces", () => { botCreateOpts: { displayName: "BotBob" }, }); - test("should allow user to create public space", { tag: "@screenshot" }, async ({ page, app, user }) => { - const contextMenu = await openSpaceCreateMenu(page); - await expect(contextMenu).toMatchScreenshot("space-create-menu.png"); - - await contextMenu.getByRole("button", { name: /Public/ }).click(); - - await contextMenu - .locator('.mx_SpaceBasicSettings_avatarContainer input[type="file"]') - .setInputFiles("playwright/sample-files/riot.png"); - await contextMenu.getByRole("textbox", { name: "Name" }).fill("Let's have a Riot"); - await expect(contextMenu.getByRole("textbox", { name: "Address" })).toHaveValue("lets-have-a-riot"); - await contextMenu.getByRole("textbox", { name: "Description" }).fill("This is a space to reminisce Riot.im!"); - await contextMenu.getByRole("button", { name: "Create" }).click(); - - // Create the default General & Random rooms, as well as a custom "Jokes" room - await expect(page.getByPlaceholder("General")).toBeVisible(); - await expect(page.getByPlaceholder("Random")).toBeVisible(); - await page.getByPlaceholder("Support").fill("Jokes"); - await page.getByRole("button", { name: "Continue" }).click(); - - // Copy matrix.to link - await page.getByRole("button", { name: "Share invite link" }).click(); - expect(await app.getClipboardText()).toEqual("https://matrix.to/#/#lets-have-a-riot:localhost"); - - // Go to space home - await page.getByRole("button", { name: "Go to my first room" }).click(); - - // Assert rooms exist in the room list - await expect(page.getByRole("treeitem", { name: "General" })).toBeVisible(); - await expect(page.getByRole("treeitem", { name: "Random" })).toBeVisible(); - await expect(page.getByRole("treeitem", { name: "Jokes" })).toBeVisible(); - }); + test( + "should allow user to create public space", + { tag: ["@screenshot", "@no-webkit"] }, + async ({ page, app, user }) => { + const contextMenu = await openSpaceCreateMenu(page); + await expect(contextMenu).toMatchScreenshot("space-create-menu.png"); + + await contextMenu.getByRole("button", { name: /Public/ }).click(); + + await contextMenu + .locator('.mx_SpaceBasicSettings_avatarContainer input[type="file"]') + .setInputFiles("playwright/sample-files/riot.png"); + await contextMenu.getByRole("textbox", { name: "Name" }).fill("Let's have a Riot"); + await expect(contextMenu.getByRole("textbox", { name: "Address" })).toHaveValue("lets-have-a-riot"); + await contextMenu + .getByRole("textbox", { name: "Description" }) + .fill("This is a space to reminisce Riot.im!"); + await contextMenu.getByRole("button", { name: "Create" }).click(); + + // Create the default General & Random rooms, as well as a custom "Jokes" room + await expect(page.getByPlaceholder("General")).toBeVisible(); + await expect(page.getByPlaceholder("Random")).toBeVisible(); + await page.getByPlaceholder("Support").fill("Jokes"); + await page.getByRole("button", { name: "Continue" }).click(); + + // Copy matrix.to link + await page.getByRole("button", { name: "Share invite link" }).click(); + expect(await app.getClipboardText()).toEqual("https://matrix.to/#/#lets-have-a-riot:localhost"); + + // Go to space home + await page.getByRole("button", { name: "Go to my first room" }).click(); + + // Assert rooms exist in the room list + await expect(page.getByRole("treeitem", { name: "General" })).toBeVisible(); + await expect(page.getByRole("treeitem", { name: "Random" })).toBeVisible(); + await expect(page.getByRole("treeitem", { name: "Jokes" })).toBeVisible(); + }, + ); test("should allow user to create private space", { tag: "@screenshot" }, async ({ page, app, user }) => { const menu = await openSpaceCreateMenu(page); @@ -157,7 +163,7 @@ test.describe("Spaces", () => { ).toBeVisible(); }); - test("should allow user to invite another to a space", async ({ page, app, user, bot }) => { + test("should allow user to invite another to a space", { tag: "@no-webkit" }, async ({ page, app, user, bot }) => { await app.client.createSpace({ visibility: "public" as any, room_alias_name: "space", From cf1fd31460652cfe59de21e5fa5b2c39662a4f59 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 18 Dec 2024 15:25:27 +0000 Subject: [PATCH 12/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/workflows/end-to-end-tests.yaml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 6d7761af975..61a4e53536d 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -3,6 +3,9 @@ # as an artifact and run end-to-end tests. name: End to End Tests on: + # CRON to run all Projects at 6am UTC + schedule: + - cron: "0 6 * * *" pull_request: {} merge_group: types: [checks_requested] @@ -93,6 +96,18 @@ jobs: matrix: # Run multiple instances in parallel to speed up the tests runner: [1, 2, 3, 4, 5, 6] + project: + - Chrome + - Firefox + - Safari + isCron: + - ${{ github.event_name == 'pull_request' }} + # Skip the Firefox & Safari runs unless this was a cron trigger + exclude: + - isCron: false + project: Firefox + - isCron: false + project: Safari steps: - uses: actions/checkout@v4 with: @@ -132,7 +147,7 @@ jobs: - name: Install system dependencies for WebKit # Some WebKit dependencies seem to lay outside the cache and will need to be installed separately - if: steps.playwright-cache.outputs.cache-hit == 'true' + if: matrix.project == 'Safari' && steps.playwright-cache.outputs.cache-hit == 'true' run: yarn playwright install-deps webkit # We skip tests tagged with @mergequeue when running on PRs, but run them in MQ and everywhere else @@ -140,13 +155,14 @@ jobs: run: | yarn playwright test \ --shard "${{ matrix.runner }}/${{ strategy.job-total }}" \ - ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '' }} + --project="${{ matrix.project }}" \ + ${{ '--grep-invert @mergequeue' || '' }} - name: Upload blob report to GitHub Actions Artifacts if: always() uses: actions/upload-artifact@v4 with: - name: all-blob-reports-${{ matrix.runner }} + name: all-blob-reports-${{ matrix.project }}-${{ matrix.runner }} path: blob-report retention-days: 1 From 031cf93d682f7892c724e7fe0e45c0149a971892 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 18 Dec 2024 17:58:04 +0000 Subject: [PATCH 13/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/workflows/end-to-end-tests.yaml | 22 ++- package.json | 2 +- playwright.config.ts | 2 - playwright/e2e/crypto/backups.spec.ts | 176 +++++++++--------- .../e2e/crypto/device-verification.spec.ts | 2 +- playwright/element-web-test.ts | 9 + 6 files changed, 119 insertions(+), 94 deletions(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 61a4e53536d..8d43df589b9 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -35,6 +35,8 @@ concurrency: env: # fetchdep.sh needs to know our PR number PR_NUMBER: ${{ github.event.pull_request.number }} + # Use 6 runners in the default case, but 4 when running on a schedule where we run all 5 projects (20 runners total) + NUM_RUNNERS: ${{ github.event_name == 'schedule' && 4 || 6 }} permissions: {} # No permissions required @@ -43,6 +45,9 @@ jobs: name: "Build Element-Web" runs-on: ubuntu-24.04 if: inputs.skip != true + outputs: + num-runners: ${{ env.NUM_RUNNERS }} + runners-matrix: ${{ steps.runner-vars.outputs.matrix }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -82,8 +87,17 @@ jobs: path: webapp retention-days: 1 + - name: Calculate runner variables + id: runner-vars + uses: actions/github-script@v7 + with: + script: | + const numRunners = parseInt(process.env.NUM_RUNNERS, 10); + const matrix = Array.from({ length: numRunners }, (_, i) => i + 1); + core.setOutput("matrix", JSON.stringify(matrix)); + playwright: - name: "Run Tests ${{ matrix.runner }}/${{ strategy.job-total }}" + name: "Run Tests [${{ matrix.project }}] ${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}" needs: build if: inputs.skip != true runs-on: ubuntu-24.04 @@ -95,7 +109,7 @@ jobs: fail-fast: false matrix: # Run multiple instances in parallel to speed up the tests - runner: [1, 2, 3, 4, 5, 6] + runner: ${{ fromJSON(needs.build.outputs.runners-matrix) }} project: - Chrome - Firefox @@ -154,9 +168,9 @@ jobs: - name: Run Playwright tests run: | yarn playwright test \ - --shard "${{ matrix.runner }}/${{ strategy.job-total }}" \ + --shard "${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}" \ --project="${{ matrix.project }}" \ - ${{ '--grep-invert @mergequeue' || '' }} + ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '' }} - name: Upload blob report to GitHub Actions Artifacts if: always() diff --git a/package.json b/package.json index 6d082f95865..87064de7e0b 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "test:playwright:open": "yarn test:playwright --ui", "test:playwright:screenshots": "yarn test:playwright:screenshots:build && yarn test:playwright:screenshots:run", "test:playwright:screenshots:build": "docker build playwright -t element-web-playwright", - "test:playwright:screenshots:run": "docker run --rm --network host -e BASE_URL -e CI -v $(pwd):/work/ -v $(node -e 'console.log(require(`path`).dirname(require.resolve(`matrix-js-sdk/package.json`)))'):/work/node_modules/matrix-js-sdk -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/:/tmp/ -it element-web-playwright --grep @screenshot", + "test:playwright:screenshots:run": "docker run --rm --network host -e BASE_URL -e CI -v $(pwd):/work/ -v $(node -e 'console.log(require(`path`).dirname(require.resolve(`matrix-js-sdk/package.json`)))'):/work/node_modules/matrix-js-sdk -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/:/tmp/ -it element-web-playwright --grep @screenshot --project=Chrome", "coverage": "yarn test --coverage", "analyse:unused-exports": "ts-node ./scripts/analyse_unused_exports.ts", "analyse:webpack-bundles": "webpack-bundle-analyzer webpack-stats.json webapp", diff --git a/playwright.config.ts b/playwright.config.ts index b5aef11daec..ea93dbc513f 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -37,7 +37,6 @@ export default defineConfig({ serviceWorkers: "block", }, ignoreSnapshots: true, - grepInvert: /@no-firefox/, }, { name: "Safari", @@ -48,7 +47,6 @@ export default defineConfig({ serviceWorkers: "block", }, ignoreSnapshots: true, - grepInvert: /@no-webkit/, }, ], use: { diff --git a/playwright/e2e/crypto/backups.spec.ts b/playwright/e2e/crypto/backups.spec.ts index d174cc89e5a..40c7dc0ac6c 100644 --- a/playwright/e2e/crypto/backups.spec.ts +++ b/playwright/e2e/crypto/backups.spec.ts @@ -51,90 +51,94 @@ test.describe("Backups", () => { displayName: "Hanako", }); - test("Create, delete and recreate a keys backup", async ({ page, user, app }, workerInfo) => { - // Create a backup - const securityTab = await app.settings.openUserSettings("Security & Privacy"); - - await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible(); - await securityTab.getByRole("button", { name: "Set up", exact: true }).click(); - - const currentDialogLocator = page.locator(".mx_Dialog"); - - // It's the first time and secure storage is not set up, so it will create one - await expect(currentDialogLocator.getByRole("heading", { name: "Set up Secure Backup" })).toBeVisible(); - await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click(); - await expect(currentDialogLocator.getByRole("heading", { name: "Save your Security Key" })).toBeVisible(); - await currentDialogLocator.getByRole("button", { name: "Copy", exact: true }).click(); - // copy the recovery key to use it later - const securityKey = await app.getClipboard(); - await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click(); - - await expect(currentDialogLocator.getByRole("heading", { name: "Secure Backup successful" })).toBeVisible(); - await currentDialogLocator.getByRole("button", { name: "Done", exact: true }).click(); - - // Open the settings again - await app.settings.openUserSettings("Security & Privacy"); - await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible(); - - // expand the advanced section to see the active version in the reports - await page - .locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced") - .locator("..") - .click(); - - await expectBackupVersionToBe(page, "1"); - - await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click(); - await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible(); - // Delete it - await currentDialogLocator.getByTestId("dialog-primary-button").click(); // Click "Delete Backup" - - // Create another - await securityTab.getByRole("button", { name: "Set up", exact: true }).click(); - await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible(); - await currentDialogLocator.getByLabel("Security Key").fill(securityKey); - await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click(); - - // Should be successful - await expect(currentDialogLocator.getByRole("heading", { name: "Success!" })).toBeVisible(); - await currentDialogLocator.getByRole("button", { name: "OK", exact: true }).click(); - - // Open the settings again - await app.settings.openUserSettings("Security & Privacy"); - await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible(); - - // expand the advanced section to see the active version in the reports - await page - .locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced") - .locator("..") - .click(); - - await expectBackupVersionToBe(page, "2"); - - // == - // Ensure that if you don't have the secret storage passphrase the backup won't be created - // == - - // First delete version 2 - await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click(); - await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible(); - // Click "Delete Backup" - await currentDialogLocator.getByTestId("dialog-primary-button").click(); - - // Try to create another - await securityTab.getByRole("button", { name: "Set up", exact: true }).click(); - await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible(); - // But cancel the security key dialog, to simulate not having the secret storage passphrase - await currentDialogLocator.getByTestId("dialog-cancel-button").click(); - - await expect(currentDialogLocator.getByRole("heading", { name: "Starting backup…" })).toBeVisible(); - // check that it failed - await expect(currentDialogLocator.getByText("Unable to create key backup")).toBeVisible(); - // cancel - await currentDialogLocator.getByTestId("dialog-cancel-button").click(); - - // go back to the settings to check that no backup was created (the setup button should still be there) - await app.settings.openUserSettings("Security & Privacy"); - await expect(securityTab.getByRole("button", { name: "Set up", exact: true })).toBeVisible(); - }); + test( + "Create, delete and recreate a keys backup", + { tag: "@no-webkit" }, + async ({ page, user, app }, workerInfo) => { + // Create a backup + const securityTab = await app.settings.openUserSettings("Security & Privacy"); + + await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible(); + await securityTab.getByRole("button", { name: "Set up", exact: true }).click(); + + const currentDialogLocator = page.locator(".mx_Dialog"); + + // It's the first time and secure storage is not set up, so it will create one + await expect(currentDialogLocator.getByRole("heading", { name: "Set up Secure Backup" })).toBeVisible(); + await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click(); + await expect(currentDialogLocator.getByRole("heading", { name: "Save your Security Key" })).toBeVisible(); + await currentDialogLocator.getByRole("button", { name: "Copy", exact: true }).click(); + // copy the recovery key to use it later + const securityKey = await app.getClipboard(); + await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click(); + + await expect(currentDialogLocator.getByRole("heading", { name: "Secure Backup successful" })).toBeVisible(); + await currentDialogLocator.getByRole("button", { name: "Done", exact: true }).click(); + + // Open the settings again + await app.settings.openUserSettings("Security & Privacy"); + await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible(); + + // expand the advanced section to see the active version in the reports + await page + .locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced") + .locator("..") + .click(); + + await expectBackupVersionToBe(page, "1"); + + await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click(); + await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible(); + // Delete it + await currentDialogLocator.getByTestId("dialog-primary-button").click(); // Click "Delete Backup" + + // Create another + await securityTab.getByRole("button", { name: "Set up", exact: true }).click(); + await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible(); + await currentDialogLocator.getByLabel("Security Key").fill(securityKey); + await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click(); + + // Should be successful + await expect(currentDialogLocator.getByRole("heading", { name: "Success!" })).toBeVisible(); + await currentDialogLocator.getByRole("button", { name: "OK", exact: true }).click(); + + // Open the settings again + await app.settings.openUserSettings("Security & Privacy"); + await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible(); + + // expand the advanced section to see the active version in the reports + await page + .locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced") + .locator("..") + .click(); + + await expectBackupVersionToBe(page, "2"); + + // == + // Ensure that if you don't have the secret storage passphrase the backup won't be created + // == + + // First delete version 2 + await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click(); + await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible(); + // Click "Delete Backup" + await currentDialogLocator.getByTestId("dialog-primary-button").click(); + + // Try to create another + await securityTab.getByRole("button", { name: "Set up", exact: true }).click(); + await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible(); + // But cancel the security key dialog, to simulate not having the secret storage passphrase + await currentDialogLocator.getByTestId("dialog-cancel-button").click(); + + await expect(currentDialogLocator.getByRole("heading", { name: "Starting backup…" })).toBeVisible(); + // check that it failed + await expect(currentDialogLocator.getByText("Unable to create key backup")).toBeVisible(); + // cancel + await currentDialogLocator.getByTestId("dialog-cancel-button").click(); + + // go back to the settings to check that no backup was created (the setup button should still be there) + await app.settings.openUserSettings("Security & Privacy"); + await expect(securityTab.getByRole("button", { name: "Set up", exact: true })).toBeVisible(); + }, + ); }); diff --git a/playwright/e2e/crypto/device-verification.spec.ts b/playwright/e2e/crypto/device-verification.spec.ts index 83a81c260cd..032b649b8d7 100644 --- a/playwright/e2e/crypto/device-verification.spec.ts +++ b/playwright/e2e/crypto/device-verification.spec.ts @@ -21,7 +21,7 @@ import { } from "./utils"; import { Bot } from "../../pages/bot"; -test.describe("Device verification", () => { +test.describe("Device verification", { tag: "@no-webkit" }, () => { let aliceBotClient: Bot; /** The backup version that was set up by the bot client. */ diff --git a/playwright/element-web-test.ts b/playwright/element-web-test.ts index 76e57e33f70..5145347f3bf 100644 --- a/playwright/element-web-test.ts +++ b/playwright/element-web-test.ts @@ -125,6 +125,15 @@ export const test = base.extend<{ labsFlags: string[]; webserver: Webserver; }>({ + context: async ({ context }, use, testInfo) => { + // We skip tests instead of using grep-invert to still surface the counts in the html report + test.skip( + testInfo.tags.includes(`@no-${testInfo.project.name.toLowerCase()}`), + `Test does not work on ${testInfo.project.name}`, + ); + await use(context); + }, + config: CONFIG_JSON, page: async ({ context, page, config, labsFlags }, use) => { await context.route(`http://localhost:8080/config.json*`, async (route) => { From 303e639580bbd132fa0dbedcb8499d25b15efc05 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 18 Dec 2024 21:30:02 +0000 Subject: [PATCH 14/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright.config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playwright.config.ts b/playwright.config.ts index ea93dbc513f..0b2bd1bd02d 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -39,10 +39,10 @@ export default defineConfig({ ignoreSnapshots: true, }, { - name: "Safari", + name: "WebKit", use: { ...devices["Desktop Safari"], - // Seemingly Safari has the same issue as Firefox in Playwright routes not working + // Seemingly WebKit has the same issue as Firefox in Playwright routes not working // https://playwright.dev/docs/network#missing-network-events-and-service-workers serviceWorkers: "block", }, From 6581e1b5399be88ee855a33620f0a268e79f4206 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 18 Dec 2024 21:37:22 +0000 Subject: [PATCH 15/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/workflows/end-to-end-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 8d43df589b9..ba39733b40c 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -113,9 +113,9 @@ jobs: project: - Chrome - Firefox - - Safari isCron: - ${{ github.event_name == 'pull_request' }} + - WebKit # Skip the Firefox & Safari runs unless this was a cron trigger exclude: - isCron: false From 107dcccceaf6b70d6e6eae1e60cc05c946e6ca21 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 18 Dec 2024 21:41:36 +0000 Subject: [PATCH 16/20] Update end-to-end-tests.yaml --- .github/workflows/end-to-end-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index ba39733b40c..7c8af7424bb 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -113,9 +113,9 @@ jobs: project: - Chrome - Firefox + - WebKit isCron: - ${{ github.event_name == 'pull_request' }} - - WebKit # Skip the Firefox & Safari runs unless this was a cron trigger exclude: - isCron: false From 6130ce90e903bf5a514280358bd7febb69497c22 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 18 Dec 2024 21:57:48 +0000 Subject: [PATCH 17/20] Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/workflows/end-to-end-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index ba39733b40c..2738e7fd5ee 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -161,7 +161,7 @@ jobs: - name: Install system dependencies for WebKit # Some WebKit dependencies seem to lay outside the cache and will need to be installed separately - if: matrix.project == 'Safari' && steps.playwright-cache.outputs.cache-hit == 'true' + if: matrix.project == 'WebKit' && steps.playwright-cache.outputs.cache-hit == 'true' run: yarn playwright install-deps webkit # We skip tests tagged with @mergequeue when running on PRs, but run them in MQ and everywhere else From 9dfe151c3dc02c61f5fdf61c46db9bea2610c1e2 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 18 Dec 2024 22:34:54 +0000 Subject: [PATCH 18/20] Finalise Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/workflows/end-to-end-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index fbc293bd14d..787fddb93a6 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -115,7 +115,7 @@ jobs: - Firefox - WebKit isCron: - - ${{ github.event_name == 'pull_request' }} + - ${{ github.event_name == 'schedule' }} # Skip the Firefox & Safari runs unless this was a cron trigger exclude: - isCron: false From 9e871d1f13457b9e52179089b5894b7442a14b14 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Wed, 18 Dec 2024 22:36:15 +0000 Subject: [PATCH 19/20] Documentation Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- docs/playwright.md | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/playwright.md b/docs/playwright.md index 4af3194220a..73ee77228b7 100644 --- a/docs/playwright.md +++ b/docs/playwright.md @@ -53,15 +53,11 @@ yarn run test:playwright:open --headed --debug See more command line options at . -### Running with Rust cryptography +## Projects -`matrix-js-sdk` is currently in the -[process](https://github.com/vector-im/element-web/issues/21972) of being -updated to replace its end-to-end encryption implementation to use the [Matrix -Rust SDK](https://github.com/matrix-org/matrix-rust-sdk). This is not currently -enabled by default, but it is possible to have Playwright configure Element to use -the Rust crypto implementation by passing `--project="Rust Crypto"` or using -the top left options in open mode. +By default, Playwright will run all "Projects", this means tests will run against Chrome, Firefox and "Safari" (Webkit). +We only run tests against Chrome in pull request CI, but all projects in the merge queue. +Some tests are excluded from running on certain browsers due to incompatibilities in the test harness. ## How the Tests Work @@ -224,3 +220,14 @@ We use test tags to categorise tests for running subsets more efficiently. - `@mergequeue`: Tests that are slow or flaky and cover areas of the app we update seldom, should not be run on every PR commit but will be run in the Merge Queue. - `@screenshot`: Tests that use `toMatchScreenshot` to speed up a run of `test:playwright:screenshots`. A test with this tag must not also have the `@mergequeue` tag as this would cause false positives in the stale screenshot detection. +- `@no-$project`: Tests which are unsupported in $Project. These tests will be skipped when running in $Project. + +Anything testing Matrix media will need to have `@no-firefox` and `@no-webkit` as those rely on the service worker which +has to be disabled in Playwright on Firefox & Webkit to retain routing functionality. +Anything testing VoIP/microphone will need to have `@no-webkit` as fake microphone functionality is not available +there at this time. + +## Colima + +If you are running under Colima, you may need to set the environment variable `TMPDIR` to `/tmp/colima` or a path +within `$HOME` to allow bind mounting temporary directories into the Docker containers. From 5c31f198ed85281f0cfe78f7caf60012ae3b9f4e Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Thu, 19 Dec 2024 09:48:42 +0000 Subject: [PATCH 20/20] typo Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/workflows/end-to-end-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 787fddb93a6..6afabdb1fe6 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -121,7 +121,7 @@ jobs: - isCron: false project: Firefox - isCron: false - project: Safari + project: WebKit steps: - uses: actions/checkout@v4 with: