From 5dad274786b54b8e59efef4a8514776a0195d61b Mon Sep 17 00:00:00 2001
From: shibbas <26466942+shibbas@users.noreply.github.com>
Date: Thu, 26 Oct 2023 09:18:24 -0700
Subject: [PATCH] test: add E2E playwright tests for web app (#89)
* added initial test support for the playwright tests
* add snapshot and thought process test along with default network response
* playwright config update
* ux update
* test screenshot updates
* update tests
* no stream har files
* embed the body in the har files
* stream har file updated
* update the screenshot for response formatting
* add screenshots for firefox and webkit tests
* fix the testid attribute
* remove playwright workflow for now
* add the github action for running local tests and fix a broken test
* only upload artifacts on failure
* ensure playwright browsers are installed in devcontainer
* move the tests to a e2e
* update types
* keep playwright tests separate
* cleanup tests
---
.devcontainer/devcontainer.json | 5 +-
.devcontainer/postCreateCommand.sh | 7 +
.github/workflows/playwright.yml | 27 +++
.gitignore | 3 +
package-lock.json | 75 ++++++--
package.json | 6 +-
packages/chat-component/src/main.ts | 9 +-
.../SettingsButton/SettingsButton.tsx | 6 +-
playwright.config.ts | 82 +++++++++
.../hars/default-chat-response-nostream.har | 75 ++++++++
.../e2e/hars/default-chat-response-stream.har | 75 ++++++++
tests/e2e/webapp.spec.ts | 173 ++++++++++++++++++
12 files changed, 527 insertions(+), 16 deletions(-)
create mode 100755 .devcontainer/postCreateCommand.sh
create mode 100644 .github/workflows/playwright.yml
create mode 100644 playwright.config.ts
create mode 100644 tests/e2e/hars/default-chat-response-nostream.har
create mode 100644 tests/e2e/hars/default-chat-response-stream.har
create mode 100644 tests/e2e/webapp.spec.ts
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 1459e9ee..3d068995 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -33,7 +33,8 @@
"esbenp.prettier-vscode",
"humao.rest-client",
"GitHub.copilot",
- "runem.lit-plugin"
+ "runem.lit-plugin",
+ "ms-playwright.playwright"
]
}
},
@@ -42,7 +43,7 @@
"forwardPorts": [3000, 3001, 5173],
// Use 'postCreateCommand' to run commands after the container is created.
- "postCreateCommand": "npm install",
+ "postCreateCommand": "./.devcontainer/postCreateCommand.sh",
// Set minimal host requirements for the container.
"hostRequirements": {
diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh
new file mode 100755
index 00000000..a50a9893
--- /dev/null
+++ b/.devcontainer/postCreateCommand.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "Installing dependencies..."
+npm install
+
+echo "Installing playwright browsers..."
+npx playwright install --with-deps
diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml
new file mode 100644
index 00000000..a1d52131
--- /dev/null
+++ b/.github/workflows/playwright.yml
@@ -0,0 +1,27 @@
+name: Playwright Tests
+on:
+ push:
+ branches: [main]
+ pull_request:
+ branches: [main]
+jobs:
+ test:
+ timeout-minutes: 60
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-node@v3
+ with:
+ node-version: 18
+ - name: Install dependencies
+ run: npm ci
+ - name: Install Playwright Browsers
+ run: npm run install:playwright
+ - name: Run Playwright tests
+ run: npm run test:playwright
+ - uses: actions/upload-artifact@v3
+ if: failure()
+ with:
+ name: playwright-report
+ path: playwright-report/
+ retention-days: 30
diff --git a/.gitignore b/.gitignore
index 84bfa743..0bc8ef4c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,3 +20,6 @@ dist/
# misc
TODO
+/test-results/
+/playwright-report/
+/playwright/.cache/
diff --git a/package-lock.json b/package-lock.json
index ddf7c504..e94c91f6 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -12,7 +12,9 @@
"packages/*"
],
"devDependencies": {
+ "@playwright/test": "^1.39.0",
"@tapjs/nock": "^3.1.13",
+ "@types/node": "^18.15.3",
"concurrently": "^8.2.1",
"eslint-config-shared": "^1.0.0",
"lint-staged": "^14.0.1",
@@ -2794,6 +2796,21 @@
"node": ">=14"
}
},
+ "node_modules/@playwright/test": {
+ "version": "1.39.0",
+ "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.39.0.tgz",
+ "integrity": "sha512-3u1iFqgzl7zr004bGPYiN/5EZpRUSFddQBra8Rqll5N0/vfpqlP9I9EXqAoGacuAbX6c9Ulg/Cjqglp5VkK6UQ==",
+ "dev": true,
+ "dependencies": {
+ "playwright": "1.39.0"
+ },
+ "bin": {
+ "playwright": "cli.js"
+ },
+ "engines": {
+ "node": ">=16"
+ }
+ },
"node_modules/@react-spring/animated": {
"version": "9.7.3",
"resolved": "https://registry.npmjs.org/@react-spring/animated/-/animated-9.7.3.tgz",
@@ -3611,12 +3628,9 @@
}
},
"node_modules/@types/node": {
- "version": "20.8.6",
- "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.6.tgz",
- "integrity": "sha512-eWO4K2Ji70QzKUqRy6oyJWUeB7+g2cRagT3T/nxYibYcT4y2BDL8lqolRXjTHmkZCdJfIPaY73KbJAZmcryxTQ==",
- "dependencies": {
- "undici-types": "~5.25.1"
- }
+ "version": "18.15.3",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.3.tgz",
+ "integrity": "sha512-p6ua9zBxz5otCmbpb5D3U4B5Nanw6Pk3PPyX05xnxbB/fRv71N7CPmORg7uAD5P70T0xmx1pzAx/FUfa5X+3cw=="
},
"node_modules/@types/node-fetch": {
"version": "2.6.6",
@@ -10604,6 +10618,50 @@
"node": ">=8"
}
},
+ "node_modules/playwright": {
+ "version": "1.39.0",
+ "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.39.0.tgz",
+ "integrity": "sha512-naE5QT11uC/Oiq0BwZ50gDmy8c8WLPRTEWuSSFVG2egBka/1qMoSqYQcROMT9zLwJ86oPofcTH2jBY/5wWOgIw==",
+ "devOptional": true,
+ "dependencies": {
+ "playwright-core": "1.39.0"
+ },
+ "bin": {
+ "playwright": "cli.js"
+ },
+ "engines": {
+ "node": ">=16"
+ },
+ "optionalDependencies": {
+ "fsevents": "2.3.2"
+ }
+ },
+ "node_modules/playwright-core": {
+ "version": "1.39.0",
+ "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.39.0.tgz",
+ "integrity": "sha512-+k4pdZgs1qiM+OUkSjx96YiKsXsmb59evFoqv8SKO067qBA+Z2s/dCzJij/ZhdQcs2zlTAgRKfeiiLm8PQ2qvw==",
+ "devOptional": true,
+ "bin": {
+ "playwright-core": "cli.js"
+ },
+ "engines": {
+ "node": ">=16"
+ }
+ },
+ "node_modules/playwright/node_modules/fsevents": {
+ "version": "2.3.2",
+ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+ "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+ "dev": true,
+ "hasInstallScript": true,
+ "optional": true,
+ "os": [
+ "darwin"
+ ],
+ "engines": {
+ "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+ }
+ },
"node_modules/pluralize": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz",
@@ -12782,11 +12840,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
- "node_modules/undici-types": {
- "version": "5.25.3",
- "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.25.3.tgz",
- "integrity": "sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA=="
- },
"node_modules/unique-filename": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-3.0.0.tgz",
diff --git a/package.json b/package.json
index ce15601e..cd563ab3 100644
--- a/package.json
+++ b/package.json
@@ -12,6 +12,7 @@
"start:search": "npm run dev --workspace=search",
"start:indexer": "npm run dev --workspace=indexer",
"test": "npm run test -ws --if-present",
+ "test:playwright": "npx playwright test",
"build": "npm run build -ws --if-present",
"clean": "npm run clean -ws --if-present",
"docker:build": "npm run docker:build -ws --if-present",
@@ -19,7 +20,8 @@
"format:check": "prettier --check .",
"lint": "eslint .",
"lint:fix": "eslint --fix .",
- "prepare": "simple-git-hooks || echo 'simple-git-hooks install skipped'"
+ "prepare": "simple-git-hooks || echo 'simple-git-hooks install skipped'",
+ "install:playwright": "npx playwright install --with-deps"
},
"keywords": [],
"author": "Microsoft",
@@ -28,7 +30,9 @@
"packages/*"
],
"devDependencies": {
+ "@playwright/test": "^1.39.0",
"@tapjs/nock": "^3.1.13",
+ "@types/node": "^18.15.3",
"concurrently": "^8.2.1",
"eslint-config-shared": "^1.0.0",
"lint-staged": "^14.0.1",
diff --git a/packages/chat-component/src/main.ts b/packages/chat-component/src/main.ts
index 7e9b01d4..e4fedacf 100644
--- a/packages/chat-component/src/main.ts
+++ b/packages/chat-component/src/main.ts
@@ -368,6 +368,7 @@ export class ChatComponent extends LitElement {
${citation.ref}. ${citation.text}
@@ -435,6 +437,7 @@ export class ChatComponent extends LitElement {