diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3d068995..abfe3795 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -20,7 +20,8 @@ "version": "latest", "installBicep": true }, - "ghcr.io/azure/azure-dev/azd:latest": {} + "ghcr.io/azure/azure-dev/azd:latest": {}, + "ghcr.io/devcontainers-contrib/features/k6:1": {} }, // Configure tool-specific properties. diff --git a/package-lock.json b/package-lock.json index e94c91f6..d5a76cb1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "devDependencies": { "@playwright/test": "^1.39.0", "@tapjs/nock": "^3.1.13", + "@types/k6": "^0.47.1", "@types/node": "^18.15.3", "concurrently": "^8.2.1", "eslint-config-shared": "^1.0.0", @@ -3613,6 +3614,12 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "peer": true }, + "node_modules/@types/k6": { + "version": "0.47.1", + "resolved": "https://registry.npmjs.org/@types/k6/-/k6-0.47.1.tgz", + "integrity": "sha512-dQCSM3WT4f5MWCyyxmY/fO+BSYS217nvlvOhDvqg4lZGfzbAuS6pOG+UuwoH+3mLTChccEds+0Q0q7YWBbtgMQ==", + "dev": true + }, "node_modules/@types/mime": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.2.tgz", diff --git a/package.json b/package.json index cd563ab3..0db94dc3 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "start:indexer": "npm run dev --workspace=indexer", "test": "npm run test -ws --if-present", "test:playwright": "npx playwright test", + "test:load": "k6 run tests/load/index.js", "build": "npm run build -ws --if-present", "clean": "npm run clean -ws --if-present", "docker:build": "npm run docker:build -ws --if-present", @@ -33,6 +34,7 @@ "@playwright/test": "^1.39.0", "@tapjs/nock": "^3.1.13", "@types/node": "^18.15.3", + "@types/k6": "^0.47.1", "concurrently": "^8.2.1", "eslint-config-shared": "^1.0.0", "lint-staged": "^14.0.1", diff --git a/packages/eslint-config/index.js b/packages/eslint-config/index.js index 66ede11c..6bf5ded0 100644 --- a/packages/eslint-config/index.js +++ b/packages/eslint-config/index.js @@ -1,4 +1,7 @@ module.exports = { + globals: { + __ENV: 'readonly', + }, parserOptions: { ecmaVersion: 'latest', sourceType: 'module', diff --git a/tests/load/README.md b/tests/load/README.md new file mode 100644 index 00000000..a820c5a1 --- /dev/null +++ b/tests/load/README.md @@ -0,0 +1,22 @@ +The tests use [k6](https://k6.io/) to perform load testing. + +# Install k6 + +k6 is already included in the dev container, so no further installation is required. + +For manual installation, refer to [k6 installation docs](https://k6.io/docs/get-started/installation/). + +# To run the test + +Set the following environment variables to point to the deployment. + +``` +export WEBAPP_URI= +export SEARCH_API_URI= +``` + +Once set, you can now run load tests using the following command: + +``` +npm run test:load +``` diff --git a/tests/load/chat.js b/tests/load/chat.js new file mode 100644 index 00000000..00c2379d --- /dev/null +++ b/tests/load/chat.js @@ -0,0 +1,66 @@ +import http from 'k6/http'; +import { Trend } from 'k6/metrics'; +import { group, sleep } from 'k6'; + +const chatStreamLatency = new Trend('chat_stream_duration'); +const chatNoStreamLatency = new Trend('chat_nostream_duration'); + +function between(min, max) { + min = Math.ceil(min); + max = Math.floor(max); + return Math.floor(Math.random() * (max - min) + min); // The maximum is exclusive and the minimum is inclusive +} + +function choose(list) { + return list[between(0, list.length)]; +} + +export function chat(baseUrl, stream = true) { + group('Chat flow', function () { + const defaultPrompts = [ + 'How to search and book rentals?', + 'What is the refund policy?', + 'How to contact a representative?', + ]; + + const payload = JSON.stringify({ + messages: [{ content: choose(defaultPrompts), role: 'user' }], + context: { + retrieval_mode: 'hybrid', + semantic_ranker: true, + semantic_captions: false, + suggest_followup_questions: true, + retrievalMode: 'hybrid', + top: 3, + useSemanticRanker: true, + useSemanticCaptions: false, + excludeCategory: '', + promptTemplate: '', + promptTemplatePrefix: '', + promptTemplateSuffix: '', + suggestFollowupQuestions: true, + approach: 'rrr', + }, + stream, + }); + + const parameters = { + headers: { + 'Content-Type': 'application/json', + }, + tags: { type: 'API' }, + }; + + const response = http.post(`${baseUrl}/chat`, payload, parameters); + + if (response.status !== 200) { + console.log(`Response: ${response.status} ${response.body}`); + } + + // add duration property to metric + const latencyMetric = stream ? chatStreamLatency : chatNoStreamLatency; + latencyMetric.add(response.timings.duration, { type: 'API' }); + + sleep(between(5, 20)); // wait between 5 and 20 seconds between each user iteration + }); +} diff --git a/tests/load/config.js b/tests/load/config.js new file mode 100644 index 00000000..130e7b76 --- /dev/null +++ b/tests/load/config.js @@ -0,0 +1,16 @@ +export const thresholdsSettings = { + 'http_req_failed{type:API}': [{ threshold: 'rate<0.01' }], // less than 1% failed requests + 'http_req_failed{type:content}': [{ threshold: 'rate<0.01' }], // less than 1% failed requests + 'http_req_duration{type:API}': ['p(90)<40000'], // 90% of the API requests must complete below 40s + 'http_req_duration{type:content}': ['p(99)<200'], // 99% of the content requests must complete below 200ms +}; + +// 5.00 iterations/s for 1m0s (maxVUs: 100-200, gracefulStop: 30s) +export const standardWorkload = { + executor: 'constant-arrival-rate', + rate: 5, + timeUnit: '1s', + duration: '1m', + preAllocatedVUs: 100, + maxVUs: 200, +}; diff --git a/tests/load/index.js b/tests/load/index.js new file mode 100644 index 00000000..38a43108 --- /dev/null +++ b/tests/load/index.js @@ -0,0 +1,19 @@ +import { mainpage } from './mainpage.js'; +import { chat } from './chat.js'; +import { thresholdsSettings, standardWorkload } from './config.js'; + +export const options = { + scenarios: { + staged: standardWorkload, + }, + thresholds: thresholdsSettings, +}; + +const webappUrl = __ENV.WEBAPP_URI; +const searchUrl = __ENV.SEARCH_API_URI; + +export default function () { + mainpage(webappUrl); + chat(searchUrl, true); + //chat(searchUrl, false); +} diff --git a/tests/load/mainpage.js b/tests/load/mainpage.js new file mode 100644 index 00000000..98472124 --- /dev/null +++ b/tests/load/mainpage.js @@ -0,0 +1,15 @@ +import http from 'k6/http'; +import { Trend } from 'k6/metrics'; +import { group, sleep } from 'k6'; + +const mainpageLatency = new Trend('mainpage_duration'); + +export function mainpage(baseUrl) { + group('Mainpage', function () { + // save response as variable + const response = http.get(`${baseUrl}`, { tags: { type: 'content' } }); + // add duration property to metric + mainpageLatency.add(response.timings.duration, { type: 'content' }); + sleep(1); + }); +}