diff --git a/.github/workflows/detailed.yaml b/.github/workflows/detailed.yaml index d0e970ea..7dd5ed00 100644 --- a/.github/workflows/detailed.yaml +++ b/.github/workflows/detailed.yaml @@ -20,8 +20,8 @@ jobs: run: npx tsc - name: Test detailed with title run: node dist/index.js tests ctrf-reports/ctrf-report.json --title "Detailed With Title" - - name: Test default no title - run: node dist/index.js ctrf-reports/ctrf-report.json --annotate false + - name: Test list + run: node dist/index.js test-list ctrf-reports/ctrf-report.json --annotate false --title "List With Title" - name: Upload test results uses: actions/upload-artifact@v4 with: diff --git a/src/index.ts b/src/index.ts index e07d7fef..f443065c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,6 +23,7 @@ import { generateSkippedTestsDetailsTable } from './views/skipped' import { generateFailedFoldedTable } from './views/failed-folded' import { generateTestSuiteFoldedTable } from './views/suite-folded' import { generateSuiteListView } from './views/suite-list' +import { generateTestListView } from './views/test-list' interface Arguments { _: Array @@ -75,6 +76,16 @@ const argv: Arguments = yargs(hideBin(process.argv)) }) } ) + .command( + 'test-list ', + 'Generate test list from a CTRF report', + (yargs) => { + return yargs.positional('file', { + describe: 'Path to the CTRF file', + type: 'string', + }) + } + ) .command( 'failed ', 'Generate fail test report from a CTRF report', @@ -391,6 +402,29 @@ if ((commandUsed === 'all' || commandUsed === '') && argv.file) { } catch (error) { console.error('Failed to read file:', error) } +} else if (argv._.includes('test-list') && argv.file) { + try { + let report = validateCtrfFile(argv.file) + report = stripAnsiFromErrors(report) + if (report !== null) { + if (argv.title) { + addHeading(title) + } + generateTestListView(report.results.tests, useSuiteName) + write() + if (argv.prComment) { + postPullRequestComment(report, apiUrl, baseUrl, onFailOnly, title, useSuiteName, prCommentMessage) + } + if (pullRequest) { + postPullRequestComment(report, apiUrl, baseUrl, onFailOnly, title, useSuiteName, core.summary.stringify()) + } + if (exitOnFail) { + exitActionOnFail(report) + } + } + } catch (error) { + console.error('Failed to read file:', error) + } } else if (argv._.includes('failed') && argv.file) { try { let report = validateCtrfFile(argv.file) diff --git a/src/views/flaky-rate.ts b/src/views/flaky-rate.ts index 76b029aa..6c2ed224 100644 --- a/src/views/flaky-rate.ts +++ b/src/views/flaky-rate.ts @@ -50,111 +50,80 @@ export async function generateFlakyRateSummary( } >() - reports.forEach((run) => { - const { tests } = run.results - - tests.forEach((test) => { - const testName = getTestName(test, useSuiteName) - - let data = flakyTestMap.get(testName) - if (!data) { - data = { - testName, - attempts: 0, - pass: 0, - fail: 0, - flakes: 0, - flakeRate: 0, + const calculateFlakeRate = (reportSubset: CtrfReport[]) => { + reportSubset.forEach((run) => { + const { tests } = run.results + + tests.forEach((test) => { + const testName = getTestName(test, useSuiteName) + + let data = flakyTestMap.get(testName) + if (!data) { + data = { + testName, + attempts: 0, + pass: 0, + fail: 0, + flakes: 0, + flakeRate: 0, + } + flakyTestMap.set(testName, data) } - flakyTestMap.set(testName, data) - } - if (test.status === 'passed' || test.status === 'failed') { - const testRuns = 1 + (test.retries || 0) - data.attempts += testRuns + if (test.status === 'passed' || test.status === 'failed') { + const testRuns = 1 + (test.retries || 0) + data.attempts += testRuns - let isFlaky = false + let isFlaky = false - if (test.flaky) { - isFlaky = true - } else if (test.retries && test.retries > 0 && test.status === 'passed') { - isFlaky = true - } + if (test.flaky) { + isFlaky = true + } else if (test.retries && test.retries > 0 && test.status === 'passed') { + isFlaky = true + } - if (isFlaky) { - data.flakes += test.retries || 0 - } + if (isFlaky) { + data.flakes += test.retries || 0 + } - if (test.status === 'passed') { - data.pass += 1 - data.fail += test.retries || 0 - } else if (test.status === 'failed') { - data.fail += 1 + (test.retries || 0) + if (test.status === 'passed') { + data.pass += 1 + data.fail += test.retries || 0 + } else if (test.status === 'failed') { + data.fail += 1 + (test.retries || 0) + } } - } + }) }) - }) - - const flakyTestArray = Array.from(flakyTestMap.values()) - - flakyTestArray.forEach((data) => { - data.flakeRate = data.attempts > 0 ? (data.flakes / data.attempts) * 100 : 0 - }) - - const totalAttemptsAllTests = flakyTestArray.reduce( - (sum, data) => sum + data.attempts, - 0 - ) - const totalFlakesAllTests = flakyTestArray.reduce( - (sum, data) => sum + data.flakes, - 0 - ) - const overallFlakeRate = - totalAttemptsAllTests > 0 ? (totalFlakesAllTests / totalAttemptsAllTests) * 100 : 0 - const overallFlakeRateFormatted = overallFlakeRate.toFixed(2) - const overallFlakeRateMessage = `**Overall Flaky Rate:** ${overallFlakeRateFormatted}%` - - const flakyTestArrayNonZero = flakyTestArray.filter( - (data) => data.flakeRate > 0 - ) - const totalRuns = reports.length - const totalRunsMessage = `Measured over ${totalRuns} runs.` - - if (flakyTestArrayNonZero.length === 0) { - const noFlakyMessage = `No flaky tests detected over ${totalRuns} runs.` - const summary = ` -${overallFlakeRateMessage} - -${noFlakyMessage} - -[Github Test Reporter CTRF](https://github.com/ctrf-io/github-test-reporter) -` - core.summary.addRaw(summary) - return + return Array.from(flakyTestMap.values()).reduce( + (sum, data) => (data.attempts > 0 ? sum + data.flakes / data.attempts : sum), + 0 + ) * 100 } - flakyTestArrayNonZero.sort((a, b) => b.flakeRate - a.flakeRate) - - const flakyRows = flakyTestArrayNonZero.map((data) => { - const { testName, attempts, pass, fail, flakeRate } = data - return `| ${testName} | ${attempts} | ${pass} | ${fail} | ${flakeRate.toFixed( - 2 - )}% |` - }) + // Normal flake rate using all test results + const overallFlakeRate = calculateFlakeRate(reports) - const limitedSummaryRows = flakyRows.slice(0, rows) + // Adjusted flake rate by excluding the latest 5 test results + const adjustedReports = reports.slice(5) + const adjustedFlakeRate = calculateFlakeRate(adjustedReports) - const summaryTable = ` -${overallFlakeRateMessage} + // Calculate flake rate change + const flakeRateChange = overallFlakeRate - adjustedFlakeRate + const flakeRateChangeMessage = `**Flake Rate Change:** ${flakeRateChange.toFixed( + 2 + )}%` -| Test 📝| Attempts 🎯| Pass ✅| Fail ❌| Flaky Rate 🍂| -| --- | --- | --- | --- | --- | -${limitedSummaryRows.join('\n')} + const overallFlakeRateMessage = `**Overall Flaky Rate:** ${overallFlakeRate.toFixed(2)}%` + const adjustedFlakeRateMessage = `**Adjusted Flaky Rate:** ${adjustedFlakeRate.toFixed(2)}%` -${totalRunsMessage} + const summary = ` +${overallFlakeRateMessage} +${adjustedFlakeRateMessage} +${flakeRateChangeMessage} [Github Test Reporter CTRF](https://github.com/ctrf-io/github-test-reporter-ctrf) ` - core.summary.addRaw(summaryTable) + core.summary.addRaw(summary) } diff --git a/src/views/suite-list.ts b/src/views/suite-list.ts index 8baf1d9c..94963855 100644 --- a/src/views/suite-list.ts +++ b/src/views/suite-list.ts @@ -1,5 +1,7 @@ import * as core from '@actions/core' import { CtrfTest } from '../../types/ctrf' +import { getEmojiForStatus } from './common' +import { stripAnsi } from '../common' export function generateSuiteListView(tests: CtrfTest[], useSuite: boolean): void { try { @@ -11,8 +13,8 @@ export function generateSuiteListView(tests: CtrfTest[], useSuite: boolean): voi tests.forEach((test) => { const groupKey = useSuite - ? test.suite || 'Unknown Suite' - : (test.filePath || 'Unknown File').replace(workspacePath, '').replace(/^\//, '') + ? test.suite || 'No suite provided' + : (test.filePath || 'No file path provided').replace(workspacePath, '').replace(/^\//, '') if (!testResultsByGroup[groupKey]) { testResultsByGroup[groupKey] = { tests: [], statusEmoji: '✅' } @@ -33,18 +35,15 @@ export function generateSuiteListView(tests: CtrfTest[], useSuite: boolean): voi markdown += `## ${groupData.statusEmoji} ${escapeMarkdown(groupKey)}\n\n` groupData.tests.forEach((test) => { - const statusEmoji = - test.status === 'passed' ? '✅' : - test.status === 'failed' ? '❌' : - test.status === 'skipped' ? '⏭️' : - test.status === 'pending' ? '⏳' : '❓' + const statusEmoji = getEmojiForStatus(test.status) - const testName = escapeMarkdown(test.name || 'Unnamed Test') + const testName = escapeMarkdown(test.name) markdown += `      **${statusEmoji} ${testName}**\n` if (test.status === 'failed' && test.message) { - const message = test.message.replace(/\n{2,}/g, '\n').trim() + let message = stripAnsi(test.message || "No failure message") + message = message.replace(/\n{2,}/g, '\n').trim() const escapedMessage = escapeMarkdown(message) diff --git a/src/views/test-list.ts b/src/views/test-list.ts new file mode 100644 index 00000000..1c4fa5b6 --- /dev/null +++ b/src/views/test-list.ts @@ -0,0 +1,48 @@ +import * as core from '@actions/core' +import { CtrfTest } from '../../types/ctrf' +import { getTestName, stripAnsi } from '../common' +import { getEmojiForStatus } from './common' + +export function generateTestListView(tests: CtrfTest[], useSuiteName: boolean): void { + try { + let markdown = `\n` + + function escapeMarkdown(text: string): string { + return text.replace(/([\\*_{}[\]()#+\-.!])/g, '\\$1') + } + + tests.forEach((test) => { + const statusEmoji = getEmojiForStatus(test.status) + + const testName = escapeMarkdown(getTestName(test, useSuiteName)) + + markdown += `**${statusEmoji} ${testName}**\n` + + if (test.status === 'failed') { + let message = stripAnsi(test.message || "No failure message") + message = message.replace(/\n{2,}/g, '\n').trim() + + const escapedMessage = escapeMarkdown(message) + + const indentedMessage = escapedMessage + .split('\n') + .filter(line => line.trim() !== '') + .map(line => `      ${line}`) + .join('\n') + + markdown += `${indentedMessage}\n` + } + }) + + markdown += `\n[Github Test Reporter](https://github.com/ctrf-io/github-test-reporter)` + + core.summary.addRaw(markdown) + + } catch (error) { + if (error instanceof Error) { + core.setFailed(`Failed to display test list: ${error.message}`) + } else { + core.setFailed('An unknown error occurred') + } + } +}