test: [M3-6855, M3-6876] - Add script to generate JUnit test summaries (

#9998)

* Allow Cypress test suite name to be read by tests and plugins

* Add dev dependencies for JUnit summarizer

* Add "yarn junit:summary" package script

package.json

@@ -44,6 +44,7 @@
     "generate-changelogs": "node scripts/changelog/generate-changelogs.mjs",
     "coverage": "yarn workspace linode-manager coverage",
     "coverage:summary": "yarn workspace linode-manager coverage:summary",
+    "junit:summary": "ts-node scripts/junit-summary/index.ts",
     "docs": "bunx vitepress@1.0.0-rc.35 dev docs"
   },
   "resolutions": {

packages/manager/cypress/support/plugins/configure-test-suite.ts

@@ -9,6 +9,9 @@ const envVarName = 'CY_TEST_SUITE';
  * If `CY_TEST_SUITE` is undefined or invalid, the 'core' test suite will be run
  * by default.
  *
+ * The resolved test suite name can be read by tests and other plugins via
+ * `Cypress.env('cypress_test_suite')`.
+ *
  * @returns Cypress configuration object.
  */
 export const configureTestSuite: CypressPlugin = (_on, config) => {
@@ -32,6 +35,7 @@ export const configureTestSuite: CypressPlugin = (_on, config) => {
     }
   })();
 
+  config.env['cypress_test_suite'] = suiteName;
   config.specPattern = `cypress/e2e/${suiteName}/**/*.spec.{ts,tsx}`;
   return config;
 };

packages/manager/cypress/support/plugins/junit-report.ts

@@ -3,17 +3,31 @@ import { CypressPlugin } from './plugin';
 // The name of the environment variable to read when checking report configuration.
 const envVarName = 'CY_TEST_JUNIT_REPORT';
 
+const capitalize = (str: string): string => {
+  return `${str.charAt(0).toUpperCase()}${str.slice(1)}`;
+};
+
 /**
  * Enables and configures JUnit reporting when `CY_TEST_JUNIT_REPORT` is defined.
  *
  * @returns Cypress configuration object.
  */
 export const enableJunitReport: CypressPlugin = (_on, config) => {
   if (!!config.env[envVarName]) {
-    config.reporter = 'junit';
+    const testSuite = config.env['cypress_test_suite'] || 'core';
+    const testSuiteName = `${capitalize(testSuite)} Test Suite`;
+
+    // Cypress doesn't know to look for modules in the root `node_modules`
+    // directory, so we have to pass a relative path.
+    // See also: https://github.com/cypress-io/cypress/issues/6406
+    config.reporter = '../../node_modules/mocha-junit-reporter';
+
+    // See also: https://www.npmjs.com/package/mocha-junit-reporter#full-configuration-options
     config.reporterOptions = {
       mochaFile: 'cypress/results/test-results-[hash].xml',
-      testsuitesTitle: 'Cloud Manager Cypress Tests',
+      rootSuiteTitle: 'Cloud Manager Cypress Tests',
+      testsuitesTitle: testSuiteName,
+      jenkinsMode: false,
     };
   }
   return config;

packages/manager/package.json

@@ -167,6 +167,7 @@
     "@vitest/ui": "^1.0.4",
     "chai-string": "^1.5.0",
     "chalk": "^5.2.0",
+    "commander": "^6.2.1",
     "css-mediaquery": "^0.1.2",
     "cypress": "^13.5.0",
     "cypress-axe": "^1.0.0",
@@ -195,14 +196,17 @@
     "glob": "^10.3.1",
     "jest-axe": "^8.0.0",
     "jsdom": "^22.1.0",
+    "junit2json": "^3.1.4",
     "lint-staged": "^13.2.2",
+    "mocha-junit-reporter": "^2.2.1",
     "msw": "~1.3.2",
     "prettier": "~2.2.1",
     "redux-mock-store": "^1.5.3",
     "reselect-tools": "^0.0.7",
     "serve": "^14.0.1",
     "storybook": "~7.6.4",
     "storybook-dark-mode": "^3.0.3",
+    "ts-node": "^10.9.2",
     "vite": "^5.0.7",
     "vite-plugin-svgr": "^3.2.0",
     "vitest": "^1.0.4"

scripts/junit-summary/formatters/formatter.ts

@@ -0,0 +1,21 @@
+import type { TestSuites } from 'junit2json';
+import type { RunInfo } from '../results/run-info';
+import type { TestResult } from '../results/test-result';
+import type { Metadata } from '../metadata/metadata';
+
+/**
+ * A function that outputs a test result summary in some format.
+ */
+export type Formatter = (
+  /** Test run information. */
+  runInfo: RunInfo,
+
+  /** Test results. */
+  results: TestResult[],
+
+  /** Additional test run metadata. */
+  metadata: Metadata,
+
+  /** Raw JUnit test result data. */
+  junitData: TestSuites[]
+) => string;

scripts/junit-summary/formatters/github-formatter.ts

@@ -0,0 +1,90 @@
+import type { Formatter } from './formatter';
+import type { TestResult } from '../results/test-result';
+import type { TestSuites } from 'junit2json';
+import type { RunInfo } from '../results/run-info';
+import type { Metadata } from '../metadata/metadata';
+import { pluralize } from '../util/pluralize';
+import { secondsToTimeString } from '../util';
+import * as path from 'path';
+import { cypressRunCommand } from '../util/cypress';
+import { escapeHtmlString } from '../util/escape';
+
+/**
+ * Outputs test result summary formatted as a GitHub comment.
+ *
+ * @param info - Run info.
+ * @param results - Test results.
+ * @param metadata - Run metadata.
+ * @param _junitData - Raw JUnit test result data (unused).
+ */
+export const githubFormatter: Formatter = (
+  runInfo: RunInfo,
+  results: TestResult[],
+  metadata: Metadata,
+  _junitData: TestSuites[]
+) => {
+  const headline = (() => {
+    const headingMarkdown = '## ';
+    const description = runInfo.failing
+      ? `${runInfo.failing} failing ${pluralize(runInfo.failing, 'test', 'tests')} on`
+      : `Passing`;
+
+    // If available, render a link for the run.
+    const runLink = (metadata.runId && metadata.runUrl)
+      ? `[test run #${escapeHtmlString(metadata.runId)} ↗︎](${escapeHtmlString(metadata.runUrl)})`
+      : 'test run';
+
+    return `${headingMarkdown}${description} ${runLink}`;
+  })();
+
+  const breakdown = `:x: ${runInfo.failing} Failing | :green_heart: ${runInfo.passing} Passing | :arrow_right_hook: ${runInfo.skipped} Skipped | :clock1: ${secondsToTimeString(runInfo.time)}\n\n`;
+
+  const failedTestSummary = (() => {
+    const heading = `### Details`;
+    const failedTestHeader = `<table><thead><tr><th colspan="3">Failing Tests</th></tr><tr><th></th><th>Spec</th><th>Test</th></tr></thead><tbody>`;
+    const failedTestRows = results
+      .filter((result: TestResult) => result.failing)
+      .map((result: TestResult) => {
+        const specFile = path.basename(result.testFilename);
+        return `<tr><td>:x:</td><td><code>${specFile}</code></td><td><em>${result.groupName} » ${result.testName}</em></td></tr>`;
+      });
+    const failedTestFooter = `</tbody></table>`;
+
+    return [
+      heading,
+      failedTestHeader,
+      ...failedTestRows,
+      failedTestFooter,
+      '',
+    ].join('\n');
+  })();
+
+  const rerunNote = (() => {
+    const heading = `### Debugging`;
+    const failingTestFiles = results
+      .filter((result: TestResult) => result.failing)
+      .map((result: TestResult) => result.testFilename);
+
+    const rerunTip = 'Use this command to re-run the failing tests:';
+
+    return [
+      heading,
+      rerunTip,
+      '',
+      '```bash',
+      cypressRunCommand(failingTestFiles),
+      '```',
+      '',
+    ].join('\n');
+  })();
+
+  return [
+    headline,
+    '',
+    breakdown,
+    runInfo.failing > 0 ? failedTestSummary : null,
+    runInfo.failing > 0 ? rerunNote : null,
+  ]
+    .filter((item) => item !== null)
+    .join('\n');
+};

scripts/junit-summary/formatters/json-formatter.ts

@@ -0,0 +1,27 @@
+import type { Formatter } from './formatter';
+import type { TestResult } from '../results/test-result';
+import type { TestSuite, TestSuites } from 'junit2json';
+import type { RunInfo } from '../results/run-info';
+import type { Metadata } from '../metadata/metadata';
+
+
+/**
+ * Outputs test result data in JSON format.
+ *
+ * @param info - Run info.
+ * @param results - Test results.
+ * @param metadata - Run metadata.
+ * @param _junitData - Raw JUnit test result data (unused).
+ */
+export const jsonFormatter: Formatter = (
+  info: RunInfo,
+  results: TestResult[],
+  metadata: Metadata,
+  _junitData: TestSuites[]
+) => {
+  return JSON.stringify({
+    info,
+    metadata,
+    results,
+  });
+};

scripts/junit-summary/formatters/slack-formatter.ts

@@ -0,0 +1,121 @@
+import type { Formatter } from './formatter';
+import type { TestResult } from '../results/test-result';
+import type { TestSuites } from 'junit2json';
+import type { RunInfo } from '../results/run-info';
+import type { Metadata } from '../metadata/metadata';
+import { pluralize } from '../util/pluralize';
+import { secondsToTimeString } from '../util';
+import * as path from 'path';
+import { cypressRunCommand } from '../util/cypress';
+
+/**
+ * Outputs test result summary formatted as a Slack message.
+ *
+ * @param info - Run info.
+ * @param results - Test results.
+ * @param metadata - Run metadata.
+ * @param _junitData - Raw JUnit test result data (unused).
+ */
+export const slackFormatter: Formatter = (
+  runInfo: RunInfo,
+  results: TestResult[],
+  metadata: Metadata,
+  _junitData: TestSuites[]
+) => {
+  const indicator = runInfo.failing ? ':x-mark:' : ':check-mark:';
+  const headline = (metadata.runId && metadata.runUrl)
+    ? `*Cypress test results for run <${metadata.runUrl}|#${metadata.runId}>*\n`
+    : `*Cypress test results*\n`;
+
+  const breakdown = `:small_red_triangle: ${runInfo.failing} Failing | :thumbs_up_green: ${runInfo.passing} Passing | :small_blue_diamond: ${runInfo.skipped} Skipped\n\n`;
+
+  // Show a human-readable summary of what was tested and whether it succeeded.
+  const summary = (() => {
+    const info = !runInfo.failing
+     ? `> ${indicator} ${runInfo.passing} passing ${pluralize(runInfo.passing, 'test', 'tests')}`
+     : `> ${indicator} ${runInfo.failing} failed ${pluralize(runInfo.failing, 'test', 'tests')}`;
+
+    const prInfo = (metadata.changeId && metadata.changeUrl)
+      ? ` on PR <${metadata.changeUrl}|#${metadata.changeId}>${metadata.changeTitle ? ` - _${metadata.changeTitle}_` : ''}`
+      : '';
+
+    const runLength = `(${secondsToTimeString(runInfo.time)})`;
+    const endingPunctuation = !runInfo.failing ? '.' : ':';
+
+    return `${info}${prInfo} ${runLength}${endingPunctuation}`
+  })();
+
+  // Display a list of failed tests and collection of actions when applicable.
+  const failedTestSummary = (() => {
+    const failedTestLines = results
+      .filter((result: TestResult) => result.failing)
+      .map((result: TestResult) => {
+        const specFile = path.basename(result.testFilename);
+        return `• \`${specFile}\` — _${result.groupName}_ » _${result.testName}_`;
+      });
+
+    // When applicable, display actions that can be taken by the user.
+    const failedTestActions = [
+      metadata.resultsUrl ? `<${metadata.resultsUrl}|View results>` : '',
+      metadata.artifactsUrl ? `<${metadata.artifactsUrl}|View artifacts>` : '',
+      metadata.rerunUrl ? `<${metadata.rerunUrl}|Replay tests>` : '',
+    ]
+      .filter((item) => item !== '')
+      .join(' | ');
+
+    return [
+      '',
+      ...failedTestLines,
+      '',
+      failedTestActions ? failedTestActions : null,
+    ]
+      .filter((item) => item !== null)
+      .map((item) => `> ${item}`)
+      .join('\n');
+  })();
+
+  // Display re-run command to help with troubleshooting.
+  const rerunNote = (() => {
+    const failingTestFiles = results
+      .filter((result: TestResult) => result.failing)
+      .map((result: TestResult) => result.testFilename);
+
+    const rerunTip = 'Use this command to re-run the failing tests:';
+    const cypressCommand = `${'```'}${cypressRunCommand(failingTestFiles)}${'```'}`;
+
+    return `${rerunTip}\n${cypressCommand}`;
+  })();
+
+  // Display test run details (author, PR number, run number, etc.) when applicable.
+  const footer = (() => {
+    const authorIdentifier = (metadata.authorSlack ? `@${metadata.authorSlack}` : null)
+      || (metadata.authorGitHub ? `<https://github.com/${metadata.authorGitHub}|${metadata.authorGitHub}>` : null)
+      || (metadata.authorName ? metadata.authorName : null);
+
+    return [
+      authorIdentifier ? `Authored by ${authorIdentifier}` : null,
+      metadata.changeId && metadata.changeUrl ? `PR <${metadata.changeUrl}|#${metadata.changeId}>` : null,
+      metadata.runId && metadata.runUrl ? `Run <${metadata.runUrl}|#${metadata.runId}>` : null,
+      metadata.branchName ? `\`${metadata.branchName}\`` : null,
+    ]
+    .filter((item) => item !== null)
+    .join(' | ');
+  })();
+
+  return [
+    headline,
+    breakdown,
+    summary,
+
+    // Add an extra line after the summary when no failures are listed.
+    runInfo.failing > 0 ? null : '',
+
+    // When one or more test has failed, display the list of failed tests as
+    // well as a command that can be used to re-run failed tests locally.
+    runInfo.failing > 0 ? `${failedTestSummary}\n` : null,
+    runInfo.failing > 0 ? `${rerunNote}\n` : null,
+
+    // Show run details footer.
+    `:cypress: ${footer}`,
+  ].filter((item) => item !== null).join('\n');
+};

scripts/junit-summary/formatters/status-formatter.ts

@@ -0,0 +1,13 @@
+import type { Formatter } from './formatter';
+import type { RunInfo } from '../results/run-info';
+
+/**
+ * Outputs "passing" if all tests have passed, or "failing" if one or more has failed.
+ *
+ * @param info - Run info.
+ */
+export const statusFormatter: Formatter = (
+  info: RunInfo,
+) => {
+  return info.failing ? 'failing' : 'passing';
+};