Native coverage integration for the CTest test controller (#4094)

* Added lcov-parser as dependency Used to parse lcov coverage info files. * Update vscode engine to 1.88.0 Needed for accessing the Test Coverage API in vscode * Native test coverage implementation Test coverage implementation for the CTest test controller. It relies on lcov coverage info files being specefied by the user in the settings.json of the project. Optionally, the user can specify CMake (utility) targets that should be built before and/or after the tests are/have been executed. These targets could reasonably zero the coverage counters (pre) and filter the coverage info files (post). * CMake fixes in single-root-UI test * Added end-to-end test for coverage * Unit test for coverage * Removed specifying gcov tool as it is not needed Also removed dynamic `--dynamic-list-data` linker flag * Use setup-lcov action * Removed log from coverage unit test * Test additions and fixes * Rationale notes in code * grab from feed * Review fixes + disable coverage test on Win MSVC can not produce gcov based coverage data, therefore the coverage end-to-end tests are disabled on Windows. * Added changelog entry for the coverage feature --------- Co-authored-by: Garrett Campbell <86264750+gcampbell-msft@users.noreply.github.com> Co-authored-by: Garrett Campbell <gcampbell@microsoft.com>
microsoft · Dec 2, 2024 · 9a24c14 · 9a24c14
1 parent a5775cc
commit 9a24c14
Show file tree

Hide file tree

Showing 24 changed files with 432 additions and 24 deletions.
diff --git a/.github/workflows/ci-main-linux.yml b/.github/workflows/ci-main-linux.yml
@@ -13,6 +13,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Setup LCOV
+        uses: hrishikesh-kadam/setup-lcov@v1
+
       - name: Setup Node environment
         uses: actions/setup-node@v3
         with:

diff --git a/.github/workflows/ci-main-mac.yml b/.github/workflows/ci-main-mac.yml
@@ -13,6 +13,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Setup LCOV
+        uses: hrishikesh-kadam/setup-lcov@v1
+
       - name: Setup Node environment
         uses: actions/setup-node@v3
         with:
@@ -43,8 +46,6 @@ jobs:
           configure-options: -DCMAKE_INSTALL_PREFIX:STRING=${{ github.workspace }}/test/fakebin
           install-build: true
 
-
-
       - name: Run successful-build test
         run: yarn endToEndTestsSuccessfulBuild
 

diff --git a/.github/workflows/ci-main.win.yml b/.github/workflows/ci-main.win.yml
@@ -14,6 +14,9 @@ jobs:
       - name: Checkout source code
         uses: actions/checkout@v3
 
+      - name: Setup LCOV
+        uses: hrishikesh-kadam/setup-lcov@v1
+
       - name: Setup Node environment
         uses: actions/setup-node@v3
         with:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,10 @@ Features:
 
 - Add support for Presets v9, which enables more macro expansion for the `include` field. [#3946](https://github.com/microsoft/vscode-cmake-tools/issues/3946)
 - Add support to configure default folder in workspace setting. [#1078](https://github.com/microsoft/vscode-cmake-tools/issues/1078)
+- Add support for processing LCOV based coverage info files when tests are
+  executed. This adds test execution type, "Run with coverage", on the `ctest`
+  section of the Testing tab.
+  [#4040](https://github.com/microsoft/vscode-cmake-tools/issues/4040)
 
 Improvements:
 

diff --git a/docs/cmake-settings.md b/docs/cmake-settings.md
@@ -38,6 +38,9 @@ Options that support substitution, in the table below, allow variable references
 | `cmake.saveBeforeBuild` | If `true` (the default), saves open text documents when build or configure is invoked before running CMake. | `true` | no |
 | `cmake.sourceDirectory` | A directory or a list of directories where the root `CMakeLists.txt`s are stored. | `${workspaceFolder}` | yes |
 | `cmake.testEnvironment` | An object containing `key:value` pairs of environment variables, which will be available when debugging, running and testing with CTest. | `null` (no environment variables) | yes |
+| `cmake.preRunCoverageTarget` | Target to build before running tests with coverage using the test explorer | null | no |
+| `cmake.postRunCoverageTarget` | Target to build after running tests with coverage using the test explorer | null | no |
+| `cmake.coverageInfoFiles` | LCOV coverage info files to be processed after running tests with coverage using the test explorer | [] | yes |
 
 ## Variable substitution
 

diff --git a/package.json b/package.json
@@ -22,7 +22,7 @@
     "syntaxes"
   ],
   "engines": {
-    "vscode": "^1.67.0"
+    "vscode": "^1.88.0"
   },
   "categories": [
     "Other",
@@ -3589,6 +3589,27 @@
           "default": true,
           "description": "%cmake-tools.configuration.cmake.enableAutomaticKitScan.description%",
           "scope": "resource"
+        },
+        "cmake.preRunCoverageTarget": {
+          "type": "string",
+          "default": null,
+          "description": "%cmake-tools.configuration.cmake.preRunCoverageTarget.description%",
+          "scope": "resource"
+        },
+        "cmake.postRunCoverageTarget": {
+          "type": "string",
+          "default": null,
+          "description": "%cmake-tools.configuration.cmake.postRunCoverageTarget.description%",
+          "scope": "resource"
+        },
+        "cmake.coverageInfoFiles": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "default": [],
+          "description": "%cmake-tools.configuration.cmake.coverageInfoFiles.description%",
+          "scope": "resource"
         }
       }
     },
@@ -3716,7 +3737,7 @@
     "@types/rimraf": "^3.0.0",
     "@types/sinon": "~9.0.10",
     "@types/tmp": "^0.2.0",
-    "@types/vscode": "1.63.0",
+    "@types/vscode": "1.88.0",
     "@types/which": "~2.0.0",
     "@types/xml2js": "^0.4.8",
     "@types/uuid": "~8.3.3",
@@ -3758,6 +3779,7 @@
     "webpack-cli": "^4.5.0"
   },
   "dependencies": {
+    "@friedemannsommer/lcov-parser": "^4.0.1",
     "@types/string.prototype.matchall": "^4.0.4",
     "ajv": "^7.1.0",
     "chokidar": "^3.5.1",

diff --git a/package.nls.json b/package.nls.json
@@ -273,6 +273,9 @@
     "cmake-tools.configuration.cmake.automaticReconfigure.description": "Automatically configure CMake project directories when the kit or the configuration preset is changed.",
     "cmake-tools.configuration.cmake.pinnedCommands.description":"List of CMake commands to always pin by default.",
     "cmake-tools.configuration.cmake.enableAutomaticKitScan.description": "Enable automatic scanning for kits when a kit isn't selected. This will only take affect when CMake Presets aren't being used.",
+    "cmake-tools.configuration.cmake.preRunCoverageTarget.description": "Target to build before running tests with coverage using the test explorer",
+    "cmake-tools.configuration.cmake.postRunCoverageTarget.description": "Target to build after running tests with coverage using the test explorer",
+    "cmake-tools.configuration.cmake.coverageInfoFiles.description": "LCOV coverage info files to be processed after running tests with coverage using the test explorer.",
     "cmake-tools.debugger.pipeName.description": "Name of the pipe (on Windows) or domain socket (on Unix) to use for debugger communication.",
     "cmake-tools.debugger.clean.description": "Clean prior to configuring.",
     "cmake-tools.debugger.configureAll.description": "Configure for all projects.",

diff --git a/src/config.ts b/src/config.ts
@@ -217,6 +217,9 @@ export interface ExtensionConfigurationSettings {
     automaticReconfigure: boolean;
     pinnedCommands: string[];
     enableAutomaticKitScan: boolean;
+    preRunCoverageTarget: string | null;
+    postRunCoverageTarget: string | null;
+    coverageInfoFiles: string[];
 }
 
 type EmittersOf<T> = {
@@ -565,6 +568,18 @@ export class ConfigurationReader implements vscode.Disposable {
         return this.configData.enableAutomaticKitScan;
     }
 
+    get preRunCoverageTarget(): string | null {
+        return this.configData.preRunCoverageTarget;
+    }
+
+    get postRunCoverageTarget(): string | null {
+        return this.configData.postRunCoverageTarget;
+    }
+
+    get coverageInfoFiles(): string[] {
+        return this.configData.coverageInfoFiles;
+    }
+
     private readonly emitters: EmittersOf<ExtensionConfigurationSettings> = {
         autoSelectActiveFolder: new vscode.EventEmitter<boolean>(),
         defaultActiveFolder: new vscode.EventEmitter<string | null>(),
@@ -629,7 +644,10 @@ export class ConfigurationReader implements vscode.Disposable {
         launchBehavior: new vscode.EventEmitter<string>(),
         automaticReconfigure: new vscode.EventEmitter<boolean>(),
         pinnedCommands: new vscode.EventEmitter<string[]>(),
-        enableAutomaticKitScan: new vscode.EventEmitter<boolean>()
+        enableAutomaticKitScan: new vscode.EventEmitter<boolean>(),
+        preRunCoverageTarget: new vscode.EventEmitter<string | null>(),
+        postRunCoverageTarget: new vscode.EventEmitter<string | null>(),
+        coverageInfoFiles: new vscode.EventEmitter<string[]>()
     };
 
     /**

diff --git a/src/coverage.ts b/src/coverage.ts
@@ -0,0 +1,64 @@
+import * as vscode from 'vscode';
+import { lcovParser } from "@friedemannsommer/lcov-parser";
+import * as nls from 'vscode-nls';
+import * as logging from '@cmt/logging';
+
+nls.config({ messageFormat: nls.MessageFormat.bundle, bundleFormat: nls.BundleFormat.standalone })();
+const localize: nls.LocalizeFunc = nls.loadMessageBundle();
+
+const log = logging.createLogger('ctest-coverage');
+
+export async function handleCoverageInfoFiles(run: vscode.TestRun, coverageInfoFiles: string[], coverageData: WeakMap<vscode.FileCoverage, vscode.FileCoverageDetail[]>) {
+    for (const coverageInfoFile of coverageInfoFiles) {
+        let contents: Uint8Array;
+        try {
+            contents = await vscode.workspace.fs.readFile(vscode.Uri.file(coverageInfoFile));
+        } catch (e) {
+            log.warning(localize('test.openCoverageInfoFile', 'Could not open coverage info file: {0}. Skipping...', coverageInfoFile));
+            return;
+        }
+        const sections = await lcovParser({ from: contents });
+        for (const section of sections) {
+            const coverage = new vscode.FileCoverage(vscode.Uri.file(section.path),
+                new vscode.TestCoverageCount(
+                    section.lines.hit,
+                    section.lines.instrumented
+                ), new vscode.TestCoverageCount(
+                    section.branches.hit,
+                    section.branches.instrumented
+                ), new vscode.TestCoverageCount(
+                    section.functions.hit,
+                    section.functions.instrumented
+                ));
+
+            const lineBranches = new Map<number, vscode.BranchCoverage[]>();
+            for (const branch of section.branches.details) {
+                const branchCoverage = new vscode.BranchCoverage(branch.hit,
+                    new vscode.Position(branch.line - 1, 0), branch.branch);
+
+                const curr = lineBranches.get(branch.line);
+                if (curr === undefined) {
+                    lineBranches.set(branch.line, [branchCoverage]);
+                } else {
+                    curr.push(branchCoverage);
+                    lineBranches.set(branch.line, curr);
+                }
+            }
+
+            const declarations: vscode.DeclarationCoverage[] = [];
+            for (const declaration of section.functions.details) {
+                declarations.push(new vscode.DeclarationCoverage(declaration.name, declaration.hit,
+                    new vscode.Position(declaration.line - 1, 0)));
+            }
+
+            const statements: vscode.StatementCoverage[] = [];
+            for (const line of section.lines.details) {
+                statements.push(new vscode.StatementCoverage(line.hit,
+                    new vscode.Position(line.line - 1, 0),
+                    lineBranches.get(line.line) ?? []));
+            }
+            coverageData.set(coverage, [...statements, ...declarations]);
+            run.addCoverage(coverage);
+        }
+    }
+}
diff --git a/src/ctest.ts b/src/ctest.ts
@@ -15,6 +15,8 @@ import { expandString } from '@cmt/expand';
 import * as proc from '@cmt/proc';
 import { ProjectController } from '@cmt/projectController';
 import { extensionManager } from '@cmt/extension';
+import { CMakeProject } from '@cmt/cmakeProject';
+import { handleCoverageInfoFiles } from '@cmt/coverage';
 
 nls.config({ messageFormat: nls.MessageFormat.bundle, bundleFormat: nls.BundleFormat.standalone })();
 const localize: nls.LocalizeFunc = nls.loadMessageBundle();
@@ -118,6 +120,18 @@ interface CTestInfo {
     version: { major: number; minor: number };
 }
 
+/* The preRunCoverageTarget and postRunCoverageTarget are optional user
+configured targets that are run before and after the tests when a "Run with
+Coverage" test execution is triggered. These cae be used to zero the
+coverage counters, filter coverage results etc. */
+interface ProjectCoverageConfig {
+    project: CMakeProject;
+    driver: CMakeDriver;
+    preRunCoverageTarget: string | null;
+    postRunCoverageTarget: string | null;
+    coverageInfoFiles: string[];
+}
+
 function parseXmlString<T>(xml: string): Promise<T> {
     return new Promise((resolve, reject) => {
         xml2js.parseString(xml, (err, result) => {
@@ -229,6 +243,8 @@ export class CTestDriver implements vscode.Disposable {
     private readonly testingEnabledEmitter = new vscode.EventEmitter<boolean>();
     readonly onTestingEnabledChanged = this.testingEnabledEmitter.event;
 
+    private coverageData = new WeakMap<vscode.FileCoverage, vscode.FileCoverageDetail[]>();
+
     dispose() {
         this.testingEnabledEmitter.dispose();
         this.testsChangedEmitter.dispose();
@@ -710,7 +726,7 @@ export class CTestDriver implements vscode.Disposable {
             return -1;
         }
 
-        if (util.isTestMode()) {
+        if (util.isTestMode() && !util.overrideTestModeForTestExplorer()) {
             // ProjectController can't be initialized in test mode, so we don't have a usable test explorer
             return 0;
         }
@@ -905,7 +921,65 @@ export class CTestDriver implements vscode.Disposable {
         return true;
     }
 
-    private async runTestHandler(request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) {
+    private async handleCoverageOnProjects(run: vscode.TestRun, projectCoverageConfigs: ProjectCoverageConfig[]) {
+        // Currently only LCOV coverage info files are supported
+        for (const projectCoverageConfig of projectCoverageConfigs) {
+            if (projectCoverageConfig.coverageInfoFiles.length === 0) {
+                log.warning(localize('test.noCoverageInfoFiles', 'No coverage info files for CMake project {0}. No coverage data will be analyzed for this project.', projectCoverageConfig.project.sourceDir));
+                continue;
+            }
+            const expandedCoverageInfoFiles = await Promise.all(projectCoverageConfig.coverageInfoFiles.map(async (coverageInfoFile: string) => expandString(coverageInfoFile, projectCoverageConfig.driver.expansionOptions)));
+            await handleCoverageInfoFiles(run, expandedCoverageInfoFiles, this.coverageData);
+        }
+    }
+
+    private async coverageCTestHelper(tests: vscode.TestItem[], run: vscode.TestRun, cancellation: vscode.CancellationToken): Promise<number> {
+        const projectRoots = new Set<string>();
+        for (const test of tests) {
+            projectRoots.add(this.getTestRootFolder(test));
+        }
+
+        const projectCoverageConfigs: ProjectCoverageConfig[] = [];
+        for (const folder of projectRoots) {
+            const project = await this.projectController?.getProjectForFolder(folder);
+            if (project) {
+                const driver = await project.getCMakeDriverInstance();
+                if (driver) {
+                    projectCoverageConfigs.push({
+                        project: project, driver: driver, preRunCoverageTarget: driver.config.preRunCoverageTarget, postRunCoverageTarget: driver.config.postRunCoverageTarget, coverageInfoFiles: driver.config.coverageInfoFiles
+                    });
+                }
+            }
+        }
+
+        for (const projectCoverageConfig of projectCoverageConfigs) {
+            if (projectCoverageConfig.preRunCoverageTarget) {
+                log.info(localize('test.buildingPreRunCoverageTarget', 'Building the preRunCoverageTarget for project {0} before running tests with coverage.', projectCoverageConfig.project.sourceDir));
+                const rc = await projectCoverageConfig.project.build([projectCoverageConfig.preRunCoverageTarget]);
+                if (rc !== 0) {
+                    log.error(localize('test.preRunCoverageTargetFailure', 'Building the preRunCoverageTarget \'{0}\' on project in {1} failed. Skipping running tests.', projectCoverageConfig.preRunCoverageTarget, projectCoverageConfig.project.sourceDir));
+                    run.end();
+                    return rc;
+                }
+            }
+        }
+        const runResult = await this.runCTestHelper(tests, run, cancellation, undefined, undefined, undefined, false, undefined, RunCTestHelperEntryPoint.TestExplorer);
+        for (const projectCoverageConfig of projectCoverageConfigs) {
+            if (projectCoverageConfig.postRunCoverageTarget) {
+                log.info(localize('test.buildingPostRunCoverageTarget', 'Building the postRunCoverageTarget \'{0}\' for project {1} after the tests have run with coverage.', projectCoverageConfig.postRunCoverageTarget, projectCoverageConfig.project.sourceDir));
+                const rc = await projectCoverageConfig.project.build([projectCoverageConfig.postRunCoverageTarget]);
+                if (rc !== 0) {
+                    log.error(localize('test.postRunCoverageTargetFailure', 'Building target postRunCoverageTarget on project in {0} failed. Skipping handling of coverage data.', projectCoverageConfig.project.sourceDir));
+                    return rc;
+                }
+            }
+        }
+
+        await this.handleCoverageOnProjects(run, projectCoverageConfigs);
+        return runResult;
+    }
+
+    private async runTestHandler(request: vscode.TestRunRequest, cancellation: vscode.CancellationToken, isCoverageRun = false) {
         // NOTE: We expect the testExplorer to be undefined when the cmake.ctest.testExplorerIntegrationEnabled is disabled.
         if (!testExplorer) {
             return;
@@ -922,7 +996,12 @@ export class CTestDriver implements vscode.Disposable {
         this.ctestsEnqueued(tests, run);
         const buildSucceeded = await this.buildTests(tests, run);
         if (buildSucceeded) {
-            await this.runCTestHelper(tests, run, cancellation, undefined, undefined, undefined, false, undefined, RunCTestHelperEntryPoint.TestExplorer);
+            if (isCoverageRun) {
+                await this.coverageCTestHelper(tests, run, cancellation);
+
+            } else {
+                await this.runCTestHelper(tests, run, cancellation, undefined, undefined, undefined, false, undefined, RunCTestHelperEntryPoint.TestExplorer);
+            }
         } else {
             log.info(localize('test.skip.run.build.failure', "Not running tests due to build failure."));
         }
@@ -1231,6 +1310,12 @@ export class CTestDriver implements vscode.Disposable {
                 (request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) => this.runTestHandler(request, cancellation),
                 true
             );
+            testExplorer.createRunProfile(
+                'Run Tests with Coverage',
+                vscode.TestRunProfileKind.Coverage,
+                (request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) => this.runTestHandler(request, cancellation, true),
+                true
+            ).loadDetailedCoverage = async (_, fileCoverage) => this.coverageData.get(fileCoverage) ?? [];
             testExplorer.createRunProfile(
                 'Debug Tests',
                 vscode.TestRunProfileKind.Debug,

diff --git a/src/util.ts b/src/util.ts
@@ -835,6 +835,13 @@ export function isTestMode(): boolean {
     return process.env['CMT_TESTING'] === '1';
 }
 
+/**
+ * Returns true if the test explorer should be enabled even when in test mode.
+ */
+export function overrideTestModeForTestExplorer(): boolean {
+    return process.env['CMT_TESTING_OVERRIDE_TEST_EXPLORER'] === '1';
+}
+
 export async function getAllCMakeListsPaths(path: string): Promise<string[] | undefined> {
     const regex: RegExp = new RegExp(/(\/|\\)CMakeLists\.txt$/);
     return recGetAllFilePaths(path, regex, await readDir(path), []);

diff --git a/test/end-to-end-tests/single-root-UI/project-folder/.vscode/settings.json b/test/end-to-end-tests/single-root-UI/project-folder/.vscode/settings.json
@@ -1,5 +1,10 @@
 {
     "cmake.buildDirectory": "${workspaceFolder}/build",
     "cmake.useCMakePresets": "never",
-    "cmake.configureOnOpen": false
+    "cmake.configureOnOpen": false,
+    "cmake.preRunCoverageTarget": "init-coverage",
+    "cmake.postRunCoverageTarget": "capture-coverage",
+    "cmake.coverageInfoFiles": [
+        "${workspaceFolder}/build/lcov.info"
+    ]
 }