From a2e39f2b76d5adb0abbda1ba8596fa2fd557a53c Mon Sep 17 00:00:00 2001 From: Dario Valdespino Date: Thu, 12 Nov 2026 10:43:33 -0300 Subject: [PATCH] feat(testing): read run-local json coverage with lcov fallback --- README.md | 15 +- packages/core/src/coverage.ts | 168 ++++++++++++- packages/core/test/coverage.test.ts | 189 ++++++++++++++- packages/vscode/CHANGELOG.md | 3 + packages/vscode/README.md | 3 +- packages/vscode/src/testing.ts | 61 +++-- packages/vscode/test/testingCoverage.test.ts | 235 +++++++++++++++++++ 7 files changed, 646 insertions(+), 28 deletions(-) create mode 100644 packages/vscode/test/testingCoverage.test.ts diff --git a/README.md b/README.md index 2206188..ba8c497 100644 --- a/README.md +++ b/README.md @@ -212,12 +212,15 @@ The **Debug** profile runs the same command with a bare `--debugger` and attache agent announces itself. That flag takes no address on `elide test`: the agent always binds 5005, so one debug run at a time. -The **Run with Coverage** profile adds `--coverage` and, once the run exits, shows the reports it wrote in the -editor's coverage gutter and the Test Coverage view: guest (JavaScript/TypeScript) LCOV from -`.dev/reports/coverage/js/lcov.info`, and JVM coverage from the JaCoCo XML report under -`.dev/reports/coverage/jvm//` (`.dev/artifacts/coverage/jvm//` before Elide 1.6), with per-line branch -counts. Elide writes JVM coverage as JaCoCo execution data (`.exec`) by default, which cannot be read without the -compiled classes; ask for the XML report in `elide.pkl` to see it in the editor: +The **Run with Coverage** profile adds `--coverage` and shows line and branch coverage in the editor's gutter and +Test Coverage view. On CLIs supporting `ELIDE_COVERAGE_REPORT`, it reads a private, run-local JSON artifact; JVM +coverage needs no XML-report configuration. Incomplete collection is reported, and sources whose recorded digest +no longer matches the file are omitted. Temporary artifacts are removed after completion or cancellation. +A failed coverage minimum errors the project run without changing individual passing test results. + +Older CLIs fall back to guest LCOV (`.dev/reports/coverage/js/lcov.info`) and JaCoCo XML +(`.dev/reports/coverage/jvm//`, or `.dev/artifacts/coverage/jvm//` before Elide 1.6). +For that fallback, Elide's default JVM execution data (`.exec`) is not sufficient; configure an XML report: ```pkl import "elide:Coverage.pkl" as Coverage diff --git a/packages/core/src/coverage.ts b/packages/core/src/coverage.ts index 137986c..e947b11 100644 --- a/packages/core/src/coverage.ts +++ b/packages/core/src/coverage.ts @@ -1,5 +1,6 @@ +import { createHash } from "node:crypto"; import { existsSync, realpathSync } from "node:fs"; -import { readFile, readdir, stat } from "node:fs/promises"; +import { open, readFile, readdir, realpath, stat } from "node:fs/promises"; import path from "node:path"; import { OUTPUT_DIR } from "./elide.js"; @@ -196,6 +197,171 @@ export interface LoadedCoverage { reports: CoverageReports; } +const MAX_ARTIFACT_BYTES = 8 * 1024 * 1024; +const MAX_ARTIFACT_FILES = 2_000; +const MAX_ARTIFACT_LINES = 200_000; +// Compact branch counters must not expand into an unbounded number of editor objects. +const MAX_ARTIFACT_BRANCHES = 1_000_000; + +interface CoverageArtifactFile { + engine: string; + path: string; + sourceDigest?: { algorithm: "sha256-utf16be"; value: string }; + lines: { line: number; covered: boolean; branches?: { covered: number; total: number } }[]; +} + +function isCoverageArtifact(value: unknown): value is { version: 1; complete: boolean; files: CoverageArtifactFile[] } { + if ( + typeof value !== "object" || value === null || + !("version" in value) || value.version !== 1 || + !("complete" in value) || typeof value.complete !== "boolean" || + !("files" in value) || !Array.isArray(value.files) || value.files.length > MAX_ARTIFACT_FILES + ) return false; + let lines = 0; + let branches = 0; + const identities = new Set(); + for (const file of value.files as unknown[]) { + if ( + typeof file !== "object" || file === null || + !("engine" in file) || typeof file.engine !== "string" || !file.engine || + !("path" in file) || typeof file.path !== "string" || !file.path || + !("lines" in file) || !Array.isArray(file.lines) + ) return false; + // The producer writes normalized, relative slash paths, including on Windows. + if ( + path.posix.isAbsolute(file.path) || path.win32.parse(file.path).root || + file.path.includes("\\") || file.path.includes("\0") || + file.path.split("/").some((part) => !part || part === "." || part === "..") + ) return false; + const identity = JSON.stringify([file.engine, file.path]); + if (identities.has(identity)) return false; + identities.add(identity); + if ("sourceDigest" in file && file.sourceDigest !== undefined) { + const digest = file.sourceDigest; + if ( + typeof digest !== "object" || digest === null || + !("algorithm" in digest) || digest.algorithm !== "sha256-utf16be" || + !("value" in digest) || typeof digest.value !== "string" || !/^[a-fA-F0-9]{64}$/.test(digest.value) + ) return false; + } + lines += file.lines.length; + if (lines > MAX_ARTIFACT_LINES) return false; + const seenLines = new Set(); + for (const line of file.lines as unknown[]) { + if ( + typeof line !== "object" || line === null || + !("line" in line) || typeof line.line !== "number" || !Number.isSafeInteger(line.line) || line.line <= 0 || + !("covered" in line) || typeof line.covered !== "boolean" + ) return false; + if (seenLines.has(line.line)) return false; + seenLines.add(line.line); + if ("branches" in line && line.branches !== undefined) { + const counter = line.branches; + if ( + typeof counter !== "object" || counter === null || + !("covered" in counter) || typeof counter.covered !== "number" || !Number.isSafeInteger(counter.covered) || counter.covered < 0 || + !("total" in counter) || typeof counter.total !== "number" || !Number.isSafeInteger(counter.total) || counter.total < counter.covered + ) return false; + branches += counter.total; + if (branches > MAX_ARTIFACT_BRANCHES) return false; + } + } + } + return true; +} + +/** + * Read a run-local coverage artifact, retaining incomplete subsets and excluding sources whose digest changed. + * Unknown or malformed documents return `undefined` to allow the legacy-report fallback. + */ +export async function loadCoverageArtifact(file: string, projectRoot: string): Promise<{ files: FileCoverageData[]; complete: boolean } | undefined> { + const text = await readArtifactText(file); + if (text === undefined) return undefined; + let artifact: unknown; + try { + artifact = JSON.parse(text); + } catch { + return undefined; + } + if (!isCoverageArtifact(artifact)) return undefined; + const root = path.resolve(projectRoot); + let realRoot: string; + try { + realRoot = await realpath(root); + } catch { + return undefined; + } + const files: FileCoverageData[] = []; + let complete = artifact.complete; + for (const entry of artifact.files) { + const source = path.resolve(root, entry.path); + if (!isWithin(root, source) || !(await artifactSourceWithinRoot(root, realRoot, source))) return undefined; + if (entry.sourceDigest) { + try { + const sourceText = await readFile(source, "utf8"); + const digest = createHash("sha256").update(Buffer.from(sourceText, "utf16le").swap16()).digest("hex"); + if (digest !== entry.sourceDigest.value.toLowerCase()) { + complete = false; + continue; + } + } catch { + complete = false; + continue; + } + } + files.push({ + path: source, + lines: entry.lines.map((line) => ({ + line: line.line, + hits: line.covered ? 1 : 0, + branches: Array.from({ length: line.branches?.total ?? 0 }, (_, i) => ({ hits: i < (line.branches?.covered ?? 0) ? 1 : 0 })), + })), + }); + } + return { files: mergeCoverage(files), complete }; +} + +function isWithin(root: string, file: string): boolean { + const relative = path.relative(root, file); + return relative !== "" && relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative); +} + +async function artifactSourceWithinRoot(root: string, realRoot: string, source: string): Promise { + // Check the nearest existing ancestor too, so a missing file below an escaping symlink is rejected. + for (let candidate = source; candidate !== root; candidate = path.dirname(candidate)) { + try { + const canonical = await realpath(candidate); + return isWithin(realRoot, canonical) || (candidate !== source && canonical === realRoot); + } catch { + // A source without a digest can still be reported after it has been removed. + } + } + return true; +} + +async function readArtifactText(file: string): Promise { + let handle; + try { + handle = await open(file, "r"); + const info = await handle.stat(); + if (!info.isFile() || info.size > MAX_ARTIFACT_BYTES) return undefined; + // Read at most the observed size plus one byte, detecting growth without an unbounded readFile. + const buffer = Buffer.alloc(info.size + 1); + let length = 0; + while (length < buffer.length) { + const { bytesRead } = await handle.read(buffer, length, buffer.length - length, null); + if (bytesRead === 0) break; + length += bytesRead; + } + if (length > info.size) return undefined; + return new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, length)); + } catch { + return undefined; + } finally { + await handle?.close(); + } +} + /** * Read the coverage a run left under `projectRoot` since `modifiedSince`: guest LCOV as it is, JaCoCo XML resolved * against `sourceRoots`. JaCoCo execution data is reported in `reports` but not decoded. diff --git a/packages/core/test/coverage.test.ts b/packages/core/test/coverage.test.ts index 107b71c..e3d7845 100644 --- a/packages/core/test/coverage.test.ts +++ b/packages/core/test/coverage.test.ts @@ -1,8 +1,9 @@ import { afterAll, describe, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, utimesSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; -import { findCoverageReports, loadCoverage, mergeCoverage, parseJacocoXml, parseLcov, sourceResolver } from "../src/index.js"; +import { findCoverageReports, loadCoverage, loadCoverageArtifact, mergeCoverage, parseJacocoXml, parseLcov, sourceResolver } from "../src/index.js"; const fixture = (name: string) => path.join(import.meta.dir, "fixtures", "coverage", name); /** `elide test --coverage` (Elide 1.6.0-rc.2) on a TypeScript project, with the project root replaced by `/project`. */ @@ -143,3 +144,189 @@ describe("loadCoverage", () => { expect(files.map((f) => f.path).filter((p) => p.endsWith(".ts")).sort()).toEqual([path.join(link, "src", "math.test.ts"), path.join(link, "src", "math.ts")]); }); }); + +describe("loadCoverageArtifact", () => { + const dirs: string[] = []; + afterAll(() => { + for (const dir of dirs) rmSync(dir, { recursive: true, force: true }); + }); + const project = () => { + const root = realpathSync(mkdtempSync(path.join(tmpdir(), "elide-cov-artifact-"))); + dirs.push(root); + return root; + }; + const row = (source = "src/Main.kt") => ({ + engine: "jvm", + path: source, + lines: [{ line: 1, covered: true }], + }); + const document = (files: unknown[] = [row()], complete = true) => ({ version: 1, complete, files }); + const write = (root: string, value: unknown) => { + const file = path.join(root, "coverage.json"); + writeFileSync(file, JSON.stringify(value)); + return file; + }; + + test("maps boolean hits and exact branch totals, sorts lines, and merges engines", async () => { + const root = project(); + const artifact = document([ + { + ...row(), + lines: [ + { line: 5, covered: false, branches: { covered: 0, total: 2 } }, + { line: 3, covered: true, branches: { covered: 2, total: 3 } }, + { line: 1, covered: false }, + { line: 6, covered: true, branches: { covered: 0, total: 0 } }, + ], + }, + { ...row(), engine: "js", lines: [{ line: 3, covered: true, branches: { covered: 1, total: 3 } }] }, + { ...row("other.ts"), engine: "future-engine", language: "typescript", unknown: true }, + ]); + expect(await loadCoverageArtifact(write(root, artifact), root)).toEqual({ + complete: true, + files: [ + { + path: path.join(root, "src", "Main.kt"), + lines: [ + { line: 1, hits: 0, branches: [] }, + { line: 3, hits: 2, branches: [{ hits: 2 }, { hits: 1 }, { hits: 0 }] }, + { line: 5, hits: 0, branches: [{ hits: 0 }, { hits: 0 }] }, + { line: 6, hits: 1, branches: [] }, + ], + }, + { path: path.join(root, "other.ts"), lines: [{ line: 1, hits: 1, branches: [] }] }, + ], + }); + }); + + test("retains incomplete subsets and recognizes empty complete or incomplete artifacts", async () => { + const root = project(); + const partial = await loadCoverageArtifact(write(root, document([row()], false)), root); + expect(partial?.complete).toBe(false); + expect(partial?.files).toHaveLength(1); + for (const complete of [true, false]) { + expect(await loadCoverageArtifact(write(root, document([], complete)), root)).toEqual({ files: [], complete }); + } + }); + + test("ignores absent, unreadable, invalid JSON and invalid UTF-8 artifacts", async () => { + const root = project(); + const file = path.join(root, "coverage.json"); + expect(await loadCoverageArtifact(file, root)).toBeUndefined(); + expect(await loadCoverageArtifact(root, root)).toBeUndefined(); + writeFileSync(file, "{"); + expect(await loadCoverageArtifact(file, root)).toBeUndefined(); + writeFileSync(file, Buffer.concat([Buffer.from('{"version":1,"complete":true,"files":[],"extra":"'), Buffer.from([0xff]), Buffer.from('"}')])); + expect(await loadCoverageArtifact(file, root)).toBeUndefined(); + }); + + test("rejects unsupported versions and malformed envelopes, files, lines and counters", async () => { + const root = project(); + const badLines: unknown[] = [ + null, {}, { line: 0, covered: true }, { line: -1, covered: true }, + { line: 1.5, covered: true }, { line: Number.MAX_SAFE_INTEGER + 1, covered: true }, + { line: "1", covered: true }, { line: 1, covered: 1 }, + { line: 1, covered: true, branches: null }, + ...[ + {}, { covered: -1, total: 1 }, { covered: 2, total: 1 }, + { covered: 0, total: -1 }, { covered: 0.5, total: 1 }, + { covered: 0, total: 1.5 }, { covered: 0, total: "2" }, + { covered: 0, total: Number.MAX_SAFE_INTEGER + 1 }, + { covered: 0, total: 1_000_000_000 }, + ].map((branches) => ({ line: 1, covered: true, branches })), + ]; + const invalid: unknown[] = [ + null, [], {}, { ...document(), version: 2 }, { ...document(), complete: "true" }, + { ...document(), files: {} }, document([null]), document([{}]), + document([{ ...row(), engine: 1 }]), document([{ ...row(), engine: "" }]), + document([{ ...row(), lines: {} }]), document([row(), row()]), + document([{ ...row(), lines: [{ line: 1, covered: true }, { line: 1, covered: false }] }]), + ...badLines.map((line) => document([{ ...row(), lines: [line] }])), + ...[ + null, {}, { algorithm: "sha256", value: "0".repeat(64) }, + { algorithm: "sha256-utf16be", value: "0".repeat(63) }, + { algorithm: "sha256-utf16be", value: "g".repeat(64) }, + ].map((sourceDigest) => document([{ ...row(), sourceDigest }])), + ]; + for (const value of invalid) { + expect(await loadCoverageArtifact(write(root, value), root)).toBeUndefined(); + } + }); + + test("rejects traversal and absolute paths on every platform", async () => { + const root = project(); + for (const source of ["", ".", "..", "../outside.kt", "src/../../outside.kt", "/outside.kt", "C:/outside.kt", "C:outside.kt", "\\\\host\\outside.kt", "src\\Main.kt", "src/./Main.kt", "src//Main.kt", "src/\0Main.kt"]) { + expect(await loadCoverageArtifact(write(root, document([row(source)])), root)).toBeUndefined(); + } + }); + + test("rejects symlinked source files or directories escaping the project", async () => { + const root = project(); + const outside = project(); + writeFileSync(path.join(outside, "Outside.kt"), ""); + symlinkSync(path.join(outside, "Outside.kt"), path.join(root, "Outside.kt"), "file"); + symlinkSync(outside, path.join(root, "linked"), "dir"); + for (const source of ["Outside.kt", "linked/Outside.kt", "linked/missing/nested.kt"]) { + expect(await loadCoverageArtifact(write(root, document([row(source)])), root)).toBeUndefined(); + } + }); + + test("bounds bytes, file count, total lines and expanded branches", async () => { + const root = project(); + const file = path.join(root, "coverage.json"); + writeFileSync(file, " ".repeat(8 * 1024 * 1024 + 1)); + expect(await loadCoverageArtifact(file, root)).toBeUndefined(); + expect(await loadCoverageArtifact(write(root, document(Array.from({ length: 2_001 }, (_, i) => row(`${i}.kt`)))), root)).toBeUndefined(); + const lines = Array.from({ length: 100_001 }, (_, i) => ({ line: i + 1, covered: true })); + expect(await loadCoverageArtifact(write(root, document([{ ...row("a.kt"), lines }, { ...row("b.kt"), lines }])), root)).toBeUndefined(); + expect(await loadCoverageArtifact(write(root, document([{ + ...row(), + lines: [ + { line: 1, covered: true, branches: { covered: 0, total: 600_000 } }, + { line: 2, covered: true, branches: { covered: 0, total: 600_000 } }, + ], + }])), root)).toBeUndefined(); + }); + + test("checks UTF-16BE source digests without normalizing Unicode or line endings", async () => { + const root = project(); + const source = "A😀\r\né"; + // Explicit UTF-16BE code units include the astral character's surrogate pair, and no BOM. + const digest = createHash("sha256").update(Buffer.from([0x00, 0x41, 0xd8, 0x3d, 0xde, 0x00, 0x00, 0x0d, 0x00, 0x0a, 0x00, 0xe9])).digest("hex"); + const sourcePath = path.join(root, "Main.kt"); + writeFileSync(sourcePath, source); + const file = write(root, document([ + { ...row("Main.kt"), sourceDigest: { algorithm: "sha256-utf16be", value: digest.toUpperCase() } }, + row("unversioned.ts"), + ])); + expect((await loadCoverageArtifact(file, root))?.complete).toBe(true); + writeFileSync(sourcePath, source.replace("\r\n", "\n")); + expect(await loadCoverageArtifact(file, root)).toEqual({ + complete: false, + files: [{ path: path.join(root, "unversioned.ts"), lines: [{ line: 1, hits: 1, branches: [] }] }], + }); + rmSync(sourcePath); + expect((await loadCoverageArtifact(file, root))?.files.map((entry) => path.basename(entry.path))).toEqual(["unversioned.ts"]); + expect((await loadCoverageArtifact(file, root))?.complete).toBe(false); + }); + + test("drops all stale rows without treating the recognized artifact as absent", async () => { + const root = project(); + const file = write(root, document([{ ...row(), sourceDigest: { algorithm: "sha256-utf16be", value: "0".repeat(64) } }])); + expect(await loadCoverageArtifact(file, root)).toEqual({ files: [], complete: false }); + }); + + test("keeps editor paths for a symlinked project root while validating current source", async () => { + const real = project(); + const parent = project(); + const link = path.join(parent, "project"); + symlinkSync(real, link, "dir"); + writeFileSync(path.join(real, "Main.kt"), "a"); + const digest = createHash("sha256").update(Buffer.from([0x00, 0x61])).digest("hex"); + const file = write(real, document([{ ...row("Main.kt"), sourceDigest: { algorithm: "sha256-utf16be", value: digest } }])); + expect(await loadCoverageArtifact(file, link)).toEqual({ + complete: true, + files: [{ path: path.join(link, "Main.kt"), lines: [{ line: 1, hits: 1, branches: [] }] }], + }); + }); +}); diff --git a/packages/vscode/CHANGELOG.md b/packages/vscode/CHANGELOG.md index e2babff..6d06e9e 100644 --- a/packages/vscode/CHANGELOG.md +++ b/packages/vscode/CHANGELOG.md @@ -14,11 +14,14 @@ All notable changes to the Elide extension are documented here. The format follo - Structured task progress beside normal task output inside VS Code's existing Terminal container. Inspect live build steps, progress, durations and skipped tasks; cancel a running task or open its step log. Task history is also available in Activity. Native stdin/stdout and task reruns are preserved, as are subsequent user layout changes. +- Run-local JSON test coverage with line and branch counters, source-digest freshness checks, incomplete-collection + reporting and temporary-artifact cleanup. Older CLIs retain LCOV/JaCoCo XML support. ### Fixed - Enable colorized process output for Elide task terminals, including tasks launched from sidebar targets and Android builds. Preserve color opt-outs and keep failure messages free of ANSI escapes. +- Report coverage-minimum and other run-level failures when all individual test results passed. - Komodo Compose previews no longer fail with "the preview session closed" after an earlier preview session was closed or replaced; the next preview starts a new session. diff --git a/packages/vscode/README.md b/packages/vscode/README.md index 36b023b..9bc3700 100644 --- a/packages/vscode/README.md +++ b/packages/vscode/README.md @@ -40,7 +40,8 @@ test, and debug commands are available from the editor. to their text progress. Explicit CLI color options and color opt-outs (`NO_COLOR`, `FORCE_COLOR=0`) are preserved. - **Test Explorer.** JUnit tests of the project's test source roots in the **Testing** view, run and debugged through `elide test --reporter=tap` with live results, failure messages and stack-trace locations. **Run with Coverage** - shows the run's LCOV and JaCoCo XML reports in the editor's coverage gutter and Test Coverage view. + shows run-local JSON coverage without requiring JaCoCo XML on compatible CLIs, with LCOV/JaCoCo XML fallback for + older CLIs. Coverage minimum failures error the project run while preserving individual passing test results. - **Elide sidebar.** An activity-bar view with each project's entrypoints, tasks, build targets (`elide build --inspect`), source sets and dependencies; run, debug, build and sync straight from the tree. A workspace's **Tasks → Build targets** groups targets by project in the CLI's order; standalone projects list them directly. diff --git a/packages/vscode/src/testing.ts b/packages/vscode/src/testing.ts index b3ebfd0..f3f835e 100644 --- a/packages/vscode/src/testing.ts +++ b/packages/vscode/src/testing.ts @@ -1,4 +1,6 @@ import type { ChildProcess } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; import path from "node:path"; import { ElideCli, @@ -8,6 +10,7 @@ import { isPathUnder, jvmTestNamePattern, loadCoverage, + loadCoverageArtifact, mergeElideInvocationOptions, normalizePath, parseTapLabel, @@ -72,7 +75,7 @@ interface RunGroup { * Discovery is static: the test source roots of the project model are scanned for JUnit declarations, so the tree is * populated without compiling or running anything. A run shells out to `elide test --reporter=tap` and maps the TAP * stream back onto those items; the Debug profile runs the same command under a JDWP agent and attaches, and the - * Coverage profile adds `--coverage` and publishes the reports the run leaves under `.dev`. + * Coverage profile adds `--coverage` and publishes run-local coverage, falling back to reports under `.dev`. */ export class ElideTestController implements vscode.Disposable { private readonly controller = vscode.tests.createTestController("elide", "Elide"); @@ -254,13 +257,20 @@ export class ElideTestController implements vscode.Disposable { private async run(request: vscode.TestRunRequest, token: vscode.CancellationToken, summary?: TestRunSummary): Promise { const run = this.controller.createTestRun(request); + let coverageDirectory: string | undefined; try { + if (request.profile?.kind === vscode.TestRunProfileKind.Coverage) { + coverageDirectory = await mkdtemp(path.join(tmpdir(), "elide-coverage-")); + } + let invocation = 0; for (const group of this.groupByProject(request)) { if (token.isCancellationRequested) break; - await this.runGroup(run, request, group, token, summary); + const coverageFile = coverageDirectory ? path.join(coverageDirectory, `${invocation++}.json`) : undefined; + await this.runGroup(run, request, group, token, summary, coverageFile); } } finally { run.end(); + if (coverageDirectory) await rm(coverageDirectory, { recursive: true, force: true }); } } @@ -339,6 +349,7 @@ export class ElideTestController implements vscode.Disposable { group: RunGroup, token: vscode.CancellationToken, summary?: TestRunSummary, + coverageFile?: string, ): Promise { const { project, projectItem, leaves } = group; for (const leaf of leaves) run.enqueued(leaf); @@ -358,6 +369,7 @@ export class ElideTestController implements vscode.Disposable { // `elide.flags` and `elide.test.options` apply here too, except for the reporter: this run parses TAP. const invocation = mergeElideInvocationOptions(configuredInvocation(settings, "test"), { options: { ...(pattern ? { "test-name-pattern": pattern } : {}), reporter: "tap", ...(coverage ? { coverage: true } : {}) }, + ...(coverageFile ? { env: { ELIDE_COVERAGE_REPORT: coverageFile } } : {}), }); const argv = elideInvocationArgs("test", invocation); this.ui.log(`test: ${dist.bin} ${argv.join(" ")} (cwd ${project.root})`); @@ -377,6 +389,7 @@ export class ElideTestController implements vscode.Disposable { const invocations = new Map(); const stderrTail: string[] = []; let sawResult = false; + let sawFailure = false; const claim = (label: string): vscode.TestItem => { const base = this.matchLabel(index, projectItem, project.root, label, selectedIds); @@ -420,7 +433,10 @@ export class ElideTestController implements vscode.Disposable { const todo = event.directive?.type === "todo"; if (skip || (todo && !event.ok)) this.settle(run, summary, "skipped", item); else if (event.ok) this.settle(run, summary, "passed", item); - else this.settle(run, summary, "failed", item, failureMessage(event.diagnostics, item)); + else { + sawFailure = true; + this.settle(run, summary, "failed", item, failureMessage(event.diagnostics, item)); + } return; } case "other": @@ -495,23 +511,36 @@ export class ElideTestController implements vscode.Disposable { } parser.end(); - if (coverage && !token.isCancellationRequested) await this.publishCoverage(run, group, startedAt, summary); + if (coverage && !token.isCancellationRequested) await this.publishCoverage(run, group, startedAt, summary, coverageFile); if (token.isCancellationRequested || failure === undefined) return; - // A failing test already exits non-zero: only a run that reported nothing at all is an error of its own. + // Run-level failures (including coverage minimums) must not turn an all-passing TAP stream green. if (sawResult) { run.appendOutput(terminalText(failure)); + if (!sawFailure) this.settle(run, summary, "errored", projectItem, new vscode.TestMessage(failure)); return; } const message = new vscode.TestMessage(failure); for (const leaf of leaves) this.settle(run, summary, "errored", leaf, message); } - /** - * Publish the coverage the run left in each project it covered: guest LCOV, and JVM coverage when the manifest asks - * for a JaCoCo XML report. JaCoCo execution data alone cannot be read without the JaCoCo analyzer, so its presence - * only earns a hint. - */ - private async publishCoverage(run: vscode.TestRun, group: RunGroup, startedAt: number, summary?: TestRunSummary): Promise { + /** Prefer the invocation's JSON artifact; older CLIs fall back to configured reports under `.dev`. */ + private async publishCoverage(run: vscode.TestRun, group: RunGroup, startedAt: number, summary?: TestRunSummary, coverageFile?: string): Promise { + const publish = (files: readonly FileCoverageData[]) => { + for (const file of files) { + const details = statementsOf(file); + const coverage = vscode.FileCoverage.fromDetails(vscode.Uri.file(file.path), details); + this.coverageDetails.set(coverage, details); + run.addCoverage(coverage); + if (summary?.coverage) summary.coverage[file.path] = { covered: coverage.statementCoverage.covered, total: coverage.statementCoverage.total }; + } + }; + const artifact = coverageFile ? await loadCoverageArtifact(coverageFile, group.project.root) : undefined; + if (artifact) { + publish(artifact.files); + if (!artifact.complete) run.appendOutput(terminalText("Coverage collection was incomplete; only collected, unchanged sources are shown.")); + this.ui.log(`coverage: ${artifact.files.length} file(s) published`); + return; + } let published = 0; let reports = 0; for (const item of group.coveredItems) { @@ -525,14 +554,8 @@ export class ElideTestController implements vscode.Disposable { continue; } reports += loaded.reports.lcov.length + loaded.reports.jacocoXml.length + loaded.reports.jacocoExec.length; - for (const file of loaded.files) { - const details = statementsOf(file); - const coverage = vscode.FileCoverage.fromDetails(vscode.Uri.file(file.path), details); - this.coverageDetails.set(coverage, details); - run.addCoverage(coverage); - if (summary?.coverage) summary.coverage[file.path] = { covered: coverage.statementCoverage.covered, total: coverage.statementCoverage.total }; - published++; - } + publish(loaded.files); + published += loaded.files.length; if (loaded.reports.jacocoExec.length > 0 && loaded.reports.jacocoXml.length === 0) { run.appendOutput( terminalText( diff --git a/packages/vscode/test/testingCoverage.test.ts b/packages/vscode/test/testingCoverage.test.ts new file mode 100644 index 0000000..6728ee9 --- /dev/null +++ b/packages/vscode/test/testingCoverage.test.ts @@ -0,0 +1,235 @@ +import { expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; + +const source = new URL("../src/", import.meta.url).pathname; + +// Exercise the real CLI transport and artifact/report readers; only editor services and distribution lookup are mocked. +const executable = ` +const fs = require("node:fs"); +const path = require("node:path"); +const { createHash } = require("node:crypto"); +const root = process.cwd(); +const mode = fs.readFileSync(path.join(root, "mode"), "utf8"); +const artifact = process.env.ELIDE_COVERAGE_REPORT; +fs.writeFileSync(path.join(root, "invocation.json"), JSON.stringify({ artifact, pid: process.pid, argv: process.argv.slice(2) })); +if (mode === "legacy") { + const reports = path.join(root, ".dev/reports/coverage/js"); + fs.mkdirSync(reports, { recursive: true }); + fs.writeFileSync(path.join(reports, "lcov.info"), "SF:" + path.join(root, "src/ExampleTest.kt") + "\\nDA:3,1\\nDA:4,0\\nend_of_record\\n"); +} else if (mode !== "normal" && artifact) { + const utf16 = Buffer.from(fs.readFileSync(path.join(root, "src/ExampleTest.kt"), "utf8"), "utf16le"); + const value = createHash("sha256").update(utf16.swap16()).digest("hex"); + fs.writeFileSync(artifact, JSON.stringify({ version: 1, complete: true, files: [{ + engine: "jvm", path: "src/ExampleTest.kt", + sourceDigest: { algorithm: "sha256-utf16be", value }, + lines: [{ line: 3, covered: true }, { line: 4, covered: false }, { line: 5, covered: true, branches: { covered: 1, total: 2 } }], + }] })); +} +if (mode === "cancelled") { + // Written after the artifact so the parent cancels an actual in-flight coverage invocation. + fs.writeFileSync(path.join(root, "ready"), "ready"); + // A real subprocess must stay alive until cancellation; the parent's fake clock cannot drive it. + setInterval(() => {}, 1000); +} else { + console.log("TAP version 13\\n1..1\\nok 1 - sample.ExampleTest > works()"); + if (mode === "gate") { + console.error("JVM line coverage 66.7% is below the required minimum of 90.0%"); + // TestCommand.COVERAGE_BELOW_MINIMUM is 1, after publishing the artifact and passing TAP. + process.exitCode = 1; + } +} +`; + +type Scenario = "gate" | "normal" | "legacy" | "cancelled"; + +function runScenario(scenario: Scenario): void { + // Keep VS Code module mocks out of Bun's shared registry for other extension tests. + const result = spawnSync(process.execPath, ["--eval", ` + import assert from "node:assert/strict"; + import * as fs from "node:fs"; + import path from "node:path"; + import os from "node:os"; + import { setTimeout as delay } from "node:timers/promises"; + import { mock } from "bun:test"; + import * as core from "@elide/ide-core"; + const source = ${JSON.stringify(source)}; + const scenario = ${JSON.stringify(scenario)}; + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "elide-coverage-test-"))); + const file = path.join(root, "src/ExampleTest.kt"); + const bin = path.join(root, "elide"); + const events = [], published = [], output = [], cancellations = []; + let controller; + const disposable = { dispose() {} }; + const collection = (owner) => { + const items = new Map(); + return { + get size() { return items.size; }, + get: (id) => items.get(id), + add(item) { item.parent = owner; items.set(item.id, item); }, + delete(id) { items.delete(id); }, + replace(list) { items.clear(); for (const item of list) this.add(item); }, + [Symbol.iterator]: () => items.entries(), + }; + }; + const item = (id, label, uri) => { + const created = { id, label, uri, parent: undefined }; + created.children = collection(created); + return created; + }; + class CancellationTokenSource { + listeners = new Set(); + token = { + isCancellationRequested: false, + onCancellationRequested: (callback) => { + this.listeners.add(callback); + return { dispose: () => this.listeners.delete(callback) }; + }, + }; + constructor() { cancellations.push(this); } + cancel() { this.token.isCancellationRequested = true; for (const callback of this.listeners) callback(); } + dispose() { this.listeners.clear(); } + } + mock.module("@elide/ide-core", () => ({ ...core, resolveElideDistribution: () => ({ bin, home: root }) })); + mock.module("vscode", () => ({ + tests: { + createTestController: () => ({ + items: collection(undefined), createTestItem: item, + createRunProfile: (label, kind) => ({ label, kind }), + createTestRun: () => ({ + enqueued() {}, started() {}, end() {}, + appendOutput: (text) => output.push(text), + passed: (test) => events.push(["passed", test.id]), + skipped: (test) => events.push(["skipped", test.id]), + failed: (test) => events.push(["failed", test.id]), + errored: (test, message) => events.push(["errored", test.id, message.message]), + addCoverage: (coverage) => published.push(coverage), + }), + dispose() {}, + }), + }, + CancellationTokenSource, + TestRunProfileKind: { Run: 1, Debug: 2, Coverage: 3 }, + TestRunRequest: class { constructor(include, exclude, profile) { Object.assign(this, { include, exclude, profile }); } }, + TestMessage: class { constructor(message) { this.message = message; } }, + RelativePattern: class { constructor(base, pattern) { Object.assign(this, { base, pattern }); } }, + Range: class {}, + Position: class { constructor(line, character) { Object.assign(this, { line, character }); } }, + BranchCoverage: class { constructor(executed) { this.executed = executed; } }, + StatementCoverage: class { constructor(executed, location, branches) { Object.assign(this, { executed, location, branches }); } }, + FileCoverage: class { + static fromDetails(uri, details) { + return { + uri, details, + statementCoverage: { covered: details.filter((d) => d.executed === true || d.executed > 0).length, total: details.length }, + }; + } + }, + Uri: { file: (fsPath) => ({ fsPath, scheme: "file" }) }, + workspace: { + textDocuments: [], + fs: { readFile: async (uri) => fs.promises.readFile(uri.fsPath) }, + findFiles: async (pattern) => { + assert.equal(pattern.base.fsPath, path.join(root, "src")); + assert.ok(fs.statSync(file).isFile()); + return [{ fsPath: file, scheme: "file" }]; + }, + createFileSystemWatcher: () => ({ onDidCreate: () => disposable, onDidChange: () => disposable, onDidDelete: () => disposable, dispose() {} }), + onDidChangeTextDocument: () => disposable, + }, + })); + mock.module(source + "config.js", () => ({ + readConfig: () => ({}), + configuredInvocation: () => scenario === "gate" ? { env: { ELIDE_COVERAGE_REPORT: path.join(root, "must-not-be-used.json") } } : {}, + })); + mock.module(source + "debug.js", () => ({ launchWithJdwp() {}, resolveAttachType() {} })); + mock.module(source + "jetbrains.js", () => ({ isJetBrainsDebugger: () => false })); + const invocation = () => JSON.parse(fs.readFileSync(path.join(root, "invocation.json"), "utf8")); + const assertCleaned = (artifact) => { + assert.equal(typeof artifact, "string", "coverage CLI must receive a private artifact destination"); + assert.equal(path.isAbsolute(artifact), true); + assert.equal(fs.existsSync(artifact), false, "artifact must be removed after the run"); + assert.equal(fs.existsSync(path.dirname(artifact)), false, "private coverage directory must be removed too"); + }; + try { + fs.mkdirSync(path.dirname(file)); + fs.writeFileSync(file, "package sample\\nimport org.junit.jupiter.api.Test\\nclass ExampleTest {\\n // Unicode source: \\u03bb \\ud83d\\ude00\\n @Test fun works() {}\\n}\\n"); + fs.writeFileSync(bin, "#!" + process.execPath + "\\n" + ${JSON.stringify(executable)}, { mode: 0o755 }); + fs.writeFileSync(path.join(root, "mode"), scenario); + // Import after installing editor mocks: static import would capture the real VS Code module. + const { ElideTestController } = await import(source + "testing.ts"); + const project = { root, folder: {}, model: { name: "app", modules: [{ kind: "test", contentRoots: [{ path: root, sourceRoots: [{ kind: "test", path: path.join(root, "src") }] }] }] } }; + controller = new ElideTestController( + { projects: [project], projectAt: (value) => value === root ? project : undefined, membersOf: () => [], onDidChange: () => disposable }, + { log() {} }, new Map(), + ); + const pending = controller.runAll({ coverage: scenario !== "normal" }); + if (scenario === "cancelled") { + const deadline = Date.now() + 5000; + // Poll persistent child-process state; fake timers cannot advance another process's filesystem writes. + while (!fs.existsSync(path.join(root, "ready"))) { + assert.ok(Date.now() < deadline, "coverage executable did not become ready"); + await delay(10); + } + const running = invocation(); + assert.ok(fs.existsSync(running.artifact)); + cancellations.at(-1).cancel(); + const summary = await pending; + assertCleaned(running.artifact); + assert.throws(() => process.kill(running.pid, 0), "cancelled CLI must have exited"); + assert.deepEqual(summary.failed, []); + assert.deepEqual(summary.errored, []); + assert.deepEqual(published, []); + } else { + const summary = await pending; + const method = "m:" + root + ":sample.ExampleTest#works"; + assert.deepEqual(summary.passed, [method], "passing TAP must match the test discovered from disk"); + assert.deepEqual(summary.failed, []); + assert.deepEqual(summary.skipped, []); + if (scenario === "normal") { + assert.equal(invocation().artifact, undefined, "ordinary runs must not request a coverage artifact"); + assert.equal(summary.coverage, undefined); + assert.deepEqual(summary.errored, []); + assert.deepEqual(published, []); + } else { + const first = invocation().artifact; + assertCleaned(first); + assert.deepEqual(summary.coverage, { [file]: { covered: scenario === "gate" ? 2 : 1, total: scenario === "gate" ? 3 : 2 } }); + assert.equal(published.length, 1, "publish coverage without requiring JaCoCo XML"); + assert.equal(published[0].uri.fsPath, file); + if (scenario === "gate") { + assert.deepEqual(summary.errored, ["p:" + root], "unmet minimum errors the project, not its passing test"); + assert.equal(events.some((e) => e[0] === "errored" && e[1] === method), false); + assert.match(events.find((e) => e[0] === "errored")[2], /minimum|coverage|exited/i); + assert.equal(fs.existsSync(path.join(root, "must-not-be-used.json")), false, "artifact env overrides configured invocation env"); + assert.equal(fs.existsSync(path.join(root, ".dev")), false, "fixture deliberately produces no XML or legacy coverage reports"); + assert.deepEqual(published[0].details[2].branches.map((b) => b.executed), [1, 0]); + const again = await controller.runAll({ coverage: true }); + const second = invocation().artifact; + assert.notEqual(second, first, "coverage runs must not reuse artifact destinations"); + assertCleaned(second); + assert.deepEqual(again.coverage, summary.coverage); + assert.deepEqual(again.errored, ["p:" + root]); + } else { + assert.deepEqual(summary.errored, [], "old CLI can ignore the artifact variable and publish LCOV instead"); + } + } + } + } finally { + controller?.dispose(); + fs.rmSync(root, { recursive: true, force: true }); + } + `], { + cwd: new URL("..", import.meta.url).pathname, + env: Object.fromEntries(Object.entries(process.env).filter(([key]) => key !== "ELIDE_COVERAGE_REPORT")), + encoding: "utf8", + timeout: 20_000, + }); + expect(result.error).toBeUndefined(); + expect(result.stderr).toBe(""); + expect(result.status).toBe(0); +} + +test("Coverage minimum failure publishes version 1 coverage without XML, preserves passing tests, and cleans isolated artifacts", () => runScenario("gate")); +test("Ordinary test runs do not request coverage artifacts", () => runScenario("normal")); +test("Coverage falls back to legacy LCOV when an older CLI ignores the artifact destination", () => runScenario("legacy")); +test("Cancelling a coverage run terminates its CLI and removes the private artifact directory", () => runScenario("cancelled"));