From 686ba475b08075e71c7c5156aece189c0d9475ed Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 3 Oct 2026 22:08:07 +0000 Subject: [PATCH] fix(scan): record baselined findings in scan output vg scan --baseline previously kept only a numeric drift delta, so matched findings left no auditable trace in JSON or SARIF. The artifact now includes baselineComparison (compared, suppressedCount, and suppressed sorted by ruleId, location, then id). Findings stay in the primary array, human reports include the count, and SARIF suppressions carry the same ids. Fixes #161 Signed-off-by: Cursor Agent Co-authored-by: vibgrate-team --- DOCS.md | 37 +++- README.md | 1 + src/core-open/baseline-comparison.ts | 93 ++++++++ src/core-open/formatters/markdown.ts | 15 +- src/core-open/formatters/sarif.ts | 21 +- src/core-open/formatters/text.ts | 15 +- src/core-open/index.ts | 7 + src/core-open/run-core-scan.ts | 7 + src/core-open/types.ts | 40 ++++ src/reporting/commands/scan.ts | 2 +- src/reporting/formatters/formatters.test.ts | 37 ++++ src/reporting/formatters/markdown.ts | 15 +- src/reporting/formatters/text.ts | 15 +- src/reporting/types.ts | 10 + test/baseline-comparison.test.ts | 234 ++++++++++++++++++++ test/fixtures/baseline-suppressions.json | 48 ++++ 16 files changed, 585 insertions(+), 12 deletions(-) create mode 100644 src/core-open/baseline-comparison.ts create mode 100644 test/baseline-comparison.test.ts create mode 100644 test/fixtures/baseline-suppressions.json diff --git a/DOCS.md b/DOCS.md index 6ad775e..553a3d6 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1081,7 +1081,7 @@ vg scan [path] [--vulns] [--full] [--format text|json|sarif|md] [--out ] [ | `--format` | `text` | Output format: `text`, `json`, `sarif`, or `md` | | `--out ` | — | Write output to a file | | `--fail-on ` | — | Exit with code 2 if findings at this level exist. `warn` / `error` gate on drift findings. `architecture-finding` (hard boundary violations) and `architecture-warning` (violations and warnings) gate on the architecture module's boundary findings, judged under the policy pack in force — `hexagonal-v1` unless `.vibgrate/architecture.toml`, `VIBGRATE_ARCHITECTURE_POLICY` or `vg build --policy` says `layered-v1`. The output names the pack whether the gate passes or fails; each failing row is `file:line symbol violation: … (rule)`. Pick the pack before turning this on: see [Architecture policy packs](./docs/architecture-policies.md) | -| `--baseline ` | — | Compare against a previous baseline | +| `--baseline ` | — | Compare against a previous baseline. Matched findings stay in the report and are listed in `baselineComparison` (see [Drift Baselines](#drift-baselines--fitness-functions)) | | `--changed-only` | — | Only scan changed files | | `--concurrency ` | `8` | Max concurrent npm registry calls | | `--drift-budget ` | — | Fitness gate: fail if drift score is above this budget | @@ -2808,6 +2808,37 @@ Recommended workflow: This makes drift a formal quality gate (fitness function), not just reporting. +### What a baseline comparison records + +`vg scan --baseline` still reports the numeric drift delta (`delta`, and `--drift-worsening` uses that delta). It also writes an additive `baselineComparison` block on the scan artifact so a matched finding is not a silent drop. Findings stay in `findings`. The block is omitted when no baseline file was read. + +```json +"baseline": ".vibgrate/baseline.json", +"delta": 2, +"baselineComparison": { + "compared": true, + "suppressedCount": 2, + "suppressed": [ + { + "ruleId": "vibgrate/dependency-rot", + "location": "package.json", + "id": "c0ffee…" + } + ] +} +``` + +| Field | Meaning | +| ----- | ------- | +| `baseline` | Repo-relative path of the file that was compared (basename if the file is outside the repo) | +| `baselineComparison.compared` | `true` when that file was read | +| `baselineComparison.suppressedCount` | How many current findings were already in the baseline | +| `baselineComparison.suppressed` | `{ ruleId, location, id }` for each match, sorted by `ruleId`, then `location`, then `id` | + +`id` is 32 lowercase hex characters: the first 128 bits of SHA-256 over the length-prefixed `ruleId`, `level`, `location`, and `message`. The same finding always produces the same id. A changed message is a different finding and is not listed as suppressed. Text and Markdown reports include `Baseline suppressions: N` and mark matched rows `(baselined)`. + +`baseline` remains the path string it has always been. `baselineComparison` is the new audit record. + ## DriftScore ### How the Score Is Calculated @@ -2854,10 +2885,14 @@ The default output. A coloured, human-readable report showing: The full scan artifact in JSON format. Contains all raw data, scores, findings, and VCS metadata. Stable schema (`schemaVersion: "1.0"`). This is the same artifact saved to `.vibgrate/scan_result.json`. +When the scan compared a baseline, the artifact also carries `baseline` (the file path) and `baselineComparison` (`compared`, `suppressedCount`, and `suppressed` — see [What a baseline comparison records](#what-a-baseline-comparison-records)). Matched findings remain in `findings`. + ### SARIF [Static Analysis Results Interchange Format](https://sarifweb.azurewebsites.net/) — compatible with GitHub Code Scanning and Azure DevOps. Contains findings only (not all metrics). Ideal for integrating drift findings directly into your PR review workflow. +A finding that matched the baseline stays in `runs[0].results`. Its result gains a SARIF `suppressions` entry (`kind: "external"`, `status: "accepted"`) whose `properties.id` is the same id as `baselineComparison.suppressed`. Results that did not match have no `suppressions` field. + ### Markdown A clean Markdown report suitable for PRs, wikis, or documentation. diff --git a/README.md b/README.md index 3346e40..188a184 100644 --- a/README.md +++ b/README.md @@ -557,6 +557,7 @@ vg scan --baseline .vibgrate/baseline.json --drift-budget 40 --drift-worsening 5 - `--drift-budget ` fails the build if drift exceeds your budget. - `--drift-worsening ` fails the build if drift worsens by more than X% vs baseline. +- Matched findings stay in the report. JSON records them in `baselineComparison` (`compared`, `suppressedCount`, `suppressed` sorted by `ruleId`, `location`, `id`), and SARIF puts those ids on `suppressions`. See [DOCS.md](./DOCS.md#what-a-baseline-comparison-records). Copy-paste CI templates live in `examples/github-actions/`. Azure DevOps and GitLab CI snippets are in [DOCS.md](./DOCS.md#ci-integration). diff --git a/src/core-open/baseline-comparison.ts b/src/core-open/baseline-comparison.ts new file mode 100644 index 0000000..0648a51 --- /dev/null +++ b/src/core-open/baseline-comparison.ts @@ -0,0 +1,93 @@ +import { createHash } from 'node:crypto'; +import type { BaselineComparison, BaselineSuppressedFinding } from './types.js'; + +/** + * Fields that identify a drift finding across a baseline and a later scan. + * The message is part of the id, so a changed finding is not recorded as + * suppressed. + */ +export interface DriftFindingIdentity { + ruleId: string; + level: string; + location: string; + message: string; +} + +/** + * Content id for a drift finding. 32 lowercase hex characters (128 bits of + * SHA-256). Parts are length-prefixed so concatenation cannot collide, and + * the digest does not depend on object key order, clock, or map iteration. + */ +export function driftFindingId(finding: DriftFindingIdentity): string { + const payload = [finding.ruleId, finding.level, finding.location, finding.message] + .map(encodePart) + .join('\n'); + return createHash('sha256').update(payload).digest('hex').slice(0, 32); +} + +function encodePart(value: string): string { + return `${Buffer.byteLength(value, 'utf8')}:${value}`; +} + +function isIdentity(value: unknown): value is DriftFindingIdentity { + if (!value || typeof value !== 'object') return false; + const finding = value as Record; + return typeof finding.ruleId === 'string' + && typeof finding.level === 'string' + && typeof finding.location === 'string' + && typeof finding.message === 'string'; +} + +/** + * Findings in `current` that already appear in `baseline`. Does not remove + * or reorder `current`. `suppressed` is sorted by ruleId, then location, + * then id. Duplicate ids collapse to one record. + */ +export function compareBaselineFindings( + current: readonly unknown[], + baseline: readonly unknown[] | undefined, +): BaselineComparison { + const baselineIds = new Set(); + if (baseline) { + for (const finding of baseline) { + if (isIdentity(finding)) baselineIds.add(driftFindingId(finding)); + } + } + + const suppressed: BaselineSuppressedFinding[] = []; + const seen = new Set(); + for (const finding of current) { + if (!isIdentity(finding)) continue; + const id = driftFindingId(finding); + if (!baselineIds.has(id) || seen.has(id)) continue; + seen.add(id); + suppressed.push({ ruleId: finding.ruleId, location: finding.location, id }); + } + + suppressed.sort(compareSuppressed); + return { + compared: true, + suppressedCount: suppressed.length, + suppressed, + }; +} + +function compareSuppressed(a: BaselineSuppressedFinding, b: BaselineSuppressedFinding): number { + if (a.ruleId !== b.ruleId) return a.ruleId < b.ruleId ? -1 : 1; + if (a.location !== b.location) return a.location < b.location ? -1 : 1; + if (a.id !== b.id) return a.id < b.id ? -1 : 1; + return 0; +} + +/** Count line shared by text and Markdown reports. */ +export function baselineSuppressionSummary(count: number): string { + return `Baseline suppressions: ${count}`; +} + +/** Ids recorded on a comparison, or an empty set when no baseline was compared. */ +export function baselinedIdSet(comparison: BaselineComparison | undefined): Set { + const ids = new Set(); + if (!comparison) return ids; + for (const entry of comparison.suppressed) ids.add(entry.id); + return ids; +} diff --git a/src/core-open/formatters/markdown.ts b/src/core-open/formatters/markdown.ts index be83d21..0fc7af8 100644 --- a/src/core-open/formatters/markdown.ts +++ b/src/core-open/formatters/markdown.ts @@ -1,7 +1,8 @@ // VENDORED from @vibgrate/core-open (packages/vibgrate-core-open) by // scripts/vendor-core-open.mjs. Do not edit here — change the source package // and re-run the vendor script. Apache-2.0. -import type { ScanArtifact } from '../types.js'; +import type { ScanArtifact, Finding } from '../types.js'; +import { baselineSuppressionSummary, baselinedIdSet, driftFindingId } from '../baseline-comparison.js'; import { securityPacksLabel } from './text.js'; /** Rows shown before the infrastructure-findings table is cut with an "… N more" line. */ @@ -19,6 +20,11 @@ function formatBillable(value: number): string { /** Generate a Markdown report from scan artifact */ export function formatMarkdown(artifact: ScanArtifact): string { const lines: string[] = []; + const baselined = baselinedIdSet(artifact.baselineComparison); + const locationOf = (finding: Finding): string => { + const marked = baselined.has(driftFindingId(finding)); + return marked ? `${finding.location} (baselined)` : finding.location; + }; // Billing (micro-project pricing) is a commercial signal attached by the full // scan; the open base scan omits it. @@ -175,7 +181,7 @@ export function formatMarkdown(artifact: ScanArtifact): string { lines.push(`|-------|------|---------|----------|`); for (const f of artifact.findings) { const emoji = f.level === 'error' ? '🔴' : f.level === 'warning' ? '🟡' : '🔵'; - lines.push(`| ${emoji} ${f.level} | ${f.ruleId} | ${f.message} | ${f.location} |`); + lines.push(`| ${emoji} ${f.level} | ${f.ruleId} | ${f.message} | ${locationOf(f)} |`); } lines.push(''); } @@ -186,5 +192,10 @@ export function formatMarkdown(artifact: ScanArtifact): string { lines.push(''); } + if (artifact.baselineComparison) { + lines.push(baselineSuppressionSummary(artifact.baselineComparison.suppressedCount)); + lines.push(''); + } + return lines.join('\n'); } diff --git a/src/core-open/formatters/sarif.ts b/src/core-open/formatters/sarif.ts index 9825a8d..e4af4de 100644 --- a/src/core-open/formatters/sarif.ts +++ b/src/core-open/formatters/sarif.ts @@ -2,6 +2,7 @@ // scripts/vendor-core-open.mjs. Do not edit here — change the source package // and re-run the vendor script. Apache-2.0. import type { ScanArtifact, Finding, SecurityFinding, SecuritySection, SecuritySeverity } from '../types.js'; +import { driftFindingId } from '../baseline-comparison.js'; /** * Generate a SARIF 2.1.0 document from scan artifact. @@ -14,8 +15,9 @@ import type { ScanArtifact, Finding, SecurityFinding, SecuritySection, SecurityS * bytes as before. */ export function formatSarif(artifact: ScanArtifact): object { + const suppressedIds = new Set(artifact.baselineComparison?.suppressed.map((entry) => entry.id) ?? []); const rules = buildRules(artifact.findings); - const results = artifact.findings.map((f) => toSarifResult(f)); + const results = artifact.findings.map((f) => toSarifResult(f, suppressedIds)); const runs: object[] = [ { @@ -179,7 +181,8 @@ function buildRules(findings: Finding[]) { }); } -function toSarifResult(finding: Finding) { +function toSarifResult(finding: Finding, suppressedIds: ReadonlySet) { + const id = driftFindingId(finding); return { ruleId: finding.ruleId, level: finding.level === 'error' ? 'error' : finding.level === 'warning' ? 'warning' : 'note', @@ -196,5 +199,19 @@ function toSarifResult(finding: Finding) { // Surface structured finding detail (e.g. advisory id, CVSS, fixed version) // to consumers like GitHub code scanning without bloating the message text. ...(finding.details && Object.keys(finding.details).length > 0 ? { properties: finding.details } : {}), + // The finding stays in `results`. The suppression carries the same id as + // `baselineComparison.suppressed` so a baseline match is not a silent drop. + ...(suppressedIds.has(id) + ? { + suppressions: [ + { + kind: 'external', + status: 'accepted', + justification: 'Matched the compared drift baseline', + properties: { id }, + }, + ], + } + : {}), }; } diff --git a/src/core-open/formatters/text.ts b/src/core-open/formatters/text.ts index 9f1ed28..eb0a62f 100644 --- a/src/core-open/formatters/text.ts +++ b/src/core-open/formatters/text.ts @@ -2,7 +2,8 @@ // scripts/vendor-core-open.mjs. Do not edit here — change the source package // and re-run the vendor script. Apache-2.0. import chalk from 'chalk'; -import type { ScanArtifact, BillingSummary, ExtendedScanResults, InventoryItem, ServiceDependencyItem, ArchitectureResult, SecurityFinding, SecuritySection } from '../types.js'; +import type { ScanArtifact, BillingSummary, ExtendedScanResults, InventoryItem, ServiceDependencyItem, ArchitectureResult, SecurityFinding, SecuritySection, Finding } from '../types.js'; +import { baselineSuppressionSummary, baselinedIdSet, driftFindingId } from '../baseline-comparison.js'; import { driftBar } from '../ui/bar.js'; import { titleBox, panelBox } from '../ui/box.js'; @@ -44,6 +45,11 @@ export interface FormatTextOptions { export function formatText(artifact: ScanArtifact, opts: FormatTextOptions = {}): string { const lines: string[] = []; + const baselined = baselinedIdSet(artifact.baselineComparison); + const locationOf = (finding: Finding): string => { + const marked = baselined.has(driftFindingId(finding)); + return marked ? `${finding.location} (baselined)` : finding.location; + }; lines.push(''); lines.push(...titleBox('Vibgrate Drift Report')); @@ -94,6 +100,11 @@ export function formatText(artifact: ScanArtifact, opts: FormatTextOptions = {}) ? chalk.red(`+${artifact.delta}`) : chalk.dim('0'); lines.push(chalk.bold(' Drift Delta: ') + deltaStr + ' (vs baseline)'); + } + if (artifact.baselineComparison) { + lines.push(chalk.bold(` ${baselineSuppressionSummary(artifact.baselineComparison.suppressedCount)}`)); + } + if (artifact.delta !== undefined || artifact.baselineComparison) { lines.push(''); } @@ -117,7 +128,7 @@ export function formatText(artifact: ScanArtifact, opts: FormatTextOptions = {}) for (const f of artifact.findings) { const icon = f.level === 'error' ? chalk.red('✖') : f.level === 'warning' ? chalk.yellow('⚠') : chalk.blue('ℹ'); lines.push(` ${icon} ${f.message}`); - lines.push(chalk.dim(` ${f.ruleId} in ${f.location}`)); + lines.push(chalk.dim(` ${f.ruleId} in ${locationOf(f)}`)); } lines.push(''); } diff --git a/src/core-open/index.ts b/src/core-open/index.ts index 2142847..0c80193 100644 --- a/src/core-open/index.ts +++ b/src/core-open/index.ts @@ -66,6 +66,13 @@ export { export { formatText } from './formatters/text.js'; export { formatSarif } from './formatters/sarif.js'; export { formatMarkdown } from './formatters/markdown.js'; +export { + compareBaselineFindings, + driftFindingId, + baselineSuppressionSummary, + baselinedIdSet, +} from './baseline-comparison.js'; +export type { DriftFindingIdentity } from './baseline-comparison.js'; // ── Scanners (fact collection) ─────────────────────────────────────────────── export { scanNodeProjects } from './scanners/node-scanner.js'; diff --git a/src/core-open/run-core-scan.ts b/src/core-open/run-core-scan.ts index 5375ff9..c1ffca7 100644 --- a/src/core-open/run-core-scan.ts +++ b/src/core-open/run-core-scan.ts @@ -31,6 +31,7 @@ import { ComposerCache } from './scanners/composer-cache.js'; import { PubCache } from './scanners/pub-cache.js'; import { Semaphore } from './utils/semaphore.js'; import { computeDriftScore, generateFindings, computeProjectId, computeSolutionId } from './scoring/drift-score.js'; +import { compareBaselineFindings } from './baseline-comparison.js'; import { formatText } from './formatters/text.js'; import { formatSarif } from './formatters/sarif.js'; import { formatMarkdown } from './formatters/markdown.js'; @@ -824,6 +825,12 @@ export async function runCoreScan( const relBaseline = path.relative(rootDir, baselinePath); artifact.baseline = !relBaseline || relBaseline.startsWith('..') ? path.basename(baselinePath) : relBaseline; artifact.delta = artifact.drift.score - baseline.drift.score; + // Matched findings stay in `findings`. This block is the auditable + // record that they were already in the baseline. + artifact.baselineComparison = compareBaselineFindings( + artifact.findings, + Array.isArray(baseline.findings) ? baseline.findings : [], + ); } catch { console.error(chalk.yellow(`Warning: Could not read baseline file: ${baselinePath}`)); } diff --git a/src/core-open/types.ts b/src/core-open/types.ts index 07f0335..33ff8d0 100644 --- a/src/core-open/types.ts +++ b/src/core-open/types.ts @@ -696,6 +696,35 @@ export interface BillingSummary { billableProjects: number; } +/** + * One current finding that was already present in the compared baseline. + * The full finding (message, level, details) stays in `findings`; this is + * the auditable identifier, not a replacement for that row. + */ +export interface BaselineSuppressedFinding { + ruleId: string; + /** Same `location` string as the finding in `findings`. */ + location: string; + /** + * 32 lowercase hex characters. Derived from the finding's rule, level, + * location, and message, so the same finding always yields the same id + * and a changed message yields a different one. + */ + id: string; +} + +/** + * Additive record written when `vg scan --baseline` reads a baseline file. + * Omitted entirely when no baseline was compared. `suppressed` is sorted by + * `ruleId`, then `location`, then `id`. + */ +export interface BaselineComparison { + compared: true; + /** Number of entries in `suppressed`. */ + suppressedCount: number; + suppressed: BaselineSuppressedFinding[]; +} + // ── Full scan artifact (stable schema) ── export interface ScanArtifact { @@ -709,7 +738,18 @@ export interface ScanArtifact { solutions?: SolutionScan[]; drift: DriftScore; findings: Finding[]; + /** + * Repo-relative path of the baseline file this scan was compared against, + * or the file's basename when it lives outside the repo. Absent when + * `--baseline` was not used or the file could not be read. + */ baseline?: string; + /** + * Which current findings were already in the baseline. Those findings stay + * in `findings`; this block is the trace that they matched. Absent when no + * baseline was compared. + */ + baselineComparison?: BaselineComparison; delta?: number; extended?: ExtendedScanResults; /** Scan wall-clock duration in milliseconds */ diff --git a/src/reporting/commands/scan.ts b/src/reporting/commands/scan.ts index 76a4e3b..b511ae8 100644 --- a/src/reporting/commands/scan.ts +++ b/src/reporting/commands/scan.ts @@ -376,7 +376,7 @@ export const scanCommand = new Command('scan') '--fail-on ', 'Fail on warn or error. architecture-finding (hard boundary violations) or architecture-warning (violations and warnings) gate on the architecture module\'s boundary findings, judged under the policy pack in force: .vibgrate/architecture.toml (policy = "hexagonal-v1" | "layered-v1" | "vertical-v1", plus any [[overlay]] rules), VIBGRATE_ARCHITECTURE_POLICY, or vg build --policy; default hexagonal-v1. The pack is named in the output. See docs/architecture-policies.md. iac-finding[=] fails on infrastructure findings from the iac-cis-v1 pack at or above (critical|high|medium|low|info; default high) and needs --iac (or --full) plus the code map — it exits 2 when the Architecture module is missing rather than passing an unevaluated tree; security-finding[=] is the umbrella across every security pack that ran. Comma-separated: at most one of warn/error/architecture-* plus any security gates, e.g. --fail-on error,iac-finding=medium', ) - .option('--baseline ', 'Compare against baseline') + .option('--baseline ', 'Compare against a baseline and record matched findings') .option('--changed-only', 'Only scan changed files') .option( '-e, --exclude ', diff --git a/src/reporting/formatters/formatters.test.ts b/src/reporting/formatters/formatters.test.ts index 9019427..80940fc 100644 --- a/src/reporting/formatters/formatters.test.ts +++ b/src/reporting/formatters/formatters.test.ts @@ -3,6 +3,7 @@ import { describe, it, expect } from 'vitest'; // reporting-side copy was a stale duplicate and is gone. These tests keep // exercising the live writer. import { formatSarif as formatSarifCore } from '../../core-open/formatters/sarif.js'; +import { compareBaselineFindings } from '../../core-open/baseline-comparison.js'; import { formatMarkdown } from '../formatters/markdown.js'; import { formatText } from '../formatters/text.js'; import type { ScanArtifact } from '../types.js'; @@ -145,6 +146,25 @@ describe('formatSarif', () => { expect(sarif.runs[0].invocations[0].startTimeUtc).toBe('2026-02-16T00:00:00.000Z'); expect(sarif.runs[0].invocations[0].executionSuccessful).toBe(true); }); + + it('keeps baselined findings and copies their ids into SARIF suppressions', () => { + const artifact = makeArtifact(); + const matched = artifact.findings[1]!; + artifact.baselineComparison = compareBaselineFindings(artifact.findings, [matched]); + const sarif = formatSarif(artifact) as any; + const results = sarif.runs[0].results; + + expect(results).toHaveLength(artifact.findings.length); + expect(results[0].suppressions).toBeUndefined(); + expect(results[1].suppressions).toEqual([ + { + kind: 'external', + status: 'accepted', + justification: 'Matched the compared drift baseline', + properties: { id: artifact.baselineComparison.suppressed[0]!.id }, + }, + ]); + }); }); // ── Markdown formatter ── @@ -212,6 +232,15 @@ describe('formatMarkdown', () => { expect(md).toContain('📉'); }); + it('includes the baseline suppression count', () => { + const artifact = makeArtifact(); + artifact.baselineComparison = compareBaselineFindings(artifact.findings, [artifact.findings[1]!]); + const md = formatMarkdown(artifact); + expect(md).toContain('Baseline suppressions: 1'); + expect(md).toContain('/test/app (baselined)'); + expect(md).toContain('vibgrate/runtime-lag'); + }); + it('handles empty findings', () => { const md = formatMarkdown(makeArtifact({ findings: [] })); expect(md).not.toContain('## Findings'); @@ -368,6 +397,14 @@ describe('formatText', () => { expect(text).toContain('vs baseline'); }); + it('includes the baseline suppression count', () => { + const artifact = makeArtifact({ delta: 1 }); + artifact.baselineComparison = compareBaselineFindings(artifact.findings, artifact.findings); + const text = formatText(artifact); + expect(text).toContain('Baseline suppressions: 2'); + expect(text).toContain('/test/app (baselined)'); + }); + it('handles empty projects', () => { const text = formatText(makeArtifact({ projects: [], findings: [] })); expect(text).toContain('Vibgrate Drift Report'); diff --git a/src/reporting/formatters/markdown.ts b/src/reporting/formatters/markdown.ts index 5778589..74cba12 100644 --- a/src/reporting/formatters/markdown.ts +++ b/src/reporting/formatters/markdown.ts @@ -1,8 +1,14 @@ -import type { ScanArtifact } from '../types.js'; +import type { ScanArtifact, Finding } from '../types.js'; +import { baselineSuppressionSummary, baselinedIdSet, driftFindingId } from '../../core-open/baseline-comparison.js'; /** Generate a Markdown report from scan artifact */ export function formatMarkdown(artifact: ScanArtifact): string { const lines: string[] = []; + const baselined = baselinedIdSet(artifact.baselineComparison); + const locationOf = (finding: Finding): string => { + const marked = baselined.has(driftFindingId(finding)); + return marked ? `${finding.location} (baselined)` : finding.location; + }; lines.push('# Vibgrate Drift Report'); lines.push(''); @@ -117,7 +123,7 @@ export function formatMarkdown(artifact: ScanArtifact): string { lines.push(`|-------|------|---------|----------|`); for (const f of artifact.findings) { const emoji = f.level === 'error' ? '🔴' : f.level === 'warning' ? '🟡' : '🔵'; - lines.push(`| ${emoji} ${f.level} | ${f.ruleId} | ${f.message} | ${f.location} |`); + lines.push(`| ${emoji} ${f.level} | ${f.ruleId} | ${f.message} | ${locationOf(f)} |`); } lines.push(''); } @@ -130,5 +136,10 @@ export function formatMarkdown(artifact: ScanArtifact): string { lines.push(''); } + if (artifact.baselineComparison) { + lines.push(baselineSuppressionSummary(artifact.baselineComparison.suppressedCount)); + lines.push(''); + } + return lines.join('\n'); } diff --git a/src/reporting/formatters/text.ts b/src/reporting/formatters/text.ts index 5275e34..76c6c02 100644 --- a/src/reporting/formatters/text.ts +++ b/src/reporting/formatters/text.ts @@ -1,11 +1,17 @@ import chalk from 'chalk'; -import type { ScanArtifact, ExtendedScanResults, InventoryItem, ServiceDependencyItem, ArchitectureResult } from '../types.js'; +import type { ScanArtifact, ExtendedScanResults, InventoryItem, ServiceDependencyItem, ArchitectureResult, Finding } from '../types.js'; +import { baselineSuppressionSummary, baselinedIdSet, driftFindingId } from '../../core-open/baseline-comparison.js'; import { VERSION } from '../version.js'; import { driftBar } from '../../core-open/ui/bar.js'; import { titleBox } from '../../core-open/ui/box.js'; export function formatText(artifact: ScanArtifact): string { const lines: string[] = []; + const baselined = baselinedIdSet(artifact.baselineComparison); + const locationOf = (finding: Finding): string => { + const marked = baselined.has(driftFindingId(finding)); + return marked ? `${finding.location} (baselined)` : finding.location; + }; // Brand colours from docs/design logo bundle const teal = chalk.hex('#3FB0A4'); @@ -65,6 +71,11 @@ export function formatText(artifact: ScanArtifact): string { ? chalk.green(`${artifact.delta}`) : chalk.dim('0'); lines.push(chalk.bold(' Drift Delta: ') + deltaStr + ' (vs baseline)'); + } + if (artifact.baselineComparison) { + lines.push(chalk.bold(` ${baselineSuppressionSummary(artifact.baselineComparison.suppressedCount)}`)); + } + if (artifact.delta !== undefined || artifact.baselineComparison) { lines.push(''); } @@ -88,7 +99,7 @@ export function formatText(artifact: ScanArtifact): string { for (const f of artifact.findings) { const icon = f.level === 'error' ? chalk.red('✖') : f.level === 'warning' ? chalk.yellow('⚠') : chalk.blue('ℹ'); lines.push(` ${icon} ${f.message}`); - lines.push(chalk.dim(` ${f.ruleId} in ${f.location}`)); + lines.push(chalk.dim(` ${f.ruleId} in ${locationOf(f)}`)); } lines.push(''); } diff --git a/src/reporting/types.ts b/src/reporting/types.ts index 0ac11bf..1f38e47 100644 --- a/src/reporting/types.ts +++ b/src/reporting/types.ts @@ -1,5 +1,9 @@ // ── Core types for Vibgrate CLI ── +import type { BaselineComparison } from '../core-open/types.js'; + +export type { BaselineComparison }; + export type DepSection = 'dependencies' | 'devDependencies' | 'peerDependencies' | 'optionalDependencies'; export type RiskLevel = 'low' | 'moderate' | 'high' | 'none'; @@ -245,7 +249,13 @@ export interface ScanArtifact { solutions?: SolutionScan[]; drift: DriftScore; findings: Finding[]; + /** Repo-relative baseline path. See core-open `ScanArtifact.baseline`. */ baseline?: string; + /** + * Auditable record of findings already present in the baseline. Findings + * stay in `findings`. Absent when no baseline was compared. + */ + baselineComparison?: BaselineComparison; delta?: number; extended?: ExtendedScanResults; /** Scan wall-clock duration in milliseconds */ diff --git a/test/baseline-comparison.test.ts b/test/baseline-comparison.test.ts new file mode 100644 index 0000000..3047b0a --- /dev/null +++ b/test/baseline-comparison.test.ts @@ -0,0 +1,234 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { tmpdir } from 'node:os'; +import { + compareBaselineFindings, + driftFindingId, + baselineSuppressionSummary, +} from '../src/core-open/baseline-comparison.js'; +import { formatSarif } from '../src/core-open/formatters/sarif.js'; +import { formatText } from '../src/core-open/formatters/text.js'; +import { formatMarkdown } from '../src/core-open/formatters/markdown.js'; +import { runCoreScan } from '../src/core-open/index.js'; +import type { Finding, ScanArtifact } from '../src/core-open/types.js'; + +const fixture = JSON.parse( + fs.readFileSync(new URL('./fixtures/baseline-suppressions.json', import.meta.url), 'utf8'), +) as { baselineFindings: Finding[]; currentFindings: Finding[] }; + +const LOCKED_COMPARISON = { + compared: true as const, + suppressedCount: 2, + suppressed: [ + { + ruleId: 'vibgrate/dependency-major-lag', + location: 'package.json', + id: '4b6faf5c74f7e7066f2434c6fb0bf1b0', + }, + { + ruleId: 'vibgrate/dependency-rot', + location: 'package.json', + id: 'b3d2b31a1ca6957f33dd815aa237a031', + }, + ], +}; + +function miniArtifact(findings: Finding[], comparison = compareBaselineFindings(findings, fixture.baselineFindings)): ScanArtifact { + return { + schemaVersion: '1.0', + timestamp: '2026-01-01T00:00:00.000Z', + vibgrateVersion: 'test', + rootPath: 'demo', + projects: [], + drift: { + score: 10, + riskLevel: 'low', + components: { runtimeScore: 0, frameworkScore: 0, dependencyScore: 0, eolScore: 0 }, + }, + findings, + delta: 0, + baseline: '.vibgrate/baseline.json', + baselineComparison: comparison, + }; +} + +describe('baseline comparison record', () => { + it('records matched findings without removing them from the primary array', () => { + const findings = fixture.currentFindings.map((finding) => ({ ...finding })); + const before = JSON.stringify(findings); + const comparison = compareBaselineFindings(findings, fixture.baselineFindings); + + expect(JSON.stringify(findings)).toBe(before); + expect(comparison).toEqual(LOCKED_COMPARISON); + expect(comparison.suppressedCount).toBe(comparison.suppressed.length); + + const document = { findings, baselineComparison: comparison }; + expect(document.findings).toHaveLength(fixture.currentFindings.length); + expect(document.findings.map((finding) => finding.ruleId)).toContain('vibgrate/framework-major-lag'); + for (const entry of document.baselineComparison.suppressed) { + expect(document.findings.some((finding) => + finding.ruleId === entry.ruleId + && finding.location === entry.location + && driftFindingId(finding) === entry.id, + )).toBe(true); + } + }); + + it('is deterministic and sorts by ruleId, location, then id', () => { + const shuffled = [...fixture.currentFindings].reverse(); + const once = compareBaselineFindings(shuffled, [...fixture.baselineFindings].reverse()); + const twice = compareBaselineFindings(fixture.currentFindings, fixture.baselineFindings); + expect(JSON.stringify(once)).toBe(JSON.stringify(twice)); + expect(JSON.stringify(once)).toBe(JSON.stringify(LOCKED_COMPARISON)); + }); + + it('derives a stable content id and ignores a changed message', () => { + const matched = fixture.currentFindings[2]!; + expect(driftFindingId(matched)).toBe('b3d2b31a1ca6957f33dd815aa237a031'); + expect(driftFindingId(matched)).toBe(driftFindingId({ ...matched })); + expect(driftFindingId({ ...matched, message: '10% of dependencies are 2+ major versions behind in demo.' })) + .not.toBe(driftFindingId(matched)); + }); + + it('collapses duplicate ids and skips entries that are not findings', () => { + const matched = fixture.baselineFindings[1]!; + const comparison = compareBaselineFindings( + [matched, { ...matched }, { ruleId: 'vibgrate/dependency-rot' }], + [null, { ruleId: 'incomplete' }, matched], + ); + expect(comparison).toEqual({ + compared: true, + suppressedCount: 1, + suppressed: [{ ruleId: matched.ruleId, location: matched.location, id: driftFindingId(matched) }], + }); + }); + + it('reports an empty suppression list when nothing matched', () => { + expect(compareBaselineFindings([], [])).toEqual({ + compared: true, + suppressedCount: 0, + suppressed: [], + }); + expect(compareBaselineFindings(fixture.currentFindings, undefined).suppressed).toEqual([]); + }); + + it('keeps every finding in SARIF and puts the same ids on suppressions', () => { + const findings = fixture.currentFindings; + const comparison = compareBaselineFindings(findings, fixture.baselineFindings); + const sarif = formatSarif(miniArtifact(findings, comparison)) as { + runs: Array<{ results: Array<{ suppressions?: Array<{ kind: string; status: string; properties: { id: string } }> }> }>; + }; + const results = sarif.runs[0]!.results; + expect(results).toHaveLength(findings.length); + const suppressedIds = results + .filter((result) => result.suppressions) + .map((result) => result.suppressions![0]!.properties.id); + expect(suppressedIds).toEqual(comparison.suppressed.map((entry) => entry.id)); + for (const result of results) { + if (!result.suppressions) continue; + expect(result.suppressions[0]).toMatchObject({ + kind: 'external', + status: 'accepted', + properties: { id: result.suppressions[0]!.properties.id }, + }); + } + const unmatched = formatSarif(miniArtifact(findings, { compared: true, suppressedCount: 0, suppressed: [] })) as { + runs: Array<{ results: Array<{ suppressions?: unknown }> }>; + }; + expect(unmatched.runs[0]!.results.every((result) => result.suppressions === undefined)).toBe(true); + }); + + it('includes the suppression count in human output', () => { + const artifact = miniArtifact(fixture.currentFindings); + const text = formatText(artifact); + const markdown = formatMarkdown(artifact); + const summary = baselineSuppressionSummary(2); + expect(text).toContain(summary); + expect(markdown).toContain(summary); + expect(text).toContain('package.json (baselined)'); + expect(markdown).toContain('package.json (baselined)'); + expect(text).toContain('vibgrate/framework-major-lag'); + expect(markdown).toContain('vibgrate/framework-major-lag'); + }); +}); + +describe('vg scan --baseline wiring', () => { + let dir: string; + let logSpy: ReturnType; + + beforeEach(() => { + dir = fs.mkdtempSync(path.join(tmpdir(), 'vg-baseline-')); + fs.writeFileSync( + path.join(dir, 'package.json'), + JSON.stringify({ name: 'baseline-fixture', version: '0.0.0', private: true }), + ); + vi.stubEnv('VIBGRATE_DSN', ''); + vi.stubGlobal('fetch', async () => { + throw new Error('unexpected network access during baseline scan'); + }); + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + function scanOpts(extra: Partial[1]> = {}) { + return { + format: 'text' as const, + concurrency: 2, + offline: true, + noLocalArtifacts: true, + quiet: true, + vibgrateVersion: 'test', + ...extra, + }; + } + + it('omits the block when the baseline file is missing or unreadable', async () => { + const missing = await runCoreScan(dir, scanOpts({ baseline: path.join(dir, 'missing.json') })); + expect(missing.baseline).toBeUndefined(); + expect(missing.baselineComparison).toBeUndefined(); + expect(missing.delta).toBeUndefined(); + + const badPath = path.join(dir, 'bad.json'); + fs.writeFileSync(badPath, '{'); + const bad = await runCoreScan(dir, scanOpts({ baseline: badPath })); + expect(bad.baselineComparison).toBeUndefined(); + expect(bad.findings.length).toBe(missing.findings.length); + }); + + it('writes the comparison onto the artifact and the human report', async () => { + const first = await runCoreScan(dir, scanOpts()); + const baselinePath = path.join(dir, '.vibgrate', 'baseline.json'); + fs.mkdirSync(path.dirname(baselinePath), { recursive: true }); + fs.writeFileSync(baselinePath, JSON.stringify(first)); + + logSpy.mockClear(); + const second = await runCoreScan(dir, scanOpts({ baseline: baselinePath })); + + expect(second.findings).toEqual(first.findings); + expect(second.baseline).toBe(path.join('.vibgrate', 'baseline.json')); + expect(second.baselineComparison).toEqual(compareBaselineFindings(second.findings, first.findings)); + expect(second.baselineComparison?.compared).toBe(true); + expect(logSpy.mock.calls.flat().join('\n')).toContain( + baselineSuppressionSummary(second.baselineComparison?.suppressedCount ?? -1), + ); + + const sarif = formatSarif(second) as { + runs: Array<{ results: Array<{ suppressions?: Array<{ properties: { id: string } }> }> }>; + }; + const ids = new Set(second.baselineComparison?.suppressed.map((entry) => entry.id)); + expect(sarif.runs[0]!.results).toHaveLength(second.findings.length); + for (const result of sarif.runs[0]!.results) { + expect(result.suppressions).toHaveLength(1); + expect(ids.has(result.suppressions![0]!.properties.id)).toBe(true); + } + }); +}); diff --git a/test/fixtures/baseline-suppressions.json b/test/fixtures/baseline-suppressions.json new file mode 100644 index 0000000..8db1218 --- /dev/null +++ b/test/fixtures/baseline-suppressions.json @@ -0,0 +1,48 @@ +{ + "baselineFindings": [ + { + "ruleId": "vibgrate/runtime-lag", + "level": "warning", + "message": "Node.js runtime \"20\" is 2 major versions behind (latest: 24).", + "location": "services/api" + }, + { + "ruleId": "vibgrate/dependency-rot", + "level": "error", + "message": "40% of dependencies are 2+ major versions behind in demo.", + "location": "package.json" + }, + { + "ruleId": "vibgrate/dependency-major-lag", + "level": "error", + "message": "left-pad is 3 major versions behind (spec: 1.0.0, latest: 9.0.0).", + "location": "package.json" + } + ], + "currentFindings": [ + { + "ruleId": "vibgrate/dependency-major-lag", + "level": "error", + "message": "left-pad is 3 major versions behind (spec: 1.0.0, latest: 9.0.0).", + "location": "package.json" + }, + { + "ruleId": "vibgrate/dependency-rot", + "level": "warning", + "message": "10% of dependencies are 2+ major versions behind in demo.", + "location": "package.json" + }, + { + "ruleId": "vibgrate/dependency-rot", + "level": "error", + "message": "40% of dependencies are 2+ major versions behind in demo.", + "location": "package.json" + }, + { + "ruleId": "vibgrate/framework-major-lag", + "level": "warning", + "message": "react is 1 major version behind (current: 18.0.0, latest: 19.0.0).", + "location": "package.json" + } + ] +}