From 4dcaff7d1d347d9ba3342c0ff357cc12e8ff1685 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 3 Oct 2026 22:09:16 +0000 Subject: [PATCH] feat(scan): record baselined findings in scan output vg scan --baseline used the baseline only for the numeric drift delta, so matched findings left no trace in JSON or SARIF. The artifact now includes a baseline block (compared, file, suppressedCount, suppressed) sorted by ruleId, location, and id. Findings stay in the primary array. Text and Markdown reports include the count, and SARIF attaches the same ids as suppressions. Fixes #161 Signed-off-by: Cursor Agent Co-authored-by: vibgrate-team --- DOCS.md | 21 +- README.md | 1 + src/core-open/baseline-audit.ts | 181 ++++++++++++ src/core-open/formatters/markdown.ts | 7 + src/core-open/formatters/sarif.ts | 48 ++- src/core-open/formatters/text.ts | 25 +- src/core-open/index.ts | 10 + src/core-open/run-core-scan.ts | 24 +- src/core-open/types.ts | 35 ++- src/reporting/baseline-audit.test.ts | 311 ++++++++++++++++++++ src/reporting/commands/scan.ts | 2 +- src/reporting/formatters/formatters.test.ts | 76 +++++ src/reporting/formatters/markdown.ts | 7 + src/reporting/formatters/text.ts | 25 +- src/reporting/types.ts | 25 +- 15 files changed, 762 insertions(+), 36 deletions(-) create mode 100644 src/core-open/baseline-audit.ts create mode 100644 src/reporting/baseline-audit.test.ts diff --git a/DOCS.md b/DOCS.md index 6ad775e..e99e138 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1081,7 +1081,7 @@ vg scan [path] [--vulns] [--full] [--format text|json|sarif|md] [--out ] [ | `--format` | `text` | Output format: `text`, `json`, `sarif`, or `md` | | `--out ` | — | Write output to a file | | `--fail-on ` | — | Exit with code 2 if findings at this level exist. `warn` / `error` gate on drift findings. `architecture-finding` (hard boundary violations) and `architecture-warning` (violations and warnings) gate on the architecture module's boundary findings, judged under the policy pack in force — `hexagonal-v1` unless `.vibgrate/architecture.toml`, `VIBGRATE_ARCHITECTURE_POLICY` or `vg build --policy` says `layered-v1`. The output names the pack whether the gate passes or fails; each failing row is `file:line symbol violation: … (rule)`. Pick the pack before turning this on: see [Architecture policy packs](./docs/architecture-policies.md) | -| `--baseline ` | — | Compare against a previous baseline | +| `--baseline ` | — | Compare against a previous baseline. Matched findings stay in the report and are listed on `baseline.suppressed` (and as SARIF suppressions); the text and Markdown reports include the count | | `--changed-only` | — | Only scan changed files | | `--concurrency ` | `8` | Max concurrent npm registry calls | | `--drift-budget ` | — | Fitness gate: fail if drift score is above this budget | @@ -2808,6 +2808,21 @@ Recommended workflow: This makes drift a formal quality gate (fitness function), not just reporting. +`vg scan --baseline` does not drop findings that were already in the baseline. The scan artifact records them so a suppression is visible in the same document: + +```json +"baseline": { + "compared": true, + "file": ".vibgrate/baseline.json", + "suppressedCount": 2, + "suppressed": [ + { "ruleId": "vibgrate/dependency-rot", "location": "package.json", "id": "af173ea1c1efa4fd" } + ] +} +``` + +`suppressed` is sorted by `ruleId`, then `location`, then `id`. `file` is the repo-relative path (`/` separators), or the file's basename when the baseline sits outside the repo — an absolute path is never stored. `id` is 16 hex characters of SHA-256 over the rule, the location, and a stable subject: advisory id, ecosystem, and package for a vulnerability; the package or framework name for a major-lag finding; otherwise the rule and location. Text and Markdown reports include `suppressedCount`. SARIF keeps the same results and sets `suppressions[].properties.id` to that id. + ## DriftScore ### How the Score Is Calculated @@ -2854,10 +2869,14 @@ The default output. A coloured, human-readable report showing: The full scan artifact in JSON format. Contains all raw data, scores, findings, and VCS metadata. Stable schema (`schemaVersion: "1.0"`). This is the same artifact saved to `.vibgrate/scan_result.json`. +When the scan was run with `--baseline`, the artifact also contains a `baseline` object: `compared`, `file`, `suppressedCount`, and `suppressed` (`ruleId`, `location`, `id`). Those entries are the findings that were already in the baseline. The same findings remain in `findings`. See [Drift Baselines & Fitness Functions](#drift-baselines--fitness-functions) for the field meanings and the id algorithm. + ### SARIF [Static Analysis Results Interchange Format](https://sarifweb.azurewebsites.net/) — compatible with GitHub Code Scanning and Azure DevOps. Contains findings only (not all metrics). Ideal for integrating drift findings directly into your PR review workflow. +A `--baseline` scan keeps every result. Each result that matches the baseline gets a SARIF `suppressions` entry whose `properties.id` is the same id as `baseline.suppressed` in the JSON artifact. The run invocation records `baselineCompared` and `baselineSuppressedCount`, including when the count is zero. + ### Markdown A clean Markdown report suitable for PRs, wikis, or documentation. diff --git a/README.md b/README.md index 3346e40..180783a 100644 --- a/README.md +++ b/README.md @@ -557,6 +557,7 @@ vg scan --baseline .vibgrate/baseline.json --drift-budget 40 --drift-worsening 5 - `--drift-budget ` fails the build if drift exceeds your budget. - `--drift-worsening ` fails the build if drift worsens by more than X% vs baseline. +- `--baseline ` keeps every finding and records which ones were already in the baseline (`baseline.suppressed` in JSON, the same ids on SARIF suppressions). The text report includes the count. Copy-paste CI templates live in `examples/github-actions/`. Azure DevOps and GitLab CI snippets are in [DOCS.md](./DOCS.md#ci-integration). diff --git a/src/core-open/baseline-audit.ts b/src/core-open/baseline-audit.ts new file mode 100644 index 0000000..c9d0a0c --- /dev/null +++ b/src/core-open/baseline-audit.ts @@ -0,0 +1,181 @@ +// VENDORED from @vibgrate/core-open (packages/vibgrate-core-open) by +// scripts/vendor-core-open.mjs. Do not edit here — change the source package +// and re-run the vendor script. Apache-2.0. +import { createHash } from 'node:crypto'; +import * as path from 'node:path'; +import type { BaselineComparison, BaselineSuppressedFinding, Finding } from './types.js'; + +/** + * Rules that emit at most one finding per location. Their id ignores the + * message, so a version-number edit does not look like a new finding. + */ +const LOCATION_ONLY_RULES = new Set([ + 'vibgrate/runtime-eol', + 'vibgrate/runtime-lag', + 'vibgrate/dependency-rot', +]); + +const MAJOR_LAG_RULES = new Set([ + 'vibgrate/framework-major-lag', + 'vibgrate/dependency-major-lag', +]); + +/** "React is 2 major versions behind …" / "lodash is 3 major versions behind …" */ +const MAJOR_LAG_NAME = /^(.+?) is \d+ major versions behind/; + +/** + * Content-derived id for a drift finding. + * + * First 16 hex characters of SHA-256 over `[ruleId, location, subject]`. + * The subject keeps two findings at the same path distinct without copying + * the message (which changes as versions move, and which must not be repeated + * into the suppression record): + * + * - vulnerability findings: `advisoryId`, `ecosystem`, and `package` from `details` + * - framework / dependency major-lag: the name that precedes "is N major versions behind" + * - runtime EOL, runtime lag, dependency rot: nothing (rule + location is enough) + * - any other rule: the message, so distinct findings do not collapse + */ +export function findingId(finding: Finding): string { + const payload = JSON.stringify([finding.ruleId, finding.location, findingSubject(finding)]); + return createHash('sha256').update(payload).digest('hex').slice(0, 16); +} + +/** + * Current findings whose id also appears in the baseline. + * Sorted by ruleId, then location, then id. The input arrays are not modified, + * and the result never includes the finding message. + */ +export function suppressedBaselineFindings( + current: readonly Finding[], + baseline: readonly Finding[], +): BaselineSuppressedFinding[] { + const baselineIds = new Set(); + for (const finding of baseline) { + const normalized = asFinding(finding); + if (normalized) baselineIds.add(findingId(normalized)); + } + + const suppressed: BaselineSuppressedFinding[] = []; + const seen = new Set(); + for (const finding of current) { + const normalized = asFinding(finding); + if (!normalized) continue; + const id = findingId(normalized); + if (!baselineIds.has(id) || seen.has(id)) continue; + seen.add(id); + suppressed.push({ ruleId: normalized.ruleId, location: normalized.location, id }); + } + + suppressed.sort(compareSuppressed); + return suppressed; +} + +/** + * Reference to the baseline file that is safe to store in a scan artifact. + * Repo-relative, with `/` separators. A file outside the repo contributes + * only its basename, so a home-directory path never enters the document. + */ +export function baselineFileReference(rootDir: string, baselinePath: string): string { + const absRoot = path.resolve(rootDir); + const absBaseline = path.resolve(baselinePath); + const rel = path.relative(absRoot, absBaseline); + if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) { + return path.basename(absBaseline); + } + return rel.split(path.sep).join('/'); +} + +/** "1 finding suppressed" / "2 findings suppressed". */ +export function baselineSuppressionPhrase(count: number): string { + const n = Number.isFinite(count) ? count : 0; + return `${n} ${n === 1 ? 'finding' : 'findings'} suppressed`; +} + +/** + * Read a `baseline` block from a scan artifact. Returns undefined for the + * legacy string path and for anything that is not a comparison record, so + * older artifacts still format. + */ +export function readBaselineComparison(value: unknown): BaselineComparison | undefined { + if (!value || typeof value !== 'object') return undefined; + const raw = value as Record; + if (raw.compared !== true || !Array.isArray(raw.suppressed)) return undefined; + + const suppressed: BaselineSuppressedFinding[] = []; + for (const item of raw.suppressed) { + if (!item || typeof item !== 'object') continue; + const row = item as Record; + if (typeof row.ruleId !== 'string' || typeof row.location !== 'string' || typeof row.id !== 'string') continue; + suppressed.push({ ruleId: row.ruleId, location: row.location, id: row.id }); + } + + const suppressedCount = typeof raw.suppressedCount === 'number' && Number.isFinite(raw.suppressedCount) + ? raw.suppressedCount + : suppressed.length; + + return { + compared: true, + file: typeof raw.file === 'string' ? raw.file : '', + suppressedCount, + suppressed, + }; +} + +/** Phrase for text/markdown, or undefined when this artifact did not compare a baseline. */ +export function baselineSuppressionPhraseFrom(baseline: unknown): string | undefined { + const comparison = readBaselineComparison(baseline); + if (!comparison) return undefined; + return baselineSuppressionPhrase(comparison.suppressedCount); +} + +function findingSubject(finding: Finding): string { + const fromDetails = subjectFromDetails(finding); + if (fromDetails) return fromDetails; + + if (MAJOR_LAG_RULES.has(finding.ruleId)) { + const name = nameBeforeMajorLag(finding.message); + return name ? `name=${name}` : `message=${finding.message}`; + } + + if (LOCATION_ONLY_RULES.has(finding.ruleId)) return ''; + return finding.message ? `message=${finding.message}` : ''; +} + +function subjectFromDetails(finding: Finding): string { + const details = finding.details; + if (!details || typeof details.advisoryId !== 'string' || !details.advisoryId) return ''; + const ecosystem = typeof details.ecosystem === 'string' ? details.ecosystem : ''; + const pkg = typeof details.package === 'string' ? details.package : ''; + return `advisoryId=${details.advisoryId}\necosystem=${ecosystem}\npackage=${pkg}`; +} + +function nameBeforeMajorLag(message: string): string { + const match = MAJOR_LAG_NAME.exec(message); + return match?.[1]?.trim() ?? ''; +} + +function asFinding(value: unknown): Finding | undefined { + if (!value || typeof value !== 'object') return undefined; + const raw = value as Partial; + if (typeof raw.ruleId !== 'string' || typeof raw.location !== 'string') return undefined; + const level = raw.level === 'error' || raw.level === 'note' || raw.level === 'warning' ? raw.level : 'warning'; + const finding: Finding = { + ruleId: raw.ruleId, + level, + message: typeof raw.message === 'string' ? raw.message : '', + location: raw.location, + }; + if (raw.details && typeof raw.details === 'object') finding.details = raw.details; + return finding; +} + +function compareSuppressed(a: BaselineSuppressedFinding, b: BaselineSuppressedFinding): number { + return cmp(a.ruleId, b.ruleId) || cmp(a.location, b.location) || cmp(a.id, b.id); +} + +function cmp(a: string, b: string): number { + if (a < b) return -1; + if (a > b) return 1; + return 0; +} diff --git a/src/core-open/formatters/markdown.ts b/src/core-open/formatters/markdown.ts index be83d21..be01999 100644 --- a/src/core-open/formatters/markdown.ts +++ b/src/core-open/formatters/markdown.ts @@ -2,6 +2,7 @@ // scripts/vendor-core-open.mjs. Do not edit here — change the source package // and re-run the vendor script. Apache-2.0. import type { ScanArtifact } from '../types.js'; +import { baselineSuppressionPhraseFrom } from '../baseline-audit.js'; import { securityPacksLabel } from './text.js'; /** Rows shown before the infrastructure-findings table is cut with an "… N more" line. */ @@ -186,5 +187,11 @@ export function formatMarkdown(artifact: ScanArtifact): string { lines.push(''); } + const baselinePhrase = baselineSuppressionPhraseFrom(artifact.baseline); + if (baselinePhrase) { + lines.push(`**Baseline:** ${baselinePhrase}`); + lines.push(''); + } + return lines.join('\n'); } diff --git a/src/core-open/formatters/sarif.ts b/src/core-open/formatters/sarif.ts index 9825a8d..26213a0 100644 --- a/src/core-open/formatters/sarif.ts +++ b/src/core-open/formatters/sarif.ts @@ -2,20 +2,25 @@ // scripts/vendor-core-open.mjs. Do not edit here — change the source package // and re-run the vendor script. Apache-2.0. import type { ScanArtifact, Finding, SecurityFinding, SecuritySection, SecuritySeverity } from '../types.js'; +import { findingId, readBaselineComparison } from '../baseline-audit.js'; /** * Generate a SARIF 2.1.0 document from scan artifact. * - * The first run carries the drift findings and is byte-for-byte what it has - * always been. When the artifact carries security-pack findings - * (`extended.security`, from `vg scan --iac`), a second run is appended for - * them — one run per artifact, with every pack listed under - * `tool.extensions`, so a scan that ran no pack produces exactly the same - * bytes as before. + * The first run carries the drift findings. With no baseline comparison and + * no security-pack findings, that run is byte-for-byte what it has always + * been. A baseline comparison keeps every result and adds `suppressions` + * whose `properties.id` matches `baseline.suppressed` in the JSON artifact, + * plus an invocation property for the count (including zero). When the + * artifact carries security-pack findings (`extended.security`, from + * `vg scan --iac`), a second run is appended for them — one run per artifact, + * with every pack listed under `tool.extensions`. */ export function formatSarif(artifact: ScanArtifact): object { + const comparison = readBaselineComparison(artifact.baseline); + const suppressedIds = comparison ? new Set(comparison.suppressed.map((row) => row.id)) : null; const rules = buildRules(artifact.findings); - const results = artifact.findings.map((f) => toSarifResult(f)); + const results = artifact.findings.map((f) => toSarifResult(f, suppressedIds)); const runs: object[] = [ { @@ -32,6 +37,17 @@ export function formatSarif(artifact: ScanArtifact): object { { executionSuccessful: true, startTimeUtc: artifact.timestamp, + // Present only after a baseline comparison, so a scan that did not + // compare stays byte-for-byte what it was. A count of zero is still + // recorded: comparison with nothing matched is not a silent omission. + ...(comparison + ? { + properties: { + baselineCompared: true, + baselineSuppressedCount: comparison.suppressedCount, + }, + } + : {}), }, ], }, @@ -179,7 +195,9 @@ function buildRules(findings: Finding[]) { }); } -function toSarifResult(finding: Finding) { +function toSarifResult(finding: Finding, suppressedIds: ReadonlySet | null) { + const id = suppressedIds ? findingId(finding) : undefined; + const suppressed = id !== undefined && suppressedIds?.has(id) === true; return { ruleId: finding.ruleId, level: finding.level === 'error' ? 'error' : finding.level === 'warning' ? 'warning' : 'note', @@ -196,5 +214,19 @@ function toSarifResult(finding: Finding) { // Surface structured finding detail (e.g. advisory id, CVSS, fixed version) // to consumers like GitHub code scanning without bloating the message text. ...(finding.details && Object.keys(finding.details).length > 0 ? { properties: finding.details } : {}), + // The result stays in `results`. The suppression carries the same id as + // `baseline.suppressed` in the JSON artifact. + ...(suppressed && id + ? { + suppressions: [ + { + kind: 'external', + status: 'accepted', + justification: 'Matched a finding recorded in the drift baseline.', + properties: { id }, + }, + ], + } + : {}), }; } diff --git a/src/core-open/formatters/text.ts b/src/core-open/formatters/text.ts index 9f1ed28..0a7267a 100644 --- a/src/core-open/formatters/text.ts +++ b/src/core-open/formatters/text.ts @@ -3,6 +3,7 @@ // and re-run the vendor script. Apache-2.0. import chalk from 'chalk'; import type { ScanArtifact, BillingSummary, ExtendedScanResults, InventoryItem, ServiceDependencyItem, ArchitectureResult, SecurityFinding, SecuritySection } from '../types.js'; +import { baselineSuppressionPhraseFrom } from '../baseline-audit.js'; import { driftBar } from '../ui/bar.js'; import { titleBox, panelBox } from '../ui/box.js'; @@ -85,15 +86,21 @@ export function formatText(artifact: ScanArtifact, opts: FormatTextOptions = {}) lines.push(''); } - if (artifact.delta !== undefined) { - // Drift is "lower is better": a negative delta means drift fell (good), - // a positive delta means drift rose (bad). - const deltaStr = artifact.delta < 0 - ? chalk.green(`${artifact.delta}`) - : artifact.delta > 0 - ? chalk.red(`+${artifact.delta}`) - : chalk.dim('0'); - lines.push(chalk.bold(' Drift Delta: ') + deltaStr + ' (vs baseline)'); + const baselinePhrase = baselineSuppressionPhraseFrom(artifact.baseline); + if (artifact.delta !== undefined || baselinePhrase) { + if (artifact.delta !== undefined) { + // Drift is "lower is better": a negative delta means drift fell (good), + // a positive delta means drift rose (bad). + const deltaStr = artifact.delta < 0 + ? chalk.green(`${artifact.delta}`) + : artifact.delta > 0 + ? chalk.red(`+${artifact.delta}`) + : chalk.dim('0'); + lines.push(chalk.bold(' Drift Delta: ') + deltaStr + ' (vs baseline)'); + } + if (baselinePhrase) { + lines.push(chalk.bold(' Baseline: ') + baselinePhrase); + } lines.push(''); } diff --git a/src/core-open/index.ts b/src/core-open/index.ts index 2142847..e2a9c72 100644 --- a/src/core-open/index.ts +++ b/src/core-open/index.ts @@ -67,6 +67,16 @@ export { formatText } from './formatters/text.js'; export { formatSarif } from './formatters/sarif.js'; export { formatMarkdown } from './formatters/markdown.js'; +// ── Baseline comparison (audit record; findings are not dropped) ──────────── +export { + baselineFileReference, + baselineSuppressionPhrase, + baselineSuppressionPhraseFrom, + findingId, + readBaselineComparison, + suppressedBaselineFindings, +} from './baseline-audit.js'; + // ── Scanners (fact collection) ─────────────────────────────────────────────── export { scanNodeProjects } from './scanners/node-scanner.js'; export { scanDotnetProjects } from './scanners/dotnet-scanner.js'; diff --git a/src/core-open/run-core-scan.ts b/src/core-open/run-core-scan.ts index 5375ff9..6e0a5b6 100644 --- a/src/core-open/run-core-scan.ts +++ b/src/core-open/run-core-scan.ts @@ -33,6 +33,7 @@ import { Semaphore } from './utils/semaphore.js'; import { computeDriftScore, generateFindings, computeProjectId, computeSolutionId } from './scoring/drift-score.js'; import { formatText } from './formatters/text.js'; import { formatSarif } from './formatters/sarif.js'; +import { baselineFileReference, suppressedBaselineFindings } from './baseline-audit.js'; import { formatMarkdown } from './formatters/markdown.js'; import { loadConfig, appendExcludePatterns } from './config.js'; import { pathExists, readJsonFile, writeJsonFile, writeTextFile, ensureDir, FileCache, quickTreeCount } from './utils/fs.js'; @@ -818,12 +819,23 @@ export async function runCoreScan( if (await pathExists(baselinePath)) { try { const baseline = await readJsonFile(baselinePath); - // Only a repo-relative reference may enter the artifact: the absolute - // path leaks the local username/home layout to the ingest server. A - // baseline outside the repo degrades to its basename for the same reason. - const relBaseline = path.relative(rootDir, baselinePath); - artifact.baseline = !relBaseline || relBaseline.startsWith('..') ? path.basename(baselinePath) : relBaseline; - artifact.delta = artifact.drift.score - baseline.drift.score; + const score = baseline?.drift?.score; + if (typeof score !== 'number' || !Number.isFinite(score) || !Array.isArray(baseline.findings)) { + throw new Error('baseline file is not a scan artifact'); + } + // Findings stay on `artifact.findings`. This block is the auditable + // record of which of them were already in the baseline. Only a + // repo-relative reference (or a basename, when the file sits outside + // the repo) may enter the artifact: an absolute path leaks the local + // username and home layout. + const suppressed = suppressedBaselineFindings(artifact.findings, baseline.findings); + artifact.baseline = { + compared: true, + file: baselineFileReference(rootDir, baselinePath), + suppressedCount: suppressed.length, + suppressed, + }; + artifact.delta = artifact.drift.score - score; } catch { console.error(chalk.yellow(`Warning: Could not read baseline file: ${baselinePath}`)); } diff --git a/src/core-open/types.ts b/src/core-open/types.ts index 07f0335..3c391d8 100644 --- a/src/core-open/types.ts +++ b/src/core-open/types.ts @@ -626,6 +626,34 @@ export interface Finding { details?: Record; } +/** + * One current finding that was already present in the compared baseline. + * `id` is the content-derived finding id (see `findingId`). It is the + * identifier copied into SARIF suppressions. The finding message is absent. + */ +export interface BaselineSuppressedFinding { + ruleId: string; + location: string; + id: string; +} + +/** + * Auditable `--baseline` comparison. Matching findings stay in + * {@link ScanArtifact.findings}; this block is the record that they were + * recognized as already baselined. `suppressed` is sorted by `ruleId`, then + * `location`, then `id`. + */ +export interface BaselineComparison { + compared: true; + /** + * Repo-relative path of the baseline file (`/` separators), or its basename + * when the file lives outside the repo. Absolute paths never enter the artifact. + */ + file: string; + suppressedCount: number; + suppressed: BaselineSuppressedFinding[]; +} + // ── Version control info ── export type VcsType = 'git' | 'unknown'; @@ -709,7 +737,12 @@ export interface ScanArtifact { solutions?: SolutionScan[]; drift: DriftScore; findings: Finding[]; - baseline?: string; + /** + * Present when `--baseline` read a scan artifact. Records which current + * findings were already in that baseline. Findings are not removed from + * {@link ScanArtifact.findings}. + */ + baseline?: BaselineComparison; delta?: number; extended?: ExtendedScanResults; /** Scan wall-clock duration in milliseconds */ diff --git a/src/reporting/baseline-audit.test.ts b/src/reporting/baseline-audit.test.ts new file mode 100644 index 0000000..02a6c0b --- /dev/null +++ b/src/reporting/baseline-audit.test.ts @@ -0,0 +1,311 @@ +import { describe, it, expect, afterEach, vi } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + baselineFileReference, + findingId, + formatSarif, + readBaselineComparison, + runCoreScan, + suppressedBaselineFindings, + type Finding, + type ScanOptions, +} from '../core-open/index.js'; + +function finding(overrides: Partial & Pick): Finding { + return { + level: 'warning', + message: 'placeholder', + ...overrides, + }; +} + +describe('baseline finding ids', () => { + it('locks the id for a location-only rule', () => { + expect(findingId(finding({ + ruleId: 'vibgrate/dependency-rot', + message: '40% of dependencies are 2+ major versions behind in app.', + location: 'package.json', + }))).toBe('af173ea1c1efa4fd'); + }); + + it('keeps the same id when a location-only message changes', () => { + const older = finding({ ruleId: 'vibgrate/runtime-lag', location: 'app', message: 'is 2 behind' }); + const newer = finding({ ruleId: 'vibgrate/runtime-lag', location: 'app', message: 'is 3 behind' }); + expect(findingId(older)).toBe(findingId(newer)); + expect(findingId(older)).toBe('a0b24c8fa1d80216'); + }); + + it('distinguishes major-lag findings by package name, not by how far behind they are', () => { + const lodash = finding({ + ruleId: 'vibgrate/dependency-major-lag', + level: 'error', + location: 'app', + message: 'lodash is 3 major versions behind (spec: ^1.0.0, latest: 4.0.0).', + }); + const lodashLater = finding({ + ...lodash, + message: 'lodash is 4 major versions behind (spec: ^1.0.0, latest: 5.0.0).', + }); + const react = finding({ + ...lodash, + message: 'react is 3 major versions behind (spec: ^17.0.0, latest: 19.0.0).', + }); + expect(findingId(lodash)).toBe(findingId(lodashLater)); + expect(findingId(lodash)).toBe('07cc9e594585d05c'); + expect(findingId(lodash)).not.toBe(findingId(react)); + }); + + it('identifies a vulnerability by advisory, not by the message', () => { + const first = finding({ + ruleId: 'vibgrate/vulnerability', + level: 'error', + location: 'lodash', + message: 'lodash@1.0.0: GHSA-example (high) — introduced by a maintainer', + details: { advisoryId: 'GHSA-example', ecosystem: 'npm', package: 'lodash' }, + }); + const rewritten = finding({ ...first, message: 'wording changed; still the same advisory' }); + const other = finding({ + ...first, + message: first.message, + details: { advisoryId: 'GHSA-other', ecosystem: 'npm', package: 'lodash' }, + }); + expect(findingId(first)).toBe(findingId(rewritten)); + expect(findingId(first)).toBe('db61f217df4c7b5e'); + expect(findingId(first)).not.toBe(findingId(other)); + }); +}); + +describe('suppressedBaselineFindings', () => { + const rot = finding({ ruleId: 'vibgrate/dependency-rot', location: 'package.json', message: 'rot' }); + const lagA = finding({ ruleId: 'vibgrate/runtime-lag', location: 'a', message: 'lag a' }); + const lagB = finding({ ruleId: 'vibgrate/runtime-lag', location: 'b', message: 'lag b' }); + + it('sorts by ruleId, then location, then id, independent of input order', () => { + const forward = suppressedBaselineFindings([lagB, rot, lagA], [lagA, lagB, rot]); + const backward = suppressedBaselineFindings([rot, lagA, lagB], [lagB, rot, lagA]); + expect(forward).toEqual(backward); + expect(forward.map((row) => `${row.ruleId} ${row.location}`)).toEqual([ + 'vibgrate/dependency-rot package.json', + 'vibgrate/runtime-lag a', + 'vibgrate/runtime-lag b', + ]); + expect(forward.map((row) => row.id)).toEqual([ + 'af173ea1c1efa4fd', + findingId(lagA), + findingId(lagB), + ]); + }); + + it('lists only findings that are in the baseline and does not copy the message', () => { + const current = finding({ + ruleId: 'vibgrate/dependency-rot', + location: 'package.json', + message: 'placeholder-token-do-not-copy', + }); + const fresh = finding({ ruleId: 'vibgrate/runtime-eol', location: 'package.json', message: 'new' }); + const rows = suppressedBaselineFindings([current, fresh], [current]); + expect(rows).toEqual([{ + ruleId: 'vibgrate/dependency-rot', + location: 'package.json', + id: 'af173ea1c1efa4fd', + }]); + expect(JSON.stringify(rows)).not.toContain('placeholder-token'); + expect(Object.keys(rows[0]).sort()).toEqual(['id', 'location', 'ruleId']); + }); + + it('does not mutate either findings array', () => { + const current = [rot, lagA]; + const baseline = [rot]; + const beforeCurrent = JSON.stringify(current); + const beforeBaseline = JSON.stringify(baseline); + suppressedBaselineFindings(current, baseline); + expect(JSON.stringify(current)).toBe(beforeCurrent); + expect(JSON.stringify(baseline)).toBe(beforeBaseline); + }); + + it('collapses duplicate current findings to one record', () => { + expect(suppressedBaselineFindings([rot, rot], [rot])).toHaveLength(1); + }); + + it('returns an empty list when nothing matches', () => { + expect(suppressedBaselineFindings([lagA], [rot])).toEqual([]); + }); +}); + +describe('baselineFileReference', () => { + it('stores a repo-relative path and only a basename for a file outside the repo', () => { + expect(baselineFileReference('/repo', '/repo/.vibgrate/baseline.json')).toBe('.vibgrate/baseline.json'); + expect(baselineFileReference('/repo', '/tmp/elsewhere/baseline.json')).toBe('baseline.json'); + }); +}); + +describe('readBaselineComparison', () => { + it('ignores the legacy string path', () => { + expect(readBaselineComparison('.vibgrate/baseline.json')).toBeUndefined(); + expect(readBaselineComparison(undefined)).toBeUndefined(); + }); +}); + +describe('vg scan --baseline audit record', () => { + const dirs: string[] = []; + let logSpy: ReturnType; + + afterEach(() => { + logSpy?.mockRestore(); + vi.unstubAllEnvs(); + for (const dir of dirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); + }); + + function tempDir(prefix: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + dirs.push(dir); + return dir; + } + + function scanOpts(extra: Partial = {}): ScanOptions { + return { + format: 'text', + concurrency: 2, + offline: true, + noLocalArtifacts: true, + quiet: true, + vibgrateVersion: 'test', + ...extra, + }; + } + + it('records matched findings without removing them, and echoes the ids in SARIF', async () => { + vi.stubEnv('VIBGRATE_DSN', ''); + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const dir = tempDir('vg-baseline-audit-'); + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ + name: 'baseline-audit-fixture', + version: '0.0.0', + private: true, + engines: { node: '18.0.0' }, + })); + + const first = await runCoreScan(dir, scanOpts()); + expect(first.findings.length).toBeGreaterThan(0); + expect(first.findings.some((f) => f.ruleId === 'vibgrate/runtime-eol')).toBe(true); + expect(first.baseline).toBeUndefined(); + + const baselinePath = path.join(dir, 'baseline.json'); + fs.writeFileSync(baselinePath, JSON.stringify(first)); + + logSpy.mockClear(); + const second = await runCoreScan(dir, scanOpts({ baseline: baselinePath })); + + expect(second.findings.map((f) => `${f.ruleId}\0${f.location}\0${f.message}`)).toEqual( + first.findings.map((f) => `${f.ruleId}\0${f.location}\0${f.message}`), + ); + expect(second.baseline).toMatchObject({ + compared: true, + file: 'baseline.json', + suppressedCount: second.baseline?.suppressed.length, + }); + expect(second.baseline?.suppressedCount).toBeGreaterThan(0); + expect(second.baseline?.suppressed).toEqual(suppressedBaselineFindings(second.findings, first.findings)); + expect(second.delta).toBe(0); + + const printed = logSpy.mock.calls.map((call: unknown[]) => String(call[0] ?? '')).join('\n'); + expect(printed).toContain(`${second.baseline?.suppressedCount} finding`); + expect(printed).toContain('suppressed'); + for (const findingRow of second.findings) expect(printed).toContain(findingRow.ruleId); + + const sarif = formatSarif(second) as { + runs: Array<{ + results: Array<{ ruleId: string; suppressions?: Array<{ properties?: { id?: string } }> }>; + invocations: Array<{ properties?: { baselineCompared?: boolean; baselineSuppressedCount?: number } }>; + }>; + }; + const results = sarif.runs[0].results; + expect(results).toHaveLength(second.findings.length); + const suppressedIds = results.flatMap((result) => result.suppressions?.map((s) => s.properties?.id) ?? []); + expect(suppressedIds.sort()).toEqual(second.baseline?.suppressed.map((row) => row.id).sort()); + expect(sarif.runs[0].invocations[0].properties).toEqual({ + baselineCompared: true, + baselineSuppressedCount: second.baseline?.suppressedCount, + }); + expect(JSON.stringify(second.baseline)).not.toContain(dir); + }); + + it('records a comparison with zero matches and still keeps every finding', async () => { + vi.stubEnv('VIBGRATE_DSN', ''); + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const dir = tempDir('vg-baseline-empty-'); + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ + name: 'baseline-audit-fixture', + version: '0.0.0', + private: true, + engines: { node: '18.0.0' }, + })); + + const first = await runCoreScan(dir, scanOpts()); + const baselinePath = path.join(dir, 'baseline.json'); + fs.writeFileSync(baselinePath, JSON.stringify({ ...first, findings: [] })); + + logSpy.mockClear(); + const second = await runCoreScan(dir, scanOpts({ baseline: baselinePath })); + expect(second.findings.length).toBe(first.findings.length); + expect(second.baseline).toMatchObject({ + compared: true, + file: 'baseline.json', + suppressedCount: 0, + suppressed: [], + }); + const printed = logSpy.mock.calls.map((call: unknown[]) => String(call[0] ?? '')).join('\n'); + expect(printed).toContain('0 findings suppressed'); + + const sarif = formatSarif(second) as { + runs: Array<{ + results: Array<{ suppressions?: unknown }>; + invocations: Array<{ properties?: { baselineSuppressedCount?: number } }>; + }>; + }; + expect(sarif.runs[0].results.every((result) => result.suppressions === undefined)).toBe(true); + expect(sarif.runs[0].invocations[0].properties?.baselineSuppressedCount).toBe(0); + }); + + it('stores only the basename when the baseline file is outside the repo', async () => { + vi.stubEnv('VIBGRATE_DSN', ''); + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const dir = tempDir('vg-baseline-root-'); + const outside = tempDir('vg-baseline-outside-'); + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ + name: 'baseline-audit-fixture', + version: '0.0.0', + private: true, + engines: { node: '18.0.0' }, + })); + const first = await runCoreScan(dir, scanOpts()); + const outsideFile = path.join(outside, 'baseline.json'); + fs.writeFileSync(outsideFile, JSON.stringify(first)); + + const second = await runCoreScan(dir, scanOpts({ baseline: outsideFile })); + expect(second.baseline?.file).toBe('baseline.json'); + expect(JSON.stringify(second.baseline)).not.toContain(outside); + }); + + it('does not pretend a comparison happened when the baseline cannot be read', async () => { + vi.stubEnv('VIBGRATE_DSN', ''); + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const dir = tempDir('vg-baseline-bad-'); + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ + name: 'baseline-audit-fixture', + version: '0.0.0', + private: true, + })); + const baselinePath = path.join(dir, 'baseline.json'); + fs.writeFileSync(baselinePath, '{ not json'); + + const artifact = await runCoreScan(dir, scanOpts({ baseline: baselinePath })); + expect(artifact.baseline).toBeUndefined(); + expect(artifact.delta).toBeUndefined(); + expect(errSpy.mock.calls.flat().join('\n')).toContain('Could not read baseline file'); + errSpy.mockRestore(); + }); +}); diff --git a/src/reporting/commands/scan.ts b/src/reporting/commands/scan.ts index 76a4e3b..d141a60 100644 --- a/src/reporting/commands/scan.ts +++ b/src/reporting/commands/scan.ts @@ -376,7 +376,7 @@ export const scanCommand = new Command('scan') '--fail-on ', 'Fail on warn or error. architecture-finding (hard boundary violations) or architecture-warning (violations and warnings) gate on the architecture module\'s boundary findings, judged under the policy pack in force: .vibgrate/architecture.toml (policy = "hexagonal-v1" | "layered-v1" | "vertical-v1", plus any [[overlay]] rules), VIBGRATE_ARCHITECTURE_POLICY, or vg build --policy; default hexagonal-v1. The pack is named in the output. See docs/architecture-policies.md. iac-finding[=] fails on infrastructure findings from the iac-cis-v1 pack at or above (critical|high|medium|low|info; default high) and needs --iac (or --full) plus the code map — it exits 2 when the Architecture module is missing rather than passing an unevaluated tree; security-finding[=] is the umbrella across every security pack that ran. Comma-separated: at most one of warn/error/architecture-* plus any security gates, e.g. --fail-on error,iac-finding=medium', ) - .option('--baseline ', 'Compare against baseline') + .option('--baseline ', 'Compare against a baseline and record which findings it already contains') .option('--changed-only', 'Only scan changed files') .option( '-e, --exclude ', diff --git a/src/reporting/formatters/formatters.test.ts b/src/reporting/formatters/formatters.test.ts index 9019427..2d8cd73 100644 --- a/src/reporting/formatters/formatters.test.ts +++ b/src/reporting/formatters/formatters.test.ts @@ -3,6 +3,7 @@ import { describe, it, expect } from 'vitest'; // reporting-side copy was a stale duplicate and is gone. These tests keep // exercising the live writer. import { formatSarif as formatSarifCore } from '../../core-open/formatters/sarif.js'; +import { findingId } from '../../core-open/baseline-audit.js'; import { formatMarkdown } from '../formatters/markdown.js'; import { formatText } from '../formatters/text.js'; import type { ScanArtifact } from '../types.js'; @@ -144,6 +145,45 @@ describe('formatSarif', () => { expect(sarif.runs[0].invocations[0].startTimeUtc).toBe('2026-02-16T00:00:00.000Z'); expect(sarif.runs[0].invocations[0].executionSuccessful).toBe(true); + expect(sarif.runs[0].invocations[0].properties).toBeUndefined(); + expect(sarif.runs[0].results[0].suppressions).toBeUndefined(); + }); + + it('puts baseline ids on SARIF suppressions and keeps every result', () => { + const artifact = makeArtifact(); + const rot = artifact.findings[1]; + const id = findingId(rot); + artifact.baseline = { + compared: true, + file: '.vibgrate/baseline.json', + suppressedCount: 1, + suppressed: [{ ruleId: rot.ruleId, location: rot.location, id }], + }; + const sarif = formatSarif(artifact) as any; + const results = sarif.runs[0].results; + + expect(results).toHaveLength(2); + expect(results[0].suppressions).toBeUndefined(); + expect(results[1].suppressions).toEqual([ + { + kind: 'external', + status: 'accepted', + justification: 'Matched a finding recorded in the drift baseline.', + properties: { id }, + }, + ]); + expect(sarif.runs[0].invocations[0].properties).toEqual({ + baselineCompared: true, + baselineSuppressedCount: 1, + }); + }); + + it('does not treat a legacy baseline path string as suppressions', () => { + const artifact = makeArtifact(); + (artifact as { baseline?: unknown }).baseline = '.vibgrate/baseline.json'; + const sarif = formatSarif(artifact) as any; + expect(sarif.runs[0].results[0].suppressions).toBeUndefined(); + expect(sarif.runs[0].invocations[0].properties).toBeUndefined(); }); }); @@ -212,6 +252,21 @@ describe('formatMarkdown', () => { expect(md).toContain('📉'); }); + it('includes the baselined finding count without dropping findings', () => { + const md = formatMarkdown(makeArtifact({ + delta: 0, + baseline: { + compared: true, + file: '.vibgrate/baseline.json', + suppressedCount: 2, + suppressed: [], + }, + })); + expect(md).toContain('**Baseline:** 2 findings suppressed'); + expect(md).toContain('vibgrate/runtime-lag'); + expect(md).toContain('vibgrate/dependency-rot'); + }); + it('handles empty findings', () => { const md = formatMarkdown(makeArtifact({ findings: [] })); expect(md).not.toContain('## Findings'); @@ -368,6 +423,27 @@ describe('formatText', () => { expect(text).toContain('vs baseline'); }); + it('includes the baselined finding count', () => { + const text = formatText(makeArtifact({ + delta: 1, + baseline: { + compared: true, + file: 'baseline.json', + suppressedCount: 1, + suppressed: [], + }, + })); + expect(text).toContain('Drift Delta'); + expect(text).toContain('1 finding suppressed'); + expect(text).toContain('vibgrate/dependency-rot'); + }); + + it('ignores a legacy baseline path string', () => { + const artifact = makeArtifact({ delta: 1 }); + (artifact as { baseline?: unknown }).baseline = 'baseline.json'; + expect(formatText(artifact)).not.toContain('suppressed'); + }); + it('handles empty projects', () => { const text = formatText(makeArtifact({ projects: [], findings: [] })); expect(text).toContain('Vibgrate Drift Report'); diff --git a/src/reporting/formatters/markdown.ts b/src/reporting/formatters/markdown.ts index 5778589..61cb283 100644 --- a/src/reporting/formatters/markdown.ts +++ b/src/reporting/formatters/markdown.ts @@ -1,4 +1,5 @@ import type { ScanArtifact } from '../types.js'; +import { baselineSuppressionPhraseFrom } from '../../core-open/baseline-audit.js'; /** Generate a Markdown report from scan artifact */ export function formatMarkdown(artifact: ScanArtifact): string { @@ -130,5 +131,11 @@ export function formatMarkdown(artifact: ScanArtifact): string { lines.push(''); } + const baselinePhrase = baselineSuppressionPhraseFrom(artifact.baseline); + if (baselinePhrase) { + lines.push(`**Baseline:** ${baselinePhrase}`); + lines.push(''); + } + return lines.join('\n'); } diff --git a/src/reporting/formatters/text.ts b/src/reporting/formatters/text.ts index 5275e34..26020e1 100644 --- a/src/reporting/formatters/text.ts +++ b/src/reporting/formatters/text.ts @@ -1,5 +1,6 @@ import chalk from 'chalk'; import type { ScanArtifact, ExtendedScanResults, InventoryItem, ServiceDependencyItem, ArchitectureResult } from '../types.js'; +import { baselineSuppressionPhraseFrom } from '../../core-open/baseline-audit.js'; import { VERSION } from '../version.js'; import { driftBar } from '../../core-open/ui/bar.js'; import { titleBox } from '../../core-open/ui/box.js'; @@ -56,15 +57,21 @@ export function formatText(artifact: ScanArtifact): string { lines.push(''); } - if (artifact.delta !== undefined) { - // Lower is better: a positive delta means drift increased (worse → red), - // a negative delta means drift decreased (improved → green). - const deltaStr = artifact.delta > 0 - ? chalk.red(`+${artifact.delta}`) - : artifact.delta < 0 - ? chalk.green(`${artifact.delta}`) - : chalk.dim('0'); - lines.push(chalk.bold(' Drift Delta: ') + deltaStr + ' (vs baseline)'); + const baselinePhrase = baselineSuppressionPhraseFrom(artifact.baseline); + if (artifact.delta !== undefined || baselinePhrase) { + if (artifact.delta !== undefined) { + // Lower is better: a positive delta means drift increased (worse → red), + // a negative delta means drift decreased (improved → green). + const deltaStr = artifact.delta > 0 + ? chalk.red(`+${artifact.delta}`) + : artifact.delta < 0 + ? chalk.green(`${artifact.delta}`) + : chalk.dim('0'); + lines.push(chalk.bold(' Drift Delta: ') + deltaStr + ' (vs baseline)'); + } + if (baselinePhrase) { + lines.push(chalk.bold(' Baseline: ') + baselinePhrase); + } lines.push(''); } diff --git a/src/reporting/types.ts b/src/reporting/types.ts index 0ac11bf..e547ba2 100644 --- a/src/reporting/types.ts +++ b/src/reporting/types.ts @@ -204,6 +204,25 @@ export interface Finding { details?: Record; } +/** One current finding that was already present in the compared baseline. */ +export interface BaselineSuppressedFinding { + ruleId: string; + location: string; + id: string; +} + +/** + * Auditable `--baseline` comparison. Matching findings stay in `findings`. + * `suppressed` is sorted by `ruleId`, then `location`, then `id`. + */ +export interface BaselineComparison { + compared: true; + /** Repo-relative baseline path (`/` separators), or the basename when outside the repo. */ + file: string; + suppressedCount: number; + suppressed: BaselineSuppressedFinding[]; +} + // ── Version control info ── export type VcsType = 'git' | 'unknown'; @@ -245,7 +264,11 @@ export interface ScanArtifact { solutions?: SolutionScan[]; drift: DriftScore; findings: Finding[]; - baseline?: string; + /** + * Present when `--baseline` read a scan artifact. Records which current + * findings were already in that baseline. Findings stay in `findings`. + */ + baseline?: BaselineComparison; delta?: number; extended?: ExtendedScanResults; /** Scan wall-clock duration in milliseconds */