diff --git a/DOCS.md b/DOCS.md index 6ad775e..e99e138 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1081,7 +1081,7 @@ vg scan [path] [--vulns] [--full] [--format text|json|sarif|md] [--out ] [ | `--format` | `text` | Output format: `text`, `json`, `sarif`, or `md` | | `--out ` | — | Write output to a file | | `--fail-on ` | — | Exit with code 2 if findings at this level exist. `warn` / `error` gate on drift findings. `architecture-finding` (hard boundary violations) and `architecture-warning` (violations and warnings) gate on the architecture module's boundary findings, judged under the policy pack in force — `hexagonal-v1` unless `.vibgrate/architecture.toml`, `VIBGRATE_ARCHITECTURE_POLICY` or `vg build --policy` says `layered-v1`. The output names the pack whether the gate passes or fails; each failing row is `file:line symbol violation: … (rule)`. Pick the pack before turning this on: see [Architecture policy packs](./docs/architecture-policies.md) | -| `--baseline ` | — | Compare against a previous baseline | +| `--baseline ` | — | Compare against a previous baseline. Matched findings stay in the report and are listed on `baseline.suppressed` (and as SARIF suppressions); the text and Markdown reports include the count | | `--changed-only` | — | Only scan changed files | | `--concurrency ` | `8` | Max concurrent npm registry calls | | `--drift-budget ` | — | Fitness gate: fail if drift score is above this budget | @@ -2808,6 +2808,21 @@ Recommended workflow: This makes drift a formal quality gate (fitness function), not just reporting. +`vg scan --baseline` does not drop findings that were already in the baseline. The scan artifact records them so a suppression is visible in the same document: + +```json +"baseline": { + "compared": true, + "file": ".vibgrate/baseline.json", + "suppressedCount": 2, + "suppressed": [ + { "ruleId": "vibgrate/dependency-rot", "location": "package.json", "id": "af173ea1c1efa4fd" } + ] +} +``` + +`suppressed` is sorted by `ruleId`, then `location`, then `id`. `file` is the repo-relative path (`/` separators), or the file's basename when the baseline sits outside the repo — an absolute path is never stored. `id` is 16 hex characters of SHA-256 over the rule, the location, and a stable subject: advisory id, ecosystem, and package for a vulnerability; the package or framework name for a major-lag finding; otherwise the rule and location. Text and Markdown reports include `suppressedCount`. SARIF keeps the same results and sets `suppressions[].properties.id` to that id. + ## DriftScore ### How the Score Is Calculated @@ -2854,10 +2869,14 @@ The default output. A coloured, human-readable report showing: The full scan artifact in JSON format. Contains all raw data, scores, findings, and VCS metadata. Stable schema (`schemaVersion: "1.0"`). This is the same artifact saved to `.vibgrate/scan_result.json`. +When the scan was run with `--baseline`, the artifact also contains a `baseline` object: `compared`, `file`, `suppressedCount`, and `suppressed` (`ruleId`, `location`, `id`). Those entries are the findings that were already in the baseline. The same findings remain in `findings`. See [Drift Baselines & Fitness Functions](#drift-baselines--fitness-functions) for the field meanings and the id algorithm. + ### SARIF [Static Analysis Results Interchange Format](https://sarifweb.azurewebsites.net/) — compatible with GitHub Code Scanning and Azure DevOps. Contains findings only (not all metrics). Ideal for integrating drift findings directly into your PR review workflow. +A `--baseline` scan keeps every result. Each result that matches the baseline gets a SARIF `suppressions` entry whose `properties.id` is the same id as `baseline.suppressed` in the JSON artifact. The run invocation records `baselineCompared` and `baselineSuppressedCount`, including when the count is zero. + ### Markdown A clean Markdown report suitable for PRs, wikis, or documentation. diff --git a/README.md b/README.md index 3346e40..180783a 100644 --- a/README.md +++ b/README.md @@ -557,6 +557,7 @@ vg scan --baseline .vibgrate/baseline.json --drift-budget 40 --drift-worsening 5 - `--drift-budget ` fails the build if drift exceeds your budget. - `--drift-worsening ` fails the build if drift worsens by more than X% vs baseline. +- `--baseline ` keeps every finding and records which ones were already in the baseline (`baseline.suppressed` in JSON, the same ids on SARIF suppressions). The text report includes the count. Copy-paste CI templates live in `examples/github-actions/`. Azure DevOps and GitLab CI snippets are in [DOCS.md](./DOCS.md#ci-integration). diff --git a/src/core-open/baseline-audit.ts b/src/core-open/baseline-audit.ts new file mode 100644 index 0000000..c9d0a0c --- /dev/null +++ b/src/core-open/baseline-audit.ts @@ -0,0 +1,181 @@ +// VENDORED from @vibgrate/core-open (packages/vibgrate-core-open) by +// scripts/vendor-core-open.mjs. Do not edit here — change the source package +// and re-run the vendor script. Apache-2.0. +import { createHash } from 'node:crypto'; +import * as path from 'node:path'; +import type { BaselineComparison, BaselineSuppressedFinding, Finding } from './types.js'; + +/** + * Rules that emit at most one finding per location. Their id ignores the + * message, so a version-number edit does not look like a new finding. + */ +const LOCATION_ONLY_RULES = new Set([ + 'vibgrate/runtime-eol', + 'vibgrate/runtime-lag', + 'vibgrate/dependency-rot', +]); + +const MAJOR_LAG_RULES = new Set([ + 'vibgrate/framework-major-lag', + 'vibgrate/dependency-major-lag', +]); + +/** "React is 2 major versions behind …" / "lodash is 3 major versions behind …" */ +const MAJOR_LAG_NAME = /^(.+?) is \d+ major versions behind/; + +/** + * Content-derived id for a drift finding. + * + * First 16 hex characters of SHA-256 over `[ruleId, location, subject]`. + * The subject keeps two findings at the same path distinct without copying + * the message (which changes as versions move, and which must not be repeated + * into the suppression record): + * + * - vulnerability findings: `advisoryId`, `ecosystem`, and `package` from `details` + * - framework / dependency major-lag: the name that precedes "is N major versions behind" + * - runtime EOL, runtime lag, dependency rot: nothing (rule + location is enough) + * - any other rule: the message, so distinct findings do not collapse + */ +export function findingId(finding: Finding): string { + const payload = JSON.stringify([finding.ruleId, finding.location, findingSubject(finding)]); + return createHash('sha256').update(payload).digest('hex').slice(0, 16); +} + +/** + * Current findings whose id also appears in the baseline. + * Sorted by ruleId, then location, then id. The input arrays are not modified, + * and the result never includes the finding message. + */ +export function suppressedBaselineFindings( + current: readonly Finding[], + baseline: readonly Finding[], +): BaselineSuppressedFinding[] { + const baselineIds = new Set(); + for (const finding of baseline) { + const normalized = asFinding(finding); + if (normalized) baselineIds.add(findingId(normalized)); + } + + const suppressed: BaselineSuppressedFinding[] = []; + const seen = new Set(); + for (const finding of current) { + const normalized = asFinding(finding); + if (!normalized) continue; + const id = findingId(normalized); + if (!baselineIds.has(id) || seen.has(id)) continue; + seen.add(id); + suppressed.push({ ruleId: normalized.ruleId, location: normalized.location, id }); + } + + suppressed.sort(compareSuppressed); + return suppressed; +} + +/** + * Reference to the baseline file that is safe to store in a scan artifact. + * Repo-relative, with `/` separators. A file outside the repo contributes + * only its basename, so a home-directory path never enters the document. + */ +export function baselineFileReference(rootDir: string, baselinePath: string): string { + const absRoot = path.resolve(rootDir); + const absBaseline = path.resolve(baselinePath); + const rel = path.relative(absRoot, absBaseline); + if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) { + return path.basename(absBaseline); + } + return rel.split(path.sep).join('/'); +} + +/** "1 finding suppressed" / "2 findings suppressed". */ +export function baselineSuppressionPhrase(count: number): string { + const n = Number.isFinite(count) ? count : 0; + return `${n} ${n === 1 ? 'finding' : 'findings'} suppressed`; +} + +/** + * Read a `baseline` block from a scan artifact. Returns undefined for the + * legacy string path and for anything that is not a comparison record, so + * older artifacts still format. + */ +export function readBaselineComparison(value: unknown): BaselineComparison | undefined { + if (!value || typeof value !== 'object') return undefined; + const raw = value as Record; + if (raw.compared !== true || !Array.isArray(raw.suppressed)) return undefined; + + const suppressed: BaselineSuppressedFinding[] = []; + for (const item of raw.suppressed) { + if (!item || typeof item !== 'object') continue; + const row = item as Record; + if (typeof row.ruleId !== 'string' || typeof row.location !== 'string' || typeof row.id !== 'string') continue; + suppressed.push({ ruleId: row.ruleId, location: row.location, id: row.id }); + } + + const suppressedCount = typeof raw.suppressedCount === 'number' && Number.isFinite(raw.suppressedCount) + ? raw.suppressedCount + : suppressed.length; + + return { + compared: true, + file: typeof raw.file === 'string' ? raw.file : '', + suppressedCount, + suppressed, + }; +} + +/** Phrase for text/markdown, or undefined when this artifact did not compare a baseline. */ +export function baselineSuppressionPhraseFrom(baseline: unknown): string | undefined { + const comparison = readBaselineComparison(baseline); + if (!comparison) return undefined; + return baselineSuppressionPhrase(comparison.suppressedCount); +} + +function findingSubject(finding: Finding): string { + const fromDetails = subjectFromDetails(finding); + if (fromDetails) return fromDetails; + + if (MAJOR_LAG_RULES.has(finding.ruleId)) { + const name = nameBeforeMajorLag(finding.message); + return name ? `name=${name}` : `message=${finding.message}`; + } + + if (LOCATION_ONLY_RULES.has(finding.ruleId)) return ''; + return finding.message ? `message=${finding.message}` : ''; +} + +function subjectFromDetails(finding: Finding): string { + const details = finding.details; + if (!details || typeof details.advisoryId !== 'string' || !details.advisoryId) return ''; + const ecosystem = typeof details.ecosystem === 'string' ? details.ecosystem : ''; + const pkg = typeof details.package === 'string' ? details.package : ''; + return `advisoryId=${details.advisoryId}\necosystem=${ecosystem}\npackage=${pkg}`; +} + +function nameBeforeMajorLag(message: string): string { + const match = MAJOR_LAG_NAME.exec(message); + return match?.[1]?.trim() ?? ''; +} + +function asFinding(value: unknown): Finding | undefined { + if (!value || typeof value !== 'object') return undefined; + const raw = value as Partial; + if (typeof raw.ruleId !== 'string' || typeof raw.location !== 'string') return undefined; + const level = raw.level === 'error' || raw.level === 'note' || raw.level === 'warning' ? raw.level : 'warning'; + const finding: Finding = { + ruleId: raw.ruleId, + level, + message: typeof raw.message === 'string' ? raw.message : '', + location: raw.location, + }; + if (raw.details && typeof raw.details === 'object') finding.details = raw.details; + return finding; +} + +function compareSuppressed(a: BaselineSuppressedFinding, b: BaselineSuppressedFinding): number { + return cmp(a.ruleId, b.ruleId) || cmp(a.location, b.location) || cmp(a.id, b.id); +} + +function cmp(a: string, b: string): number { + if (a < b) return -1; + if (a > b) return 1; + return 0; +} diff --git a/src/core-open/formatters/markdown.ts b/src/core-open/formatters/markdown.ts index be83d21..be01999 100644 --- a/src/core-open/formatters/markdown.ts +++ b/src/core-open/formatters/markdown.ts @@ -2,6 +2,7 @@ // scripts/vendor-core-open.mjs. Do not edit here — change the source package // and re-run the vendor script. Apache-2.0. import type { ScanArtifact } from '../types.js'; +import { baselineSuppressionPhraseFrom } from '../baseline-audit.js'; import { securityPacksLabel } from './text.js'; /** Rows shown before the infrastructure-findings table is cut with an "… N more" line. */ @@ -186,5 +187,11 @@ export function formatMarkdown(artifact: ScanArtifact): string { lines.push(''); } + const baselinePhrase = baselineSuppressionPhraseFrom(artifact.baseline); + if (baselinePhrase) { + lines.push(`**Baseline:** ${baselinePhrase}`); + lines.push(''); + } + return lines.join('\n'); } diff --git a/src/core-open/formatters/sarif.ts b/src/core-open/formatters/sarif.ts index 9825a8d..26213a0 100644 --- a/src/core-open/formatters/sarif.ts +++ b/src/core-open/formatters/sarif.ts @@ -2,20 +2,25 @@ // scripts/vendor-core-open.mjs. Do not edit here — change the source package // and re-run the vendor script. Apache-2.0. import type { ScanArtifact, Finding, SecurityFinding, SecuritySection, SecuritySeverity } from '../types.js'; +import { findingId, readBaselineComparison } from '../baseline-audit.js'; /** * Generate a SARIF 2.1.0 document from scan artifact. * - * The first run carries the drift findings and is byte-for-byte what it has - * always been. When the artifact carries security-pack findings - * (`extended.security`, from `vg scan --iac`), a second run is appended for - * them — one run per artifact, with every pack listed under - * `tool.extensions`, so a scan that ran no pack produces exactly the same - * bytes as before. + * The first run carries the drift findings. With no baseline comparison and + * no security-pack findings, that run is byte-for-byte what it has always + * been. A baseline comparison keeps every result and adds `suppressions` + * whose `properties.id` matches `baseline.suppressed` in the JSON artifact, + * plus an invocation property for the count (including zero). When the + * artifact carries security-pack findings (`extended.security`, from + * `vg scan --iac`), a second run is appended for them — one run per artifact, + * with every pack listed under `tool.extensions`. */ export function formatSarif(artifact: ScanArtifact): object { + const comparison = readBaselineComparison(artifact.baseline); + const suppressedIds = comparison ? new Set(comparison.suppressed.map((row) => row.id)) : null; const rules = buildRules(artifact.findings); - const results = artifact.findings.map((f) => toSarifResult(f)); + const results = artifact.findings.map((f) => toSarifResult(f, suppressedIds)); const runs: object[] = [ { @@ -32,6 +37,17 @@ export function formatSarif(artifact: ScanArtifact): object { { executionSuccessful: true, startTimeUtc: artifact.timestamp, + // Present only after a baseline comparison, so a scan that did not + // compare stays byte-for-byte what it was. A count of zero is still + // recorded: comparison with nothing matched is not a silent omission. + ...(comparison + ? { + properties: { + baselineCompared: true, + baselineSuppressedCount: comparison.suppressedCount, + }, + } + : {}), }, ], }, @@ -179,7 +195,9 @@ function buildRules(findings: Finding[]) { }); } -function toSarifResult(finding: Finding) { +function toSarifResult(finding: Finding, suppressedIds: ReadonlySet | null) { + const id = suppressedIds ? findingId(finding) : undefined; + const suppressed = id !== undefined && suppressedIds?.has(id) === true; return { ruleId: finding.ruleId, level: finding.level === 'error' ? 'error' : finding.level === 'warning' ? 'warning' : 'note', @@ -196,5 +214,19 @@ function toSarifResult(finding: Finding) { // Surface structured finding detail (e.g. advisory id, CVSS, fixed version) // to consumers like GitHub code scanning without bloating the message text. ...(finding.details && Object.keys(finding.details).length > 0 ? { properties: finding.details } : {}), + // The result stays in `results`. The suppression carries the same id as + // `baseline.suppressed` in the JSON artifact. + ...(suppressed && id + ? { + suppressions: [ + { + kind: 'external', + status: 'accepted', + justification: 'Matched a finding recorded in the drift baseline.', + properties: { id }, + }, + ], + } + : {}), }; } diff --git a/src/core-open/formatters/text.ts b/src/core-open/formatters/text.ts index 9f1ed28..0a7267a 100644 --- a/src/core-open/formatters/text.ts +++ b/src/core-open/formatters/text.ts @@ -3,6 +3,7 @@ // and re-run the vendor script. Apache-2.0. import chalk from 'chalk'; import type { ScanArtifact, BillingSummary, ExtendedScanResults, InventoryItem, ServiceDependencyItem, ArchitectureResult, SecurityFinding, SecuritySection } from '../types.js'; +import { baselineSuppressionPhraseFrom } from '../baseline-audit.js'; import { driftBar } from '../ui/bar.js'; import { titleBox, panelBox } from '../ui/box.js'; @@ -85,15 +86,21 @@ export function formatText(artifact: ScanArtifact, opts: FormatTextOptions = {}) lines.push(''); } - if (artifact.delta !== undefined) { - // Drift is "lower is better": a negative delta means drift fell (good), - // a positive delta means drift rose (bad). - const deltaStr = artifact.delta < 0 - ? chalk.green(`${artifact.delta}`) - : artifact.delta > 0 - ? chalk.red(`+${artifact.delta}`) - : chalk.dim('0'); - lines.push(chalk.bold(' Drift Delta: ') + deltaStr + ' (vs baseline)'); + const baselinePhrase = baselineSuppressionPhraseFrom(artifact.baseline); + if (artifact.delta !== undefined || baselinePhrase) { + if (artifact.delta !== undefined) { + // Drift is "lower is better": a negative delta means drift fell (good), + // a positive delta means drift rose (bad). + const deltaStr = artifact.delta < 0 + ? chalk.green(`${artifact.delta}`) + : artifact.delta > 0 + ? chalk.red(`+${artifact.delta}`) + : chalk.dim('0'); + lines.push(chalk.bold(' Drift Delta: ') + deltaStr + ' (vs baseline)'); + } + if (baselinePhrase) { + lines.push(chalk.bold(' Baseline: ') + baselinePhrase); + } lines.push(''); } diff --git a/src/core-open/index.ts b/src/core-open/index.ts index 2142847..e2a9c72 100644 --- a/src/core-open/index.ts +++ b/src/core-open/index.ts @@ -67,6 +67,16 @@ export { formatText } from './formatters/text.js'; export { formatSarif } from './formatters/sarif.js'; export { formatMarkdown } from './formatters/markdown.js'; +// ── Baseline comparison (audit record; findings are not dropped) ──────────── +export { + baselineFileReference, + baselineSuppressionPhrase, + baselineSuppressionPhraseFrom, + findingId, + readBaselineComparison, + suppressedBaselineFindings, +} from './baseline-audit.js'; + // ── Scanners (fact collection) ─────────────────────────────────────────────── export { scanNodeProjects } from './scanners/node-scanner.js'; export { scanDotnetProjects } from './scanners/dotnet-scanner.js'; diff --git a/src/core-open/run-core-scan.ts b/src/core-open/run-core-scan.ts index 5375ff9..6e0a5b6 100644 --- a/src/core-open/run-core-scan.ts +++ b/src/core-open/run-core-scan.ts @@ -33,6 +33,7 @@ import { Semaphore } from './utils/semaphore.js'; import { computeDriftScore, generateFindings, computeProjectId, computeSolutionId } from './scoring/drift-score.js'; import { formatText } from './formatters/text.js'; import { formatSarif } from './formatters/sarif.js'; +import { baselineFileReference, suppressedBaselineFindings } from './baseline-audit.js'; import { formatMarkdown } from './formatters/markdown.js'; import { loadConfig, appendExcludePatterns } from './config.js'; import { pathExists, readJsonFile, writeJsonFile, writeTextFile, ensureDir, FileCache, quickTreeCount } from './utils/fs.js'; @@ -818,12 +819,23 @@ export async function runCoreScan( if (await pathExists(baselinePath)) { try { const baseline = await readJsonFile(baselinePath); - // Only a repo-relative reference may enter the artifact: the absolute - // path leaks the local username/home layout to the ingest server. A - // baseline outside the repo degrades to its basename for the same reason. - const relBaseline = path.relative(rootDir, baselinePath); - artifact.baseline = !relBaseline || relBaseline.startsWith('..') ? path.basename(baselinePath) : relBaseline; - artifact.delta = artifact.drift.score - baseline.drift.score; + const score = baseline?.drift?.score; + if (typeof score !== 'number' || !Number.isFinite(score) || !Array.isArray(baseline.findings)) { + throw new Error('baseline file is not a scan artifact'); + } + // Findings stay on `artifact.findings`. This block is the auditable + // record of which of them were already in the baseline. Only a + // repo-relative reference (or a basename, when the file sits outside + // the repo) may enter the artifact: an absolute path leaks the local + // username and home layout. + const suppressed = suppressedBaselineFindings(artifact.findings, baseline.findings); + artifact.baseline = { + compared: true, + file: baselineFileReference(rootDir, baselinePath), + suppressedCount: suppressed.length, + suppressed, + }; + artifact.delta = artifact.drift.score - score; } catch { console.error(chalk.yellow(`Warning: Could not read baseline file: ${baselinePath}`)); } diff --git a/src/core-open/types.ts b/src/core-open/types.ts index 07f0335..3c391d8 100644 --- a/src/core-open/types.ts +++ b/src/core-open/types.ts @@ -626,6 +626,34 @@ export interface Finding { details?: Record; } +/** + * One current finding that was already present in the compared baseline. + * `id` is the content-derived finding id (see `findingId`). It is the + * identifier copied into SARIF suppressions. The finding message is absent. + */ +export interface BaselineSuppressedFinding { + ruleId: string; + location: string; + id: string; +} + +/** + * Auditable `--baseline` comparison. Matching findings stay in + * {@link ScanArtifact.findings}; this block is the record that they were + * recognized as already baselined. `suppressed` is sorted by `ruleId`, then + * `location`, then `id`. + */ +export interface BaselineComparison { + compared: true; + /** + * Repo-relative path of the baseline file (`/` separators), or its basename + * when the file lives outside the repo. Absolute paths never enter the artifact. + */ + file: string; + suppressedCount: number; + suppressed: BaselineSuppressedFinding[]; +} + // ── Version control info ── export type VcsType = 'git' | 'unknown'; @@ -709,7 +737,12 @@ export interface ScanArtifact { solutions?: SolutionScan[]; drift: DriftScore; findings: Finding[]; - baseline?: string; + /** + * Present when `--baseline` read a scan artifact. Records which current + * findings were already in that baseline. Findings are not removed from + * {@link ScanArtifact.findings}. + */ + baseline?: BaselineComparison; delta?: number; extended?: ExtendedScanResults; /** Scan wall-clock duration in milliseconds */ diff --git a/src/reporting/baseline-audit.test.ts b/src/reporting/baseline-audit.test.ts new file mode 100644 index 0000000..02a6c0b --- /dev/null +++ b/src/reporting/baseline-audit.test.ts @@ -0,0 +1,311 @@ +import { describe, it, expect, afterEach, vi } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + baselineFileReference, + findingId, + formatSarif, + readBaselineComparison, + runCoreScan, + suppressedBaselineFindings, + type Finding, + type ScanOptions, +} from '../core-open/index.js'; + +function finding(overrides: Partial & Pick): Finding { + return { + level: 'warning', + message: 'placeholder', + ...overrides, + }; +} + +describe('baseline finding ids', () => { + it('locks the id for a location-only rule', () => { + expect(findingId(finding({ + ruleId: 'vibgrate/dependency-rot', + message: '40% of dependencies are 2+ major versions behind in app.', + location: 'package.json', + }))).toBe('af173ea1c1efa4fd'); + }); + + it('keeps the same id when a location-only message changes', () => { + const older = finding({ ruleId: 'vibgrate/runtime-lag', location: 'app', message: 'is 2 behind' }); + const newer = finding({ ruleId: 'vibgrate/runtime-lag', location: 'app', message: 'is 3 behind' }); + expect(findingId(older)).toBe(findingId(newer)); + expect(findingId(older)).toBe('a0b24c8fa1d80216'); + }); + + it('distinguishes major-lag findings by package name, not by how far behind they are', () => { + const lodash = finding({ + ruleId: 'vibgrate/dependency-major-lag', + level: 'error', + location: 'app', + message: 'lodash is 3 major versions behind (spec: ^1.0.0, latest: 4.0.0).', + }); + const lodashLater = finding({ + ...lodash, + message: 'lodash is 4 major versions behind (spec: ^1.0.0, latest: 5.0.0).', + }); + const react = finding({ + ...lodash, + message: 'react is 3 major versions behind (spec: ^17.0.0, latest: 19.0.0).', + }); + expect(findingId(lodash)).toBe(findingId(lodashLater)); + expect(findingId(lodash)).toBe('07cc9e594585d05c'); + expect(findingId(lodash)).not.toBe(findingId(react)); + }); + + it('identifies a vulnerability by advisory, not by the message', () => { + const first = finding({ + ruleId: 'vibgrate/vulnerability', + level: 'error', + location: 'lodash', + message: 'lodash@1.0.0: GHSA-example (high) — introduced by a maintainer', + details: { advisoryId: 'GHSA-example', ecosystem: 'npm', package: 'lodash' }, + }); + const rewritten = finding({ ...first, message: 'wording changed; still the same advisory' }); + const other = finding({ + ...first, + message: first.message, + details: { advisoryId: 'GHSA-other', ecosystem: 'npm', package: 'lodash' }, + }); + expect(findingId(first)).toBe(findingId(rewritten)); + expect(findingId(first)).toBe('db61f217df4c7b5e'); + expect(findingId(first)).not.toBe(findingId(other)); + }); +}); + +describe('suppressedBaselineFindings', () => { + const rot = finding({ ruleId: 'vibgrate/dependency-rot', location: 'package.json', message: 'rot' }); + const lagA = finding({ ruleId: 'vibgrate/runtime-lag', location: 'a', message: 'lag a' }); + const lagB = finding({ ruleId: 'vibgrate/runtime-lag', location: 'b', message: 'lag b' }); + + it('sorts by ruleId, then location, then id, independent of input order', () => { + const forward = suppressedBaselineFindings([lagB, rot, lagA], [lagA, lagB, rot]); + const backward = suppressedBaselineFindings([rot, lagA, lagB], [lagB, rot, lagA]); + expect(forward).toEqual(backward); + expect(forward.map((row) => `${row.ruleId} ${row.location}`)).toEqual([ + 'vibgrate/dependency-rot package.json', + 'vibgrate/runtime-lag a', + 'vibgrate/runtime-lag b', + ]); + expect(forward.map((row) => row.id)).toEqual([ + 'af173ea1c1efa4fd', + findingId(lagA), + findingId(lagB), + ]); + }); + + it('lists only findings that are in the baseline and does not copy the message', () => { + const current = finding({ + ruleId: 'vibgrate/dependency-rot', + location: 'package.json', + message: 'placeholder-token-do-not-copy', + }); + const fresh = finding({ ruleId: 'vibgrate/runtime-eol', location: 'package.json', message: 'new' }); + const rows = suppressedBaselineFindings([current, fresh], [current]); + expect(rows).toEqual([{ + ruleId: 'vibgrate/dependency-rot', + location: 'package.json', + id: 'af173ea1c1efa4fd', + }]); + expect(JSON.stringify(rows)).not.toContain('placeholder-token'); + expect(Object.keys(rows[0]).sort()).toEqual(['id', 'location', 'ruleId']); + }); + + it('does not mutate either findings array', () => { + const current = [rot, lagA]; + const baseline = [rot]; + const beforeCurrent = JSON.stringify(current); + const beforeBaseline = JSON.stringify(baseline); + suppressedBaselineFindings(current, baseline); + expect(JSON.stringify(current)).toBe(beforeCurrent); + expect(JSON.stringify(baseline)).toBe(beforeBaseline); + }); + + it('collapses duplicate current findings to one record', () => { + expect(suppressedBaselineFindings([rot, rot], [rot])).toHaveLength(1); + }); + + it('returns an empty list when nothing matches', () => { + expect(suppressedBaselineFindings([lagA], [rot])).toEqual([]); + }); +}); + +describe('baselineFileReference', () => { + it('stores a repo-relative path and only a basename for a file outside the repo', () => { + expect(baselineFileReference('/repo', '/repo/.vibgrate/baseline.json')).toBe('.vibgrate/baseline.json'); + expect(baselineFileReference('/repo', '/tmp/elsewhere/baseline.json')).toBe('baseline.json'); + }); +}); + +describe('readBaselineComparison', () => { + it('ignores the legacy string path', () => { + expect(readBaselineComparison('.vibgrate/baseline.json')).toBeUndefined(); + expect(readBaselineComparison(undefined)).toBeUndefined(); + }); +}); + +describe('vg scan --baseline audit record', () => { + const dirs: string[] = []; + let logSpy: ReturnType; + + afterEach(() => { + logSpy?.mockRestore(); + vi.unstubAllEnvs(); + for (const dir of dirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true }); + }); + + function tempDir(prefix: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + dirs.push(dir); + return dir; + } + + function scanOpts(extra: Partial = {}): ScanOptions { + return { + format: 'text', + concurrency: 2, + offline: true, + noLocalArtifacts: true, + quiet: true, + vibgrateVersion: 'test', + ...extra, + }; + } + + it('records matched findings without removing them, and echoes the ids in SARIF', async () => { + vi.stubEnv('VIBGRATE_DSN', ''); + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const dir = tempDir('vg-baseline-audit-'); + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ + name: 'baseline-audit-fixture', + version: '0.0.0', + private: true, + engines: { node: '18.0.0' }, + })); + + const first = await runCoreScan(dir, scanOpts()); + expect(first.findings.length).toBeGreaterThan(0); + expect(first.findings.some((f) => f.ruleId === 'vibgrate/runtime-eol')).toBe(true); + expect(first.baseline).toBeUndefined(); + + const baselinePath = path.join(dir, 'baseline.json'); + fs.writeFileSync(baselinePath, JSON.stringify(first)); + + logSpy.mockClear(); + const second = await runCoreScan(dir, scanOpts({ baseline: baselinePath })); + + expect(second.findings.map((f) => `${f.ruleId}\0${f.location}\0${f.message}`)).toEqual( + first.findings.map((f) => `${f.ruleId}\0${f.location}\0${f.message}`), + ); + expect(second.baseline).toMatchObject({ + compared: true, + file: 'baseline.json', + suppressedCount: second.baseline?.suppressed.length, + }); + expect(second.baseline?.suppressedCount).toBeGreaterThan(0); + expect(second.baseline?.suppressed).toEqual(suppressedBaselineFindings(second.findings, first.findings)); + expect(second.delta).toBe(0); + + const printed = logSpy.mock.calls.map((call: unknown[]) => String(call[0] ?? '')).join('\n'); + expect(printed).toContain(`${second.baseline?.suppressedCount} finding`); + expect(printed).toContain('suppressed'); + for (const findingRow of second.findings) expect(printed).toContain(findingRow.ruleId); + + const sarif = formatSarif(second) as { + runs: Array<{ + results: Array<{ ruleId: string; suppressions?: Array<{ properties?: { id?: string } }> }>; + invocations: Array<{ properties?: { baselineCompared?: boolean; baselineSuppressedCount?: number } }>; + }>; + }; + const results = sarif.runs[0].results; + expect(results).toHaveLength(second.findings.length); + const suppressedIds = results.flatMap((result) => result.suppressions?.map((s) => s.properties?.id) ?? []); + expect(suppressedIds.sort()).toEqual(second.baseline?.suppressed.map((row) => row.id).sort()); + expect(sarif.runs[0].invocations[0].properties).toEqual({ + baselineCompared: true, + baselineSuppressedCount: second.baseline?.suppressedCount, + }); + expect(JSON.stringify(second.baseline)).not.toContain(dir); + }); + + it('records a comparison with zero matches and still keeps every finding', async () => { + vi.stubEnv('VIBGRATE_DSN', ''); + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const dir = tempDir('vg-baseline-empty-'); + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ + name: 'baseline-audit-fixture', + version: '0.0.0', + private: true, + engines: { node: '18.0.0' }, + })); + + const first = await runCoreScan(dir, scanOpts()); + const baselinePath = path.join(dir, 'baseline.json'); + fs.writeFileSync(baselinePath, JSON.stringify({ ...first, findings: [] })); + + logSpy.mockClear(); + const second = await runCoreScan(dir, scanOpts({ baseline: baselinePath })); + expect(second.findings.length).toBe(first.findings.length); + expect(second.baseline).toMatchObject({ + compared: true, + file: 'baseline.json', + suppressedCount: 0, + suppressed: [], + }); + const printed = logSpy.mock.calls.map((call: unknown[]) => String(call[0] ?? '')).join('\n'); + expect(printed).toContain('0 findings suppressed'); + + const sarif = formatSarif(second) as { + runs: Array<{ + results: Array<{ suppressions?: unknown }>; + invocations: Array<{ properties?: { baselineSuppressedCount?: number } }>; + }>; + }; + expect(sarif.runs[0].results.every((result) => result.suppressions === undefined)).toBe(true); + expect(sarif.runs[0].invocations[0].properties?.baselineSuppressedCount).toBe(0); + }); + + it('stores only the basename when the baseline file is outside the repo', async () => { + vi.stubEnv('VIBGRATE_DSN', ''); + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const dir = tempDir('vg-baseline-root-'); + const outside = tempDir('vg-baseline-outside-'); + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ + name: 'baseline-audit-fixture', + version: '0.0.0', + private: true, + engines: { node: '18.0.0' }, + })); + const first = await runCoreScan(dir, scanOpts()); + const outsideFile = path.join(outside, 'baseline.json'); + fs.writeFileSync(outsideFile, JSON.stringify(first)); + + const second = await runCoreScan(dir, scanOpts({ baseline: outsideFile })); + expect(second.baseline?.file).toBe('baseline.json'); + expect(JSON.stringify(second.baseline)).not.toContain(outside); + }); + + it('does not pretend a comparison happened when the baseline cannot be read', async () => { + vi.stubEnv('VIBGRATE_DSN', ''); + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const dir = tempDir('vg-baseline-bad-'); + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ + name: 'baseline-audit-fixture', + version: '0.0.0', + private: true, + })); + const baselinePath = path.join(dir, 'baseline.json'); + fs.writeFileSync(baselinePath, '{ not json'); + + const artifact = await runCoreScan(dir, scanOpts({ baseline: baselinePath })); + expect(artifact.baseline).toBeUndefined(); + expect(artifact.delta).toBeUndefined(); + expect(errSpy.mock.calls.flat().join('\n')).toContain('Could not read baseline file'); + errSpy.mockRestore(); + }); +}); diff --git a/src/reporting/commands/scan.ts b/src/reporting/commands/scan.ts index 76a4e3b..d141a60 100644 --- a/src/reporting/commands/scan.ts +++ b/src/reporting/commands/scan.ts @@ -376,7 +376,7 @@ export const scanCommand = new Command('scan') '--fail-on ', 'Fail on warn or error. architecture-finding (hard boundary violations) or architecture-warning (violations and warnings) gate on the architecture module\'s boundary findings, judged under the policy pack in force: .vibgrate/architecture.toml (policy = "hexagonal-v1" | "layered-v1" | "vertical-v1", plus any [[overlay]] rules), VIBGRATE_ARCHITECTURE_POLICY, or vg build --policy; default hexagonal-v1. The pack is named in the output. See docs/architecture-policies.md. iac-finding[=] fails on infrastructure findings from the iac-cis-v1 pack at or above (critical|high|medium|low|info; default high) and needs --iac (or --full) plus the code map — it exits 2 when the Architecture module is missing rather than passing an unevaluated tree; security-finding[=] is the umbrella across every security pack that ran. Comma-separated: at most one of warn/error/architecture-* plus any security gates, e.g. --fail-on error,iac-finding=medium', ) - .option('--baseline ', 'Compare against baseline') + .option('--baseline ', 'Compare against a baseline and record which findings it already contains') .option('--changed-only', 'Only scan changed files') .option( '-e, --exclude ', diff --git a/src/reporting/formatters/formatters.test.ts b/src/reporting/formatters/formatters.test.ts index 9019427..2d8cd73 100644 --- a/src/reporting/formatters/formatters.test.ts +++ b/src/reporting/formatters/formatters.test.ts @@ -3,6 +3,7 @@ import { describe, it, expect } from 'vitest'; // reporting-side copy was a stale duplicate and is gone. These tests keep // exercising the live writer. import { formatSarif as formatSarifCore } from '../../core-open/formatters/sarif.js'; +import { findingId } from '../../core-open/baseline-audit.js'; import { formatMarkdown } from '../formatters/markdown.js'; import { formatText } from '../formatters/text.js'; import type { ScanArtifact } from '../types.js'; @@ -144,6 +145,45 @@ describe('formatSarif', () => { expect(sarif.runs[0].invocations[0].startTimeUtc).toBe('2026-02-16T00:00:00.000Z'); expect(sarif.runs[0].invocations[0].executionSuccessful).toBe(true); + expect(sarif.runs[0].invocations[0].properties).toBeUndefined(); + expect(sarif.runs[0].results[0].suppressions).toBeUndefined(); + }); + + it('puts baseline ids on SARIF suppressions and keeps every result', () => { + const artifact = makeArtifact(); + const rot = artifact.findings[1]; + const id = findingId(rot); + artifact.baseline = { + compared: true, + file: '.vibgrate/baseline.json', + suppressedCount: 1, + suppressed: [{ ruleId: rot.ruleId, location: rot.location, id }], + }; + const sarif = formatSarif(artifact) as any; + const results = sarif.runs[0].results; + + expect(results).toHaveLength(2); + expect(results[0].suppressions).toBeUndefined(); + expect(results[1].suppressions).toEqual([ + { + kind: 'external', + status: 'accepted', + justification: 'Matched a finding recorded in the drift baseline.', + properties: { id }, + }, + ]); + expect(sarif.runs[0].invocations[0].properties).toEqual({ + baselineCompared: true, + baselineSuppressedCount: 1, + }); + }); + + it('does not treat a legacy baseline path string as suppressions', () => { + const artifact = makeArtifact(); + (artifact as { baseline?: unknown }).baseline = '.vibgrate/baseline.json'; + const sarif = formatSarif(artifact) as any; + expect(sarif.runs[0].results[0].suppressions).toBeUndefined(); + expect(sarif.runs[0].invocations[0].properties).toBeUndefined(); }); }); @@ -212,6 +252,21 @@ describe('formatMarkdown', () => { expect(md).toContain('📉'); }); + it('includes the baselined finding count without dropping findings', () => { + const md = formatMarkdown(makeArtifact({ + delta: 0, + baseline: { + compared: true, + file: '.vibgrate/baseline.json', + suppressedCount: 2, + suppressed: [], + }, + })); + expect(md).toContain('**Baseline:** 2 findings suppressed'); + expect(md).toContain('vibgrate/runtime-lag'); + expect(md).toContain('vibgrate/dependency-rot'); + }); + it('handles empty findings', () => { const md = formatMarkdown(makeArtifact({ findings: [] })); expect(md).not.toContain('## Findings'); @@ -368,6 +423,27 @@ describe('formatText', () => { expect(text).toContain('vs baseline'); }); + it('includes the baselined finding count', () => { + const text = formatText(makeArtifact({ + delta: 1, + baseline: { + compared: true, + file: 'baseline.json', + suppressedCount: 1, + suppressed: [], + }, + })); + expect(text).toContain('Drift Delta'); + expect(text).toContain('1 finding suppressed'); + expect(text).toContain('vibgrate/dependency-rot'); + }); + + it('ignores a legacy baseline path string', () => { + const artifact = makeArtifact({ delta: 1 }); + (artifact as { baseline?: unknown }).baseline = 'baseline.json'; + expect(formatText(artifact)).not.toContain('suppressed'); + }); + it('handles empty projects', () => { const text = formatText(makeArtifact({ projects: [], findings: [] })); expect(text).toContain('Vibgrate Drift Report'); diff --git a/src/reporting/formatters/markdown.ts b/src/reporting/formatters/markdown.ts index 5778589..61cb283 100644 --- a/src/reporting/formatters/markdown.ts +++ b/src/reporting/formatters/markdown.ts @@ -1,4 +1,5 @@ import type { ScanArtifact } from '../types.js'; +import { baselineSuppressionPhraseFrom } from '../../core-open/baseline-audit.js'; /** Generate a Markdown report from scan artifact */ export function formatMarkdown(artifact: ScanArtifact): string { @@ -130,5 +131,11 @@ export function formatMarkdown(artifact: ScanArtifact): string { lines.push(''); } + const baselinePhrase = baselineSuppressionPhraseFrom(artifact.baseline); + if (baselinePhrase) { + lines.push(`**Baseline:** ${baselinePhrase}`); + lines.push(''); + } + return lines.join('\n'); } diff --git a/src/reporting/formatters/text.ts b/src/reporting/formatters/text.ts index 5275e34..26020e1 100644 --- a/src/reporting/formatters/text.ts +++ b/src/reporting/formatters/text.ts @@ -1,5 +1,6 @@ import chalk from 'chalk'; import type { ScanArtifact, ExtendedScanResults, InventoryItem, ServiceDependencyItem, ArchitectureResult } from '../types.js'; +import { baselineSuppressionPhraseFrom } from '../../core-open/baseline-audit.js'; import { VERSION } from '../version.js'; import { driftBar } from '../../core-open/ui/bar.js'; import { titleBox } from '../../core-open/ui/box.js'; @@ -56,15 +57,21 @@ export function formatText(artifact: ScanArtifact): string { lines.push(''); } - if (artifact.delta !== undefined) { - // Lower is better: a positive delta means drift increased (worse → red), - // a negative delta means drift decreased (improved → green). - const deltaStr = artifact.delta > 0 - ? chalk.red(`+${artifact.delta}`) - : artifact.delta < 0 - ? chalk.green(`${artifact.delta}`) - : chalk.dim('0'); - lines.push(chalk.bold(' Drift Delta: ') + deltaStr + ' (vs baseline)'); + const baselinePhrase = baselineSuppressionPhraseFrom(artifact.baseline); + if (artifact.delta !== undefined || baselinePhrase) { + if (artifact.delta !== undefined) { + // Lower is better: a positive delta means drift increased (worse → red), + // a negative delta means drift decreased (improved → green). + const deltaStr = artifact.delta > 0 + ? chalk.red(`+${artifact.delta}`) + : artifact.delta < 0 + ? chalk.green(`${artifact.delta}`) + : chalk.dim('0'); + lines.push(chalk.bold(' Drift Delta: ') + deltaStr + ' (vs baseline)'); + } + if (baselinePhrase) { + lines.push(chalk.bold(' Baseline: ') + baselinePhrase); + } lines.push(''); } diff --git a/src/reporting/types.ts b/src/reporting/types.ts index 0ac11bf..e547ba2 100644 --- a/src/reporting/types.ts +++ b/src/reporting/types.ts @@ -204,6 +204,25 @@ export interface Finding { details?: Record; } +/** One current finding that was already present in the compared baseline. */ +export interface BaselineSuppressedFinding { + ruleId: string; + location: string; + id: string; +} + +/** + * Auditable `--baseline` comparison. Matching findings stay in `findings`. + * `suppressed` is sorted by `ruleId`, then `location`, then `id`. + */ +export interface BaselineComparison { + compared: true; + /** Repo-relative baseline path (`/` separators), or the basename when outside the repo. */ + file: string; + suppressedCount: number; + suppressed: BaselineSuppressedFinding[]; +} + // ── Version control info ── export type VcsType = 'git' | 'unknown'; @@ -245,7 +264,11 @@ export interface ScanArtifact { solutions?: SolutionScan[]; drift: DriftScore; findings: Finding[]; - baseline?: string; + /** + * Present when `--baseline` read a scan artifact. Records which current + * findings were already in that baseline. Findings stay in `findings`. + */ + baseline?: BaselineComparison; delta?: number; extended?: ExtendedScanResults; /** Scan wall-clock duration in milliseconds */