diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e7126f7..395aef9 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -541,7 +541,8 @@ forgekit/ doctor.js # health checks emit/ # one module per tool (claude, codex, cursor, gemini, aider, copilot, windsurf, zed, continue) + mcp ledger.js # PCM core: content-addressed claims, oracle taxonomy, decayed Beta val, Eq. 3 retrieval, semilattice merge (ADR-0006) - ledger_store.js # git-native on-disk ledger (.forge/ledger/): sharded claims, append-only evidence/tombstone logs, normal-form verify + ledger_store.js # git-native on-disk ledger (.forge/ledger/): sharded claims, append-only evidence/tombstone logs, normal-form verify, local usage log + ledger_retention.js # retention learned from the ledger's own history: archive never-served claims, idle ones past the longest observed comeback, and BIC-detected near-duplicates (`ledger compact`) ledger_bridge.js # legacy-store bridge, dormant by default (ledger-only); `FORGE_LEDGER_ONLY=0` re-enables cortex/recall/brain shadow-writes + idempotent `ledger import` ledger_read.js # ledger-only read path by default (`FORGE_LEDGER_ONLY=0` merges legacy∪ledger instead): cortex lesson/fact injection, `recall list`, brain's AGENTS.md index all see teammate knowledge from `ledger merge` learn_consolidate.js # bin/learn-consolidate.sh: deterministic consolidation of ~/.claude/skills/learned — merge duplicates, drop only ledger-refuted (dormant/retracted/attic) lessons; no model call @@ -620,8 +621,8 @@ from the tree it describes. ```mermaid %%{init: {'theme':'base','themeVariables':{'primaryColor':'#201a15','primaryTextColor':'#f2ede7','primaryBorderColor':'#372c22','lineColor':'#f26430','secondaryColor':'#272019','tertiaryColor':'#171310','edgeLabelBackground':'#201a15','clusterBkg':'#171310','clusterBorder':'#4a3b2e','fontFamily':'ui-sans-serif, system-ui, sans-serif','fontSize':'14px'},'flowchart':{'curve':'basis','padding':10,'nodeSpacing':36,'rankSpacing':44}}}%% flowchart LR - test["test
117 files"] - src["src
109 files"] + test["test
119 files"] + src["src
110 files"] landing["landing
61 files"] research["research
37 files"] global["global
5 files"] @@ -629,7 +630,7 @@ flowchart LR scripts["scripts
2 files"] docs["docs
1 file"] examples["examples
1 file"] - test -- 240 --> src + test -- 247 --> src bench -- 8 --> src examples -- 4 --> src test -- 2 --> bench diff --git a/CHANGELOG.md b/CHANGELOG.md index c5f8a66..098c540 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,35 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- **`forge ledger compact [--dry-run]`: ledger retention learned from the ledger's own + history.** It prints every learned number and archives three kinds of claim to the + attic. The attic is reversible: new evidence restores a claim, and `show`/`blame` still + read it. + - **Never-served claims:** tombstoned or dormant claims, which retrieval skips. + - **Idle claims:** idle longer than the longest stretch any claim of this ledger came back + from. + - **Near-duplicates:** a boundary exists only when BIC prefers two groups of + nearest-neighbour similarities to one. +- **A local usage log** (`.forge/ledger/.usage.jsonl`, gitignored) records which claims were + served: the session lesson block, pre-edit lessons, the déjà-vu advisory, + `forge ledger query` and the MCP query. Retention learns from it. Before this, nothing + recorded use. + +### Changed + +- **The Stop hook's ledger pruning no longer uses a fixed 2 × 45-day window.** Tombstoned and + dormant claims are archived at once. Live claims are archived by the learned idle cut-off, + and only once the usage log spans longer than that cut-off. + - This bounds the per-session summary claims, which were never contradicted and so grew + forever: one is archived once it has been idle past the cut-off. The bound is the ledger's + longest comeback, so a single claim that came back after a long silence raises it for + every claim — deliberately, because an archived claim is no longer served and so cannot + prove itself useful again. + - `forge ledger show` and `forge ledger blame` read the attic, so a fresh retraction stays + inspectable. + ## [1.1.1] - 2026-09-22 ### Fixed diff --git a/README.md b/README.md index 8043649..817fbb6 100644 --- a/README.md +++ b/README.md @@ -417,7 +417,7 @@ and output live in [`docs/GUIDE.md`](docs/GUIDE.md). | | `forge recall` | manage cross-session memory (list / add / consolidate) | | | `forge remember` | add a durable fact to this repo's portable memory (forge brain) | | | `forge brain` | show / rebuild the portable project memory index | -| | `forge ledger` | evidence-referenced memory — stats / verify / show / blame / query / at / diff / root / ratify / retract / merge / sync / import | +| | `forge ledger` | evidence-referenced memory — stats / verify / show / blame / query / compact / at / diff / root / ratify / retract / merge / sync / import | | | `forge handoff` | bounded session snapshot — rewrite .forge/state.md, re-injected each session start | | | `forge decide` | append-only decision log — D-#### ADR-lite entries in .forge/decisions.md | | | `forge know` | route any fact to its storage home (decision / ledger / recall / …) — total, never dropped | diff --git a/bench/impact_cases.mjs b/bench/impact_cases.mjs index b2202f7..a5bd765 100644 --- a/bench/impact_cases.mjs +++ b/bench/impact_cases.mjs @@ -47,7 +47,7 @@ // - test/ledger.test.js imports { mergeStates } (:14) and calls it // (src/ledger_sync.js:3 also names it in the module header — same file, already labeled.) // -// claimText (src/ledger.js) — 9 files +// claimText (src/ledger.js) — 10 files // - src/ledger.js defines it (:610); sketchOf() (:636), termsOf() (:637) and :880 call it // - src/context.js imports { claimText } (:13) and calls it (:185) // - src/dash.js imports { claimText } (:16) and calls it (:58, :389, :400) @@ -57,6 +57,7 @@ // - src/cortex_mcp.js dynamic-imports { claimText } (:91) and calls it (:96, :106) // - test/ledger.test.js imports { claimText } (:8) and calls it // - src/learn_consolidate.js imports { claimText } (:32) and calls it (:110) +// - src/ledger_retention.js imports { claimText } (:29) and calls it (:184) // (test/dash.test.js:69 mentions the name only inside an assertion message — a string, // not a reference — so it is NOT labeled as a dependent.) // @@ -116,6 +117,7 @@ export const IMPACT_CASES = [ "src/dash.js", "src/deja.js", "src/learn_consolidate.js", + "src/ledger_retention.js", "src/ledger_store.js", "src/cli.js", "src/cortex_mcp.js", diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 172f6cf..1e437f4 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -843,6 +843,38 @@ freshly minted copy of the same fact stay two entries until you run it. It moves evidence and provenance logs with it, unions them into an existing twin rather than overwriting, and is idempotent. +`forge ledger compact [--dry-run]` archives what this ledger's own history says will not be +used again. It prints every number it learned, and nothing in it is a fixed threshold: + +```console +$ forge ledger compact --dry-run +Forge ledger — compact (every cut-off learned from this ledger) [dry run] + + claims: 11 · claims with logged use: 10 + retention: idle cut-off 4 d = the longest idle stretch any claim came back from (199 comebacks, typical gap 4 d; usage log spans 90 d) + duplicates: boundary 0.28 (two components beat one: BIC -72.6 < 3.5) · 1 group(s) + + archive: 3 + 34a49b8d036e idle 86 d > learned cut-off 4 d + a5e218fd1814 tombstoned (never served) + d3a5a1c9941e near-duplicate of c70ee7d4f505 (similarity 0.55 ≥ learned 0.28) + + dry run: nothing written +``` + +**The three archive rules:** +- **Never served:** a tombstoned or dormant claim goes at once, because retrieval never serves it. +- **Idle too long:** a live claim goes once it has been idle longer than any claim here has ever been idle and then used again. Until the usage log covers that long, no live claim is archived. +- **Near-duplicates:** each claim's similarity to its closest claim of the same kind is modelled as one group or two, and BIC decides which fits. Only two groups produce a duplicate boundary. The claim kept from each group is the one with the highest val. + +**Where use comes from:** forge writes `.forge/ledger/.usage.jsonl`, a gitignored local log. It records each claim that the session lesson block, pre-edit lessons, the déjà-vu advisory, `ledger query` or the MCP query served. + +**What happens to archived claims:** +- They move to `.forge/ledger/attic/`, and their logs stay where they are. +- `forge ledger show` and `blame` still read them. +- New evidence brings one back. +- The Stop hook applies the first two rules on its own; duplicates are grouped only by this command. + `forge ledger blame ` is the accountability view — every mint, every oracle outcome, every retraction, and per-author trust: diff --git a/src/cli.js b/src/cli.js index 27d6fb5..f375979 100755 --- a/src/cli.js +++ b/src/cli.js @@ -740,7 +740,7 @@ HANDLERS.ledger = async (argv) => { } if (sub === "show") { const id = args[2]; - const hit = id && id.length >= 2 ? ls.getClaimByPrefix(dir, id) : null; + const hit = id && id.length >= 2 ? ls.getClaimByPrefix(dir, id, { attic: true }) : null; if (!hit) { console.error( id ? ` no claim matching ${id}` : "usage: forge ledger show ", @@ -874,6 +874,37 @@ HANDLERS.ledger = async (argv) => { ); return; } + // `compact` — archive what this ledger's own history says will not be used again, and + // near-duplicates, printing every learned number (ledger_retention.js). Reversible. + if (sub === "compact") { + const dryRun = argv.includes("--dry-run"); + const r = ls.compactLedger(dir, nowDay, { dryRun }); + if (json) return console.log(JSON.stringify(r, null, 2)); + const rt = r.retention; + const d = r.duplicates; + const lines = [ + `Forge ledger — compact (every cut-off learned from this ledger)${dryRun ? " [dry run]" : ""}`, + "", + ` claims: ${r.claims} · claims with logged use: ${r.servedClaims}`, + rt.learned + ? ` retention: idle cut-off ${rt.cutoff} d = the longest idle stretch any claim came back from (${rt.comebacks} comebacks, typical gap ${rt.typicalGap} d; usage log spans ${rt.usageSpan} d)` + : ` retention: not learned — ${rt.reason}`, + d?.boundary != null + ? ` duplicates: boundary ${d.boundary.toFixed(2)} (two components beat one: BIC ${d.bic2?.toFixed(1)} < ${d.bic1?.toFixed(1)}) · ${d.groups.length} group(s)` + : ` duplicates: none — ${d?.compared ? `one component fits the ${d.compared} nearest-neighbour similarities better` : "fewer than two claims of one kind are still live to compare"}`, + "", + ` archive: ${r.archive.length}`, + ]; + for (const a of r.archive.slice(0, 20)) lines.push(` ${a.id.slice(0, 12)} ${a.reason}`); + if (r.archive.length > 20) lines.push(` … ${r.archive.length - 20} more (--json for all)`); + lines.push( + "", + dryRun + ? " dry run: nothing written" + : ` archived ${r.archived.length} claim(s) to .forge/ledger/attic/ — new evidence brings one back; show/blame still read it`, + ); + return console.log(lines.join("\n")); + } if (sub === "query") { const q = args.slice(2).join(" "); if (!q) { @@ -889,6 +920,11 @@ HANDLERS.ledger = async (argv) => { const claims = ls.loadClaims(dir); const sim = claimSim(root, q, claims, claimText); const ranked = retrieve(q, claims, { nowDay, budget: 8, sim }); + ls.recordUse( + dir, + ranked.map((r) => r.claim.id), + { via: "cli.query", t: nowDay }, + ); if (json) return console.log( JSON.stringify( diff --git a/src/commands.js b/src/commands.js index 8bd433f..ee71c3e 100644 --- a/src/commands.js +++ b/src/commands.js @@ -87,7 +87,7 @@ export const COMMANDS = { cortex: "self-correcting project memory — status / why ", deja: "anti-repetition — have you done this task before? ranks prior solved/verified sessions", ledger: - "evidence-referenced memory — stats / verify / show / blame / query / at / diff / root / ratify / retract / merge / sync / import", + "evidence-referenced memory — stats / verify / show / blame / query / compact / at / diff / root / ratify / retract / merge / sync / import", reuse: "proof-carrying code cache — query / mint --file / stats", context: "budgeted context assembly + completeness gate — what an edit NEEDS known", preflight: "assumption check — what a task names that the repo doesn't define", diff --git a/src/cortex.js b/src/cortex.js index e2d2401..6409ced 100644 --- a/src/cortex.js +++ b/src/cortex.js @@ -6,6 +6,7 @@ import { recordLessonEvent, supersedeLessonClaim } from "./ledger_bridge.js"; import { ledgerLessons, mergedLessons } from "./ledger_read.js"; +import { recordUse, repoLedger } from "./ledger_store.js"; import { confidenceOf, confirm, @@ -188,15 +189,39 @@ export function lessonsForContext(root, context, opts = {}) { return selectForInjection(mergedLessons(root, opts.nowDay ?? 0), context, opts); } +/** Log that these lessons were served (ledger retention learns from it). Only lessons + * backed by a ledger claim have an id to log; best-effort, never throws. + * @param {string} root + * @param {{provenance?: {claim?: string}}[]} lessons + * @param {{via: string, t: number}} opts */ +export function recordServedLessons(root, lessons, { via, t }) { + recordUse( + repoLedger(root), + lessons.map((l) => l?.provenance?.claim).filter((id) => typeof id === "string"), + { via, t }, + ); +} + /** Repo-wide top active lessons — what a SessionStart hook injects (no file context yet). - * Merged view: a teammate's outcome-confirmed lesson surfaces here too. */ -export function startupBlock(root, nowDay = 0, budget = 8) { + * Merged view: a teammate's outcome-confirmed lesson surfaces here too. `record` logs the + * shown lessons as served; only the hook sets it (AGENTS.md emission is not a use). + * @param {string} root + * @param {number} [nowDay] + * @param {number} [budget] + * @param {{record?: boolean}} [opts] */ +export function startupBlock(root, nowDay = 0, budget = 8, { record = false } = {}) { const active = mergedLessons(root, nowDay).filter((l) => l.status === "active"); if (!active.length) return ""; const ranked = active .map((l) => ({ lesson: l, conf: confidenceOf(l, nowDay) })) .sort((a, b) => b.conf - a.conf); const shown = ranked.slice(0, budget); + if (record) + recordServedLessons( + root, + shown.map((x) => x.lesson), + { via: "session-start", t: nowDay }, + ); const rows = shown.map((x) => `- **${x.lesson.id}** — ${x.lesson.correctedBehavior}`.slice(0, 200), ); diff --git a/src/cortex_hook_main.js b/src/cortex_hook_main.js index 2546908..4077055 100644 --- a/src/cortex_hook_main.js +++ b/src/cortex_hook_main.js @@ -10,7 +10,12 @@ // stop (Stop) — distill the session into lessons // stop-gate (Stop, synchronous) — completion gate: block once if code moved but no doc/state did // session-start (SessionStart) — inject learned lessons as context -import { applyDistillation, lessonsForContext, startupBlock } from "./cortex.js"; +import { + applyDistillation, + lessonsForContext, + recordServedLessons, + startupBlock, +} from "./cortex.js"; import { appendSessionEvent, classifyEvent, @@ -130,7 +135,7 @@ async function main() { const { stateBlock } = await import("./handoff.js"); const { rehydrationBlock } = await import("./session.js"); const block = [ - startupBlock(root, today), + startupBlock(root, today, undefined, { record: true }), goalBlock(root), stateBlock(root), rehydrationBlock(root), @@ -254,7 +259,10 @@ async function preEditAdvisory(root, input, today) { { files: [file], symbols: [], keywords: [file] }, { nowDay: today, budget: 3 }, ); - if (selected.length) return block; // learned lessons for this file win + if (selected.length) { + recordServedLessons(root, selected, { via: "pre-edit", t: today }); + return block; // learned lessons for this file win + } const { riskFor } = await import("./predictor.js"); const features = await liveEditFeatures(root, file, input, today); const { band } = riskFor(features, { mode: "heuristic" }); diff --git a/src/cortex_mcp.js b/src/cortex_mcp.js index 6b85a4a..41b09e1 100644 --- a/src/cortex_mcp.js +++ b/src/cortex_mcp.js @@ -87,7 +87,9 @@ async function callTool(name, args = {}) { } if (name === "forge_ledger_query") { try { - const { loadClaims, repoLedger, retractionProposals } = await import("./ledger_store.js"); + const { loadClaims, recordUse, repoLedger, retractionProposals } = await import( + "./ledger_store.js" + ); const { retrieve, claimText } = await import("./ledger.js"); const { claimSim, simLabel } = await import("./embed.js"); const dir = repoLedger(root); @@ -95,6 +97,11 @@ async function callTool(name, args = {}) { const claims = loadClaims(dir); const sim = claimSim(root, q, claims, claimText); const ranked = retrieve(q, claims, { nowDay: today(), budget: 8, sim }); + recordUse( + dir, + ranked.map((r) => r.claim.id), + { via: "mcp.query", t: today() }, + ); const pending = retractionProposals(claims); return JSON.stringify( { diff --git a/src/deja.js b/src/deja.js index acbecbe..38bfbf4 100644 --- a/src/deja.js +++ b/src/deja.js @@ -22,6 +22,7 @@ import { loadClaims, pruneLedger, putClaim, + recordUse, reindex, repoLedger, } from "./ledger_store.js"; @@ -124,9 +125,10 @@ export function recordSessionSummary(root, sid, events, nowDay = epochDay()) { }); if (o.ok) appendEvidence(dir, minted.claim.id, o.outcome); } - // Session-end housekeeping (the murāja'a job): archive what the protocol says is - // forgotten — tombstoned or dormant with nothing new for 2·T — so the ledger the next - // prompt reads stays bounded. Nothing is deleted; new evidence un-archives a claim. + // Session-end housekeeping (the murāja'a job): archive what this ledger's own history + // says will not be served again (ledger_retention.js — never-served claims, and live ones + // idle past the longest comeback), so the ledger the next prompt reads stays bounded. + // Nothing is deleted; new evidence un-archives a claim. pruneLedger(dir, nowDay); reindex(dir, nowDay); return { ok: true, id: minted.claim.id, tested: s.tested }; @@ -194,7 +196,10 @@ export function dejaAdvisory(root, task, nowDay = epochDay()) { if (!task || !String(task).trim()) return ""; try { const hits = dejaFromLedger(root, task, { nowDay, budget: 3 }); - return dejaLine(hits[0], nowDay); + const line = dejaLine(hits[0], nowDay); + // Only a surfaced hit counts as use; a hit below the relevance floor was never shown. + if (line) recordUse(repoLedger(root), [hits[0].claim.id], { via: "deja", t: nowDay }); + return line; } catch { return ""; } diff --git a/src/ledger_retention.js b/src/ledger_retention.js new file mode 100644 index 0000000..6114bb3 --- /dev/null +++ b/src/ledger_retention.js @@ -0,0 +1,316 @@ +// Ledger retention and compaction, learned from THIS ledger's own history. Pure: no fs, no +// clock. ledger_store.js supplies the claims and the usage log and applies the plan. +// +// The rule it replaces was a fixed review window (archive a tombstoned or dormant claim +// once 2 × 45 days passed since its last event) and a never-called clusters() at τ = 0.7. +// Nothing learned from use, and the session summaries the Stop hook mints every session +// were never contradicted, so they never went dormant and the ledger grew without bound. +// Here every cut-off is estimated from the ledger being compacted: +// +// - ARCHIVE an unservable claim (tombstoned, or dormant: retrieve() already skips it, so +// its chance of being served is zero by construction, not by a tuned threshold). +// - ARCHIVE a live claim once it has been idle longer than any claim in this ledger has +// ever been idle and then come back: the longest gap between two consecutive +// activities of one claim. Past that point the ledger has never seen a reuse, so the +// empirical chance of one is zero. An archived claim is no longer served and so can no +// longer be used, which would make a wrong archive self-confirming. That is why the +// cut-off is the longest comeback, not a typical one. (A replay that fit a cut-off by +// F1 was tried first: it archived a claim used every 3 days on the day it fell due.) +// The rule only switches on once the usage log spans longer than that gap. Before +// then, "not used again" only means "use was not recorded". +// - GROUP near-duplicates of one kind: each claim's nearest-neighbour similarity is +// modelled as one Gaussian or two (hard split, Otsu), BIC picks the model, and only a +// two-component fit yields a duplicate boundary (where the two posteriors are equal). +// A ledger without duplicates yields none. +// +// Everything this plans is reversible: an archived claim keeps its bytes in the attic and +// its logs in place, and any new evidence brings it back (ledger_store appendRecord). + +import { claimText, isDormant, jaccard, SKETCH_K, sketch, val } from "./ledger.js"; + +/** @typedef {{id: string, kind?: string, body?: any, provenance?: {t?: number}, + * evidence?: {t?: number}[], tombstone?: {t?: number} | null}} Claim */ + +const median = (xs) => { + const s = [...xs].sort((a, b) => a - b); + if (!s.length) return null; + const m = s.length >> 1; + return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; +}; + +/** + * The days a claim was active: minted, given evidence, or served (usage log). Sorted, + * unique, integers. + * @param {Claim} claim + * @param {number[]} [useDays] + * @returns {number[]} + */ +export function activityDays(claim, useDays = []) { + const days = new Set(); + const add = (t) => { + if (Number.isFinite(t)) days.add(Math.floor(t)); + }; + add(claim.provenance?.t); + for (const e of claim.evidence ?? []) add(e.t); + for (const t of useDays) add(t); + return [...days].sort((a, b) => a - b); +} + +/** + * Learn the idle cut-off from the ledger's own history (see the header): the longest gap + * after which any claim was active again. + * @param {{days: number[]}[]} histories one per claim (or per id seen only in the usage + * log): its sorted activity days + * @param {number} nowDay + * @param {{usageSince?: number | null}} opts the first day the usage log recorded anything + * @returns {{learned: boolean, reason?: string, cutoff?: number, typicalGap?: number, + * comebacks?: number, usageSpan?: number}} + */ +export function learnIdleCutoff(histories, nowDay, { usageSince = null } = {}) { + if (usageSince == null) + return { learned: false, reason: "no usage recorded yet — live claims are kept" }; + const gaps = []; + for (const h of histories) + for (let i = 1; i < h.days.length; i++) gaps.push(h.days[i] - h.days[i - 1]); + if (!gaps.length) + return { + learned: false, + reason: "no claim has been active twice — no comeback to learn from, live claims are kept", + }; + const cutoff = Math.max(...gaps); + const typicalGap = median(gaps) ?? undefined; + const usageSpan = nowDay - usageSince; + if (usageSpan <= cutoff) + return { + learned: false, + cutoff, + typicalGap, + comebacks: gaps.length, + usageSpan, + reason: `the usage log spans ${usageSpan} d, not yet longer than the longest comeback (${cutoff} d) — live claims are kept`, + }; + return { learned: true, cutoff, typicalGap, comebacks: gaps.length, usageSpan }; +} + +// ── Duplicates ──────────────────────────────────────────────────────────────────────── + +/** Log-likelihood of xs under N(mu, v). */ +const gaussLL = (xs, mu, v) => + xs.reduce((s, x) => s - 0.5 * (Math.log(2 * Math.PI * v) + (x - mu) ** 2 / v), 0); + +const meanVar = (xs, floor) => { + const mu = xs.reduce((s, x) => s + x, 0) / xs.length; + const v = xs.reduce((s, x) => s + (x - mu) ** 2, 0) / xs.length; + return { mu, v: Math.max(v, floor) }; +}; + +/** + * One Gaussian or two, chosen by BIC, over nearest-neighbour similarities. The variance + * floor is the MinHash estimate's own resolution (a step of 1/k, uniform quantisation + * noise 1/(12k²)), so identical duplicates (similarity exactly 1) cannot make the + * likelihood infinite. Returns the duplicate boundary, or null when one component wins. + * @param {number[]} xs + * @param {number} [k] sketch size + * @returns {{boundary: number | null, bic1: number, bic2: number | null, + * low?: {mu: number, v: number, w: number}, high?: {mu: number, v: number, w: number}}} + */ +export function similarityBoundary(xs, k = SKETCH_K) { + const n = xs.length; + const floor = 1 / (12 * k * k); + const one = meanVar(xs, floor); + const bic1 = -2 * gaussLL(xs, one.mu, one.v) + 2 * Math.log(n); + // A two-component fit has 5 parameters; with no more points than that it is not + // identifiable, so one component stands. + if (n <= 5) return { boundary: null, bic1, bic2: null }; + const s = [...xs].sort((a, b) => a - b); + // Otsu: the split that maximises the between-class variance. + let bestSplit = -1; + let bestBetween = -1; + let sumLeft = 0; + const total = s.reduce((a, b) => a + b, 0); + for (let i = 0; i < n - 1; i++) { + sumLeft += s[i]; + if (s[i] === s[i + 1]) continue; // split only between distinct values + const w0 = (i + 1) / n; + const m0 = sumLeft / (i + 1); + const m1 = (total - sumLeft) / (n - i - 1); + const between = w0 * (1 - w0) * (m0 - m1) ** 2; + if (between > bestBetween) { + bestBetween = between; + bestSplit = i; + } + } + if (bestSplit < 0) return { boundary: null, bic1, bic2: null }; // all values equal + const lowXs = s.slice(0, bestSplit + 1); + const highXs = s.slice(bestSplit + 1); + const lo = { ...meanVar(lowXs, floor), w: lowXs.length / n }; + const hi = { ...meanVar(highXs, floor), w: highXs.length / n }; + const ll2 = xs.reduce( + (acc, x) => + acc + + Math.log( + (lo.w * Math.exp((-0.5 * (x - lo.mu) ** 2) / lo.v)) / Math.sqrt(2 * Math.PI * lo.v) + + (hi.w * Math.exp((-0.5 * (x - hi.mu) ** 2) / hi.v)) / Math.sqrt(2 * Math.PI * hi.v), + ), + 0, + ); + const bic2 = -2 * ll2 + 5 * Math.log(n); + if (!(bic2 < bic1)) return { boundary: null, bic1, bic2, low: lo, high: hi }; + // Where the weighted densities cross between the two means: the Bayes decision point. + const logDens = (x, c) => Math.log(c.w) - 0.5 * Math.log(c.v) - (0.5 * (x - c.mu) ** 2) / c.v; + let a = lo.mu; + let b = hi.mu; + for (let it = 0; it < 64 && b - a > 1e-12; it++) { + const m = (a + b) / 2; + if (logDens(m, hi) >= logDens(m, lo)) b = m; + else a = m; + } + return { boundary: b, bic1, bic2, low: lo, high: hi }; +} + +/** + * Near-duplicate groups among live claims of the same kind, with one survivor each: the + * highest val, then the most evidence, then the earliest minted, then the smallest id. + * @param {Claim[]} claims live (servable) claims + * @param {number} nowDay + * @returns {{boundary: number | null, bic1?: number, bic2?: number | null, compared: number, + * groups: {keep: string, drop: {id: string, similarity: number}[]}[]}} + */ +export function duplicateGroups(claims, nowDay) { + const byKind = new Map(); + for (const c of claims) { + const k = c.kind ?? ""; + if (!byKind.has(k)) byKind.set(k, []); + byKind.get(k).push({ c, s: sketch(claimText(c)) }); + } + /** @type {{i: any, j: any, sim: number}[]} */ + const pairs = []; + const nn = []; + for (const items of byKind.values()) { + if (items.length < 2) continue; + const best = new Array(items.length).fill(0); + for (let i = 0; i < items.length; i++) + for (let j = i + 1; j < items.length; j++) { + const sim = jaccard(items[i].s, items[j].s); + pairs.push({ i: items[i], j: items[j], sim }); + if (sim > best[i]) best[i] = sim; + if (sim > best[j]) best[j] = sim; + } + nn.push(...best); + } + if (!nn.length) return { boundary: null, compared: 0, groups: [] }; + const fit = similarityBoundary(nn); + if (fit.boundary == null) + return { boundary: null, bic1: fit.bic1, bic2: fit.bic2, compared: nn.length, groups: [] }; + // Union-find over the pairs at or above the boundary. + const parent = new Map(); + const find = (x) => { + while (parent.get(x) !== x) { + parent.set(x, parent.get(parent.get(x))); + x = parent.get(x); + } + return x; + }; + const pairKey = (a, b) => (a < b ? `${a}\n${b}` : `${b}\n${a}`); + /** @type {Map} similarity of each pair at or above the boundary */ + const close = new Map(); + for (const p of pairs) { + if (p.sim < fit.boundary) continue; + for (const it of [p.i, p.j]) if (!parent.has(it.c.id)) parent.set(it.c.id, it.c.id); + parent.set(find(p.i.c.id), find(p.j.c.id)); + close.set(pairKey(p.i.c.id, p.j.c.id), p.sim); + } + const members = new Map(); + const claimById = new Map(claims.map((c) => [c.id, c])); + for (const id of parent.keys()) { + const r = find(id); + if (!members.has(r)) members.set(r, []); + members.get(r).push(claimById.get(id)); + } + const rank = (c) => [ + -val(c, nowDay), + -(c.evidence?.length ?? 0), + c.provenance?.t ?? Number.POSITIVE_INFINITY, + c.id, + ]; + const cmp = (a, b) => { + const ra = rank(a); + const rb = rank(b); + for (let i = 0; i < ra.length; i++) if (ra[i] !== rb[i]) return ra[i] < rb[i] ? -1 : 1; + return 0; + }; + // A member is dropped only as a duplicate of the SURVIVOR, judged by their own pair. The + // union-find groups can chain (A–B and B–C close, A–C not): C is then no duplicate of A and + // stays, and the reported similarity is always the pair that decided. + const groups = [...members.values()] + .filter((g) => g.length >= 2) + .map((g) => { + const [keep, ...rest] = [...g].sort(cmp); + const drop = rest + .map((c) => ({ id: c.id, similarity: close.get(pairKey(keep.id, c.id)) })) + .filter((d) => d.similarity != null) + .map((d) => ({ id: d.id, similarity: /** @type {number} */ (d.similarity) })); + return { keep: keep.id, drop }; + }) + .filter((g) => g.drop.length) + .sort((a, b) => (a.keep < b.keep ? -1 : 1)); + return { boundary: fit.boundary, bic1: fit.bic1, bic2: fit.bic2, compared: nn.length, groups }; +} + +// ── The plan ────────────────────────────────────────────────────────────────────────── + +/** + * What to archive, and why. `uses` maps claim id → the days it was served. + * @param {Claim[]} claims every claim in the live store (not the attic) + * @param {Map} uses + * @param {number} nowDay + * @param {{halfLife?: number, duplicates?: boolean}} [opts] halfLife only feeds isDormant + * @returns {{archive: {id: string, reason: string}[], retention: ReturnType, + * duplicates: ReturnType | null}} + */ +export function retentionPlan(claims, uses, nowDay, { halfLife, duplicates = false } = {}) { + let usageSince = null; + for (const days of uses.values()) + for (const d of days) if (usageSince == null || d < usageSince) usageSince = d; + const histories = claims.map((c) => ({ days: activityDays(c, uses.get(c.id) ?? []) })); + // Claims already in the attic still count as evidence of how long a claim can idle and + // come back: their served days stay in the usage log. + const known = new Set(claims.map((c) => c.id)); + const learnFrom = [...histories]; + for (const [id, days] of uses) + if (!known.has(id)) learnFrom.push({ days: [...new Set(days)].sort((a, b) => a - b) }); + const retention = learnIdleCutoff(learnFrom, nowDay, { usageSince }); + /** @type {{id: string, reason: string}[]} */ + const archive = []; + const live = []; + const dormantOpts = halfLife == null ? {} : { halfLife }; + for (let i = 0; i < claims.length; i++) { + const c = claims[i]; + if (c.tombstone) { + archive.push({ id: c.id, reason: "tombstoned (never served)" }); + continue; + } + if (isDormant(c, nowDay, dormantOpts)) { + archive.push({ id: c.id, reason: "dormant (never served)" }); + continue; + } + const days = histories[i].days; + const idle = days.length ? nowDay - days[days.length - 1] : null; + if (retention.learned && idle != null && idle > /** @type {number} */ (retention.cutoff)) { + archive.push({ id: c.id, reason: `idle ${idle} d > learned cut-off ${retention.cutoff} d` }); + continue; + } + live.push(c); + } + let dup = null; + if (duplicates) { + dup = duplicateGroups(live, nowDay); + for (const g of dup.groups) + for (const d of g.drop) + archive.push({ + id: d.id, + reason: `near-duplicate of ${g.keep.slice(0, 12)} (similarity ${d.similarity.toFixed(2)} ≥ learned ${dup.boundary?.toFixed(2)})`, + }); + } + return { archive, retention, duplicates: dup }; +} diff --git a/src/ledger_store.js b/src/ledger_store.js index c3ed698..8d69e96 100644 --- a/src/ledger_store.js +++ b/src/ledger_store.js @@ -29,7 +29,6 @@ import { DORMANT_VAL, emptyState, hasSecret, - isDormant, legacyClaimId, liveClaims, mergeStates, @@ -41,6 +40,7 @@ import { validateRef, validOutcome, } from "./ledger.js"; +import { retentionPlan } from "./ledger_retention.js"; import { redactSecrets } from "./secrets.js"; import { contentHash, epochDay, readJsonSafe } from "./util.js"; @@ -429,18 +429,93 @@ function readStateCache(dir, sig) { return cached.state; } +/** Keep a machine-local file out of git: add `name` to the ledger's .gitignore if absent. */ +function ensureLocalIgnored(dir, name, comment) { + const path = join(dir, GITIGNORE_FILE); + let current = ""; + try { + current = readFileSync(path, "utf8"); + } catch {} // no file yet + if (current.split(/\r?\n/).includes(name)) return; + // APPEND, never rewrite: two processes (a hook and a CLI) can reach this at once, and a + // read-modify-write would drop the other's line. Appending can at worst duplicate a + // comment, which git ignores. + appendLine(path, `# ${comment} (forge)\n${name}`); +} + function writeStateCache(dir, sig, state) { try { mkdirSync(dir, { recursive: true }); - if (!existsSync(join(dir, GITIGNORE_FILE))) - writeFileSync( - join(dir, GITIGNORE_FILE), - `# derived read cache — rebuilt from the claim files whenever they change (forge)\n${CACHE_FILE}\n`, - ); + ensureLocalIgnored( + dir, + CACHE_FILE, + "derived read cache — rebuilt from the claim files whenever they change", + ); writeFileSync(join(dir, CACHE_FILE), JSON.stringify({ sig, state })); } catch {} // a read-only checkout just pays the full read every time } +// --------------------------------------------------------------------------- +// Usage log. Retention learns from which claims actually get served (ledger_retention.js), +// and nothing recorded that: retrieve() is pure and every caller discarded the ids. Each +// place that SERVES claims to an agent or a person now appends one line here: the session +// lesson block, pre-edit lessons, the déjà-vu advisory, `forge ledger query` and the MCP +// query. It is machine-local (gitignored) and outside ledgerSignature, so appending never +// invalidates the snapshot cache. +// --------------------------------------------------------------------------- + +export const USAGE_FILE = ".usage.jsonl"; + +/** + * Record that these claims were served. Best-effort: never throws (hooks call it), and a + * missing ledger directory records nothing. + * @param {string} dir + * @param {string[]} ids + * @param {{via?: string, t?: number}} [opts] + */ +export function recordUse(dir, ids, { via = "", t = epochDay() } = {}) { + try { + const list = [...new Set((ids ?? []).filter((x) => typeof x === "string" && x))]; + if (!list.length || !existsSync(dir)) return; + ensureLocalIgnored(dir, USAGE_FILE, "which claims were served, and when — local use log"); + // appendLine terminates a line a killed process left torn, so the next record never + // glues onto it and both are lost (the same guard the claim logs use). + appendLine(join(dir, USAGE_FILE), JSON.stringify({ t, via, ids: list })); + } catch {} +} + +/** + * Claim id → the days it was served. Malformed lines are skipped. + * @param {string} dir + * @returns {Map} + */ +export function readUses(dir) { + /** @type {Map} */ + const out = new Map(); + let text = ""; + try { + text = readFileSync(join(dir, USAGE_FILE), "utf8"); + } catch { + return out; + } + for (const line of text.split("\n")) { + if (!line.trim()) continue; + let rec; + try { + rec = JSON.parse(line); + } catch { + continue; + } + if (!Number.isFinite(rec?.t) || !Array.isArray(rec?.ids)) continue; + for (const id of rec.ids) { + if (typeof id !== "string") continue; + if (!out.has(id)) out.set(id, []); + out.get(id)?.push(rec.t); + } + } + return out; +} + function readStateFromDisk(dir, verifyHashes) { const state = emptyState(); for (const { id, claim } of walkClaimFiles(dir)) { @@ -480,21 +555,40 @@ export function loadClaims(dir) { /** Find one claim by id prefix without scanning the whole ledger (ids are sharded by * their first two hex chars, so any prefix ≥ 2 chars pins the shard). An AMBIGUOUS prefix * (≥2 claims match) returns null — silently picking the first sorted match let a short - * prefix ratify or retract a claim nobody named. */ -export function getClaimByPrefix(dir, prefix) { + * prefix ratify or retract a claim nobody named. + * @param {string} dir + * @param {string} prefix + * @param {{attic?: boolean}} [opts] also look in the attic (read-only callers only); an + * archived hit carries `archived: true` */ +export function getClaimByPrefix(dir, prefix, { attic = false } = {}) { if (!prefix || prefix.length < 2) return null; const shardDir = join(dir, "claims", prefix.slice(0, 2)); - if (!existsSync(shardDir)) return null; - const matches = readdirSync(shardDir).filter((f) => f.endsWith(".json") && f.startsWith(prefix)); + const live = existsSync(shardDir) + ? readdirSync(shardDir) + .filter((f) => f.endsWith(".json") && f.startsWith(prefix)) + .map((f) => join(shardDir, f)) + : []; + // Read-only callers (show, blame) may also look in the attic: pruning archives a + // tombstoned claim at once, and the attic is its audit trail. Writers never do — new + // evidence on an archived claim goes through appendEvidence, which restores it. + const atticDir = join(dir, "attic"); + const archived = + attic && !live.length && existsSync(atticDir) + ? readdirSync(atticDir) + .filter((f) => f.endsWith(".json") && f.startsWith(prefix)) + .map((f) => join(atticDir, f)) + : []; + const matches = live.length ? live : archived; if (matches.length !== 1) return null; - const f = matches[0]; - const id = f.replace(/\.json$/, ""); - const claim = readJsonSafe(join(shardDir, f)); + const path = matches[0]; + const id = path.replace(/^.*[\\/]/, "").replace(/\.json$/, ""); + const claim = readJsonSafe(path); if (!claim || claimId(claim.kind, claim.body, claim.scope) !== id) return null; const state = emptyState(); state.claims[id] = { ...claim, id }; for (const log of LOGS) state[log][id] = readLog(dir, log, id); - return liveClaims(state)[0]; + const view = liveClaims(state)[0]; + return live.length ? view : { ...view, archived: true }; } /** Try to import one raw source log line into `dir`; returns {ok, deduped} on success or @@ -564,7 +658,7 @@ export function mergeDirs(dstDir, srcDir, { nowDay = epochDay() } = {}) { * trail: every channel the agent used can be questioned). */ export function blame(dir, prefix, nowDay = 0) { - const claim = getClaimByPrefix(dir, prefix); + const claim = getClaimByPrefix(dir, prefix, { attic: true }); if (!claim) return null; const trust = authorTrust(loadClaims(dir)); return { @@ -787,30 +881,52 @@ export function verify(dir) { } /** - * Prune to the attic — the spec's forgetting rule (01-pcm-protocol.md §3) made real: a claim - * is archived once it is tombstoned, or dormant, AND nothing new has landed on it for more - * than 2·T. (The spec prunes a tombstone immediately; waiting the same 2·T keeps - * `forge ledger show/blame` able to answer for a recent retraction — the attic is the audit - * trail, not a deletion.) Nothing is lost: the claim bytes move to attic/, every log stays, - * and new evidence un-archives the claim. Idempotent. + * Prune to the attic, by the retention plan LEARNED from this ledger (ledger_retention.js): + * a tombstoned or dormant claim is archived at once (retrieve() never serves it), and a live + * claim is archived once its idle time passes the cut-off the ledger's own history supports + * (none until the usage log covers a full learned horizon). This replaced a fixed 2 × 45-day + * window. Nothing is lost: the claim bytes move to attic/, every log stays, `forge ledger + * show/blame` still read the attic, and new evidence un-archives the claim. Idempotent. + * Near-duplicates are only grouped by `compactLedger` (an explicit command): the pairwise + * pass is too slow for the Stop hook that calls this. * @param {string} dir * @param {number} [nowDay] - * @param {{halfLife?:number}} [opts] - * @returns {{pruned:string[]}} ids archived by this pass + * @param {{halfLife?:number}} [opts] only feeds isDormant + * @returns {{pruned:string[], retention: ReturnType["retention"]}} */ export function pruneLedger(dir, nowDay = epochDay(), { halfLife = DEFAULT_HALF_LIFE_DAYS } = {}) { + const plan = retentionPlan(loadClaims(dir), readUses(dir), nowDay, { halfLife }); const pruned = []; - for (const c of loadClaims(dir)) { - const last = Math.max( - c.provenance?.t ?? 0, - c.tombstone?.t ?? 0, - ...(c.evidence ?? []).map((e) => e.t ?? 0), - ); - if (nowDay - last <= 2 * halfLife) continue; // still within the review window - if (!c.tombstone && !isDormant(c, nowDay, { halfLife })) continue; - if (pruneToAttic(dir, c.id).ok) pruned.push(c.id); - } - return { pruned }; + for (const { id } of plan.archive) if (pruneToAttic(dir, id).ok) pruned.push(id); + return { pruned, retention: plan.retention }; +} + +/** + * `forge ledger compact`: the prune plan plus near-duplicate grouping, with every learned + * number reported so a person can see why each claim was archived. `dryRun` plans only. + * @param {string} dir + * @param {number} [nowDay] + * @param {{dryRun?: boolean, halfLife?: number}} [opts] + */ +export function compactLedger( + dir, + nowDay = epochDay(), + { dryRun = false, halfLife = DEFAULT_HALF_LIFE_DAYS } = {}, +) { + const claims = loadClaims(dir); + const uses = readUses(dir); + const plan = retentionPlan(claims, uses, nowDay, { halfLife, duplicates: true }); + const archived = []; + if (!dryRun) for (const a of plan.archive) if (pruneToAttic(dir, a.id).ok) archived.push(a.id); + return { + dryRun, + claims: claims.length, + servedClaims: uses.size, + retention: plan.retention, + duplicates: plan.duplicates, + archive: plan.archive, + archived, + }; } /** Move one dormant/tombstoned claim file to the attic (audit trail, never retrieved). */ diff --git a/test/deja.test.js b/test/deja.test.js index 4a6170d..608bb9e 100644 --- a/test/deja.test.js +++ b/test/deja.test.js @@ -13,7 +13,7 @@ import { recordSessionSummary, } from "../src/deja.js"; import { mintClaim, val } from "../src/ledger.js"; -import { loadClaims, putClaim, repoLedger } from "../src/ledger_store.js"; +import { loadClaims, putClaim, readUses, repoLedger } from "../src/ledger_store.js"; const fixture = () => mkdtempSync(join(tmpdir(), "forge-deja-")); @@ -149,8 +149,13 @@ test("dejaAdvisory actually fires for a repeated task (DEJA_REL_FLOOR is inside ); const hit = dejaAdvisory(root, "add oauth login flow with pkce to the auth module", 200); assert.ok(hit.includes("déjà vu"), "a repeated task surfaces the advisory"); - const miss = dejaAdvisory(root, "optimize the image resizing pipeline for thumbnails", 200); + // A surfaced hit is a use of that claim (ledger retention learns from it)… + const [summary] = loadClaims(repoLedger(root)); + assert.deepEqual(readUses(repoLedger(root)).get(summary.id), [200]); + const miss = dejaAdvisory(root, "optimize the image resizing pipeline for thumbnails", 201); assert.equal(miss, "", "an unrelated task stays silent (below the noise floor)"); + // …and a silent miss is not. + assert.deepEqual(readUses(repoLedger(root)).get(summary.id), [200]); }); test("recordSessionSummary is best-effort and returns cleanly on an empty session", () => { diff --git a/test/ledger_retention.test.js b/test/ledger_retention.test.js new file mode 100644 index 0000000..dde3a72 --- /dev/null +++ b/test/ledger_retention.test.js @@ -0,0 +1,260 @@ +// Ledger retention and compaction learned from the ledger's own history (ledger_retention.js). +// The point of these tests is that NOTHING is a fixed threshold: the same code gives +// different cut-offs for ledgers with different rhythms, and refuses to act where the +// data cannot support a decision. +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { mintClaim } from "../src/ledger.js"; +import { + activityDays, + duplicateGroups, + learnIdleCutoff, + retentionPlan, + similarityBoundary, +} from "../src/ledger_retention.js"; +import { + compactLedger, + loadClaims, + loadState, + pruneLedger, + putClaim, + readUses, + recordUse, + repoLedger, + tombstone, + USAGE_FILE, +} from "../src/ledger_store.js"; + +const tmp = () => mkdtempSync(join(tmpdir(), "forge-retention-")); +const fact = (name, text, t = 0) => + mintClaim({ kind: "fact", body: { name, text }, provenance: { author: "tester" }, t }).claim; + +/** A ledger's history: `hot` claims used every `step` days until `end`, `cold` ones used + * twice and then abandoned. Scaling `step` scales the whole calendar. */ +function rhythm(step, { hot = 8, cold = 8, periods = 30 } = {}) { + const histories = []; + for (let i = 0; i < hot; i++) + histories.push({ days: Array.from({ length: periods }, (_, k) => k * step) }); + for (let i = 0; i < cold; i++) histories.push({ days: [0, step] }); + return { histories, nowDay: periods * step }; +} + +// ── The idle cut-off ────────────────────────────────────────────────────────────────── + +test("learnIdleCutoff: the cut-off follows the ledger's own rhythm (no fixed window)", () => { + const fast = rhythm(3); + const slow = rhythm(30); + const f = learnIdleCutoff(fast.histories, fast.nowDay, { usageSince: 0 }); + const s = learnIdleCutoff(slow.histories, slow.nowDay, { usageSince: 0 }); + assert.equal(f.learned, true, f.reason); + assert.equal(s.learned, true, s.reason); + assert.equal(f.cutoff, 3, "cut-off = the longest idle stretch a claim came back from"); + // Every quantity scaled by 10, so the learned cut-off scales by exactly 10. + assert.equal(s.cutoff, 10 * /** @type {number} */ (f.cutoff)); + assert.equal(s.typicalGap, 10 * /** @type {number} */ (f.typicalGap)); +}); + +test("learnIdleCutoff: a claim with a slower rhythm is never archived while it is due", () => { + // The flaw a fitted cut-off had: with a 3-day claim in the ledger it learned "idle > 2 ⇒ + // gone" and archived the 10-day claim — which, archived, could never be served again. + const histories = [ + { days: Array.from({ length: 30 }, (_, k) => k * 3) }, // every 3 days + { days: Array.from({ length: 9 }, (_, k) => k * 10) }, // every 10 days + { days: [0, 3] }, // abandoned + ]; + const r = learnIdleCutoff(histories, 90, { usageSince: 0 }); + assert.equal(r.learned, true); + assert.equal(r.cutoff, 10, "the slowest comeback sets the cut-off"); +}); + +test("learnIdleCutoff: refuses to act without the data to back it", () => { + const { histories, nowDay } = rhythm(3); + assert.match( + learnIdleCutoff(histories, nowDay, { usageSince: null }).reason ?? "", + /no usage recorded/, + ); + const once = histories.map((h) => ({ days: h.days.slice(0, 1) })); + assert.match(learnIdleCutoff(once, nowDay, { usageSince: 0 }).reason ?? "", /active twice/); + // The usage log is younger than the longest comeback: it could not yet have seen a reuse + // after that long, so "not used" would only mean "not recorded". + const young = learnIdleCutoff(histories, nowDay, { usageSince: nowDay - 1 }); + assert.equal(young.learned, false); + assert.match(young.reason ?? "", /longest comeback/); +}); + +test("activityDays: mint, evidence and served days, sorted and unique", () => { + const c = { id: "x", provenance: { t: 5 }, evidence: [{ t: 9 }, { t: 5 }] }; + assert.deepEqual(activityDays(c, [7, 9, 12]), [5, 7, 9, 12]); +}); + +// ── The plan ────────────────────────────────────────────────────────────────────────── + +test("retentionPlan: archives idle live claims only once a cut-off is learned", () => { + const { histories, nowDay } = rhythm(3); + // Claims whose only activity is use (no evidence), ids by index. + const claims = histories.map((_, i) => ({ id: `c${i}`, kind: "fact", body: { text: `t${i}` } })); + const uses = new Map(histories.map((h, i) => [`c${i}`, h.days])); + // `now` is the hot claims' due day (idle = the full 3-day gap): they must stay. + const plan = retentionPlan(claims, uses, nowDay); + assert.equal(plan.retention.learned, true); + const archived = new Set(plan.archive.map((a) => a.id)); + for (let i = 0; i < 8; i++) assert.ok(!archived.has(`c${i}`), `hot claim c${i} is kept`); + for (let i = 8; i < 16; i++) assert.ok(archived.has(`c${i}`), `abandoned claim c${i} goes`); + assert.match(plan.archive[0].reason, /learned cut-off/); + // With no usage log the same claims are all kept. + assert.equal(retentionPlan(claims, new Map(), nowDay).archive.length, 0); +}); + +test("retentionPlan: tombstoned claims are never served, so they are archived at once", () => { + const claims = [ + { id: "a", kind: "fact", body: { text: "a" }, provenance: { t: 99 } }, + { id: "b", kind: "fact", body: { text: "b" }, provenance: { t: 99 }, tombstone: { t: 99 } }, + ]; + const plan = retentionPlan(claims, new Map(), 100); + assert.deepEqual( + plan.archive.map((a) => a.id), + ["b"], + ); + assert.match(plan.archive[0].reason, /tombstoned/); +}); + +// ── Duplicates ──────────────────────────────────────────────────────────────────────── + +test("similarityBoundary: two separated groups give a boundary between them; one group none", () => { + const two = [0.02, 0.05, 0.03, 0.04, 0.06, 0.01, 0.05, 0.9, 0.95, 0.92]; + const fit = similarityBoundary(two); + assert.ok(fit.boundary != null && fit.boundary > 0.06 && fit.boundary < 0.9, `${fit.boundary}`); + assert.equal(similarityBoundary([0.1, 0.12, 0.09, 0.11, 0.1, 0.13, 0.08]).boundary, null); + assert.equal(similarityBoundary([0.1, 0.9, 0.95]).boundary, null, "too few points to fit two"); + // Exact duplicates (similarity 1) must not make the likelihood infinite. + const exact = similarityBoundary([0.02, 0.03, 0.01, 0.04, 0.02, 1, 1, 1]); + assert.ok(Number.isFinite(exact.bic1) && exact.boundary != null && exact.boundary < 1); +}); + +const TOPICS = [ + "the build uses esbuild with a custom plugin for svg imports", + "database migrations run in a single transaction per file", + "the staging deploy needs the vpn to reach the metrics endpoint", + "feature flags are read once at process start and cached", + "logging goes through pino and is shipped to loki every minute", + "image uploads are resized by a lambda before they hit the bucket", + "the cron worker locks jobs with redis to avoid double runs", + "api keys rotate every ninety days via the secrets operator", +]; + +test("duplicateGroups: near-duplicates collapse to one survivor; distinct facts are untouched", () => { + const dir = tmp(); + const base = "the payments service retries failed webhooks three times with backoff"; + const dupA = fact("wh1", base, 1); + const dupB = fact("wh2", `${base} and jitter`, 2); + const dupC = fact("wh3", `in production ${base}`, 3); + for (const c of [dupA, dupB, dupC, ...TOPICS.map((t, i) => fact(`t${i}`, t, 1))]) + putClaim(dir, c); + const r = duplicateGroups(loadClaims(dir), 10); + assert.ok(r.boundary != null, "the similarity distribution has a duplicate mode"); + assert.equal(r.groups.length, 1, JSON.stringify(r.groups)); + const g = r.groups[0]; + assert.deepEqual([g.keep, ...g.drop.map((d) => d.id)].sort(), [dupA.id, dupB.id, dupC.id].sort()); + assert.equal(g.keep, dupA.id, "equal val and evidence: the earliest minted survives"); + for (const d of g.drop) assert.ok(d.similarity >= /** @type {number} */ (r.boundary)); + // A ledger of distinct facts has no duplicate mode at all. + const clean = tmp(); + for (const [i, t] of TOPICS.entries()) putClaim(clean, fact(`t${i}`, t, 1)); + assert.deepEqual(duplicateGroups(loadClaims(clean), 10).groups, []); +}); + +test("duplicateGroups: a chain A–B–C drops only what is close to the survivor itself", () => { + // B contains A's text and C's text; A and C share nothing. Union-find links all three, but + // C is no duplicate of the survivor A, so only B goes. + const words = Array.from({ length: 14 }, (_, i) => `w${i}x`); + const claims = [ + { id: "a", kind: "fact", body: { text: words.slice(0, 8).join(" ") }, provenance: { t: 1 } }, + { id: "b", kind: "fact", body: { text: words.join(" ") }, provenance: { t: 2 } }, + { id: "c", kind: "fact", body: { text: words.slice(6).join(" ") }, provenance: { t: 3 } }, + ...TOPICS.map((t, i) => ({ + id: `t${i}`, + kind: "fact", + body: { text: t }, + provenance: { t: 1 }, + })), + ]; + const r = duplicateGroups(claims, 10); + assert.ok(r.boundary != null); + assert.equal(r.groups.length, 1, JSON.stringify(r.groups)); + assert.equal(r.groups[0].keep, "a"); + assert.deepEqual( + r.groups[0].drop.map((d) => d.id), + ["b"], + "c is not a duplicate of a — it stays", + ); +}); + +test("duplicateGroups: claims of different kinds are never grouped", () => { + const text = "always run the migrations before seeding the database"; + const claims = [ + { id: "f1", kind: "fact", body: { text } }, + { id: "l1", kind: "lesson", body: { text } }, + ]; + assert.deepEqual(duplicateGroups(claims, 0).groups, []); +}); + +// ── Store integration ───────────────────────────────────────────────────────────────── + +test("recordUse/readUses: a gitignored, append-only local log; never throws", () => { + const dir = tmp(); + putClaim(dir, fact("a", "alpha fact", 0)); + const [c] = loadClaims(dir); + const sigBefore = statSync(join(dir, ".state-cache.json")).mtimeMs; + recordUse(dir, [c.id, c.id, "", 42], { via: "test", t: 7 }); + recordUse(dir, [c.id], { via: "test", t: 9 }); + writeFileSync(join(dir, USAGE_FILE), `${readFileSync(join(dir, USAGE_FILE), "utf8")}{broken\n`); + assert.deepEqual(readUses(dir).get(c.id), [7, 9], "deduped per call, malformed lines skipped"); + assert.match(readFileSync(join(dir, ".gitignore"), "utf8"), /^\.usage\.jsonl$/m, "gitignored"); + // Logging use does not invalidate the snapshot cache (it is outside the signature). + loadState(dir); + assert.equal(statSync(join(dir, ".state-cache.json")).mtimeMs, sigBefore); + assert.doesNotThrow(() => recordUse(join(dir, "missing"), ["x"])); + assert.equal(existsSync(join(dir, "missing")), false, "a missing ledger records nothing"); +}); + +test("pruneLedger + compactLedger: learned from the usage log, reversible, dry run writes nothing", () => { + const root = tmp(); + const dir = repoLedger(root); + const hot = fact("hot", "the api gateway strips x-forwarded-host", 0); + const cold = fact("cold", "the old admin panel lived under /legacy", 0); + const gone = fact("gone", "retracted belief", 0); + for (const c of [hot, cold, gone]) putClaim(dir, c); + tombstone(dir, gone.id, { author: "alice", reason: "wrong", t: 1 }); + // Both served early on; only `hot` keeps being served, every 3 days. + for (let d = 0; d <= 60; d += 3) recordUse(dir, [hot.id], { via: "test", t: d }); + recordUse(dir, [cold.id], { via: "test", t: 3 }); + const dry = compactLedger(dir, 61, { dryRun: true }); + assert.equal(dry.retention.learned, true, dry.retention.reason); + assert.deepEqual(dry.archive.map((a) => a.id).sort(), [cold.id, gone.id].sort()); + assert.deepEqual(dry.archived, [], "dry run"); + assert.equal(loadClaims(dir).length, 3, "dry run wrote nothing"); + const { pruned } = pruneLedger(dir, 61); + assert.deepEqual(pruned.sort(), [cold.id, gone.id].sort()); + assert.deepEqual( + loadClaims(dir).map((c) => c.id), + [hot.id], + ); +}); + +test("recordServedLessons: logs ledger-backed lessons, skips legacy ones", async () => { + const { recordServedLessons } = await import("../src/cortex.js"); + const root = tmp(); + const dir = repoLedger(root); + putClaim(dir, fact("x", "a fact so the ledger directory exists", 0)); + recordServedLessons( + root, + [{ provenance: { claim: "c-ledger" } }, { provenance: {} }, { id: "legacy-only" }], + { via: "pre-edit", t: 12 }, + ); + const uses = readUses(dir); + assert.deepEqual([...uses.keys()], ["c-ledger"]); + assert.deepEqual(uses.get("c-ledger"), [12]); +}); diff --git a/test/ledger_store.test.js b/test/ledger_store.test.js index e17f8cc..2cc190c 100644 --- a/test/ledger_store.test.js +++ b/test/ledger_store.test.js @@ -627,7 +627,7 @@ test("mergeDirs: imported forged/unresolvable evidence is quarantined and cannot assert.equal(val(loadClaims(dst)[0], 5), before, "val still untouched after re-merge"); }); -test("pruneLedger (C7): tombstoned and long-dormant claims go to the attic, new evidence brings them back", () => { +test("pruneLedger (C7): never-served claims go to the attic at once, new evidence brings them back", () => { // A real repo: a human.revert must cite a git object that resolves here (review C2). const root = mkdtempSync(join(tmpdir(), "forge-prune-")); const g = (...args) => execFileSync("git", args, { cwd: root, stdio: "ignore" }); @@ -652,12 +652,21 @@ test("pruneLedger (C7): tombstoned and long-dormant claims go to the attic, new assert.equal(appendEvidence(dir, recent.id, revert(now - 1)).ok, true); tombstone(dir, retracted.id, { author: "alice", reason: "superseded", t: 0 }); - const { pruned } = pruneLedger(dir, now); - assert.deepEqual(pruned.sort(), [refuted.id, retracted.id].sort()); + // Tombstoned and dormant claims are never served (retrieve() skips them), so they go now — + // the recently refuted one included; the old fixed 2 × 45-day window is gone. The live + // claim stays: no use has been logged, so no idle cut-off could be learned. + const { pruned, retention } = pruneLedger(dir, now); + assert.deepEqual(pruned.sort(), [refuted.id, retracted.id, recent.id].sort()); + assert.equal(retention.learned, false, "no usage log → nothing learned about live claims"); const ids = loadClaims(dir).map((c) => c.id); - assert.ok(!ids.includes(refuted.id) && !ids.includes(retracted.id), "archived, not retrieved"); - assert.ok(ids.includes(live.id) && ids.includes(recent.id), "live and recently-refuted stay"); + assert.deepEqual(ids, [live.id], "only the live claim is still retrieved"); assert.ok(existsSync(join(dir, "attic", `${refuted.id}.json`)), "the bytes are kept for audit"); + // show/blame still answer for an archived claim: the attic is its audit trail. + const audit = blame(dir, recent.id.slice(0, 10), now); + assert.ok(audit, "blame reads the attic"); + assert.equal(audit.evidence.length, 1, "with its evidence"); + assert.equal(getClaimByPrefix(dir, recent.id.slice(0, 10)), null, "writers never see the attic"); + assert.equal(getClaimByPrefix(dir, recent.id.slice(0, 10), { attic: true })?.archived, true); assert.deepEqual(pruneLedger(dir, now).pruned, [], "idempotent"); // Re-importing the same state must not resurrect a pruned claim… importState(dir, loadState(dir), { nowDay: now }); diff --git a/test/ledger_usage.test.js b/test/ledger_usage.test.js new file mode 100644 index 0000000..613d4ff --- /dev/null +++ b/test/ledger_usage.test.js @@ -0,0 +1,94 @@ +// Every place that SERVES ledger claims logs the use (ledger retention learns from it). This +// drives the real entrypoints — hook, CLI, MCP server — in the default ledger-only mode, where +// lessons live in the ledger and carry their claim id. +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import { processSession } from "../src/cortex_hook.js"; +import { loadClaims, readUses, repoLedger } from "../src/ledger_store.js"; +import { epochDay } from "../src/util.js"; + +delete process.env.FORGE_LEDGER_ONLY; // the default: ledger-only + +const HOOK = fileURLToPath(new URL("../src/cortex_hook_main.js", import.meta.url)); +const CLI = fileURLToPath(new URL("../src/cli.js", import.meta.url)); +const SERVER = fileURLToPath(new URL("../src/cortex_mcp.js", import.meta.url)); + +const session = () => [ + { type: "bash", command: "npm test", exitCode: 1 }, + { type: "edit", file: "src/tax.ts" }, + { type: "edit", file: "src/tax.ts" }, + { type: "edit", file: "src/tax.ts" }, + { type: "bash", command: "npm test", exitCode: 0 }, +]; + +/** Total use events per claim id, summed over the whole log. */ +const useCount = (dir, ids) => ids.reduce((n, id) => n + (readUses(dir).get(id)?.length ?? 0), 0); + +test("session-start, pre-edit, `ledger query` and the MCP query each log what they served", () => { + const root = mkdtempSync(join(tmpdir(), "forge-usage-")); + // Recent days: the hooks judge a ledger lesson's status by its decayed val TODAY. + const today = epochDay(); + processSession(root, session(), today - 1); + processSession(root, session(), today); // → an active lesson on src/tax.ts, in the ledger + const dir = repoLedger(root); + const lessons = loadClaims(dir) + .filter((c) => c.kind === "lesson") + .map((c) => c.id); + assert.ok(lessons.length, "the seeded lesson is a ledger claim"); + assert.equal(useCount(dir, lessons), 0, "nothing served yet"); + + const start = spawnSync("node", [HOOK, "session-start"], { + input: JSON.stringify({ session_id: "s-usage", cwd: root }), + encoding: "utf8", + timeout: 20000, + }); + assert.equal(start.status, 0, start.stderr); + assert.match(start.stdout, /Lessons learned on this repo/); + const afterStart = useCount(dir, lessons); + assert.ok(afterStart >= 1, "the session-start lesson block logged its lessons"); + + const pre = spawnSync("node", [HOOK, "pre-edit"], { + input: JSON.stringify({ cwd: root, tool_input: { file_path: "src/tax.ts" } }), + encoding: "utf8", + timeout: 20000, + }); + assert.equal(pre.status, 0, pre.stderr); + assert.match(pre.stdout, /tax\.ts/); + const afterPre = useCount(dir, lessons); + assert.ok(afterPre > afterStart, "the pre-edit advisory logged the lesson it showed"); + + const query = spawnSync("node", [CLI, "ledger", "query", "tax.ts tests fail"], { + cwd: root, + encoding: "utf8", + env: { ...process.env, FORGE_NO_HINT: "1" }, + timeout: 20000, + }); + assert.equal(query.status, 0, query.stderr); + const afterQuery = useCount(dir, lessons); + assert.ok(afterQuery > afterPre, "`forge ledger query` logged its results"); + + const requests = [ + { jsonrpc: "2.0", id: 1, method: "initialize", params: {} }, + { + jsonrpc: "2.0", + id: 2, + method: "tools/call", + params: { name: "forge_ledger_query", arguments: { query: "tax.ts tests fail" } }, + }, + ] + .map((r) => JSON.stringify(r)) + .join("\n"); + const mcp = spawnSync("node", [SERVER], { + input: `${requests}\n`, + encoding: "utf8", + env: { ...process.env, FORGE_ROOT: root }, + timeout: 20000, + }); + assert.equal(mcp.status, 0, mcp.stderr); + assert.ok(useCount(dir, lessons) > afterQuery, "the MCP ledger query logged its results"); +});