From 489a4fa4200e213f7e304592d187921dcbe5a905 Mon Sep 17 00:00:00 2001 From: Juber Shaikh Date: Tue, 22 Sep 2026 04:15:05 +0000 Subject: [PATCH 01/13] fix(commit-gate): scan binary files with the format grammars only The staged secret scan reads every file with `git diff --text`, and the entropy leg of hasSecret flagged the XMP packet id W5M0MpCehiHzreSzNTczkc9d that the XMP packet wrapper carries inside PDFs, JPEGs and PNGs, so ordinary binary commits were refused. - A staged file that git reports as binary (`--numstat` prints `-\t-`) AND whose added bytes contain a NUL (git's own content test) is scanned with SECRET_RE / CASE_RE only. A `binary` attribute on a text file does not qualify, so .gitattributes cannot switch the entropy leg off. - The XMP packet id is a public constant, exempt from the entropy leg like lockfile integrity digests (whole-token match only). - hasSecret / redactSecrets take `{ entropy: false }`; defaults unchanged. The unscanned-file fail-closed path is untouched. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JpQ15QdqdDUJLavNhExwvL --- CHANGELOG.md | 12 +++++++ docs/GUIDE.md | 2 +- src/commit_gate.js | 52 +++++++++++++++++++++++++-- src/secrets.js | 25 +++++++++---- test/commit_gate.test.js | 76 ++++++++++++++++++++++++++++++++++++++++ test/secrets.test.js | 22 ++++++++++++ 6 files changed, 180 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5d9a15..fe232df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Fixed + +- **Binary files no longer trip the commit gate's secret scan.** The staged scan reads every + file with `git diff --text`, and the entropy leg flagged the XMP packet id + (`W5M0MpCehiHzreSzNTczkc9d`, a constant fixed by Adobe's XMP spec) that the XMP packet + wrapper carries inside PDFs, JPEGs and PNGs, so ordinary binary commits were refused. A staged file that git + reports as binary (`--numstat` prints `-`/`-`) and that contains a NUL byte now gets the + credential-format grammars only; a `binary` attribute on a text file does not qualify, so + `.gitattributes` cannot switch the entropy leg off. The XMP packet id is also exempt from the + entropy leg everywhere, like lockfile integrity digests. A `ghp_…` token inside a binary is + still refused, and an unreadable diff still fails closed. + ## [1.0.0] - 2026-09-22 ### Added diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 8113326..d30d8c2 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -1207,7 +1207,7 @@ Plain `forge cost` remains the per-day spend view via `ccusage`. | `forge cost` | Real per-day spend (via `ccusage`) + the cost ceiling; `--stages` for the measured report. | | `forge scan ` | Vet a skill/MCP (SKILL.md/.mcp.json) for injection/RCE/exfil before install. | | `forge harden` | Wire the pre-commit gate (gitleaks-if-present + `forge precommit`) + sandbox settings; never clobbers a user-authored hook. | -| `forge precommit` | Commit-level gate rung: staged code with no doc/state artifact → finding (same classifier as the Stop gate) + built-in secret scan over staged added lines. `FORGE_COMMIT_GATE=block` refuses the commit, `warn` (default) prints and allows, `0` disables; a detected secret blocks in every mode. | +| `forge precommit` | Commit-level gate rung: staged code with no doc/state artifact → finding (same classifier as the Stop gate) + built-in secret scan over staged added lines (binary files get the credential-format grammars only, not the entropy leg). `FORGE_COMMIT_GATE=block` refuses the commit, `warn` (default) prints and allows, `0` disables; a detected secret blocks in every mode. | | `forge spec [init\|lock\|check]` | Spec-as-contract drift check. | | `forge brand` | Print the active brand token map. | | `forge lean ""` | Scope-minimality footprint for a task — advisory (the Lean Path as a command). | diff --git a/src/commit_gate.js b/src/commit_gate.js index 6e4eaf0..5809a69 100644 --- a/src/commit_gate.js +++ b/src/commit_gate.js @@ -11,6 +11,8 @@ // the same commit is a finding. // (ii) SECRETS — hasSecret (secrets.js) over the staged ADDED lines only (context // lines predate the commit), as the built-in fallback when gitleaks is absent. +// A BINARY file gets the format grammars only: its bytes are compressed or random +// by nature, so the entropy leg would measure the encoding, not a credential. // // Modes via FORGE_COMMIT_GATE: "warn" (default — print findings, allow), "block" // (completeness findings refuse the commit), "0"/"off" (kill switch). A secret finding @@ -36,7 +38,11 @@ import { IGNORE_DIRS } from "./util.js"; // So detection is confirmed by the same module's NARROWER redaction rules (format // grammars, PEM, entropy tokens, opaque assigned literals — never a code expression): // one source of truth (secrets.js), calibrated to the verb (mizan). -const lineBlockSecret = (text) => hasSecret(text) && redactSecrets(text) !== text; +/** + * @param {string} text + * @param {{entropy?: boolean}} [opts] + */ +const lineBlockSecret = (text, opts) => hasSecret(text, opts) && redactSecrets(text, opts) !== text; // Exact bytes, no trim — same discipline as gate.js's gitRaw. `gitStrict` THROWS on a git // error or an over-large output (ENOBUFS) — the secret scan turns that into an unscanned @@ -69,7 +75,11 @@ function gitRaw(root, args) { // `--text` defeats a `.gitattributes` `-diff`/`binary` marking (which printed "Binary // files differ" and hid every added line), `--no-textconv` a `diff=` textconv // that rewrites what is shown, `--no-ext-diff` a configured external diff tool. +// `core.quotePath=false` keeps non-ASCII header paths raw, so they match the `-z` paths of +// the binary probe below (a path that still differs only loses the binary relaxation). const DIFF_ARGS = [ + "-c", + "core.quotePath=false", "--literal-pathspecs", "diff", "--cached", @@ -150,6 +160,39 @@ export function scanStagedAdded(root) { return { byFile, unscanned }; } +/** + * Staged paths git itself treats as BINARY (`--numstat` prints `-\t-` for them). A repo can + * mark a text file binary with `.gitattributes`, so the caller also requires content + * evidence (a NUL byte, git's own binary test) before relaxing the scan. Best effort: if + * git fails the set is empty and every file gets the full scan — the strict direction. + * @param {string} root + * @returns {Set} + */ +export function stagedBinaryFiles(root) { + const out = new Set(); + const tokens = gitRaw(root, [ + "--literal-pathspecs", + "diff", + "--cached", + "--numstat", + "-z", + "--no-ext-diff", + "--no-textconv", + ]).split("\0"); + for (let i = 0; i < tokens.length; i++) { + const m = /^(-|\d+)\t(-|\d+)\t(.*)$/s.exec(tokens[i]); + if (!m) continue; + // `-z` rename/copy record: `added\tdeleted\t` NUL old NUL new — the new path is staged. + let path = m[3]; + if (path === "") { + path = tokens[i + 2] ?? ""; + i += 2; + } + if (m[1] === "-" && m[2] === "-" && path) out.add(path); + } + return out; +} + /** * Parse a `--unified=0` diff into the added lines of each file. * @param {string} raw @@ -278,8 +321,13 @@ export function commitGate(root, { env = process.env } = {}) { }; const secretFiles = []; const { byFile, unscanned } = scanStagedAdded(root); + const binary = stagedBinaryFiles(root); for (const [file, lines] of byFile) { - if (lineBlockSecret(lines.join("\n"))) secretFiles.push(file); + const text = lines.join("\n"); + // Binary per git AND per content: a `binary` attribute on a text file (no NUL) + // keeps the entropy leg, so .gitattributes cannot switch it off. + const isBinary = binary.has(file) && text.includes("\0"); + if (lineBlockSecret(text, { entropy: !isBinary })) secretFiles.push(file); } return { ...commitGateDecision({ staged, secretFiles, unscanned, mode }), diff --git a/src/secrets.js b/src/secrets.js index efc499d..3051cc7 100644 --- a/src/secrets.js +++ b/src/secrets.js @@ -89,7 +89,13 @@ const TOKEN_RE = /[A-Za-z0-9+=_-]{20,}/g; // leg flagged 90-100% of them, so every lockfile commit was refused. Such a digest is // consumed whole (group 1) and never scored; format grammars still apply to it. const INTEGRITY = "\\b(?:sha(?:1|256|384|512)-[A-Za-z0-9+/]{16,}={0,2}|h1:[A-Za-z0-9+/]{43}=)"; -const ENTROPY_SCAN_G = new RegExp(`(${INTEGRITY})|${TOKEN_RE.source}`, "g"); +// Published constants that clear the entropy bar by accident. The XMP packet wrapper id +// `W5M0MpCehiHzreSzNTczkc9d` is fixed by Adobe's XMP specification and written verbatim +// into every XMP packet wrapper (PDF, JPEG, PNG, TIFF metadata), so it is no more a secret +// than a lockfile digest. Matched as a WHOLE token only (the lookarounds use TOKEN_RE's class): +// a longer run that merely contains it is still scored as one token. +const PUBLIC_CONSTANT = "(? val.startsWith("/") && !isHighEntropyToken(val.replaceAll("/", "")) ? m : `${key}[REDACTED]`, ); - s = s.replace(ENTROPY_SCAN_G, (t, integrity) => - !integrity && isHighEntropyToken(t) ? "[REDACTED]" : t, + if (!entropy) return s; + s = s.replace(ENTROPY_SCAN_G, (t, exempt) => + !exempt && isHighEntropyToken(t) ? "[REDACTED]" : t, ); return s; } diff --git a/test/commit_gate.test.js b/test/commit_gate.test.js index 268eaaa..9e341af 100644 --- a/test/commit_gate.test.js +++ b/test/commit_gate.test.js @@ -11,6 +11,7 @@ import { gateMode, renderCommitGate, stagedAddedLines, + stagedBinaryFiles, stagedFiles, } from "../src/commit_gate.js"; import { fakeGithubPat } from "./_fixtures.js"; @@ -298,3 +299,78 @@ test("a lockfile integrity line passes the commit gate (B4)", () => { assert.equal(r.allow, true, renderCommitGate(r)); assert.equal(r.findings.filter((f) => f.kind.startsWith("secret")).length, 0); }); + +// ── Binary files: the staged scan reads every file with `--text`, and the entropy leg +// flagged the XMP packet id that every PDF/JPEG/PNG with XMP metadata carries, so ordinary +// binary commits were refused. Binary files now get the format grammars only. +const XMP_ID = "W5M0MpCehiHzreSzNTczkc9d"; +/** A small PDF-shaped binary: an XMP packet plus a compressed stream holding NUL bytes. */ +const binaryPdf = (extra = "") => { + const xmp = `\n\n`; + return Buffer.concat([ + Buffer.from( + `%PDF-1.7\n%\xE2\xE3\xCF\xD3\n1 0 obj\n<< /Type /Metadata /Subtype /XML >>\nstream\n${xmp}\nendstream\nendobj\n2 0 obj\n<< /Length 8 /Filter /FlateDecode >>\nstream\n`, + "latin1", + ), + Buffer.from([0x78, 0x9c, 0x00, 0x01, 0xff, 0x00, 0x10, 0x0a]), + Buffer.from(`${extra}\nendstream\nendobj\n%%EOF\n`, "latin1"), + ]); +}; + +test("a staged XMP-bearing binary PDF is allowed (git reports it binary)", () => { + const { root, git } = gitFixture(); + writeFileSync(join(root, "report.pdf"), binaryPdf()); + git("add", "report.pdf"); + assert.ok(stagedBinaryFiles(root).has("report.pdf"), "git's numstat marks the PDF binary"); + const r = commitGate(root, { env: env() }); + assert.equal(r.allow, true, renderCommitGate(r)); + assert.equal(r.findings.filter((f) => f.kind.startsWith("secret")).length, 0); + assert.equal(cli(root).status, 0); +}); + +test("a binary file holding a credential format is still refused", () => { + const { root, git } = gitFixture(); + writeFileSync(join(root, "leak.pdf"), binaryPdf(`/Token (${fakeGithubPat()})`)); + git("add", "leak.pdf"); + const r = commitGate(root, { env: env() }); + assert.equal(r.allow, false, "format grammars still apply to binary content"); + assert.ok(r.findings.some((f) => f.kind === "secret" && f.files.includes("leak.pdf"))); + assert.equal(cli(root).status, 1); +}); + +test("binary scope: the entropy leg is skipped for binaries, kept for text files", () => { + const unknown = ["Zq7Rt2", "Xk9Lp4", "Vm1Nc8", "Yb5Ws3", "Hd6Fg0"].join(""); + // A random-looking run inside binary bytes is not refused (it is what compression looks + // like); the same run in a text file is. + const bin = gitFixture(); + writeFileSync(join(bin.root, "blob.pdf"), binaryPdf(unknown)); + bin.git("add", "blob.pdf"); + assert.equal(commitGate(bin.root, { env: env() }).allow, true); + const txt = gitFixture(); + writeFileSync(join(txt.root, "cfg.txt"), `key ${unknown}\n`); + txt.git("add", "cfg.txt"); + assert.equal(commitGate(txt.root, { env: env() }).allow, false); +}); + +test("a `binary` attribute on a text file does not switch the entropy leg off", () => { + const unknown = ["Zq7Rt2", "Xk9Lp4", "Vm1Nc8", "Yb5Ws3", "Hd6Fg0"].join(""); + const { root, git } = gitFixture(); + writeFileSync(join(root, ".gitattributes"), "*.txt binary\n"); + writeFileSync(join(root, "cfg.txt"), `key ${unknown}\n`); + git("add", "-A"); + assert.ok(stagedBinaryFiles(root).has("cfg.txt"), "git reports it binary by attribute"); + const r = commitGate(root, { env: env() }); + assert.equal(r.allow, false, "no NUL byte, so it is scanned as text"); + assert.ok(r.findings.some((f) => f.kind === "secret" && f.files.includes("cfg.txt"))); +}); + +test("an XMP packet in a text sidecar passes too (public constant)", () => { + const { root, git } = gitFixture(); + writeFileSync( + join(root, "photo.xmp"), + `\n\n\n`, + ); + git("add", "photo.xmp"); + const r = commitGate(root, { env: env() }); + assert.equal(r.allow, true, renderCommitGate(r)); +}); diff --git a/test/secrets.test.js b/test/secrets.test.js index 4461cb8..8f01eb7 100644 --- a/test/secrets.test.js +++ b/test/secrets.test.js @@ -294,3 +294,25 @@ test("hasSecret/redactSecrets: lockfile / SRI / go.sum integrity digests are not // The exemption is shape-bound: a real token beside a digest is still caught. assert.ok(hasSecret(`${leftPad} ${fakeGithubPat()}`)); }); + +// ── The XMP packet id is a published constant (Adobe XMP spec), present in every +// PDF/JPEG/PNG with XMP metadata; the entropy leg flagged it, refusing binary commits. +test("hasSecret/redactSecrets: the XMP packet id is a public constant, not a secret", () => { + const xmp = "W5M0MpCehiHzreSzNTczkc9d"; + assert.ok(isHighEntropyToken(xmp), "it clears the entropy bar on its own — hence the exemption"); + const packet = ``; + assert.equal(hasSecret(packet), false); + assert.equal(redactSecrets(packet), packet); + // Whole-token only: a longer run that merely contains the constant is still scored. + assert.ok(hasSecret(`x ${xmp}Qx7Lp2`)); + assert.ok(hasSecret(`${packet} ${fakeGithubPat()}`), "a real token beside it is still caught"); +}); + +test("hasSecret/redactSecrets {entropy:false}: format grammars only", () => { + const tok = fakeUnknownVendor(); + assert.equal(hasSecret(tok, { entropy: false }), false, "entropy-only token passes"); + assert.equal(redactSecrets(tok, { entropy: false }), tok); + assert.ok(hasSecret(fakeGithubPat(), { entropy: false }), "format grammar still applies"); + assert.notEqual(redactSecrets(fakeGithubPat(), { entropy: false }), fakeGithubPat()); + assert.ok(hasSecret(tok), "the default keeps the entropy leg"); +}); From 33be6de6788549d29addeca62cfafef69cd257b1 Mon Sep 17 00:00:00 2001 From: Juber Shaikh Date: Tue, 22 Sep 2026 04:17:53 +0000 Subject: [PATCH 02/13] docs(gate): drop the claim that repeated gates multiply catch rates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The formal synthesis (§5.3, corrected 2026-09-21) withdrew the claim that running the same check at Stop, pre-commit and CI multiplies catch rates: on the same diff the copies fire together, so they are nested checks and the residual is (1−p)(1−c_max), not (1−p)·∏(1−cⱼ). Rewrites the headers of src/commit_gate.js and src/gate.js, the two ARCHITECTURE.md passages (§5 intro and the commit-boundary gate), and the Mintlify verification-gates intro to the corrected law, and states what the commit rung does add: catches for edits after the turn, hosts or sessions where the Stop hook never ran, and sessions whose one Stop block was spent. The gate.js header no longer calls the gate a cⱼ≈1 layer; its catch rate on real misses is unmeasured. No behaviour change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JpQ15QdqdDUJLavNhExwvL --- ARCHITECTURE.md | 15 +++++++++++---- CHANGELOG.md | 11 +++++++++++ mintlify/concepts/verification-gates.mdx | 8 +++++--- src/commit_gate.js | 13 ++++++++++--- src/gate.js | 10 +++++++--- 5 files changed, 44 insertions(+), 13 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f54cea9..d6506cc 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -200,8 +200,12 @@ to decide whether an edit is safe to start. Surface: `forge reuse query | mint | Two failure modes this layer exists to kill: **partial work** (code changes without the artifacts that depend on it) and **session amnesia** (the next session re-assumes what this one knew). Instructions raise the _probability_ of correct behavior; deterministic -hooks guarantee a _floor_ — with per-task miss rate `1−p` and gate catch rate `c`, -silent misses fall to `(1−p)(1−c)`, and every layer here is one more `c`. +hooks guarantee a _floor_ — with per-task miss rate `1−p`, silent misses fall to +`(1−p)·P(no check fires | miss)`: `(1−p)(1−c)` for one check with catch rate `c`. A second +check lowers that only where it catches what the first cannot; the product `∏(1−cⱼ)` holds +only if the checks fire independently. The same check repeated at another point (Stop, +pre-commit, CI on the same diff) is nested, so the residual is `(1−p)(1−c_max)` (formal +synthesis §5.3, corrected 2026-09-21). **The completion gate (Stop, `src/gate.js`).** The only Stop-path guard that may answer: `completion-gate.sh` runs synchronously (the lesson-mining `cortex.sh stop` stays @@ -332,8 +336,11 @@ the gate lattice (turn ⊂ commit ⊂ PR): the Stop hook gates the turn and CI's gates the PR, so this runs the SAME registry-derived completeness classifier (`classifyPath` from `gate.js`) plus `hasSecret` over staged added lines at the commit boundary — code staged without its doc/state artifact, or a staged secret, is caught -while the fix is still one `git add` away. Each rung is an independent catch layer, so -the silent-miss probability falls multiplicatively. +while the fix is still one `git add` away. The rungs are **not** independent catch +layers: on the same diff the copies fire together, so they do not multiply the catch rate +and the residual stays `(1−p)(1−c_max)`. This rung adds catches only where it sees what the +Stop hook could not — edits made after the turn ended, a host or session where the Stop +hook never ran, or a session whose one Stop block was already spent. **Deep verification (`src/consensus.js`, `forge verify --deep`).** Where plain `verify` asks one oracle (the tests) plus one heuristic, this runs a table of independent lenses diff --git a/CHANGELOG.md b/CHANGELOG.md index fe232df..90baf9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,17 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). entropy leg everywhere, like lockfile integrity digests. A `ghp_…` token inside a binary is still refused, and an unreadable diff still fails closed. +### Changed + +- **The gate docs no longer claim that repeated gates multiply their catch rates.** The + headers of `src/commit_gate.js` and `src/gate.js`, ARCHITECTURE.md §5 and the Mintlify + verification-gates page said each rung (Stop, pre-commit, CI) was an independent catch + layer, so the silent-miss probability fell multiplicatively. The formal synthesis withdrew + that (§5.3, corrected 2026-09-21): the same classifier run on the same diff fires together, + so the residual is `(1−p)(1−c_max)`, and a later rung adds catches only where it sees what + the earlier one could not (edits after the turn, a host where the Stop hook never ran). + Comments and docs only; no behaviour change. + ## [1.0.0] - 2026-09-22 ### Added diff --git a/mintlify/concepts/verification-gates.mdx b/mintlify/concepts/verification-gates.mdx index c00b610..159af18 100644 --- a/mintlify/concepts/verification-gates.mdx +++ b/mintlify/concepts/verification-gates.mdx @@ -4,9 +4,11 @@ description: "Independent verification, the hallucinated-symbol flag, spec-as-co --- Nothing is "done" without a check you can run — a test, a build exit code, a screenshot. -Forge's verification gates each add one more catch. With per-task miss rate `1 − p` and a -gate catch rate `c`, silent misses fall to `(1 − p)(1 − c)`, and every gate here is one -more `c`. +With per-task miss rate `1 − p`, silent misses fall to `(1 − p)` times the chance that no +check fires on the miss: `(1 − p)(1 − c)` for one gate with catch rate `c`. A further gate +lowers that only where it catches something the others cannot. The same check repeated at +another point (Stop hook, pre-commit, CI on the same diff) fires together with the first, +so the residual stays `(1 − p)(1 − c_max)` rather than a product. **Verification reduces, does not certify.** Crew verifiers and the hallucinated-symbol diff --git a/src/commit_gate.js b/src/commit_gate.js index 5809a69..d8e5c8e 100644 --- a/src/commit_gate.js +++ b/src/commit_gate.js @@ -1,9 +1,16 @@ // forge precommit — the commit-level rung of the gate lattice (turn ⊂ commit ⊂ PR). // The Stop hook gates the TURN and CI's docs check gates the PR; this module runs the // same F1 classifier at the commit boundary so a commit that ships code without its -// doc/state artifact is caught while the fix is still one `git add` away. Same math as -// the paper's Theorem D: each rung is an independent cⱼ layer over the identical -// structural signal, so P(silent miss) falls multiplicatively, not by hope. +// doc/state artifact is caught while the fix is still one `git add` away. +// What this rung does NOT buy (formal synthesis §5.3, corrected 2026-09-21): the rungs +// are not independent. On the same diff the copies of one classifier fire together or +// not at all, so they are nested checks and the residual is (1−p)(1−c_max), not +// (1−p)·∏(1−cⱼ) — repeating a check does not multiply its catch rate. The rung adds +// catches only where it sees what the Stop hook could not: edits made after the turn +// ended, a host or session where the Stop hook never ran (no hook support, +// FORGE_STOPGATE=0, a human committing), or a session whose one Stop block was already +// spent. That is also the portability argument: without hooks, the same check re-binds +// at pre-commit without changing the math. // // Two detectors, both reused — never reimplemented: // (i) COMPLETENESS — classifyPath (gate.js), the same registry-derived total function diff --git a/src/gate.js b/src/gate.js index 4316952..869bce2 100644 --- a/src/gate.js +++ b/src/gate.js @@ -3,9 +3,13 @@ // guarantees a floor: a session that changed code but produced no TEST EVIDENCE (a test // file moved, or a fresh passing `verify` provenance stamp) or moved no doc/state // artifact is blocked ONCE, with the exact repair procedure as the reason. P(silent miss) = -// (1−p)·∏(1−cⱼ) — the gate is the cⱼ≈1 layer for the structural signal "code moved, -// nothing followed". Loop-safe (stop_hook_active + once-per-session marker), fail-open -// on every error path, kill switch FORGE_STOPGATE=0. +// (1−p)·P(no check fires | miss) (formal synthesis Theorem D, corrected 2026-09-21). On +// its proxy, "code moved, nothing followed", the first stop fires exactly (T3). Its catch +// rate on real misses depends on the agent (touching state.md satisfies the docs leg, and +// the block fires once per session) and has not been measured. The same classifier re-run +// at pre-commit or in CI is a NESTED check on the same diff: (1−p)(1−c_max), not a +// product. Loop-safe (stop_hook_active + once-per-session marker), fail-open on every +// error path, kill switch FORGE_STOPGATE=0. // // Classification derives from the SAME registries the atlas is built from (CODE_EXTS/ // DOC_EXTS/config rules) + the shared test-file predicate — no parallel regex lists that From d4b437be10de4f57a6ab73984c871f30ea529bd8 Mon Sep 17 00:00:00 2001 From: Juber Shaikh Date: Tue, 22 Sep 2026 04:21:49 +0000 Subject: [PATCH 03/13] fix(handoff): one byte budget shared by the handoff writer and loader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit writeState bounded the snapshot at 150 lines while stateBlock injected only 80 at SessionStart, so rows 81-150 of a valid handoff were silently dropped: the A4/A5 budget mismatch the formal synthesis's T4 correction names, reproduced in lines. - STATE_BUDGET_BYTES (8192, the synthesis's A5 cap) is the single budget, in one unit (UTF-8 bytes of the snapshot body), used by both sides. - selectSnapshot keeps rows in A4 priority order (goal + criteria, next, decisions, gotchas + assumptions, in-progress, done) until the body fits; every header stays, and a cut section ends with "(+N more not kept …)". Sections are written in that order, so even a loader cut of a hand-edited file drops the least important rows first. - stateBlock applies the same budget; a snapshot writeState produced always arrives whole. `maxLines` options are replaced by `budget`. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JpQ15QdqdDUJLavNhExwvL --- ARCHITECTURE.md | 5 +- CHANGELOG.md | 10 ++ docs/GUIDE.md | 10 +- global/tools/handoff/SKILL.md | 6 +- mintlify/concepts/cross-session-memory.mdx | 2 +- src/handoff.js | 147 ++++++++++++++++----- test/handoff.test.js | 113 ++++++++++++++-- 7 files changed, 246 insertions(+), 47 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d6506cc..3713604 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -229,7 +229,10 @@ injects: learned lessons, the anchored goal, the handoff snapshot, recent commit uncommitted changes — a fresh session orients on evidence, not priors. **The state/decision stores (`src/handoff.js`, `src/decide.js`).** `state.md` is a -bounded REWRITE (snapshot semantics — loader cost stays O(bound) forever); +bounded REWRITE (snapshot semantics — loader cost stays O(bound) forever). Writer and +loader share ONE budget in one unit (`STATE_BUDGET_BYTES`, 8 KB): the writer keeps rows in +priority order (goal, next, decisions, gotchas, in-progress, done) until the body fits, so +the SessionStart loader never cuts what the handoff wrote; `decisions.md` is append-only ADR-lite with a machine-readable `decision` ledger twin (log semantics — supersede, never edit). Both refuse secrets at write. diff --git a/CHANGELOG.md b/CHANGELOG.md index 90baf9f..2bc48be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,16 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). entropy leg everywhere, like lockfile integrity digests. A `ghp_…` token inside a binary is still refused, and an unreadable diff still fails closed. +- **A handoff snapshot is read back whole at session start.** `forge handoff` wrote up to + 150 lines to `.forge/state.md` but the SessionStart loader injected only the first 80, so + rows 81–150 of a valid handoff were silently dropped (the budget mismatch the formal + synthesis's T4 correction names). Writer and loader now share one budget in one unit, + `STATE_BUDGET_BYTES` (8 KB of snapshot body), and the writer keeps rows in priority order + (goal and acceptance criteria, next steps, decisions, gotchas and open assumptions, + in-progress files, then done) until the body fits. A section that lost rows ends with + "(+N more not kept …)". Sections are now written in that priority order. Only a hand-edited + or pre-budget file can still overflow the loader, and then the cut names the file. + ### Changed - **The gate docs no longer claim that repeated gates multiply their catch rates.** The diff --git a/docs/GUIDE.md b/docs/GUIDE.md index d30d8c2..c2c0716 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -410,9 +410,13 @@ Thursday's session — no more each-session re-assumption of what you're working ### `forge handoff ""` — the bounded session snapshot Session memory is volatile; `.forge/state.md` is the checkpoint that survives. One -command rewrites it (never appends — it stays ≤150 lines forever) with what got done, -what comes next, the gotchas, and any assumptions this session proceeded under (gathered -automatically from the session log, along with in-progress git files): +command rewrites it (never appends) with what got done, what comes next, the gotchas, and +any assumptions this session proceeded under (gathered automatically from the session log, +along with in-progress git files). It stays within one 8 KB budget forever — the same budget +the session-start loader injects, so the next session reads back everything the handoff +wrote. When the rows do not fit, they are kept in priority order (goal and acceptance +criteria, next steps, decisions, gotchas and assumptions, in-progress files, then done) and +each cut section says how many rows it dropped: ```bash forge handoff "built the export endpoint" \ diff --git a/global/tools/handoff/SKILL.md b/global/tools/handoff/SKILL.md index 279c966..0d429c8 100644 --- a/global/tools/handoff/SKILL.md +++ b/global/tools/handoff/SKILL.md @@ -6,8 +6,10 @@ description: End-of-session checkpoint. Use when finishing, pausing, or switchin # handoff — persist what this session knows Session memory is volatile; `.forge/state.md` is the committed-brain checkpoint the -SessionStart hook re-injects. Rewritten every time (bounded ≤150 lines), never appended — -the next session reads a snapshot, not an archive. +SessionStart hook re-injects. Rewritten every time, never appended, and bounded by the +same 8 KB budget the SessionStart loader injects — the next session reads back the whole +snapshot, not an archive. When rows do not fit, `done` rows go first and next steps and +gotchas stay. ## When - Ending or pausing a work session, or before a risky context switch. diff --git a/mintlify/concepts/cross-session-memory.mdx b/mintlify/concepts/cross-session-memory.mdx index 99b5865..3df5856 100644 --- a/mintlify/concepts/cross-session-memory.mdx +++ b/mintlify/concepts/cross-session-memory.mdx @@ -68,7 +68,7 @@ Two stores keep knowledge across sessions: | Store | Semantics | | ------------------- | ------------------------------------------------------------------------------------ | -| `.forge/state.md` | A bounded **rewrite** (snapshot) — loader cost stays `O(bound)` forever. | +| `.forge/state.md` | A bounded **rewrite** (snapshot); writer and loader share one 8 KB budget, so it is read back whole. | | `.forge/decisions.md` | Append-only **ADR-lite** (`D-####`) with a machine-readable decision ledger twin. | Both refuse secrets at write. `state.md` is re-injected each session start; diff --git a/src/handoff.js b/src/handoff.js index b08af0f..49b6fc3 100644 --- a/src/handoff.js +++ b/src/handoff.js @@ -13,6 +13,23 @@ import { git } from "./util.js"; export const statePath = (root) => join(root, ".forge", "state.md"); +/** + * The ONE size budget for the snapshot, shared by the writer (writeState) and the loader + * (stateBlock), in one unit: UTF-8 bytes of the snapshot body (the provenance line, which + * the loader strips, is not counted). The formal synthesis's T4 correction (2026-09-21) + * found its handoff bounded LINES while its loader injected at most 8 KB, so a valid + * snapshot could be cut at session start; forge had reproduced that mismatch in lines + * (150 written, 80 injected), silently dropping everything past line 80. The writer now + * selects rows until the body fits this budget, so the loader never truncates what the + * writer wrote. 8192 is the synthesis's A5 cap (roughly 2k tokens per session start). + */ +export const STATE_BUDGET_BYTES = 8192; + +const byteLen = (s) => Buffer.byteLength(s, "utf8"); +/** Bytes of `lines` joined by newlines — the measure both sides apply. */ +const bodyBytes = (lines) => + lines.reduce((n, l) => n + byteLen(l), 0) + Math.max(0, lines.length - 1); + /** Branch, dirty files (capped), recent commits — empty-safe outside a git repo. */ export function gatherGitFacts(root, { statusCap = 20 } = {}) { const branch = git(root, ["rev-parse", "--abbrev-ref", "HEAD"]); @@ -61,21 +78,67 @@ export function gatherAssumptions(root, { cap = 5 } = {}) { const arr = (v) => (Array.isArray(v) ? v : v ? [v] : []).map((x) => String(x).trim()).filter(Boolean); -const section = (title, rows, fallback = "- (none)") => [ - `## ${title}`, - ...(rows.length ? rows.map((r) => `- ${r}`) : [fallback]), - "", -]; +const omitted = (n, budget) => `- (+${n} more not kept — over the ${budget}-byte snapshot budget)`; + +/** + * Choose rows in PRIORITY order until the snapshot fits `budget` bytes. `sections` arrive + * in priority order, which is also the display order, so even a loader cut of a + * hand-edited file loses the least important rows first. Every header stays; a section + * whose rows did not all fit ends with an explicit "(+N more not kept)" row, so a drop is + * never silent. Within a section rows keep their given order; a row that does not fit is + * dropped (with the rest of its section) and later, smaller sections may still fit. + * @param {{title: string, rows: string[], fallback?: string}[]} sections + * @param {number} budget + * @returns {string[]} + */ +export function selectSnapshot(sections, budget = STATE_BUDGET_BYTES) { + const head = ["# Session state", ""]; + // Fixed cost first: every header, its blank line, and one reserved line per section — + // its fallback when empty, else the worst-case omission marker. + const reserve = sections.map((sec) => + sec.rows.length ? omitted(sec.rows.length, budget) : sec.fallback || "- (none)", + ); + let used = bodyBytes([ + ...head, + ...sections.flatMap((sec, i) => [`## ${sec.title}`, reserve[i], ""]), + ]); + const kept = sections.map(() => /** @type {string[]} */ ([])); + sections.forEach((sec, i) => { + for (const r of sec.rows) { + const line = `- ${r}`; + const cost = byteLen(line) + 1; // the row plus its newline + if (used + cost > budget) break; + used += cost; + kept[i].push(line); + } + }); + const out = [...head]; + sections.forEach((sec, i) => { + const dropped = sec.rows.length - kept[i].length; + out.push(`## ${sec.title}`, ...kept[i]); + if (!sec.rows.length) out.push(sec.fallback || "- (none)"); + else if (dropped) out.push(omitted(dropped, budget)); + out.push(""); + }); + return out; +} /** * Rewrite the whole snapshot from this session's fields + auto-gathered git facts. * Refuses secrets in the human-supplied fields (same rule as every forge store) and - * truncates to `maxLines` so the session-start injection can never balloon. + * selects rows in the synthesis's A4 priority order (goal, next, decisions, gotchas, + * in-progress, done) until the body fits `budget` — the SAME budget stateBlock injects, + * so the next session reads back exactly what was written. * @param {string} root * @param {{done?:string[]|string, next?:string[]|string, gotchas?:string[]|string, * criteria?:string[]|string, goal?:string, phase?:string}} fields + * @param {{t?: number, budget?: number}} [opts] */ -export function writeState(root, fields = {}, { t = Date.now(), maxLines = 150 } = {}) { +export function writeState( + root, + fields = {}, + { t = Date.now(), budget = STATE_BUDGET_BYTES } = {}, +) { const done = arr(fields.done); const next = arr(fields.next); const gotchas = arr(fields.gotchas); @@ -104,30 +167,33 @@ export function writeState(root, fields = {}, { t = Date.now(), maxLines = 150 } const progress = facts.status.length ? [...facts.status, ...(facts.overflow ? [`(+${facts.overflow} more dirty files)`] : [])] : []; - const lines = [ - "# Session state", - "", - ...section("Goal / Phase", [`${goal}${fields.phase ? ` — phase: ${fields.phase}` : ""}`]), - ...section("Acceptance criteria", criteria), - ...section("Done this session", done), - ...section("Next steps", next), - ...section("Gotchas", gotchas), - ...section("Open assumptions", assumptions), - ...section("In-progress files (git, at handoff)", progress, "- (clean tree)"), - "## Decisions", - `- append-only log: \`.forge/decisions.md\` (\`${BRAND.cli} decide\`)`, - "", - ]; + // A4 priority: goal (with its acceptance criteria), next, decisions, gotchas (with the + // open assumptions — both are "what could bite the next session"), in-progress, done. + const kept = selectSnapshot( + [ + { + title: "Goal / Phase", + rows: [`${goal}${fields.phase ? ` — phase: ${fields.phase}` : ""}`], + }, + { title: "Acceptance criteria", rows: criteria }, + { title: "Next steps", rows: next }, + { + title: "Decisions", + rows: [`append-only log: \`.forge/decisions.md\` (\`${BRAND.cli} decide\`)`], + }, + { title: "Gotchas", rows: gotchas }, + { title: "Open assumptions", rows: assumptions }, + { title: "In-progress files (git, at handoff)", rows: progress, fallback: "- (clean tree)" }, + { title: "Done this session", rows: done }, + ], + budget, + ); const provenance = ``; - const kept = - lines.length + 1 > maxLines - ? [...lines.slice(0, maxLines - 2), "- (truncated to stay bounded)"] - : lines; mkdirSync(join(root, ".forge"), { recursive: true }); writeFileSync(statePath(root), [...kept, provenance, ""].join("\n")); - return { ok: true, path: statePath(root), lines: kept.length + 1 }; + return { ok: true, path: statePath(root), lines: kept.length + 1, bytes: bodyBytes(kept) }; } // Only the EXACT provenance line is stripped — a naive slice at the first " src + test -- 230 --> src bench -- 7 --> src examples -- 4 --> src test -- 2 --> bench diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bc48be..e0cb5ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,26 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed +- **The everyday blast-radius checks walk sibling and forward relations, tagged.** The + substrate pre-action check (so also the ambient prompt hook and the `FORGE_ENFORCE` gate) + and the Stop gate's repair checklist ran the reverse-only walk that the empirical + refutation measured at recall 0.022, where 94.7% of the misses were sibling files. They now + walk reverse + the paper's repaired sibling and forward relations at the frozen parameters + already in `src/atlas.js`, and every file is tagged with the relation that reached it: + `forge substrate` and the ambient advisory print `path (reverse|sibling|forward)` with a + per-relation count, `--json` adds `impact.fileRelations` and `impact.relationCounts`, and the + Stop gate's block reason lists the untouched co-change candidates. The enforce gate still + counts only dependents toward its 25-file block (the wide walk would put 79 of this repo's + 98 source files over it, against 35 today, at precision about 0.09) and names the other + candidates in its reason; `blastRelations` changes what it counts. A wide walk never relabels + a reverse dependent, so its reverse-tagged set equals the reverse-only answer. Scope + decomposition and lesson matching keep using dependents only. `relations: ["reverse"]` + (`substrateCheck`, `repairReason`) is the explicit reverse-only option; `forge impact` and + `predict_impact` are unchanged (reverse-only unless `--all-relations`). The + `source/substrate.json` impact faculties move from `operational-v1` to + `operational-v2-recall`, with a guarantee that says the frozen parameters were tuned on a + different graph builder and are not held-out validated here. + - **The gate docs no longer claim that repeated gates multiply their catch rates.** The headers of `src/commit_gate.js` and `src/gate.js`, ARCHITECTURE.md §5 and the Mintlify verification-gates page said each rung (Stop, pre-commit, CI) was an independent catch diff --git a/README.md b/README.md index 377ea7b..78a3d67 100644 --- a/README.md +++ b/README.md @@ -232,9 +232,12 @@ from a fresh repository graph. - **Git-native team merge.** Claims and append-only logs merge by set union. The join is property-tested for commutativity, associativity, and idempotence. - **Heuristic impact prediction.** Forgekit builds a regex-derived code graph and walks - reverse dependencies to estimate affected files and tests. It is not conservative: it can - miss affected files (including constructs its parser does not recognize) as well as produce - false positives. + reverse dependencies to estimate affected files and tests; the pre-action check and the + Stop gate's repair checklist also walk the empirical refutation's sibling and forward + relations and tag each file with the relation that reached it. It is not conservative: it + can miss affected files (including constructs its parser does not recognize) as well as + produce false positives, and the sibling/forward files are lower-precision co-change + candidates. - **Budgeted context assembly.** Definitions, direct dependants, sibling tests, and trusted lessons are selected under a token budget. Missing required context becomes a question rather than invented context. diff --git a/docs/GUIDE.md b/docs/GUIDE.md index c2c0716..97026b5 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -123,10 +123,10 @@ Forge substrate — pre-action check context: complete — 4 required item(s), 1840/12000 tokens (`forge context` for the assembly) - impact: 3 file(s) predicted - - src/auth.js - - src/login.js - - src/session.js + impact: 3 file(s) predicted — 3 reverse + - src/auth.js (reverse) + - src/login.js (reverse) + - src/session.js (reverse) verify: - review impacted files before editing @@ -136,6 +136,17 @@ Forge substrate — pre-action check It found `login.js` and `session.js` — the two files that import `verifyToken` but you never named. That's the "forgot the coupled file" bug, caught _before_ the edit. +Each impacted file carries the relation that reached it. `reverse` files depend on the +change. The pre-action check (and so the ambient hook and the enforce gate) also walks the +empirical refutation's repaired **sibling** relation — a file that shares a dependency with +the target, like `deserializer.js` beside a changed `serializer.js` when both use +`wire_format.js` — and its **forward** relation, what the target itself depends on. The +reverse-only walk measured recall 0.022 on nine real repositories, and 94.7% of its misses +were siblings. Sibling and forward files are co-change candidates to check, not certain +breaks: on forgekit itself they take the median answer from 15 files to about 80 (precision +0.09). Pass `relations: ["reverse"]` to `substrateCheck` for the old answer; `forge impact` +stays reverse-only unless you add `--all-relations`. + **A vague task — it tells you to ask first:** ```console @@ -282,8 +293,9 @@ impacts all co-members) and a data-driven threshold from PageRank centrality and ledger incident history. `--basic` reverts to the fixed-threshold mode. Run `forge atlas build` first. -By default the walk follows **reverse dependencies only** — the files that actually -reference the target. `--all-relations` additionally walks the empirical refutation's +By default `forge impact` follows **reverse dependencies only** — the files that actually +reference the target. (`forge substrate`, the ambient prompt hook and the Stop gate's repair +checklist walk all three relations and tag each file.) `--all-relations` additionally walks the empirical refutation's repaired **sibling** and **forward** rules at their frozen parameters (a file that shares a dependency with the target, and what the target itself depends on). That is a recall instrument, not an everyday view: on forgekit itself the median answer goes from 15 files @@ -1228,7 +1240,11 @@ forge substrate "update verifyToken in src/auth.js" --json "okToProceed": false, "assumption": { "risk": "high", "shouldAsk": true, "questions": ["…"] }, "route": { "tier": "simple", "model": { "name": "Haiku 4.5" } }, - "impact": { "impactedFiles": ["src/auth.js", "src/login.js"] }, + "impact": { + "impactedFiles": ["src/auth.js", "src/login.js"], + "fileRelations": { "src/auth.js": "reverse", "src/login.js": "reverse" }, + "relationCounts": { "reverse": 2 }, + }, "verification": { "checklist": ["npm test", "npm run typecheck"] }, } ``` @@ -1253,7 +1269,7 @@ Forge substrate — pre-action advisory (advisory, never blocks): - Under-specified (high risk). Ask before editing: • What constraints must be respected: performance, dependencies, style, compatibility? - Suggested model: Haiku 4.5 (simple); escalate only on a verifier failure. -- Predicted blast radius (2): login.js, auth.js. Review these before editing. +- Predicted blast radius (2: 2 reverse): auth.js (reverse), login.js (reverse). Review these before editing. - Verify with: review impacted files before editing · run the narrowest affected test first ``` @@ -1416,7 +1432,7 @@ ambient pre-action guard there. | ------------------------------------------- | --------------------------------- | ----------------------------------------------- | | `proceed: ASK FIRST` / `okToProceed: false` | task is under-specified | ask the `clarify` questions, don't guess | | `route` | cheapest capable model | start there; escalate only if a verifier fails | -| `impact` | predicted blast radius | read these files before editing | +| `impact` | predicted blast radius, tagged | read the `reverse` files; check `sibling`/`forward` ones for a needed co-change | | `scope` | independent vs. coupled work | split independent groups into separate sessions | | `memory` | past Cortex lessons for this area | context, not law — tests override it | | `verify` | how to prove it works | run it, show the output, then say "done" | @@ -1485,7 +1501,7 @@ Create `global/crew/.md` with frontmatter. It installs into `~/.claude/age | when the ambient hook speaks | `src/substrate.js` → `substrateContext()` | | the cross-tool rule wording | `source/rules.json` → `substrate` section (then `forge init`) | | opt-in LLM adjudication | `FORGE_LLM=1` (+ `FORGE_LLM_AMBIENT=1` for the hook); config in `source/substrate.json` → `llm` | -| opt-in enforcing gate (halt, don't just advise) | `FORGE_ENFORCE=1` — blocks a no-anchor prompt or a very-large-blast action; `src/substrate.js` → `enforceDecision()`. Off by default. | +| opt-in enforcing gate (halt, don't just advise) | `FORGE_ENFORCE=1` — blocks a no-anchor prompt or a very-large-blast action (counted over dependents; sibling/forward candidates are named, not counted — `blastRelations` changes that); `src/substrate.js` → `enforceDecision()`. Off by default. | | verify test timeout | `FORGE_VERIFY_TIMEOUT_MS` (default 600000) | ### Opt into LLM-assisted judgments diff --git a/docs/cognitive-substrate/README.md b/docs/cognitive-substrate/README.md index 03891d6..b73f808 100644 --- a/docs/cognitive-substrate/README.md +++ b/docs/cognitive-substrate/README.md @@ -44,7 +44,7 @@ Forge substrate — pre-action advisory (advisory, never blocks): - Under-specified (high risk). Ask before editing: • What constraints must be respected: performance, dependencies, style, or compatibility? - Suggested model: Haiku 4.5 (simple); escalate only on a verifier failure. -- Predicted blast radius (2): invoice.js, math.js. Review these before editing. +- Predicted blast radius (2: 2 reverse): invoice.js (reverse), math.js (reverse). Review these before editing. - Verify with: review impacted files before editing · run the narrowest affected test first ``` @@ -82,16 +82,21 @@ $ forge substrate "Change verifyToken in src/auth.js to require length > 20; upd proceed: yes assumption: medium risk · completeness 0.63 route: Haiku 4.5 (simple) - impact: 3 file(s) predicted - - src/auth.js - - src/login.js (imports verifyToken — you didn't mention it) - - src/session.js (imports verifyToken — you didn't mention it) + impact: 3 file(s) predicted — 3 reverse + - src/auth.js (reverse) + - src/login.js (reverse) ← imports verifyToken; you didn't mention it + - src/session.js (reverse) ← imports verifyToken; you didn't mention it verify: - run the narrowest affected test first, then the broader suite ``` The second run found the two files that import `verifyToken` but you never named — the -"forgot the coupled file" bug, caught _before_ the edit. Add `--json` for machine-readable +"forgot the coupled file" bug, caught _before_ the edit. Each file is tagged with the +relation that reached it: `reverse` (depends on the change), `sibling` (shares a dependency +with it) or `forward` (the change depends on it). The check walks all three by default +because the reverse-only walk missed the sibling files that were 94.7% of the empirical +refutation's misses; siblings and forward files are co-change candidates, lower precision +than dependents, and the enforce gate counts dependents only. Add `--json` for machine-readable output (see [Use it in a script](#use-it-in-a-script)). --- diff --git a/global/tools/cognitive-substrate/references/capability-map.md b/global/tools/cognitive-substrate/references/capability-map.md index e6b9ca9..3d47d87 100644 --- a/global/tools/cognitive-substrate/references/capability-map.md +++ b/global/tools/cognitive-substrate/references/capability-map.md @@ -6,7 +6,7 @@ | Learning | `forge cortex`, ledger oracles | External outcomes (tests, CI, human accept/revert) move claim confidence; model weights do not change. | | Imagination | `forge imagine [--run]`, `forge impact` | Predicted breaks + minimal covering test suite; `--run` dry-runs it in a sandboxed worktree. | | Self-correction | `forge verify`, `forge diagnose` | Tests/builds beat model claims; 3× the same failure signature mints a diagnosis + escalation. | -| Impact-awareness | `forge atlas`, `forge impact` | Known symbols/files and likely dependents are surfaced. | +| Impact-awareness | `forge atlas`, `forge impact`, `forge substrate` | Known symbols/files, likely dependents, and sibling/forward co-change candidates are surfaced, each tagged by relation (`forge impact` alone is reverse-only unless `--all-relations`). | | M1 routing | `forge route` | Transparent model-tier recommendation. | | M2 assumption gate | `forge preflight`, `forge context` | Under-specified tasks return *computed* missing-set questions. | | M3 decomposition | `forge scope` | Import clusters show independent vs coupled files. | diff --git a/mintlify/concepts/pre-action-gate.mdx b/mintlify/concepts/pre-action-gate.mdx index 551b28e..f681cd5 100644 --- a/mintlify/concepts/pre-action-gate.mdx +++ b/mintlify/concepts/pre-action-gate.mdx @@ -61,7 +61,10 @@ on. **Blast radius** — the set of files an edit is predicted to impact, read from the code graph. `forge impact` computes it; the pipeline surfaces it before the model touches -anything. +anything. The pipeline tags each file with the relation that reached it: `reverse` +(depends on the change), `sibling` (shares a dependency with it) or `forward` (the change +depends on it). `forge impact` alone walks reverse dependents unless you pass +`--all-relations`. ```bash forge impact verifyToken # predicted impacted files for a symbol @@ -81,7 +84,7 @@ The verdict is **advisory by default** — it reports, it does not block. Set the completeness gate cannot cover the predicted edit set. - the impacted set exceeds the default ~25-file threshold. + the dependents in the impacted set exceed the default ~25-file threshold. diff --git a/source/substrate.json b/source/substrate.json index 53e11f8..21c682b 100644 --- a/source/substrate.json +++ b/source/substrate.json @@ -4,9 +4,9 @@ "faculties": [ { "id": "memory", "forge": "recall + cortex", "status": "partial", "guarantee": "facts and lessons are persisted as auditable files; relevance is advisory" }, { "id": "learning", "forge": "cortex lessons", "status": "partial", "guarantee": "external outcomes update lesson confidence; no model weights are changed" }, - { "id": "imagination", "forge": "impact graph", "status": "operational-v1", "guarantee": "reverse dependency traversal predicts possible blast radius" }, + { "id": "imagination", "forge": "impact graph", "status": "operational-v2-recall", "guarantee": "the pre-action check walks reverse dependents plus the empirical refutation's repaired sibling and forward relations (frozen parameters) and tags each file with its relation; a recall instrument (precision about 0.09 on this repo), with parameters frozen on a different graph builder, so not held-out validated here" }, { "id": "self-correction", "forge": "verify + doom-loop guard", "status": "partial", "guarantee": "tests/builds are trusted over model claims" }, - { "id": "impact-awareness", "forge": "atlas + impact", "status": "operational-v1", "guarantee": "known symbols/files and likely dependents are surfaced before edits" } + { "id": "impact-awareness", "forge": "atlas + impact", "status": "operational-v2-recall", "guarantee": "known symbols/files, their dependents, and sibling/forward co-change candidates are surfaced before edits, each tagged by relation; the enforce gate counts dependents only, and `forge impact` stays reverse-only unless --all-relations" } ], "mechanisms": [ { "id": "M1", "name": "complexity-aware routing", "command": "forge route", "status": "solved-with-transparency-layer" }, diff --git a/src/atlas.js b/src/atlas.js index c32a416..7d4a288 100644 --- a/src/atlas.js +++ b/src/atlas.js @@ -1225,13 +1225,58 @@ export const SIBLING = Object.freeze({ }); export const FORWARD = Object.freeze({ maxHops: 2, weight: 0.5 }); export const IMPACT_RELATIONS = Object.freeze(["reverse", "sibling", "forward"]); -/** What `impact()` walks unless a caller asks for more. The sibling/forward rules above are - * the paper's repair and they work — but they are a RECALL instrument: on this repo the - * median answer goes from 15 files to 78 of ~450 (max 196), recall 1.00, precision 0.093. - * An everyday "what does this change touch?" wants the focused answer, and a gate whose - * blast threshold is 25 files would otherwise trip on almost every edit. So the wider walk - * is opt-in: `impact(atlas, f, { relations: IMPACT_RELATIONS })`, or `--all-relations`. */ +/** What a bare `impact()` call (and `forge impact`) walks: the focused "what references + * this?" answer. The sibling/forward rules above are the paper's repair and a RECALL + * instrument: on this repo the median answer goes from 15 files to about 80 of ~450 + * (max ~197), recall 1.00, precision 0.093. The recall-critical callers — the substrate + * pre-action check (and so the ambient prompt hook and the enforce gate), and the Stop + * gate's repair checklist — pass IMPACT_RELATIONS and tag every file with the relation + * that reached it; `relations: DEFAULT_IMPACT_RELATIONS` is their explicit reverse-only + * option, and `forge impact --all-relations` is the CLI's wide walk. */ export const DEFAULT_IMPACT_RELATIONS = Object.freeze(["reverse"]); +/** Relations ranked by the strength of their structural claim: a reverse file DEPENDS on + * the change; an llm-verified one was graph- and grep-confirmed to reference it; a + * sibling shares a dependency with it; a forward file is something the change depends on. */ +export const RELATION_ORDER = Object.freeze(["reverse", "llm-verified", "sibling", "forward"]); +/** The relations that mean "this file depends on the change" — what a blocking count uses. */ +export const DEPENDENT_RELATIONS = Object.freeze(["reverse", "llm-verified"]); +/** Position of a relation in RELATION_ORDER (unknown relations sort last). */ +export const relationRank = (r) => { + const i = RELATION_ORDER.indexOf(r); + return i < 0 ? RELATION_ORDER.length : i; +}; + +/** + * Per-file relation tags over one or more impact() reports: each impacted file gets the + * strongest relation (RELATION_ORDER) any of its items was reached by, so output can say + * WHY a file is listed and a count can be taken per relation. + * @param {{impacted?: {relation?: string, node?: {file?: string}}[]}[]} reports + * @returns {Record} file → relation + */ +export function fileRelations(reports) { + /** @type {Record} */ + const out = {}; + for (const r of reports || []) + for (const x of r?.impacted || []) { + const file = x?.node?.file; + const rel = x?.relation || "reverse"; + if (!file) continue; + if (!(file in out) || relationRank(rel) < relationRank(out[file])) out[file] = rel; + } + return out; +} + +/** + * Files ordered strongest relation first, then by path — the display order every + * relation-aware caller uses so the dependents lead and co-change candidates follow. + * @param {string[]} files + * @param {Record} rels file → relation (from fileRelations) + */ +export function byRelation(files, rels) { + return [...files].sort( + (a, b) => relationRank(rels[a]) - relationRank(rels[b]) || (a < b ? -1 : a > b ? 1 : 0), + ); +} const round4 = (x) => Number(x.toFixed(4)); @@ -1357,11 +1402,14 @@ export function impact( // Sibling/forward items carry `relation` + `relationHops`; `hopDistance` stays the // REVERSE-dependency distance (null here), so "direct dependents" filters keep meaning. + // A reverse dependent keeps its label even when a sibling/forward path scores higher: + // the wide walk ADDS files and never relabels a dependent, so the reverse-tagged set of + // a wide walk is exactly the reverse-only answer (a count over it cannot drift). const offer = (id, confidence, relation, path, edgeKinds) => { const node = nodeById.get(id); if (!node || confidence < threshold) return; const prev = visited.get(id); - if (prev && prev.confidence >= round4(confidence)) return; + if (prev && (prev.relation === "reverse" || prev.confidence >= round4(confidence))) return; visited.set(id, { id, node, diff --git a/src/gate.js b/src/gate.js index 869bce2..d5bb9e5 100644 --- a/src/gate.js +++ b/src/gate.js @@ -21,7 +21,16 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { extname, join } from "node:path"; import { cusum } from "./anchor.js"; -import { CODE_EXTS, DOC_EXTS, impact, isConfigFile, load as loadAtlas } from "./atlas.js"; +import { + byRelation, + CODE_EXTS, + DOC_EXTS, + fileRelations, + IMPACT_RELATIONS, + impact, + isConfigFile, + load as loadAtlas, +} from "./atlas.js"; import { BRAND } from "./brand.js"; import { readSession, sessionPath } from "./cortex_hook.js"; import { decisionsPath } from "./decide.js"; @@ -281,23 +290,43 @@ export function obligationsFor(classes = {}) { * so it leads with the MISSING leg (test evidence vs docs vs config docs); the old * "handoff alone satisfies the gate" claim survives only on the config-only row, where * that lighter bar is real. Stale-doc candidates come from the CACHED atlas only (a - * hook never builds). + * hook never builds). The same walk names the code files the graph predicts should + * co-change but the session never touched, tagged by relation — the reverse-only walk + * missed the sibling files that were 94.7% of the empirical refutation's misses, so the + * default walks IMPACT_RELATIONS; `relations: ["reverse"]` is the reverse-only option. * @param {string} root * @param {{codeFiles?: string[], driftAlarm?: boolean, - * classes?: {code?: string[], config?: string[], test?: string[]}, row?: string}} [opts] */ + * classes?: {code?: string[], config?: string[], test?: string[], docs?: string[]}, + * row?: string, relations?: readonly string[]}} [opts] */ export function repairReason( root, - { codeFiles = [], driftAlarm = false, classes = {}, row = "code-without-docs" } = {}, + { + codeFiles = [], + driftAlarm = false, + classes = {}, + row = "code-without-docs", + relations = IMPACT_RELATIONS, + } = {}, ) { let likelyDocs = []; + /** @type {string[]} */ + let coChange = []; try { const atlas = loadAtlas(root); if (atlas) { const docs = new Set(); - for (const f of codeFiles.slice(0, 10)) - for (const d of impact(atlas, f, { maxHops: 2 }).impactedFiles) - if (d.endsWith(".md")) docs.add(d); + const reports = codeFiles + .slice(0, 10) + .map((f) => impact(atlas, f, { maxHops: 2, relations })); + for (const r of reports) for (const d of r.impactedFiles) if (d.endsWith(".md")) docs.add(d); likelyDocs = [...docs].slice(0, 5); + const touched = new Set(Object.values(classes).flat()); + for (const f of codeFiles) touched.add(f); + const rels = fileRelations(reports); + coChange = byRelation( + Object.keys(rels).filter((f) => !touched.has(f) && classifyPath(f) === "code"), + rels, + ).map((f) => `${f} (${rels[f]})`); } } catch {} const cited = codeFiles.length ? codeFiles : (classes.config ?? []); @@ -310,6 +339,13 @@ export function repairReason( const handoffStep = (suffix = "") => `\`${BRAND.cli} handoff "" --next ""\` — rewrite the session snapshot the next session resumes from${suffix}.`; const decideStep = `\`${BRAND.cli} decide ""\` if a non-obvious decision was made.`; + const coChangeStep = coChange.length + ? `Co-change candidates the graph predicts but this session never touched — confirm each needs no change: ${coChange + .slice(0, 8) + .join( + ", ", + )}${coChange.length > 8 ? ` (+${coChange.length - 8} more)` : ""}. (reverse = depends on the change · sibling = shares a dependency with it · forward = the change depends on it)` + : ""; let headline; const steps = []; if (row === "code-without-test-evidence") { @@ -333,6 +369,8 @@ export function repairReason( "END-TO-END COMPLETENESS: code changed this session but no doc or state artifact moved with it."; steps.push(docsSyncStep, handoffStep(), decideStep); } + // Second, right after the row's lead step: the files the diff may still owe a change. + if (coChangeStep) steps.splice(1, 0, coChangeStep); if (driftAlarm) steps.push( `Sustained goal drift this session (CUSUM alarm) — re-read the goal: \`${BRAND.cli} anchor\`.`, diff --git a/src/substrate.js b/src/substrate.js index d22d6d8..9219dfd 100644 --- a/src/substrate.js +++ b/src/substrate.js @@ -11,8 +11,13 @@ import { isStale as atlasIsStale, build as buildAtlas, buildSccIndex, + byRelation, + DEPENDENT_RELATIONS, + fileRelations, + IMPACT_RELATIONS, impact as impactGraph, load as loadAtlas, + relationRank, } from "./atlas.js"; import { assemble as assembleContext } from "./context.js"; import { matchingLessons } from "./cortex.js"; @@ -103,17 +108,27 @@ function siblingTestCandidates(file) { return out; } -/** Predict the tests likely to fail if the impacted files change (impacted tests + siblings). */ -export function predictFailingTests(root, impactedFiles) { +/** + * Predict the tests likely to fail if the impacted files change (impacted tests + siblings). + * @param {string} root + * @param {string[]} impactedFiles + * @param {Record} [rels] file → relation (atlas fileRelations) + * @returns {string[]} + */ +export function predictFailingTests(root, impactedFiles, rels) { const out = new Set(); - for (const f of impactedFiles) { + // With relation tags, tests predicted by a DEPENDENT come before a sibling's or forward + // file's (a Set keeps first insertion), so the capped "run these first" list leads with + // them; untagged input keeps the plain sorted order. + const files = rels ? byRelation(impactedFiles, rels) : impactedFiles; + for (const f of files) { if (isTestFile(f)) { out.add(f); continue; } for (const c of siblingTestCandidates(f)) if (existsSync(join(root, c))) out.add(c); } - return [...out].sort(); + return rels ? [...out] : [...out].sort(); } // Grep-style verify for the LLM impact pass: a proposed dependent is only kept if the target @@ -201,6 +216,9 @@ export function predictImpact( * @param {string} [opts.model] * @param {number} [opts.timeoutMs] * @param {boolean} [opts.bidirectional] + * @param {readonly string[]} [opts.relations] impact relations to walk — default + * IMPACT_RELATIONS (reverse + the paper's sibling/forward repair, each file tagged); + * pass DEFAULT_IMPACT_RELATIONS (["reverse"]) for the reverse-only walk. */ export function substrateCheck( root, @@ -213,6 +231,7 @@ export function substrateCheck( model, timeoutMs, bidirectional, + relations = IMPACT_RELATIONS, } = {}, ) { const text = String(task || ""); @@ -285,10 +304,15 @@ export function substrateCheck( const impactTargets = [...new Set([...entities.symbols, ...entities.files])].slice(0, 8); const impactRun = useLLM ? buildRunner({ model, timeoutMs }) : undefined; const impactVerify = makeImpactVerify(root); + // Recall-critical: the reverse-only walk the empirical refutation measured at recall 0.022 + // (94.7% of its misses were sibling files) is not the default here. Every file is tagged + // with the relation that reached it, so a reader can tell a dependent from a co-change + // candidate, and the enforce gate can count dependents only. const impacts = atlas ? impactTargets.map((target) => impactGraph(atlas, target, { threshold, + relations, llm: useLLM, run: impactRun, verify: impactVerify, @@ -296,12 +320,26 @@ export function substrateCheck( ) : []; const impactedFiles = [...new Set(impacts.flatMap((r) => r.impactedFiles || []))].sort(); + const impactRelations = fileRelations(impacts); + /** @type {Record} */ + const relationCounts = {}; + for (const f of impactedFiles) { + const r = impactRelations[f] ?? "reverse"; + relationCounts[r] = (relationCounts[r] ?? 0) + 1; + } + // Scope decomposition and lesson matching keep the DEPENDENT set they always used: they + // describe the work itself, and co-change candidates are for review, not for scoping. + const dependentFiles = impactedFiles.filter((f) => + DEPENDENT_RELATIONS.includes(impactRelations[f] ?? "reverse"), + ); // Consequence simulation (Eq 4), class "failing tests": which tests likely break if the // impacted files change — the impacted files that ARE tests, plus each impacted source file's // sibling test. Cheap, exact-ish, and surfaced BEFORE the edit (not after, like verify). // Gated on atlas freshness (belt and braces with the null atlas above): predictions from a // stale graph are not trustworthy and must not be presented as consequence evidence. - const predictedTests = atlasFresh ? predictFailingTests(root, impactedFiles) : []; + const predictedTests = atlasFresh + ? predictFailingTests(root, impactedFiles, impactRelations) + : []; // P3 reuse stage: has this team already built (and verified) this? The explicit gate // meters + writes evidence (reuseQuery); the ambient hook path stays read-only // (reusePeek) so a per-prompt hook never appends to the ledger or metrics. @@ -336,7 +374,7 @@ export function substrateCheck( } })() : null; - const scopedFiles = [...new Set([...entities.files, ...impactedFiles])]; + const scopedFiles = [...new Set([...entities.files, ...dependentFiles])]; const scope = scopedFiles.length ? decompose(root, scopedFiles) : { clusters: [], independentGroups: 0 }; @@ -365,6 +403,11 @@ export function substrateCheck( targets: impactTargets, reports: impacts, impactedFiles, + // Which relations were walked, the relation each impacted file was reached by + // (strongest claim wins: reverse > llm-verified > sibling > forward), and the counts. + relations: [...relations], + fileRelations: impactRelations, + relationCounts, predictedTests, // Truthful freshness: false when the atlas is missing/stale and couldn't be rebuilt. // Consumers must not present impactedFiles as trustworthy when this is false. @@ -447,12 +490,21 @@ export function substrateCheck( * so it halts a vacuous prompt ("fix it", "make it better") or an edit into a very large blast * radius, and never a specified task. Off unless `FORGE_ENFORCE=1` (or `enforce:true`); default * behaviour is unchanged. `reason` is written to be shown to the agent. + * The blast-radius count is taken over DEPENDENTS (reverse / llm-verified files) by default: + * the 25-file threshold was set on that walk, and the sibling/forward relations are a recall + * instrument (precision 0.093 on this repo) — counting them would block most edits here, + * against this gate's "strongest, lowest-false-positive signals only" contract. The block + * reason still names them, and `blastRelations` counts other relations when wanted. * @param {object} result - substrateCheck() result * @param {object} [opts] * @param {boolean} [opts.enforce] * @param {number} [opts.blastThreshold] + * @param {readonly string[]} [opts.blastRelations] relations counted toward blastThreshold */ -export function enforceDecision(result, { enforce, blastThreshold = 25 } = {}) { +export function enforceDecision( + result, + { enforce, blastThreshold = 25, blastRelations = DEPENDENT_RELATIONS } = {}, +) { const on = typeof enforce === "boolean" ? enforce : process.env.FORGE_ENFORCE === "1"; if (!on || !result) return { block: false }; const tail = "\n(Set FORGE_ENFORCE=0 to make Forge advisory again.)"; @@ -476,17 +528,47 @@ export function enforceDecision(result, { enforce, blastThreshold = 25 } = {}) { // untrustworthy) impacted set, and stale predictions must never hard-block an edit — // the explicit guard documents the intent even though a stale atlas now yields blast 0. if (result.impact?.atlasFresh !== false) { - const blast = result.impact?.impactedFiles?.length ?? 0; + const files = result.impact?.impactedFiles ?? []; + const rels = result.impact?.fileRelations ?? {}; + // An untagged file (a result built without relation tags) counts, as it always did. + const counted = files.filter((f) => !rels[f] || blastRelations.includes(rels[f])); + const blast = counted.length; if (blast >= blastThreshold) { + const others = files.length - blast; return { block: true, - reason: `Forge gate (enforcing): this touches a large blast radius (${blast} files predicted). Review the impacted files (or narrow the change) before editing.${tail}`, + reason: `Forge gate (enforcing): this touches a large blast radius (${blast} files predicted${ + others ? `, plus ${others} co-change candidate(s): ${relationSummary(result.impact)}` : "" + }). Review the impacted files (or narrow the change) before editing.${tail}`, }; } } return { block: false }; } +/** "2 reverse, 3 sibling, 1 forward" — counts in RELATION_ORDER; "" when untagged. */ +function relationSummary(impact) { + const counts = impact?.relationCounts ?? {}; + return Object.keys(counts) + .sort((a, b) => relationRank(a) - relationRank(b)) + .map((r) => `${counts[r]} ${r}`) + .join(", "); +} + +/** What each relation tag claims — printed once, only when a non-reverse tag is shown. */ +const RELATION_LEGEND = + "reverse = depends on the change · sibling = shares a dependency with it · forward = the change depends on it"; + +/** Impacted files strongest relation first, each with its tag when tags exist. */ +function taggedFiles(impact) { + const rels = impact?.fileRelations ?? {}; + return byRelation(impact?.impactedFiles ?? [], rels).map((f) => + rels[f] ? `${f} (${rels[f]})` : f, + ); +} +const hasCoChange = (impact) => + Object.values(impact?.fileRelations ?? {}).some((r) => !DEPENDENT_RELATIONS.includes(r)); + export function renderSubstrate(result) { const lines = ["Forge substrate — pre-action check", ""]; lines.push(` proceed: ${result.okToProceed ? "yes" : "ASK FIRST"}`); @@ -519,10 +601,15 @@ export function renderSubstrate(result) { if (result.impact.atlasFresh === false) { lines.push("", " impact: unavailable — atlas missing or stale (predictions not trustworthy)"); } else { - lines.push("", ` impact: ${result.impact.impactedFiles.length} file(s) predicted`); - for (const file of result.impact.impactedFiles.slice(0, 10)) lines.push(` - ${file}`); - if (result.impact.impactedFiles.length > 10) - lines.push(` … ${result.impact.impactedFiles.length - 10} more`); + const summary = relationSummary(result.impact); + lines.push( + "", + ` impact: ${result.impact.impactedFiles.length} file(s) predicted${summary ? ` — ${summary}` : ""}`, + ); + const shown = taggedFiles(result.impact); + for (const file of shown.slice(0, 10)) lines.push(` - ${file}`); + if (shown.length > 10) lines.push(` … ${shown.length - 10} more`); + if (hasCoChange(result.impact)) lines.push(` (${RELATION_LEGEND})`); } // Predicted tests only speak for a FRESH atlas — right after an "impact: unavailable" // notice, a likely-affected-tests list would contradict it with stale data (RA-07). @@ -575,10 +662,12 @@ export function substrateContext(result) { "- Impact unavailable: atlas missing or stale — predicted blast radius is not trustworthy (rebuild the atlas to get it).", ); } else if (result.impact.impactedFiles.length) { - const files = result.impact.impactedFiles; + const files = taggedFiles(result.impact); + const summary = relationSummary(result.impact); lines.push( - `- Predicted blast radius (${files.length}): ${files.slice(0, 8).join(", ")}${files.length > 8 ? " …" : ""}. Review these before editing.`, + `- Predicted blast radius (${files.length}${summary ? `: ${summary}` : ""}): ${files.slice(0, 8).join(", ")}${files.length > 8 ? " …" : ""}. Review these before editing.`, ); + if (hasCoChange(result.impact)) lines.push(` (${RELATION_LEGEND})`); } // Same freshness rule as the renderer: never advise stale test predictions (RA-07). const predTests = result.impact.atlasFresh === false ? [] : result.impact.predictedTests || []; diff --git a/test/impact_callers.test.js b/test/impact_callers.test.js new file mode 100644 index 0000000..d5aef19 --- /dev/null +++ b/test/impact_callers.test.js @@ -0,0 +1,179 @@ +// E03 (research-to-code audit): the everyday blast-radius callers ran the reverse-only walk +// the empirical refutation measured at recall 0.022, where 94.7% of the misses were sibling +// files. The recall-critical callers — the substrate check (and through it the ambient +// prompt hook and the enforce gate) and the Stop gate's repair checklist — now walk the +// sibling and forward relations too, at the frozen parameters in atlas.js, and tag every +// file with the relation that reached it. Fixture: serializer.js and deserializer.js both +// use wire_format.js; app.js imports both (test/fixtures/impact_repos.mjs, sibFiles). +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import { + build, + DEFAULT_IMPACT_RELATIONS, + fileRelations, + IMPACT_RELATIONS, + impact, +} from "../src/atlas.js"; +import { repairReason } from "../src/gate.js"; +import { + enforceDecision, + renderSubstrate, + substrateCheck, + substrateContext, +} from "../src/substrate.js"; +import { sibFiles, writeRepo } from "./fixtures/impact_repos.mjs"; + +const TASK = "Change serialize in src/serializer.js to add a version byte; update tests"; +const HOOK = fileURLToPath(new URL("../src/cortex_hook_main.js", import.meta.url)); + +function sibRepo() { + const root = writeRepo(sibFiles); + build({ root }); // writes the cached atlas the hook-grade (allowBuild:false) paths read + return root; +} + +test("substrateCheck reports the sibling file, tagged by relation", () => { + const root = sibRepo(); + const r = substrateCheck(root, TASK, { allowBuild: false }); + assert.equal(r.impact.atlasFresh, true); + assert.ok(r.impact.impactedFiles.includes("src/deserializer.js"), r.impact.impactedFiles.join()); + assert.equal(r.impact.fileRelations["src/deserializer.js"], "sibling"); + assert.equal(r.impact.fileRelations["src/app.js"], "reverse"); + assert.equal(r.impact.fileRelations["src/wire_format.js"], "forward"); + assert.deepEqual(r.impact.relations, [...IMPACT_RELATIONS]); + assert.equal(r.impact.relationCounts.sibling, 1); + const out = renderSubstrate(r); + assert.match(out, /src\/deserializer\.js \(sibling\)/); + assert.match(out, /1 reverse, 1 sibling, 1 forward/); +}); + +test("substrateCheck keeps an explicit reverse-only option", () => { + const root = sibRepo(); + const r = substrateCheck(root, TASK, { allowBuild: false, relations: DEFAULT_IMPACT_RELATIONS }); + assert.ok(!r.impact.impactedFiles.includes("src/deserializer.js")); + assert.ok(r.impact.impactedFiles.includes("src/app.js")); + assert.deepEqual(Object.values(r.impact.fileRelations), ["reverse"]); +}); + +test("the ambient advisory names the sibling file and says what the tag means", () => { + const root = sibRepo(); + const text = substrateContext(substrateCheck(root, TASK, { allowBuild: false })); + assert.match(text, /Predicted blast radius \(3: 1 reverse, 1 sibling, 1 forward\)/); + assert.match(text, /src\/app\.js \(reverse\), src\/deserializer\.js \(sibling\)/); + assert.match(text, /sibling = shares a dependency with it/); +}); + +test("the ambient prompt hook itself surfaces the sibling file", () => { + const root = sibRepo(); + const r = spawnSync("node", [HOOK, "preflight"], { + input: JSON.stringify({ session_id: "s-e03", cwd: root, prompt: TASK }), + encoding: "utf8", + }); + assert.equal(r.status, 0, r.stderr); + const ctx = JSON.parse(r.stdout).hookSpecificOutput.additionalContext; + assert.match(ctx, /src\/deserializer\.js \(sibling\)/, ctx); +}); + +test("enforce gate: dependents drive the block count; co-change candidates are named", () => { + const impactOf = (reverse, sibling) => { + const files = [ + ...Array.from({ length: reverse }, (_, i) => `r${i}.js`), + ...Array.from({ length: sibling }, (_, i) => `s${i}.js`), + ]; + const fileRelations = Object.fromEntries( + files.map((f) => [f, f.startsWith("r") ? "reverse" : "sibling"]), + ); + return { + assumption: { hardUnderspecified: false, questions: [] }, + impact: { + impactedFiles: files, + fileRelations, + relationCounts: { reverse, sibling }, + }, + }; + }; + const opts = { enforce: true, blastThreshold: 25 }; + // 10 dependents + 40 siblings: the siblings alone never block (precision-first gate). + assert.equal(enforceDecision(impactOf(10, 40), opts).block, false); + const g = enforceDecision(impactOf(30, 5), opts); + assert.equal(g.block, true); + assert.match( + g.reason, + /30 files predicted, plus 5 co-change candidate\(s\): 30 reverse, 5 sibling/, + ); + // The explicit option counts every relation toward the threshold. + assert.equal( + enforceDecision(impactOf(10, 40), { ...opts, blastRelations: IMPACT_RELATIONS }).block, + true, + ); + // The real substrate result carries the tags the gate reads. + const root = sibRepo(); + const real = substrateCheck(root, TASK, { allowBuild: false }); + assert.equal(enforceDecision(real, { enforce: true, blastThreshold: 1 }).block, true); + assert.equal( + enforceDecision(real, { enforce: true, blastThreshold: 2 }).block, + false, + "1 dependent (app.js) — the sibling and forward files are not counted by default", + ); + assert.equal( + enforceDecision(real, { enforce: true, blastThreshold: 3, blastRelations: IMPACT_RELATIONS }) + .block, + true, + ); +}); + +test("the Stop gate's repair checklist names the untouched sibling, tagged", () => { + const root = sibRepo(); + const reason = repairReason(root, { + codeFiles: ["src/serializer.js"], + classes: { code: ["src/serializer.js"] }, + }); + assert.match(reason, /Co-change candidates the graph predicts/); + assert.match(reason, /src\/app\.js \(reverse\), src\/deserializer\.js \(sibling\)/); + assert.match(reason, /src\/wire_format\.js \(forward\)/); + const touched = repairReason(root, { + codeFiles: ["src/serializer.js", "src/deserializer.js"], + classes: { code: ["src/serializer.js", "src/deserializer.js"] }, + }); + assert.doesNotMatch(touched, /deserializer\.js \(sibling\)/, "a changed file is not a candidate"); + const reverseOnly = repairReason(root, { + codeFiles: ["src/serializer.js"], + classes: { code: ["src/serializer.js"] }, + relations: ["reverse"], + }); + assert.doesNotMatch(reverseOnly, /deserializer/, "reverse-only option: the old answer"); +}); + +test("a wide walk adds files but never relabels a reverse dependent", () => { + // b reaches a by a 4-hop reverse chain (b → x → y → z → a) AND is a's sibling via c. The + // sibling path scores higher, but b stays a dependent. + const mod = (name) => ({ id: `module:${name}`, name, kind: "module", file: `${name}.js` }); + const imp = (s, t) => ({ + source: `module:${s}`, + target: `module:${t}`, + kind: "imports", + confidence: 1, + }); + const atlas = { + nodes: ["a", "b", "c", "x", "y", "z"].map(mod), + edges: [ + imp("a", "c"), + imp("b", "c"), + imp("b", "x"), + imp("x", "y"), + imp("y", "z"), + imp("z", "a"), + ], + symbols: [], + }; + const wide = impact(atlas, "a.js", { threshold: 0.01, relations: IMPACT_RELATIONS }); + const rels = fileRelations([wide]); + assert.equal(rels["b.js"], "reverse"); + const reverseOnly = impact(atlas, "a.js", { threshold: 0.01 }); + const reverseTagged = Object.keys(rels) + .filter((f) => rels[f] === "reverse") + .sort(); + assert.deepEqual(reverseTagged, reverseOnly.impactedFiles, "same dependents either way"); +}); From af2a5cc4d057add5cfe1779caaaf17c643530351 Mon Sep 17 00:00:00 2001 From: Juber Shaikh Date: Tue, 22 Sep 2026 04:35:44 +0000 Subject: [PATCH 05/13] docs(preflight): show the verifyToken example at the 0.88 the code computes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ARCHITECTURE.md, docs/GUIDE.md, docs/cognitive-substrate/README.md and the src/preflight.js comment said "Change verifyToken in src/auth.js to require length > 20; update tests" scores ≈ 0.63 (medium risk); the code gives 0.878 (low risk). Re-running the pre-df1c5a3 preflight gives exactly 0.630: the prior was hand-set when the task had one concrete anchor (the filename), and df1c5a3 (2026-09-21) made a named code identifier a second anchor. The docs now show 0.88 and explain the change; the weights are untouched. A test pins 0.23 / 0.88 (0.63 at one anchor) and checks that the two documented example outputs print the computed line. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JpQ15QdqdDUJLavNhExwvL --- ARCHITECTURE.md | 7 +++++-- CHANGELOG.md | 8 ++++++++ docs/GUIDE.md | 2 +- docs/cognitive-substrate/README.md | 2 +- src/preflight.js | 11 +++++++---- test/preflight.test.js | 26 +++++++++++++++++++++++++- 6 files changed, 47 insertions(+), 9 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 4a80612..f36e9e5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -330,8 +330,11 @@ completeness score `s(x)` is a **logistic** over its features (concreteness, nam vagueness, a smooth `tanh` length term) instead of an additive rubric with magic coefficients and discontinuous word-count steps — the `sigmoid` bounds it to (0,1) with no clamp, every feature's pull stays attributable, and a labeled bank could refine the weights via `predictor.js`'s -`trainLogistic`. The calibrated prior still lands the paper's own examples where they were -(a bare "make the auth better" ≈ 0.23 → ask; a concrete verifyToken edit ≈ 0.63 → proceed). +`trainLogistic`. The hand-set prior (not fit to data) puts the paper's own examples on the +right side of the 0.6 threshold: a bare "make the auth better" ≈ 0.23 → ask; the concrete +verifyToken edit ≈ 0.88 → proceed. That edit scored ≈ 0.63 when the weights were set, with one +concrete anchor (the filename); since a named code identifier became a second anchor it scores +≈ 0.88, and the weights were not re-fit. **The evidence trail (preflight).** Once a goal is anchored, every prompt appends its graded `driftScore` to the session log; `cusum` (until now test-only math) accumulates diff --git a/CHANGELOG.md b/CHANGELOG.md index e0cb5ee..6b49eda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,14 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). "(+N more not kept …)". Sections are now written in that priority order. Only a hand-edited or pre-budget file can still overflow the loader, and then the cut names the file. +- **The verifyToken example's completeness score matches the code again.** The docs + (ARCHITECTURE.md, GUIDE, the cognitive-substrate README) and the `src/preflight.js` comment + said "Change verifyToken in src/auth.js to require length > 20; update tests" scores ≈ 0.63 + (medium risk), but the code gives 0.878 (low risk). The prior was hand-set when that task + had one concrete anchor (the filename, 0.63); since 2026-09-21 a named code identifier is a + second anchor. The docs now show 0.88 and say why; the weights are unchanged. A test pins the + value and checks that the two example outputs print it. + ### Changed - **The everyday blast-radius checks walk sibling and forward relations, tagged.** The diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 97026b5..172f6cf 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -116,7 +116,7 @@ $ forge substrate "Change verifyToken in src/auth.js to require length > 20; upd Forge substrate — pre-action check proceed: yes - assumption: medium risk · completeness 0.63 + assumption: low risk · completeness 0.88 route: Haiku 4.5 (simple) · complexity 0.15 driven by: base cost of any task diff --git a/docs/cognitive-substrate/README.md b/docs/cognitive-substrate/README.md index b73f808..4b15995 100644 --- a/docs/cognitive-substrate/README.md +++ b/docs/cognitive-substrate/README.md @@ -80,7 +80,7 @@ $ forge substrate "make the auth better" $ forge substrate "Change verifyToken in src/auth.js to require length > 20; update tests" proceed: yes - assumption: medium risk · completeness 0.63 + assumption: low risk · completeness 0.88 route: Haiku 4.5 (simple) impact: 3 file(s) predicted — 3 reverse - src/auth.js (reverse) diff --git a/src/preflight.js b/src/preflight.js index b6153f9..6d7046c 100644 --- a/src/preflight.js +++ b/src/preflight.js @@ -225,10 +225,13 @@ export function ambiguityMarkers(text) { // the audit flagged as "graded-but-uncalibrated". Each feature contributes a signed amount to the // log-odds and the sigmoid maps the sum to [0,1] — so the estimate is smooth (no step jumps), // self-bounding (no ad-hoc clamp), and every feature's pull stays attributable (transparent rubric, -// the substrate's core commitment). Weights are a documented PRIOR calibrated so the paper's own -// examples land where they should (a bare "make the auth better" ≈ 0.23 → ask; a concrete -// "Change verifyToken … length > 20; update tests" ≈ 0.63 → proceed); a labeled task bank could -// refine them via predictor.js's trainLogistic without changing this call site. +// the substrate's core commitment). Weights are a documented PRIOR, hand-set so the paper's own +// examples land on the right side of τ = 0.6: a bare "make the auth better" scores 0.23 → ask. +// "Change verifyToken … length > 20; update tests" scored 0.63 when the prior was set, with one +// concrete anchor (the filename); since 2026-09-21 a named code identifier is a second anchor +// (countAnchors), so it now scores 0.88 → proceed, low risk. The weights were not re-fit, and +// they are not calibrated on data; a labeled task bank could refine them via predictor.js's +// trainLogistic without changing this call site. export const COMPLETENESS_WEIGHTS = { bias: -0.858, concreteness: 1.44, // each concrete anchor (example, call signature, quoted literal, filename) diff --git a/test/preflight.test.js b/test/preflight.test.js index e335959..2383dbd 100644 --- a/test/preflight.test.js +++ b/test/preflight.test.js @@ -1,8 +1,9 @@ import assert from "node:assert/strict"; -import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; +import { fileURLToPath } from "node:url"; import { ambiguityMarkers, assessTask, @@ -200,6 +201,29 @@ test("assessTask: reproduces the paper's calibrated anchor examples", () => { assert.equal(completenessFeatures("make the auth better").concreteness, 0); }); +// The docs quoted the verifyToken example at ≈ 0.63 while the code gave 0.878: the prior was +// set when the task had one concrete anchor, and a named identifier became a second one. Pin +// what the code computes and that the docs' example output says the same. +test("assessTask: the documented examples score what the docs say (0.23 and 0.88)", () => { + const task = "Change verifyToken in src/auth.js to require length > 20; update tests"; + const clear = assessTask(task); + assert.equal(clear.completeness.toFixed(2), "0.88"); + assert.equal(clear.risk, "low"); + assert.equal(assessTask("make the auth better").completeness.toFixed(2), "0.23"); + const f = completenessFeatures(task); + assert.equal(f.concreteness, 2, "the filename plus the named identifier"); + assert.equal( + completenessScore({ ...f, concreteness: 1 }).toFixed(2), + "0.63", + "one anchor — the value the prior was hand-set against", + ); + const line = `assumption: ${clear.risk} risk · completeness ${clear.completeness.toFixed(2)}`; + for (const doc of ["docs/GUIDE.md", "docs/cognitive-substrate/README.md"]) { + const text = readFileSync(fileURLToPath(new URL(`../${doc}`, import.meta.url)), "utf8"); + assert.ok(text.includes(line), `${doc} shows "${line}"`); + } +}); + test("assessTaskLLM: parses a completeness reading, rejects junk", () => { const p = assessTaskLLM("do a thing", { run: () => '{"completeness":0.3,"missing":["target_scope"],"questions":["Which file?"]}', From acdbda4523657837c5905b831bd06cd73967c3aa Mon Sep 17 00:00:00 2001 From: Juber Shaikh Date: Tue, 22 Sep 2026 04:43:19 +0000 Subject: [PATCH 06/13] fix(learn): consolidate learned lessons from ledger evidence, not a model bin/learn-consolidate.sh sent every learned lesson to Haiku with "DROP anything ... contradicted" and rewrote ~/.claude/skills/learned from the answer: memory pruned by the model's own judgment, which the research rejects (audit A13 / H25). src/learn_consolidate.js makes the default path deterministic: - exact and near-duplicate lessons within a project merge into their first occurrence (MinHash Jaccard >= 0.7, the ledger's clusters() threshold); - a lesson is dropped only when its matching ledger claim (lesson/fact) in the same project is dormant (isDormant), tombstoned, or in the attic; a lesson with no matching claim is kept; - originals are archived first, as before; --dry-run and --json report. The script execs it by default (`--repo ` names the ledger, default the cwd). The Haiku rewrite stays behind an explicit `--llm` first argument, with "contradicted" removed from its prompt. Tests stub `claude` on PATH so the suite never calls a model. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JpQ15QdqdDUJLavNhExwvL --- ARCHITECTURE.md | 7 +- CHANGELOG.md | 12 ++ bin/learn-consolidate.sh | 38 ++++- docs/legacy/PLAYBOOK.md | 2 +- docs/legacy/RUN.md | 2 +- src/learn_consolidate.js | 280 +++++++++++++++++++++++++++++++++ test/learn_consolidate.test.js | 199 +++++++++++++++++++++++ 7 files changed, 531 insertions(+), 9 deletions(-) mode change 100644 => 100755 bin/learn-consolidate.sh create mode 100644 src/learn_consolidate.js create mode 100644 test/learn_consolidate.test.js diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f36e9e5..9a99f68 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -544,6 +544,7 @@ forgekit/ ledger_store.js # git-native on-disk ledger (.forge/ledger/): sharded claims, append-only evidence/tombstone logs, normal-form verify ledger_bridge.js # legacy-store bridge, dormant by default (ledger-only); `FORGE_LEDGER_ONLY=0` re-enables cortex/recall/brain shadow-writes + idempotent `ledger import` ledger_read.js # ledger-only read path by default (`FORGE_LEDGER_ONLY=0` merges legacy∪ledger instead): cortex lesson/fact injection, `recall list`, brain's AGENTS.md index all see teammate knowledge from `ledger merge` + learn_consolidate.js # bin/learn-consolidate.sh: deterministic consolidation of ~/.claude/skills/learned — merge duplicates, drop only ledger-refuted (dormant/retracted/attic) lessons; no model call reuse.js # proof-carrying artifact cache: fingerprint (MinHash+LSH), exact→near→adapt→miss ladder, atlas revalidation embed.js # optional embeddings tier (ADR-0005): FORGE_EMBED=cmd:|http:, swaps MinHash/Jaccard for cosine in `reuse query`/`ledger query`, disk-cached at .forge/embed-cache.jsonl, silent fallback to MinHash context.js # budgeted context assembly + completeness gate: R(edit) set cover, compression ladder, computed missing-set @@ -619,8 +620,8 @@ from the tree it describes. ```mermaid %%{init: {'theme':'base','themeVariables':{'primaryColor':'#201a15','primaryTextColor':'#f2ede7','primaryBorderColor':'#372c22','lineColor':'#f26430','secondaryColor':'#272019','tertiaryColor':'#171310','edgeLabelBackground':'#201a15','clusterBkg':'#171310','clusterBorder':'#4a3b2e','fontFamily':'ui-sans-serif, system-ui, sans-serif','fontSize':'14px'},'flowchart':{'curve':'basis','padding':10,'nodeSpacing':36,'rankSpacing':44}}}%% flowchart LR - test["test
114 files"] - src["src
98 files"] + test["test
115 files"] + src["src
99 files"] landing["landing
61 files"] research["research
37 files"] global["global
5 files"] @@ -628,7 +629,7 @@ flowchart LR scripts["scripts
2 files"] docs["docs
1 file"] examples["examples
1 file"] - test -- 230 --> src + test -- 233 --> src bench -- 7 --> src examples -- 4 --> src test -- 2 --> bench diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b49eda..541c67f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,18 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). `operational-v2-recall`, with a guarantee that says the frozen parameters were tuned on a different graph builder and are not held-out validated here. +- **`bin/learn-consolidate.sh` no longer lets a model prune memory.** It sent every learned + lesson to Haiku with "DROP anything … contradicted" and rewrote `~/.claude/skills/learned` + from the answer, which is pruning by the model's own judgment (the research requires pruning + by ground truth). Consolidation is now deterministic (`src/learn_consolidate.js`): exact and + near-duplicate lessons within a project merge (MinHash Jaccard ≥ 0.7, the ledger's own + consolidation threshold), and a lesson is dropped only when its matching ledger claim in + that project is dormant, retracted or pruned to the attic; a lesson the ledger knows nothing + about is kept. `--repo ` names the ledger (default: the current directory), and + `--dry-run` / `--json` report without writing. Originals are archived first, as before. The + model rewrite remains behind an explicit `--llm` first argument, with "contradicted" removed + from its prompt. + - **The gate docs no longer claim that repeated gates multiply their catch rates.** The headers of `src/commit_gate.js` and `src/gate.js`, ARCHITECTURE.md §5 and the Mintlify verification-gates page said each rung (Stop, pre-commit, CI) was an independent catch diff --git a/bin/learn-consolidate.sh b/bin/learn-consolidate.sh old mode 100644 new mode 100755 index 56c2c14..44808d6 --- a/bin/learn-consolidate.sh +++ b/bin/learn-consolidate.sh @@ -1,9 +1,37 @@ #!/usr/bin/env bash -# Consolidate accumulated learned lessons: merge duplicates, prune trivia, -# keep only durable rules. Run weekly (manually or via cron). Uses Haiku. -# Fixes the append-only bloat of the session-learning hook. +# Consolidate the learned lessons the opt-in session-learner appends to +# ~/.claude/skills/learned: merge duplicates, and drop a lesson ONLY when the forge +# ledger refutes it (its matching claim is dormant, retracted or pruned to the attic). +# Deterministic, no model call — src/learn_consolidate.js does the work. Run weekly. +# +# learn-consolidate.sh [--dir ] [--repo ]... [--dry-run] [--json] +# +# `--repo` names a project whose .forge/ledger supplies the evidence (default: the current +# directory, when it has a ledger). A lesson with no matching ledger claim is always kept. +# +# `--llm` (explicit opt-in, first argument) runs the old Haiku rewrite instead. It prunes +# by the model's own judgment, which the research this project follows rejects for +# memory (prune by ground truth, not by the model's say-so), so it is never the default. set -uo pipefail +# Resolve symlinks (the script is usually linked onto PATH) to find the package root. +SELF="${BASH_SOURCE[0]}" +while [ -L "$SELF" ]; do + link="$(readlink "$SELF")" + case "$link" in + /*) SELF="$link" ;; + *) SELF="$(dirname "$SELF")/$link" ;; + esac +done +ROOT="$(cd "$(dirname "$SELF")/.." && pwd)" + +if [ "${1:-}" != "--llm" ]; then + command -v node >/dev/null 2>&1 || { echo "node not found on PATH"; exit 1; } + exec node "$ROOT/src/learn_consolidate.js" "$@" +fi +shift + +echo "! --llm: consolidating by model judgment (not ledger evidence); originals are archived first" DIR="$HOME/.claude/skills/learned" command -v claude >/dev/null 2>&1 || { echo "claude CLI not found on PATH"; exit 1; } @@ -18,9 +46,11 @@ mkdir -p "$DIR/archive" ts="$(date +%Y%m%d-%H%M%S)" for f in $inputs; do cp "$f" "$DIR/archive/$(basename "$f").$ts.bak"; done +# Contradiction is deliberately NOT a model decision even here: only ledger evidence +# (the default path) may refute a lesson. prompt="You are consolidating a developer's accumulated learned lessons from AI coding sessions. MERGE duplicates and near-duplicates into one rule. DROP anything -trivial, one-off, session-specific, or contradicted. KEEP only durable, reusable +trivial, one-off, or session-specific. KEEP only durable, reusable rules (project gotchas, error->fix patterns, workflow rules). Group under '## ' headers (use '## General' for cross-project). Each rule = one markdown bullet. Do NOT invent anything — only compress what is given. NEVER diff --git a/docs/legacy/PLAYBOOK.md b/docs/legacy/PLAYBOOK.md index 94502a4..1b8cec5 100644 --- a/docs/legacy/PLAYBOOK.md +++ b/docs/legacy/PLAYBOOK.md @@ -17,7 +17,7 @@ graphify install --project # optional: adds a code-graph skill to this re Then in Claude Code, once per big repo: `/graphify .` (builds the graph) and `graphify hook install` (keeps it current on every commit). -Housekeeping: run `claude-learn-consolidate` weekly to dedupe/prune learned lessons. +Housekeeping: run `claude-learn-consolidate` weekly to merge duplicate learned lessons; it drops a lesson only when the forge ledger refutes it (`--repo ` supplies the ledger). Minimalism enforcer is always on via Ponytail (`/ponytail`, `/ponytail-review`). --- diff --git a/docs/legacy/RUN.md b/docs/legacy/RUN.md index c543d51..9d9c889 100644 --- a/docs/legacy/RUN.md +++ b/docs/legacy/RUN.md @@ -44,7 +44,7 @@ echo 'export ENABLE_SESSION_LEARNING=1' >> ~/.zshrc && source ~/.zshrc # alrea claude-init # in a repo: write AGENTS.md + thin CLAUDE.md (auto-detect stack) claude-taste # list per-repo UI taste skills claude-taste minimalist-ui # enable one taste for the current repo -claude-learn-consolidate # merge/dedupe/prune learned lessons (weekly; ~1-2 min) +claude-learn-consolidate # merge duplicate learned lessons; drop only ledger-refuted ones (weekly; no model call; --llm for the old Haiku rewrite) ``` ## 5. Skills / agents (auto-fire, or force with /name) diff --git a/src/learn_consolidate.js b/src/learn_consolidate.js new file mode 100644 index 0000000..f2a8326 --- /dev/null +++ b/src/learn_consolidate.js @@ -0,0 +1,280 @@ +// forge learn-consolidate — DETERMINISTIC consolidation of the legacy learned-lessons store +// (~/.claude/skills/learned: `lessons-YYYY-MM.md`, appended by the opt-in session-learner +// guard, plus the `CONSOLIDATED.md` a previous run wrote). bin/learn-consolidate.sh used to +// send every lesson to a model with "DROP anything … contradicted" and rewrite the store +// from its answer: pruning memory by the model's own judgment, which the research rejects +// (white paper §3, memory residual gap; §7.1, val = validity from an external oracle). +// Here nothing is judged, reworded or invented: +// - MERGE: an exact duplicate (normalized text) or a near-duplicate (MinHash Jaccard ≥ τ, +// the ledger's own consolidation threshold, ledger.clusters) within one project +// collapses into its first occurrence. +// - DROP: only on ledger ground truth. A lesson is dropped when its best-matching ledger +// claim (lesson/fact, Jaccard ≥ τ against claimText) is dormant (ledger.isDormant: its +// oracle-evidenced val fell below DORMANT_VAL and no confirmation restored it), +// retracted (tombstoned), or archived to the ledger attic by `forge ledger prune`. +// A lesson with no matching claim is KEPT: absence of evidence is not refutation. +// Claims are matched only within the lesson's project (a repo whose directory name is the +// project), so a lesson refuted in one repo is not dropped from another. + +import { + copyFileSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + realpathSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { basename, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { claimText, isDormant, jaccard, sketch } from "./ledger.js"; +import { loadClaims, repoLedger } from "./ledger_store.js"; +import { epochDay } from "./util.js"; + +/** Same τ as ledger.clusters — "these two say the same thing". */ +export const CONSOLIDATE_TAU = 0.7; +export const GENERAL = "General"; +export const learnedDir = () => join(homedir(), ".claude", "skills", "learned"); + +const norm = (s) => + String(s) + .toLowerCase() + .replace(/[`*_"'.,;:!?()[\]{}]/g, " ") + .replace(/\s+/g, " ") + .trim(); + +/** + * Parse learned-lesson markdown into `{project, text}` entries. Understands both shapes: + * the session-learner's `## 2026-07-04 12:00 — ` headers and a consolidated + * file's `## ` headers. Bullets are `- ` / `* ` lines; an indented non-bullet + * line continues the bullet above it. Everything else (titles, prose) is ignored. + * @param {string} text + * @returns {{project: string, text: string}[]} + */ +export function parseLearned(text) { + const out = []; + let project = GENERAL; + let last = null; + for (const line of String(text).split(/\r?\n/)) { + const h = /^##\s+(.+?)\s*$/.exec(line); + if (h) { + const dated = /^\d{4}-\d{2}-\d{2}(?:\s+\d{1,2}:\d{2})?\s+[—–-]\s+(.+)$/.exec(h[1]); + project = (dated ? dated[1] : h[1]).trim() || GENERAL; + last = null; + continue; + } + const b = /^\s{0,3}[-*]\s+(.+?)\s*$/.exec(line); + if (b) { + last = { project, text: b[1] }; + out.push(last); + continue; + } + if (last && /^\s{2,}\S/.test(line)) last.text += ` ${line.trim()}`; + else if (!line.trim()) last = null; + } + return out; +} + +/** + * @typedef {{project: string, text: string}} Learned + * @typedef {{id?: string, kind?: string, body?: any, tombstone?: any, attic?: boolean, + * project?: string}} LedgerClaim + */ + +/** Why a claim refutes the lessons that match it, or null when it does not. */ +function refutation(claim, nowDay) { + if (claim.attic) return "archived to the ledger attic (dormant or retracted)"; + if (claim.tombstone) return "retracted in the ledger"; + try { + if (isDormant(claim, nowDay)) return "dormant in the ledger (oracle evidence refuted it)"; + } catch {} + return null; +} + +/** + * Consolidate deterministically: merge duplicates, drop only ledger-refuted lessons. + * @param {Learned[]} entries + * @param {{claims?: LedgerClaim[], nowDay?: number, tau?: number}} [opts] each claim may + * carry `project` (the repo directory name); a claim without one matches any project. + * @returns {{kept: Learned[], merged: {text: string, into: string}[], + * dropped: {project: string, text: string, claim: string, reason: string}[]}} + */ +export function consolidateLearned( + entries, + { claims = [], nowDay = epochDay(), tau = CONSOLIDATE_TAU } = {}, +) { + const usable = claims + .filter((c) => c && (c.kind === "lesson" || c.kind === "fact")) + .map((c) => ({ c, s: sketch(claimText(c)), why: refutation(c, nowDay) })); + /** @type {(Learned & {s: any, n: string})[]} */ + const kept = []; + const merged = []; + const dropped = []; + for (const e of entries) { + const text = String(e.text || "").trim(); + if (!text) continue; + const project = e.project || GENERAL; + const s = sketch(text); + const n = norm(text); + const dup = kept.find((k) => k.project === project && (k.n === n || jaccard(k.s, s) >= tau)); + if (dup) { + merged.push({ text, into: dup.text }); + continue; + } + let best = null; + for (const u of usable) { + if (u.c.project && project !== GENERAL && u.c.project !== project) continue; + const j = jaccard(u.s, s); + if (j >= tau && (!best || j > best.j)) best = { ...u, j }; + } + if (best?.why) { + dropped.push({ + project, + text, + claim: String(best.c.id ?? "").slice(0, 12), + reason: best.why, + }); + continue; + } + kept.push({ project, text, s, n }); + } + return { kept: kept.map(({ project, text }) => ({ project, text })), merged, dropped }; +} + +/** + * The consolidated file: one `## ` section per project in first-seen order + * (General first when present), one bullet per kept lesson, in input order. + * @param {Learned[]} kept + * @param {{date?: string}} [opts] + */ +export function renderConsolidated(kept, { date = new Date().toISOString().slice(0, 10) } = {}) { + const order = [...new Set(kept.map((k) => k.project))].sort((a, b) => + a === GENERAL ? -1 : b === GENERAL ? 1 : 0, + ); + const lines = [`# Learned — consolidated ${date}`, ""]; + for (const p of order) { + lines.push(`## ${p}`); + for (const k of kept) if (k.project === p) lines.push(`- ${k.text}`); + lines.push(""); + } + return lines.join("\n"); +} + +/** + * Ledger claims of each repo (live + attic), tagged with the repo's directory name as + * their project. Unreadable ledgers contribute nothing. + * @param {string[]} repos + * @returns {LedgerClaim[]} + */ +export function ledgerClaimsFor(repos) { + const out = []; + for (const root of repos) { + const dir = repoLedger(root); + if (!existsSync(dir)) continue; + const project = basename(resolve(root)); + try { + for (const c of loadClaims(dir)) out.push({ ...c, project }); + } catch {} + try { + const attic = join(dir, "attic"); + for (const f of existsSync(attic) ? readdirSync(attic) : []) + if (f.endsWith(".json")) + out.push({ ...JSON.parse(readFileSync(join(attic, f), "utf8")), attic: true, project }); + } catch {} + } + return out; +} + +/** + * The whole job over a learned-lessons directory. Originals are archived under + * `archive/` before anything is rewritten; `dryRun` reports without writing. + * @param {{dir?: string, repos?: string[], nowDay?: number, dryRun?: boolean, date?: string}} [opts] + */ +export function consolidateDir({ + dir = learnedDir(), + repos = [], + nowDay = epochDay(), + dryRun = false, + date, +} = {}) { + const monthly = existsSync(dir) + ? readdirSync(dir) + .filter((f) => /^lessons-.*\.md$/.test(f)) + .sort() + : []; + const inputs = [ + ...(existsSync(join(dir, "CONSOLIDATED.md")) ? ["CONSOLIDATED.md"] : []), + ...monthly, + ]; + const none = { kept: [], merged: [], dropped: [] }; + if (!inputs.length) return { ok: true, row: "nothing", dir, inputs, ...none }; + const entries = inputs.flatMap((f) => parseLearned(readFileSync(join(dir, f), "utf8"))); + if (!entries.length) return { ok: true, row: "empty", dir, inputs, ...none }; + const r = consolidateLearned(entries, { claims: ledgerClaimsFor(repos), nowDay }); + const result = { ok: true, row: dryRun ? "dry-run" : "written", dir, inputs, ...r }; + if (dryRun) return result; + const ts = new Date().toISOString().replace(/[-:]/g, "").replace(/\..*$/, ""); + mkdirSync(join(dir, "archive"), { recursive: true }); + for (const f of inputs) copyFileSync(join(dir, f), join(dir, "archive", `${f}.${ts}.bak`)); + writeFileSync(join(dir, "CONSOLIDATED.md"), renderConsolidated(r.kept, { date })); + for (const f of monthly) unlinkSync(join(dir, f)); + return result; +} + +/** @param {ReturnType} r */ +export function renderReport(r) { + if (r.row === "nothing") return `nothing to consolidate in ${r.dir}`; + if (r.row === "empty") return "no lesson content"; + const head = + r.row === "dry-run" + ? `(dry run) would keep ${r.kept.length} lesson(s) — nothing written` + : `✓ consolidated → ${join(r.dir, "CONSOLIDATED.md")} (${r.kept.length} lesson(s); originals archived in ${join(r.dir, "archive")}/)`; + const lines = [ + head, + ` merged duplicates: ${r.merged.length}`, + ` dropped on ledger evidence: ${r.dropped.length}`, + ]; + for (const d of r.dropped.slice(0, 20)) + lines.push(` - [${d.project}] ${d.text.slice(0, 80)} — ${d.reason} (claim ${d.claim})`); + return lines.join("\n"); +} + +/** CLI: node src/learn_consolidate.js [--dir ] [--repo ]… [--dry-run] [--json] */ +export function main(argv = process.argv.slice(2)) { + const repos = []; + let dir; + let dryRun = false; + let json = false; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--dir") dir = argv[++i]; + else if (a === "--repo") repos.push(argv[++i]); + else if (a === "--dry-run") dryRun = true; + else if (a === "--json") json = true; + } + // Default ledger: the current directory's, when it has one. + if (!repos.length && existsSync(repoLedger(process.cwd()))) repos.push(process.cwd()); + const r = consolidateDir({ dir: dir || learnedDir(), repos: repos.filter(Boolean), dryRun }); + console.log(json ? JSON.stringify(r, null, 2) : renderReport(r)); + return r; +} + +// Run as a script (bin/learn-consolidate.sh execs this file); importing it has no effect. +// realpath: node resolves the main module through symlinks (/tmp → /private/tmp on macOS). +const isMain = () => { + try { + return realpathSync(process.argv[1] ?? "") === fileURLToPath(import.meta.url); + } catch { + return false; + } +}; +if (isMain()) { + try { + main(); + } catch (e) { + console.error(`learn-consolidate: ${e instanceof Error ? e.message : e} — originals kept`); + process.exitCode = 1; + } +} diff --git a/test/learn_consolidate.test.js b/test/learn_consolidate.test.js new file mode 100644 index 0000000..8d9a9d4 --- /dev/null +++ b/test/learn_consolidate.test.js @@ -0,0 +1,199 @@ +// A13 (research-to-code audit): bin/learn-consolidate.sh asked a model to "DROP anything … +// contradicted" and rewrote the learned-lessons store from its answer — memory pruned by the +// model's own judgment, which the research rejects. Consolidation is now deterministic: +// duplicates merge, and a lesson is dropped only when the ledger refutes it. +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { delimiter, join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import { + consolidateDir, + consolidateLearned, + ledgerClaimsFor, + parseLearned, + renderConsolidated, +} from "../src/learn_consolidate.js"; +import { isDormant, mintClaim, outcomeRecord } from "../src/ledger.js"; +import { appendEvidence, putClaim, repoLedger, tombstone } from "../src/ledger_store.js"; + +const SCRIPT = fileURLToPath(new URL("../bin/learn-consolidate.sh", import.meta.url)); +const tmp = (p = "forge-learn-") => mkdtempSync(join(tmpdir(), p)); + +const FLAKY = "Run the db migration before the integration tests or they fail with a missing table"; +const TRIVIA = "The staging deploy needs VPN access from the office network first"; +const RETRY = "The HTTP client retries three times with exponential backoff before giving up"; + +/** A repo named `name` whose ledger holds `text` as a lesson claim, optionally refuted. */ +function repoWith(name, text, { refute = false, retract = false } = {}) { + const root = join(tmp(), name); + mkdirSync(root, { recursive: true }); + const dir = repoLedger(root); + const minted = mintClaim({ + kind: "lesson", + body: { + correctedBehavior: text, + trigger: { action: "edit", files: [], keywords: [], symbols: [] }, + whatWentWrong: "", + }, + scope: { level: "repo" }, + provenance: { agent: "cortex", author: "t", task: `lsn_${name}` }, + t: 100, + }); + assert.equal(minted.ok, true); + assert.equal(putClaim(dir, minted.claim).ok, true); + if (refute) + for (const ref of ["human:alice@shop-review", "human:bob@shop-review"]) { + const o = outcomeRecord({ oracle: "human.revert", result: "contradict", ref, t: 101 }); + assert.equal(o.ok, true); + assert.equal(appendEvidence(dir, minted.claim.id, o.outcome).ok, true); + } + if (retract) tombstone(dir, minted.claim.id, { author: "t", reason: "wrong", t: 101 }); + return { root, claim: minted.claim }; +} + +test("parseLearned reads the session-learner and consolidated shapes", () => { + const entries = parseLearned( + [ + "# Learned — consolidated 2026-09-01", + "", + "## General", + `- ${TRIVIA}`, + "## 2026-09-10 14:02 — shop", + `- ${FLAKY}`, + " (seen twice)", + "* use pnpm, not npm", + ].join("\n"), + ); + assert.deepEqual(entries, [ + { project: "General", text: TRIVIA }, + { project: "shop", text: `${FLAKY} (seen twice)` }, + { project: "shop", text: "use pnpm, not npm" }, + ]); +}); + +test("duplicates merge; nothing is dropped without ledger evidence", () => { + const r = consolidateLearned( + [ + { project: "shop", text: FLAKY }, + { project: "shop", text: `${FLAKY}.` }, // exact after normalization + { project: "shop", text: FLAKY.replace("Run", "Always run") }, // near-duplicate + { project: "blog", text: FLAKY }, // another project keeps its own copy + { project: "shop", text: TRIVIA }, // "trivial" is not a reason to delete + ], + { claims: [] }, + ); + assert.deepEqual( + r.kept.map((k) => `${k.project}: ${k.text}`), + [`shop: ${FLAKY}`, `blog: ${FLAKY}`, `shop: ${TRIVIA}`], + ); + assert.equal(r.merged.length, 2); + assert.equal(r.dropped.length, 0, "no claim matched, so nothing is refuted"); +}); + +test("a lesson is dropped only when its matching ledger claim is dormant or retracted", () => { + const refuted = repoWith("shop", FLAKY, { refute: true }); + assert.ok(isDormant(ledgerClaimsFor([refuted.root])[0], 102), "fixture claim is dormant"); + const confirmedish = repoWith("shop", RETRY); // live, never refuted + const retracted = repoWith("shop", TRIVIA, { retract: true }); + const claims = ledgerClaimsFor([refuted.root, confirmedish.root, retracted.root]); + const r = consolidateLearned( + [ + { project: "shop", text: FLAKY }, + { project: "shop", text: RETRY }, + { project: "shop", text: TRIVIA }, + { project: "shop", text: "an unrelated lesson the ledger knows nothing about" }, + ], + { claims, nowDay: 102 }, + ); + assert.deepEqual( + r.kept.map((k) => k.text), + [RETRY, "an unrelated lesson the ledger knows nothing about"], + ); + assert.deepEqual( + r.dropped.map((d) => d.reason), + ["dormant in the ledger (oracle evidence refuted it)", "retracted in the ledger"], + ); + assert.equal(r.dropped[0].claim, refuted.claim.id.slice(0, 12), "the drop cites its claim"); +}); + +test("ledger evidence is scoped to its project", () => { + const refutedElsewhere = repoWith("blog", FLAKY, { refute: true }); + const r = consolidateLearned([{ project: "shop", text: FLAKY }], { + claims: ledgerClaimsFor([refutedElsewhere.root]), + nowDay: 102, + }); + assert.equal(r.kept.length, 1, "refuted in blog does not drop it from shop"); +}); + +test("renderConsolidated groups by project, General first", () => { + const md = renderConsolidated( + [ + { project: "shop", text: "a" }, + { project: "General", text: "b" }, + { project: "shop", text: "c" }, + ], + { date: "2026-09-22" }, + ); + assert.equal(md, "# Learned — consolidated 2026-09-22\n\n## General\n- b\n\n## shop\n- a\n- c\n"); +}); + +test("consolidateDir archives originals, rewrites CONSOLIDATED.md, removes monthly files", () => { + const dir = tmp(); + writeFileSync( + join(dir, "lessons-2026-09.md"), + `\n## 2026-09-10 14:02 — shop\n- ${FLAKY}\n- ${FLAKY}\n`, + ); + const dry = consolidateDir({ dir, dryRun: true }); + assert.equal(dry.row, "dry-run"); + assert.ok(existsSync(join(dir, "lessons-2026-09.md")), "a dry run writes nothing"); + const r = consolidateDir({ dir, date: "2026-09-22" }); + assert.equal(r.kept.length, 1); + assert.equal( + readFileSync(join(dir, "CONSOLIDATED.md"), "utf8"), + `# Learned — consolidated 2026-09-22\n\n## shop\n- ${FLAKY}\n`, + ); + assert.ok(!existsSync(join(dir, "lessons-2026-09.md"))); + assert.equal(readdirSync(join(dir, "archive")).length, 1, "the original is archived"); + assert.equal(consolidateDir({ dir: tmp() }).row, "nothing"); +}); + +// A STUB `claude` sits first on PATH, so no real model is ever called from the suite: the +// default path must never invoke it, and the opt-in `--llm` path reaches the stub only. +test("the script runs deterministically with no model call; --llm is opt-in", { + skip: process.platform === "win32" && "bash script test", +}, () => { + const home = tmp("forge-learn-home-"); + const dir = join(home, ".claude", "skills", "learned"); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "lessons-2026-09.md"), `## 2026-09-10 14:02 — shop\n- ${FLAKY}\n`); + const stubBin = tmp("forge-learn-bin-"); + const calls = join(stubBin, "calls.log"); + writeFileSync( + join(stubBin, "claude"), + `#!/bin/sh\necho called >> "${calls}"\necho "Not logged in - please run /login"\n`, + ); + chmodSync(join(stubBin, "claude"), 0o755); + const refuted = repoWith("shop", FLAKY, { refute: true }); + const env = { ...process.env, HOME: home, PATH: `${stubBin}${delimiter}${process.env.PATH}` }; + const run = spawnSync("bash", [SCRIPT, "--repo", refuted.root], { env, encoding: "utf8" }); + assert.equal(run.status, 0, run.stdout + run.stderr); + assert.match(run.stdout, /dropped on ledger evidence: 1/); + assert.doesNotMatch(readFileSync(join(dir, "CONSOLIDATED.md"), "utf8"), /migration/); + assert.equal(existsSync(calls), false, "the default path never calls a model"); + const llm = spawnSync("bash", [SCRIPT, "--llm"], { env, encoding: "utf8" }); + assert.equal(readFileSync(calls, "utf8").trim(), "called", "only --llm reaches the model"); + assert.equal(llm.status, 1, "the stub's login error keeps the originals"); + assert.match(llm.stdout, /by model judgment/); + assert.match(llm.stdout, /originals kept/); +}); From bb63a64ed857c5fb01db63370c96f779e7d6237d Mon Sep 17 00:00:00 2001 From: Juber Shaikh Date: Tue, 22 Sep 2026 05:03:27 +0000 Subject: [PATCH 07/13] feat(router): universal cross-provider router core (MIRT + cost model + correlated cascade policy) Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JpQ15QdqdDUJLavNhExwvL --- bench/universal-router/fit_prior.mjs | 22 +++ data/models.json | 190 ++++++++++++++++++++ data/router_prior.json | 235 ++++++++++++++++++++++++ src/router/cost.js | 77 ++++++++ src/router/features.js | 62 +++++++ src/router/index.js | 234 ++++++++++++++++++++++++ src/router/lbfgs.js | 79 ++++++++ src/router/linalg.js | 40 +++++ src/router/mirt.js | 258 +++++++++++++++++++++++++++ src/router/policy.js | 105 +++++++++++ src/router/prior.js | 59 ++++++ src/router/quadrature.js | 110 ++++++++++++ src/router/registry.js | 51 ++++++ 13 files changed, 1522 insertions(+) create mode 100644 bench/universal-router/fit_prior.mjs create mode 100644 data/models.json create mode 100644 data/router_prior.json create mode 100644 src/router/cost.js create mode 100644 src/router/features.js create mode 100644 src/router/index.js create mode 100644 src/router/lbfgs.js create mode 100644 src/router/linalg.js create mode 100644 src/router/mirt.js create mode 100644 src/router/policy.js create mode 100644 src/router/prior.js create mode 100644 src/router/quadrature.js create mode 100644 src/router/registry.js diff --git a/bench/universal-router/fit_prior.mjs b/bench/universal-router/fit_prior.mjs new file mode 100644 index 0000000..841bae0 --- /dev/null +++ b/bench/universal-router/fit_prior.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node +// Fit the universal router's shipped prior (data/router_prior.json) from public per-task +// results. Input: a JSON file built from SWE-bench Verified and SWE-bench/experiments +// (harness-bench writes it; see bench/universal-router/README.md): +// { source: {...}, tasks: [{id, text}], outcomes: { : { : {resolved, cost} } } } +// +// node bench/universal-router/fit_prior.mjs [--out data/router_prior.json] [--only ] +import { readFileSync, writeFileSync } from "node:fs"; +import { buildPrior } from "../../src/router/prior.js"; + +const args = process.argv.slice(2); +const opt = (n, d) => (args.includes(n) ? args[args.indexOf(n) + 1] : d); +const input = JSON.parse(readFileSync(args[0], "utf8")); +const out = opt("--out", new URL("../../data/router_prior.json", import.meta.url).pathname); +const only = opt("--only") ? new Set(JSON.parse(readFileSync(opt("--only"), "utf8"))) : null; +const t0 = Date.now(); +const prior = buildPrior(input, only); +writeFileSync(out, `${JSON.stringify(prior, null, 2)}\n`); +console.log( + `wrote ${out}: ${prior.models.length} models, ${prior.provenance.tasks} tasks, k=${prior.mirt.k}, ` + + `scale=${prior.selection.chosen.scale}, ${((Date.now() - t0) / 1000).toFixed(1)}s`, +); diff --git a/data/models.json b/data/models.json new file mode 100644 index 0000000..9085022 --- /dev/null +++ b/data/models.json @@ -0,0 +1,190 @@ +{ + "$comment": "Model registry for the universal router. Data only: add models, prices and provider ids here or in .forge/models.json. Prices are USD per million tokens and are used only for models without observed attempt costs. `providers` maps a provider name (as in providers.js) to the id that provider serves; an empty map means the model is recommended by id but cannot be applied until you add a provider id.", + "models": [ + { + "id": "claude-haiku-4.5", + "label": "Claude Haiku 4.5", + "org": "Anthropic", + "run_model_id": "claude-haiku-4-5-20251001", + "benchmark_run": "20260217_mini-v2.0.0_claude-4-5-haiku-high", + "evidence": "SWE-bench Verified, mini-SWE-agent 2.0.0, one attempt per task (SWE-bench/experiments@40f164d, runs dated 2026-02-17)", + "price_in": 1, + "price_out": 5, + "price_source": "https://platform.claude.com/docs/en/about-claude/pricing (checked 2026-09-22)", + "providers": { + "anthropic": "claude-haiku-4-5-20251001" + } + }, + { + "id": "claude-sonnet-4.5", + "label": "Claude Sonnet 4.5", + "org": "Anthropic", + "run_model_id": "claude-sonnet-4-5-20250929", + "benchmark_run": "20260217_mini-v2.0.0_claude-4-5-sonnet-high", + "evidence": "SWE-bench Verified, mini-SWE-agent 2.0.0, one attempt per task (SWE-bench/experiments@40f164d, runs dated 2026-02-17)", + "price_in": 3, + "price_out": 15, + "price_source": "https://platform.claude.com/docs/en/about-claude/pricing (checked 2026-09-22)", + "providers": { + "anthropic": "claude-sonnet-4-5-20250929" + } + }, + { + "id": "claude-opus-4.5", + "label": "Claude Opus 4.5", + "org": "Anthropic", + "run_model_id": "claude-4-5-opus", + "benchmark_run": "20260217_mini-v2.0.0_claude-4-5-opus-high", + "evidence": "SWE-bench Verified, mini-SWE-agent 2.0.0, one attempt per task (SWE-bench/experiments@40f164d, runs dated 2026-02-17)", + "price_in": 5, + "price_out": 25, + "price_source": "https://platform.claude.com/docs/en/about-claude/pricing (checked 2026-09-22)", + "providers": { + "anthropic": "claude-opus-4-5" + } + }, + { + "id": "claude-opus-4.6", + "label": "Claude Opus 4.6", + "org": "Anthropic", + "run_model_id": "claude-opus-4-6", + "benchmark_run": "20260217_mini-v2.0.0_claude-4-6-opus", + "evidence": "SWE-bench Verified, mini-SWE-agent 2.0.0, one attempt per task (SWE-bench/experiments@40f164d, runs dated 2026-02-17)", + "price_in": null, + "price_out": null, + "price_source": null, + "providers": { + "anthropic": "claude-opus-4-6" + } + }, + { + "id": "deepseek-v3.2", + "label": "DeepSeek V3.2", + "org": "DeepSeek", + "run_model_id": "deepseek-v3.2", + "benchmark_run": "20260217_mini-v2.0.0_deepseek-3-2-high", + "evidence": "SWE-bench Verified, mini-SWE-agent 2.0.0, one attempt per task (SWE-bench/experiments@40f164d, runs dated 2026-02-17)", + "price_in": null, + "price_out": null, + "price_source": null, + "providers": {} + }, + { + "id": "gemini-3-flash", + "label": "Gemini 3 Flash", + "org": "Google DeepMind", + "run_model_id": "gemini-3-flash-preview", + "benchmark_run": "20260217_mini-v2.0.0_gemini-3-flash-high", + "evidence": "SWE-bench Verified, mini-SWE-agent 2.0.0, one attempt per task (SWE-bench/experiments@40f164d, runs dated 2026-02-17)", + "price_in": null, + "price_out": null, + "price_source": null, + "providers": {} + }, + { + "id": "glm-5", + "label": "GLM 5", + "org": "Z-AI", + "run_model_id": "glm-5", + "benchmark_run": "20260217_mini-v2.0.0_glm-5-high", + "evidence": "SWE-bench Verified, mini-SWE-agent 2.0.0, one attempt per task (SWE-bench/experiments@40f164d, runs dated 2026-02-17)", + "price_in": null, + "price_out": null, + "price_source": null, + "providers": {} + }, + { + "id": "gpt-5.2", + "label": "GPT 5.2", + "org": "OpenAI", + "run_model_id": "gpt-5-2", + "benchmark_run": "20260217_mini-v2.0.0_gpt-5-2-high", + "evidence": "SWE-bench Verified, mini-SWE-agent 2.0.0, one attempt per task (SWE-bench/experiments@40f164d, runs dated 2026-02-17)", + "price_in": null, + "price_out": null, + "price_source": null, + "providers": {} + }, + { + "id": "gpt-5-mini", + "label": "GPT 5 mini", + "org": "OpenAI", + "run_model_id": "gpt-5-mini-2025-08-07", + "benchmark_run": "20260217_mini-v2.0.0_gpt-5-mini", + "evidence": "SWE-bench Verified, mini-SWE-agent 2.0.0, one attempt per task (SWE-bench/experiments@40f164d, runs dated 2026-02-17)", + "price_in": null, + "price_out": null, + "price_source": null, + "providers": {} + }, + { + "id": "kimi-k2.5", + "label": "Kimi K2.5", + "org": "Moonshot AI", + "run_model_id": "kimi-k2.5", + "benchmark_run": "20260217_mini-v2.0.0_kimi-k2-5-high", + "evidence": "SWE-bench Verified, mini-SWE-agent 2.0.0, one attempt per task (SWE-bench/experiments@40f164d, runs dated 2026-02-17)", + "price_in": null, + "price_out": null, + "price_source": null, + "providers": {} + }, + { + "id": "minimax-m2.5", + "label": "MiniMax M2.5", + "org": "MiniMax", + "run_model_id": "minimax-m2.5", + "benchmark_run": "20260217_mini-v2.0.0_minimax-2-5-high", + "evidence": "SWE-bench Verified, mini-SWE-agent 2.0.0, one attempt per task (SWE-bench/experiments@40f164d, runs dated 2026-02-17)", + "price_in": null, + "price_out": null, + "price_source": null, + "providers": {} + }, + { + "id": "claude-sonnet-5", + "label": "Claude Sonnet 5", + "org": "Anthropic", + "run_model_id": null, + "benchmark_run": null, + "evidence": null, + "price_in": 3, + "price_out": 15, + "price_source": "forgekit src/model_tiers.json (pricingVerified 2026-07-17)", + "providers": { + "anthropic": "claude-sonnet-5" + }, + "note": "no public per-task runs in the prior: enters cold (population-mean ability, cost from price) until outcomes are recorded" + }, + { + "id": "claude-opus-4.8", + "label": "Claude Opus 4.8", + "org": "Anthropic", + "run_model_id": null, + "benchmark_run": null, + "evidence": null, + "price_in": 5, + "price_out": 25, + "price_source": "forgekit src/model_tiers.json (pricingVerified 2026-07-17)", + "providers": { + "anthropic": "claude-opus-4-8" + }, + "note": "cold until outcomes are recorded" + }, + { + "id": "claude-fable-5", + "label": "Claude Fable 5", + "org": "Anthropic", + "run_model_id": null, + "benchmark_run": null, + "evidence": null, + "price_in": 10, + "price_out": 50, + "price_source": "forgekit src/model_tiers.json (pricingVerified 2026-07-17)", + "providers": { + "anthropic": "claude-fable-5" + }, + "note": "cold until outcomes are recorded" + } + ] +} \ No newline at end of file diff --git a/data/router_prior.json b/data/router_prior.json new file mode 100644 index 0000000..6bbf8b7 --- /dev/null +++ b/data/router_prior.json @@ -0,0 +1,235 @@ +{ + "version": 1, + "models": [ + "claude-haiku-4.5", + "claude-opus-4.5", + "claude-opus-4.6", + "claude-sonnet-4.5", + "deepseek-v3.2", + "gemini-3-flash", + "glm-5", + "gpt-5-mini", + "gpt-5.2", + "kimi-k2.5", + "minimax-m2.5" + ], + "features": { + "names": [ + "log_chars", + "log_lines", + "code_fences", + "log_constraints", + "log_steps", + "rubric_knn", + "rubric_score", + "log_files", + "log_fanout", + "log_churn", + "past_mistakes", + "ambiguity" + ], + "mean": [ + 7.062787772579041, + 3.2878803054715684, + 1.1190000000000007, + 0.983066988785852, + 0.06182653266665947, + 0.2883249334784194, + 0.2611751963393547, + 0, + 0, + 0, + 0, + 0 + ], + "std": [ + 0.8417200458615252, + 0.9530457540503706, + 1.7333029163997837, + 0.9800798580583188, + 0.2789095760387337, + 0.19965141825734764, + 0.11687710780681453, + 1, + 1, + 1, + 1, + 1 + ] + }, + "mirt": { + "k": 1, + "a": [ + 1.952730331922173, + 3.7109748349346297, + 2.6916944244558723, + 3.0128015273548874, + 2.3548805451623376, + 3.144600475417333, + 2.4060353731375135, + 0.39030369993098823, + 2.4010665137393263, + 2.3135806472117144, + 2.9358804100992435 + ], + "w": [ + 0.11079596035046817, + 0.17624688394608876, + 0.1998995319450095, + -0.07873031584057663, + 0.17729210660376551, + -0.8135197867382431, + 0.7557548283615448, + 0, + 0, + 0, + 0, + 0 + ], + "L": [ + [ + 4.679573546562638 + ], + [ + 4.630794405823491 + ], + [ + 3.377363771855404 + ], + [ + 5.236017986598532 + ], + [ + 4.329023846786565 + ], + [ + 4.051818718273076 + ], + [ + 3.560734724743931 + ], + [ + 2.6691845328496466 + ], + [ + 3.5510834991146014 + ], + [ + 3.9583284784459396 + ], + [ + 3.723177648186695 + ] + ] + }, + "cost": { + "alpha": [ + -1.2900175533689853, + -0.49152431275772285, + -0.9824225464118961, + -0.5433188627351587, + -0.9991377763653935, + -1.107762151289058, + -0.9880484211079457, + -3.2422984987701424, + -1.0022113078344923, + -2.272123352441629, + -2.9650546991343676 + ], + "beta": [ + 0.023932639055548002, + 0.1304987203901819, + -0.00815546199250494, + 0.013730794119413263, + 0.059450774896442855, + -0.010695224225947497, + -0.020293167249973074, + 0, + 0, + 0, + 0, + 0 + ], + "s2": 0.43787253519418, + "rho": 0, + "kappa": -3.2870748890887924, + "source": [ + "observed", + "observed", + "observed", + "observed", + "observed", + "observed", + "observed", + "observed", + "observed", + "observed", + "observed" + ], + "pricedModels": 3, + "n": 5498 + }, + "selection": { + "chosen": { + "k": 1, + "scale": 2, + "heldOutLogLik": -1755.6790297911962 + }, + "table": [ + { + "k": 1, + "scale": 0.5, + "heldOutLogLik": -1824.7244269606604 + }, + { + "k": 1, + "scale": 1, + "heldOutLogLik": -1772.9470699393087 + }, + { + "k": 1, + "scale": 2, + "heldOutLogLik": -1755.6790297911962 + }, + { + "k": 2, + "scale": 0.5, + "heldOutLogLik": -1830.3849115902071 + }, + { + "k": 2, + "scale": 1, + "heldOutLogLik": -1793.7513544765607 + }, + { + "k": 2, + "scale": 2, + "heldOutLogLik": -1792.166234344018 + }, + { + "k": 3, + "scale": 0.5, + "heldOutLogLik": -1842.7640146747044 + }, + { + "k": 3, + "scale": 1, + "heldOutLogLik": -1801.7336988486554 + }, + { + "k": 3, + "scale": 2, + "heldOutLogLik": -1775.4902564936888 + } + ], + "folds": 3 + }, + "provenance": { + "benchmark": "SWE-bench Verified (rev 78f471b)", + "runs": "SWE-bench/experiments@40f164d, mini-SWE-agent 2.0.0, dated 2026-02-17", + "split": "all", + "tasks": 500, + "outcomes": 5500, + "fittedAt": "2026-09-22T05:00:30.522Z" + } +} diff --git a/src/router/cost.js b/src/router/cost.js new file mode 100644 index 0000000..edc225e --- /dev/null +++ b/src/router/cost.js @@ -0,0 +1,77 @@ +// Expected cost of one attempt by a model on a task. +// +// log cost = α_m + β·x + ε, ε ~ N(0, s²) ⇒ E[cost] = exp(α_m + β·x + s²/2) +// +// α_m (per model) and the shared slope β are fitted by least squares on observed attempt costs. +// A model with prices but no observed attempts gets α_m from its price: across the models that +// have both, α_k − log(blended price_k) is nearly constant (same agent, same token volume), so +// α_m = log(blended price_m) + mean(α_k − log blended price_k). The input/output blend ρ is the +// value that makes that difference most constant across those models (chosen from data). +import { leastSquares } from "./linalg.js"; + +/** + * @param {{model: number, x: number[], cost: number}[]} obs + * @param {number} nModels + * @param {number} nFeatures + * @param {{priceIn?: number|null, priceOut?: number|null}[]} [prices] per model, USD per Mtok + */ +export function fitCost(obs, nModels, nFeatures, prices = []) { + const used = obs.filter((o) => o.cost > 0 && Number.isFinite(o.cost)); + const seen = new Set(used.map((o) => o.model)); + const models = [...seen].sort((a, b) => a - b); + const col = new Map(models.map((m, i) => [m, i])); + const X = used.map((o) => [...models.map((m) => (m === o.model ? 1 : 0)), ...o.x]); + const y = used.map((o) => Math.log(o.cost)); + // A tiny ridge on the slopes only keeps the system well-posed with few observations. + const ridge = [...models.map(() => 0), ...new Array(nFeatures).fill(1e-6 * Math.max(1, used.length))]; + const coef = used.length > models.length ? leastSquares(X, y, ridge) : [...models.map(() => 0), ...new Array(nFeatures).fill(0)]; + const alpha = new Array(nModels).fill(null); + for (const m of models) alpha[m] = coef[col.get(m)]; + const beta = coef.slice(models.length); + let rss = 0; + used.forEach((o, i) => { + const pred = X[i].reduce((s, v, c) => s + v * coef[c], 0); + rss += (y[i] - pred) ** 2; + }); + const dof = Math.max(1, used.length - models.length - nFeatures); + const s2 = used.length ? rss / dof : 0; + + // Cold start from prices. + const priced = models.filter((m) => prices[m]?.priceIn > 0 && prices[m]?.priceOut > 0); + let rho = null; + let kappa = null; + if (priced.length >= 1) { + const spread = (r) => { + const d = priced.map((m) => alpha[m] - Math.log(r * prices[m].priceIn + (1 - r) * prices[m].priceOut)); + const mean = d.reduce((s, v) => s + v, 0) / d.length; + return { mean, var: d.reduce((s, v) => s + (v - mean) ** 2, 0) / d.length }; + }; + let best = null; + for (let i = 0; i <= 100; i++) { + const r = i / 100; + const sp = spread(r); + // Ties (e.g. a single priced model) resolve to the smallest ρ that is exactly as good, + // which is the least committal blend; the tie is reported via `pricedModels`. + if (!best || sp.var < best.var - 1e-15) best = { r, ...sp }; + } + rho = best.r; + kappa = best.mean; + } + const source = new Array(nModels).fill(null); + for (const m of models) source[m] = "observed"; + for (let m = 0; m < nModels; m++) { + if (alpha[m] !== null) continue; + const p = prices[m]; + if (kappa !== null && p?.priceIn > 0 && p?.priceOut > 0) { + alpha[m] = Math.log(rho * p.priceIn + (1 - rho) * p.priceOut) + kappa; + source[m] = "price"; + } + } + return { alpha, beta, s2, rho, kappa, source, pricedModels: priced.length, n: used.length }; +} + +/** Expected attempt cost per model for a task (null where the model's cost is unknown). */ +export function expectedCosts(costModel, x) { + const bx = costModel.beta.reduce((s, b, d) => s + b * (x[d] ?? 0), 0); + return costModel.alpha.map((a) => (a === null ? null : Math.exp(a + bx + costModel.s2 / 2))); +} diff --git a/src/router/features.js b/src/router/features.js new file mode 100644 index 0000000..7455bc7 --- /dev/null +++ b/src/router/features.js @@ -0,0 +1,62 @@ +// Task features for the universal router: countable properties of the task text and, when a +// repository is available, of the code it touches. The router learns how each feature shifts +// difficulty; nothing here decides a model. +import { routeTask, rubricComplexity, rubricSignals } from "../route.js"; + +export const FEATURE_NAMES = [ + "log_chars", + "log_lines", + "code_fences", + "log_constraints", + "log_steps", + "rubric_knn", + "rubric_score", + "log_files", + "log_fanout", + "log_churn", + "past_mistakes", + "ambiguity", +]; + +/** + * Raw (unstandardised) features. `root` may be null for text-only use (no repo signals). + * @param {string|null} root + * @param {string} task + */ +export function rawFeatures(root, task) { + const text = String(task ?? ""); + const sig = rubricSignals(text); + const rub = rubricComplexity(text); + let repo = { files: 0, fanout: 0, churn: 0, pastMistakes: 0, ambiguity: 0 }; + if (root) { + try { + repo = routeTask(root, text, { llm: false }).signals ?? repo; + } catch {} + } + return [ + Math.log1p(text.length), + Math.log1p((text.match(/\n/g) || []).length), + (text.match(/```/g) || []).length / 2, + Math.log1p(sig.nConstraints), + Math.log1p(sig.nSteps), + Number(rub.knn) || 0, + Number(rub.score) || 0, + Math.log1p(repo.files || 0), + Math.log1p(repo.fanout || 0), + Math.log1p(repo.churn || 0), + Number(repo.pastMistakes) || 0, + Number(repo.ambiguity) || 0, + ]; +} + +/** Standardisation fitted on training data (a zero-variance feature keeps scale 1). */ +export function fitScaler(rows) { + const d = rows[0]?.length ?? FEATURE_NAMES.length; + const mean = new Array(d).fill(0); + const std = new Array(d).fill(0); + for (const r of rows) for (let i = 0; i < d; i++) mean[i] += r[i] / rows.length; + for (const r of rows) for (let i = 0; i < d; i++) std[i] += (r[i] - mean[i]) ** 2 / rows.length; + return { names: FEATURE_NAMES.slice(0, d), mean, std: std.map((v) => (v > 1e-12 ? Math.sqrt(v) : 1)) }; +} + +export const standardise = (scaler, raw) => raw.map((v, i) => (v - scaler.mean[i]) / scaler.std[i]); diff --git a/src/router/index.js b/src/router/index.js new file mode 100644 index 0000000..4411971 --- /dev/null +++ b/src/router/index.js @@ -0,0 +1,234 @@ +// Universal router: pick the model, or the cascade of models, that minimises expected cost for +// the success probability the user asks for — across any provider in the registry. +// +// Pieces (each in its own module, each documented with its equation): +// features.js task → feature vector +// mirt.js P(model solves task) with correlated failures (multidimensional IRT) +// cost.js E[cost of one attempt] +// policy.js best single model or cascade under the chosen objective +// registry.js which models exist and who can serve them (data) +// +// Learning: `recordOutcome` appends (task features, model, verified pass/fail, cost) to +// .forge/route_outcomes.jsonl, and `fitRouter` refits with the shipped fit as the prior +// (a Bayesian update), writing .forge/router_model.json. The shipped fit comes from public +// per-task results (data/router_prior.json records its source and date). +import { createHash } from "node:crypto"; +import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { expectedCosts, fitCost } from "./cost.js"; +import { rawFeatures, standardise } from "./features.js"; +import { fitMirt, marginals, nodeProbabilities } from "./mirt.js"; +import { choose, parseObjective } from "./policy.js"; +import { loadRegistry, servableBy } from "./registry.js"; + +const SHIPPED_PRIOR = new URL("../../data/router_prior.json", import.meta.url); +const LOCAL_MODEL = (root) => join(root, ".forge", "router_model.json"); +const OUTCOMES = (root) => join(root, ".forge", "route_outcomes.jsonl"); + +const readJson = (p) => { + try { + return JSON.parse(readFileSync(p, "utf8")); + } catch { + return null; + } +}; + +/** The fitted router in effect: the project's refit if present, else the shipped prior fit. */ +export function loadRouterModel(root) { + const local = root && existsSync(LOCAL_MODEL(root)) ? readJson(LOCAL_MODEL(root)) : null; + if (local?.mirt) return { ...local, origin: ".forge/router_model.json" }; + const shipped = readJson(SHIPPED_PRIOR); + return shipped?.mirt ? { ...shipped, origin: "data/router_prior.json" } : null; +} + +/** + * Align a fitted model with the registry: models known to the fit keep their parameters; models + * only in the registry enter "cold" — ability and loadings at the mean of the fitted models (the + * population prior), cost from their price if the registry has one. + */ +export function alignModels(fitted, registry) { + const ids = registry.models.map((m) => m.id); + const idx = new Map(fitted.models.map((id, i) => [id, i])); + const k = fitted.mirt.k; + const meanA = fitted.mirt.a.reduce((s, v) => s + v, 0) / fitted.mirt.a.length; + const meanL = Array.from({ length: k }, (_, d) => fitted.mirt.L.reduce((s, r) => s + r[d], 0) / fitted.mirt.L.length); + const a = []; + const L = []; + const alpha = []; + const status = []; + const prices = registry.models.map((m) => ({ priceIn: m.price_in ?? null, priceOut: m.price_out ?? null })); + for (let i = 0; i < ids.length; i++) { + const j = idx.get(ids[i]); + if (j !== undefined) { + a.push(fitted.mirt.a[j]); + L.push(fitted.mirt.L[j]); + alpha.push(fitted.cost.alpha[j]); + status.push("fitted"); + } else { + a.push(meanA); + L.push(meanL); + const p = prices[i]; + const c = fitted.cost; + alpha.push( + c.kappa !== null && c.rho !== null && p.priceIn > 0 && p.priceOut > 0 + ? Math.log(c.rho * p.priceIn + (1 - c.rho) * p.priceOut) + c.kappa + : null, + ); + status.push("cold"); + } + } + return { + ids, + mirt: { k, a, w: fitted.mirt.w, L }, + cost: { ...fitted.cost, alpha }, + status, + }; +} + +/** + * Recommend a model or cascade for a task. + * @param {string|null} root + * @param {string} task + * @param {{objective?: string, maxDepth?: number, provider?: string, candidates?: string[], features?: number[]}} [opts] + */ +export function routeUniversal(root, task, opts = {}) { + const fitted = opts.model ?? loadRouterModel(root); + if (!fitted) return { ok: false, reason: "no fitted router model (data/router_prior.json missing)" }; + const registry = opts.registry ?? loadRegistry(root); + const aligned = alignModels(fitted, registry); + const raw = opts.features ?? rawFeatures(root, task); + const x = standardise(fitted.features, raw); + const objective = parseObjective(opts.objective ?? readConfigObjective(root)); + const allowed = new Set(opts.candidates ?? servableBy(registry, opts.provider ?? "any")); + const candidates = aligned.ids.map((id, i) => (allowed.has(id) ? i : -1)).filter((i) => i >= 0); + const nodes = nodeProbabilities(aligned.mirt, x); + const costs = expectedCosts(aligned.cost, x); + const maxDepth = Math.max(1, Math.min(opts.maxDepth ?? 3, candidates.length)); + const pick = choose(nodes, costs, candidates, objective, maxDepth); + if (!pick) return { ok: false, reason: "no candidate model has a known cost for this provider" }; + const p1 = marginals(nodes); + return { + ok: true, + cascade: pick.seq.map((i) => ({ + model: aligned.ids[i], + status: aligned.status[i], + pSolveAlone: p1[i], + expectedAttemptCost: costs[i], + })), + pSuccess: pick.p, + expectedCost: pick.cost, + objective, + target: pick.target, + targetMet: pick.targetMet, + bestSingle: { model: aligned.ids[pick.bestSingle.model], pSuccess: pick.bestSingle.p, expectedCost: pick.bestSingle.cost }, + candidates: candidates.length, + cascadesEvaluated: pick.evaluated, + fit: { origin: fitted.origin, k: fitted.mirt.k, provenance: fitted.provenance ?? null }, + taskRef: taskRef(task), + }; +} + +function readConfigObjective(root) { + if (!root) return undefined; + const cfg = readJson(join(root, ".forge", "config.json")); + return cfg?.route?.objective; +} + +export const taskRef = (task) => createHash("sha256").update(String(task)).digest("hex").slice(0, 16); + +/** + * Record a verified outcome of one attempt (the only evidence the router learns from). The task + * text is not stored: only its hash and features. + */ +export function recordOutcome(root, { task, model, passed, cost = null, features = null }) { + if (!model || typeof passed !== "boolean") throw new Error("recordOutcome needs model and passed (boolean)"); + const dir = join(root, ".forge"); + mkdirSync(dir, { recursive: true }); + const row = { + at: new Date().toISOString(), + task: taskRef(task), + features: features ?? rawFeatures(root, task), + model, + passed, + cost: Number.isFinite(cost) ? cost : null, + }; + appendFileSync(OUTCOMES(root), `${JSON.stringify(row)}\n`); + return row; +} + +export function readOutcomes(root) { + if (!existsSync(OUTCOMES(root))) return []; + return readFileSync(OUTCOMES(root), "utf8") + .split("\n") + .filter(Boolean) + .map((l) => { + try { + return JSON.parse(l); + } catch { + return null; + } + }) + .filter(Boolean); +} + +/** + * Refit on the project's recorded outcomes with the shipped fit as the prior mean (Bayesian + * update: few local outcomes barely move it, many outcomes dominate). Writes + * .forge/router_model.json and returns it. + */ +export function fitRouter(root, { outcomes = readOutcomes(root), registry = loadRegistry(root) } = {}) { + const shipped = readJson(SHIPPED_PRIOR); + if (!shipped?.mirt) throw new Error("data/router_prior.json missing: cannot fit without a prior"); + const base = alignModels(shipped, registry); + const index = new Map(base.ids.map((id, i) => [id, i])); + const byTask = new Map(); + const costObs = []; + let skipped = 0; + for (const o of outcomes) { + const m = index.get(o.model); + if (m === undefined || !Array.isArray(o.features)) { + skipped++; + continue; + } + const x = standardise(shipped.features, o.features); + if (!byTask.has(o.task)) byTask.set(o.task, { x, obs: [] }); + byTask.get(o.task).obs.push([m, o.passed ? 1 : 0]); + if (o.cost > 0) costObs.push({ model: m, x, cost: o.cost }); + } + const data = { nModels: base.ids.length, nFeatures: shipped.features.mean.length, tasks: [...byTask.values()] }; + const scale = shipped.selection?.chosen?.scale ?? 1; + const { params } = fitMirt(data, base.mirt.k, { + a: base.mirt.a, + w: base.mirt.w, + L: base.mirt.L, + scaleA: 2 * scale, + scaleW: scale, + scaleL: scale, + }); + // Cost: shipped α is worth one observation (unit-information prior); local attempts update it. + const alpha = base.cost.alpha.slice(); + for (let m = 0; m < alpha.length; m++) { + const mine = costObs.filter((c) => c.model === m); + if (!mine.length) continue; + const resid = mine.map((c) => Math.log(c.cost) - base.cost.beta.reduce((s, b, d) => s + b * c.x[d], 0)); + const prior = alpha[m] ?? resid.reduce((s, v) => s + v, 0) / resid.length; + alpha[m] = (prior + resid.reduce((s, v) => s + v, 0)) / (1 + resid.length); + } + const model = { + version: 1, + models: base.ids, + features: shipped.features, + mirt: params, + cost: { ...base.cost, alpha }, + selection: shipped.selection, + provenance: { + prior: shipped.provenance, + local: { outcomes: outcomes.length - skipped, skipped, tasks: data.tasks.length, fittedAt: new Date().toISOString() }, + }, + }; + mkdirSync(join(root, ".forge"), { recursive: true }); + writeFileSync(LOCAL_MODEL(root), JSON.stringify(model, null, 2)); + return model; +} + +export { fitCost }; diff --git a/src/router/lbfgs.js b/src/router/lbfgs.js new file mode 100644 index 0000000..da2b95f --- /dev/null +++ b/src/router/lbfgs.js @@ -0,0 +1,79 @@ +// Limited-memory BFGS with a backtracking Armijo line search. Minimises f: R^n -> R given a +// function returning { value, grad }. Used for maximum-a-posteriori fits of the router's models. + +const dot = (a, b) => { + let s = 0; + for (let i = 0; i < a.length; i++) s += a[i] * b[i]; + return s; +}; + +/** + * @param {(x: number[]) => {value: number, grad: number[]}} f + * @param {number[]} x0 + * @param {{maxIter?: number, memory?: number, gradTol?: number, relTol?: number}} [opts] + */ +export function minimize(f, x0, { maxIter = 500, memory = 10, gradTol = 1e-6, relTol = 1e-10 } = {}) { + let x = x0.slice(); + let { value: fx, grad: g } = f(x); + const S = []; + const Y = []; + let iter = 0; + for (; iter < maxIter; iter++) { + const gnorm = Math.sqrt(dot(g, g)); + if (!Number.isFinite(fx) || gnorm < gradTol * Math.max(1, Math.abs(fx))) break; + // Two-loop recursion: d = -H g. + const q = g.slice(); + const alpha = []; + for (let i = S.length - 1; i >= 0; i--) { + const rho = 1 / dot(Y[i], S[i]); + const a = rho * dot(S[i], q); + alpha[i] = a; + for (let j = 0; j < q.length; j++) q[j] -= a * Y[i][j]; + } + let gamma = 1; + if (S.length) gamma = dot(S.at(-1), Y.at(-1)) / dot(Y.at(-1), Y.at(-1)); + else gamma = 1 / Math.max(gnorm, 1); + for (let j = 0; j < q.length; j++) q[j] *= gamma; + for (let i = 0; i < S.length; i++) { + const rho = 1 / dot(Y[i], S[i]); + const b = rho * dot(Y[i], q); + for (let j = 0; j < q.length; j++) q[j] += S[i][j] * (alpha[i] - b); + } + let d = q.map((v) => -v); + let slope = dot(g, d); + if (!(slope < 0)) { + // Not a descent direction (curvature information went stale): restart from steepest descent. + S.length = 0; + Y.length = 0; + d = g.map((v) => -v / Math.max(gnorm, 1)); + slope = dot(g, d); + } + let step = 1; + let next; + let xn; + for (let ls = 0; ls < 40; ls++) { + xn = x.map((v, i) => v + step * d[i]); + next = f(xn); + if (Number.isFinite(next.value) && next.value <= fx + 1e-4 * step * slope) break; + step *= 0.5; + next = undefined; + } + if (!next) break; + const s = xn.map((v, i) => v - x[i]); + const y = next.grad.map((v, i) => v - g[i]); + if (dot(s, y) > 1e-12) { + S.push(s); + Y.push(y); + if (S.length > memory) { + S.shift(); + Y.shift(); + } + } + const improved = fx - next.value; + x = xn; + fx = next.value; + g = next.grad; + if (improved >= 0 && improved < relTol * Math.max(1, Math.abs(fx))) break; + } + return { x, value: fx, grad: g, iterations: iter }; +} diff --git a/src/router/linalg.js b/src/router/linalg.js new file mode 100644 index 0000000..9da127e --- /dev/null +++ b/src/router/linalg.js @@ -0,0 +1,40 @@ +// Small dense linear algebra for the router's least-squares fits. + +/** Solve A x = b (A square) by Gaussian elimination with partial pivoting. */ +export function solve(A, b) { + const n = A.length; + const M = A.map((row, i) => [...row, b[i]]); + for (let c = 0; c < n; c++) { + let p = c; + for (let r = c + 1; r < n; r++) if (Math.abs(M[r][c]) > Math.abs(M[p][c])) p = r; + if (Math.abs(M[p][c]) < 1e-12) throw new Error("singular system"); + [M[c], M[p]] = [M[p], M[c]]; + for (let r = 0; r < n; r++) { + if (r === c) continue; + const f = M[r][c] / M[c][c]; + for (let k = c; k <= n; k++) M[r][k] -= f * M[c][k]; + } + } + return M.map((row, i) => row[n] / row[i]); +} + +/** + * Ridge-regularised least squares: argmin ||y - X β||² + Σ_i ridge_i β_i². + * @param {number[][]} X rows + * @param {number[]} y + * @param {number[]} ridge per-coefficient penalty (0 = unpenalised) + */ +export function leastSquares(X, y, ridge) { + const p = X[0].length; + const A = Array.from({ length: p }, () => new Array(p).fill(0)); + const b = new Array(p).fill(0); + for (let i = 0; i < X.length; i++) { + const xi = X[i]; + for (let r = 0; r < p; r++) { + b[r] += xi[r] * y[i]; + for (let c = 0; c < p; c++) A[r][c] += xi[r] * xi[c]; + } + } + for (let r = 0; r < p; r++) A[r][r] += ridge[r] ?? 0; + return solve(A, b); +} diff --git a/src/router/mirt.js b/src/router/mirt.js new file mode 100644 index 0000000..a53948b --- /dev/null +++ b/src/router/mirt.js @@ -0,0 +1,258 @@ +// Multidimensional item response theory (MIRT) for "which model solves which task". +// +// P(model m solves task j | θ_j) = σ(a_m − w·x_j + λ_m·θ_j), θ_j ~ N(0, I_k) +// +// a_m ability of model m (any provider; no tiers) +// w·x_j difficulty of task j predicted from its features x_j (so unseen tasks get a difficulty) +// θ_j the part of task j's difficulty the features do not explain, shared by all models +// through their loadings λ_m. This is what makes failures correlated: when one model +// fails a task, others are more likely to fail it too, which is exactly what a cascade +// ("try A, and if it fails try B") must account for. +// +// Parameters are fitted by maximum a posteriori over the MARGINAL likelihood (θ integrated out +// with Gauss–Hermite quadrature), from sparse observations: each task may have been tried by +// any subset of models. The latent dimension k and the prior scale are chosen by K-fold +// cross-validated likelihood, not fixed in code. +import { minimize } from "./lbfgs.js"; +import { nodesFor, normalGrid } from "./quadrature.js"; + +const logSigmoid = (z) => (z >= 0 ? -Math.log1p(Math.exp(-z)) : z - Math.log1p(Math.exp(z))); +export const sigmoid = (z) => (z >= 0 ? 1 / (1 + Math.exp(-z)) : Math.exp(z) / (1 + Math.exp(z))); + +/** + * @typedef {{ nModels: number, nFeatures: number, tasks: {x: number[], obs: [number, 0|1][]}[] }} MirtData + * obs entries are [modelIndex, solved]. + * @typedef {{ k: number, a: number[], w: number[], L: number[][] }} MirtParams + * @typedef {{ a?: number[], w?: number[], L?: number[][], scaleA?: number, scaleW?: number, scaleL?: number }} MirtPrior + */ + +function pack(p) { + return [...p.a, ...p.w, ...p.L.flat()]; +} +function unpack(v, M, D, k) { + const a = v.slice(0, M); + const w = v.slice(M, M + D); + const L = []; + for (let m = 0; m < M; m++) L.push(v.slice(M + D + m * k, M + D + (m + 1) * k)); + return { k, a, w, L }; +} + +/** + * Negative log posterior and its gradient. + * @param {number[]} v packed parameters + * @param {MirtData} data + * @param {number} k + * @param {{points: number[][], weights: number[]}} grid + * @param {Required} prior + * @param {number[]} [taskIdx] subset of tasks (defaults to all) + */ +export function objective(v, data, k, grid, prior, taskIdx) { + const M = data.nModels; + const D = data.nFeatures; + const { a, w, L } = unpack(v, M, D, k); + const Q = grid.weights.length; + const logW = grid.weights.map(Math.log); + // LT[m][q] = λ_m · θ_q + const LT = L.map((lm) => + grid.points.map((t) => { + let s = 0; + for (let d = 0; d < k; d++) s += lm[d] * t[d]; + return s; + }), + ); + const ga = new Array(M).fill(0); + const gw = new Array(D).fill(0); + const gL = L.map(() => new Array(k).fill(0)); + let nll = 0; + const lq = new Array(Q); + const idx = taskIdx ?? data.tasks.map((_, j) => j); + for (const j of idx) { + const task = data.tasks[j]; + if (!task.obs.length) continue; + let b = 0; + for (let d = 0; d < D; d++) b += w[d] * task.x[d]; + let mx = -Infinity; + for (let q = 0; q < Q; q++) { + let s = logW[q]; + for (const [m, y] of task.obs) { + const z = a[m] - b + LT[m][q]; + s += y ? logSigmoid(z) : logSigmoid(-z); + } + lq[q] = s; + if (s > mx) mx = s; + } + let tot = 0; + for (let q = 0; q < Q; q++) { + lq[q] = Math.exp(lq[q] - mx); + tot += lq[q]; + } + nll -= mx + Math.log(tot); + // Posterior node weights r_q; d(-ll)/dz_mq = -r_q (y - σ(z)). + let gb = 0; + for (let q = 0; q < Q; q++) { + const r = lq[q] / tot; + if (r < 1e-300) continue; + for (const [m, y] of task.obs) { + const z = a[m] - b + LT[m][q]; + const g = r * (y - sigmoid(z)); + ga[m] -= g; + gb += g; + const t = grid.points[q]; + for (let d = 0; d < k; d++) gL[m][d] -= g * t[d]; + } + } + for (let d = 0; d < D; d++) gw[d] += gb * task.x[d]; + } + // Gaussian priors (MAP). + for (let m = 0; m < M; m++) { + const da = a[m] - prior.a[m]; + nll += (0.5 * da * da) / prior.scaleA ** 2; + ga[m] += da / prior.scaleA ** 2; + for (let d = 0; d < k; d++) { + const dl = L[m][d] - (prior.L[m]?.[d] ?? 0); + nll += (0.5 * dl * dl) / prior.scaleL ** 2; + gL[m][d] += dl / prior.scaleL ** 2; + } + } + for (let d = 0; d < D; d++) { + const dw = w[d] - prior.w[d]; + nll += (0.5 * dw * dw) / prior.scaleW ** 2; + gw[d] += dw / prior.scaleW ** 2; + } + return { value: nll, grad: [...ga, ...gw, ...gL.flat()] }; +} + +function fullPrior(M, D, k, prior = {}) { + return { + a: prior.a ?? new Array(M).fill(0), + w: prior.w ?? new Array(D).fill(0), + // Default loading prior mean: a common positive first factor (models agree on which tasks + // are hard), zero on further factors. It only centres the prior; the data moves it. + L: prior.L ?? Array.from({ length: M }, () => Array.from({ length: k }, (_, d) => (d === 0 ? 1 : 0))), + scaleA: prior.scaleA ?? 2, + scaleW: prior.scaleW ?? 1, + scaleL: prior.scaleL ?? 1, + }; +} + +/** + * MAP fit for a given k and prior. + * @param {MirtData} data + * @param {number} k + * @param {MirtPrior} [prior] + * @param {number[]} [taskIdx] + */ +export function fitMirt(data, k, prior = {}, taskIdx) { + const M = data.nModels; + const D = data.nFeatures; + const pr = fullPrior(M, D, k, prior); + const grid = normalGrid(k, nodesFor(k)); + // Start at the prior mean, with a small deterministic asymmetry on extra factors so they can + // leave the symmetric saddle at zero. + const init = { + k, + a: pr.a.slice(), + w: pr.w.slice(), + L: pr.L.map((row, m) => row.map((v, d) => (d === 0 ? v : v + 0.1 * Math.cos(1 + m * (d + 1))))), + }; + const res = minimize((v) => objective(v, data, k, grid, pr, taskIdx), pack(init), { + maxIter: 400, + }); + return { params: unpack(res.x, M, D, k), nlp: res.value, iterations: res.iterations }; +} + +/** Held-out marginal log-likelihood of tasks (no prior terms). */ +export function heldOutLogLik(params, data, taskIdx) { + const M = data.nModels; + const D = data.nFeatures; + const k = params.k; + const grid = normalGrid(k, nodesFor(k)); + // Flat, zero-strength prior: objective() adds prior terms, so neutralise them with huge scales. + const flat = { ...fullPrior(M, D, k), scaleA: 1e12, scaleW: 1e12, scaleL: 1e12 }; + return -objective(pack(params), data, k, grid, flat, taskIdx).value; +} + +/** + * Choose k and the prior scale by K-fold cross-validated held-out likelihood, then refit on all + * tasks with the winner. Folds are assigned deterministically. + * @param {MirtData} data + * @param {{ks?: number[], scales?: number[], folds?: number, prior?: MirtPrior}} [opts] + */ +export function selectAndFit(data, { ks = [1, 2, 3], scales = [0.5, 1, 2], folds = 3, prior = {}, maxExpand = 4 } = {}) { + const J = data.tasks.length; + const order = data.tasks.map((_, j) => j).sort((x, y) => hash32(x) - hash32(y)); + const foldOf = new Array(J); + order.forEach((j, i) => { + foldOf[j] = i % folds; + }); + const table = []; + let best = null; + const cv = (k, s) => { + let ll = 0; + for (let f = 0; f < folds; f++) { + const tr = []; + const te = []; + for (let j = 0; j < J; j++) (foldOf[j] === f ? te : tr).push(j); + const { params } = fitMirt(data, k, { ...prior, scaleA: 2 * s, scaleW: s, scaleL: s }, tr); + ll += heldOutLogLik(params, data, te); + } + table.push({ k, scale: s, heldOutLogLik: ll }); + if (!best || ll > best.heldOutLogLik) best = { k, scale: s, heldOutLogLik: ll }; + return ll; + }; + for (const k of ks) { + const grid = [...scales].sort((x, y) => x - y); + const lls = grid.map((s) => cv(k, s)); + // If the best scale sits on an edge of the grid, the optimum may lie beyond it: step outward + // (halving or doubling) while the held-out likelihood keeps improving. + for (let n = 0; n < maxExpand; n++) { + const bi = lls.indexOf(Math.max(...lls)); + if (bi !== 0 && bi !== grid.length - 1) break; + const low = bi === 0; + const s = low ? grid[0] / 2 : grid.at(-1) * 2; + const ll = cv(k, s); + if (low) { + grid.unshift(s); + lls.unshift(ll); + } else { + grid.push(s); + lls.push(ll); + } + if (ll <= lls[low ? 1 : lls.length - 2]) break; + } + } + const fit = fitMirt(data, best.k, { ...prior, scaleA: 2 * best.scale, scaleW: best.scale, scaleL: best.scale }); + return { ...fit, selection: { chosen: best, table, folds } }; +} + +function hash32(n) { + let h = (n + 0x9e3779b9) | 0; + h = Math.imul(h ^ (h >>> 16), 0x85ebca6b); + h = Math.imul(h ^ (h >>> 13), 0xc2b2ae35); + return (h ^ (h >>> 16)) >>> 0; +} + +/** + * Conditional success probabilities at each quadrature node, for one task. + * @param {MirtParams} params + * @param {number[]} x standardised feature vector + * @returns {{P: number[][], weights: number[]}} P[m][q] + */ +export function nodeProbabilities(params, x) { + const grid = normalGrid(params.k, nodesFor(params.k)); + let b = 0; + for (let d = 0; d < params.w.length; d++) b += params.w[d] * x[d]; + const P = params.a.map((am, m) => + grid.points.map((t) => { + let s = am - b; + for (let d = 0; d < params.k; d++) s += params.L[m][d] * t[d]; + return sigmoid(s); + }), + ); + return { P, weights: grid.weights }; +} + +/** Marginal P(model m solves the task) = E_θ[σ(...)]. */ +export function marginals({ P, weights }) { + return P.map((row) => row.reduce((s, p, q) => s + p * weights[q], 0)); +} diff --git a/src/router/policy.js b/src/router/policy.js new file mode 100644 index 0000000..4042933 --- /dev/null +++ b/src/router/policy.js @@ -0,0 +1,105 @@ +// Choosing a model, or a cascade of models, for one task. +// +// A cascade s = (m1, m2, …) runs m1; only if an external check says it failed does it run m2, +// and so on. With node probabilities P[m][q] = P(m solves | θ_q) and weights w_q: +// +// P(s solves) = 1 − Σ_q w_q Π_{m∈s} (1 − P[m][q]) +// E[cost of s] = Σ_i c_{m_i} · Σ_q w_q Π_{l 1); + let cost = 0; + for (const m of seq) { + let reach = 0; + for (let q = 0; q < w.length; q++) reach += w[q] * fail[q]; + cost += costs[m] * reach; + for (let q = 0; q < w.length; q++) fail[q] *= 1 - P[m][q]; + } + let pf = 0; + for (let q = 0; q < w.length; q++) pf += w[q] * fail[q]; + return { p: 1 - pf, cost }; +} + +/** All ordered cascades of distinct candidates up to `maxDepth` models long. */ +export function* cascades(candidates, maxDepth) { + function* rec(prefix, used) { + if (prefix.length) yield prefix; + if (prefix.length >= maxDepth) return; + for (const m of candidates) if (!used.has(m)) yield* rec([...prefix, m], new Set([...used, m])); + } + yield* rec([], new Set()); +} + +/** + * Parse an objective string: "match-best-single" | "target:0.85" | "value:2" | "budget:0.5". + * @param {string|undefined} spec + */ +export function parseObjective(spec) { + if (!spec || spec === "match-best-single" || spec === "match") return { kind: "match-best-single" }; + const [kind, raw] = String(spec).split(":"); + const v = Number(raw); + if (kind === "target" && v > 0 && v < 1) return { kind, target: v }; + if (kind === "value" && v > 0) return { kind, value: v }; + if (kind === "budget" && v > 0) return { kind, budget: v }; + throw new Error(`unknown objective "${spec}" (use match-best-single, target:<0..1>, value:<$>, budget:<$>)`); +} + +/** + * @param {{P: number[][], weights: number[]}} nodes + * @param {(number|null)[]} costs expected attempt cost per model (null = unknown, excluded) + * @param {number[]} candidates model indices allowed + * @param {{kind: string, target?: number, value?: number, budget?: number}} objective + * @param {number} maxDepth + */ +export function choose(nodes, costs, candidates, objective, maxDepth) { + const { P, weights } = nodes; + const usable = candidates.filter((m) => costs[m] !== null && Number.isFinite(costs[m])); + if (!usable.length) return null; + const single = usable.map((m) => ({ m, ...cascadeStats(P, weights, costs, [m]) })); + const bestSingle = single.reduce((a, b) => (b.p > a.p ? b : a)); + const target = + objective.kind === "match-best-single" ? bestSingle.p : objective.kind === "target" ? objective.target : null; + let best = null; + let bestKey = null; + let evaluated = 0; + const eps = 1e-12; + for (const seq of cascades(usable, maxDepth)) { + const st = cascadeStats(P, weights, costs, seq); + evaluated++; + let key; + if (target !== null) key = st.p >= target - eps ? [0, st.cost, -st.p, seq.length] : [1, -st.p, st.cost, seq.length]; + else if (objective.kind === "value") key = [0, -(objective.value * st.p - st.cost), seq.length]; + else key = st.cost <= objective.budget + eps ? [0, -st.p, st.cost, seq.length] : [1, st.cost, -st.p, seq.length]; + if (!bestKey || lexLess(key, bestKey)) { + bestKey = key; + best = { seq, ...st }; + } + } + return { + ...best, + target, + targetMet: target === null ? null : best.p >= target - eps, + bestSingle: { model: bestSingle.m, p: bestSingle.p, cost: bestSingle.cost }, + evaluated, + }; +} + +function lexLess(a, b) { + for (let i = 0; i < a.length; i++) { + if (a[i] < b[i] - 1e-15) return true; + if (a[i] > b[i] + 1e-15) return false; + } + return false; +} diff --git a/src/router/prior.js b/src/router/prior.js new file mode 100644 index 0000000..70a7ee5 --- /dev/null +++ b/src/router/prior.js @@ -0,0 +1,59 @@ +// Fit the universal router from per-task outcome data (tasks × models, sparse allowed): +// features → standardisation → MIRT with k and prior scale chosen by cross-validation → cost model. +import { fitCost } from "./cost.js"; +import { fitScaler, rawFeatures, standardise } from "./features.js"; +import { selectAndFit } from "./mirt.js"; +import { loadRegistry } from "./registry.js"; + +/** + * @param {{source?: object, tasks: {id: string, text: string, features?: number[]}[], + * outcomes: Record>}} input + * @param {Set|null} [only] restrict to these task ids + * @param {{registry?: {models: object[]}, root?: string|null}} [opts] + */ +export function buildPrior(input, only = null, { registry = loadRegistry(null), root = null } = {}) { + const models = Object.keys(input.outcomes).filter((id) => registry.models.some((m) => m.id === id)); + const tasks = input.tasks.filter((t) => !only || only.has(t.id)); + const raw = tasks.map((t) => t.features ?? rawFeatures(root, t.text)); + const scaler = fitScaler(raw); + const X = raw.map((r) => standardise(scaler, r)); + const data = { + nModels: models.length, + nFeatures: scaler.mean.length, + tasks: tasks.map((t, j) => ({ + x: X[j], + obs: models + .map((id, m) => [m, input.outcomes[id][t.id]]) + .filter(([, o]) => o) + .map(([m, o]) => [m, o.resolved ? 1 : 0]), + })), + }; + const fit = selectAndFit(data); + const costObs = []; + tasks.forEach((t, j) => + models.forEach((id, m) => { + const o = input.outcomes[id][t.id]; + if (o?.cost > 0) costObs.push({ model: m, x: X[j], cost: o.cost }); + }), + ); + const prices = models.map((id) => { + const r = registry.models.find((m) => m.id === id); + return { priceIn: r?.price_in ?? null, priceOut: r?.price_out ?? null }; + }); + const cost = fitCost(costObs, models.length, scaler.mean.length, prices); + return { + version: 1, + models, + features: scaler, + mirt: fit.params, + cost, + selection: fit.selection, + provenance: { + ...(input.source ?? {}), + tasks: tasks.length, + outcomes: data.tasks.reduce((s, t) => s + t.obs.length, 0), + fittedAt: new Date().toISOString(), + }, + }; +} + diff --git a/src/router/quadrature.js b/src/router/quadrature.js new file mode 100644 index 0000000..ad23e0f --- /dev/null +++ b/src/router/quadrature.js @@ -0,0 +1,110 @@ +// Gauss–Hermite quadrature for expectations under a standard normal, computed (not tabulated) +// with the Golub–Welsch algorithm: the nodes are the eigenvalues of the Jacobi matrix of the +// probabilists' Hermite polynomials (zero diagonal, off-diagonal sqrt(i)), and each weight is the +// squared first component of the matching normalised eigenvector. + +/** + * Eigen-decomposition of a symmetric tridiagonal matrix (implicit QL with Wilkinson shifts), + * returning eigenvalues and the first component of each eigenvector. + * @param {number[]} diag + * @param {number[]} off off[i] couples i and i+1 (length n-1) + */ +function tridiagEigen(diag, off) { + const n = diag.length; + const d = diag.slice(); + const e = [...off, 0]; + // z holds the first row of the accumulated rotation matrix (= first eigenvector components). + const z = Array.from({ length: n }, (_, i) => (i === 0 ? 1 : 0)); + for (let l = 0; l < n; l++) { + for (let iter = 0; iter < 200; iter++) { + let m = l; + for (; m < n - 1; m++) { + const dd = Math.abs(d[m]) + Math.abs(d[m + 1]); + if (Math.abs(e[m]) <= Number.EPSILON * dd) break; + } + if (m === l) break; + let g = (d[l + 1] - d[l]) / (2 * e[l]); + let r = Math.hypot(g, 1); + g = d[m] - d[l] + e[l] / (g + (g >= 0 ? Math.abs(r) : -Math.abs(r))); + let s = 1; + let c = 1; + let p = 0; + let i = m - 1; + for (; i >= l; i--) { + let f = s * e[i]; + const b = c * e[i]; + r = Math.hypot(f, g); + e[i + 1] = r; + if (r === 0) { + d[i + 1] -= p; + e[m] = 0; + break; + } + s = f / r; + c = g / r; + g = d[i + 1] - p; + r = (d[i] - g) * s + 2 * c * b; + p = s * r; + d[i + 1] = g + p; + g = c * r - b; + f = z[i + 1]; + z[i + 1] = s * z[i] + c * f; + z[i] = c * z[i] - s * f; + } + if (r === 0 && i >= l) continue; + d[l] -= p; + e[l] = g; + e[m] = 0; + } + } + return { values: d, first: z }; +} + +/** + * Nodes and weights with Σ w·f(x) ≈ E[f(X)], X ~ N(0, 1). Exact for polynomials of degree + * below 2q. + * @param {number} q number of nodes (≥ 1) + */ +export function normalNodes(q) { + if (!Number.isInteger(q) || q < 1) throw new Error("normalNodes: q must be a positive integer"); + if (q === 1) return { x: [0], w: [1] }; + const off = Array.from({ length: q - 1 }, (_, i) => Math.sqrt(i + 1)); + const { values, first } = tridiagEigen(new Array(q).fill(0), off); + const pairs = values.map((x, i) => [x, first[i] ** 2]).sort((a, b) => a[0] - b[0]); + const total = pairs.reduce((s, [, w]) => s + w, 0); + return { x: pairs.map(([x]) => x), w: pairs.map(([, w]) => w / total) }; +} + +/** + * Product grid for E over N(0, I_k): Q = q^k points. + * @param {number} k dimensions + * @param {number} q nodes per dimension + * @returns {{points: number[][], weights: number[]}} + */ +export function normalGrid(k, q) { + const { x, w } = normalNodes(q); + let points = [[]]; + let weights = [1]; + for (let d = 0; d < k; d++) { + const np = []; + const nw = []; + for (let i = 0; i < points.length; i++) + for (let j = 0; j < x.length; j++) { + np.push([...points[i], x[j]]); + nw.push(weights[i] * w[j]); + } + points = np; + weights = nw; + } + return { points, weights }; +} + +/** + * Nodes per dimension so the grid stays near a fixed evaluation budget: exactness degree grows + * with q, cost grows as q^k. Budget ≈ 256 points keeps a fit over a few thousand outcomes fast. + * @param {number} k + * @param {number} [budget] + */ +export function nodesFor(k, budget = 256) { + return Math.max(3, Math.floor(budget ** (1 / Math.max(1, k)))); +} diff --git a/src/router/registry.js b/src/router/registry.js new file mode 100644 index 0000000..b50c272 --- /dev/null +++ b/src/router/registry.js @@ -0,0 +1,51 @@ +// The model registry is data, not code: which models exist, who serves them, what they cost, +// and where any prior evidence about them came from. The shipped file (data/models.json) is a +// starting point; `.forge/models.json` in a project adds models, overrides prices or provider +// ids, and can disable entries. No model, vendor or tier is named anywhere in router code. +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +const SHIPPED = new URL("../../data/models.json", import.meta.url); + +function readJson(path) { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return null; + } +} + +/** + * @param {string|null} root project root (for .forge/models.json), or null + * @returns {{models: object[], sources: string[]}} + */ +export function loadRegistry(root) { + const base = readJson(SHIPPED) ?? { models: [] }; + const byId = new Map(base.models.map((m) => [m.id, { ...m }])); + const sources = ["data/models.json"]; + const localPath = root ? join(root, ".forge", "models.json") : null; + if (localPath && existsSync(localPath)) { + const local = readJson(localPath); + if (local?.models) { + sources.push(".forge/models.json"); + for (const m of local.models) { + if (!m?.id) continue; + const prev = byId.get(m.id) ?? {}; + byId.set(m.id, { ...prev, ...m, providers: { ...(prev.providers ?? {}), ...(m.providers ?? {}) } }); + } + } + } + const models = [...byId.values()].filter((m) => m.enabled !== false); + return { models, sources }; +} + +/** + * Registry ids a provider can serve. "any" returns every enabled model (advice only: the caller + * must still map ids to a provider before applying). + * @param {{models: object[]}} registry + * @param {string} provider + */ +export function servableBy(registry, provider) { + if (!provider || provider === "any") return registry.models.map((m) => m.id); + return registry.models.filter((m) => m.providers?.[provider]).map((m) => m.id); +} From 271e84e34709d87a4722d99d7fa496ef1ffbf8e1 Mon Sep 17 00:00:00 2001 From: Juber Shaikh Date: Tue, 22 Sep 2026 05:21:44 +0000 Subject: [PATCH 08/13] feat(router): forge route universal|outcome|fit|models, tests, shipped data in package files Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JpQ15QdqdDUJLavNhExwvL --- bench/universal-router/README.md | 21 ++++ package.json | 3 +- src/cli.js | 93 ++++++++++++++++++ test/router_math.test.js | 164 +++++++++++++++++++++++++++++++ test/router_universal.test.js | 87 ++++++++++++++++ 5 files changed, 367 insertions(+), 1 deletion(-) create mode 100644 bench/universal-router/README.md create mode 100644 test/router_math.test.js create mode 100644 test/router_universal.test.js diff --git a/bench/universal-router/README.md b/bench/universal-router/README.md new file mode 100644 index 0000000..0c9be56 --- /dev/null +++ b/bench/universal-router/README.md @@ -0,0 +1,21 @@ +# Universal router: shipped prior + +`data/router_prior.json` is the fit the universal router uses until a project records its own +outcomes (`forge route outcome`, then `forge route fit`). It was fitted from public per-task +results: + +- **Tasks:** the 500 issues of SWE-bench Verified (dataset revision `78f471b`); the issue text is the task. +- **Runs:** eleven models from seven providers, each run once per issue with the same agent scaffold (mini-SWE-agent 2.0.0), taken from SWE-bench/experiments @ `40f164d` (runs dated 2026-02-17). Each run gives a verified resolved/unresolved outcome and the observed cost. + +To regenerate it, build the input JSON, then run: + +```bash +# harness-bench writes the input from SWE-bench Verified + SWE-bench/experiments: +# python3 -m hbench.cli build universal --swe-exp --forgekit +# python3 -c "from hbench.tracks import universal as U; U.export_for_forgekit(Path('universal_all.json'))" +node bench/universal-router/fit_prior.mjs universal_all.json --out data/router_prior.json +``` + +The fit chooses the latent dimension k and the prior scale by 3-fold cross-validation. That selection is recorded in `selection` inside the file. + +Raw per-task results are not redistributed here, only the fitted parameters and their provenance. diff --git a/package.json b/package.json index 0141bd7..f0c4bd7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@codewithjuber/forgekit", "version": "1.0.0", - "description": "Shared memory, impact analysis, and guardrail hooks for AI coding agents — authored once, emitted as native config for Claude Code, Codex, Cursor, Gemini, Aider, and more.", + "description": "Shared memory, impact analysis, and guardrail hooks for AI coding agents \u2014 authored once, emitted as native config for Claude Code, Codex, Cursor, Gemini, Aider, and more.", "type": "module", "bin": { "forge": "src/cli.js" @@ -34,6 +34,7 @@ }, "files": [ "src", + "data", "source", "global", "templates", diff --git a/src/cli.js b/src/cli.js index 9c60d4a..4af2130 100755 --- a/src/cli.js +++ b/src/cli.js @@ -1988,6 +1988,9 @@ HANDLERS.route = async (argv) => { console.log(" and calibrating on real routing outcomes needs data forge does not record"); return; } + if (["universal", "outcome", "fit", "models"].includes(argv[1]) || argv.includes("--universal")) { + return routeUniversalCli(argv); + } const json = argv.includes("--json"); const apply = argv.includes("--apply"); const providerIdx = argv.indexOf("--provider"); @@ -2051,6 +2054,96 @@ HANDLERS.route = async (argv) => { } return; }; +// Universal router (src/router): any provider's models, chosen by expected cost for the success +// probability asked for. Models come from data/models.json and .forge/models.json. +async function routeUniversalCli(argv) { + const U = await import("./router/index.js"); + const { loadRegistry, servableBy } = await import("./router/registry.js"); + const json = argv.includes("--json"); + const val = (flag) => (argv.includes(flag) ? argv[argv.indexOf(flag) + 1] : undefined); + const VALUED = new Set(["--objective", "--provider", "--model", "--cost", "--depth"]); + const words = argv.slice(1).filter((a, i, arr) => !a.startsWith("--") && !VALUED.has(arr[i - 1] ?? "")); + const sub = ["outcome", "fit", "models", "universal"].includes(words[0]) ? words.shift() : "universal"; + const root = process.cwd(); + if (sub === "models") { + const reg = loadRegistry(root); + const fit = U.loadRouterModel(root); + const rows = reg.models.map((m) => ({ + id: m.id, + org: m.org ?? null, + status: fit?.models.includes(m.id) ? "fitted" : "cold", + providers: Object.keys(m.providers ?? {}), + price: m.price_in != null ? `${m.price_in}/${m.price_out}` : null, + })); + if (json) return console.log(JSON.stringify({ sources: reg.sources, fit: fit?.origin ?? null, models: rows }, null, 2)); + heading(`${BRAND.brand} route models — registry (${reg.sources.join(" + ")})\n`); + for (const r of rows) + console.log( + ` ${r.id.padEnd(22)} ${String(r.org ?? "").padEnd(16)} ${r.status.padEnd(7)} ${r.price ? `$${r.price}/Mtok`.padEnd(14) : "".padEnd(14)} ${r.providers.join(", ") || "(no provider id: advice only)"}`, + ); + console.log(`\n fit in use: ${fit?.origin ?? "none"}`); + return; + } + if (sub === "fit") { + const model = U.fitRouter(root); + if (json) return console.log(JSON.stringify(model.provenance, null, 2)); + console.log(` refit on ${model.provenance.local.outcomes} recorded outcome(s) over ${model.provenance.local.tasks} task(s); wrote .forge/router_model.json`); + return; + } + const task = words.join(" "); + if (!task) { + console.error( + 'usage: forge route universal "" [--objective match-best-single|target:

|value:<$>|budget:<$>] [--provider |any] [--depth ] [--json]\n' + + ' forge route outcome "" --model --pass|--fail [--cost ]\n' + + " forge route fit | forge route models", + ); + process.exitCode = 1; + return; + } + if (sub === "outcome") { + const passed = argv.includes("--pass") ? true : argv.includes("--fail") ? false : undefined; + const cost = val("--cost") !== undefined ? Number(val("--cost")) : null; + try { + const row = U.recordOutcome(root, { task, model: val("--model"), passed, cost }); + if (json) return console.log(JSON.stringify(row, null, 2)); + console.log(` recorded ${row.model} ${row.passed ? "pass" : "fail"} for task ${row.task} (.forge/route_outcomes.jsonl)`); + } catch (e) { + console.error(` ${e.message}`); + process.exitCode = 1; + } + return; + } + let rec; + try { + rec = U.routeUniversal(root, task, { + objective: val("--objective"), + provider: val("--provider") ?? "any", + maxDepth: val("--depth") ? Number(val("--depth")) : undefined, + }); + } catch (e) { + console.error(` ${e.message}`); + process.exitCode = 1; + return; + } + if (json) return console.log(JSON.stringify(rec, null, 2)); + if (!rec.ok) { + console.error(` ${rec.reason}`); + process.exitCode = 1; + return; + } + heading(`${BRAND.brand} route universal — ${rec.objective.kind}${rec.target != null ? ` (target ${rec.target.toFixed(2)})` : ""}\n`); + rec.cascade.forEach((c, i) => + console.log( + ` ${i === 0 ? "→" : "then, if a check fails →"} ${paint(c.model, "accent")} P(solve alone) ${c.pSolveAlone.toFixed(2)} · ~$${c.expectedAttemptCost.toFixed(3)}/attempt${c.status === "cold" ? " · cold (no outcomes yet)" : ""}`, + ), + ); + console.log( + `\n P(success) ${rec.pSuccess.toFixed(2)} · expected cost $${rec.expectedCost.toFixed(3)} · best single: ${rec.bestSingle.model} ${rec.bestSingle.pSuccess.toFixed(2)} at $${rec.bestSingle.expectedCost.toFixed(3)}`, + ); + console.log(` ${rec.candidates} candidate model(s), ${rec.cascadesEvaluated} cascade(s) compared · fit: ${rec.fit.origin}`); + console.log(` learn from results: \`${BRAND.cli} route outcome "" --model --pass|--fail --cost \`, then \`${BRAND.cli} route fit\``); +} + HANDLERS.anchor = async (argv) => { const { goalDrift, renderAnchor } = await import("./anchor.js"); const { clearGoal, getGoal, setGoal } = await import("./goal.js"); diff --git a/test/router_math.test.js b/test/router_math.test.js new file mode 100644 index 0000000..04b769e --- /dev/null +++ b/test/router_math.test.js @@ -0,0 +1,164 @@ +// Universal router: the numerical pieces (quadrature, optimiser, MIRT, cost, cascade policy). +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { expectedCosts, fitCost } from "../src/router/cost.js"; +import { minimize } from "../src/router/lbfgs.js"; +import { fitMirt, heldOutLogLik, marginals, nodeProbabilities, objective, sigmoid } from "../src/router/mirt.js"; +import { cascadeStats, cascades, choose, parseObjective } from "../src/router/policy.js"; +import { normalGrid, normalNodes } from "../src/router/quadrature.js"; + +// Deterministic PRNG so every run sees the same synthetic data. +function rng(seed) { + let s = seed >>> 0; + const u = () => { + s = (Math.imul(s, 1664525) + 1013904223) >>> 0; + return (s + 0.5) / 4294967296; + }; + return { u, g: () => Math.sqrt(-2 * Math.log(u())) * Math.cos(2 * Math.PI * u()) }; +} + +test("quadrature: Gauss–Hermite nodes reproduce standard-normal moments exactly", () => { + const { x, w } = normalNodes(12); + const m = (p) => x.reduce((s, xi, i) => s + w[i] * xi ** p, 0); + assert.ok(Math.abs(m(0) - 1) < 1e-12); + assert.ok(Math.abs(m(1)) < 1e-12); + assert.ok(Math.abs(m(2) - 1) < 1e-12); + assert.ok(Math.abs(m(4) - 3) < 1e-10); + assert.ok(Math.abs(m(6) - 15) < 1e-8); + const g = normalGrid(2, 4); + assert.equal(g.points.length, 16); + assert.ok(Math.abs(g.weights.reduce((s, v) => s + v, 0) - 1) < 1e-12); +}); + +test("lbfgs: finds the Rosenbrock minimum", () => { + const f = ([a, b]) => ({ + value: (1 - a) ** 2 + 100 * (b - a * a) ** 2, + grad: [-2 * (1 - a) - 400 * a * (b - a * a), 200 * (b - a * a)], + }); + const r = minimize(f, [-1.2, 1]); + assert.ok(Math.abs(r.x[0] - 1) < 1e-4 && Math.abs(r.x[1] - 1) < 1e-4, JSON.stringify(r.x)); +}); + +function synthetic(seed, { M = 5, J = 600, missing = 0.3 } = {}) { + const { u, g } = rng(seed); + const a = [1.5, 0.8, 0.2, -0.4, 2].slice(0, M); + const w = [0.9, -0.6]; + const L = [[1.2], [1.0], [0.9], [1.1], [1.3]].slice(0, M); + const tasks = []; + for (let j = 0; j < J; j++) { + const x = [g(), g()]; + const th = g(); + const obs = []; + for (let m = 0; m < M; m++) { + if (u() < missing) continue; + const z = a[m] - (w[0] * x[0] + w[1] * x[1]) + L[m][0] * th; + obs.push([m, u() < sigmoid(z) ? 1 : 0]); + } + tasks.push({ x, obs }); + } + return { truth: { a, w, L }, data: { nModels: M, nFeatures: 2, tasks } }; +} + +test("mirt: analytic gradient matches finite differences (sparse observations)", () => { + const { data } = synthetic(3, { J: 60 }); + const grid = normalGrid(2, 5); + const prior = { a: [0, 0, 0, 0, 0], w: [0, 0], L: [[1, 0], [1, 0], [1, 0], [1, 0], [1, 0]], scaleA: 2, scaleW: 1, scaleL: 1 }; + const v = [0.5, 0.1, -0.2, 0.3, 0.9, 0.4, -0.3, 1.1, 0.2, 0.9, -0.1, 1.2, 0.3, 0.8, 0.1, 1.0, -0.2]; + const { grad } = objective(v, data, 2, grid, prior); + for (let i = 0; i < v.length; i++) { + const h = 1e-5; + const up = v.slice(); + const dn = v.slice(); + up[i] += h; + dn[i] -= h; + const num = (objective(up, data, 2, grid, prior).value - objective(dn, data, 2, grid, prior).value) / (2 * h); + assert.ok(Math.abs(num - grad[i]) < 1e-5 * Math.max(1, Math.abs(num)), `param ${i}: ${num} vs ${grad[i]}`); + } +}); + +test("mirt: recovers abilities and difficulty weights from synthetic outcomes", () => { + const { truth, data } = synthetic(11); + const { params } = fitMirt(data, 1); + // Order of abilities is what routing depends on. + const order = (v) => v.map((x, i) => [x, i]).sort((p, q) => p[0] - q[0]).map(([, i]) => i); + assert.deepEqual(order(params.a), order(truth.a)); + assert.ok(Math.sign(params.w[0]) === 1 && Math.sign(params.w[1]) === -1); + // Predicted marginal success at x = 0 is close to the true marginal. + const p = marginals(nodeProbabilities(params, [0, 0])); + const { g } = rng(5); + truth.a.forEach((am, m) => { + let s = 0; + for (let i = 0; i < 20000; i++) s += sigmoid(am + truth.L[m][0] * g()); + assert.ok(Math.abs(p[m] - s / 20000) < 0.06, `model ${m}: ${p[m]} vs ${s / 20000}`); + }); + assert.ok(Number.isFinite(heldOutLogLik(params, data))); +}); + +test("policy: cascade success and cost integrate correlated failures (matches Monte Carlo)", () => { + const params = { k: 1, a: [0.5, 1.0], w: [0], L: [[2], [2]] }; + const nodes = nodeProbabilities(params, [0]); + const costs = [1, 3]; + const st = cascadeStats(nodes.P, nodes.weights, costs, [0, 1]); + const { g, u } = rng(9); + let solved = 0; + let spent = 0; + const n = 200000; + for (let i = 0; i < n; i++) { + const th = g(); + spent += 1; + if (u() < sigmoid(0.5 + 2 * th)) { + solved++; + continue; + } + spent += 3; + if (u() < sigmoid(1 + 2 * th)) solved++; + } + assert.ok(Math.abs(st.p - solved / n) < 0.005, `${st.p} vs ${solved / n}`); + assert.ok(Math.abs(st.cost - spent / n) < 0.02, `${st.cost} vs ${spent / n}`); + // Treating the two models as independent would overstate what the cascade buys. + const m = marginals(nodes); + assert.ok(1 - (1 - m[0]) * (1 - m[1]) > st.p + 0.02); +}); + +test("policy: objectives choose as specified and enumerate every ordered cascade", () => { + assert.equal([...cascades([0, 1, 2], 2)].length, 3 + 6); + const nodes = { P: [[0.5], [0.8], [0.9]], weights: [1] }; + const costs = [0.1, 1, 5]; + // match-best-single: at least model 2's 0.9, as cheap as possible → 0 then 1 (0.9 at 0.6). + const m = choose(nodes, costs, [0, 1, 2], parseObjective("match-best-single"), 3); + assert.deepEqual(m.seq, [0, 1]); + assert.ok(m.targetMet && Math.abs(m.p - 0.9) < 1e-12); + const t = choose(nodes, costs, [0, 1, 2], parseObjective("target:0.99"), 3); + assert.equal(t.targetMet, true); + assert.ok(t.p >= 0.99); + const b = choose(nodes, costs, [0, 1, 2], parseObjective("budget:0.2"), 3); + assert.deepEqual(b.seq, [0]); + const v = choose(nodes, costs, [0, 1, 2], parseObjective("value:1"), 3); + assert.ok(v.seq.length >= 1); + assert.throws(() => parseObjective("target:2")); + // A model with unknown cost is never chosen. + assert.deepEqual(choose(nodes, [null, 1, 5], [0, 1, 2], parseObjective("budget:2"), 1).seq, [1]); +}); + +test("cost: fitted per model; a model with only a price enters from the price ratio", () => { + const { g } = rng(21); + const obs = []; + for (let i = 0; i < 300; i++) { + const x = [g()]; + obs.push({ model: 0, x, cost: Math.exp(-2 + 0.5 * x[0] + 0.1 * g()) }); + obs.push({ model: 1, x, cost: Math.exp(-1 + 0.5 * x[0] + 0.1 * g()) }); + } + const prices = [ + { priceIn: 1, priceOut: 5 }, + { priceIn: Math.E, priceOut: 5 * Math.E }, + { priceIn: 2, priceOut: 10 }, + ]; + const c = fitCost(obs, 3, 1, prices); + assert.ok(Math.abs(c.alpha[0] + 2) < 0.05 && Math.abs(c.alpha[1] + 1) < 0.05); + assert.ok(Math.abs(c.beta[0] - 0.5) < 0.05); + assert.equal(c.source[2], "price"); + // Price 2x the first model's → about 2x its cost (log 2 ≈ 0.693 above α₀). + assert.ok(Math.abs(c.alpha[2] - (c.alpha[0] + Math.log(2))) < 0.05); + const e = expectedCosts(c, [0]); + assert.ok(e[1] > e[0] && e.every((v) => v > 0)); +}); diff --git a/test/router_universal.test.js b/test/router_universal.test.js new file mode 100644 index 0000000..c2631b6 --- /dev/null +++ b/test/router_universal.test.js @@ -0,0 +1,87 @@ +// Universal router: registry as data, provider filtering, outcome recording and Bayesian refit. +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fitRouter, loadRouterModel, readOutcomes, recordOutcome, routeUniversal } from "../src/router/index.js"; +import { loadRegistry, servableBy } from "../src/router/registry.js"; + +const project = () => { + const d = mkdtempSync(join(tmpdir(), "forge-router-")); + mkdirSync(join(d, ".forge"), { recursive: true }); + return d; +}; +const TASK = "Fix the off-by-one error in the pagination helper so the last page is included."; + +test("router code names no vendor, model or tier: they all come from data", () => { + for (const f of ["index.js", "policy.js", "mirt.js", "cost.js", "features.js", "registry.js", "prior.js"]) { + const src = readFileSync(new URL(`../src/router/${f}`, import.meta.url), "utf8"); + assert.doesNotMatch(src, /claude|haiku|sonnet|opus|gpt-|gemini|kimi|minimax|deepseek|glm/i, f); + } +}); + +test("registry: shipped models load; .forge/models.json adds, overrides and disables", () => { + const d = project(); + const base = loadRegistry(d); + assert.ok(base.models.length >= 11); + const [first, second] = base.models; + writeFileSync( + join(d, ".forge", "models.json"), + JSON.stringify({ + models: [ + { id: "local-model", label: "Local", price_in: 0.1, price_out: 0.2, providers: { mygw: "local/x" } }, + { id: first.id, providers: { mygw: "gw/first" } }, + { id: second.id, enabled: false }, + ], + }), + ); + const reg = loadRegistry(d); + assert.ok(reg.models.some((m) => m.id === "local-model")); + assert.ok(!reg.models.some((m) => m.id === second.id)); + assert.deepEqual(servableBy(reg, "mygw").sort(), [first.id, "local-model"].sort()); + assert.ok(reg.sources.includes(".forge/models.json")); +}); + +test("routeUniversal: returns a cascade within the provider's models, with probabilities and costs", () => { + const d = project(); + const reg = loadRegistry(d); + const anthropic = servableBy(reg, "anthropic"); + assert.ok(anthropic.length >= 1); + const r = routeUniversal(d, TASK, { provider: "anthropic" }); + assert.equal(r.ok, true); + assert.ok(r.cascade.every((c) => anthropic.includes(c.model))); + assert.ok(r.pSuccess > 0 && r.pSuccess <= 1 && r.expectedCost > 0); + assert.ok(r.targetMet, "match-best-single always reaches the best single model"); + const any = routeUniversal(d, TASK, { objective: "target:0.95" }); + assert.equal(any.ok, true); + assert.ok(any.cascade.length <= 3); + // A model the fit has never seen enters cold, at the population mean. + const cold = routeUniversal(d, TASK, { candidates: reg.models.filter((m) => !m.evidence).map((m) => m.id) }); + assert.ok(!cold.ok || cold.cascade.every((c) => c.status === "cold")); +}); + +test("recordOutcome stores features and a hash, never the task text", () => { + const d = project(); + const model = loadRegistry(d).models[0].id; + recordOutcome(d, { task: "secret project name zeta", model, passed: true, cost: 0.12, features: new Array(12).fill(0) }); + const raw = readFileSync(join(d, ".forge", "route_outcomes.jsonl"), "utf8"); + assert.doesNotMatch(raw, /zeta/); + assert.equal(readOutcomes(d).length, 1); + assert.throws(() => recordOutcome(d, { task: "x", model })); +}); + +test("fitRouter: local outcomes move a model's ability in their direction (Bayesian update)", () => { + const d = project(); + const prior = loadRouterModel(d); + const target = prior.models[0]; + const before = prior.mirt.a[0]; + const feats = new Array(prior.features.mean.length).fill(0).map((_, i) => prior.features.mean[i]); + for (let i = 0; i < 40; i++) + recordOutcome(d, { task: `task ${i}`, model: target, passed: false, cost: 0.2, features: feats }); + const fitted = fitRouter(d); + const after = fitted.mirt.a[fitted.models.indexOf(target)]; + assert.ok(after < before - 0.3, `ability should fall after 40 verified failures: ${before} -> ${after}`); + assert.equal(loadRouterModel(d).origin, ".forge/router_model.json"); + assert.equal(fitted.provenance.local.outcomes, 40); +}); From e5305b0b30ec89b7d65f4eb75d4bcedc1c671ae6 Mon Sep 17 00:00:00 2001 From: Juber Shaikh Date: Tue, 22 Sep 2026 05:26:44 +0000 Subject: [PATCH 09/13] fix: impact label for learn_consolidate.js; router typecheck and lint clean Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JpQ15QdqdDUJLavNhExwvL --- bench/impact_cases.mjs | 4 ++- src/cli.js | 41 +++++++++++++++++++--------- src/router/cost.js | 18 +++++++++---- src/router/features.js | 6 ++++- src/router/index.js | 50 ++++++++++++++++++++++++++-------- src/router/lbfgs.js | 6 ++++- src/router/mirt.js | 23 ++++++++++++---- src/router/policy.js | 23 ++++++++++++---- src/router/prior.js | 39 +++++++++++++++------------ src/router/quadrature.js | 1 + src/router/registry.js | 6 ++++- test/router_math.test.js | 43 ++++++++++++++++++++++++----- test/router_universal.test.js | 51 ++++++++++++++++++++++++++++++----- 13 files changed, 239 insertions(+), 72 deletions(-) diff --git a/bench/impact_cases.mjs b/bench/impact_cases.mjs index 2a38893..b2202f7 100644 --- a/bench/impact_cases.mjs +++ b/bench/impact_cases.mjs @@ -47,7 +47,7 @@ // - test/ledger.test.js imports { mergeStates } (:14) and calls it // (src/ledger_sync.js:3 also names it in the module header — same file, already labeled.) // -// claimText (src/ledger.js) — 8 files +// claimText (src/ledger.js) — 9 files // - src/ledger.js defines it (:610); sketchOf() (:636), termsOf() (:637) and :880 call it // - src/context.js imports { claimText } (:13) and calls it (:185) // - src/dash.js imports { claimText } (:16) and calls it (:58, :389, :400) @@ -56,6 +56,7 @@ // - src/cli.js dynamic-imports { claimText } (:874, :1644) and calls it // - src/cortex_mcp.js dynamic-imports { claimText } (:91) and calls it (:96, :106) // - test/ledger.test.js imports { claimText } (:8) and calls it +// - src/learn_consolidate.js imports { claimText } (:32) and calls it (:110) // (test/dash.test.js:69 mentions the name only inside an assertion message — a string, // not a reference — so it is NOT labeled as a dependent.) // @@ -114,6 +115,7 @@ export const IMPACT_CASES = [ "src/context.js", "src/dash.js", "src/deja.js", + "src/learn_consolidate.js", "src/ledger_store.js", "src/cli.js", "src/cortex_mcp.js", diff --git a/src/cli.js b/src/cli.js index 4af2130..27d6fb5 100755 --- a/src/cli.js +++ b/src/cli.js @@ -2058,12 +2058,16 @@ HANDLERS.route = async (argv) => { // probability asked for. Models come from data/models.json and .forge/models.json. async function routeUniversalCli(argv) { const U = await import("./router/index.js"); - const { loadRegistry, servableBy } = await import("./router/registry.js"); + const { loadRegistry } = await import("./router/registry.js"); const json = argv.includes("--json"); const val = (flag) => (argv.includes(flag) ? argv[argv.indexOf(flag) + 1] : undefined); const VALUED = new Set(["--objective", "--provider", "--model", "--cost", "--depth"]); - const words = argv.slice(1).filter((a, i, arr) => !a.startsWith("--") && !VALUED.has(arr[i - 1] ?? "")); - const sub = ["outcome", "fit", "models", "universal"].includes(words[0]) ? words.shift() : "universal"; + const words = argv + .slice(1) + .filter((a, i, arr) => !a.startsWith("--") && !VALUED.has(arr[i - 1] ?? "")); + const sub = ["outcome", "fit", "models", "universal"].includes(words[0]) + ? words.shift() + : "universal"; const root = process.cwd(); if (sub === "models") { const reg = loadRegistry(root); @@ -2075,7 +2079,10 @@ async function routeUniversalCli(argv) { providers: Object.keys(m.providers ?? {}), price: m.price_in != null ? `${m.price_in}/${m.price_out}` : null, })); - if (json) return console.log(JSON.stringify({ sources: reg.sources, fit: fit?.origin ?? null, models: rows }, null, 2)); + if (json) + return console.log( + JSON.stringify({ sources: reg.sources, fit: fit?.origin ?? null, models: rows }, null, 2), + ); heading(`${BRAND.brand} route models — registry (${reg.sources.join(" + ")})\n`); for (const r of rows) console.log( @@ -2087,7 +2094,9 @@ async function routeUniversalCli(argv) { if (sub === "fit") { const model = U.fitRouter(root); if (json) return console.log(JSON.stringify(model.provenance, null, 2)); - console.log(` refit on ${model.provenance.local.outcomes} recorded outcome(s) over ${model.provenance.local.tasks} task(s); wrote .forge/router_model.json`); + console.log( + ` refit on ${model.provenance.local.outcomes} recorded outcome(s) over ${model.provenance.local.tasks} task(s); wrote .forge/router_model.json`, + ); return; } const task = words.join(" "); @@ -2106,7 +2115,9 @@ async function routeUniversalCli(argv) { try { const row = U.recordOutcome(root, { task, model: val("--model"), passed, cost }); if (json) return console.log(JSON.stringify(row, null, 2)); - console.log(` recorded ${row.model} ${row.passed ? "pass" : "fail"} for task ${row.task} (.forge/route_outcomes.jsonl)`); + console.log( + ` recorded ${row.model} ${row.passed ? "pass" : "fail"} for task ${row.task} (.forge/route_outcomes.jsonl)`, + ); } catch (e) { console.error(` ${e.message}`); process.exitCode = 1; @@ -2131,17 +2142,23 @@ async function routeUniversalCli(argv) { process.exitCode = 1; return; } - heading(`${BRAND.brand} route universal — ${rec.objective.kind}${rec.target != null ? ` (target ${rec.target.toFixed(2)})` : ""}\n`); - rec.cascade.forEach((c, i) => + heading( + `${BRAND.brand} route universal — ${rec.objective.kind}${rec.target != null ? ` (target ${rec.target.toFixed(2)})` : ""}\n`, + ); + rec.cascade.forEach((c, i) => { console.log( ` ${i === 0 ? "→" : "then, if a check fails →"} ${paint(c.model, "accent")} P(solve alone) ${c.pSolveAlone.toFixed(2)} · ~$${c.expectedAttemptCost.toFixed(3)}/attempt${c.status === "cold" ? " · cold (no outcomes yet)" : ""}`, - ), - ); + ); + }); console.log( `\n P(success) ${rec.pSuccess.toFixed(2)} · expected cost $${rec.expectedCost.toFixed(3)} · best single: ${rec.bestSingle.model} ${rec.bestSingle.pSuccess.toFixed(2)} at $${rec.bestSingle.expectedCost.toFixed(3)}`, ); - console.log(` ${rec.candidates} candidate model(s), ${rec.cascadesEvaluated} cascade(s) compared · fit: ${rec.fit.origin}`); - console.log(` learn from results: \`${BRAND.cli} route outcome "" --model --pass|--fail --cost \`, then \`${BRAND.cli} route fit\``); + console.log( + ` ${rec.candidates} candidate model(s), ${rec.cascadesEvaluated} cascade(s) compared · fit: ${rec.fit.origin}`, + ); + console.log( + ` learn from results: \`${BRAND.cli} route outcome "" --model --pass|--fail --cost \`, then \`${BRAND.cli} route fit\``, + ); } HANDLERS.anchor = async (argv) => { diff --git a/src/router/cost.js b/src/router/cost.js index edc225e..01e1477 100644 --- a/src/router/cost.js +++ b/src/router/cost.js @@ -23,16 +23,22 @@ export function fitCost(obs, nModels, nFeatures, prices = []) { const X = used.map((o) => [...models.map((m) => (m === o.model ? 1 : 0)), ...o.x]); const y = used.map((o) => Math.log(o.cost)); // A tiny ridge on the slopes only keeps the system well-posed with few observations. - const ridge = [...models.map(() => 0), ...new Array(nFeatures).fill(1e-6 * Math.max(1, used.length))]; - const coef = used.length > models.length ? leastSquares(X, y, ridge) : [...models.map(() => 0), ...new Array(nFeatures).fill(0)]; + const ridge = [ + ...models.map(() => 0), + ...new Array(nFeatures).fill(1e-6 * Math.max(1, used.length)), + ]; + const coef = + used.length > models.length + ? leastSquares(X, y, ridge) + : [...models.map(() => 0), ...new Array(nFeatures).fill(0)]; const alpha = new Array(nModels).fill(null); for (const m of models) alpha[m] = coef[col.get(m)]; const beta = coef.slice(models.length); let rss = 0; - used.forEach((o, i) => { + for (let i = 0; i < used.length; i++) { const pred = X[i].reduce((s, v, c) => s + v * coef[c], 0); rss += (y[i] - pred) ** 2; - }); + } const dof = Math.max(1, used.length - models.length - nFeatures); const s2 = used.length ? rss / dof : 0; @@ -42,7 +48,9 @@ export function fitCost(obs, nModels, nFeatures, prices = []) { let kappa = null; if (priced.length >= 1) { const spread = (r) => { - const d = priced.map((m) => alpha[m] - Math.log(r * prices[m].priceIn + (1 - r) * prices[m].priceOut)); + const d = priced.map( + (m) => alpha[m] - Math.log(r * prices[m].priceIn + (1 - r) * prices[m].priceOut), + ); const mean = d.reduce((s, v) => s + v, 0) / d.length; return { mean, var: d.reduce((s, v) => s + (v - mean) ** 2, 0) / d.length }; }; diff --git a/src/router/features.js b/src/router/features.js index 7455bc7..14244b2 100644 --- a/src/router/features.js +++ b/src/router/features.js @@ -56,7 +56,11 @@ export function fitScaler(rows) { const std = new Array(d).fill(0); for (const r of rows) for (let i = 0; i < d; i++) mean[i] += r[i] / rows.length; for (const r of rows) for (let i = 0; i < d; i++) std[i] += (r[i] - mean[i]) ** 2 / rows.length; - return { names: FEATURE_NAMES.slice(0, d), mean, std: std.map((v) => (v > 1e-12 ? Math.sqrt(v) : 1)) }; + return { + names: FEATURE_NAMES.slice(0, d), + mean, + std: std.map((v) => (v > 1e-12 ? Math.sqrt(v) : 1)), + }; } export const standardise = (scaler, raw) => raw.map((v, i) => (v - scaler.mean[i]) / scaler.std[i]); diff --git a/src/router/index.js b/src/router/index.js index 4411971..5b51387 100644 --- a/src/router/index.js +++ b/src/router/index.js @@ -51,12 +51,18 @@ export function alignModels(fitted, registry) { const idx = new Map(fitted.models.map((id, i) => [id, i])); const k = fitted.mirt.k; const meanA = fitted.mirt.a.reduce((s, v) => s + v, 0) / fitted.mirt.a.length; - const meanL = Array.from({ length: k }, (_, d) => fitted.mirt.L.reduce((s, r) => s + r[d], 0) / fitted.mirt.L.length); + const meanL = Array.from( + { length: k }, + (_, d) => fitted.mirt.L.reduce((s, r) => s + r[d], 0) / fitted.mirt.L.length, + ); const a = []; const L = []; const alpha = []; const status = []; - const prices = registry.models.map((m) => ({ priceIn: m.price_in ?? null, priceOut: m.price_out ?? null })); + const prices = registry.models.map((m) => ({ + priceIn: m.price_in ?? null, + priceOut: m.price_out ?? null, + })); for (let i = 0; i < ids.length; i++) { const j = idx.get(ids[i]); if (j !== undefined) { @@ -89,11 +95,13 @@ export function alignModels(fitted, registry) { * Recommend a model or cascade for a task. * @param {string|null} root * @param {string} task - * @param {{objective?: string, maxDepth?: number, provider?: string, candidates?: string[], features?: number[]}} [opts] + * @param {{objective?: string, maxDepth?: number, provider?: string, candidates?: string[], features?: number[], + * model?: any, registry?: {models: any[], sources?: string[]}}} [opts] */ export function routeUniversal(root, task, opts = {}) { const fitted = opts.model ?? loadRouterModel(root); - if (!fitted) return { ok: false, reason: "no fitted router model (data/router_prior.json missing)" }; + if (!fitted) + return { ok: false, reason: "no fitted router model (data/router_prior.json missing)" }; const registry = opts.registry ?? loadRegistry(root); const aligned = alignModels(fitted, registry); const raw = opts.features ?? rawFeatures(root, task); @@ -120,7 +128,11 @@ export function routeUniversal(root, task, opts = {}) { objective, target: pick.target, targetMet: pick.targetMet, - bestSingle: { model: aligned.ids[pick.bestSingle.model], pSuccess: pick.bestSingle.p, expectedCost: pick.bestSingle.cost }, + bestSingle: { + model: aligned.ids[pick.bestSingle.model], + pSuccess: pick.bestSingle.p, + expectedCost: pick.bestSingle.cost, + }, candidates: candidates.length, cascadesEvaluated: pick.evaluated, fit: { origin: fitted.origin, k: fitted.mirt.k, provenance: fitted.provenance ?? null }, @@ -134,14 +146,16 @@ function readConfigObjective(root) { return cfg?.route?.objective; } -export const taskRef = (task) => createHash("sha256").update(String(task)).digest("hex").slice(0, 16); +export const taskRef = (task) => + createHash("sha256").update(String(task)).digest("hex").slice(0, 16); /** * Record a verified outcome of one attempt (the only evidence the router learns from). The task * text is not stored: only its hash and features. */ export function recordOutcome(root, { task, model, passed, cost = null, features = null }) { - if (!model || typeof passed !== "boolean") throw new Error("recordOutcome needs model and passed (boolean)"); + if (!model || typeof passed !== "boolean") + throw new Error("recordOutcome needs model and passed (boolean)"); const dir = join(root, ".forge"); mkdirSync(dir, { recursive: true }); const row = { @@ -176,7 +190,10 @@ export function readOutcomes(root) { * update: few local outcomes barely move it, many outcomes dominate). Writes * .forge/router_model.json and returns it. */ -export function fitRouter(root, { outcomes = readOutcomes(root), registry = loadRegistry(root) } = {}) { +export function fitRouter( + root, + { outcomes = readOutcomes(root), registry = loadRegistry(root) } = {}, +) { const shipped = readJson(SHIPPED_PRIOR); if (!shipped?.mirt) throw new Error("data/router_prior.json missing: cannot fit without a prior"); const base = alignModels(shipped, registry); @@ -195,7 +212,11 @@ export function fitRouter(root, { outcomes = readOutcomes(root), registry = load byTask.get(o.task).obs.push([m, o.passed ? 1 : 0]); if (o.cost > 0) costObs.push({ model: m, x, cost: o.cost }); } - const data = { nModels: base.ids.length, nFeatures: shipped.features.mean.length, tasks: [...byTask.values()] }; + const data = { + nModels: base.ids.length, + nFeatures: shipped.features.mean.length, + tasks: [...byTask.values()], + }; const scale = shipped.selection?.chosen?.scale ?? 1; const { params } = fitMirt(data, base.mirt.k, { a: base.mirt.a, @@ -210,7 +231,9 @@ export function fitRouter(root, { outcomes = readOutcomes(root), registry = load for (let m = 0; m < alpha.length; m++) { const mine = costObs.filter((c) => c.model === m); if (!mine.length) continue; - const resid = mine.map((c) => Math.log(c.cost) - base.cost.beta.reduce((s, b, d) => s + b * c.x[d], 0)); + const resid = mine.map( + (c) => Math.log(c.cost) - base.cost.beta.reduce((s, b, d) => s + b * c.x[d], 0), + ); const prior = alpha[m] ?? resid.reduce((s, v) => s + v, 0) / resid.length; alpha[m] = (prior + resid.reduce((s, v) => s + v, 0)) / (1 + resid.length); } @@ -223,7 +246,12 @@ export function fitRouter(root, { outcomes = readOutcomes(root), registry = load selection: shipped.selection, provenance: { prior: shipped.provenance, - local: { outcomes: outcomes.length - skipped, skipped, tasks: data.tasks.length, fittedAt: new Date().toISOString() }, + local: { + outcomes: outcomes.length - skipped, + skipped, + tasks: data.tasks.length, + fittedAt: new Date().toISOString(), + }, }, }; mkdirSync(join(root, ".forge"), { recursive: true }); diff --git a/src/router/lbfgs.js b/src/router/lbfgs.js index da2b95f..8fc4232 100644 --- a/src/router/lbfgs.js +++ b/src/router/lbfgs.js @@ -12,7 +12,11 @@ const dot = (a, b) => { * @param {number[]} x0 * @param {{maxIter?: number, memory?: number, gradTol?: number, relTol?: number}} [opts] */ -export function minimize(f, x0, { maxIter = 500, memory = 10, gradTol = 1e-6, relTol = 1e-10 } = {}) { +export function minimize( + f, + x0, + { maxIter = 500, memory = 10, gradTol = 1e-6, relTol = 1e-10 } = {}, +) { let x = x0.slice(); let { value: fx, grad: g } = f(x); const S = []; diff --git a/src/router/mirt.js b/src/router/mirt.js index a53948b..8ab6c0f 100644 --- a/src/router/mirt.js +++ b/src/router/mirt.js @@ -128,7 +128,9 @@ function fullPrior(M, D, k, prior = {}) { w: prior.w ?? new Array(D).fill(0), // Default loading prior mean: a common positive first factor (models agree on which tasks // are hard), zero on further factors. It only centres the prior; the data moves it. - L: prior.L ?? Array.from({ length: M }, () => Array.from({ length: k }, (_, d) => (d === 0 ? 1 : 0))), + L: + prior.L ?? + Array.from({ length: M }, () => Array.from({ length: k }, (_, d) => (d === 0 ? 1 : 0))), scaleA: prior.scaleA ?? 2, scaleW: prior.scaleW ?? 1, scaleL: prior.scaleL ?? 1, @@ -176,9 +178,12 @@ export function heldOutLogLik(params, data, taskIdx) { * Choose k and the prior scale by K-fold cross-validated held-out likelihood, then refit on all * tasks with the winner. Folds are assigned deterministically. * @param {MirtData} data - * @param {{ks?: number[], scales?: number[], folds?: number, prior?: MirtPrior}} [opts] + * @param {{ks?: number[], scales?: number[], folds?: number, prior?: MirtPrior, maxExpand?: number}} [opts] */ -export function selectAndFit(data, { ks = [1, 2, 3], scales = [0.5, 1, 2], folds = 3, prior = {}, maxExpand = 4 } = {}) { +export function selectAndFit( + data, + { ks = [1, 2, 3], scales = [0.5, 1, 2], folds = 3, prior = {}, maxExpand = 4 } = {}, +) { const J = data.tasks.length; const order = data.tasks.map((_, j) => j).sort((x, y) => hash32(x) - hash32(y)); const foldOf = new Array(J); @@ -186,6 +191,7 @@ export function selectAndFit(data, { ks = [1, 2, 3], scales = [0.5, 1, 2], folds foldOf[j] = i % folds; }); const table = []; + /** @type {{k: number, scale: number, heldOutLogLik: number} | null} */ let best = null; const cv = (k, s) => { let ll = 0; @@ -221,8 +227,15 @@ export function selectAndFit(data, { ks = [1, 2, 3], scales = [0.5, 1, 2], folds if (ll <= lls[low ? 1 : lls.length - 2]) break; } } - const fit = fitMirt(data, best.k, { ...prior, scaleA: 2 * best.scale, scaleW: best.scale, scaleL: best.scale }); - return { ...fit, selection: { chosen: best, table, folds } }; + if (!best) throw new Error("selectAndFit: no candidate configuration"); + const chosen = /** @type {{k: number, scale: number, heldOutLogLik: number}} */ (best); + const fit = fitMirt(data, chosen.k, { + ...prior, + scaleA: 2 * chosen.scale, + scaleW: chosen.scale, + scaleL: chosen.scale, + }); + return { ...fit, selection: { chosen, table, folds } }; } function hash32(n) { diff --git a/src/router/policy.js b/src/router/policy.js index 4042933..e9135c2 100644 --- a/src/router/policy.js +++ b/src/router/policy.js @@ -47,13 +47,16 @@ export function* cascades(candidates, maxDepth) { * @param {string|undefined} spec */ export function parseObjective(spec) { - if (!spec || spec === "match-best-single" || spec === "match") return { kind: "match-best-single" }; + if (!spec || spec === "match-best-single" || spec === "match") + return { kind: "match-best-single" }; const [kind, raw] = String(spec).split(":"); const v = Number(raw); if (kind === "target" && v > 0 && v < 1) return { kind, target: v }; if (kind === "value" && v > 0) return { kind, value: v }; if (kind === "budget" && v > 0) return { kind, budget: v }; - throw new Error(`unknown objective "${spec}" (use match-best-single, target:<0..1>, value:<$>, budget:<$>)`); + throw new Error( + `unknown objective "${spec}" (use match-best-single, target:<0..1>, value:<$>, budget:<$>)`, + ); } /** @@ -70,7 +73,11 @@ export function choose(nodes, costs, candidates, objective, maxDepth) { const single = usable.map((m) => ({ m, ...cascadeStats(P, weights, costs, [m]) })); const bestSingle = single.reduce((a, b) => (b.p > a.p ? b : a)); const target = - objective.kind === "match-best-single" ? bestSingle.p : objective.kind === "target" ? objective.target : null; + objective.kind === "match-best-single" + ? bestSingle.p + : objective.kind === "target" + ? objective.target + : null; let best = null; let bestKey = null; let evaluated = 0; @@ -79,9 +86,15 @@ export function choose(nodes, costs, candidates, objective, maxDepth) { const st = cascadeStats(P, weights, costs, seq); evaluated++; let key; - if (target !== null) key = st.p >= target - eps ? [0, st.cost, -st.p, seq.length] : [1, -st.p, st.cost, seq.length]; + if (target !== null) + key = + st.p >= target - eps ? [0, st.cost, -st.p, seq.length] : [1, -st.p, st.cost, seq.length]; else if (objective.kind === "value") key = [0, -(objective.value * st.p - st.cost), seq.length]; - else key = st.cost <= objective.budget + eps ? [0, -st.p, st.cost, seq.length] : [1, st.cost, -st.p, seq.length]; + else + key = + st.cost <= objective.budget + eps + ? [0, -st.p, st.cost, seq.length] + : [1, st.cost, -st.p, seq.length]; if (!bestKey || lexLess(key, bestKey)) { bestKey = key; best = { seq, ...st }; diff --git a/src/router/prior.js b/src/router/prior.js index 70a7ee5..4d4a078 100644 --- a/src/router/prior.js +++ b/src/router/prior.js @@ -11,31 +11,37 @@ import { loadRegistry } from "./registry.js"; * @param {Set|null} [only] restrict to these task ids * @param {{registry?: {models: object[]}, root?: string|null}} [opts] */ -export function buildPrior(input, only = null, { registry = loadRegistry(null), root = null } = {}) { - const models = Object.keys(input.outcomes).filter((id) => registry.models.some((m) => m.id === id)); +export function buildPrior( + input, + only = null, + { registry = loadRegistry(null), root = null } = {}, +) { + const models = Object.keys(input.outcomes).filter((id) => + registry.models.some((m) => m.id === id), + ); const tasks = input.tasks.filter((t) => !only || only.has(t.id)); const raw = tasks.map((t) => t.features ?? rawFeatures(root, t.text)); const scaler = fitScaler(raw); const X = raw.map((r) => standardise(scaler, r)); - const data = { - nModels: models.length, - nFeatures: scaler.mean.length, - tasks: tasks.map((t, j) => ({ - x: X[j], - obs: models - .map((id, m) => [m, input.outcomes[id][t.id]]) - .filter(([, o]) => o) - .map(([m, o]) => [m, o.resolved ? 1 : 0]), - })), - }; + /** @type {import("./mirt.js").MirtData} */ + const data = { nModels: models.length, nFeatures: scaler.mean.length, tasks: [] }; + tasks.forEach((t, j) => { + /** @type {[number, 0|1][]} */ + const obs = []; + models.forEach((id, m) => { + const o = input.outcomes[id][t.id]; + if (o) obs.push([m, o.resolved ? 1 : 0]); + }); + data.tasks.push({ x: X[j], obs }); + }); const fit = selectAndFit(data); const costObs = []; - tasks.forEach((t, j) => + tasks.forEach((t, j) => { models.forEach((id, m) => { const o = input.outcomes[id][t.id]; if (o?.cost > 0) costObs.push({ model: m, x: X[j], cost: o.cost }); - }), - ); + }); + }); const prices = models.map((id) => { const r = registry.models.find((m) => m.id === id); return { priceIn: r?.price_in ?? null, priceOut: r?.price_out ?? null }; @@ -56,4 +62,3 @@ export function buildPrior(input, only = null, { registry = loadRegistry(null), }, }; } - diff --git a/src/router/quadrature.js b/src/router/quadrature.js index ad23e0f..03fe33b 100644 --- a/src/router/quadrature.js +++ b/src/router/quadrature.js @@ -14,6 +14,7 @@ function tridiagEigen(diag, off) { const d = diag.slice(); const e = [...off, 0]; // z holds the first row of the accumulated rotation matrix (= first eigenvector components). + /** @type {number[]} */ const z = Array.from({ length: n }, (_, i) => (i === 0 ? 1 : 0)); for (let l = 0; l < n; l++) { for (let iter = 0; iter < 200; iter++) { diff --git a/src/router/registry.js b/src/router/registry.js index b50c272..08a47a6 100644 --- a/src/router/registry.js +++ b/src/router/registry.js @@ -31,7 +31,11 @@ export function loadRegistry(root) { for (const m of local.models) { if (!m?.id) continue; const prev = byId.get(m.id) ?? {}; - byId.set(m.id, { ...prev, ...m, providers: { ...(prev.providers ?? {}), ...(m.providers ?? {}) } }); + byId.set(m.id, { + ...prev, + ...m, + providers: { ...(prev.providers ?? {}), ...(m.providers ?? {}) }, + }); } } } diff --git a/test/router_math.test.js b/test/router_math.test.js index 04b769e..f22bfd7 100644 --- a/test/router_math.test.js +++ b/test/router_math.test.js @@ -3,7 +3,14 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { expectedCosts, fitCost } from "../src/router/cost.js"; import { minimize } from "../src/router/lbfgs.js"; -import { fitMirt, heldOutLogLik, marginals, nodeProbabilities, objective, sigmoid } from "../src/router/mirt.js"; +import { + fitMirt, + heldOutLogLik, + marginals, + nodeProbabilities, + objective, + sigmoid, +} from "../src/router/mirt.js"; import { cascadeStats, cascades, choose, parseObjective } from "../src/router/policy.js"; import { normalGrid, normalNodes } from "../src/router/quadrature.js"; @@ -62,8 +69,23 @@ function synthetic(seed, { M = 5, J = 600, missing = 0.3 } = {}) { test("mirt: analytic gradient matches finite differences (sparse observations)", () => { const { data } = synthetic(3, { J: 60 }); const grid = normalGrid(2, 5); - const prior = { a: [0, 0, 0, 0, 0], w: [0, 0], L: [[1, 0], [1, 0], [1, 0], [1, 0], [1, 0]], scaleA: 2, scaleW: 1, scaleL: 1 }; - const v = [0.5, 0.1, -0.2, 0.3, 0.9, 0.4, -0.3, 1.1, 0.2, 0.9, -0.1, 1.2, 0.3, 0.8, 0.1, 1.0, -0.2]; + const prior = { + a: [0, 0, 0, 0, 0], + w: [0, 0], + L: [ + [1, 0], + [1, 0], + [1, 0], + [1, 0], + [1, 0], + ], + scaleA: 2, + scaleW: 1, + scaleL: 1, + }; + const v = [ + 0.5, 0.1, -0.2, 0.3, 0.9, 0.4, -0.3, 1.1, 0.2, 0.9, -0.1, 1.2, 0.3, 0.8, 0.1, 1.0, -0.2, + ]; const { grad } = objective(v, data, 2, grid, prior); for (let i = 0; i < v.length; i++) { const h = 1e-5; @@ -71,8 +93,13 @@ test("mirt: analytic gradient matches finite differences (sparse observations)", const dn = v.slice(); up[i] += h; dn[i] -= h; - const num = (objective(up, data, 2, grid, prior).value - objective(dn, data, 2, grid, prior).value) / (2 * h); - assert.ok(Math.abs(num - grad[i]) < 1e-5 * Math.max(1, Math.abs(num)), `param ${i}: ${num} vs ${grad[i]}`); + const num = + (objective(up, data, 2, grid, prior).value - objective(dn, data, 2, grid, prior).value) / + (2 * h); + assert.ok( + Math.abs(num - grad[i]) < 1e-5 * Math.max(1, Math.abs(num)), + `param ${i}: ${num} vs ${grad[i]}`, + ); } }); @@ -80,7 +107,11 @@ test("mirt: recovers abilities and difficulty weights from synthetic outcomes", const { truth, data } = synthetic(11); const { params } = fitMirt(data, 1); // Order of abilities is what routing depends on. - const order = (v) => v.map((x, i) => [x, i]).sort((p, q) => p[0] - q[0]).map(([, i]) => i); + const order = (v) => + v + .map((x, i) => [x, i]) + .sort((p, q) => p[0] - q[0]) + .map(([, i]) => i); assert.deepEqual(order(params.a), order(truth.a)); assert.ok(Math.sign(params.w[0]) === 1 && Math.sign(params.w[1]) === -1); // Predicted marginal success at x = 0 is close to the true marginal. diff --git a/test/router_universal.test.js b/test/router_universal.test.js index c2631b6..a932c54 100644 --- a/test/router_universal.test.js +++ b/test/router_universal.test.js @@ -4,7 +4,13 @@ import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; -import { fitRouter, loadRouterModel, readOutcomes, recordOutcome, routeUniversal } from "../src/router/index.js"; +import { + fitRouter, + loadRouterModel, + readOutcomes, + recordOutcome, + routeUniversal, +} from "../src/router/index.js"; import { loadRegistry, servableBy } from "../src/router/registry.js"; const project = () => { @@ -15,7 +21,15 @@ const project = () => { const TASK = "Fix the off-by-one error in the pagination helper so the last page is included."; test("router code names no vendor, model or tier: they all come from data", () => { - for (const f of ["index.js", "policy.js", "mirt.js", "cost.js", "features.js", "registry.js", "prior.js"]) { + for (const f of [ + "index.js", + "policy.js", + "mirt.js", + "cost.js", + "features.js", + "registry.js", + "prior.js", + ]) { const src = readFileSync(new URL(`../src/router/${f}`, import.meta.url), "utf8"); assert.doesNotMatch(src, /claude|haiku|sonnet|opus|gpt-|gemini|kimi|minimax|deepseek|glm/i, f); } @@ -30,7 +44,13 @@ test("registry: shipped models load; .forge/models.json adds, overrides and disa join(d, ".forge", "models.json"), JSON.stringify({ models: [ - { id: "local-model", label: "Local", price_in: 0.1, price_out: 0.2, providers: { mygw: "local/x" } }, + { + id: "local-model", + label: "Local", + price_in: 0.1, + price_out: 0.2, + providers: { mygw: "local/x" }, + }, { id: first.id, providers: { mygw: "gw/first" } }, { id: second.id, enabled: false }, ], @@ -57,14 +77,22 @@ test("routeUniversal: returns a cascade within the provider's models, with proba assert.equal(any.ok, true); assert.ok(any.cascade.length <= 3); // A model the fit has never seen enters cold, at the population mean. - const cold = routeUniversal(d, TASK, { candidates: reg.models.filter((m) => !m.evidence).map((m) => m.id) }); + const cold = routeUniversal(d, TASK, { + candidates: reg.models.filter((m) => !m.evidence).map((m) => m.id), + }); assert.ok(!cold.ok || cold.cascade.every((c) => c.status === "cold")); }); test("recordOutcome stores features and a hash, never the task text", () => { const d = project(); const model = loadRegistry(d).models[0].id; - recordOutcome(d, { task: "secret project name zeta", model, passed: true, cost: 0.12, features: new Array(12).fill(0) }); + recordOutcome(d, { + task: "secret project name zeta", + model, + passed: true, + cost: 0.12, + features: new Array(12).fill(0), + }); const raw = readFileSync(join(d, ".forge", "route_outcomes.jsonl"), "utf8"); assert.doesNotMatch(raw, /zeta/); assert.equal(readOutcomes(d).length, 1); @@ -78,10 +106,19 @@ test("fitRouter: local outcomes move a model's ability in their direction (Bayes const before = prior.mirt.a[0]; const feats = new Array(prior.features.mean.length).fill(0).map((_, i) => prior.features.mean[i]); for (let i = 0; i < 40; i++) - recordOutcome(d, { task: `task ${i}`, model: target, passed: false, cost: 0.2, features: feats }); + recordOutcome(d, { + task: `task ${i}`, + model: target, + passed: false, + cost: 0.2, + features: feats, + }); const fitted = fitRouter(d); const after = fitted.mirt.a[fitted.models.indexOf(target)]; - assert.ok(after < before - 0.3, `ability should fall after 40 verified failures: ${before} -> ${after}`); + assert.ok( + after < before - 0.3, + `ability should fall after 40 verified failures: ${before} -> ${after}`, + ); assert.equal(loadRouterModel(d).origin, ".forge/router_model.json"); assert.equal(fitted.provenance.local.outcomes, 40); }); From 1e1fb68e38506527de0d8842a0bcb11509d819da Mon Sep 17 00:00:00 2001 From: Juber Shaikh Date: Tue, 22 Sep 2026 05:32:50 +0000 Subject: [PATCH 10/13] docs(router): universal routing design, measured results and limits; command table; changelog Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JpQ15QdqdDUJLavNhExwvL --- ARCHITECTURE.md | 10 ++-- CHANGELOG.md | 7 +++ README.md | 2 +- docs/UNIVERSAL_ROUTING.md | 101 ++++++++++++++++++++++++++++++++++++++ src/commands.js | 3 +- 5 files changed, 116 insertions(+), 7 deletions(-) create mode 100644 docs/UNIVERSAL_ROUTING.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9a99f68..e7126f7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -620,17 +620,17 @@ from the tree it describes. ```mermaid %%{init: {'theme':'base','themeVariables':{'primaryColor':'#201a15','primaryTextColor':'#f2ede7','primaryBorderColor':'#372c22','lineColor':'#f26430','secondaryColor':'#272019','tertiaryColor':'#171310','edgeLabelBackground':'#201a15','clusterBkg':'#171310','clusterBorder':'#4a3b2e','fontFamily':'ui-sans-serif, system-ui, sans-serif','fontSize':'14px'},'flowchart':{'curve':'basis','padding':10,'nodeSpacing':36,'rankSpacing':44}}}%% flowchart LR - test["test
115 files"] - src["src
99 files"] + test["test
117 files"] + src["src
109 files"] landing["landing
61 files"] research["research
37 files"] global["global
5 files"] - bench["bench
2 files"] + bench["bench
3 files"] scripts["scripts
2 files"] docs["docs
1 file"] examples["examples
1 file"] - test -- 233 --> src - bench -- 7 --> src + test -- 240 --> src + bench -- 8 --> src examples -- 4 --> src test -- 2 --> bench test -- 2 --> global diff --git a/CHANGELOG.md b/CHANGELOG.md index 541c67f..39a60dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- **Universal router** (`src/router`, `forge route universal|outcome|fit|models`). It recommends a model or a cascade across any provider's models, with no vendor, tier or threshold in code. + - **Model:** multidimensional IRT for who solves what, with correlated failures integrated by Gauss–Hermite quadrature; a log-linear cost model; and a cascade policy with a parameter-free default objective (`match-best-single`) plus `target`, `value` and `budget`. + - **Data:** the models live in `data/models.json` and `.forge/models.json`. A shipped prior is fitted on public SWE-bench Verified runs of 11 models from 7 providers. `route outcome` and `route fit` perform a Bayesian update on the project's own outcomes. + - **Measured** (harness-bench run 4, pre-registered, 350 held-out issues): 76.3% solved at $0.093 per task, against 75.1% at $0.364 for the best single model chosen on dev (non-inferior, 74% cheaper). See docs/UNIVERSAL_ROUTING.md for the limits. + ### Fixed - **Binary files no longer trip the commit gate's secret scan.** The staged scan reads every diff --git a/README.md b/README.md index 78a3d67..8043649 100644 --- a/README.md +++ b/README.md @@ -410,7 +410,7 @@ and output live in [`docs/GUIDE.md`](docs/GUIDE.md). | | `forge impact` | hazard-aware blast radius — SCC-aware propagation + data-driven threshold from PageRank centrality and ledger incident history | | | `forge scope` | decompose files into independent clusters (+ coupled files you didn't name) | | | `forge context` | budgeted context assembly + completeness gate — what an edit NEEDS known | -| | `forge route` | recommend the cheapest capable model for a task (+ gateway config) | +| | `forge route` | recommend the cheapest capable model for a task (+ gateway config); `route universal`: any provider's models, lowest expected cost for the success asked for, learned from outcomes | | | `forge verify` | independent verification gate — tests + hallucinated-symbol + provenance (--deep: multi-lens consensus) | | | `forge precommit` | commit-level gate — staged code w/o docs + secret scan (FORGE_COMMIT_GATE=block|warn|0) | | **Memory** | `forge cortex` | self-correcting project memory — status / why | diff --git a/docs/UNIVERSAL_ROUTING.md b/docs/UNIVERSAL_ROUTING.md new file mode 100644 index 0000000..0c79059 --- /dev/null +++ b/docs/UNIVERSAL_ROUTING.md @@ -0,0 +1,101 @@ +# Universal routing + +`forge route universal` recommends a model, or a cascade of models ("try A; if a check fails, try B"), for a task. It works with any provider's models. + +The router code names no vendor, model, tier or threshold, and a test enforces that. Models come from data (`data/models.json`, plus `.forge/models.json` in a project). What each model can do and what it costs is learned from verified outcomes. + +## The model + +**1. Who solves what: multidimensional item response theory.** + +``` +P(model m solves task j | θ_j) = σ( a_m − w·x_j + λ_m·θ_j ), θ_j ~ N(0, I_k) +``` + +| Symbol | Meaning | +|---|---| +| `a_m` | the ability of model m | +| `x_j` | task features (text; repository signals when a repo is present) | +| `w` | learned weights that turn features into a difficulty, so unseen tasks get one | +| `θ_j` | the difficulty the features miss, shared by all models through their loadings `λ_m` | + +The shared `θ_j` is what makes failures correlated: if one model fails a task, others are more likely to fail it too. On the public data, P(Opus 4.5 solves | MiniMax M2.5 failed) is 0.25, against 0.77 unconditionally. + +**Fitting.** +- The fit maximises the marginal posterior, with θ integrated by Gauss–Hermite quadrature. The nodes are computed with Golub–Welsch, not tabulated. +- The optimiser is L-BFGS with analytic gradients. +- Observations can be sparse: each task may have been tried by any subset of models. +- The latent dimension k and the prior scale are chosen by K-fold cross-validated likelihood, and the scale grid expands past its edge while the likelihood improves. + +**2. What an attempt costs.** + +``` +log cost = α_m + β·x + ε, E[cost] = exp(α_m + β·x + s²/2) +``` + +- `α_m` and the shared slope `β` are fitted by least squares on observed attempt costs. +- A model that has prices but no observed attempts takes `α_m` from its price. For models with both, `α − log(blended price)` is close to constant, and the input/output blend is chosen to make it most constant. + +**3. Choosing a cascade.** For a cascade s = (m₁, m₂, …), with node probabilities `P[m][q]` and weights `w_q`: + +``` +P(s solves) = 1 − Σ_q w_q Π_{m∈s} (1 − P[m][q]) +E[cost of s] = Σ_i c_{m_i} · Σ_q w_q Π_{l" --model --pass|--fail --cost ` records a verified result. Only a hash of the task and its features are stored, never the text. +- `forge route fit` refits with the shipped fit as the prior mean. This is a Bayesian update: a few local outcomes barely move it, and many outcomes dominate. +- Cost intercepts are updated with a unit-information prior. +- A model in the registry but not in the fit enters "cold", at the population-mean ability, until outcomes arrive. + +**5. Candidates.** +- `--provider ` limits the candidates to models that provider can serve (the `providers` map in the registry). +- `--provider any` (the default) gives advice across every model. +- The shipped registry gives provider ids only for Anthropic models. Add ids for OpenRouter, a LiteLLM gateway or a native API in `.forge/models.json`. + +## Shipped prior + +`data/router_prior.json` was fitted on public per-task results, and `bench/universal-router/README.md` shows how to regenerate it: + +- **Tasks:** the 500 SWE-bench Verified issues. +- **Runs:** eleven models from seven providers with the same scaffold (mini-SWE-agent 2.0.0), one attempt each, dated February 2026. + - Anthropic: Claude Haiku / Sonnet / Opus 4.5 and Opus 4.6 + - OpenAI: GPT-5.2 and GPT-5 mini + - Google: Gemini 3 Flash + - Moonshot: Kimi K2.5 + - MiniMax: M2.5 + - DeepSeek: V3.2 + - Z-AI: GLM-5 +- **Selection:** cross-validation chose k = 1 at prior scale 2. + +## Measured + +harness-bench run 4 is pre-registered. It fits on 150 dev issues and scores 350 held-out issues against each model's real outcome and cost. + +| Policy | Solved | $ per task | +|---|---|---| +| universal router, `match-best-single` | 76.3% | $0.093 | +| best single model chosen on dev (Gemini 3 Flash) | 75.1% | $0.364 | +| always Claude Opus 4.5 | 77.4% | $0.760 | +| universal router, `target:0.9` | 81.4% | $0.260 | + +**Against the best single model:** non-inferior (+1.1 points, CI [−2.0, +4.3]) at 74% lower cost. In 5-fold cross-validation it is +3.2 points (CI [+0.4, +6.2]) at −$0.58 per task. + +**Limits (measured):** +- **Where the gain comes from.** Most of it comes from choosing across providers. On the 150-issue fit the router does not beat a fixed cascade chosen on the same dev data; with 400 training issues its target modes are cheaper than the fixed equivalents. +- **Targets are optimistic.** Predicted cascade success is optimistic by 4 to 6 points on the test split, so `target:p` lands below p. A cross-validated calibration map is the planned fix. +- **One scaffold, text-only features.** The data is one agent scaffold and text-only features, with February 2026 prices. Re-fit on your own outcomes. + +## Relation to `forge route` + +`forge route` (tiered: haiku / sonnet / opus / fable) is unchanged and remains the default for Claude Code model selection. The universal router is opt-in: run `forge route universal`, or set `route.objective` in `.forge/config.json` to choose its default objective. diff --git a/src/commands.js b/src/commands.js index 217aa76..8bd433f 100644 --- a/src/commands.js +++ b/src/commands.js @@ -92,7 +92,8 @@ export const COMMANDS = { context: "budgeted context assembly + completeness gate — what an edit NEEDS known", preflight: "assumption check — what a task names that the repo doesn't define", config: "provider setup — show / switch / add providers, set default model", - route: "recommend the cheapest capable model for a task (+ gateway config)", + route: + "recommend the cheapest capable model for a task (+ gateway config); `route universal`: any provider's models, lowest expected cost for the success asked for, learned from outcomes", impact: { summary: "hazard-aware blast radius — SCC-aware propagation + data-driven threshold from PageRank centrality and ledger incident history", From 3983b047613596c68a71eade20a97de9a1b0e424 Mon Sep 17 00:00:00 2001 From: Juber Shaikh Date: Tue, 22 Sep 2026 05:37:08 +0000 Subject: [PATCH 11/13] data(router): refit shipped prior with the expanding CV grid (k=1, scale 4) Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JpQ15QdqdDUJLavNhExwvL --- data/router_prior.json | 94 ++++++++++++++++++++++++++------------- docs/UNIVERSAL_ROUTING.md | 2 +- 2 files changed, 63 insertions(+), 33 deletions(-) diff --git a/data/router_prior.json b/data/router_prior.json index 6bbf8b7..ed74c2e 100644 --- a/data/router_prior.json +++ b/data/router_prior.json @@ -60,26 +60,26 @@ "mirt": { "k": 1, "a": [ - 1.952730331922173, - 3.7109748349346297, - 2.6916944244558723, - 3.0128015273548874, - 2.3548805451623376, - 3.144600475417333, - 2.4060353731375135, - 0.39030369993098823, - 2.4010665137393263, - 2.3135806472117144, - 2.9358804100992435 + 2.177858803755454, + 4.063503002402341, + 2.8359347040377507, + 3.3986989106967864, + 2.5626646675266604, + 3.3649044433622177, + 2.5468476514321257, + 0.45157979433268414, + 2.532025999749253, + 2.489473808690017, + 3.1102620827554137 ], "w": [ - 0.11079596035046817, - 0.17624688394608876, - 0.1998995319450095, - -0.07873031584057663, - 0.17729210660376551, - -0.8135197867382431, - 0.7557548283615448, + 0.07070713276101422, + 0.1733322346323156, + 0.2222672533855528, + -0.08134656643250822, + 0.16847366289629023, + -0.855541323999771, + 0.7965984160287538, 0, 0, 0, @@ -88,37 +88,37 @@ ], "L": [ [ - 4.679573546562638 + 5.1911704471943665 ], [ - 4.630794405823491 + 5.220184004941034 ], [ - 3.377363771855404 + 3.649997500447575 ], [ - 5.236017986598532 + 5.99852893744851 ], [ - 4.329023846786565 + 4.7570847722368645 ], [ - 4.051818718273076 + 4.449400899896764 ], [ - 3.560734724743931 + 3.839364815356646 ], [ - 2.6691845328496466 + 2.804764383436117 ], [ - 3.5510834991146014 + 3.8106589710829795 ], [ - 3.9583284784459396 + 4.315707369830679 ], [ - 3.723177648186695 + 4.045751091860787 ] ] }, @@ -172,8 +172,8 @@ "selection": { "chosen": { "k": 1, - "scale": 2, - "heldOutLogLik": -1755.6790297911962 + "scale": 4, + "heldOutLogLik": -1752.9128596102591 }, "table": [ { @@ -191,6 +191,16 @@ "scale": 2, "heldOutLogLik": -1755.6790297911962 }, + { + "k": 1, + "scale": 4, + "heldOutLogLik": -1752.9128596102591 + }, + { + "k": 1, + "scale": 8, + "heldOutLogLik": -1753.1403085829472 + }, { "k": 2, "scale": 0.5, @@ -206,6 +216,21 @@ "scale": 2, "heldOutLogLik": -1792.166234344018 }, + { + "k": 2, + "scale": 4, + "heldOutLogLik": -1791.7523579503713 + }, + { + "k": 2, + "scale": 8, + "heldOutLogLik": -1784.0834221415564 + }, + { + "k": 2, + "scale": 16, + "heldOutLogLik": -1796.2047119381166 + }, { "k": 3, "scale": 0.5, @@ -220,6 +245,11 @@ "k": 3, "scale": 2, "heldOutLogLik": -1775.4902564936888 + }, + { + "k": 3, + "scale": 4, + "heldOutLogLik": -1789.8919516415144 } ], "folds": 3 @@ -230,6 +260,6 @@ "split": "all", "tasks": 500, "outcomes": 5500, - "fittedAt": "2026-09-22T05:00:30.522Z" + "fittedAt": "2026-09-22T05:36:55.196Z" } } diff --git a/docs/UNIVERSAL_ROUTING.md b/docs/UNIVERSAL_ROUTING.md index 0c79059..749c1b5 100644 --- a/docs/UNIVERSAL_ROUTING.md +++ b/docs/UNIVERSAL_ROUTING.md @@ -76,7 +76,7 @@ Every ordered cascade of up to 3 candidates is evaluated; `--depth` bounds the s - MiniMax: M2.5 - DeepSeek: V3.2 - Z-AI: GLM-5 -- **Selection:** cross-validation chose k = 1 at prior scale 2. +- **Selection:** cross-validation chose k = 1 at prior scale 4. Scale 2 had been the edge of the first grid, so the grid kept expanding until the held-out likelihood stopped improving. ## Measured From d6014b5d732a6e4bc60b52fe22436b2865fae325 Mon Sep 17 00:00:00 2001 From: Juber Shaikh <40266375+CodeWithJuber@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:23:29 +0200 Subject: [PATCH 12/13] chore(package): keep the description's em dash literal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The router patch series re-serialized package.json and wrote the description's em dash as a — escape. Same string once parsed, but an unrelated diff line; restore the literal character so the only package.json change on this branch is the new "data" entry in "files". Co-Authored-By: Claude Opus 5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f0c4bd7..69d83ef 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@codewithjuber/forgekit", "version": "1.0.0", - "description": "Shared memory, impact analysis, and guardrail hooks for AI coding agents \u2014 authored once, emitted as native config for Claude Code, Codex, Cursor, Gemini, Aider, and more.", + "description": "Shared memory, impact analysis, and guardrail hooks for AI coding agents — authored once, emitted as native config for Claude Code, Codex, Cursor, Gemini, Aider, and more.", "type": "module", "bin": { "forge": "src/cli.js" From f64c5401b09749edfbe9948ab2dde1b622e72fa5 Mon Sep 17 00:00:00 2001 From: Juber Shaikh <40266375+CodeWithJuber@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:07:04 +0200 Subject: [PATCH 13/13] fix(learn): run the --llm model call without GNU timeout on macOS bin/learn-consolidate.sh wrapped `claude -p` in `timeout 180`, a GNU coreutils command stock macOS does not ship. With stderr sent to /dev/null the missing command failed silently, claude never ran, and every --llm run on a Mac ended in "response too short". The stub-claude test added in this branch exposed it: Install smoke (macos-latest) failed reading calls.log. The call now goes through `limited`: `timeout`, else Homebrew's `gtimeout`, else the bare command. Checked in Git Bash with a PATH that has neither (stub called once, exit 1, "originals kept") and with `timeout` present. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 5 +++++ bin/learn-consolidate.sh | 11 ++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39a60dd..4d8af25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,11 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). model rewrite remains behind an explicit `--llm` first argument, with "contradicted" removed from its prompt. +- **`learn-consolidate.sh --llm` works on macOS.** It wrapped the model call in GNU `timeout`, + which stock macOS does not ship; with stderr discarded the missing command failed silently, + the model was never called, and every run ended in "response too short". The call now uses + `timeout`, else Homebrew's `gtimeout`, else runs unwrapped. + - **The gate docs no longer claim that repeated gates multiply their catch rates.** The headers of `src/commit_gate.js` and `src/gate.js`, ARCHITECTURE.md §5 and the Mintlify verification-gates page said each rung (Stop, pre-commit, CI) was an independent catch diff --git a/bin/learn-consolidate.sh b/bin/learn-consolidate.sh index 44808d6..d5e6c65 100755 --- a/bin/learn-consolidate.sh +++ b/bin/learn-consolidate.sh @@ -59,8 +59,17 @@ include secrets/tokens/PII. Output only the markdown, no preamble. LESSONS: $all" +# `timeout` is GNU coreutils: stock macOS has none (Homebrew coreutils installs it as +# `gtimeout`). Calling a missing `timeout` failed silently here (stderr is discarded), so +# claude never ran and every --llm run on a Mac ended in "response too short". +limited() { + if command -v timeout >/dev/null 2>&1; then timeout 180 "$@" + elif command -v gtimeout >/dev/null 2>&1; then gtimeout 180 "$@" + else "$@"; fi +} + # Uses your logged-in session (slower startup, but authed). Weekly/cron task. -out="$(printf '%s' "$prompt" | timeout 180 claude -p --model haiku 2>/dev/null)" +out="$(printf '%s' "$prompt" | limited claude -p --model haiku 2>/dev/null)" out="$(printf '%s' "$out" | sed '/^[[:space:]]*$/d')" # Guard: never overwrite/delete on an error or empty/too-short response.