diff --git a/tools/renovate/agent-image-fod-vehicle.nix b/tools/renovate/agent-image-fod-vehicle.nix new file mode 100644 index 00000000..8235fc21 --- /dev/null +++ b/tools/renovate/agent-image-fod-vehicle.nix @@ -0,0 +1,74 @@ +# A VERIFICATION VEHICLE for the agent image's view of one shared FOD hash. +# It is NOT how the agent image is built, and nothing in the image, the publish +# lane, or the moon gates evaluates it. Its only consumer is +# tools/renovate/refresh-fod-hashes.ts, whose FOD_ENTRIES table names this file +# and the `compass-agent` attribute below as one entry's build vehicle. +# +# ── WHY IT EXISTS ── +# `agent-image/entrypoint.nix` carries a SINGLE `outputHash` literal over the +# installed `node_modules` tree, and it is imported by TWO consumers resolving +# TWO different nixpkgs revisions: +# +# guest-image/default.nix:66 with ROOT's `pkgs` (../devenv.lock) +# agent-image/devenv.nix:34 with AGENT-IMAGE's `pkgs` (../agent-image/devenv.lock) +# +# That FOD's builder takes `nativeBuildInputs = [ pkgs.bun ]`, so each consumer +# realises it with its own bun derivation, and the one committed hash is correct +# for both only while those two buns produce a byte-identical install tree. +# +# The refresher realises a vehicle to recompute the hash, and the only vehicle in +# the repo was `guest-image/default.nix` — which takes ROOT's pkgs. Nothing built +# `agent-image/devenv.nix` (it is a devenv container definition, realised by +# `devenv container build`, not by `nix build`). So the agent-image consumer's +# builder was never exercised by the refresh: if the two channel revs drifted +# onto bun versions with different install trees, the pin stayed right for +# guest-image and was silently wrong for the agent image, surfacing only when the +# heavy OCI build ran. This file closes that gap by giving that consumer a plain +# `nix build`-able expression with the SAME pkgs the image uses. +# +# ── WHY IT IS A SEPARATE FILE, NOT A CHANGE TO agent-image/ ── +# It lives under tools/renovate/ beside its sole consumer, deliberately: +# +# * agent-image/moon.yml's build task declares `inputs: ['**/*']`, so a file +# added under agent-image/ would reschedule the image build (the dominant CI +# cost, the reason ci.yml's timeout is 90m) on every edit to a file that +# cannot affect the image. +# * a nix file inside agent-image/ reads as part of the image definition. This +# one is test scaffolding for a Renovate task; putting it beside +# refresh-fod-hashes.ts is what tells the next reader that. +# +# ── HOW IT IS USED ── +# nix build -f tools/renovate/agent-image-fod-vehicle.nix compass-agent +# +# The refresher first fakes the `outputHash` in entrypoint.nix, so the build +# fails AT the fixed-output derivation with the `got: ` line it parses, +# and never proceeds to the `bun build --compile` bundle. That fail-fast is the +# same property `guest-image/default.nix` gives the authoritative entry. +let + # The AGENT-IMAGE devenv lock's nixpkgs, resolved exactly as the repo's other + # plain-nix vehicles resolve theirs (guest-image/default.nix:51-57, + # tools/toolchain/gate-tools.nix:36-42) — read the lock, fetch that rev, import + # it. The lock path is the whole point of this file: `../../devenv.lock` (the + # root lock) is what the authoritative vehicle already uses, and reading it + # here would make this vehicle re-derive the identical value and verify + # nothing. + lock = builtins.fromJSON (builtins.readFile ../../agent-image/devenv.lock); + node = lock.nodes.nixpkgs.locked; + nixpkgsSrc = builtins.fetchTarball { + url = "https://github.com/${node.owner}/${node.repo}/archive/${node.rev}.tar.gz"; + sha256 = node.narHash; + }; + pkgs = import nixpkgsSrc { }; + lib = pkgs.lib; +in +{ + # The same import agent-image/devenv.nix performs, with the same `pkgs`. + # entrypoint.nix is `{ pkgs, lib }:` and returns the wrapper derivation, not + # the FOD — the FOD is an internal `let` binding. Realising the wrapper pulls + # the FOD in as a dependency, which is all the refresh needs: nix builds + # dependencies first, so a faked `outputHash` errors there before any of the + # bundle work starts. entrypoint.nix's own relative imports + # (../packages/compass-agent, ../package.json, …) resolve against agent-image/, + # not against this file, so importing it from here is transparent. + compass-agent = import ../../agent-image/entrypoint.nix { inherit pkgs lib; }; +} diff --git a/tools/renovate/config.json5 b/tools/renovate/config.json5 index 351c00e7..1e5ec815 100644 --- a/tools/renovate/config.json5 +++ b/tools/renovate/config.json5 @@ -908,6 +908,12 @@ // Omit the latter and Renovate would run the refresh but silently DROP // it, shipping a channel bump whose outputHash never moved. // + // Step 3 also REALISES tools/renovate/agent-image-fod-vehicle.nix, to check + // the same outputHash through the bun this lock resolves. That file is only + // read, never written, so it must NOT be listed: the allowlist governs what + // Renovate commits, and adding a file the task cannot modify is dead + // surface that reads as a claim the task writes it. + // // Self-hosted bot config MUST allow the command: // allowedCommands: [… , "^bun tools/renovate/refresh-agent-image-nixpkgs\\.ts$"] // see tools/renovate/bot-config.json5 (config.test.ts pins the two diff --git a/tools/renovate/config.test.ts b/tools/renovate/config.test.ts index 63102cde..87ddf36c 100644 --- a/tools/renovate/config.test.ts +++ b/tools/renovate/config.test.ts @@ -1621,16 +1621,21 @@ describe("tools/renovate FOD trigger coverage (every task site, derived from FOD test("the coupled (site, entry) set has its expected shape (guard is not vacuous)", () => { // The guard below iterates `coupled`, so a set that emptied or thinned out // would leave it passing while checking nothing. A bare `> 0` cannot see - // the thinning: all six pairs today bind the SAME entry - // (agent-image/entrypoint.nix, whose triggers are bun.lock, devenv.lock and - // agent-image/devenv.lock), so renaming just ONE of that entry's triggers - // dissolves the coupling at some sites while the rest keep - // the count positive. Pinning the exact count catches a partial trigger - // rename, or a fileFilters edit, that dissolves any single pairing. It does - // NOT check WHICH sites are coupled — the per-site describes above pin - // that — only that the population has not shrunk or grown. A newly coupled - // site is a deliberate edit: update this number in the same change. - expect(coupled.length).toBe(6); + // the thinning: the pairs bind only two of the three entries, and the + // entrypoint.nix pin is now carried by TWO entries (the authoritative + // write and its verify sibling) sharing one trigger list, so renaming a + // single trigger dissolves many pairings while leaving the count positive. + // Pinning the exact count catches a partial trigger rename, or a + // fileFilters edit, that dissolves any single pairing. It does NOT check + // WHICH sites are coupled — the per-site describes above pin that — only + // that the population has not shrunk or grown. A newly coupled site is a + // deliberate edit: update this number in the same change. + // + // 12 = six sites naming a trigger of the two entrypoint.nix entries + // (2 entries × 6 sites), plus the guestd vendorHash entry's zero pairs — + // its triggers are go/go.mod and go/go.sum, which the gomod MANAGER writes + // and no fileFilters names. + expect(coupled.length).toBe(12); expect(taskSites.length).toBeGreaterThan(0); }); @@ -1669,21 +1674,25 @@ describe("tools/renovate FOD trigger coverage (every task site, derived from FOD // A command that refreshes the FOD pin IN-PROCESS rather than by shelling // FOD_COMMAND. The guard's job is to ensure a site committing a trigger - // recomputes the pin; a script that imports the refresher and drives the same - // table satisfies that as completely as the standalone command, and adding the - // command beside it would pay a SECOND faked-pin realise (a full FOD cache - // miss plus a networked bun install) to rewrite a value already correct. Each - // entry is listed with the call site that makes it true, so this stays a - // per-script statement of fact and never a blanket exemption. + // recomputes the pin; a script that imports refreshFodEntries and drives the + // same table satisfies that as completely as the standalone command, and + // adding the command beside it would pay a SECOND faked-pin realise (a full + // FOD cache miss plus a networked bun install) to rewrite a value already + // correct. Each entry is listed with the call site that makes it true, so this + // stays a per-script statement of fact and never a blanket exemption: a script + // added here without that call fails the assertion below. const IN_PROCESS_REFRESHERS: Record = { // refresh-agent-image-nixpkgs.ts: relocks the agent-image channel, then - // awaits the FOD refresh over that scope's entries. - "bun tools/renovate/refresh-agent-image-nixpkgs.ts": "refreshEntry", + // awaits refreshFodEntries(agentImageFodEntries()) — the authoritative write + // and its verify sibling, in the order refreshFodEntries enforces. + "bun tools/renovate/refresh-agent-image-nixpkgs.ts": "refreshFodEntries", }; test("every declared in-process refresher really drives the FOD table", async () => { // Guards the exemption itself: the claim above is only sound while each // listed script genuinely calls the refresher, so read the source and check. + // Without this, a later edit could strip the call and the site would keep + // its pass through this table alone. for (const [command, symbol] of Object.entries(IN_PROCESS_REFRESHERS)) { const script = command.replace(/^bun /, ""); const source = await Bun.file(join(repoRoot, script)).text(); @@ -1709,7 +1718,7 @@ describe("tools/renovate FOD trigger coverage (every task site, derived from FOD const found: string[] = []; // A site refreshes the pin either by shelling FOD_COMMAND — which must run // LAST, after whatever wrote the trigger — or by running a script that - // drives the same table in-process, which orders the two itself. + // drives the table in-process, which orders the two itself. const inProcess = commands.filter((c) => c in IN_PROCESS_REFRESHERS); const fodIndex = commands.indexOf(FOD_COMMAND); if (fodIndex !== -1) { diff --git a/tools/renovate/moon.yml b/tools/renovate/moon.yml index 6f83bf5b..15ed86d6 100644 --- a/tools/renovate/moon.yml +++ b/tools/renovate/moon.yml @@ -24,6 +24,10 @@ tasks: test: inputs: - '*.ts' + # The FOD verification vehicle (agent-image-fod-vehicle.nix) lives here and + # is named by refresh-fod-hashes.ts's table; the suite asserts it exists, so + # deleting or renaming it must schedule this project. + - '*.nix' - '*.json5' - 'tsconfig.json' - '/tsconfig.base.json' diff --git a/tools/renovate/refresh-agent-image-nixpkgs.test.ts b/tools/renovate/refresh-agent-image-nixpkgs.test.ts index 12e5e996..0d225d49 100644 --- a/tools/renovate/refresh-agent-image-nixpkgs.test.ts +++ b/tools/renovate/refresh-agent-image-nixpkgs.test.ts @@ -101,8 +101,16 @@ describe("agent-image scope geometry", () => { // The single named input, not a bare `devenv update`: relocking every input // would bloat the PR's diff past the channel advance the branch is about. - test("the relocked input is the channel input alone", () => { - expect(NIXPKGS_INPUT).toBe("nixpkgs"); + // Asserted against the real devenv.yaml rather than against the constant's own + // literal — the hazard is the input being RENAMED upstream, which a + // self-referential equality check cannot see. + test("the relocked input names a real input of this scope", () => { + const yaml = Bun.YAML.parse( + readFileSync(join(repoRoot, AGENT_IMAGE_DIR, "devenv.yaml"), "utf8"), + ) as { inputs?: Record }; + const inputs = yaml.inputs ?? {}; + expect(Object.keys(inputs)).toContain(NIXPKGS_INPUT); + expect(inputs[NIXPKGS_INPUT]?.url).toContain("devenv-nixpkgs"); }); // Ground truth: the lock the config's manager governs must exist at this @@ -114,18 +122,42 @@ describe("agent-image scope geometry", () => { // The rev read here is the one the FOD refresh hangs off: the channel // resolves the bun entrypoint.nix's builder uses, so the FOD table must gate - // that entry on this lock. If the table and this task disagree, the entry - // point throws rather than shipping a possibly-stale outputHash — assert the - // agreement here too so the drift fails in a unit test, not on a branch. - test("the FOD entry for entrypoint.nix is gated on this lock", () => { - const entry = FOD_ENTRIES.find( + // EVERY entry over that pin on this lock. If the table and this task disagree, + // the entry point throws rather than shipping a possibly-stale outputHash — + // assert the agreement here too so the drift fails in a unit test, not on a + // branch. + test("every FOD entry for entrypoint.nix is gated on this lock", () => { + const entries = FOD_ENTRIES.filter( (e) => e.file === "agent-image/entrypoint.nix", ); - expect(entry).toBeDefined(); - expect(entry?.triggers).toContain(AGENT_IMAGE_LOCK); - // And still on bun.lock — the manifest trigger that already existed; this - // task ADDS a cause, it does not replace one. - expect(entry?.triggers).toContain("bun.lock"); + expect(entries.length).toBeGreaterThan(0); + for (const entry of entries) { + expect(entry.triggers).toContain(AGENT_IMAGE_LOCK); + // And still on bun.lock — the manifest trigger that already existed; this + // task ADDS a cause, it does not replace one. + expect(entry.triggers).toContain("bun.lock"); + } + }); + + // This lock resolves the bun the OCI image build uses, so the shared + // outputHash is only actually CHECKED for that builder if some entry realises + // a vehicle whose pkgs come from this lock. Without one, the refresh reverts + // to a one-builder rewrite whose divergence surfaces only on the image build. + // The vehicle file must exist too — a table naming a deleted file would fail + // at realise time on a branch, not here. + test("a FOD entry realises entrypoint.nix through this lock's own pkgs", () => { + const viaThisLock = FOD_ENTRIES.filter( + (e) => + e.file === "agent-image/entrypoint.nix" && + e.vehicleChannelLock === AGENT_IMAGE_LOCK, + ); + expect(viaThisLock.length).toBe(1); + const [entry] = viaThisLock; + if (!entry) throw new Error("expected exactly one entry via this lock"); + // It VERIFIES; the authoritative write stays with the root-pkgs vehicle, or + // two writers over one marker would be last-write-wins. + expect(entry.verifyOf).toBeDefined(); + expect(existsSync(join(repoRoot, entry.buildFile))).toBe(true); }); }); @@ -277,9 +309,10 @@ async function buildEntryRepo(): Promise { join(repo, "agent-image", "entrypoint.nix"), `{ pkgs, lib }:\n outputHash = "${STUB_SRI}";\n`, ); - // The single realise vehicle this scope's refresh drives. - await mkdir(join(repo, "guest-image"), { recursive: true }); - await Bun.write(join(repo, "guest-image", "default.nix"), "{ }\n"); + for (const entry of FOD_ENTRIES) { + await mkdir(join(repo, entry.buildFile, ".."), { recursive: true }); + await Bun.write(join(repo, entry.buildFile), "{ }\n"); + } await Bun.write(join(repo, "stubbin", "nix"), STUB_NIX); await chmod(join(repo, "stubbin", "nix"), 0o755); diff --git a/tools/renovate/refresh-agent-image-nixpkgs.ts b/tools/renovate/refresh-agent-image-nixpkgs.ts index 9cbdddd9..c2d3d5e3 100644 --- a/tools/renovate/refresh-agent-image-nixpkgs.ts +++ b/tools/renovate/refresh-agent-image-nixpkgs.ts @@ -50,18 +50,24 @@ // deliberate). One hash for two builders is safe only while both revs resolve to // a bun producing a byte-identical install tree. // -// The refresh has a REACH LIMIT worth stating plainly, because it is not what -// the coupling suggests. refresh-fod-hashes.ts realises exactly one vehicle, -// `guest-image/default.nix`, and that vehicle takes ROOT's `pkgs`. Nothing in -// the repo builds `agent-image/devenv.nix`. So on a branch that moves ONLY the -// agent-image channel, the recomputed SRI is derived through root's bun and -// equals the committed value: the step is a correct, cheap no-op rewrite rather -// than a verification of this scope's builder. It is wired anyway so the pin is -// recomputed whenever the shared hash's inputs move for any reason, and so the -// coupling is declared where a reader looks for it. The consequence to keep in -// view: if the two channel revs diverge onto bun versions with different install -// trees, this hash stays right for guest-image and is silently wrong for the -// agent image, and that break surfaces on the agent-image OCI build. +// BOTH builders are checked, and that is what makes this step a verification +// rather than a bookkeeping rewrite. refresh-fod-hashes.ts's table carries the +// entrypoint pin TWICE: an AUTHORITATIVE entry realised through +// `guest-image/default.nix` (ROOT's pkgs), which writes the canonical SRI, and a +// VERIFY entry realised through `tools/renovate/agent-image-fod-vehicle.nix` — +// a plain-nix file that imports the SAME entrypoint.nix with the pkgs resolved +// from THIS lock, i.e. the bun the OCI image build actually uses. The verify +// entry recomputes and compares; it never writes. +// +// So on a branch that moves ONLY the agent-image channel, the two vehicles are +// realised in order and their SRIs must agree. Agreement is the quiet path (the +// authoritative rewrite is a no-op and the check logs a confirmation). +// Disagreement means the two channel revs' buns produce different install trees, +// so NO single `outputHash` literal can satisfy both consumers: the refresh +// throws, naming both SRIs, both vehicles and both channel revs, and this task +// exits non-zero. That is a red `renovate/artifacts` on the very PR that moved +// the pin — instead of a green PR whose break waits for the agent-image OCI +// build to find it. // // Steps: // 1. Self-gate — exit 0 unless agent-image/devenv.lock differs from the base @@ -72,11 +78,14 @@ // lock (narHash + lastModified + the transitive nixpkgs-src node), not just // the rev the regex bumped. Networked, but light: it re-locks inputs, it // does NOT build the image. -// 3. FOD refresh — recompute agent-image/entrypoint.nix's `outputHash` through -// refresh-fod-hashes.ts's own machinery (its exported table entry + -// refreshEntry), so there is ONE realise-and-parse implementation rather -// than a second copy that drifts from its got:-attribution and -// restore-on-failure discipline. +// 3. FOD refresh + cross-builder verify — recompute agent-image/entrypoint.nix's +// `outputHash` through refresh-fod-hashes.ts's own machinery (its exported +// table entries + refreshFodEntries), so there is ONE realise-and-parse +// implementation rather than a second copy that drifts from its +// got:-attribution and restore-on-failure discipline. refreshFodEntries +// writes the authoritative value first, then verifies it through the +// agent-image vehicle; it fails loud if either build reports no `got:`, or +// if the two vehicles disagree. // // Invoked by the agent-image channel packageRule's postUpgradeTasks // (config.json5) as `bun tools/renovate/refresh-agent-image-nixpkgs.ts`, @@ -100,10 +109,11 @@ // that block still names both caches + keys. // // Exit codes: -// 0 - the lock was relocked and the FOD hash refreshed (or a no-op branch: the -// lock does not differ from base). +// 0 - the lock was relocked, the FOD hash refreshed, and both builders agreed +// on it (or a no-op branch: the lock does not differ from base). // 1 - a step failed (the relock itself, a lock-shape change, a relock that -// wrote nothing, an unresolvable base ref, or the FOD realise). +// wrote nothing, an unresolvable base ref, a FOD realise, or the two +// vehicles disagreeing about the shared outputHash). // // What a non-zero exit actually BUYS — do NOT overclaim it. It does NOT abort // the Renovate branch: the failure is caught in Renovate's post-upgrade command @@ -124,35 +134,55 @@ import { agentImageNixpkgsRev, NIXPKGS_INPUT, } from "./refresh-agent-image-nixpkgs.core.ts"; -import { FOD_ENTRIES, refreshEntry } from "./refresh-fod-hashes.ts"; +import { + FOD_ENTRIES, + type FodEntry, + refreshFodEntries, +} from "./refresh-fod-hashes.ts"; -// The FOD whose `outputHash` this scope's bun realises. Resolved from -// refresh-fod-hashes.ts's SHIPPED table (by the file it pins) rather than -// restated here, so a table edit that moves or renames the entry fails LOUD at -// this lookup instead of silently skipping the refresh — the half-refreshed PR -// this step exists to prevent. +// The FOD entries over the pin this scope's bun realises — BOTH of them: the +// authoritative one (realised through guest-image/default.nix, root's pkgs) and +// the verify one (realised through the agent-image vehicle, THIS lock's pkgs). +// Resolved from refresh-fod-hashes.ts's SHIPPED table by the file they pin, +// rather than restated here, so a table edit that moves or renames an entry +// fails LOUD at this lookup instead of silently skipping the refresh — the +// half-refreshed PR this step exists to prevent. const ENTRYPOINT_NIX = "agent-image/entrypoint.nix"; -function agentImageFodEntry() { - const entry = FOD_ENTRIES.find((e) => e.file === ENTRYPOINT_NIX); - if (!entry) { +function agentImageFodEntries(): FodEntry[] { + const entries = FOD_ENTRIES.filter((e) => e.file === ENTRYPOINT_NIX); + if (entries.length === 0) { throw new Error( `refresh-agent-image-nixpkgs: no FOD_ENTRIES entry pins ${ENTRYPOINT_NIX} — ` + "refresh-fod-hashes.ts's table moved, so this relock cannot refresh the outputHash " + "its new bun may have invalidated. Refusing to ship a possibly-stale FOD pin.", ); } - // The entry must ALSO be gated on this lock, or a real Renovate run of - // refresh-fod-hashes.ts on this branch would no-op while we refresh here — - // two sites disagreeing about what invalidates the pin. - if (!entry.triggers.includes(AGENT_IMAGE_LOCK)) { + // One of them must realise a vehicle resolved from THIS lock, or the relock + // below would move a bun that nothing then builds against — exactly the blind + // spot the verify entry was added to close. Checked here rather than assumed, + // so dropping the vehicle from the table reds this task instead of quietly + // reverting it to a one-builder rewrite. + if (!entries.some((e) => e.vehicleChannelLock === AGENT_IMAGE_LOCK)) { throw new Error( - `refresh-agent-image-nixpkgs: the ${ENTRYPOINT_NIX} FOD entry does not list ` + - `${AGENT_IMAGE_LOCK} among its triggers, so refresh-fod-hashes.ts no longer agrees ` + - "that an agent-image channel relock can move that outputHash. Reconcile the table " + - "with this task before shipping.", + `refresh-agent-image-nixpkgs: no ${ENTRYPOINT_NIX} FOD entry realises a vehicle ` + + `resolved from ${AGENT_IMAGE_LOCK}, so this relock's new bun would never build the ` + + "pin it can invalidate. Restore the agent-image verification vehicle before shipping.", ); } - return entry; + // Every entry must ALSO be gated on this lock, or a real Renovate run of + // refresh-fod-hashes.ts on this branch would no-op while we refresh here — + // two sites disagreeing about what invalidates the pin. + for (const entry of entries) { + if (!entry.triggers.includes(AGENT_IMAGE_LOCK)) { + throw new Error( + `refresh-agent-image-nixpkgs: the ${ENTRYPOINT_NIX} FOD entry '${entry.id}' does not ` + + `list ${AGENT_IMAGE_LOCK} among its triggers, so refresh-fod-hashes.ts no longer ` + + "agrees that an agent-image channel relock can move that outputHash. Reconcile the " + + "table with this task before shipping.", + ); + } + } + return entries; } async function main(): Promise { @@ -206,10 +236,10 @@ async function main(): Promise { return 0; } - // Resolve the FOD entry BEFORE the relock: a table drift must fail the task + // Resolve the FOD entries BEFORE the relock: a table drift must fail the task // without having networked or rewritten the lock, so the branch is left in // the state Renovate handed it rather than half-processed. - const fodEntry = agentImageFodEntry(); + const fodEntries = agentImageFodEntries(); // ── Step 2: relock the channel input in the agent-image scope. ── // The regex update moved only the rev string, leaving the lock's narHash / @@ -251,23 +281,33 @@ async function main(): Promise { `refresh-agent-image-nixpkgs: ${AGENT_IMAGE_LOCK} relocked — '${NIXPKGS_INPUT}' now at ${agentImageNixpkgsRev(after)}.`, ); - // ── Step 3: recompute the shared FOD hash. ── + // ── Step 3: recompute the shared FOD hash, and check BOTH builders. ── // entrypoint.nix's `outputHash` content-addresses the installed node_modules // tree, and a stale value fails the image build with `hash mismatch in - // fixed-output derivation`. This realises the build vehicle against a faked - // pin and writes back the SRI nix reports — refresh-fod-hashes.ts's own - // machinery, unchanged, so got:-attribution and restore-on-failure are shared - // rather than re-implemented. It fails loud if the build reports no `got:`. + // fixed-output derivation`. refreshFodEntries drives refresh-fod-hashes.ts's + // own machinery, unchanged, so got:-attribution and restore-on-failure are + // shared rather than re-implemented. It fails loud if a build reports no + // `got:`. + // + // Two entries, one pin, in a fixed order refreshFodEntries enforces: + // + // * the authoritative realise through guest-image/default.nix (ROOT's pkgs) + // WRITES the canonical SRI. On an agent-image-channel-only branch that + // rewrite is a no-op — root's bun did not move — which is expected. + // * the verify realise through tools/renovate/agent-image-fod-vehicle.nix + // re-derives the same pin with the pkgs from the lock JUST relocked above, + // i.e. the bun this image actually builds with, and COMPARES. This is the + // leg that makes the relock's effect on the pin observable at all. // - // Expect a no-op on an agent-image-channel-only branch: the vehicle is - // guest-image/default.nix, which resolves through ROOT's pkgs, so it cannot - // see this scope's bun move (the header's REACH LIMIT). Recomputing anyway - // keeps the pin correct whenever the shared hash's inputs do move, and costs - // one realise. + // Agreement is the quiet path. Disagreement throws — one `outputHash` literal + // cannot serve two buns with different install trees — reddening + // `renovate/artifacts` on the branch that moved the pin, rather than letting + // the break wait for the agent-image OCI build. console.log( - `refresh-agent-image-nixpkgs: recomputing the ${fodEntry.file} outputHash (vehicle resolves through root's pkgs; a no-op unless the shared inputs moved) ...`, + `refresh-agent-image-nixpkgs: recomputing the ${ENTRYPOINT_NIX} outputHash and verifying it ` + + `through this scope's own bun (${fodEntries.length} vehicle(s)) ...`, ); - await refreshEntry(fodEntry); + await refreshFodEntries(fodEntries); console.log("refresh-agent-image-nixpkgs: done."); return 0; diff --git a/tools/renovate/refresh-fod-hashes.test.ts b/tools/renovate/refresh-fod-hashes.test.ts index 2658c46e..4ed38b97 100644 --- a/tools/renovate/refresh-fod-hashes.test.ts +++ b/tools/renovate/refresh-fod-hashes.test.ts @@ -4,9 +4,12 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { $ } from "bun"; import { + assertFodTableInvariants, FOD_ENTRIES, type FodEntry, + hashOnMarker, parseGotForFragment, + refreshEntry, rewriteInlineHash, } from "./refresh-fod-hashes.ts"; @@ -19,6 +22,13 @@ import { // `hash mismatch in fixed-output derivation` and the bump PR goes red. The // refresher recomputes the hash IN the bump branch so the PR lands green. // +// It also guards the SECOND half of that pin's story: `outputHash` is realised by +// two builders (guest-image with root's pkgs, the agent image with its own), so +// the table pairs an AUTHORITATIVE entry that writes the value with a VERIFY +// entry that recomputes it through the agent-image vehicle and compares. Equal is +// a quiet no-op; a difference must throw naming both SRIs, because no single +// literal can then satisfy both builders. +// // The failure mode is precisely the script's cwd × git-cwd-relative-pathspec // gate interaction plus the fake-hash→build→parse-`got:` recovery — only a real // run in a real git repo exercises it. So this drives the ACTUAL shipped script @@ -28,27 +38,35 @@ import { // Network-free & deterministic: the stub `nix` derives a stable `got:` SRI from // the drv fragment it's asked to build, so a run is fully offline and asserting a // pin took the fragment-specific stub value proves both that the per-FOD gate -// fired AND that the correct derivation's hash was parsed and written back. +// fired AND that the correct derivation's hash was parsed and written back. The +// stub also honours a per-vehicle divergence knob, which is how the two-builders +// disagreement is exercised without two real nixpkgs. // -// The fixture layout, trigger files, and pin markers are DERIVED from the -// script's own exported FOD_ENTRIES table, so a rebase that edits the table keeps -// this test honest without a second edit. +// The fixture layout, trigger files, vehicles, and pin markers are DERIVED from +// the script's own exported FOD_ENTRIES table, so a rebase that edits the table +// keeps this honest without a second edit. const SCRIPT_REL = "tools/renovate/refresh-fod-hashes.ts"; const REAL_SCRIPT = join(import.meta.dir, "refresh-fod-hashes.ts"); -// Resolve each FOD entry from the shipped table, throwing on drift so the fixture -// follows the script. A returning helper (not a top-level `if (!x) throw`) gives a -// non-nullable type that narrows into the closures below. -function mustFind(fragment: string): FodEntry { - const entry = FOD_ENTRIES.find((e) => e.drvFragment === fragment); +// Resolve each FOD entry from the shipped table by its stable id, throwing on +// drift so the fixture follows the script. A returning helper (not a top-level +// `if (!x) throw`) gives a non-nullable type that narrows into the closures +// below. Keyed on `id`, not `drvFragment`: the two entrypoint.nix entries share a +// fragment by design — they are the same derivation seen through two nixpkgs. +function mustFind(id: string): FodEntry { + const entry = FOD_ENTRIES.find((e) => e.id === id); if (!entry) { - throw new Error(`fixture drift: expected a '${fragment}' FOD entry`); + throw new Error(`fixture drift: expected a '${id}' FOD entry`); } return entry; } -const GO_ENTRY = mustFind("go-modules"); -const BUN_ENTRY = mustFind("node-modules"); +const GO_ENTRY = mustFind("guestd-go-vendor"); +const BUN_WRITE_ENTRY = mustFind("agent-node-modules-root-pkgs"); +const BUN_VERIFY_ENTRY = mustFind("agent-node-modules-agent-image-pkgs"); +// The real repo, for the table properties that must hold against files on disk +// (as opposed to the hermetic fixture repo the gate tests build below). +const repoRoot = join(import.meta.dir, "..", ".."); // Hermetic git: no user/global/system config leakage, identity from env only. const HERMETIC_ENV = { @@ -90,21 +108,39 @@ in compass-app `; const BUN_NIX_FIXTURE = `let nodeModules = pkgs.stdenv.mkDerivation { - ${BUN_ENTRY.marker}${bodyOf(PLACEHOLDER_BUN)}"; + ${BUN_WRITE_ENTRY.marker}${bodyOf(PLACEHOLDER_BUN)}"; }; in nodeModules `; - -// A fake `nix`: parse the `-f ` build request and emit a -// `hash mismatch` block for BOTH FODs (the real build with --keep-going reports -// every stale FOD), each with a fragment-derived deterministic `got:` SRI. Shape -// matches what parseGotForFragment scans for. Offline. +// A stand-in for each vehicle the table names that is not itself a pin file. The +// stub nix never evaluates it; it exists so the fixture tree has the file the +// script passes to `nix build -f`, matching the real layout. +const VEHICLE_FIXTURE = "{ }\n"; + +// A fake `nix`: emit a `hash mismatch` block for BOTH FODs (the real build with +// --keep-going reports every stale FOD), each with a fragment-derived +// deterministic `got:` SRI. Shape matches what parseGotForFragment scans for. +// Offline. +// +// STUB_DIVERGE_VEHICLE, when set to a `-f ` value, makes THAT vehicle +// report a different SRI for the same fragment — the two-builders-disagree case, +// which is otherwise only reachable with two real, skewed nixpkgs revisions. const STUB_NIX = `#!/usr/bin/env bash -# Ignore all args; emit the mismatch shape for both known FODs to stderr. +# Find the \`-f \` vehicle so a per-vehicle divergence can be simulated. +vehicle="" +prev="" +for arg in "$@"; do + if [ "$prev" = "-f" ]; then vehicle="$arg"; fi + prev="$arg" +done +salt="" +if [ -n "\${STUB_DIVERGE_VEHICLE:-}" ] && [ "$vehicle" = "$STUB_DIVERGE_VEHICLE" ]; then + salt="diverged" +fi emit() { local frag="$1" local digest - digest="$(printf %s "$frag" | sha256sum | cut -d' ' -f1)" + digest="$(printf %s "$salt$frag" | sha256sum | cut -d' ' -f1)" echo "error: hash mismatch in fixed-output derivation '/nix/store/deadbeef-compass-\${frag}.drv':" >&2 echo " specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" >&2 echo " got: sha256-stub-\${digest}" >&2 @@ -115,21 +151,27 @@ exit 1 `; // Mirror the stub's fragment→SRI derivation so the expected value is computable -// in-process. -function stubSriForFragment(fragment: string): string { +// in-process. `salt` matches the stub's divergence prefix. +function stubSriForFragment(fragment: string, salt = ""): string { const hasher = new Bun.CryptoHasher("sha256"); - hasher.update(fragment); + hasher.update(`${salt}${fragment}`); return `sha256-stub-${hasher.digest("hex")}`; } -// The `sha256-…` value on the marker line of a pin file. -function hashOnMarker(nixText: string, marker: string): string | undefined { +// The `sha256-…` value on the marker line, tolerating absence — the shipped +// hashOnMarker throws instead (a verify that cannot find its pin must fail +// loud), so tests asserting "unchanged/absent" use this local reader. +function maybeHashOnMarker( + nixText: string, + marker: string, +): string | undefined { const line = nixText.split("\n").find((l) => l.includes(marker)); return line?.match(/sha256-[^"]*/)?.[0]; } // Build a throwaway repo mirroring the minimal repo-root layout and commit it as -// the `main` baseline: both pin files, both trigger manifests, the shipped script. +// the `main` baseline: both pin files, every vehicle, both trigger manifests, the +// shipped script. async function buildBaselineRepo(): Promise { const repo = await mkdtemp(join(tmpdir(), "pr579-")); const write = async (rel: string, body: string) => { @@ -142,7 +184,15 @@ async function buildBaselineRepo(): Promise { await write(SCRIPT_REL, await readFile(REAL_SCRIPT, "utf8")); // Pin files at the paths the TABLE declares. await write(GO_ENTRY.file, GO_NIX_FIXTURE); - await write(BUN_ENTRY.file, BUN_NIX_FIXTURE); + await write(BUN_WRITE_ENTRY.file, BUN_NIX_FIXTURE); + // Every vehicle the table names, unless it IS a pin file already written + // above — derived from the shipped table so a new vehicle needs no second + // edit here. + const pinFiles = FOD_ENTRIES.map((e) => e.file); + for (const entry of FOD_ENTRIES) { + if (pinFiles.includes(entry.buildFile)) continue; + await write(entry.buildFile, VEHICLE_FIXTURE); + } // Mirror pin files the Go entry declares (same vendorHash, refreshed in // lockstep — derived from the shipped table so a rebase that edits mirrorFiles // keeps this honest with no second edit). @@ -150,8 +200,29 @@ async function buildBaselineRepo(): Promise { await write(mirror, GO_MIRROR_FIXTURE); } // Trigger manifests (content is irrelevant; only their diff-vs-base matters). - for (const trigger of [...GO_ENTRY.triggers, ...BUN_ENTRY.triggers]) { - await write(trigger, "baseline\n"); + for (const entry of FOD_ENTRIES) { + for (const trigger of entry.triggers) { + await write(trigger, "baseline\n"); + } + } + // Channel locks the divergence diagnostic quotes. Real devenv locks; only + // `nodes.nixpkgs.locked.rev` is read, and only to name the two revs in the + // error, so a minimal shape with distinguishable revs is enough. + for (const entry of FOD_ENTRIES) { + await write( + entry.vehicleChannelLock, + `${JSON.stringify( + { + nodes: { + nixpkgs: { + locked: { rev: revFor(entry.vehicleChannelLock) }, + }, + }, + }, + null, + 2, + )}\n`, + ); } // The fake nix. const binDir = join(repo, "stubbin"); @@ -173,15 +244,24 @@ async function buildBaselineRepo(): Promise { return repo; } +// A deterministic 40-hex rev per lock path, so the divergence error's two revs +// are distinguishable and assertable. +function revFor(lockPath: string): string { + const hasher = new Bun.CryptoHasher("sha256"); + hasher.update(lockPath); + return hasher.digest("hex").slice(0, 40); +} + // Run the shipped script exactly as Renovate does: cwd = repo root, stub nix // first on PATH, base branch = the committed baseline. -async function runRefresh(repo: string) { +async function runRefresh(repo: string, extraEnv: Record = {}) { return await $`bun ${SCRIPT_REL}` .cwd(repo) .env({ ...HERMETIC_ENV, PATH: `${join(repo, "stubbin")}:${process.env.PATH}`, RENOVATE_BASE_BRANCH: "main", + ...extraEnv, }) .quiet() .nothrow(); @@ -209,7 +289,7 @@ describe("tools/renovate/refresh-fod-hashes.ts gate (PR #579)", () => { expect(res.exitCode).toBe(0); expect(stdout).not.toContain("nothing to do"); const goNix = await readFile(join(repo, GO_ENTRY.file), "utf8"); - expect(hashOnMarker(goNix, GO_ENTRY.marker)).toBe( + expect(maybeHashOnMarker(goNix, GO_ENTRY.marker)).toBe( stubSriForFragment("go-modules"), ); }); @@ -229,7 +309,7 @@ describe("tools/renovate/refresh-fod-hashes.ts gate (PR #579)", () => { expect(mirrors.length).toBeGreaterThan(0); // the Go entry declares flake.nix for (const mirror of mirrors) { const text = await readFile(join(repo, mirror), "utf8"); - expect(hashOnMarker(text, GO_ENTRY.marker)).toBe( + expect(maybeHashOnMarker(text, GO_ENTRY.marker)).toBe( stubSriForFragment("go-modules"), ); expect(text).not.toContain(bodyOf(PLACEHOLDER_GO_MIRROR)); @@ -239,17 +319,19 @@ describe("tools/renovate/refresh-fod-hashes.ts gate (PR #579)", () => { // A gomod-only bump refreshes the Go vendorHash and leaves the bun outputHash // pin byte-for-byte untouched — the per-FOD self-gate granularity. test("a go/go.mod bump leaves the bun outputHash pin untouched", async () => { - const bunBefore = await readFile(join(repo, BUN_ENTRY.file), "utf8"); + const bunBefore = await readFile(join(repo, BUN_WRITE_ENTRY.file), "utf8"); await Bun.write(join(repo, "go/go.mod"), "bumped\n"); const res = await runRefresh(repo); expect(res.exitCode).toBe(0); const goNix = await readFile(join(repo, GO_ENTRY.file), "utf8"); - expect(hashOnMarker(goNix, GO_ENTRY.marker)).toBe( + expect(maybeHashOnMarker(goNix, GO_ENTRY.marker)).toBe( stubSriForFragment("go-modules"), ); - expect(await readFile(join(repo, BUN_ENTRY.file), "utf8")).toBe(bunBefore); + expect(await readFile(join(repo, BUN_WRITE_ENTRY.file), "utf8")).toBe( + bunBefore, + ); }); // A bun.lock bump refreshes only the bun outputHash; the Go vendorHash is left @@ -261,8 +343,8 @@ describe("tools/renovate/refresh-fod-hashes.ts gate (PR #579)", () => { const res = await runRefresh(repo); expect(res.exitCode).toBe(0); - const bunNix = await readFile(join(repo, BUN_ENTRY.file), "utf8"); - expect(hashOnMarker(bunNix, BUN_ENTRY.marker)).toBe( + const bunNix = await readFile(join(repo, BUN_WRITE_ENTRY.file), "utf8"); + expect(maybeHashOnMarker(bunNix, BUN_WRITE_ENTRY.marker)).toBe( stubSriForFragment("node-modules"), ); expect(await readFile(join(repo, GO_ENTRY.file), "utf8")).toBe(goBefore); @@ -283,10 +365,10 @@ describe("tools/renovate/refresh-fod-hashes.ts gate (PR #579)", () => { const res = await runRefresh(repo); expect(res.exitCode).toBe(0); - const bunNix = await readFile(join(repo, BUN_ENTRY.file), "utf8"); - expect(hashOnMarker(bunNix, BUN_ENTRY.marker)).toBe( - stubSriForFragment("node-modules"), - ); + const bunNix = await readFile(join(repo, BUN_WRITE_ENTRY.file), "utf8"); + expect( + hashOnMarker(bunNix, BUN_WRITE_ENTRY.marker, BUN_WRITE_ENTRY.file), + ).toBe(stubSriForFragment("node-modules")); expect(await readFile(join(repo, GO_ENTRY.file), "utf8")).toBe(goBefore); }); @@ -309,7 +391,7 @@ describe("tools/renovate/refresh-fod-hashes.ts gate (PR #579)", () => { expect(res.exitCode).toBe(0); expect(stdout).toContain("nothing to do"); const goNix = await readFile(join(repo, GO_ENTRY.file), "utf8"); - expect(hashOnMarker(goNix, GO_ENTRY.marker)).toBe(PLACEHOLDER_GO); + expect(maybeHashOnMarker(goNix, GO_ENTRY.marker)).toBe(PLACEHOLDER_GO); }); // Silent-no-op defense: if the build reports no `got:` for the gated FOD (a @@ -329,7 +411,111 @@ describe("tools/renovate/refresh-fod-hashes.ts gate (PR #579)", () => { expect(res.stderr.toString()).toMatch(/no 'got:' SRI/); // The pin was restored to its original placeholder, not left faked. const goNix = await readFile(join(repo, GO_ENTRY.file), "utf8"); - expect(hashOnMarker(goNix, GO_ENTRY.marker)).toBe(PLACEHOLDER_GO); + expect(maybeHashOnMarker(goNix, GO_ENTRY.marker)).toBe(PLACEHOLDER_GO); + }); +}); + +describe("two builders, one outputHash: the verify entry", () => { + let repo: string; + beforeEach(async () => { + repo = await buildBaselineRepo(); + }); + afterEach(async () => { + if (repo) await rm(repo, { recursive: true, force: true }); + }); + + // The EQUAL path — the expected outcome on a healthy branch. Both vehicles + // report the same SRI, so the verify entry writes nothing, changes nothing, + // and does not fail the task. The pin ends at the authoritative value. + test("agreement between the two vehicles is a silent no-op", async () => { + await Bun.write(join(repo, "bun.lock"), "bumped\n"); + + const res = await runRefresh(repo); + const stdout = res.stdout.toString(); + + expect(res.exitCode).toBe(0); + expect(stdout).toContain("verified"); + expect(stdout).not.toContain("cannot satisfy both"); + const bunNix = await readFile(join(repo, BUN_WRITE_ENTRY.file), "utf8"); + expect(maybeHashOnMarker(bunNix, BUN_WRITE_ENTRY.marker)).toBe( + stubSriForFragment("node-modules"), + ); + // And it left no fake behind on the second realise either. + expect(bunNix).not.toContain("sha256-AAAAAAAA"); + }); + + // The MISMATCH path — the whole reason the second vehicle exists. The + // agent-image vehicle reports a different SRI for the same FOD, meaning the + // two channel revs' buns produce different install trees. One literal cannot + // serve both, so the task must exit non-zero (reddening `renovate/artifacts`) + // with a diagnosis naming BOTH SRIs, both vehicles, and both channel revs. + test("a divergent second vehicle throws naming both SRIs, vehicles and revs", async () => { + await Bun.write(join(repo, "bun.lock"), "bumped\n"); + + const res = await runRefresh(repo, { + STUB_DIVERGE_VEHICLE: BUN_VERIFY_ENTRY.buildFile, + }); + const stderr = res.stderr.toString(); + + expect(res.exitCode).not.toBe(0); + expect(stderr).toContain("cannot satisfy both"); + // Both SRIs, named. + expect(stderr).toContain(stubSriForFragment("node-modules")); + expect(stderr).toContain(stubSriForFragment("node-modules", "diverged")); + // Both vehicles, named. + expect(stderr).toContain(BUN_WRITE_ENTRY.buildFile); + expect(stderr).toContain(BUN_VERIFY_ENTRY.buildFile); + // Both channel revs, named. + expect(stderr).toContain(revFor(BUN_WRITE_ENTRY.vehicleChannelLock)); + expect(stderr).toContain(revFor(BUN_VERIFY_ENTRY.vehicleChannelLock)); + }); + + // A verify entry must never leave its own recomputed value in the tree: the + // authoritative write is what ships. Even on the throwing path the pin on disk + // is the authoritative SRI, never the divergent one and never the fake. + test("the verify entry never writes its own value, even when it diverges", async () => { + await Bun.write(join(repo, "bun.lock"), "bumped\n"); + + const res = await runRefresh(repo, { + STUB_DIVERGE_VEHICLE: BUN_VERIFY_ENTRY.buildFile, + }); + expect(res.exitCode).not.toBe(0); + + const bunNix = await readFile(join(repo, BUN_WRITE_ENTRY.file), "utf8"); + expect(maybeHashOnMarker(bunNix, BUN_WRITE_ENTRY.marker)).toBe( + stubSriForFragment("node-modules"), + ); + expect(bunNix).not.toContain("sha256-AAAAAAAA"); + }); + + // Ordering is load-bearing: the authoritative write must land BEFORE the + // verify reads its baseline, or every ordinary refresh would be reported as a + // divergence (the verify would compare the new value against the stale pin). + // Assert it on the observable log order, so a refactor that reorders the run + // set fails here. + test("writes the authoritative value before the verify reads its baseline", async () => { + await Bun.write(join(repo, "bun.lock"), "bumped\n"); + + const res = await runRefresh(repo); + expect(res.exitCode).toBe(0); + + const stdout = res.stdout.toString(); + const wrote = stdout.indexOf( + `${BUN_WRITE_ENTRY.file} -> ${stubSriForFragment("node-modules")}`, + ); + const verified = stdout.indexOf("verifying"); + expect(wrote).toBeGreaterThanOrEqual(0); + expect(verified).toBeGreaterThanOrEqual(0); + expect(wrote).toBeLessThan(verified); + }); + + // Structural guard, not just ordering: refreshEntry itself refuses a verify + // entry, so no future caller can write through one by accident and clobber the + // authoritative value with the second builder's. + test("refreshEntry refuses to write a verify entry", async () => { + await expect(refreshEntry(BUN_VERIFY_ENTRY)).rejects.toThrow( + /must never write/, + ); }); }); @@ -399,7 +585,7 @@ describe("rewriteInlineHash", () => { "sha256-newgo=", GO_ENTRY.file, ); - expect(hashOnMarker(out, GO_ENTRY.marker)).toBe("sha256-newgo="); + expect(maybeHashOnMarker(out, GO_ENTRY.marker)).toBe("sha256-newgo="); }); test("is idempotent — rewriting to the same SRI yields identical content", () => { @@ -440,22 +626,210 @@ describe("rewriteInlineHash", () => { withDollars, GO_ENTRY.file, ); - expect(hashOnMarker(out, GO_ENTRY.marker)).toBe(withDollars); + expect(maybeHashOnMarker(out, GO_ENTRY.marker)).toBe(withDollars); + }); +}); + +describe("hashOnMarker", () => { + test("reads the committed SRI off the marker line", () => { + expect(hashOnMarker(BUN_NIX_FIXTURE, BUN_WRITE_ENTRY.marker, "f.nix")).toBe( + PLACEHOLDER_BUN, + ); + }); + + // The verify baseline must never degrade to "no mismatch observed" when the + // pin cannot be read — that would pass a branch whose pin was never checked. + test("throws when the marker is missing", () => { + expect(() => + hashOnMarker(BUN_NIX_FIXTURE, 'noSuch = "sha256-', "f.nix"), + ).toThrow(/marker .* not found/); + }); + + test("throws when the marker line carries no SRI", () => { + expect(() => + hashOnMarker(' outputHash = "";\n', "outputHash", "f.nix"), + ).toThrow(/no sha256- SRI/); }); }); -describe("FOD_ENTRIES table invariant", () => { - test("no drvFragment is a substring of another (got: attribution safety)", () => { - // parseGotForFragment attributes a mismatch block by includes(fragment); - // if one fragment contained another, one entry's got: could bind the - // other's block and write the wrong hash. The module also asserts this at - // load time — this test states the property explicitly and pins it against - // a future table edit. - for (const a of FOD_ENTRIES) { - for (const b of FOD_ENTRIES) { - if (a === b) continue; - expect(b.drvFragment.includes(a.drvFragment)).toBe(false); - } +describe("FOD_ENTRIES table invariants", () => { + // A base row to mutate per case, so each violating table differs from a legal + // one in exactly the property under test. + const write: FodEntry = { + id: "w", + file: "a.nix", + marker: 'outputHash = "sha256-', + drvFragment: "node-modules", + buildFile: "veh-a.nix", + buildTarget: "t", + vehicleChannelLock: "a.lock", + triggers: ["bun.lock"], + }; + const verify: FodEntry = { + ...write, + id: "v", + buildFile: "veh-b.nix", + vehicleChannelLock: "b.lock", + verifyOf: "w", + }; + + test("the shipped table satisfies every invariant", () => { + expect(() => assertFodTableInvariants(FOD_ENTRIES)).not.toThrow(); + }); + + // `vehicleChannelLock` is a CLAIM about `buildFile`: it names the lock whose + // nixpkgs node supplies that vehicle's `pkgs`. Nothing in the type system ties + // the two, so a vehicle pointed at the sibling scope's lock would satisfy + // every structural invariant (different file name, different target) and then + // re-derive the identical hash — a verify that always agrees and checks + // nothing. Read the vehicle and assert it really reads the lock it declares. + test("every entry's vehicle reads the channel lock the table declares", async () => { + for (const entry of FOD_ENTRIES) { + const vehicle = await readFile(join(repoRoot, entry.buildFile), "utf8"); + expect( + vehicle.includes(entry.vehicleChannelLock), + `${entry.buildFile} must read ${entry.vehicleChannelLock} ` + + `(declared by entry '${entry.id}')`, + ).toBe(true); + } + }); + + // buildFile became a per-entry field, so its existence is a property of the + // TABLE, not of the one entry a scope-specific suite happens to look at. A + // renamed or moved vehicle must fail here rather than at Renovate-branch time. + test("every entry's vehicle file exists", async () => { + for (const entry of FOD_ENTRIES) { + expect( + await Bun.file(join(repoRoot, entry.buildFile)).exists(), + `${entry.buildFile} (entry '${entry.id}') must exist`, + ).toBe(true); } }); + + // No gate in the repo EVALUATES the renovate vehicle: the only nix eval in CI + // is over tools/toolchain/gate-tools.nix, and guest-image/default.nix is + // covered by the guest-image project's own build. So a syntax break in a + // vehicle would pass every check on the PR that ships it and first surface on + // a live Renovate branch, where the realise fails with an eval error, the + // refresh exits non-zero, and the bumped lock still lands beside an + // unrefreshed pin — the silent drift this table exists to prevent, through a + // different door. `--parse` is offline and sub-second, so it runs here. + // + // The precondition uses `which`, NOT `command -v`: the latter is a shell + // BUILTIN that Bun's own shell does not implement, so it returns non-zero + // even where nix is installed — which would skip this test everywhere and + // leave it permanently vacuous. + test("every entry's vehicle parses as nix", async () => { + const nixOnPath = await $`which nix-instantiate`.nothrow().quiet(); + if (nixOnPath.exitCode !== 0) return; + for (const entry of FOD_ENTRIES) { + const res = + await $`nix-instantiate --parse ${join(repoRoot, entry.buildFile)}` + .nothrow() + .quiet(); + expect( + res.exitCode, + `${entry.buildFile} (entry '${entry.id}') must parse: ` + + res.stderr.toString(), + ).toBe(0); + } + }); + + // The shipped pairing is the point of the whole mechanism: one writer, one + // verifier, two vehicles over the SAME pin. + test("the shipped table pairs the entrypoint outputHash across two vehicles", () => { + expect(BUN_VERIFY_ENTRY.verifyOf).toBe(BUN_WRITE_ENTRY.id); + expect(BUN_VERIFY_ENTRY.file).toBe(BUN_WRITE_ENTRY.file); + expect(BUN_VERIFY_ENTRY.marker).toBe(BUN_WRITE_ENTRY.marker); + expect(BUN_VERIFY_ENTRY.buildFile).not.toBe(BUN_WRITE_ENTRY.buildFile); + expect(BUN_VERIFY_ENTRY.vehicleChannelLock).not.toBe( + BUN_WRITE_ENTRY.vehicleChannelLock, + ); + }); + + test("rejects duplicate ids", () => { + expect(() => assertFodTableInvariants([write, { ...write }])).toThrow( + /duplicate FodEntry id/, + ); + }); + + // got: attribution safety, scoped to a vehicle: within ONE vehicle's build + // output, `drvName.includes(fragment)` would let the shorter fragment bind the + // longer entry's block and write the wrong hash. + test("rejects a drvFragment that is a substring of another IN THE SAME vehicle", () => { + expect(() => + assertFodTableInvariants([ + { ...write, id: "a", drvFragment: "modules" }, + { + ...write, + id: "b", + file: "b.nix", + drvFragment: "node-modules", + }, + ]), + ).toThrow(/ambiguous drvFragment/); + }); + + // …but the SAME fragment across two vehicles is legal and load-bearing: it is + // literally the same derivation seen through two nixpkgs, and each realise + // reads only its own vehicle's output. + test("allows the same drvFragment across two different vehicles", () => { + expect(() => assertFodTableInvariants([write, verify])).not.toThrow(); + }); + + // Two writers over one pin are last-write-wins: the second realise's value + // silently overwrites the first, hiding the divergence the pairing exists to + // surface. + test("rejects two authoritative entries writing the same file+marker", () => { + expect(() => + assertFodTableInvariants([ + write, + { ...verify, verifyOf: undefined, id: "w2" }, + ]), + ).toThrow(/both WRITE/); + }); + + test("rejects a verifyOf naming a nonexistent entry", () => { + expect(() => + assertFodTableInvariants([{ ...verify, verifyOf: "nope" }]), + ).toThrow(/not an authoritative/); + }); + + test("rejects a verify entry checking a different pin", () => { + expect(() => + assertFodTableInvariants([write, { ...verify, file: "other.nix" }]), + ).toThrow(/SAME pin/); + }); + + // A verify entry on the same vehicle re-derives the identical value: it would + // always agree and check nothing, which is worse than absent because it reads + // as coverage. + test("rejects a verify entry realising the same vehicle as its target", () => { + expect(() => + assertFodTableInvariants([ + write, + { ...verify, buildFile: write.buildFile }, + ]), + ).toThrow(/SAME vehicle/); + }); + + // Divergent triggers would let the gate open the verify without its writer, so + // the comparison baseline would be the pre-write (stale) pin. + test("rejects a verify entry gated differently from its target", () => { + expect(() => + assertFodTableInvariants([ + write, + { ...verify, triggers: ["bun.lock", "extra"] }, + ]), + ).toThrow(/SAME triggers/); + }); + + test("rejects mirrorFiles on a verify entry", () => { + expect(() => + assertFodTableInvariants([ + write, + { ...verify, mirrorFiles: ["flake.nix"] }, + ]), + ).toThrow(/writes nothing/); + }); }); diff --git a/tools/renovate/refresh-fod-hashes.ts b/tools/renovate/refresh-fod-hashes.ts index 4f24bcb6..e3063970 100755 --- a/tools/renovate/refresh-fod-hashes.ts +++ b/tools/renovate/refresh-fod-hashes.ts @@ -30,20 +30,48 @@ // Neither is a URL hash a `nix store prefetch-file` can recompute (that is // refresh-toolchain-hashes.ts's job for the vendored-binary pins). A vendorHash / // outputHash is only knowable by REALISING the derivation: build it, read the SRI -// Nix reports on the mismatch. Both FODs live in the guest-image rootfs closure -// (guest-image/default.nix imports agent-image/entrypoint.nix with root's pkgs), -// so ONE build vehicle realises both — and with a deliberately-wrong pin the -// build fails FAST at the FOD, never proceeding to the heavy guestd compile or -// erofs pack. +// Nix reports on the mismatch. Each entry names its OWN build vehicle, and with a +// deliberately-wrong pin the build fails FAST at the FOD, never proceeding to the +// heavy guestd compile, the erofs pack, or the agent bundle. +// +// ── ONE outputHash, TWO builders: why some entries WRITE and some VERIFY ── +// `agent-image/entrypoint.nix` carries a SINGLE `outputHash` literal and is +// imported by TWO consumers with two different nixpkgs pins: +// +// guest-image/default.nix:66 with ROOT's `pkgs` (root devenv.lock) +// agent-image/devenv.nix:34 with AGENT-IMAGE's `pkgs` (agent-image/devenv.lock) +// +// The FOD's builder takes `nativeBuildInputs = [ pkgs.bun ]`, so the two consumers +// realise it with two bun derivations. One hash satisfies both only while those +// two buns produce a byte-identical install tree. The table therefore carries the +// entrypoint pin TWICE: +// +// * an AUTHORITATIVE entry, realised through guest-image/default.nix (root's +// pkgs), which WRITES the canonical SRI; and +// * a VERIFY entry (`verifyOf`), realised through the agent-image vehicle +// (agent-image's pkgs), which recomputes and COMPARES against what the +// authoritative entry just wrote. Equal → log and no-op. Different → throw, +// naming both SRIs, both vehicles and both channel revs. +// +// A verify entry must never write: two entries writing one marker would be +// last-write-wins, and the second value would silently overwrite the first — +// hiding the very divergence this pair exists to catch. The write-then-verify +// ORDER is therefore load-bearing, and refreshFodEntries() enforces it +// structurally (it partitions the run set; it never trusts table order), backed +// by the table invariants below (a verify entry must share its authoritative +// entry's file+marker+triggers and use a DIFFERENT vehicle). // // Mechanism, per gated entry: // 1. Rewrite the entry's hash to a fixed FAKE value. -// 2. `nix build` the rootfs vehicle (--keep-going so a co-stale sibling FOD +// 2. `nix build` the entry's vehicle (--keep-going so a co-stale sibling FOD // does not mask this one) — it fails with the entry's real `got:` SRI. // 3. Parse the `got:` for THIS entry's derivation (matched by a drv-name -// fragment, so a sibling FOD's mismatch can never be misattributed). -// 4. Write the real SRI back. Fail LOUD (exit 1) if no `got:` is found — a -// silent no-op would ship the stale pin this task exists to fix. +// fragment, so a sibling FOD's mismatch in the same vehicle can never be +// misattributed). +// 4. Write the real SRI back (authoritative), or compare it against the +// committed value and throw on divergence (verify). Fail LOUD (exit 1) if no +// `got:` is found — a silent no-op would ship the stale pin this task exists +// to fix. // // Self-gating: for each entry, act only when one of its trigger manifests differs // from the base branch (mirrors refresh-toolchain-hashes.ts's versions/*.nix @@ -74,34 +102,51 @@ // Design: docs/designs/repo/compass-renovate-migration.md // // Exit codes: -// 0 - hash(es) refreshed, or a no-op branch (no trigger manifest changed). -// 1 - a step failed (build produced no `got:`, or a marker was missing) — -// fail loud, never ship a half-refreshed pin set. +// 0 - hash(es) refreshed and every verify entry agreed, or a no-op branch (no +// trigger manifest changed). +// 1 - a step failed (build produced no `got:`, a marker was missing, or a +// verify entry's vehicle disagreed with the committed pin) — fail loud, +// never ship a half-refreshed or one-builder-only pin set. import { $ } from "bun"; -// The build vehicle that realises both FODs, and the Nix file it is expressed in. -// Repo-root-relative (the runner cwd = repo root; main() chdirs there). -export const BUILD_FILE = "guest-image/default.nix"; -export const BUILD_TARGET = "compass-guest-rootfs"; - // A syntactically-valid but deliberately-wrong SRI. Building any FOD against it // forces the `hash mismatch … got: ` error we parse. All-A base64 is a // canonical fake (matches the lib.fakeHash convention the nix files document). const FAKE_SRI = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; -// The two pinned FODs. Exported so the test derives its fixtures from the shipped +// The pinned FODs. Exported so the test derives its fixtures from the shipped // table rather than restating it — a rebase that edits the table keeps the test // honest with no second edit. export type FodEntry = { + // Stable identity of this table row, used by `verifyOf` and by every log line + // and error message. Unique across the table. + id: string; // Nix file carrying the pinned hash (repo-root-relative). file: string; // Unique substring on the hash line — the attr name + `= "sha256-` prefix, so // it matches exactly one line and cannot bind a comment or a sibling attr. marker: string; // A fragment of this FOD's derivation name, used to attribute the correct - // `got:` when --keep-going reports multiple mismatches. Distinct per entry. + // `got:` when --keep-going reports multiple mismatches. Distinct per entry + // WITHIN one vehicle; two entries realising different vehicles may share it + // (that is exactly the two-builders-one-hash case above). drvFragment: string; + // The build vehicle that realises this entry's FOD: the Nix file and the attr + // in it, both repo-root-relative (the runner cwd = repo root; main() chdirs + // there). Per ENTRY, not per module: the same pinned hash is realised through + // two vehicles resolving two different nixpkgs revs, and a shared global would + // make one of those unreachable. + buildFile: string; + buildTarget: string; + // The devenv lock whose `nodes.nixpkgs.locked.rev` supplies `buildFile`'s + // `pkgs` (repo-root-relative). Load-bearing, not decorative: a scope-specific + // refresher selects its entries by this field, the table invariants below + // check it names the lock `buildFile` actually reads, and the divergence error + // quotes both revs so the reader does not have to go find them. Rewriting it + // to an equivalent-looking spelling (a leading `./`, an absolute path) breaks + // those lookups. + vehicleChannelLock: string; // Manifests whose change invalidates this FOD (repo-root-relative). The // per-entry self-gate fires when any of these differs from the base branch. triggers: string[]; @@ -110,60 +155,209 @@ export type FodEntry = { // proxyVendor set over the same go/. They are NOT separately realised (the // build vehicle content-addresses only `file`'s FOD, so a faked mirror pin // would never surface in its output); each is rewritten to the SRI `file`'s - // realise reports. Absent for a lone pin. + // realise reports. Absent for a lone pin. Never set on a verify entry — a + // verify entry writes nothing at all. mirrorFiles?: string[]; + // Set => this entry VERIFIES the pin another entry writes, and never writes + // itself. The value is that authoritative entry's `id`. See the header: it + // exists to check a SECOND builder's view of one shared hash. + verifyOf?: string; }; export const FOD_ENTRIES: FodEntry[] = [ { + id: "guestd-go-vendor", file: "guest-image/default.nix", marker: 'vendorHash = "sha256-', drvFragment: "go-modules", + buildFile: "guest-image/default.nix", + buildTarget: "compass-guest-rootfs", + vehicleChannelLock: "devenv.lock", triggers: ["go/go.mod", "go/go.sum"], mirrorFiles: ["flake.nix"], }, { + id: "agent-node-modules-root-pkgs", + file: "agent-image/entrypoint.nix", + marker: 'outputHash = "sha256-', + drvFragment: "node-modules", + // The AUTHORITATIVE realise of the shared outputHash, through + // guest-image/default.nix — which imports the very same entrypoint.nix with + // ROOT's `pkgs` (that file documents the divergence as deliberate). This is + // the entry that WRITES the canonical value; the sibling below re-derives it + // through the agent-image scope and only compares. + buildFile: "guest-image/default.nix", + buildTarget: "compass-guest-rootfs", + vehicleChannelLock: "devenv.lock", + // `bun.lock` moves the installed tree's version set. The two channel locks + // are declared beside it because each moves a BUILDER of this same FOD: + // `devenv.lock` supplies root's `pkgs` to the vehicle realised here, and + // `agent-image/devenv.lock` supplies the agent-image scope's `pkgs` to + // `agent-image/devenv.nix`, the OTHER importer of entrypoint.nix. All paths + // are repo-root-relative, as the gate's `git diff` pathspec expects. The + // verify entry below carries the IDENTICAL trigger list, which is what makes + // the two always gate together. + triggers: ["bun.lock", "devenv.lock", "agent-image/devenv.lock"], + }, + { + id: "agent-node-modules-agent-image-pkgs", file: "agent-image/entrypoint.nix", marker: 'outputHash = "sha256-', + // The SAME drv name as the entry above — it is literally the same + // derivation expression, evaluated against a different nixpkgs. Sharing the + // fragment is safe because attribution is scoped to a vehicle's own build + // output, and the two entries realise different vehicles (the table + // invariants below enforce exactly that). drvFragment: "node-modules", - // `bun.lock` moves the installed tree's version set, and it is the trigger - // that actually moves this pin today. `devenv.lock` is the ROOT scope's - // channel pin: `guest-image/default.nix` imports entrypoint.nix with root's - // `pkgs`, so that lock supplies this FOD's `nativeBuildInputs = [ pkgs.bun ]` - // on the vehicle realised below. `agent-image/devenv.lock` is declared - // beside them because it moves the BUILDER in the OTHER consumer: - // `agent-image/devenv.nix` imports the SAME entrypoint.nix with the - // agent-image scope's own `pkgs`. - // - // Note the asymmetry this table cannot express. The vehicle realised below - // is `guest-image/default.nix`, so a relock of the agent-image channel alone - // recomputes a value derived through ROOT's bun, which will equal the - // committed one: the refresh is correct but cannot observe that consumer. - // The trigger is declared here so the coupling sits where a reader looks for - // it; the agent-image builder is verified by the image build, not by this - // refresh. All three paths are repo-root-relative, as the gate's `git diff` - // pathspec expects. + // The agent-image scope's own view of the shared hash. This vehicle exists + // solely for this check: it imports entrypoint.nix with the pkgs resolved + // from agent-image/devenv.lock, which is the bun the OCI image build + // actually uses. Without it, a channel-rev divergence that changes bun's + // install tree would leave this pin right for guest-image, silently wrong + // for the agent image, and unobservable until the image build. + buildFile: "tools/renovate/agent-image-fod-vehicle.nix", + buildTarget: "compass-agent", + vehicleChannelLock: "agent-image/devenv.lock", triggers: ["bun.lock", "devenv.lock", "agent-image/devenv.lock"], + verifyOf: "agent-node-modules-root-pkgs", }, ]; -// Fragment attribution (parseGotForFragment) matches a mismatch block by -// `drvName.includes(fragment)`, so two fragments where one contains the other -// (e.g. "modules" vs "node-modules") would let one entry's got: bind the other's -// block and write the WRONG hash. The current two fragments are mutually -// disjoint; assert it at load time so a future table edit that reintroduces an -// ambiguous fragment fails loud here rather than silently misattributing at run. -for (const a of FOD_ENTRIES) { - for (const b of FOD_ENTRIES) { - if (a !== b && b.drvFragment.includes(a.drvFragment)) { +// ── Table invariants, asserted at load so a bad edit fails HERE, not at run. ── +// Each one closes a way the write/verify pairing could silently do the wrong +// thing. One function per rule, so a failure's stack names the rule broken and +// each stays readable whole; assertFodTableInvariants below composes them in the +// order their diagnostics are most useful. Exported (the composer) so the test +// can prove each guard is real by feeding it a table that violates exactly one +// rule, rather than restating the properties in assertions that would drift. + +// Ids identify a row to `verifyOf` and to every diagnostic. +function assertUniqueIds(entries: FodEntry[]): void { + const seenIds = new Set(); + for (const entry of entries) { + if (seenIds.has(entry.id)) { + throw new Error( + `renovate-fod: duplicate FodEntry id '${entry.id}' — ids identify a row to ` + + "`verifyOf` and to every diagnostic, so they must be unique", + ); + } + seenIds.add(entry.id); + } +} + +// One writer per pin. Two authoritative entries over the same file+marker would +// be last-write-wins: the second realise's value would overwrite the first with +// no diagnostic, which is precisely the divergence-hiding failure the verify +// entry exists to prevent. +function assertOneWriterPerPin(entries: FodEntry[]): void { + const writers = new Map(); + for (const entry of entries) { + if (entry.verifyOf) continue; + const key = `${entry.file}\u0000${entry.marker}`; + const prior = writers.get(key); + if (prior) { throw new Error( - `renovate-fod: ambiguous drvFragment '${a.drvFragment}' is a substring of ` + - `'${b.drvFragment}' — fragments must be mutually disjoint for got: attribution`, + `renovate-fod: '${entry.id}' and '${prior.id}' both WRITE ${entry.marker} in ` + + `${entry.file} — a second writer silently clobbers the first; the later one ` + + "must carry `verifyOf` instead", ); } + writers.set(key, entry); + } +} + +// A verify entry must actually verify the pin it claims to, through a DIFFERENT +// builder, and must be gated identically so the two can never fire apart (a +// verify run whose authoritative sibling did not run would compare against a +// pre-write baseline). +function assertVerifyPairing(entries: FodEntry[]): void { + for (const entry of entries) { + if (!entry.verifyOf) continue; + const target = entries.find((e) => e.id === entry.verifyOf); + if (!target || target.verifyOf) { + throw new Error( + `renovate-fod: verify entry '${entry.id}' names verifyOf='${entry.verifyOf}', ` + + "which is not an authoritative (writing) entry in this table", + ); + } + assertVerifiesTarget(entry, target); + } +} + +// The four properties one verify/authoritative PAIR must hold, split out so each +// rule reads on its own and the enclosing scan stays a scan. +function assertVerifiesTarget(entry: FodEntry, target: FodEntry): void { + if (entry.file !== target.file || entry.marker !== target.marker) { + throw new Error( + `renovate-fod: verify entry '${entry.id}' must check the SAME pin as '${target.id}' ` + + `— got ${entry.file}:${entry.marker} vs ${target.file}:${target.marker}`, + ); + } + if ( + entry.buildFile === target.buildFile && + entry.buildTarget === target.buildTarget + ) { + throw new Error( + `renovate-fod: verify entry '${entry.id}' realises the SAME vehicle as '${target.id}' ` + + `(${entry.buildFile}#${entry.buildTarget}) — it would re-derive the identical value ` + + "and check nothing; a verify entry exists to exercise a SECOND builder", + ); + } + const sameTriggers = + entry.triggers.length === target.triggers.length && + new Set(entry.triggers).size === new Set(target.triggers).size && + entry.triggers.every((t) => target.triggers.includes(t)); + if (!sameTriggers) { + throw new Error( + `renovate-fod: verify entry '${entry.id}' and '${target.id}' must declare the SAME ` + + "triggers, or the gate could open one without the other and the verify would " + + `compare against an unrefreshed pin — got [${entry.triggers}] vs [${target.triggers}]`, + ); + } + if (entry.mirrorFiles?.length) { + throw new Error( + `renovate-fod: verify entry '${entry.id}' declares mirrorFiles — a verify entry ` + + "writes nothing, so mirrors would never be propagated from it", + ); + } +} + +// got: attribution. parseGotForFragment matches a mismatch block by +// `drvName.includes(fragment)`, so within ONE vehicle's output two fragments +// where one contains the other (e.g. "modules" vs "node-modules") would let one +// entry's got: bind the other's block and write the WRONG hash. Scoped to the +// vehicle because that is the only place the ambiguity can arise: entries +// realising different vehicles never read each other's output. +function assertDisjointFragmentsPerVehicle(entries: FodEntry[]): void { + for (const a of entries) { + for (const b of entries) { + if (a === b) continue; + if (a.buildFile !== b.buildFile || a.buildTarget !== b.buildTarget) { + continue; + } + if (b.drvFragment.includes(a.drvFragment)) { + throw new Error( + `renovate-fod: ambiguous drvFragment '${a.drvFragment}' (${a.id}) is a substring ` + + `of '${b.drvFragment}' (${b.id}) and both realise ${a.buildFile}#${a.buildTarget} — ` + + "fragments must be mutually disjoint within a vehicle for got: attribution", + ); + } + } } } +export function assertFodTableInvariants(entries: FodEntry[]): void { + assertUniqueIds(entries); + assertOneWriterPerPin(entries); + assertVerifyPairing(entries); + // LAST, so the pairing rules above give their specific diagnosis first: a + // verify entry wrongly sharing its target's vehicle also trips the fragment + // rule, and "same vehicle" is the message that names the actual mistake. + assertDisjointFragmentsPerVehicle(entries); +} + +assertFodTableInvariants(FOD_ENTRIES); + // ── Pure rewrite: replace the `sha256-…` SRI on the line carrying `marker`. ── // Unlike refresh-toolchain-hashes.ts (whose hash sits on the line AFTER the URL // marker), a vendorHash/outputHash IS the marker line. Throws on an empty SRI or @@ -193,14 +387,37 @@ export function rewriteInlineHash( return lines.join("\n"); } +// ── Pure read: the `sha256-…` SRI currently on the line carrying `marker`. ── +// The inverse of rewriteInlineHash, and the baseline a verify entry compares its +// recomputed value against. Throws rather than returning undefined: a verify that +// cannot find the pin must fail loud, never "no mismatch observed". +export function hashOnMarker( + fileText: string, + marker: string, + file: string, +): string { + const line = fileText.split("\n").find((l) => l.includes(marker)); + if (line === undefined) { + throw new Error(`renovate-fod: marker '${marker}' not found in ${file}`); + } + const sri = line.match(/sha256-[^"]*/)?.[0]; + if (!sri) { + throw new Error( + `renovate-fod: no sha256- SRI on the '${marker}' line in ${file}`, + ); + } + return sri; +} + // ── Parse the `got:` SRI for the derivation whose name contains `fragment`. ── // Nix prints, per mismatching FOD: // error: hash mismatch in fixed-output derivation '/nix/store/…-.drv': // specified: sha256- // got: sha256- -// Scoping to the fragment means a co-stale sibling FOD's mismatch (only possible -// defensively — the two triggers never change on one branch) is never -// misattributed. Returns undefined if this FOD did not report a mismatch. +// Scoping to the fragment means a co-stale sibling FOD's mismatch in the same +// vehicle (only possible defensively — the two triggers never change on one +// branch) is never misattributed. Returns undefined if this FOD did not report a +// mismatch. export function parseGotForFragment( nixOutput: string, fragment: string, @@ -227,10 +444,10 @@ export function parseGotForFragment( return undefined; } -// Recompute one entry's SRI: fake the pin, build the vehicle, read the reported -// `got:` for this FOD, then restore the file to the ORIGINAL text (the caller -// writes the real SRI). Restores on every path so a failure never leaves a fake -// pin in the tree. +// Recompute one entry's SRI: fake the pin, build THIS entry's vehicle, read the +// reported `got:` for this FOD, then restore the file to the ORIGINAL text (the +// caller decides whether to write the value or merely compare it). Restores on +// every path so a failure never leaves a fake pin in the tree. async function recompute(entry: FodEntry, origText: string): Promise { await Bun.write( entry.file, @@ -238,7 +455,7 @@ async function recompute(entry: FodEntry, origText: string): Promise { ); try { const res = - await $`nix build -f ${BUILD_FILE} ${BUILD_TARGET} --no-link --keep-going` + await $`nix build -f ${entry.buildFile} ${entry.buildTarget} --no-link --keep-going` .nothrow() .quiet(); const out = `${res.stdout.toString()}\n${res.stderr.toString()}`; @@ -246,8 +463,8 @@ async function recompute(entry: FodEntry, origText: string): Promise { if (!got) { throw new Error( `renovate-fod: no 'got:' SRI for '${entry.drvFragment}' after building ` + - `${BUILD_TARGET} with a faked ${entry.marker} — build output:\n` + - out.split("\n").slice(-30).join("\n"), + `${entry.buildTarget} (${entry.buildFile}) with a faked ${entry.marker} — ` + + `build output:\n${out.split("\n").slice(-30).join("\n")}`, ); } return got; @@ -256,19 +473,46 @@ async function recompute(entry: FodEntry, origText: string): Promise { } } -// Refresh ONE entry's pin (and its mirrors) in place: read the current text, -// recompute the SRI by realising the vehicle against a faked pin, write the real -// value back, then propagate it to every declared mirror. This is the per-entry -// body of main()'s gated loop, factored out so a scope-specific refresher can -// drive a single FOD directly — refresh-agent-image-nixpkgs.ts calls it after its -// own relock, having already established (by its own base diff) that the -// agent-image entry's trigger moved. Extracting it keeps ONE realise-and-parse +const isObj = (v: unknown): v is Record => + typeof v === "object" && v !== null; + +// The nixpkgs channel rev a vehicle's `pkgs` resolves from, for DIAGNOSTICS only. +// Never throws: it is called while building an error message, and losing the +// real failure to a secondary parse error would be strictly worse than printing +// a placeholder. +async function vehicleChannelRev(lockFile: string): Promise { + try { + let cur: unknown = JSON.parse(await Bun.file(lockFile).text()); + for (const key of ["nodes", "nixpkgs", "locked", "rev"]) { + if (!isObj(cur)) return ``; + cur = cur[key]; + } + return typeof cur === "string" ? cur : ``; + } catch (error) { + return ``; + } +} + +// Refresh ONE authoritative entry's pin (and its mirrors) in place: read the +// current text, recompute the SRI by realising the entry's vehicle against a +// faked pin, write the real value back, then propagate it to every declared +// mirror. This is the per-entry body of main()'s gated loop, factored out so a +// scope-specific refresher can drive a single FOD directly — +// refresh-agent-image-nixpkgs.ts drives it (via refreshFodEntries) after its own +// relock, having already established by its own base diff that the agent-image +// entry's trigger moved. Extracting it keeps ONE realise-and-parse // implementation: a second copy would drift from the got:-attribution and // restore-on-failure discipline above. export async function refreshEntry(entry: FodEntry): Promise { + if (entry.verifyOf) { + throw new Error( + `renovate-fod: '${entry.id}' is a verify entry and must never write ${entry.file} — ` + + `route it through verifyEntry (its write would clobber '${entry.verifyOf}'s value)`, + ); + } const origText = await Bun.file(entry.file).text(); console.log( - `renovate-fod: ${entry.file} (${entry.drvFragment}) trigger changed; recomputing ${entry.marker.replace(/ = .*/, "")} ...`, + `renovate-fod: ${entry.file} (${entry.drvFragment}) trigger changed; recomputing ${entry.marker.replace(/ = .*/, "")} via ${entry.buildTarget} (${entry.buildFile}) ...`, ); const got = await recompute(entry, origText); await Bun.write( @@ -286,6 +530,93 @@ export async function refreshEntry(entry: FodEntry): Promise { } } +// Check ONE verify entry: recompute the shared pin through the SECOND builder and +// compare it against what is on disk — which, by refreshFodEntries' ordering, is +// the value `authoritative` has already written. Writes nothing on either path +// (recompute restores the file it faked), so the authoritative value survives +// intact whatever this finds. +// +// Equal is the expected outcome and the only quiet one. A difference means the +// two vehicles' bun derivations produce different install trees, so NO single +// `outputHash` literal can satisfy both consumers: throwing reds +// `renovate/artifacts` on the branch that moved the pin, which is the whole point +// of realising this second vehicle. +export async function verifyEntry( + entry: FodEntry, + authoritative: FodEntry, +): Promise { + if (entry.verifyOf !== authoritative.id) { + throw new Error( + `renovate-fod: '${entry.id}' verifies '${entry.verifyOf}', not '${authoritative.id}'`, + ); + } + // Read AFTER the authoritative write, so the baseline is the refreshed pin — + // comparing against the pre-write text would report every ordinary refresh as + // a divergence. ONE read serves both roles: it is the value being checked AND + // the text recompute restores after faking the pin, so the file cannot end in + // a state neither of them intended. + const currentText = await Bun.file(entry.file).text(); + const committed = hashOnMarker(currentText, entry.marker, entry.file); + console.log( + `renovate-fod: verifying ${entry.file} ${entry.marker.replace(/ = .*/, "")} against ${entry.buildTarget} (${entry.buildFile}) ...`, + ); + const got = await recompute(entry, currentText); + if (got === committed) { + console.log( + `renovate-fod: ${entry.file} verified — ${entry.buildFile} and ${authoritative.buildFile} ` + + `agree on ${committed}`, + ); + return; + } + const [verifyRev, authRev] = await Promise.all([ + vehicleChannelRev(entry.vehicleChannelLock), + vehicleChannelRev(authoritative.vehicleChannelLock), + ]); + throw new Error( + `renovate-fod: ${entry.file}'s ${entry.marker.replace(/ = .*/, "")} cannot satisfy both of ` + + "its builders — one outputHash literal, two nixpkgs revs whose bun produces a " + + "different installed tree.\n" + + ` ${authoritative.id} (authoritative, wrote it): ${committed}\n` + + ` vehicle: ${authoritative.buildTarget} (${authoritative.buildFile})\n` + + ` channel: ${authoritative.vehicleChannelLock} @ ${authRev}\n` + + ` ${entry.id} (verify): ${got}\n` + + ` vehicle: ${entry.buildTarget} (${entry.buildFile})\n` + + ` channel: ${entry.vehicleChannelLock} @ ${verifyRev}\n` + + "The committed value is the authoritative one, so the agent-image OCI build would " + + "fail `hash mismatch in fixed-output derivation`. Reconcile the two channel revs " + + "onto one bun, or split the pin so each consumer carries its own.", + ); +} + +// Drive a set of gated entries in the ONE order that is correct: every +// authoritative write first, then every verify. The ordering is structural — the +// run set is partitioned here, never read in table order — because a verify that +// ran first would compare the second builder's value against the STALE pin and +// throw on an ordinary refresh, and a verify that was skipped would ship the +// unverified pin this pairing exists to catch. +export async function refreshFodEntries(entries: FodEntry[]): Promise { + const authoritative = entries.filter((e) => !e.verifyOf); + const verifiers = entries.filter((e) => e.verifyOf); + + for (const entry of authoritative) { + await refreshEntry(entry); + } + for (const entry of verifiers) { + // Resolve against the RUN SET, not the whole table: if the caller gated the + // verify entry without its writer, nothing refreshed the pin above and the + // comparison would be against a pre-write baseline. The table invariants + // make the two gate together, so reaching this is a caller bug — fail loud. + const target = authoritative.find((e) => e.id === entry.verifyOf); + if (!target) { + throw new Error( + `renovate-fod: verify entry '${entry.id}' was gated without its authoritative ` + + `entry '${entry.verifyOf}', so there is no refreshed value to verify against`, + ); + } + await verifyEntry(entry, target); + } +} + async function main(): Promise { // Resolve the repo root from git, not a hardcoded depth: Renovate invokes this // as a postUpgradeTask and the paths above are repo-root-relative, so a wrong @@ -304,7 +635,7 @@ async function main(): Promise { } // Per-entry gate: act only on FODs whose trigger manifests this branch changed - // vs base. A gomod bump opens the Go entry alone; a bun bump the bun entry + // vs base. A gomod bump opens the Go entry alone; a bun bump the bun entries // alone; a branch touching neither manifest no-ops with no build. const gated: FodEntry[] = []; for (const entry of FOD_ENTRIES) { @@ -328,9 +659,7 @@ async function main(): Promise { return; } - for (const entry of gated) { - await refreshEntry(entry); - } + await refreshFodEntries(gated); console.log("renovate-fod: FOD hashes refreshed."); }