From eef3c84f086dd7c3956e0a9cea4f4b7e003c6db1 Mon Sep 17 00:00:00 2001 From: aamir Date: Fri, 6 Nov 2026 10:08:30 -0600 Subject: [PATCH 1/2] Pin benchmark reads to an expected publication --- src/benchmarks/service.ts | 4 ++++ src/commands/benchmarks.ts | 3 ++- tests/benchmarks-command.test.mjs | 6 ++++++ tests/benchmarks-service.test.mjs | 15 +++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/benchmarks/service.ts b/src/benchmarks/service.ts index ca0e07c..92f6170 100644 --- a/src/benchmarks/service.ts +++ b/src/benchmarks/service.ts @@ -79,6 +79,7 @@ export interface BenchmarkShowInput { project?: string; workload?: string; includeReport?: boolean; + expectPublication?: string; } export interface BenchmarkShowResult { @@ -136,6 +137,9 @@ export function createBenchmarksService(options: ManagementScopeOptions): Benchm (result.publication && (result.publication.org_id !== session.organizationId || result.publication.project_id !== project!.id || result.publication.workload_id !== workload!.id))) { throw new CliError("Understudy returned a benchmark outside the selected workload."); } + if (input.expectPublication && result.publication?.id !== input.expectPublication) { + throw new CliError("The published benchmark has changed. Refresh the dashboard and copy the agent handoff again."); + } const report = result.benchmark?.report; return { project: { id: project!.id, slug: project!.slug, name: project!.name }, diff --git a/src/commands/benchmarks.ts b/src/commands/benchmarks.ts index 0164a01..246d5b3 100644 --- a/src/commands/benchmarks.ts +++ b/src/commands/benchmarks.ts @@ -30,7 +30,8 @@ export function addBenchmarksCommands( .option("--project ", "Exact project id, slug, or name; defaults to saved context.") .option("--workload ", "Exact workload id or name; defaults to saved context.") .option("--include-report", "Include the full benchmark manifest, which may contain customer prompt and eval content.") - .action(async function (this: Command, options: { project?: string; workload?: string; includeReport?: boolean }) { + .option("--expect-publication ", "Require this exact publication; fail if the current benchmark changed.") + .action(async function (this: Command, options: { project?: string; workload?: string; includeReport?: boolean; expectPublication?: string }) { const result = await service.show(options); output(formatBenchmark(this, result, options.includeReport === true)); }); diff --git a/tests/benchmarks-command.test.mjs b/tests/benchmarks-command.test.mjs index 5efb358..3d9856e 100644 --- a/tests/benchmarks-command.test.mjs +++ b/tests/benchmarks-command.test.mjs @@ -50,6 +50,12 @@ test("benchmarks show gives an agent the full JSON report only when requested", assert.equal(JSON.parse(output[0]).benchmark.report.prompt_change.diff[0], "Invented prompt line"); }); +test("benchmarks show forwards the publication pin", async () => { + const { program, calls } = setup(); + await program.parseAsync(["node", "understudy", "benchmarks", "show", "--expect-publication", "synthetic-publication"]); + assert.deepEqual(calls, [{ expectPublication: "synthetic-publication" }]); +}); + test("benchmarks list prints available publications", async () => { const { program, output } = setup(); await program.parseAsync(["node", "understudy", "benchmarks", "list"]); diff --git a/tests/benchmarks-service.test.mjs b/tests/benchmarks-service.test.mjs index 3ed8b1f..8bfd9f9 100644 --- a/tests/benchmarks-service.test.mjs +++ b/tests/benchmarks-service.test.mjs @@ -103,6 +103,21 @@ test("benchmark detail exposes an explicit empty state", async () => { assert.equal(result.manifestBytes, null); }); +test("benchmark detail refuses a replaced or missing publication when pinned", async () => { + const service = fixture(async (input) => responseFor(input)); + const current = await service.show({ project: projectId, workload: workloadId, expectPublication: publication.id }); + assert.equal(current.publication.id, publication.id); + await assert.rejects( + service.show({ project: projectId, workload: workloadId, expectPublication: "synthetic-older-publication", includeReport: true }), + /published benchmark has changed/, + ); + await assert.rejects( + fixture(async (input) => responseFor(input, { benchmark: null, publication: null })) + .show({ project: projectId, workload: workloadId, expectPublication: publication.id }), + /published benchmark has changed/, + ); +}); + test("benchmark detail rejects another workload's response", async () => { await assert.rejects( fixture(async (input) => responseFor(input, { From 84cc6f6825b45c5070c1025d6027d664a54de3e3 Mon Sep 17 00:00:00 2001 From: aamir Date: Fri, 6 Nov 2026 10:12:45 -0600 Subject: [PATCH 2/2] Reject empty expected benchmark publication --- src/benchmarks/service.ts | 5 ++++- tests/benchmarks-service.test.mjs | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/benchmarks/service.ts b/src/benchmarks/service.ts index 92f6170..b489423 100644 --- a/src/benchmarks/service.ts +++ b/src/benchmarks/service.ts @@ -121,6 +121,9 @@ export function createBenchmarksService(options: ManagementScopeOptions): Benchm }, async show(input) { + if (input.expectPublication !== undefined && input.expectPublication.length === 0) { + throw new CliError("Expected publication id must not be empty."); + } const scope = await resolveManagementScope(options, { project: input.project, workload: input.workload, @@ -137,7 +140,7 @@ export function createBenchmarksService(options: ManagementScopeOptions): Benchm (result.publication && (result.publication.org_id !== session.organizationId || result.publication.project_id !== project!.id || result.publication.workload_id !== workload!.id))) { throw new CliError("Understudy returned a benchmark outside the selected workload."); } - if (input.expectPublication && result.publication?.id !== input.expectPublication) { + if (input.expectPublication !== undefined && result.publication?.id !== input.expectPublication) { throw new CliError("The published benchmark has changed. Refresh the dashboard and copy the agent handoff again."); } const report = result.benchmark?.report; diff --git a/tests/benchmarks-service.test.mjs b/tests/benchmarks-service.test.mjs index 8bfd9f9..f449585 100644 --- a/tests/benchmarks-service.test.mjs +++ b/tests/benchmarks-service.test.mjs @@ -111,6 +111,10 @@ test("benchmark detail refuses a replaced or missing publication when pinned", a service.show({ project: projectId, workload: workloadId, expectPublication: "synthetic-older-publication", includeReport: true }), /published benchmark has changed/, ); + await assert.rejects( + service.show({ project: projectId, workload: workloadId, expectPublication: "" }), + /Expected publication id must not be empty/, + ); await assert.rejects( fixture(async (input) => responseFor(input, { benchmark: null, publication: null })) .show({ project: projectId, workload: workloadId, expectPublication: publication.id }),