diff --git a/src/benchmarks/service.ts b/src/benchmarks/service.ts index ca0e07c..b489423 100644 --- a/src/benchmarks/service.ts +++ b/src/benchmarks/service.ts @@ -79,6 +79,7 @@ export interface BenchmarkShowInput { project?: string; workload?: string; includeReport?: boolean; + expectPublication?: string; } export interface BenchmarkShowResult { @@ -120,6 +121,9 @@ export function createBenchmarksService(options: ManagementScopeOptions): Benchm }, async show(input) { + if (input.expectPublication !== undefined && input.expectPublication.length === 0) { + throw new CliError("Expected publication id must not be empty."); + } const scope = await resolveManagementScope(options, { project: input.project, workload: input.workload, @@ -136,6 +140,9 @@ export function createBenchmarksService(options: ManagementScopeOptions): Benchm (result.publication && (result.publication.org_id !== session.organizationId || result.publication.project_id !== project!.id || result.publication.workload_id !== workload!.id))) { throw new CliError("Understudy returned a benchmark outside the selected workload."); } + if (input.expectPublication !== undefined && result.publication?.id !== input.expectPublication) { + throw new CliError("The published benchmark has changed. Refresh the dashboard and copy the agent handoff again."); + } const report = result.benchmark?.report; return { project: { id: project!.id, slug: project!.slug, name: project!.name }, diff --git a/src/commands/benchmarks.ts b/src/commands/benchmarks.ts index 0164a01..246d5b3 100644 --- a/src/commands/benchmarks.ts +++ b/src/commands/benchmarks.ts @@ -30,7 +30,8 @@ export function addBenchmarksCommands( .option("--project ", "Exact project id, slug, or name; defaults to saved context.") .option("--workload ", "Exact workload id or name; defaults to saved context.") .option("--include-report", "Include the full benchmark manifest, which may contain customer prompt and eval content.") - .action(async function (this: Command, options: { project?: string; workload?: string; includeReport?: boolean }) { + .option("--expect-publication ", "Require this exact publication; fail if the current benchmark changed.") + .action(async function (this: Command, options: { project?: string; workload?: string; includeReport?: boolean; expectPublication?: string }) { const result = await service.show(options); output(formatBenchmark(this, result, options.includeReport === true)); }); diff --git a/tests/benchmarks-command.test.mjs b/tests/benchmarks-command.test.mjs index 5efb358..3d9856e 100644 --- a/tests/benchmarks-command.test.mjs +++ b/tests/benchmarks-command.test.mjs @@ -50,6 +50,12 @@ test("benchmarks show gives an agent the full JSON report only when requested", assert.equal(JSON.parse(output[0]).benchmark.report.prompt_change.diff[0], "Invented prompt line"); }); +test("benchmarks show forwards the publication pin", async () => { + const { program, calls } = setup(); + await program.parseAsync(["node", "understudy", "benchmarks", "show", "--expect-publication", "synthetic-publication"]); + assert.deepEqual(calls, [{ expectPublication: "synthetic-publication" }]); +}); + test("benchmarks list prints available publications", async () => { const { program, output } = setup(); await program.parseAsync(["node", "understudy", "benchmarks", "list"]); diff --git a/tests/benchmarks-service.test.mjs b/tests/benchmarks-service.test.mjs index 3ed8b1f..f449585 100644 --- a/tests/benchmarks-service.test.mjs +++ b/tests/benchmarks-service.test.mjs @@ -103,6 +103,25 @@ test("benchmark detail exposes an explicit empty state", async () => { assert.equal(result.manifestBytes, null); }); +test("benchmark detail refuses a replaced or missing publication when pinned", async () => { + const service = fixture(async (input) => responseFor(input)); + const current = await service.show({ project: projectId, workload: workloadId, expectPublication: publication.id }); + assert.equal(current.publication.id, publication.id); + await assert.rejects( + service.show({ project: projectId, workload: workloadId, expectPublication: "synthetic-older-publication", includeReport: true }), + /published benchmark has changed/, + ); + await assert.rejects( + service.show({ project: projectId, workload: workloadId, expectPublication: "" }), + /Expected publication id must not be empty/, + ); + await assert.rejects( + fixture(async (input) => responseFor(input, { benchmark: null, publication: null })) + .show({ project: projectId, workload: workloadId, expectPublication: publication.id }), + /published benchmark has changed/, + ); +}); + test("benchmark detail rejects another workload's response", async () => { await assert.rejects( fixture(async (input) => responseFor(input, {