diff --git a/README.md b/README.md index 1137e80..254b6dc 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ arguments and filters, for example `understudy workloads route --help`. | `projects` | `list`, `show`, `create`, `update`, `delete`, `switch`, `ensure-default` | Application/service groupings. `switch` saves the project as this directory's default. | | `workloads` | `list`, `show`, `create`, `update`, `route` | Model tasks, capture settings, and explicit model traffic routes. | | `models` | `list`, `show` | Models in your organization's catalog. | +| `benchmarks` | `list`, `show` | Read published workload benchmarks and model recommendations. | | `keys` | `list`, `create`, `exec`, `revoke` | API key metadata, private key storage, and credential delivery to a child process. | New workloads have capture disabled by default. Setting a model route enables @@ -150,6 +151,16 @@ request metadata remains available without it. During setup, choose retention for the intended traffic rather than copying settings from unrelated workloads. See the [resource command guide](docs/resource-commands.md). +To give a coding agent the current published benchmark and recommendation for +one workload, run this from the application workspace: + +```sh +understudy benchmarks show --project "" --workload "" --include-report --json +``` + +The full manifest may include prompt and evaluation content; review it before +sharing it with another agent or saving the output. + `workloads create` prints the project and workload request headers, strongly recommends session IDs for session-level evaluation, and explains turn IDs and tags for the application's model calls. diff --git a/docs/resource-commands.md b/docs/resource-commands.md index 381b5e4..30bdd44 100644 --- a/docs/resource-commands.md +++ b/docs/resource-commands.md @@ -22,6 +22,9 @@ accept an id. A selector that matches multiple resources is an error. | `workloads route --clear` | Clear its model route. | | `models show ` | Return known model identity, display name, and open-weight metadata. | | `models list --gateway` / `models show --gateway` | Read the gateway catalog's declared wire formats using saved inference access. | +| `benchmarks list` | List workloads and their current published benchmark, if any. | +| `benchmarks show --project --workload ` | Summarize the current benchmark and its recommendation without printing the full report. | +| `benchmarks show --project --workload --include-report --json` | Return the full published benchmark manifest, including any recommendation report and prompt diff. | | `keys list` | Return the key metadata page exposed by the platform. | | `keys create --name ` | Create a key and privately retain its one-time value. | | `keys exec -- [args...]` | Inject a stored key into a child environment without printing it. | @@ -124,5 +127,24 @@ formats do not establish tool behavior, parameter support, model health, prices, or which model actually serves a request. The default management catalog remains usable without inference setup. +`benchmarks` uses the authenticated organization's management API. `list` reads +the organization-wide index; it includes workloads without a publication in +JSON. `show` resolves one project and workload from exact selectors or the +current directory context, then verifies that the returned publication and +manifest belong to that scope. It reports explicitly when no benchmark has +been published. Both commands are read-only and send no inference requests. + +The default `show` output contains publication metadata, the serialized manifest +size, and a compact recommendation. Add `--include-report` to include the entire +published manifest in text or JSON output. That response may contain +customer-authored prompt, rubric, scenario, and evaluation material; the CLI +does not write it to a file. +If saving it for agent review, keep it in the application's ignored, +package-excluded `.understudy/` directory with owner-only permissions. The +management response is bounded to 16 MiB, and this command does not paginate +an individual benchmark. A report's prompt label and optional diff are not a +guaranteed, versioned deployable prompt artifact; check the application prompt +before planning an update. + These contracts are covered by invented fixtures and mocked HTTP responses. Source contracts and local tests do not establish live deployment availability. diff --git a/src/benchmarks/service.ts b/src/benchmarks/service.ts new file mode 100644 index 0000000..ca0e07c --- /dev/null +++ b/src/benchmarks/service.ts @@ -0,0 +1,154 @@ +import { z } from "zod"; + +import { resolveManagementScope, type ManagementScopeOptions } from "../context/service.js"; +import { CliError } from "../errors.js"; +import { requestManagementJson } from "../management/client.js"; +import { resolveManagementSession } from "../management/session.js"; + +const publicationSchema = z.object({ + id: z.string().min(1), + org_id: z.string().min(1), + project_id: z.string().min(1), + workload_id: z.string().min(1), + name: z.string().min(1), + artifact_sha256: z.string().regex(/^[a-f0-9]{64}$/), + model_count: z.number().int().positive(), + task_count: z.number().int().positive(), + best_score: z.number().min(0).max(1), + best_pass_rate: z.number().min(0).max(1), + published_at: z.string().min(1), +}); + +const indexSchema = z.object({ + workloads: z.array(z.object({ + project_id: z.string().min(1), + project_name: z.string().min(1), + project_slug: z.string().min(1), + workload_id: z.string().min(1), + workload_name: z.string().min(1), + benchmark: publicationSchema.nullable(), + })), +}); + +// Validate the identity and the fields used in the summary, while preserving +// every server-owned report field for the explicit full-report output. +const manifestSchema = z.object({ + id: z.string().min(1), + name: z.string().min(1), + project_id: z.string().min(1), + workload_id: z.string().min(1), + created_at: z.string().min(1), + models: z.array(z.object({ model: z.string().min(1) }).passthrough()).min(1), + report: z.object({ + recommendation: z.object({ + headline: z.string().min(1), + summary: z.string().min(1), + }).passthrough(), + arms: z.array(z.object({ + arm_id: z.string().min(1), + model: z.string().min(1), + display_name: z.string().min(1), + prompt: z.string().min(1), + role: z.enum(["current", "recommended", "compared"]), + score: z.number().min(0).max(1), + }).passthrough()).min(2), + prompt_change: z.object({ + diff: z.array(z.object({ + op: z.enum(["equal", "insert", "delete", "replace"]), + before: z.array(z.string()), + after: z.array(z.string()), + }).passthrough()).min(1), + }).passthrough().optional(), + }).passthrough().optional(), +}).passthrough(); + +const detailSchema = z.object({ + benchmark: manifestSchema.nullable(), + publication: publicationSchema.nullable(), +}); + +type Publication = z.infer; +type IndexRow = z.infer["workloads"][number]; +type Manifest = z.infer; + +export interface BenchmarkListResult { + workloads: IndexRow[]; +} + +export interface BenchmarkShowInput { + project?: string; + workload?: string; + includeReport?: boolean; +} + +export interface BenchmarkShowResult { + project: { id: string; slug: string; name: string }; + workload: { id: string; name: string }; + publication: Publication | null; + recommendation: NonNullable["recommendation"] | null; + arms: Array["arms"][number], "arm_id" | "model" | "display_name" | "role" | "score">>; + hasPromptDiff: boolean; + reportAvailable: boolean; + manifestBytes: number | null; + benchmark?: Manifest | null; +} + +export interface BenchmarksService { + list(): Promise; + show(input: BenchmarkShowInput): Promise; +} + +export function createBenchmarksService(options: ManagementScopeOptions): BenchmarksService { + return { + async list() { + const session = await resolveManagementSession(options); + const result = await requestManagementJson(session, { + path: `/admin/v1/orgs/${encodeURIComponent(session.organizationId)}/benchmarks`, + action: "Benchmark listing", + }, indexSchema); + const seen = new Set(); + for (const row of result.workloads) { + const key = `${row.project_id}\0${row.workload_id}`; + if (seen.has(key) || (row.benchmark && ( + row.benchmark.org_id !== session.organizationId || + row.benchmark.project_id !== row.project_id || + row.benchmark.workload_id !== row.workload_id + ))) throw new CliError("Understudy returned an invalid benchmark listing."); + seen.add(key); + } + return result; + }, + + async show(input) { + const scope = await resolveManagementScope(options, { + project: input.project, + workload: input.workload, + requireWorkload: true, + ignoreDefaults: input.project !== undefined, + }); + const { session, project, workload } = scope; + const result = await requestManagementJson(session, { + path: `/admin/v1/orgs/${encodeURIComponent(session.organizationId)}/projects/${encodeURIComponent(project!.id)}/workloads/${encodeURIComponent(workload!.id)}/benchmark`, + action: "Benchmark read", + }, detailSchema); + if ((result.benchmark === null) !== (result.publication === null) || + (result.benchmark && (result.benchmark.project_id !== project!.id || result.benchmark.workload_id !== workload!.id)) || + (result.publication && (result.publication.org_id !== session.organizationId || result.publication.project_id !== project!.id || result.publication.workload_id !== workload!.id))) { + throw new CliError("Understudy returned a benchmark outside the selected workload."); + } + const report = result.benchmark?.report; + return { + project: { id: project!.id, slug: project!.slug, name: project!.name }, + workload: { id: workload!.id, name: workload!.name }, + publication: result.publication, + recommendation: report?.recommendation ?? null, + arms: report?.arms.map(({ arm_id, model, display_name, role, score }) => + ({ arm_id, model, display_name, role, score })) ?? [], + hasPromptDiff: report?.prompt_change !== undefined, + reportAvailable: report !== undefined, + manifestBytes: result.benchmark ? Buffer.byteLength(JSON.stringify(result.benchmark), "utf8") : null, + ...(input.includeReport ? { benchmark: result.benchmark } : {}), + }; + }, + }; +} diff --git a/src/cli.ts b/src/cli.ts index bc2ec33..21e07ea 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -6,6 +6,7 @@ import { type AuthCommandDependencies, } from "./commands/auth.js"; import { addModelsCommand } from "./commands/models.js"; +import { addBenchmarksCommands } from "./commands/benchmarks.js"; import { addBillingCommands } from "./commands/billing.js"; import { addCapturesCommands } from "./commands/captures.js"; import { addContextCommands } from "./commands/context.js"; @@ -43,6 +44,7 @@ export function createCli( addAuthenticationCommands(program, authDependencies); addBillingCommands(program); + addBenchmarksCommands(program); addCapturesCommands(program); addContextCommands(program); addKeysCommands(program); diff --git a/src/commands/benchmarks.ts b/src/commands/benchmarks.ts new file mode 100644 index 0000000..0164a01 --- /dev/null +++ b/src/commands/benchmarks.ts @@ -0,0 +1,62 @@ +import { Command } from "commander"; + +import { createCredentialStore } from "../auth/credentials.js"; +import { createBenchmarksService, type BenchmarksService, type BenchmarkShowResult } from "../benchmarks/service.js"; +import { formatOutput } from "../output.js"; + +export function addBenchmarksCommands( + program: Command, + service: BenchmarksService = createBenchmarksService({ store: createCredentialStore() }), + output = (message: string) => process.stdout.write(`${message}\n`), +): void { + const benchmarks = program.command("benchmarks") + .description("Read published workload benchmarks and model recommendations."); + + benchmarks.command("list") + .description("List workloads and their current benchmark publication, if any.") + .action(async function (this: Command) { + const result = await service.list(); + const published = result.workloads.filter((row) => row.benchmark); + output(formatOutput(this, result, published.length === 0 + ? "No published benchmarks in the authenticated organization." + : [ + ...published.map((row) => `${row.project_slug}/${row.workload_name} ${row.benchmark!.name} published ${row.benchmark!.published_at}`), + `${published.length} published benchmark(s) across ${result.workloads.length} workload(s).`, + ])); + }); + + benchmarks.command("show") + .description("Show the current benchmark for a workload. Use --include-report for the full manifest and prompt diff.") + .option("--project ", "Exact project id, slug, or name; defaults to saved context.") + .option("--workload ", "Exact workload id or name; defaults to saved context.") + .option("--include-report", "Include the full benchmark manifest, which may contain customer prompt and eval content.") + .action(async function (this: Command, options: { project?: string; workload?: string; includeReport?: boolean }) { + const result = await service.show(options); + output(formatBenchmark(this, result, options.includeReport === true)); + }); +} + +function formatBenchmark(command: Command, result: BenchmarkShowResult, full: boolean): string { + if (full) { + return formatOutput(command, result, JSON.stringify(result, null, 2).split("\n")); + } + if (!result.publication) { + return formatOutput(command, result, `No published benchmark for ${result.project.slug}/${result.workload.name}.`); + } + const lines = [ + `Benchmark ${result.publication.name}`, + `Workload ${result.project.slug}/${result.workload.name}`, + `Published ${result.publication.published_at}`, + `Models ${result.publication.model_count}`, + `Tasks ${result.publication.task_count}`, + `Size ${result.manifestBytes} bytes (serialized manifest)`, + ...(result.recommendation ? [ + `Recommend ${result.recommendation.headline}`, + `Summary ${result.recommendation.summary}`, + ...result.arms.map((arm) => `${arm.role.padEnd(11)}${arm.model} (score ${arm.score})`), + `Prompt diff ${result.hasPromptDiff ? "available in full report" : "not published"}`, + ] : ["Recommendation report not published for this benchmark."]), + "Use --include-report --json to give a coding agent the full report and benchmark evidence.", + ]; + return formatOutput(command, result, lines); +} diff --git a/tests/benchmarks-command.test.mjs b/tests/benchmarks-command.test.mjs new file mode 100644 index 0000000..5efb358 --- /dev/null +++ b/tests/benchmarks-command.test.mjs @@ -0,0 +1,57 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { Command } from "commander"; + +import { addBenchmarksCommands } from "../dist/commands/benchmarks.js"; + +const report = { + project: { id: "synthetic-project", slug: "fictional", name: "Fictional" }, + workload: { id: "synthetic-workload", name: "writer" }, + publication: { + name: "Fictional comparison", published_at: "2030-01-01T00:00:00Z", + model_count: 2, task_count: 3, + }, + recommendation: { headline: "Try model A", summary: "Fictional evidence." }, + arms: [{ arm_id: "a", model: "example/model-a", display_name: "A", role: "recommended", score: 0.8 }], + hasPromptDiff: true, + reportAvailable: true, + manifestBytes: 2048, +}; + +function setup() { + const calls = []; + const output = []; + const program = new Command().option("--json").exitOverride(); + program.configureOutput({ writeErr: () => undefined }); + addBenchmarksCommands(program, { + list: async () => ({ workloads: [{ project_slug: "fictional", workload_name: "writer", benchmark: report.publication }] }), + show: async (input) => { + calls.push(input); + return input.includeReport ? { ...report, benchmark: { report: { prompt_change: { diff: ["Invented prompt line"] } } } } : report; + }, + }, (message) => output.push(message)); + return { program, calls, output }; +} + +test("benchmarks show prints a summary and forwards saved-context selection", async () => { + const { program, calls, output } = setup(); + await program.parseAsync(["node", "understudy", "benchmarks", "show"]); + assert.deepEqual(calls, [{}]); + assert.match(output[0], /Recommend Try model A/); + assert.match(output[0], /--include-report --json/); + assert.match(output[0], /2048 bytes/); + assert.doesNotMatch(output[0], /Invented prompt line/); +}); + +test("benchmarks show gives an agent the full JSON report only when requested", async () => { + const { program, calls, output } = setup(); + await program.parseAsync(["node", "understudy", "--json", "benchmarks", "show", "--project", "fictional", "--workload", "writer", "--include-report"]); + assert.deepEqual(calls, [{ includeReport: true, project: "fictional", workload: "writer" }]); + assert.equal(JSON.parse(output[0]).benchmark.report.prompt_change.diff[0], "Invented prompt line"); +}); + +test("benchmarks list prints available publications", async () => { + const { program, output } = setup(); + await program.parseAsync(["node", "understudy", "benchmarks", "list"]); + assert.match(output[0], /fictional\/writer Fictional comparison/); +}); diff --git a/tests/benchmarks-service.test.mjs b/tests/benchmarks-service.test.mjs new file mode 100644 index 0000000..3ed8b1f --- /dev/null +++ b/tests/benchmarks-service.test.mjs @@ -0,0 +1,143 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { createBenchmarksService } from "../dist/benchmarks/service.js"; + +const orgId = "synthetic-organization"; +const projectId = "synthetic-project"; +const workloadId = "synthetic-workload"; +const promptText = "Invented prompt line for a fictional task."; + +const jwt = (claims) => [ + Buffer.from(JSON.stringify({ alg: "none" })).toString("base64url"), + Buffer.from(JSON.stringify(claims)).toString("base64url"), + "synthetic-signature", +].join("."); + +const store = { + read: async () => ({ + version: 1, + method: "oauth", + accessToken: jwt({ org_id: orgId, exp: Date.now() / 1_000 + 3600 }), + }), + write: async () => undefined, + clear: async () => undefined, +}; + +const publication = { + id: "synthetic-publication", + org_id: orgId, + project_id: projectId, + workload_id: workloadId, + name: "Fictional comparison", + artifact_sha256: "a".repeat(64), + model_count: 2, + task_count: 3, + best_score: 0.8, + best_pass_rate: 0.7, + published_at: "2030-01-01T00:00:00.000Z", +}; + +const benchmark = { + id: "synthetic-benchmark", + name: publication.name, + project_id: projectId, + workload_id: workloadId, + created_at: publication.published_at, + models: [ + { model: "example/model-a", tasks: [{ task_id: "invented-case", score: 0.8 }] }, + ], + report: { + recommendation: { headline: "Try model A", summary: "Review its fictional results." }, + arms: [ + { arm_id: "a", model: "example/model-a", display_name: "A", prompt: "Tuned prompt", role: "recommended", score: 0.8 }, + { arm_id: "b", model: "example/model-b", display_name: "B", prompt: "Current prompt", role: "current", score: 0.5 }, + ], + prompt_change: { + diff: [{ op: "insert", before: [], after: [promptText] }], + }, + caveats: ["Small fictional sample."], + }, +}; + +function fixture(fetchImplementation) { + return createBenchmarksService({ store, baseUrl: "https://example.test", fetchImplementation }); +} + +function responseFor(input, detail = { benchmark, publication }) { + const url = new URL(String(input)); + if (url.pathname.endsWith("/projects")) return Response.json({ projects: [{ id: projectId, org_id: orgId, slug: "fictional", name: "Fictional" }] }); + if (url.pathname.endsWith("/workloads")) return Response.json({ workloads: [{ id: workloadId, project_id: projectId, name: "writer" }] }); + if (url.pathname.endsWith("/benchmark")) return Response.json(detail); + throw new Error(`Unexpected synthetic request: ${url.pathname}`); +} + +test("benchmark detail summarizes safely and preserves the full report only on request", async () => { + const urls = []; + const service = fixture(async (input, init) => { + urls.push(String(input)); + assert.equal(init.method, "GET"); + assert.match(init.headers.authorization, /^Bearer /); + return responseFor(input); + }); + const selection = { project: "fictional", workload: "writer" }; + const summary = await service.show(selection); + assert.equal(summary.recommendation.headline, "Try model A"); + assert.equal(summary.hasPromptDiff, true); + assert.equal(summary.manifestBytes, Buffer.byteLength(JSON.stringify(benchmark), "utf8")); + assert.equal(summary.benchmark, undefined); + assert.doesNotMatch(JSON.stringify(summary), /Invented prompt line|Tuned prompt/); + + const full = await service.show({ ...selection, includeReport: true }); + assert.equal(full.benchmark.report.prompt_change.diff[0].after[0], promptText); + assert.deepEqual(full.benchmark.models[0].tasks, benchmark.models[0].tasks); + assert.ok(urls.every((url) => url.startsWith("https://example.test/admin/v1/orgs/synthetic-organization/"))); +}); + +test("benchmark detail exposes an explicit empty state", async () => { + const result = await fixture(async (input) => responseFor(input, { benchmark: null, publication: null })) + .show({ project: projectId, workload: workloadId, includeReport: true }); + assert.equal(result.publication, null); + assert.equal(result.recommendation, null); + assert.equal(result.benchmark, null); + assert.equal(result.manifestBytes, null); +}); + +test("benchmark detail rejects another workload's response", async () => { + await assert.rejects( + fixture(async (input) => responseFor(input, { + publication: { ...publication, workload_id: "other-workload" }, benchmark, + })).show({ project: projectId, workload: workloadId }), + /outside the selected workload/, + ); +}); + +test("benchmark detail does not claim a missing or malformed prompt diff", async () => { + const noChange = { ...benchmark, report: { ...benchmark.report, prompt_change: undefined } }; + const result = await fixture(async (input) => responseFor(input, { benchmark: noChange, publication })) + .show({ project: projectId, workload: workloadId }); + assert.equal(result.hasPromptDiff, false); + + for (const promptChange of [null, {}, { diff: [] }]) { + const invalid = { ...benchmark, report: { ...benchmark.report, prompt_change: promptChange } }; + await assert.rejects( + fixture(async (input) => responseFor(input, { benchmark: invalid, publication })) + .show({ project: projectId, workload: workloadId }), + /invalid response for benchmark read/, + ); + } +}); + +test("benchmark listing validates publication scope", async () => { + const listed = await fixture(async (input) => { + assert.equal(new URL(String(input)).pathname, `/admin/v1/orgs/${orgId}/benchmarks`); + return Response.json({ workloads: [ + { project_id: projectId, project_name: "Fictional", project_slug: "fictional", workload_id: workloadId, workload_name: "writer", benchmark: publication }, + ] }); + }).list(); + assert.equal(listed.workloads[0].benchmark.name, publication.name); + + await assert.rejects(fixture(async () => Response.json({ workloads: [ + { project_id: projectId, project_name: "Fictional", project_slug: "fictional", workload_id: workloadId, workload_name: "writer", benchmark: { ...publication, org_id: "other-organization" } }, + ] })).list(), /invalid benchmark listing/); +});