Add CLI access to published benchmark reports

README.md

@@ -140,6 +140,7 @@ arguments and filters, for example `understudy workloads route --help`.
 | `projects` | `list`, `show`, `create`, `update`, `delete`, `switch`, `ensure-default` | Application/service groupings. `switch` saves the project as this directory's default. |
 | `workloads` | `list`, `show`, `create`, `update`, `route` | Model tasks, capture settings, and explicit model traffic routes. |
 | `models` | `list`, `show` | Models in your organization's catalog. |
+| `benchmarks` | `list`, `show` | Read published workload benchmarks and model recommendations. |
 | `keys` | `list`, `create`, `exec`, `revoke` | API key metadata, private key storage, and credential delivery to a child process. |
 
 New workloads have capture disabled by default. Setting a model route enables
@@ -150,6 +151,16 @@ request metadata remains available without it. During setup, choose retention
 for the intended traffic rather than copying settings from unrelated workloads.
 See the [resource command guide](docs/resource-commands.md).
 
+To give a coding agent the current published benchmark and recommendation for
+one workload, run this from the application workspace:
+
+```sh
+understudy benchmarks show --project "<project>" --workload "<workload>" --include-report --json
+```
+
+The full manifest may include prompt and evaluation content; review it before
+sharing it with another agent or saving the output.
+
 `workloads create` prints the project and workload request headers, strongly
 recommends session IDs for session-level evaluation, and explains turn IDs and
 tags for the application's model calls.

docs/resource-commands.md

@@ -22,6 +22,9 @@ accept an id. A selector that matches multiple resources is an error.
 | `workloads route <workload> --clear` | Clear its model route. |
 | `models show <model>` | Return known model identity, display name, and open-weight metadata. |
 | `models list --gateway` / `models show <model> --gateway` | Read the gateway catalog's declared wire formats using saved inference access. |
+| `benchmarks list` | List workloads and their current published benchmark, if any. |
+| `benchmarks show --project <project> --workload <workload>` | Summarize the current benchmark and its recommendation without printing the full report. |
+| `benchmarks show --project <project> --workload <workload> --include-report --json` | Return the full published benchmark manifest, including any recommendation report and prompt diff. |
 | `keys list` | Return the key metadata page exposed by the platform. |
 | `keys create --name <name>` | Create a key and privately retain its one-time value. |
 | `keys exec <reference> -- <command> [args...]` | Inject a stored key into a child environment without printing it. |
@@ -124,5 +127,24 @@ formats do not establish tool behavior, parameter support, model health, prices,
 or which model actually serves a request. The default management catalog remains
 usable without inference setup.
 
+`benchmarks` uses the authenticated organization's management API. `list` reads
+the organization-wide index; it includes workloads without a publication in
+JSON. `show` resolves one project and workload from exact selectors or the
+current directory context, then verifies that the returned publication and
+manifest belong to that scope. It reports explicitly when no benchmark has
+been published. Both commands are read-only and send no inference requests.
+
+The default `show` output contains publication metadata, the serialized manifest
+size, and a compact recommendation. Add `--include-report` to include the entire
+published manifest in text or JSON output. That response may contain
+customer-authored prompt, rubric, scenario, and evaluation material; the CLI
+does not write it to a file.
+If saving it for agent review, keep it in the application's ignored,
+package-excluded `.understudy/` directory with owner-only permissions. The
+management response is bounded to 16 MiB, and this command does not paginate
+an individual benchmark. A report's prompt label and optional diff are not a
+guaranteed, versioned deployable prompt artifact; check the application prompt
+before planning an update.
+
 These contracts are covered by invented fixtures and mocked HTTP responses.
 Source contracts and local tests do not establish live deployment availability.

src/benchmarks/service.ts

@@ -0,0 +1,154 @@
+import { z } from "zod";
+
+import { resolveManagementScope, type ManagementScopeOptions } from "../context/service.js";
+import { CliError } from "../errors.js";
+import { requestManagementJson } from "../management/client.js";
+import { resolveManagementSession } from "../management/session.js";
+
+const publicationSchema = z.object({
+  id: z.string().min(1),
+  org_id: z.string().min(1),
+  project_id: z.string().min(1),
+  workload_id: z.string().min(1),
+  name: z.string().min(1),
+  artifact_sha256: z.string().regex(/^[a-f0-9]{64}$/),
+  model_count: z.number().int().positive(),
+  task_count: z.number().int().positive(),
+  best_score: z.number().min(0).max(1),
+  best_pass_rate: z.number().min(0).max(1),
+  published_at: z.string().min(1),
+});
+
+const indexSchema = z.object({
+  workloads: z.array(z.object({
+    project_id: z.string().min(1),
+    project_name: z.string().min(1),
+    project_slug: z.string().min(1),
+    workload_id: z.string().min(1),
+    workload_name: z.string().min(1),
+    benchmark: publicationSchema.nullable(),
+  })),
+});
+
+// Validate the identity and the fields used in the summary, while preserving
+// every server-owned report field for the explicit full-report output.
+const manifestSchema = z.object({
+  id: z.string().min(1),
+  name: z.string().min(1),
+  project_id: z.string().min(1),
+  workload_id: z.string().min(1),
+  created_at: z.string().min(1),
+  models: z.array(z.object({ model: z.string().min(1) }).passthrough()).min(1),
+  report: z.object({
+    recommendation: z.object({
+      headline: z.string().min(1),
+      summary: z.string().min(1),
+    }).passthrough(),
+    arms: z.array(z.object({
+      arm_id: z.string().min(1),
+      model: z.string().min(1),
+      display_name: z.string().min(1),
+      prompt: z.string().min(1),
+      role: z.enum(["current", "recommended", "compared"]),
+      score: z.number().min(0).max(1),
+    }).passthrough()).min(2),
+    prompt_change: z.object({
+      diff: z.array(z.object({
+        op: z.enum(["equal", "insert", "delete", "replace"]),
+        before: z.array(z.string()),
+        after: z.array(z.string()),
+      }).passthrough()).min(1),
+    }).passthrough().optional(),
+  }).passthrough().optional(),
+}).passthrough();
+
+const detailSchema = z.object({
+  benchmark: manifestSchema.nullable(),
+  publication: publicationSchema.nullable(),
+});
+
+type Publication = z.infer<typeof publicationSchema>;
+type IndexRow = z.infer<typeof indexSchema>["workloads"][number];
+type Manifest = z.infer<typeof manifestSchema>;
+
+export interface BenchmarkListResult {
+  workloads: IndexRow[];
+}
+
+export interface BenchmarkShowInput {
+  project?: string;
+  workload?: string;
+  includeReport?: boolean;
+}
+
+export interface BenchmarkShowResult {
+  project: { id: string; slug: string; name: string };
+  workload: { id: string; name: string };
+  publication: Publication | null;
+  recommendation: NonNullable<Manifest["report"]>["recommendation"] | null;
+  arms: Array<Pick<NonNullable<Manifest["report"]>["arms"][number], "arm_id" | "model" | "display_name" | "role" | "score">>;
+  hasPromptDiff: boolean;
+  reportAvailable: boolean;
+  manifestBytes: number | null;
+  benchmark?: Manifest | null;
+}
+
+export interface BenchmarksService {
+  list(): Promise<BenchmarkListResult>;
+  show(input: BenchmarkShowInput): Promise<BenchmarkShowResult>;
+}
+
+export function createBenchmarksService(options: ManagementScopeOptions): BenchmarksService {
+  return {
+    async list() {
+      const session = await resolveManagementSession(options);
+      const result = await requestManagementJson(session, {
+        path: `/admin/v1/orgs/${encodeURIComponent(session.organizationId)}/benchmarks`,
+        action: "Benchmark listing",
+      }, indexSchema);
+      const seen = new Set<string>();
+      for (const row of result.workloads) {
+        const key = `${row.project_id}\0${row.workload_id}`;
+        if (seen.has(key) || (row.benchmark && (
+          row.benchmark.org_id !== session.organizationId ||
+          row.benchmark.project_id !== row.project_id ||
+          row.benchmark.workload_id !== row.workload_id
+        ))) throw new CliError("Understudy returned an invalid benchmark listing.");
+        seen.add(key);
+      }
+      return result;
+    },
+
+    async show(input) {
+      const scope = await resolveManagementScope(options, {
+        project: input.project,
+        workload: input.workload,
+        requireWorkload: true,
+        ignoreDefaults: input.project !== undefined,
+      });
+      const { session, project, workload } = scope;
+      const result = await requestManagementJson(session, {
+        path: `/admin/v1/orgs/${encodeURIComponent(session.organizationId)}/projects/${encodeURIComponent(project!.id)}/workloads/${encodeURIComponent(workload!.id)}/benchmark`,
+        action: "Benchmark read",
+      }, detailSchema);
+      if ((result.benchmark === null) !== (result.publication === null) ||
+          (result.benchmark && (result.benchmark.project_id !== project!.id || result.benchmark.workload_id !== workload!.id)) ||
+          (result.publication && (result.publication.org_id !== session.organizationId || result.publication.project_id !== project!.id || result.publication.workload_id !== workload!.id))) {
+        throw new CliError("Understudy returned a benchmark outside the selected workload.");
+      }
+      const report = result.benchmark?.report;
+      return {
+        project: { id: project!.id, slug: project!.slug, name: project!.name },
+        workload: { id: workload!.id, name: workload!.name },
+        publication: result.publication,
+        recommendation: report?.recommendation ?? null,
+        arms: report?.arms.map(({ arm_id, model, display_name, role, score }) =>
+          ({ arm_id, model, display_name, role, score })) ?? [],
+        hasPromptDiff: report?.prompt_change !== undefined,
+        reportAvailable: report !== undefined,
+        manifestBytes: result.benchmark ? Buffer.byteLength(JSON.stringify(result.benchmark), "utf8") : null,
+        ...(input.includeReport ? { benchmark: result.benchmark } : {}),
+      };
+    },
+  };
+}

src/cli.ts

@@ -6,6 +6,7 @@ import {
   type AuthCommandDependencies,
 } from "./commands/auth.js";
 import { addModelsCommand } from "./commands/models.js";
+import { addBenchmarksCommands } from "./commands/benchmarks.js";
 import { addBillingCommands } from "./commands/billing.js";
 import { addCapturesCommands } from "./commands/captures.js";
 import { addContextCommands } from "./commands/context.js";
@@ -43,6 +44,7 @@ export function createCli(
 
   addAuthenticationCommands(program, authDependencies);
   addBillingCommands(program);
+  addBenchmarksCommands(program);
   addCapturesCommands(program);
   addContextCommands(program);
   addKeysCommands(program);

src/commands/benchmarks.ts

@@ -0,0 +1,62 @@
+import { Command } from "commander";
+
+import { createCredentialStore } from "../auth/credentials.js";
+import { createBenchmarksService, type BenchmarksService, type BenchmarkShowResult } from "../benchmarks/service.js";
+import { formatOutput } from "../output.js";
+
+export function addBenchmarksCommands(
+  program: Command,
+  service: BenchmarksService = createBenchmarksService({ store: createCredentialStore() }),
+  output = (message: string) => process.stdout.write(`${message}\n`),
+): void {
+  const benchmarks = program.command("benchmarks")
+    .description("Read published workload benchmarks and model recommendations.");
+
+  benchmarks.command("list")
+    .description("List workloads and their current benchmark publication, if any.")
+    .action(async function (this: Command) {
+      const result = await service.list();
+      const published = result.workloads.filter((row) => row.benchmark);
+      output(formatOutput(this, result, published.length === 0
+        ? "No published benchmarks in the authenticated organization."
+        : [
+            ...published.map((row) => `${row.project_slug}/${row.workload_name}  ${row.benchmark!.name}  published ${row.benchmark!.published_at}`),
+            `${published.length} published benchmark(s) across ${result.workloads.length} workload(s).`,
+          ]));
+    });
+
+  benchmarks.command("show")
+    .description("Show the current benchmark for a workload. Use --include-report for the full manifest and prompt diff.")
+    .option("--project <selector>", "Exact project id, slug, or name; defaults to saved context.")
+    .option("--workload <selector>", "Exact workload id or name; defaults to saved context.")
+    .option("--include-report", "Include the full benchmark manifest, which may contain customer prompt and eval content.")
+    .action(async function (this: Command, options: { project?: string; workload?: string; includeReport?: boolean }) {
+      const result = await service.show(options);
+      output(formatBenchmark(this, result, options.includeReport === true));
+    });
+}
+
+function formatBenchmark(command: Command, result: BenchmarkShowResult, full: boolean): string {
+  if (full) {
+    return formatOutput(command, result, JSON.stringify(result, null, 2).split("\n"));
+  }
+  if (!result.publication) {
+    return formatOutput(command, result, `No published benchmark for ${result.project.slug}/${result.workload.name}.`);
+  }
+  const lines = [
+    `Benchmark  ${result.publication.name}`,
+    `Workload   ${result.project.slug}/${result.workload.name}`,
+    `Published  ${result.publication.published_at}`,
+    `Models     ${result.publication.model_count}`,
+    `Tasks      ${result.publication.task_count}`,
+    `Size       ${result.manifestBytes} bytes (serialized manifest)`,
+    ...(result.recommendation ? [
+      `Recommend  ${result.recommendation.headline}`,
+      `Summary    ${result.recommendation.summary}`,
+      ...result.arms.map((arm) => `${arm.role.padEnd(11)}${arm.model} (score ${arm.score})`),
+      `Prompt diff ${result.hasPromptDiff ? "available in full report" : "not published"}`,
+    ] : ["Recommendation report not published for this benchmark."]),
+    "Use --include-report --json to give a coding agent the full report and benchmark evidence.",
+  ];
+  return formatOutput(command, result, lines);
+}

tests/benchmarks-command.test.mjs

@@ -0,0 +1,57 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { Command } from "commander";
+
+import { addBenchmarksCommands } from "../dist/commands/benchmarks.js";
+
+const report = {
+  project: { id: "synthetic-project", slug: "fictional", name: "Fictional" },
+  workload: { id: "synthetic-workload", name: "writer" },
+  publication: {
+    name: "Fictional comparison", published_at: "2030-01-01T00:00:00Z",
+    model_count: 2, task_count: 3,
+  },
+  recommendation: { headline: "Try model A", summary: "Fictional evidence." },
+  arms: [{ arm_id: "a", model: "example/model-a", display_name: "A", role: "recommended", score: 0.8 }],
+  hasPromptDiff: true,
+  reportAvailable: true,
+  manifestBytes: 2048,
+};
+
+function setup() {
+  const calls = [];
+  const output = [];
+  const program = new Command().option("--json").exitOverride();
+  program.configureOutput({ writeErr: () => undefined });
+  addBenchmarksCommands(program, {
+    list: async () => ({ workloads: [{ project_slug: "fictional", workload_name: "writer", benchmark: report.publication }] }),
+    show: async (input) => {
+      calls.push(input);
+      return input.includeReport ? { ...report, benchmark: { report: { prompt_change: { diff: ["Invented prompt line"] } } } } : report;
+    },
+  }, (message) => output.push(message));
+  return { program, calls, output };
+}
+
+test("benchmarks show prints a summary and forwards saved-context selection", async () => {
+  const { program, calls, output } = setup();
+  await program.parseAsync(["node", "understudy", "benchmarks", "show"]);
+  assert.deepEqual(calls, [{}]);
+  assert.match(output[0], /Recommend  Try model A/);
+  assert.match(output[0], /--include-report --json/);
+  assert.match(output[0], /2048 bytes/);
+  assert.doesNotMatch(output[0], /Invented prompt line/);
+});
+
+test("benchmarks show gives an agent the full JSON report only when requested", async () => {
+  const { program, calls, output } = setup();
+  await program.parseAsync(["node", "understudy", "--json", "benchmarks", "show", "--project", "fictional", "--workload", "writer", "--include-report"]);
+  assert.deepEqual(calls, [{ includeReport: true, project: "fictional", workload: "writer" }]);
+  assert.equal(JSON.parse(output[0]).benchmark.report.prompt_change.diff[0], "Invented prompt line");
+});
+
+test("benchmarks list prints available publications", async () => {
+  const { program, output } = setup();
+  await program.parseAsync(["node", "understudy", "benchmarks", "list"]);
+  assert.match(output[0], /fictional\/writer  Fictional comparison/);
+});