From 07466694a31c15e3ab2121f4ac1d8dadd3675186 Mon Sep 17 00:00:00 2001 From: aamir Date: Tue, 20 Oct 2026 11:39:54 -0500 Subject: [PATCH 1/2] Include session and turn attribution in CLI setup --- README.md | 4 + docs/resource-commands.md | 29 +++++ scripts/bundle-skills.mjs | 1 + skills/setup-understudy/SKILL.md | 23 ++++ .../references/anthropic-messages.md | 4 + .../references/openai-compatible.md | 4 + .../references/session-turn-ids.md | 105 ++++++++++++++++++ src/commands/workloads.ts | 43 ++++++- tests/skills-bundle.test.mjs | 2 +- tests/skills-distribution.test.mjs | 1 + tests/workloads-command.test.mjs | 35 ++++++ 11 files changed, 247 insertions(+), 4 deletions(-) create mode 100644 skills/setup-understudy/references/session-turn-ids.md diff --git a/README.md b/README.md index 6507e5e..1137e80 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,10 @@ request metadata remains available without it. During setup, choose retention for the intended traffic rather than copying settings from unrelated workloads. See the [resource command guide](docs/resource-commands.md). +`workloads create` prints the project and workload request headers, strongly +recommends session IDs for session-level evaluation, and explains turn IDs and +tags for the application's model calls. + ### Inspect traffic, usage, and billing | Group | Subcommands | What you can learn | diff --git a/docs/resource-commands.md b/docs/resource-commands.md index 644ad64..381b5e4 100644 --- a/docs/resource-commands.md +++ b/docs/resource-commands.md @@ -16,6 +16,7 @@ accept an id. A selector that matches multiple resources is an error. | `projects delete --confirm` | Soft-delete the selected project. | | `projects switch ` (alias `use`) | Verify and save the project as the current directory context. | | `projects ensure-default` | Idempotently provision the canonical rehearsal project and default workload. | +| `workloads create --project [--capture]` | Create a workload and print its request-header integration recipe. | | `workloads update --project --capture-sample-rate ` | Set capture sampling between 0 and 1 independently of the capture enabled flag. | | `workloads route --model-id [--traffic-pct ]` | Explicitly route the selected workload to a public model, defaulting to 10 percent. | | `workloads route --clear` | Clear its model route. | @@ -42,6 +43,34 @@ capture unchanged unless `--capture on|off` is explicit. Percentages must be integers from 0 to 100. Clearing cannot be combined with model or percentage options. Responses must echo the requested model, workload, project, and share. +Workload creation prints the project and workload headers to send with inference +requests. It strongly recommends `x-understudy-session-id` for session-level +evaluation: use one opaque, case-sensitive ID across every model call in an +execution, including tool continuations and retries, and a new ID for each +execution. Online model experiments require it to keep assignment stable. + +A **turn** is an application-defined step within a session: for example, one +user message and the assistant work it triggers, or one agent step in a longer +task. It may span several model requests, tool continuations, and retries. Send +the same `x-understudy-turn-id` on those requests, and change it when the next +logical step begins. A new model request is not automatically a new turn. A +one-shot task is a one-turn session; the same value may be sent as both session +and turn ID, or the turn header may be omitted. Both IDs must contain +non-whitespace text and fit within 256 UTF-8 bytes. Neither ID signals that +an execution or turn has completed; the application defines those boundaries. + +`x-understudy-tags` carries application-owned dimensions as a flat JSON object +with string values, for example `{"cohort":"trial"}`. Tags can identify a +user, account, or cohort for filtering and targeting. Choose keys from the +application's own identifiers and avoid sensitive personal data. If an online +experiment targets a tag, send the same key and value on every call in the +session. Tags determine which sessions are eligible; the session ID groups +calls and determines a stable experiment assignment. Tags alone do not measure +quality: a session-level evaluation also needs a task outcome or verifier. +The application supplies these headers with requests; workload creation does +not store their values. `--json` returns header names and guidance in +`integration`. + The key-list API currently does not expose its upstream pagination cursor. The CLI therefore returns `completeness: "unknown"` and describes this limitation in both text and JSON. It cannot discover keys beyond that page. A key outside the diff --git a/scripts/bundle-skills.mjs b/scripts/bundle-skills.mjs index 04c57d2..5995080 100644 --- a/scripts/bundle-skills.mjs +++ b/scripts/bundle-skills.mjs @@ -14,6 +14,7 @@ const skillSources = [{ "references/mastra.md", "references/openai-compatible.md", "references/request-correlation.md", + "references/session-turn-ids.md", "references/vercel-ai-sdk.md", ], }, { diff --git a/skills/setup-understudy/SKILL.md b/skills/setup-understudy/SKILL.md index 42e070c..88e7bab 100644 --- a/skills/setup-understudy/SKILL.md +++ b/skills/setup-understudy/SKILL.md @@ -112,6 +112,8 @@ Produce a compact inventory with: - existing project/workload mapping, if any; - the application's own job/record/conversation IDs, existing durable call logs or storage, and whether each inference attempt retains its Understudy request ID; +- session and logical-turn boundaries, existing IDs with those lifetimes, and + how context reaches continuations, retries, fallbacks and background jobs; - owning process, deployment/environment and runtime configuration source; - a bounded application trigger, its environment and external effects; and - supported, unsupported or unresolved status with the reason. @@ -149,6 +151,8 @@ changing capture, activating traffic or sending live probes, show one compact pl - task purpose, call sites, proposed workload name and boundary for each task; - affected code/configuration, preserved SDK/protocol/models and existing routes; - credential delivery destination, capture choice and any effect on reused traffic; +- session/turn attribution: chosen boundaries, ID sources, propagation changes + and any unresolved paths; - request-ID storage: current coverage, proposed destination and changes, and the user's explicit Yes / No / Do it later decision described below; - local/production scope, requested commits, restart/deploy steps, and restoration; @@ -391,6 +395,21 @@ Preserve model selection, protocol, prompts, tools, schemas, streaming consumers retries, timeouts, cancellation and parsing. Use existing configuration validation rather than adding a new configuration framework. +### Attach session and turn IDs + +Follow [session and turn attribution](references/session-turn-ids.md) alongside +the SDK reference. Strongly recommend `x-understudy-session-id` for grouping an +execution's model calls; online model experiments require it for stable +assignment. Use `x-understudy-turn-id` for a logical step within that session. +A new model request does not automatically mean a new turn. + +Reuse suitable application IDs and propagate them through request-scoped +options, including tool continuations, retries, fallbacks and resumed jobs. +Never mutate a shared client's default headers to switch sessions. If suitable +boundaries or propagation hooks cannot be established, record the gap rather +than inventing new IDs per call or claiming complete attribution. These request +headers are distinct from retaining the gateway's per-request response ID below. + ### Ask whether to keep request IDs with the application's records Inspect the inference wrapper's success, error, streaming and retry paths, then @@ -466,6 +485,10 @@ gateway key fails configuration rather than silently bypassing the gateway. Check the emitted project slug and workload name against the resolved mapping; using the same incorrect ID in both configuration and a test expectation proves only that the header was forwarded. +Use the [session/turn checks](references/session-turn-ids.md#verify-and-report) +to verify stable IDs within a turn, changed IDs at the right boundaries and +isolation across concurrent executions. Sending headers locally does not prove +that the deployed gateway retained them; report hosted verification separately. For approved request-correlation changes, use synthetic transport responses to check success, HTTP error, streaming failure/cancellation, retry and missing-header paths that the application supports. Verify the stored association with the correct app diff --git a/skills/setup-understudy/references/anthropic-messages.md b/skills/setup-understudy/references/anthropic-messages.md index 346eb21..71e9d26 100644 --- a/skills/setup-understudy/references/anthropic-messages.md +++ b/skills/setup-understudy/references/anthropic-messages.md @@ -115,6 +115,10 @@ Do not create a new client for every request solely to change metadata. ## Preserve behavior +Follow [session and turn attribution](session-turn-ids.md) for execution IDs. +Use per-request `headers` in TypeScript or `extra_headers` in Python, preserve +existing headers, and never place changing IDs in shared defaults. + Change only the authentication reference, base URL, and Understudy metadata needed for the approved calls. Preserve: diff --git a/skills/setup-understudy/references/openai-compatible.md b/skills/setup-understudy/references/openai-compatible.md index 666280a..46d4005 100644 --- a/skills/setup-understudy/references/openai-compatible.md +++ b/skills/setup-understudy/references/openai-compatible.md @@ -117,6 +117,10 @@ Do not create a new client for every request solely to change metadata. ## Preserve behavior +Follow [session and turn attribution](session-turn-ids.md) for execution IDs. +Use the same per-request `headers` / `extra_headers` hook, preserve existing +headers, and never place changing session or turn IDs in shared defaults. + Change only the authentication reference, base URL, and Understudy metadata needed for the approved calls. Preserve: diff --git a/skills/setup-understudy/references/session-turn-ids.md b/skills/setup-understudy/references/session-turn-ids.md new file mode 100644 index 0000000..b5366f4 --- /dev/null +++ b/skills/setup-understudy/references/session-turn-ids.md @@ -0,0 +1,105 @@ +# Session and turn IDs during application setup + +Read this alongside the selected SDK integration reference. Connecting a gateway URL does +not identify which requests belong to one task. During setup, discover the +application's execution and turn boundaries, attach their IDs to each relevant +model request, and verify propagation with synthetic local checks. + +## Choose the boundaries + +| Header | Application meaning | Lifetime | +| --- | --- | --- | +| `x-understudy-session-id` | One conversation or task execution, as defined by the application | Reuse across its model calls, tool continuations, retries, and fallback calls. Start a new ID for a new independent execution. | +| `x-understudy-turn-id` | One logical step within that session, such as a user message and all assistant work it triggers | Reuse across that step's model calls and retries. Change when the next logical step begins. | + +A model request or tool-loop iteration is not automatically a new turn. For a +one-shot task, the session has one turn: use the same value for both IDs, or +omit the turn header. Prefer both for applications with multiple turns. +Strongly recommend session IDs for session-level evaluation; online model +experiments require them for stable assignment. + +Reuse existing opaque execution/conversation and message/step IDs when their +lifetimes match these definitions. A user/account ID, workload name, or +process-wide constant is not an execution ID. If suitable IDs do not exist, +allocate them at the application entrypoint and logical-turn boundary, then +carry them through the existing context, queue messages, and retry paths. +Persist them with an existing resumable job when necessary; do not regenerate +them inside the model-call helper or on job redelivery. + +Both IDs are opaque and case-sensitive, must contain non-whitespace text, and +must fit within 256 UTF-8 bytes. Prefer header-safe ASCII IDs such as UUIDs. +Do not truncate, lowercase, or derive IDs from prompts, email addresses, or +other sensitive content. Neither header signals completion or proves an +outcome. They also do not replace the gateway's per-request +`x-understudy-request-id`; retain that separately when the app already records +request correlation. + +Before editing, describe the chosen boundaries, where each ID comes from, +and which call sites will receive them. If the framework cannot propagate +context or the boundary is unresolved, report that gap instead of inventing +IDs per request or claiming complete session attribution. + +## Attach at request scope + +Keep credentials and base URLs on the shared client. Add session and turn +headers through the installed SDK's per-request options, merging existing +request headers. Never mutate shared client defaults to switch sessions: +concurrent executions can otherwise inherit one another's IDs. + +Illustrative TypeScript fragments below use IDs already carried in the +application's context; they do not create IDs inside the inference loop. +Keep the application's existing model, messages, tools, and stream handling. +These fragments assume `existingOptions.headers` is a plain object; if the app +uses `Headers` or header tuples, merge using that representation instead of +object spread. + +```ts +const attributionHeaders = { + "x-understudy-session-id": context.sessionId, + "x-understudy-turn-id": context.turnId, +}; + +// OpenAI SDK: the second argument contains transport options, not model input. +const result = await openai.chat.completions.create( + existingParams, + { ...existingOptions, headers: { ...existingOptions.headers, ...attributionHeaders } }, +); +``` + +```ts +// Anthropic SDK: keep consuming the stream as the application already does. +const stream = anthropic.messages.stream( + existingParams, + { ...existingOptions, headers: { ...existingOptions.headers, ...attributionHeaders } }, +); +``` + +For Vercel AI SDK, use the generation call's `headers` option. For Mastra, +LangChain, LlamaIndex, and other wrappers, inspect the installed version's +supported per-invocation header/context hook and verify that it reaches the +underlying model requests, including tool continuations. Framework memory +thread IDs do not automatically become Understudy headers. Do not invent +framework options or replace a shared client with mutable session defaults. +For raw HTTP, merge into the request's headers using its existing header +representation (`Headers`, tuples, or an object). +For the direct Python OpenAI and Anthropic clients, use `extra_headers` on the +existing model call, merging the application's existing extra headers. + +## Verify and report + +Use a local fake transport or existing integration tests with wholly synthetic +IDs and payloads. Inspect outbound headers and check: + +- two calls in one turn, including a tool continuation or retry, retain both IDs; +- the next turn retains the session ID and changes the turn ID; +- a new independent execution changes the session ID; +- interleaved concurrent executions keep their own IDs; +- background jobs, fallbacks, and resumptions preserve the intended context; +- existing authentication, workload headers, payloads, and stream behavior remain + intact. + +Report the boundary mapping, verified call paths, and any missing propagation. +Local checks prove header construction, not hosted capture persistence. If a +bounded live verification is already authorized, inspect the resulting +metadata through the supported capture surface without printing private +payloads; otherwise leave hosted verification explicitly pending. diff --git a/src/commands/workloads.ts b/src/commands/workloads.ts index 5f95511..66a4330 100644 --- a/src/commands/workloads.ts +++ b/src/commands/workloads.ts @@ -60,7 +60,7 @@ export function addWorkloadsCommands( workloads .command("create ") - .description("Create a workload with capture disabled by default.") + .description("Create a workload with capture disabled by default and show request metadata headers.") .option("--project ", "Exact project id, slug, or name; defaults to saved context.") .option("--project-id ", "Alias for --project.") .option("--capture", "Enable capture for the new workload.") @@ -75,7 +75,7 @@ export function addWorkloadsCommands( name, capture: options.capture === true, }); - dependencies.output(formatWorkloadMutation(this, "Created", result)); + dependencies.output(formatWorkloadCreation(this, result)); }); workloads @@ -212,7 +212,7 @@ function formatWorkloadDetail( function formatWorkloadMutation( command: Command, - verb: "Created" | "Updated", + verb: "Updated", result: WorkloadResult, ): string { return formatOutput( @@ -224,6 +224,43 @@ function formatWorkloadMutation( ); } +function formatWorkloadCreation(command: Command, result: WorkloadResult): string { + const guidance = { + sessionId: "Strongly recommended for session-level evaluation; required for online model experiments. Reuse it across an execution's calls, tool continuations, and retries; use a new ID for each execution.", + turnId: "A turn is one application-defined step within a session, such as one user message and the assistant work it triggers. It can span model calls, tool continuations, and retries. A one-shot session has one turn: use the session ID as its turn ID, or omit the turn ID.", + tags: "App-owned labels such as user_id, account_id, or cohort. Send a flat JSON string map on each relevant call; a tag used to target an experiment must be present on every call in the session.", + }; + const integration = { + headers: { + "x-understudy-project": result.project.slug, + "x-understudy-workload": result.workload.name, + }, + perRequestHeaders: { + sessionId: "x-understudy-session-id", + turnId: "x-understudy-turn-id", + tags: "x-understudy-tags", + }, + guidance, + }; + return formatOutput(command, { ...result, integration }, [ + `Created workload ${result.workload.name} in project ${result.project.slug}. Capture is ${result.workload.captureEnabled ? "on" : "off"}.`, + "Send these headers on each model request:", + ` x-understudy-project: ${result.project.slug}`, + ` x-understudy-workload: ${result.workload.name}`, + "Strongly recommended for session-level evaluation (required for online model experiments):", + " x-understudy-session-id: ", + "For multiple steps within a session, identify each logical turn:", + " x-understudy-turn-id: ", + "Add tags to select and compare the users or cohorts you care about:", + ' x-understudy-tags: {"cohort":"trial"}', + guidance.sessionId, + guidance.turnId, + guidance.tags, + "A new model call does not by itself mean a new turn.", + "Tags select cohorts; the session ID groups calls and keeps an experiment assignment stable.", + ]); +} + function formatRouting(workload: WorkloadResult["workload"]): string { if (workload.routeKind === "none") return "none"; if (workload.routeKind === "deployment") { diff --git a/tests/skills-bundle.test.mjs b/tests/skills-bundle.test.mjs index 0e368b9..c8f37ef 100644 --- a/tests/skills-bundle.test.mjs +++ b/tests/skills-bundle.test.mjs @@ -7,7 +7,7 @@ import test from "node:test"; import { createSkillBundle, writeSkillBundle } from "../scripts/bundle-skills.mjs"; const references = { - "setup-understudy": ["anthropic-messages", "mastra", "openai-compatible", "request-correlation", "vercel-ai-sdk"], + "setup-understudy": ["anthropic-messages", "mastra", "openai-compatible", "request-correlation", "session-turn-ids", "vercel-ai-sdk"], "try-models": ["spot-check-view", "upgrade"], "recommend-models": ["model-facts", "task-cost"], "rollout-workload": [], diff --git a/tests/skills-distribution.test.mjs b/tests/skills-distribution.test.mjs index 966313a..28a5818 100644 --- a/tests/skills-distribution.test.mjs +++ b/tests/skills-distribution.test.mjs @@ -30,6 +30,7 @@ const reviewedFiles = { "references/mastra.md", "references/openai-compatible.md", "references/request-correlation.md", + "references/session-turn-ids.md", "references/vercel-ai-sdk.md", ], "try-models": [ diff --git a/tests/workloads-command.test.mjs b/tests/workloads-command.test.mjs index 52a0b69..7615b49 100644 --- a/tests/workloads-command.test.mjs +++ b/tests/workloads-command.test.mjs @@ -112,6 +112,41 @@ test("workloads create defaults capture off and enables it only with --capture", "--capture", ]); assert.equal(withCapture.calls[0].input.capture, true); + assert.match(withCapture.output[0], /x-understudy-project: synthetic-project/); + assert.match(withCapture.output[0], /x-understudy-workload: captured-workload/); + assert.match(withCapture.output[0], /x-understudy-session-id: /); + assert.match(withCapture.output[0], /Strongly recommended for session-level evaluation/); + assert.match(withCapture.output[0], /required for online model experiments/); + assert.match(withCapture.output[0], /x-understudy-turn-id: /); + assert.match(withCapture.output[0], /x-understudy-tags: \{"cohort":"trial"\}/); + assert.match(withCapture.output[0], /tool continuations, and retries/); + assert.match(withCapture.output[0], /Tags select cohorts; the session ID groups calls/); + assert.match(withCapture.output[0], /A one-shot session has one turn/); + assert.match(withCapture.output[0], /A new model call does not by itself mean a new turn/); +}); + +test("workload creation JSON names per-request metadata without sending it in the create input", async () => { + const { program, calls, output } = programFor(); + await program.parseAsync(["node", "understudy", "--json", "workloads", "create", "new-workload"]); + + assert.deepEqual(calls[0].input, { + project: undefined, + name: "new-workload", + capture: false, + }); + const result = JSON.parse(output[0]); + assert.deepEqual(result.integration.headers, { + "x-understudy-project": project.slug, + "x-understudy-workload": "new-workload", + }); + assert.deepEqual(result.integration.perRequestHeaders, { + sessionId: "x-understudy-session-id", + turnId: "x-understudy-turn-id", + tags: "x-understudy-tags", + }); + assert.match(result.integration.guidance.sessionId, /required for online model experiments/); + assert.match(result.integration.guidance.turnId, /use the session ID as its turn ID, or omit the turn ID/); + assert.match(result.integration.guidance.tags, /every call in the session/); }); test("workloads show supports global JSON output", async () => { From 7491f093810062525df1f1ce3afd97461871823b Mon Sep 17 00:00:00 2001 From: aamir Date: Tue, 20 Oct 2026 11:42:51 -0500 Subject: [PATCH 2/2] Include attribution guide in binary smoke expectations --- tests/binary-smoke.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/binary-smoke.mjs b/tests/binary-smoke.mjs index 476ac0b..25bc720 100644 --- a/tests/binary-smoke.mjs +++ b/tests/binary-smoke.mjs @@ -648,7 +648,7 @@ function assertBundledSkills(binary, environment, scratch) { "compare-models": ["SKILL.md", "references/bundled-comparison.md", "references/comparison-view.md", "references/demo.md", "scripts/compare.mjs", "scripts/present.mjs", "templates/demo-adapter.mjs"], "setup-understudy": [ "SKILL.md", "references/anthropic-messages.md", "references/mastra.md", - "references/openai-compatible.md", "references/request-correlation.md", "references/vercel-ai-sdk.md", + "references/openai-compatible.md", "references/request-correlation.md", "references/session-turn-ids.md", "references/vercel-ai-sdk.md", ], "try-models": ["SKILL.md", "references/spot-check-view.md", "references/upgrade.md"], "recommend-models": ["SKILL.md", "references/model-facts.md", "references/task-cost.md"],