Close frame capture by default and state which frames each turn actually inspected

platform/src/agents/agent-runtime-do.ts

@@ -14,6 +14,8 @@ import { storedExtractionDiagnosticSchema, type StoredExtractionDiagnostic } fro
 import { transcriptDiagnosticSchema, type TranscriptDiagnostic } from './runtime/transcript-diagnostics';
 import { agentDraftSchema, agentRunProgressSchema, toolTrace, type AgentDraft } from './runtime/run-progress';
 import { framePreviewKey, recordedFramePreviews, saveFramePreviews, type SavedFrame } from './runtime/frame-previews';
+import { readFrameRequest } from './research/frame-request';
+import { frameList, isFrameTurn } from './runtime/frame-access';
 import { saveStoryboardPreviews } from './runtime/storyboard-previews';
 import { compactAgentRun, compactAgentResult } from './response';
 import { queuedRunIdentitySchema, type QueuedRunIdentity } from './runtime/admission-queue';
@@ -708,6 +710,7 @@ export class AgentRuntimeDO extends Agent<Env, AgentRuntimeState> {
         },
         onExtractionDiagnostic: event => this.recordExtractionDiagnostic(runId, event),
         onTranscriptDiagnostic: event => this.recordTranscriptDiagnostic(runId, event),
+        startedAt: row.created_at,
         persistedRoute: this.readRoute(runId),
         persistRoute: (selected) => {
           this.persistRoute(runId, selected);
@@ -1038,6 +1041,8 @@ export class AgentRuntimeDO extends Agent<Env, AgentRuntimeState> {
     const history = this.readConversationHistory(run);
     const citedIds=answerCitationIds(parsedInput.answer);
     const citedSessionEvidence=this.sessionStore.evidenceForCitations(citedIds);
+    const requestedSaved = readFrameRequest(run.execution_message ?? run.message).requestsSaved;
+    const needsFrameInventory = requestedSaved || isFrameTurn(decision) || citedSessionEvidence.some(packet => packet.kind === 'youtube_frames');
     const result = buildAgentTurnResult({
       runId,
       conversationId: run.conversation_id,
@@ -1050,7 +1055,9 @@ export class AgentRuntimeDO extends Agent<Env, AgentRuntimeState> {
       requestedVideoIds: decision.route === 'inspect_video' ? [decision.videoId] : decision.comparisonVideoIds ?? [],
       // An application fallback is checked against what this run had, not earlier turns' cited content.
       runPackets: this.admittedRunEvidence(runId, history),
-    } : undefined);
+    } : undefined, needsFrameInventory ? { requestedSaved, access: frameList(decision), startedAt: run.created_at,
+      inventory: this.sessionStore.brief().assets.flatMap(asset => asset.kind === 'frame' ? [{ version: asset.version, videoId: asset.videoId,
+        collectedAt: asset.collectedAt, ...(typeof asset.details.timestampMs === 'number' ? { timestampMs: asset.details.timestampMs } : {}) }] : []) } : undefined);
     // Hash the accepted text: unmatched citations may have been removed from the input.
     const answerHash = createHash('sha256').update(result.answer).digest('hex');
     const serialized = JSON.stringify(result);

platform/src/agents/contracts.ts

@@ -14,13 +14,27 @@ export type VisualEvidence = z.infer<typeof visualEvidenceSchema>;
 export const visualRequirementsSchema = z.array(z.string().trim().min(1).max(200)).min(1).max(8);
 
 const frameTimes = z.array(z.number().int().nonnegative()).max(24);
-/** A fixed capture list stays scoped to one video, including during route recovery. */
-export const frameAccessSchema = z.object({
-  videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/),
+const frameAccessEntrySchema = z.object({
+  /** Absent: capture times for whichever videos discovery selects. */
+  videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/).optional(),
   savedTimestampsMs: frameTimes,
-  captureTimestampsMs: frameTimes.min(1),
+  captureTimestampsMs: frameTimes,
 });
+/** Exact saved and new frame positions for each video in the turn. Model-facing, so no transforms. */
+export const frameAccessListSchema = z.array(frameAccessEntrySchema).min(1).max(8);
+/** A fixed capture list stays scoped to its videos, including during route recovery.
+ * Routes persisted before multi-video access stored one entry as a bare object. */
+export const frameAccessSchema = z.preprocess(
+  value => value && typeof value === 'object' && !Array.isArray(value) ? [value] : value,
+  frameAccessListSchema,
+);
 export type FrameAccess = z.infer<typeof frameAccessSchema>;
+/** Model choices for visual retrieval without a per-video exact-frame list. */
+export const unscopedImageAccessSchema = z.enum(['storyboard_only', 'agent_selected']);
+export const routeFrameAccessSchema = z.union([frameAccessSchema, unscopedImageAccessSchema,
+  z.object({ storyboardTimestampsMs: frameTimes.min(1) }),
+  z.object({ storyboardSelections: z.array(z.object({ videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), timestampsMs: frameTimes.min(1) })).min(1).max(8) })]);
+export type RouteFrameAccess = z.infer<typeof routeFrameAccessSchema>;
 
 /** Evidence the user explicitly asked to be read before answering. Research cannot complete until each is read or unavailable. */
 export const requestedEvidenceSchema = z.array(z.enum(['original_quotes', 'comments'])).min(1).max(2);
@@ -42,7 +56,7 @@ export const capabilityRouteDecisionSchema = z.discriminatedUnion('route', [
     savedOnly: z.boolean().optional(),
     skill: z.enum(['verify_saved_evidence']).optional(),
     requestedEvidence: requestedEvidenceSchema.optional(),
-    frameAccess: frameAccessSchema.optional(),
+    frameAccess: routeFrameAccessSchema.optional(),
     refreshFramesOnly: z.boolean().optional(),
     refreshEvidence: z.boolean().optional(),
     refreshDynamicData: z.boolean().optional(),
@@ -60,7 +74,7 @@ export const capabilityRouteDecisionSchema = z.discriminatedUnion('route', [
     savedOnly: z.boolean().optional(),
     skill: z.enum(['verify_saved_evidence']).optional(),
     requestedEvidence: requestedEvidenceSchema.optional(),
-    frameAccess: frameAccessSchema.optional(),
+    frameAccess: routeFrameAccessSchema.optional(),
     refreshFramesOnly: z.boolean().optional(),
     timestampedFrameQuestion: z.string().trim().min(1).max(500).optional(),
     refreshEvidence: z.boolean().optional(),

platform/src/agents/finalizer.ts

@@ -8,7 +8,9 @@ import {
   type AgentTurnResult,
   type EvidencePacket,
   type FinalizeAnswerInput,
+  type FrameAccess,
 } from './contracts';
+import { frameTurnNotes, type InventoryFrame } from './runtime/frame-access';
 import { appendNotices, playlistCoverageDisclosures } from './playlist-coverage';
 import { CITED_FRAME_PREVIEWS, citedPacketFramePreviews } from './runtime/frame-previews';
 
@@ -28,13 +30,16 @@ export function buildAgentTurnResult(
   packets: EvidencePacket[],
   creditsCharged: number,
   durationLimitContext?: DurationLimitAnswerContext,
+  /** Present when the route asked for frame work: its exact access, and the session's saved frames. */
+  frameTurn?: { requestedSaved?: boolean; access?: FrameAccess; inventory: readonly InventoryFrame[]; startedAt: number },
 ): AgentTurnResult {
   const markers = answerCitationIds(input.answer);
   const citations: AgentCitation[] = [];
   const unresolved = new Set<string>();
   // Which saved previews each cited frame excerpt used, from its own packet. A later answer can
   // hold other captures of the same moment, so time alone never identifies the pixels.
   const citedFrames: Array<{ citationId: string; frames: ReturnType<typeof citedPacketFramePreviews> }> = [];
+  const citedObservations: Parameters<typeof frameTurnNotes>[0]['cited'][number][] = [];
   for (const marker of markers) {
     const matches = packets.flatMap((packet) => packet.excerpts
       .filter((excerpt) => excerpt.id === marker)
@@ -57,6 +62,7 @@ export function buildAgentTurnResult(
       ? citedPacketFramePreviews(packet, excerpt).slice(0, 6) : [];
     // Record a frame citation even without a preview, so it never falls back to matching by time.
     if (source.kind === 'frames') citedFrames.push({ citationId: excerpt.id, frames });
+    if (source.kind === 'frames' && excerpt.startMs !== undefined) citedObservations.push({ packet, startMs: excerpt.startMs, videoId: source.videoId, title: source.title });
     citations.push({
       id: excerpt.id, sourceId: source.id, provider: source.provider,
       videoId: source.videoId, channelId: source.channelId, playlistId: source.playlistId,
@@ -75,7 +81,12 @@ export function buildAgentTurnResult(
 
   // The application states playlist totals, the listing limit and reviewed content itself.
   const playlistCoverage = playlistCoverageDisclosures(packets, citations.map(citation => citation.id));
-  const disclosedAnswer = appendNotices(answer, playlistCoverage.notes);
+  const notes = [...playlistCoverage.notes];
+  // The application, not the model, states which frames this turn actually inspected.
+  if (citedObservations.length || (frameTurn && input.intent !== 'clarification' && input.intent !== 'rejected')) {
+    notes.push(...frameTurnNotes({ runId: identity.runId, startedAt: frameTurn?.startedAt ?? 0, requestedSaved: frameTurn?.requestedSaved, access: frameTurn?.access, inventory: frameTurn?.inventory ?? [], packets, cited: citedObservations }));
+  }
+  const disclosedAnswer = appendNotices(answer, notes);
 
   const warnings = answerWarnings(packets, [
     ...input.warnings,

platform/src/agents/providers/youtube/tool-context.ts

@@ -58,6 +58,7 @@ export interface AgentToolContext {
   refreshEvidence?: boolean;
   refreshFramesOnly?: boolean;
   frameAccess?: FrameAccess;
+  storyboardSelections?: { videoId: string; timestampsMs: number[] }[];
   timestampedFrameQuestion?: string;
   session?: import('../../runtime/session-evidence').SessionAccess;
   assetCache?: RunAssetCache;

platform/src/agents/providers/youtube/tools/get-video-frames.ts

@@ -26,13 +26,14 @@ export const getVideoFramesInputSchema = frameRequestSchema.extend({
 
 export function createGetVideoFramesTool(context: AgentToolContext) {
   const schema = getVideoFramesInputSchema.omit({ focus: true });
-  const scope = context.frameAccess;
+  const captures = context.frameAccess?.filter(entry => entry.captureTimestampsMs.length);
+  const times = captures?.length ? z.array(z.literal([...new Set(captures.flatMap(entry => entry.captureTimestampsMs))])).min(1).max(6) : undefined;
+  const videos = captures?.every(entry => entry.videoId) ? captures.map(entry => entry.videoId!) : undefined;
   return tool({
     description: 'Retrieve or reuse up to six video frames at millisecond timestamps. Supply analysis with your question to analyze immediately while images are saved. Omit analysis for retrieval only. Returns saved assetVersions, dimensions and failures, without image bytes. Use analyze_video_frames for questions about versions already saved. If suitable frames already exist in session inventory, analyze them directly. Still images cannot establish motion or speech.',
-    inputSchema: scope ? schema.extend({
-      videoId: z.literal(scope.videoId),
-      timestampsMs: z.array(z.literal(scope.captureTimestampsMs)).min(1).max(6),
-    }) : schema,
+    // Per-video pairing is enforced at execution; the schema narrows what the model can propose.
+    // An entry without a video fixes only the times, for whichever video discovery selects.
+    inputSchema: !times ? schema : videos ? schema.extend({ videoId: z.literal(videos), timestampsMs: times }) : schema.extend({ timestampsMs: times }),
     outputSchema: evidencePacketSchema,
     // Previews are dashboard images with random IDs; the model analyzes assetVersions instead.
     toModelOutput: ({ output }) => ({ type: 'text', value: JSON.stringify({
@@ -48,8 +49,11 @@ export function createGetVideoFramesTool(context: AgentToolContext) {
 export function executeGetVideoFrames(input: z.input<typeof getVideoFramesInputSchema>, context: AgentToolContext, toolCallId: string) {
   const parsed = getVideoFramesInputSchema.parse(input);
   const scope = context.frameAccess;
-  if (scope && (parsed.videoId !== scope.videoId || parsed.timestampsMs.some(time => !scope.captureTimestampsMs.includes(time)))) {
-    throw new Error(`Capture is limited to ${scope.captureTimestampsMs.join(', ')} ms on ${scope.videoId}. Use saved analysis for saved positions. If it fails, report the gap.`);
+  const allowed = (scope?.find(entry => entry.videoId === parsed.videoId) ?? scope?.find(entry => !entry.videoId))?.captureTimestampsMs ?? [];
+  if (scope && parsed.timestampsMs.some(time => !allowed.includes(time))) {
+    const permitted = scope.filter(entry => entry.captureTimestampsMs.length)
+      .map(entry => `${entry.captureTimestampsMs.join(', ')} ms on ${entry.videoId ?? 'videos found by discovery'}`).join('; ');
+    throw new Error(`Capture is limited to ${permitted}. Use saved analysis for saved positions. If it fails, report the gap.`);
   }
   if (!parsed.analysis && parsed.videoId === context.pinnedVideoId &&
     (!parsed.selections || parsed.selections.every(selection => selection.basis === 'user_requested'))) {

platform/src/agents/providers/youtube/tools/get-video-storyboard.ts

@@ -36,10 +36,19 @@ function validateInput(input: z.infer<typeof retrievalInput>, ctx: z.RefinementC
   if (input.sheetIndexes && input.timestampsMs) ctx.addIssue({ code: 'custom', message: 'Choose sheetIndexes or timestampsMs, not both.' });
 }
 export const getVideoStoryboardInputSchema = retrievalInput.superRefine(validateInput);
+// Apply the same selection before advertising the tool and before any evidence lookup or billing.
+function storyboardInput(context: AgentToolContext) {
+  const selections = context.storyboardSelections;
+  return selections ? retrievalInput.extend({
+    videoId: z.literal(selections.map(item => item.videoId)),
+    timestampsMs: z.array(z.literal([...new Set(selections.flatMap(item => item.timestampsMs))])).min(1).max(MAX_STORYBOARD_SHEETS),
+    sheetIndexes: z.never().optional(),
+  }) : retrievalInput;
+}
 export function createGetVideoStoryboardTool(context: AgentToolContext) {
   return tool({
     description: 'Retrieve storyboard images and metadata. Supply analysis with a visual question to analyze immediately while saving; omit it for retrieval only. With videoId alone, retrieve up to 3 sheets spread across the video. Use maxSheets, sheetIndexes or timestampsMs to select other coverage. No preliminary metadata call is needed. maxSheets gives a spread overview; use transcript findings to choose timestamps for targeted inspection. Returns saved assetVersions and sampled coverage, without image bytes. Use analyze_video_storyboard for questions about versions already in session inventory. Up to 20 sheets and 8 MiB per selection. Metadata alone is not visual evidence.',
-    inputSchema: retrievalInput.omit({focus:true}).superRefine(validateInput),
+    inputSchema: storyboardInput(context).omit({focus:true}).superRefine(validateInput),
     outputSchema: evidencePacketSchema,
     // Previews are dashboard images with random IDs. Hidden from the model, which otherwise
     // mistakes their per-sheet assetId for the analysisAssetVersions analysis needs.
@@ -53,7 +62,11 @@ export function createGetVideoStoryboardTool(context: AgentToolContext) {
   });
 }
 export function executeGetVideoStoryboard(input: z.infer<typeof getVideoStoryboardInputSchema>, context: AgentToolContext, toolCallId: string) {
-  const requested = getVideoStoryboardInputSchema.parse(input);
+  const requested = storyboardInput(context).superRefine(validateInput).parse(input);
+  if (context.storyboardSelections) {
+    const allowed = context.storyboardSelections.find(item => item.videoId === requested.videoId)?.timestampsMs;
+    if (!allowed || !requested.timestampsMs?.every(time => allowed.includes(time))) throw new Error('Storyboard position is outside the grant for this video.');
+  }
   // Only the old metadata-only shape needs a new key. Preserve existing keys
   // for explicit selections, and use schema order for the default overview.
   const parsed = requested.maxSheets === undefined && !requested.sheetIndexes && !requested.timestampsMs
@@ -102,6 +115,7 @@ async function retrieveVideoStoryboard(parsed: z.infer<typeof getVideoStoryboard
   } : undefined, onReady => withRunDeadline(deadlineAt, context.signal, signal => timeStoryboardStage(parsed.videoId, 'retrieval', () => context.provider.storyboard!(parsed.videoId, parsed.timestampsMs, {
     maxSheets: parsed.maxSheets ?? MAX_STORYBOARD_SHEETS, sheetIndexes: parsed.sheetIndexes, metadataOnly: false,
     resolution: requestedResolution(parsed), signal, deadlineAt,
+    ...(!context.session && (context.refreshEvidence || context.refreshFramesOnly) ? { refresh: true } : {}),
   }, event => context.onExtractionDiagnostic?.({ ...event, toolCallId }), ...(onReady ? [onReady] : [])), { runId: context.runId, toolCallId }),
     'Storyboard retrieval exceeded its budget.'), async (response, signal) => {
     const storyboard = storyboardSchema.parse(response.value);
@@ -131,7 +145,7 @@ async function retrieveVideoStoryboard(parsed: z.infer<typeof getVideoStoryboard
     excerpts: [],
     artifacts: [{ type: 'youtube_storyboard_retrieval', title: `Sampled visual evidence for ${parsed.videoId}`,
       data: { timingsMs, analysisAssetVersions: response.assetVersions?.filter(version => context.session?.brief().assets.some(asset => asset.version === version && asset.kind === 'storyboard_sheet')),
-        videoId: parsed.videoId, sessionReused: response.sessionReused === true, selection: storyboard.selection, manifest: storyboard.manifest,
+        videoId: parsed.videoId, requestedTimestampsMs: parsed.timestampsMs, sessionReused: response.sessionReused === true, selection: storyboard.selection, manifest: storyboard.manifest,
         resolution: servedResolution(storyboard),
         sampledRanges: storyboard.sheets.map(sheet => ({ startMs: sheet.firstFrameIndex * sheet.intervalMs,
           endMs: (sheet.firstFrameIndex + sheet.frameCount - 1) * sheet.intervalMs })), totalFrames: storyboard.frameCount,

platform/src/agents/providers/youtube/tools/stored-analysis.ts

@@ -73,8 +73,9 @@ export async function readAnalysisAssets(
       if (scope) {
         // Read the position from the immutable payload, even when older inventory metadata omits it.
         const time = storedFramePosition.parse(stored.value).frames[0]!.timestampMs;
-        const allowed = scope.savedTimestampsMs.includes(time) || (!requiresRetrieval && scope.captureTimestampsMs.includes(time));
-        if (stored.asset.videoId !== scope.videoId || !allowed)
+        const entry = scope.find(item => item.videoId === stored.asset.videoId) ?? scope.find(item => !item.videoId);
+        const allowed = entry && (entry.savedTimestampsMs.includes(time) || (!requiresRetrieval && entry.captureTimestampsMs.includes(time)));
+        if (!allowed)
           throw new Error('This saved version is outside the requested inspection, or its position still requires a fresh capture.');
       }
       return stored;