Restrict visual retrieval to the requested frames and report what was inspected

platform/package.json

@@ -18,7 +18,8 @@
     "test:auth": "vitest run test/auth-config.test.ts test/authentication.test.ts test/cli-auth-boundary.test.ts test/account-route.test.ts",
     "test:container": "npm --prefix youtube-processor test",
     "test:agent:dashboard": "npm --prefix ../web run build -- --webpack && playwright test --config playwright.agent-dashboard.config.ts",
-    "test:agent:visual-replay": "vitest run --config scripts/replays/visual-inspection.config.ts",
+    "test:agent:visual-replay": "vitest run --config scripts/replays/visual-inspection.config.ts visual-inspection.live.test.ts",
+    "test:agent:weather-replay": "vitest run --config scripts/replays/visual-inspection.config.ts weather-map.live.test.ts",
     "test:agent:live": "tsc --noEmit -p tsconfig.agent-live.json && tsx scripts/test-youtube-agent-live.ts",
     "test:user-account": "vitest run --config vitest.user-account.config.ts",
     "types": "wrangler types",

platform/scripts/replays/visual-inspection.config.ts

@@ -4,5 +4,5 @@ import { parseEnv } from 'node:util';
 import { resolve } from 'node:path';
 const vars = process.env.AGENT_REPLAY_ENV_FILE ? parseEnv(readFileSync(process.env.AGENT_REPLAY_ENV_FILE, 'utf8')) : process.env;
 export default defineConfig({ test: { root: import.meta.dirname, environment: 'node', globals: true,
-  include: ['visual-inspection.live.test.ts'], setupFiles: [resolve(import.meta.dirname, '../../test/setup.ts')],
+  include: ['*.live.test.ts'], setupFiles: [resolve(import.meta.dirname, '../../test/setup.ts')],
   testTimeout: 900_000, env: { FIREWORKS_API_KEY: vars.FIREWORKS_API_KEY ?? vars.FIREWORKS_API_KEY_1 ?? '' } } });

platform/scripts/replays/weather-map.live.test.ts

@@ -0,0 +1,79 @@
+// Live language decisions, local image tools. The saved analysis fails deterministically.
+// This fixture contains no private session records or original video images.
+import { expect, it, vi } from 'vitest';
+import { createAgentModel, createClassifierFallbackModel } from '../../src/agents/model';
+import { classifyCapabilityWithModel } from '../../src/agents/research/capability-router';
+import { runResearchAgentWithModel } from '../../src/agents/research/research-agent';
+import { buildAgentTurnResult } from '../../src/agents/finalizer';
+import type { EvidencePacket } from '../../src/agents/contracts';
+import type { AgentToolContext } from '../../src/agents/providers/youtube/tool-context';
+import { attachTestAssetStore } from '../../test/fixtures/analysis-session';
+
+it('replays the 3:30 saved-analysis failure and the separately permitted 3:48 capture', async () => {
+  const videoId = 'abcdefghijk';
+  const message = 'Re-inspect the saved 3:30 weather map. If that fails, report the gap and do not recapture it. ' +
+    'Capture only one new still at 3:48. Read its displayed day/time and any readable temperature or wind numbers with their locations. ' +
+    'Compare the overlays and say whether Ireland becomes more involved, only if the images support that.';
+  const runId = crypto.randomUUID();
+  const signal = AbortSignal.timeout(200_000);
+  const env = { FIREWORKS_API_KEY: process.env.FIREWORKS_API_KEY, AGENT_GLM_PROVIDER: 'fireworks',
+    AGENT_FINALIZER_PROVIDER: 'fireworks', AGENT_FINALIZER_MODEL: 'glm-5p3-flash', AGENT_FINALIZER_REASONING_EFFORT: 'medium',
+    AI_GATEWAY_ID: '' } as unknown as Env;
+  const model = (model_role: string) => createAgentModel(env, runId, 'low', { model_role, agent_run_id: runId });
+  const frame = (timestampMs: number) => ({ videoId, frames: [{ timestampMs, width: 640, height: 360,
+    mimeType: 'image/jpeg', imageBase64: '/9j/2Q==' }], failures: [], meta: { partial: false, warnings: [] } });
+  const frames = vi.fn(async (request: { timestampsMs: number[] }) => {
+    expect(request.timestampsMs).toEqual([228000]);
+    return { value: frame(228000), cacheStatus: 'miss' };
+  });
+  const storyboard = vi.fn(async () => { throw new Error('Unexpected storyboard retrieval'); });
+  const packets: EvidencePacket[] = [];
+  const inspections: number[] = [];
+  const imageRequests: { name: string; input: unknown }[] = [];
+  const context: AgentToolContext = { runId, signal, transcriptPolicy: { mode: 'complete_transcript' }, provider: { frames, storyboard } as unknown as AgentToolContext['provider'],
+    traceToolCall: async call => {
+      if (call.source === 'model' && ['get_video_frames', 'get_video_storyboard'].includes(call.name))
+        imageRequests.push({ name: call.name, input: call.input });
+      return call.execute();
+    },
+    analyzeFrames: async ({ frames }) => {
+      const time = frames.frames[0]!.timestampMs;
+      inspections.push(time);
+      if (time === 210000) throw new Error('Saved image analysis failed.');
+      return { findings: [{ observation: 'The day label reads Tuesday; a 12 appears next to Dublin.', timestampsMs: [time] }], warnings: [] };
+    },
+    executeEvidenceTool: async call => { const result = await call.execute(); packets.push(result); return result; },
+    getEvidence: () => packets, finalize: vi.fn(),
+  };
+  const store = attachTestAssetStore(context);
+  const saved = store.put('frame', videoId, frame(210000), { timestampMs: 210000 });
+  const startedAt = store.runStart();
+  const decision = await classifyCapabilityWithModel({ message, sessionBrief: context.session!.brief(), model: model('classifier'),
+    fallbackModel: createClassifierFallbackModel(env, runId), signal });
+  if (decision.route !== 'inspect_video') throw new Error('Expected bounded inspection: ' + JSON.stringify(decision));
+  expect(decision.frameAccess).toEqual([{ videoId, savedTimestampsMs: [210000], captureTimestampsMs: [228000] }]);
+  if (!Array.isArray(decision.frameAccess)) throw new Error('Missing exact positions');
+  const access = decision.frameAccess;
+  let answer = '';
+  let modelAnswer = '';
+  context.finalize = async (_id, input) => {
+    modelAnswer = input.answer;
+    const result = buildAgentTurnResult({ runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() },
+      { userId: 'replay', creditsRemaining: 100 }, input, packets, 0, undefined,
+      { access, startedAt, inventory: [{ videoId, version: saved, collectedAt: 1, timestampMs: 210000 }] });
+    answer = result.answer;
+    return result;
+  };
+  await runResearchAgentWithModel({ model: model('agent_core'), finalizationModel: model('finalizer'), context, message, decision });
+  process.stdout.write(JSON.stringify({ replay: 'weather-map', inspections, imageRequests, captureRequests: frames.mock.calls.map(([request]) => request.timestampsMs), modelAnswer, answer }) + '\n');
+  expect(imageRequests).toEqual([{ name: 'get_video_frames', input: expect.objectContaining({ videoId, timestampsMs: [228000] }) }]);
+  expect(inspections).toContain(210000);
+  expect(inspections).toContain(228000);
+  expect(frames.mock.calls.map(([request]) => request.timestampsMs)).toEqual([[228000]]);
+  expect(storyboard).not.toHaveBeenCalled();
+  expect(modelAnswer).toMatch(/Tuesday/i);
+  expect(modelAnswer).toContain('3:30');
+  expect(modelAnswer).toMatch(/could(?:n't| not)|unable|cannot|failed|unavailable|couldn’t|wasn.t/i);
+  expect(answer).toContain('I could not re-inspect the saved 3:30 frame');
+  expect(answer).toContain('I retrieved the 3:48 frame and inspected it.');
+});

platform/src/agents/agent-runtime-do.ts

@@ -14,6 +14,7 @@ import { storedExtractionDiagnosticSchema, type StoredExtractionDiagnostic } fro
 import { transcriptDiagnosticSchema, type TranscriptDiagnostic } from './runtime/transcript-diagnostics';
 import { agentDraftSchema, agentRunProgressSchema, toolTrace, type AgentDraft } from './runtime/run-progress';
 import { framePreviewKey, recordedFramePreviews, saveFramePreviews, type SavedFrame } from './runtime/frame-previews';
+import { frameList, isFrameTurn } from './runtime/frame-access';
 import { saveStoryboardPreviews } from './runtime/storyboard-previews';
 import { compactAgentRun, compactAgentResult } from './response';
 import { queuedRunIdentitySchema, type QueuedRunIdentity } from './runtime/admission-queue';
@@ -708,6 +709,7 @@ export class AgentRuntimeDO extends Agent<Env, AgentRuntimeState> {
         },
         onExtractionDiagnostic: event => this.recordExtractionDiagnostic(runId, event),
         onTranscriptDiagnostic: event => this.recordTranscriptDiagnostic(runId, event),
+        startedAt: row.created_at,
         persistedRoute: this.readRoute(runId),
         persistRoute: (selected) => {
           this.persistRoute(runId, selected);
@@ -1038,6 +1040,7 @@ export class AgentRuntimeDO extends Agent<Env, AgentRuntimeState> {
     const history = this.readConversationHistory(run);
     const citedIds=answerCitationIds(parsedInput.answer);
     const citedSessionEvidence=this.sessionStore.evidenceForCitations(citedIds);
+    const needsFrameInventory = isFrameTurn(decision) || citedSessionEvidence.some(packet => packet.kind === 'youtube_frames');
     const result = buildAgentTurnResult({
       runId,
       conversationId: run.conversation_id,
@@ -1050,7 +1053,9 @@ export class AgentRuntimeDO extends Agent<Env, AgentRuntimeState> {
       requestedVideoIds: decision.route === 'inspect_video' ? [decision.videoId] : decision.comparisonVideoIds ?? [],
       // An application fallback is checked against what this run had, not earlier turns' cited content.
       runPackets: this.admittedRunEvidence(runId, history),
-    } : undefined);
+    } : undefined, needsFrameInventory ? { access: frameList(decision), startedAt: run.created_at,
+      inventory: this.sessionStore.brief().assets.flatMap(asset => asset.kind === 'frame' ? [{ version: asset.version, videoId: asset.videoId,
+        collectedAt: asset.collectedAt, ...(typeof asset.details.timestampMs === 'number' ? { timestampMs: asset.details.timestampMs } : {}) }] : []) } : undefined);
     // Hash the accepted text: unmatched citations may have been removed from the input.
     const answerHash = createHash('sha256').update(result.answer).digest('hex');
     const serialized = JSON.stringify(result);

platform/src/agents/contracts.ts

@@ -14,13 +14,31 @@ export type VisualEvidence = z.infer<typeof visualEvidenceSchema>;
 export const visualRequirementsSchema = z.array(z.string().trim().min(1).max(200)).min(1).max(8);
 
 const frameTimes = z.array(z.number().int().nonnegative()).max(24);
-/** A fixed capture list stays scoped to one video, including during route recovery. */
-export const frameAccessSchema = z.object({
-  videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/),
+const frameAccessEntrySchema = z.object({
+  /** Absent: capture times for whichever videos discovery selects. */
+  videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/).optional(),
   savedTimestampsMs: frameTimes,
-  captureTimestampsMs: frameTimes.min(1),
+  captureTimestampsMs: frameTimes,
 });
+/** Exact saved and new frame positions for each video in the turn. Model-facing, so no transforms. */
+export const frameAccessListSchema = z.array(frameAccessEntrySchema).min(1).max(8);
+/** A fixed capture list stays scoped to its videos, including during route recovery.
+ * Routes persisted before multi-video access stored one entry as a bare object. */
+export const frameAccessSchema = z.preprocess(
+  value => value && typeof value === 'object' && !Array.isArray(value) ? [value] : value,
+  frameAccessListSchema,
+);
 export type FrameAccess = z.infer<typeof frameAccessSchema>;
+/** Model choices for visual retrieval without a per-video exact-frame list. */
+export const unscopedImageAccessSchema = z.enum(['storyboard_only', 'agent_selected']);
+const storyboardSelectionsSchema = z.object({
+  storyboardSelections: z.array(z.object({ videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), timestampsMs: frameTimes.min(1) })).min(1).max(8),
+});
+export const classifierFrameAccessSchema = z.union([frameAccessListSchema, unscopedImageAccessSchema, storyboardSelectionsSchema]);
+export const routeFrameAccessSchema = z.union([frameAccessSchema, unscopedImageAccessSchema,
+  // Older single-video routes stored sheet positions without the video ID.
+  z.object({ storyboardTimestampsMs: frameTimes.min(1) }), storyboardSelectionsSchema]);
+export type RouteFrameAccess = z.infer<typeof routeFrameAccessSchema>;
 
 /** Evidence the user explicitly asked to be read before answering. Research cannot complete until each is read or unavailable. */
 export const requestedEvidenceSchema = z.array(z.enum(['original_quotes', 'comments'])).min(1).max(2);
@@ -42,7 +60,7 @@ export const capabilityRouteDecisionSchema = z.discriminatedUnion('route', [
     savedOnly: z.boolean().optional(),
     skill: z.enum(['verify_saved_evidence']).optional(),
     requestedEvidence: requestedEvidenceSchema.optional(),
-    frameAccess: frameAccessSchema.optional(),
+    frameAccess: routeFrameAccessSchema.optional(),
     refreshFramesOnly: z.boolean().optional(),
     refreshEvidence: z.boolean().optional(),
     refreshDynamicData: z.boolean().optional(),
@@ -60,7 +78,7 @@ export const capabilityRouteDecisionSchema = z.discriminatedUnion('route', [
     savedOnly: z.boolean().optional(),
     skill: z.enum(['verify_saved_evidence']).optional(),
     requestedEvidence: requestedEvidenceSchema.optional(),
-    frameAccess: frameAccessSchema.optional(),
+    frameAccess: routeFrameAccessSchema.optional(),
     refreshFramesOnly: z.boolean().optional(),
     timestampedFrameQuestion: z.string().trim().min(1).max(500).optional(),
     refreshEvidence: z.boolean().optional(),

platform/src/agents/finalizer.ts

@@ -8,7 +8,9 @@ import {
   type AgentTurnResult,
   type EvidencePacket,
   type FinalizeAnswerInput,
+  type FrameAccess,
 } from './contracts';
+import { frameTurnNotes, type InventoryFrame } from './runtime/frame-access';
 import { appendNotices, playlistCoverageDisclosures } from './playlist-coverage';
 import { CITED_FRAME_PREVIEWS, citedPacketFramePreviews } from './runtime/frame-previews';
 
@@ -28,13 +30,16 @@ export function buildAgentTurnResult(
   packets: EvidencePacket[],
   creditsCharged: number,
   durationLimitContext?: DurationLimitAnswerContext,
+  /** Present when the route asked for frame work: its exact access, and the session's saved frames. */
+  frameTurn?: { access?: FrameAccess; inventory: readonly InventoryFrame[]; startedAt: number },
 ): AgentTurnResult {
   const markers = answerCitationIds(input.answer);
   const citations: AgentCitation[] = [];
   const unresolved = new Set<string>();
   // Which saved previews each cited frame excerpt used, from its own packet. A later answer can
   // hold other captures of the same moment, so time alone never identifies the pixels.
   const citedFrames: Array<{ citationId: string; frames: ReturnType<typeof citedPacketFramePreviews> }> = [];
+  const citedObservations: Parameters<typeof frameTurnNotes>[0]['cited'][number][] = [];
   for (const marker of markers) {
     const matches = packets.flatMap((packet) => packet.excerpts
       .filter((excerpt) => excerpt.id === marker)
@@ -57,6 +62,7 @@ export function buildAgentTurnResult(
       ? citedPacketFramePreviews(packet, excerpt).slice(0, 6) : [];
     // Record a frame citation even without a preview, so it never falls back to matching by time.
     if (source.kind === 'frames') citedFrames.push({ citationId: excerpt.id, frames });
+    if (source.kind === 'frames' && excerpt.startMs !== undefined) citedObservations.push({ packet, startMs: excerpt.startMs, videoId: source.videoId, title: source.title });
     citations.push({
       id: excerpt.id, sourceId: source.id, provider: source.provider,
       videoId: source.videoId, channelId: source.channelId, playlistId: source.playlistId,
@@ -75,7 +81,12 @@ export function buildAgentTurnResult(
 
   // The application states playlist totals, the listing limit and reviewed content itself.
   const playlistCoverage = playlistCoverageDisclosures(packets, citations.map(citation => citation.id));
-  const disclosedAnswer = appendNotices(answer, playlistCoverage.notes);
+  const notes = [...playlistCoverage.notes];
+  // The application, not the model, states which frames this turn actually inspected.
+  if (citedObservations.length || (frameTurn && input.intent !== 'clarification' && input.intent !== 'rejected')) {
+    notes.push(...frameTurnNotes({ runId: identity.runId, startedAt: frameTurn?.startedAt ?? 0, access: frameTurn?.access, inventory: frameTurn?.inventory ?? [], packets, cited: citedObservations }));
+  }
+  const disclosedAnswer = appendNotices(answer, notes);
 
   const warnings = answerWarnings(packets, [
     ...input.warnings,

platform/src/agents/providers/youtube/tool-context.ts

@@ -58,6 +58,7 @@ export interface AgentToolContext {
   refreshEvidence?: boolean;
   refreshFramesOnly?: boolean;
   frameAccess?: FrameAccess;
+  storyboardSelections?: { videoId: string; timestampsMs: number[] }[];
   timestampedFrameQuestion?: string;
   session?: import('../../runtime/session-evidence').SessionAccess;
   assetCache?: RunAssetCache;

platform/src/agents/providers/youtube/tools/get-video-frames.ts

@@ -26,13 +26,14 @@ export const getVideoFramesInputSchema = frameRequestSchema.extend({
 
 export function createGetVideoFramesTool(context: AgentToolContext) {
   const schema = getVideoFramesInputSchema.omit({ focus: true });
-  const scope = context.frameAccess;
+  const captures = context.frameAccess?.filter(entry => entry.captureTimestampsMs.length);
+  const times = captures?.length ? z.array(z.literal([...new Set(captures.flatMap(entry => entry.captureTimestampsMs))])).min(1).max(6) : undefined;
+  const videos = captures?.every(entry => entry.videoId) ? captures.map(entry => entry.videoId!) : undefined;
   return tool({
     description: 'Retrieve or reuse up to six video frames at millisecond timestamps. Supply analysis with your question to analyze immediately while images are saved. Omit analysis for retrieval only. Returns saved assetVersions, dimensions and failures, without image bytes. Use analyze_video_frames for questions about versions already saved. If suitable frames already exist in session inventory, analyze them directly. Still images cannot establish motion or speech.',
-    inputSchema: scope ? schema.extend({
-      videoId: z.literal(scope.videoId),
-      timestampsMs: z.array(z.literal(scope.captureTimestampsMs)).min(1).max(6),
-    }) : schema,
+    // Per-video pairing is enforced at execution; the schema narrows what the model can propose.
+    // An entry without a video fixes only the times, for whichever video discovery selects.
+    inputSchema: !times ? schema : videos ? schema.extend({ videoId: z.literal(videos), timestampsMs: times }) : schema.extend({ timestampsMs: times }),
     outputSchema: evidencePacketSchema,
     // Previews are dashboard images with random IDs; the model analyzes assetVersions instead.
     toModelOutput: ({ output }) => ({ type: 'text', value: JSON.stringify({
@@ -48,8 +49,11 @@ export function createGetVideoFramesTool(context: AgentToolContext) {
 export function executeGetVideoFrames(input: z.input<typeof getVideoFramesInputSchema>, context: AgentToolContext, toolCallId: string) {
   const parsed = getVideoFramesInputSchema.parse(input);
   const scope = context.frameAccess;
-  if (scope && (parsed.videoId !== scope.videoId || parsed.timestampsMs.some(time => !scope.captureTimestampsMs.includes(time)))) {
-    throw new Error(`Capture is limited to ${scope.captureTimestampsMs.join(', ')} ms on ${scope.videoId}. Use saved analysis for saved positions. If it fails, report the gap.`);
+  const allowed = (scope?.find(entry => entry.videoId === parsed.videoId) ?? scope?.find(entry => !entry.videoId))?.captureTimestampsMs ?? [];
+  if (scope && parsed.timestampsMs.some(time => !allowed.includes(time))) {
+    const permitted = scope.filter(entry => entry.captureTimestampsMs.length)
+      .map(entry => `${entry.captureTimestampsMs.join(', ')} ms on ${entry.videoId ?? 'videos found by discovery'}`).join('; ');
+    throw new Error(`Capture is limited to ${permitted}. Use saved analysis for saved positions. If it fails, report the gap.`);
   }
   if (!parsed.analysis && parsed.videoId === context.pinnedVideoId &&
     (!parsed.selections || parsed.selections.every(selection => selection.basis === 'user_requested'))) {