From 1b312d99a19058bf222b18834c6cd79d0a8e7194 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sat, 7 Nov 2026 17:04:01 +0530 Subject: [PATCH 01/12] Scope frame access per video so cross-video re-inspection is real and reported A saved frame on one video plus a capture on another could not be expressed with single-video frame access. In production the route was scoped to the new video, the saved frame was never re-inspected, and the answer said it had been. Frame access is now one entry per video, every entry must be a route subject (two videos means a comparison), and the activity report covers every video. Also: a refresh with no fixed positions runs instead of asking, required captures are analyzed before completion, app warnings survive the cap, the report is its own uncited paragraph, saved-only conflicts name frameAccess, and saved frame positions are read once per run. --- docs/api-reference/openapi.json | 256 ++++++++++-------- platform/src/agents/contracts.ts | 14 +- .../youtube/tools/get-video-frames.ts | 16 +- .../youtube/tools/stored-analysis.ts | 5 +- .../src/agents/research/capability-router.ts | 42 ++- .../src/agents/research/research-agent.ts | 38 ++- platform/src/agents/runtime/frame-access.ts | 65 +++-- platform/test/youtube-agent-router.test.ts | 67 ++++- platform/test/youtube-stored-analysis.test.ts | 93 ++++++- 9 files changed, 407 insertions(+), 189 deletions(-) diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json index 8cc88d27..c5fc4da7 100644 --- a/docs/api-reference/openapi.json +++ b/docs/api-reference/openapi.json @@ -7661,38 +7661,42 @@ } }, "frameAccess": { - "type": "object", - "properties": { - "videoId": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]{11}$" - }, - "savedTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "savedTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "captureTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } } }, - "captureTimestampsMs": { - "minItems": 1, - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - } - }, - "required": [ - "videoId", - "savedTimestampsMs", - "captureTimestampsMs" - ], - "additionalProperties": false + "required": [ + "videoId", + "savedTimestampsMs", + "captureTimestampsMs" + ], + "additionalProperties": false + } }, "refreshFramesOnly": { "type": "boolean" @@ -7792,38 +7796,42 @@ } }, "frameAccess": { - "type": "object", - "properties": { - "videoId": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]{11}$" - }, - "savedTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "savedTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "captureTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } } }, - "captureTimestampsMs": { - "minItems": 1, - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - } - }, - "required": [ - "videoId", - "savedTimestampsMs", - "captureTimestampsMs" - ], - "additionalProperties": false + "required": [ + "videoId", + "savedTimestampsMs", + "captureTimestampsMs" + ], + "additionalProperties": false + } }, "refreshFramesOnly": { "type": "boolean" @@ -9521,38 +9529,42 @@ } }, "frameAccess": { - "type": "object", - "properties": { - "videoId": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]{11}$" - }, - "savedTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "savedTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "captureTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } } }, - "captureTimestampsMs": { - "minItems": 1, - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - } - }, - "required": [ - "videoId", - "savedTimestampsMs", - "captureTimestampsMs" - ], - "additionalProperties": false + "required": [ + "videoId", + "savedTimestampsMs", + "captureTimestampsMs" + ], + "additionalProperties": false + } }, "refreshFramesOnly": { "type": "boolean" @@ -9652,38 +9664,42 @@ } }, "frameAccess": { - "type": "object", - "properties": { - "videoId": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]{11}$" - }, - "savedTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "savedTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "captureTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } } }, - "captureTimestampsMs": { - "minItems": 1, - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - } - }, - "required": [ - "videoId", - "savedTimestampsMs", - "captureTimestampsMs" - ], - "additionalProperties": false + "required": [ + "videoId", + "savedTimestampsMs", + "captureTimestampsMs" + ], + "additionalProperties": false + } }, "refreshFramesOnly": { "type": "boolean" diff --git a/platform/src/agents/contracts.ts b/platform/src/agents/contracts.ts index fc752798..f92f26ef 100644 --- a/platform/src/agents/contracts.ts +++ b/platform/src/agents/contracts.ts @@ -14,12 +14,20 @@ export type VisualEvidence = z.infer; export const visualRequirementsSchema = z.array(z.string().trim().min(1).max(200)).min(1).max(8); const frameTimes = z.array(z.number().int().nonnegative()).max(24); -/** A fixed capture list stays scoped to one video, including during route recovery. */ -export const frameAccessSchema = z.object({ +const frameAccessEntrySchema = z.object({ videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), savedTimestampsMs: frameTimes, - captureTimestampsMs: frameTimes.min(1), + captureTimestampsMs: frameTimes, }); +/** Exact saved and new frame positions for each video in the turn. Model-facing, so no transforms. */ +export const frameAccessListSchema = z.array(frameAccessEntrySchema).min(1).max(8); +/** A fixed capture list stays scoped to its videos, including during route recovery. + * Routes persisted before multi-video access stored one entry as a bare object. */ +export const frameAccessSchema = z.preprocess( + value => value && typeof value === 'object' && !Array.isArray(value) ? [value] : value, + frameAccessListSchema.refine(entries => entries.some(entry => entry.captureTimestampsMs.length > 0), + 'Frame access authorizes at least one new capture.'), +); export type FrameAccess = z.infer; /** Evidence the user explicitly asked to be read before answering. Research cannot complete until each is read or unavailable. */ diff --git a/platform/src/agents/providers/youtube/tools/get-video-frames.ts b/platform/src/agents/providers/youtube/tools/get-video-frames.ts index a427ce9f..cdd939d2 100644 --- a/platform/src/agents/providers/youtube/tools/get-video-frames.ts +++ b/platform/src/agents/providers/youtube/tools/get-video-frames.ts @@ -26,12 +26,13 @@ export const getVideoFramesInputSchema = frameRequestSchema.extend({ export function createGetVideoFramesTool(context: AgentToolContext) { const schema = getVideoFramesInputSchema.omit({ focus: true }); - const scope = context.frameAccess; + const captures = context.frameAccess?.filter(entry => entry.captureTimestampsMs.length); return tool({ description: 'Retrieve or reuse up to six video frames at millisecond timestamps. Supply analysis with your question to analyze immediately while images are saved. Omit analysis for retrieval only. Returns saved assetVersions, dimensions and failures, without image bytes. Use analyze_video_frames for questions about versions already saved. If suitable frames already exist in session inventory, analyze them directly. Still images cannot establish motion or speech.', - inputSchema: scope ? schema.extend({ - videoId: z.literal(scope.videoId), - timestampsMs: z.array(z.literal(scope.captureTimestampsMs)).min(1).max(6), + // Per-video pairing is enforced at execution; the schema narrows what the model can propose. + inputSchema: captures?.length ? schema.extend({ + videoId: z.literal(captures.map(entry => entry.videoId)), + timestampsMs: z.array(z.literal([...new Set(captures.flatMap(entry => entry.captureTimestampsMs))])).min(1).max(6), }) : schema, outputSchema: evidencePacketSchema, // Previews are dashboard images with random IDs; the model analyzes assetVersions instead. @@ -48,8 +49,11 @@ export function createGetVideoFramesTool(context: AgentToolContext) { export function executeGetVideoFrames(input: z.input, context: AgentToolContext, toolCallId: string) { const parsed = getVideoFramesInputSchema.parse(input); const scope = context.frameAccess; - if (scope && (parsed.videoId !== scope.videoId || parsed.timestampsMs.some(time => !scope.captureTimestampsMs.includes(time)))) { - throw new Error(`Capture is limited to ${scope.captureTimestampsMs.join(', ')} ms on ${scope.videoId}. Use saved analysis for saved positions. If it fails, report the gap.`); + const allowed = scope?.find(entry => entry.videoId === parsed.videoId)?.captureTimestampsMs ?? []; + if (scope && parsed.timestampsMs.some(time => !allowed.includes(time))) { + const permitted = scope.filter(entry => entry.captureTimestampsMs.length) + .map(entry => `${entry.captureTimestampsMs.join(', ')} ms on ${entry.videoId}`).join('; '); + throw new Error(`Capture is limited to ${permitted}. Use saved analysis for saved positions. If it fails, report the gap.`); } if (!parsed.analysis && parsed.videoId === context.pinnedVideoId && (!parsed.selections || parsed.selections.every(selection => selection.basis === 'user_requested'))) { diff --git a/platform/src/agents/providers/youtube/tools/stored-analysis.ts b/platform/src/agents/providers/youtube/tools/stored-analysis.ts index d5bf8afd..a9cc4d61 100644 --- a/platform/src/agents/providers/youtube/tools/stored-analysis.ts +++ b/platform/src/agents/providers/youtube/tools/stored-analysis.ts @@ -73,8 +73,9 @@ export async function readAnalysisAssets( if (scope) { // Read the position from the immutable payload, even when older inventory metadata omits it. const time = storedFramePosition.parse(stored.value).frames[0]!.timestampMs; - const allowed = scope.savedTimestampsMs.includes(time) || (!requiresRetrieval && scope.captureTimestampsMs.includes(time)); - if (stored.asset.videoId !== scope.videoId || !allowed) + const entry = scope.find(item => item.videoId === stored.asset.videoId); + const allowed = entry && (entry.savedTimestampsMs.includes(time) || (!requiresRetrieval && entry.captureTimestampsMs.includes(time))); + if (!allowed) throw new Error('This saved version is outside the requested inspection, or its position still requires a fresh capture.'); } return stored; diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index d55dac2b..c595e0f8 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -8,7 +8,7 @@ import { capabilityRouteDecisionSchema, answerDetailSchema, comparisonVideoIdsSchema, - frameAccessSchema, + frameAccessListSchema, requestedEvidenceSchema, numberedItemCountSchema, visualEvidenceSchema, @@ -30,7 +30,7 @@ const classifierDecisionSchema = z.object({ skill: z.enum(['verify_saved_evidence']).optional().describe('Verify saved evidence: use for follow-ups that read saved original captions/comments, check exact quotations or precise passage links, or correct source claims. It does not by itself prohibit fetching; set savedOnly separately. Omit for broad summaries and visual analysis.'), requestedEvidence: requestedEvidenceSchema.optional().describe('Evidence the user explicitly asks to be read before answering: original_quotes for exact quotations, verbatim wording or precise passage links from captions; comments for what viewers or commenters say. Research must read each or report it unavailable. Omit when not explicitly requested.'), savedOnly: z.boolean().optional().describe('True when the user requires existing saved evidence only or prohibits fresh retrieval. All provider access is disabled for this run; use saved evidence reads and report missing assets.'), - frameAccess: z.union([frameAccessSchema, z.literal('agent_selected')]).optional().describe('For exact captures on one video, list only the permitted NEW capture timestamps in milliseconds. Put saved-only inspection positions separately in savedTimestampsMs, even if their assets are missing. Use agent_selected only for open-ended capture positions.'), + frameAccess: z.union([frameAccessListSchema, z.literal('agent_selected')]).optional().describe('For exact captures, one entry per video the turn touches. captureTimestampsMs lists only the permitted NEW capture positions in milliseconds; savedTimestampsMs lists saved positions to re-inspect without recapture, even if their assets are missing. A video may have only saved positions. Use agent_selected only for open-ended capture positions.'), timestampedFrameQuestion: z.string().trim().min(1).max(500).optional().describe('For inspect_video: when the user supplies exact timestamps and asks what is visible, copy the focused visual question here. Include this for a request to describe each specified frame. Omit for image retrieval only or saved-only requests.'), refreshFramesOnly: z.boolean().optional().describe('For inspection or research, including multi-video comparisons: true when the user asks to freshly capture or refresh frames only. Reuse verified video metadata. Leave refreshEvidence unset unless other source evidence must also be refreshed.'), refreshEvidence: z.boolean().optional().describe('Except for a frames-only refresh, true only when the user explicitly asks to fetch again, refresh or get fresh source data. Choose an executable route in that case.'), @@ -130,7 +130,8 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio // Keep the provider schema strict. Normalize a copy only at the acceptance // boundary, retaining which fields needed a fallback before the SDK parses them. const input = { ...value } as Record; - if (input.frameAccess && typeof input.frameAccess === 'object' && input.refreshFramesOnly !== true) { + // Fixed captures imply a frames refresh. A saved-only conflict is reported against frameAccess below. + if (input.frameAccess && typeof input.frameAccess === 'object' && input.refreshFramesOnly !== true && input.savedOnly !== true) { input.refreshFramesOnly = true; defaultedFields.push('refreshFramesOnly'); } @@ -173,10 +174,18 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio if (input.refreshFramesOnly === true && input.visualEvidence === 'none') issue('visualEvidence', 'Refreshing frames requires visual tool access.'); const scope = parsed.success && parsed.data.frameAccess; if (scope && scope !== 'agent_selected') { - if (!suppliedVideoIds.includes(scope.videoId) || (input.route === 'inspect_video' && input.videoId !== scope.videoId)) - issue('frameAccess', 'Scope frame access to the supplied video being inspected.'); - const times = [...scope.savedTimestampsMs, ...scope.captureTimestampsMs]; - if (new Set(times).size !== times.length) issue('frameAccess', 'Give each timestamp exactly one role, saved inspection or fresh capture.'); + const scoped = scope.map(entry => entry.videoId); + // Every scoped video must be a subject of the route, or its frames cannot be read or captured. + const subjects = input.route === 'inspect_video' ? [input.videoId] + : input.route === 'topic_research' && Array.isArray(input.comparisonVideoIds) ? input.comparisonVideoIds : []; + if (input.savedOnly === true) + issue('frameAccess', 'Saved-only requests cannot capture frames. Omit frameAccess, or omit savedOnly if the user authorized new captures.'); + else if (scoped.some(id => !suppliedVideoIds.includes(id) || !subjects.includes(id)) || new Set(scoped).size !== scoped.length) + issue('frameAccess', 'List one frameAccess entry per video, only for the route\'s subjects. When saved or new frames span several videos, use topic_research with every one in comparisonVideoIds.'); + else if (scope.some(entry => new Set([...entry.savedTimestampsMs, ...entry.captureTimestampsMs]).size !== entry.savedTimestampsMs.length + entry.captureTimestampsMs.length)) + issue('frameAccess', 'Give each timestamp exactly one role, saved inspection or fresh capture.'); + else if (!scope.some(entry => entry.captureTimestampsMs.length)) + issue('frameAccess', 'frameAccess authorizes new captures. Omit it when no new frame is requested, and analyze saved frames directly.'); } if (input.timestampedFrameQuestion && input.route !== 'inspect_video') issue('route', 'Exact frame requests require single-video inspection.'); @@ -289,7 +298,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline 'Select skill verify_saved_evidence for follow-ups that recover saved original captions/comments, verify exact quotations or passage links, or correct source claims. This replaces general discovery and analysis with a verification workflow. The skill does not prohibit fetching: a request that also needs evidence not yet saved, such as comments, still gets it unless savedOnly is set. Omit the skill for broad summaries or visual analysis.', 'Set requestedEvidence for each kind of evidence the user explicitly asks for: original_quotes for exact quotations, verbatim wording or precise passage links; comments for what viewers or commenters say. Include every requested kind in one decision, alongside any explanation. Research cannot complete until each is read or reported unavailable, so choose an executable route.', 'Set savedOnly true whenever the user prohibits fresh retrieval or asks to use existing saved sources only. Keep this restriction on follow-ups when it remains in force. Missing saved evidence is a gap, never permission to fetch. Use the saved reader instead of transcript retrieval or discovery.', - 'When a turn mixes saved and new frames, omit savedOnly and supply frameAccess. For example, re-inspect saved 10:30 and capture only 10:45 means savedTimestampsMs [630000] and captureTimestampsMs [645000]. A prohibition on recapturing 10:30 still allows the requested 10:45. Times mentioned for comparison, narration or history do not authorize capture.', + 'When a turn mixes saved and new frames, omit savedOnly and supply frameAccess with one entry per video. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. A prohibition on recapturing 10:30 still allows the requested 10:45. When the saved frame belongs to one video and the new capture to another, list both videos and choose topic_research with both in comparisonVideoIds, even if the message links only one. Times mentioned for comparison, narration or history do not authorize capture.', 'Requests for exact quotations from original saved captions/comments, precise passage-start links, or explicit original-asset reads require inspect_video or topic_research with the resolved subjects, even if prior summaries discuss the topic. Research can call search_originals without fetching YouTube. Quoting a source is different from quoting a prior conversation message. Do not route an original-source read directly to finalize.', 'For every context_answer set contextScope: history for listing, quoting, recalling or correcting conversation messages or preferences; video for claims about video content; mixed only when the requested answer needs both. A video URL inside a quoted earlier message does not require video evidence.', 'For finalize give a short routing reason, not a user-facing answer. Choose responseIntent clarification for missing scope, or rejected for unsupported requests. Do not use the legacy clarification or rejected routes for new decisions.', @@ -593,13 +602,19 @@ function candidateObject(value: unknown): Record | undefined { return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : undefined; } +/** An explicit prohibition rules out an unrestricted fallback. Only the classifier + * resolves which video and positions the user authorized. */ +const prohibitsRecapture = (message: string) => /\b(?:without|never|do not|don['’]t)\s+re-?captur(?:e|ing)\b/i.test(message); + function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[]): ClassificationIssue[] { - // An explicit prohibition rules out an unrestricted fallback. Only the classifier - // resolves which video and positions the user authorized. - const prohibitsRecapture = /\b(?:without|never|do not|don['’]t)\s+re-?captur(?:e|ing)\b/i.test(input.message); - if (decision.frameAccess === 'agent_selected' && prohibitsRecapture) { + if (decision.frameAccess === 'agent_selected' && prohibitsRecapture(input.message)) { return [{ path: 'frameAccess', code: 'prohibited_recapture', message: 'The request prohibits recapture. Supply the fixed fresh positions and keep the saved inspection positions separate.' }]; } + // A single linked video pins inspection to itself, which would leave another video's frames out of reach. + const linked = extractYouTubeVideoIds(input.message); + if (Array.isArray(decision.frameAccess) && decision.route === 'inspect_video' && linked.length === 1 && linked[0] !== decision.videoId) { + return [{ path: 'frameAccess', code: 'frame_scope_outside_link', message: `The message links ${linked[0]}. Inspect that video, or use topic_research with comparisonVideoIds for every video whose frames this turn uses.` }]; + } if (decision.refreshFramesOnly && !decision.frameAccess && !decision.comparisonVideoIds?.length && (decision.route === 'inspect_video' || videoIds.length === 1)) { return [{ path: 'frameAccess', code: 'missing_capture_scope', message: 'Specify saved and authorized new timestamps separately, or agent_selected if the user delegated the capture positions.' }]; @@ -705,6 +720,9 @@ function lastResortDecision( } if (constraints.refreshFramesOnly && !constraints.savedOnly) { fill('refreshFramesOnly', true); + // No attempt fixed the positions. Without a prohibition the refresh may choose them, + // as it did before scoped access; with one, the checks below ask the user instead. + if (decision.frameAccess === undefined && !prohibitsRecapture(input.message)) fill('frameAccess', 'agent_selected'); if (!decision.visualEvidence || decision.visualEvidence === 'none') fill('visualEvidence', 'helpful'); } if (constraints.refreshEvidence && !constraints.savedOnly) fill('refreshEvidence', true); diff --git a/platform/src/agents/research/research-agent.ts b/platform/src/agents/research/research-agent.ts index 7c2edb38..93d45ab8 100644 --- a/platform/src/agents/research/research-agent.ts +++ b/platform/src/agents/research/research-agent.ts @@ -666,17 +666,19 @@ async function runResearchAgentWithModelWithinDeadline(options: { const requiredVideos = options.decision.comparisonVideoIds?.length ?? (options.decision.route === 'topic_research' ? options.decision.requiredVideoCount : undefined); const warnings = mergeWarnings(input.warnings.filter(warning => warning.code !== 'RESEARCH_COVERAGE_SHORTFALL'), youtubeAvailabilityWarnings([...toolFailures.values()])); - if (frameScope?.savedTimestampsMs.length) { + // Application notes lead the list so the warning cap never drops them. + const notes: AgentWarning[] = []; + if (frameScope?.some(entry => entry.savedTimestampsMs.length)) { // The unified finalizer renders this in prose. Direct and fallback answers retain it as a note. const { report } = frameActivity(frameScope, options.context.runId, evidence.values()); - if (!input.answer.includes(report)) warnings.unshift({ code: 'FRAME_ACTIVITY', message: report.slice(0, 1000) }); + if (!input.answer.includes(report)) notes.push({ code: 'FRAME_ACTIVITY', message: report.slice(0, 1000) }); } if (visualRequired && !hasVisualObservations()) { - warnings.push({ code: 'VISUAL_EVIDENCE_INCOMPLETE', + notes.push({ code: 'VISUAL_EVIDENCE_INCOMPLETE', message: `The request required visual evidence${visualRequirements.length ? ` (${visualRequirements.join('; ')})` : ''}, but no analyzed visual observations were collected. Visual portions of the answer remain unverified.` }); } if (requiredVideos !== undefined && reviewedVideos.size < requiredVideos) { - warnings.push({ code: 'PARTIAL_EVIDENCE', + notes.push({ code: 'PARTIAL_EVIDENCE', message: `The user requested ${requiredVideos} source videos; usable ${visualRequired ? '' : 'transcript '}evidence was reviewed from ${reviewedVideos.size}.` }); } const artifacts = [...input.artifacts.filter(artifact => artifact.type !== 'research_coverage'), { @@ -685,10 +687,10 @@ async function runResearchAgentWithModelWithinDeadline(options: { }]; if (options.decision.route === 'topic_research' && options.decision.channelId && !playlistListing && ![...evidence.values()].some(packet => packet.kind === 'youtube_channel_videos')) { - warnings.push({ code: 'CHANNEL_INSPECTION_INCOMPLETE', + notes.push({ code: 'CHANNEL_INSPECTION_INCOMPLETE', message: 'The requested channel catalog could not be inspected. Do not treat this response as complete channel research.' }); } - const result = await options.context.finalize(id, withDurationLimitNotice({ ...input, warnings: warnings.slice(0, 50), artifacts }, currentDurationNotice())); + const result = await options.context.finalize(id, withDurationLimitNotice({ ...input, warnings: mergeWarnings(notes, warnings), artifacts }, currentDurationNotice())); finalized = true; return result; }, @@ -760,6 +762,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { const visualRequired = visualEvidenceLevel(options.decision) === 'required'; const visualRequirements = options.decision.visualRequirements ?? []; const frameScope = options.decision.frameAccess; + let savedFramePositions: Set | undefined; const hasVisualObservations = () => [...evidence.values()].some(packet => packet.excerpts.length > 0 && packet.artifacts.some(artifact => ['youtube_frame_analysis', 'youtube_storyboard_analysis'].includes(artifact.type))); @@ -769,11 +772,13 @@ async function runResearchAgentWithModelWithinDeadline(options: { const failed = new Set([...toolFailures.values()].map(failure => failure.toolName)); if (frameScope) { const { saved, captures } = frameActivity(frameScope, options.context.runId, evidence.values()); - const available = options.context.session?.brief().assets ?? []; - const needsSavedAnalysis = saved.some(item => !item.inspected && available.some(asset => asset.kind === 'frame' - && asset.videoId === frameScope.videoId && asset.details.timestampMs === item.time)); - const needsCaptureAnalysis = options.decision.route === 'inspect_video' && options.decision.timestampedFrameQuestion - && captures.some(item => item.retrieved && !item.inspected); + // Saved positions predate the run, so read the inventory once rather than on every step. + savedFramePositions ??= new Set((options.context.session?.brief().assets ?? []).flatMap(asset => + asset.kind === 'frame' && typeof asset.details.timestampMs === 'number' ? [`${asset.videoId}:${asset.details.timestampMs}`] : [])); + const needsSavedAnalysis = saved.some(item => !item.inspected && savedFramePositions!.has(`${item.videoId}:${item.time}`)); + // Exact-frame questions, and visual requests with nothing analyzed yet, need the new images read. + const needsCaptureAnalysis = ((options.decision.route === 'inspect_video' && options.decision.timestampedFrameQuestion) + || (visualRequired && !hasVisualObservations())) && captures.some(item => item.retrieved && !item.inspected); return (toolNames.includes('analyze_video_frames') && !!options.context.analyzeFrames && !failed.has('analyze_video_frames') && (needsSavedAnalysis || needsCaptureAnalysis)) || (toolNames.includes('get_video_frames') && !!options.context.provider.frames && !failed.has('get_video_frames') @@ -1097,7 +1102,7 @@ async function runUnifiedFinalizer(options: { .filter(packet => packet.sources.some(source => source.videoId && subjects.has(source.videoId))); const inspectionRecords = inspected.length ? inspectionRecordsForModel(inspected, options.context.evidenceRunRecords?.(inspected.map(packet => packet.packetId))) : undefined; - const frameReport = 'frameAccess' in options.decision && options.decision.frameAccess?.savedTimestampsMs.length + const frameReport = 'frameAccess' in options.decision && options.decision.frameAccess?.some(entry => entry.savedTimestampsMs.length) ? frameActivity(options.decision.frameAccess, options.context.runId, options.evidence).report : undefined; const baseOutputSchema = conversational ? conversationalFinalizationOutputSchema : intent === 'context_answer' ? contextFinalizationOutputSchema : finalizationOutputSchema; @@ -1201,7 +1206,7 @@ async function runUnifiedFinalizer(options: { 'Keep JSON compact. Use short ref_N citations rather than full evidence IDs. For specific-video comparisons cite every subject, or explicitly state the missing side and add ANSWER_SCOPE_SHORTFALL. If contextIncomplete is true, do not claim exhaustive coverage unless the supplied evidence establishes it.', EVIDENCE_DELIVERY_GUIDANCE, INSPECTION_RECORD_GUIDANCE, - ...(frameReport ? ['The application will insert frameActivityReport into the answer. Write the findings and their limitations. Use only the inspections confirmed in that report as current visual evidence. If a saved inspection is missing, disclose what could not be verified. Leave the account of tool actions to the application.'] : []), + ...(frameReport ? ['The application appends frameActivityReport to the answer as its own closing paragraph. Write the findings and their limitations. Use only the inspections confirmed in that report as current visual evidence. If a saved inspection is missing, disclose what could not be verified. Leave the account of tool actions to the application.'] : []), 'Recovery has a limited token budget. Preserve the requested count where evidence permits by shortening each item before reducing the count. If scope remains incomplete, state the shortfall and add ANSWER_SCOPE_SHORTFALL. Do not pad or invent findings.', 'State important evidence gaps plainly. Do not claim that a failed provider operation succeeded.', 'For list evidence such as playlists, take counts from collection: entriesShown are the entries supplied here, and collection.coverage states what was not retained, not shown, or not stated by the provider. Never report fewer available entries than are shown. Describe unshown or unretained entries as a context limit, not as absent from the source. When the answer cites a playlist entry, or reviewed transcript or visual content for a listed video, the application appends a playlist coverage note stating the provider total, the 12-video listing limit and which videos had content reviewed; do not contradict it. Otherwise no note is appended, so state any material playlist coverage limit yourself. Never present listing details as reviewed video content. evidenceOmittedFromContext counts saved list packets that did not fit at all; notListed of them are counted but not named.', @@ -1324,7 +1329,6 @@ async function runUnifiedFinalizer(options: { for (const block of output.blocks) { block.evidenceIds = block.evidenceIds.map(id => prepared.fullIds.get(id) ?? id); } - if (frameReport && output.blocks.length) output.blocks[0]!.text += ` ${frameReport}`; validationStage = 'answer_notes'; // Judge each block by every reference it will render, declared or inline. const references = output.blocks.map(block => blockReferences(block, prepared.fullIds)); @@ -1335,6 +1339,12 @@ async function runUnifiedFinalizer(options: { for (const [index, block] of output.blocks.entries()) block.text = markUnitMismatches(block.text, figures.filter(figure => figure.blockIndex === index)); validationStage = 'rendered_answer'; const input = renderStructuredAnswer({ ...output, intent, artifacts: [] }, prepared.fullIds); + // Its own uncited paragraph, so no finding's citation appears to support the account of tool actions. + if (frameReport) { + const answer = `${input.answer}\n\n${frameReport}`; + if (answer.length <= 20_000) input.answer = answer; + else notes.push({ code: 'FRAME_ACTIVITY', message: frameReport.slice(0, 1000) }); + } input.warnings = mergeWarnings(input.warnings, [...failureWarnings, ...prepared.evidence.flatMap(packet => packet.warnings.filter(warning => warning.code === 'TRANSCRIPT_CONTEXT_TRUNCATED'))]); // A history-only answer carries no source content, so incidental subjects need no citations. diff --git a/platform/src/agents/runtime/frame-access.ts b/platform/src/agents/runtime/frame-access.ts index 2e4a8956..97b9fe17 100644 --- a/platform/src/agents/runtime/frame-access.ts +++ b/platform/src/agents/runtime/frame-access.ts @@ -5,33 +5,44 @@ import { frameCoverageSchema } from './model-evidence'; /** A requested position, a returned image and an analyst result are different facts. * Only packets produced by this run can establish that it did the work. */ export function frameActivity(scope: FrameAccess, runId: string, evidence: Iterable) { - const inspected = new Set(); - const retrieved = new Set(); - const requested = new Set(); - for (const packet of evidence) { - if (!packet.packetId.startsWith(`packet:${runId}:`) || packet.kind !== 'youtube_frames' - || !packet.sources.some(source => source.videoId === scope.videoId)) continue; - // A partial inline analysis may cover fewer positions than its paired retrieval. - for (const artifact of packet.artifacts) { - if (artifact.type !== 'youtube_frame_analysis' && artifact.type !== 'youtube_frame_retrieval') continue; - const coverage = frameCoverageSchema.safeParse(artifact.data); - if (!coverage.success) continue; - const analyzing = artifact.type === 'youtube_frame_analysis'; - if (!analyzing) for (const time of coverage.data.requestedTimestampsMs) requested.add(time); - for (const frame of coverage.data.frames) (analyzing ? inspected : retrieved).add(frame.timestampMs); + const packets = [...evidence]; + const named = scope.length > 1; + const saved: { videoId: string; time: number; inspected: boolean }[] = []; + const captures: { videoId: string; time: number; requested: boolean; retrieved: boolean; inspected: boolean }[] = []; + const sentences: string[] = []; + for (const { videoId, savedTimestampsMs, captureTimestampsMs } of scope) { + const inspected = new Set(); + const retrieved = new Set(); + const requested = new Set(); + for (const packet of packets) { + if (!packet.packetId.startsWith(`packet:${runId}:`) || packet.kind !== 'youtube_frames' + || !packet.sources.some(source => source.videoId === videoId)) continue; + // A partial inline analysis may cover fewer positions than its paired retrieval. + for (const artifact of packet.artifacts) { + if (artifact.type !== 'youtube_frame_analysis' && artifact.type !== 'youtube_frame_retrieval') continue; + const coverage = frameCoverageSchema.safeParse(artifact.data); + if (!coverage.success) continue; + const analyzing = artifact.type === 'youtube_frame_analysis'; + if (!analyzing) for (const time of coverage.data.requestedTimestampsMs) requested.add(time); + for (const frame of coverage.data.frames) (analyzing ? inspected : retrieved).add(frame.timestampMs); + } + } + // With several videos, each position names its video so a gap cannot be read as another video's. + const title = named ? packets.flatMap(packet => packet.sources) + .find(source => source.videoId === videoId && source.title)?.title ?? videoId : undefined; + const frame = (time: number) => `${clockTime(time)} frame${title ? ` of "${title}"` : ''}`; + for (const time of savedTimestampsMs) { + saved.push({ videoId, time, inspected: inspected.has(time) }); + sentences.push(inspected.has(time) ? `I re-inspected the saved ${frame(time)}.` + : `I could not re-inspect the saved ${frame(time)} in this turn.`); + } + for (const time of captureTimestampsMs) { + const item = { videoId, time, requested: requested.has(time), retrieved: retrieved.has(time), inspected: inspected.has(time) }; + captures.push(item); + sentences.push(item.retrieved ? `I retrieved the ${frame(time)}${item.inspected ? ' and inspected it' : ''}.` + : item.requested ? `I requested the ${frame(time)}, but no frame was returned.` + : `I have no confirmed retrieval of the ${frame(time)} in this turn.`); } } - const saved = scope.savedTimestampsMs.map(time => ({ time, inspected: inspected.has(time) })); - const captures = scope.captureTimestampsMs.map(time => ({ - time, requested: requested.has(time), retrieved: retrieved.has(time), inspected: inspected.has(time), - })); - const report = [ - ...saved.map(({ time, inspected }) => inspected ? `I re-inspected the saved ${clockTime(time)} frame.` - : `I could not re-inspect the saved ${clockTime(time)} frame in this turn.`), - ...captures.map(({ time, requested, retrieved, inspected }) => retrieved - ? `I retrieved the ${clockTime(time)} frame${inspected ? ' and inspected it' : ''}.` - : requested ? `I requested ${clockTime(time)}, but no frame was returned.` - : `I have no confirmed retrieval of the ${clockTime(time)} frame in this turn.`), - ].join(' '); - return { saved, captures, report }; + return { saved, captures, report: sentences.join(' ') }; } diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index daaed224..e2834c14 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -6,6 +6,7 @@ import { runResearchAgentWithModel, } from '../src/agents/research/research-agent'; import { INSPECT_VIDEO_TOOL_NAMES } from '../src/agents/research/capabilities/inspect-video'; +import { capabilityRouteDecisionSchema } from '../src/agents/contracts'; import { classifyCapabilityWithModel, extractYouTubeVideoIds, @@ -1093,10 +1094,10 @@ describe('YouTube agent capability router', () => { it.each(['overlap', 'unrestricted'])('repairs an unscoped refresh and rejects %s capture permission', async invalid => { const videoId = 'abcdefghijk'; - const frameAccess = { videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; + const frameAccess = [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]; const base = { route: 'inspect_video', videoId, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['dough shape'] }; const model = sequenceClassifier([base, { ...base, frameAccess: invalid === 'overlap' - ? { ...frameAccess, captureTimestampsMs: [630000, 645000] } : 'agent_selected' }]); + ? [{ ...frameAccess[0]!, captureTimestampsMs: [630000, 645000] }] : 'agent_selected' }]); const fallbackModel = sequenceClassifier([{ ...base, frameAccess }]); const route = await classifyCapabilityWithModel({ model, fallbackModel, message: 'Reuse saved 10:30 without recapturing it and capture only 10:45.', sessionBrief: { memories: [], assets: [{ videoId, version: 'a'.repeat(64), kind: 'frame', current: true, @@ -1106,6 +1107,68 @@ describe('YouTube agent capability router', () => { expect(fallbackModel.doGenerateCalls).toHaveLength(1); }); + it('routes a saved frame on one video and a capture on another as a comparison of both', async () => { + const [saved, linked] = ['abcdefghijk', 'lmnopqrstuv']; + const frameAccess = [{ videoId: saved, savedTimestampsMs: [30000], captureTimestampsMs: [] }, + { videoId: linked, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + const visual = { refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['outfits'] }; + // Each rejected shape left the saved video's frame unreadable in production. + const model = sequenceClassifier([ + { route: 'inspect_video', videoId: linked, ...visual, frameAccess }, + { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'outfits', ...visual, frameAccess }, + ]); + const fallbackModel = sequenceClassifier([{ route: 'topic_research', comparisonVideoIds: [saved, linked], ...visual, frameAccess }]); + const route = await classifyCapabilityWithModel({ model, fallbackModel, + message: `Compare this with https://www.youtube.com/watch?v=${linked}: re-inspect the saved 0:30 frame without recapturing it, and capture only 1:00 of the new video.`, + sessionBrief: { memories: [], assets: [{ videoId: saved, version: 'a'.repeat(64), kind: 'frame', current: true, + collectedAt: 1, details: { timestampMs: 30000 } }] }, signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'topic_research', comparisonVideoIds: [saved, linked], frameAccess }); + expect(fallbackModel.doGenerateCalls).toHaveLength(1); + }); + + it('keeps frame access on the linked video when one video is linked', async () => { + const [saved, linked] = ['abcdefghijk', 'lmnopqrstuv']; + const visual = { refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['outfit'] }; + const model = sequenceClassifier([ + { route: 'inspect_video', videoId: saved, ...visual, frameAccess: [{ videoId: saved, savedTimestampsMs: [], captureTimestampsMs: [45000] }] }, + { route: 'inspect_video', videoId: linked, ...visual, frameAccess: [{ videoId: linked, savedTimestampsMs: [], captureTimestampsMs: [45000] }] }, + ]); + const route = await classifyCapabilityWithModel({ model, message: `Capture only 0:45 of https://youtu.be/${linked}`, + sessionBrief: { memories: [], assets: [{ videoId: saved, version: 'a'.repeat(64), kind: 'frame', current: true, collectedAt: 1, details: { timestampMs: 30000 } }] }, + signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'inspect_video', videoId: linked, frameAccess: [{ videoId: linked }] }); + expect(model.doGenerateCalls).toHaveLength(2); + }); + + it('reads routes persisted with single-video frame access', () => { + const legacy = { videoId: 'abcdefghijk', savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; + expect(capabilityRouteDecisionSchema.parse({ route: 'inspect_video', videoId: legacy.videoId, researchVideoCount: 1, + useStoryboard: true, answerDetail: 'standard', frameAccess: legacy })).toMatchObject({ frameAccess: [legacy] }); + }); + + it.each([ + ['runs an open refresh', 'Refresh the frames of https://youtu.be/abcdefghijk', { route: 'inspect_video', refreshFramesOnly: true }], + ['asks before recapturing against a prohibition', 'Look at https://youtu.be/abcdefghijk again without recapturing 0:30, refresh the rest', + { route: 'finalize', responseIntent: 'clarification' }], + ])('%s when no attempt fixes capture positions', async (_name, message, expected) => { + const model = sequenceClassifier([{ route: 'inspect_video', videoId: 'abcdefghijk', refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'] }]); + const route = await expectLastResort(classifyCapabilityWithModel({ model, message, signal: new AbortController().signal })); + expect(route).toMatchObject(expected); + expect(route).not.toHaveProperty('frameAccess'); + }); + + it('names frameAccess when a saved-only request also asks for captures', async () => { + const videoId = 'abcdefghijk'; + const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [45000] }]; + const model = sequenceClassifier([ + { route: 'inspect_video', videoId, savedOnly: true, visualEvidence: 'required', visualRequirements: ['scene'], frameAccess }, + { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'], frameAccess }, + ]); + const route = await classifyCapabilityWithModel({ model, message: `Capture only 0:45 of https://youtu.be/${videoId}`, signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'inspect_video', frameAccess, refreshFramesOnly: true }); + expect(JSON.stringify(model.doGenerateCalls[1]!.prompt)).toContain('Saved-only requests cannot capture frames'); + }); + it('pins inspect_video provider calls to the classified video', async () => { const video = vi.fn(async () => ({ cacheStatus: 'miss' as const, value: {} as never })); const provider = createCapabilityProvider(providerWith({ video }), { diff --git a/platform/test/youtube-stored-analysis.test.ts b/platform/test/youtube-stored-analysis.test.ts index dbc74bec..fa2da511 100644 --- a/platform/test/youtube-stored-analysis.test.ts +++ b/platform/test/youtube-stored-analysis.test.ts @@ -147,7 +147,7 @@ it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and on toolNames: ['analyze_video_frames', 'get_video_frames', 'finalize_answer'], message: 'Re-inspect the saved 10:30 image and capture only 10:45. Report the gap if reuse fails.', decision: { route: 'inspect_video', videoId, useStoryboard: true, refreshFramesOnly: true, visualEvidence: 'required', - frameAccess: { videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] } }, + frameAccess: [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }] }, }); expect(provider.frames).toHaveBeenCalledOnce(); expect(provider.frames).toHaveBeenCalledWith(expect.objectContaining({ timestampsMs: [645000] }), expect.anything(), @@ -164,9 +164,88 @@ it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and on })); }); +it('re-inspects a saved frame of one video while capturing only the other video', async () => { + const { context, provider, store, frames } = setup(); + const other = 'lmnopqrstuv'; + const at = (id: string, timestampMs: number) => ({ ...frames, videoId: id, frames: [{ ...frames.frames[0]!, timestampMs }] }); + const saved = store.put('frame', videoId, at(videoId, 30000), { timestampMs: 30000 }); + provider.frames.mockResolvedValue({ value: at(other, 60000), cacheStatus: 'miss' }); + context.analyzeFrames = vi.fn(async ({ frames }) => ({ + findings: [{ observation: 'A blue shirt.', timestampsMs: [frames.frames[0]!.timestampMs] }], warnings: [] })); + const calls = [ + { name: 'analyze_video_frames', input: { assetVersions: [saved], focus: 'What is he wearing?' } }, + { name: 'get_video_frames', input: { videoId: other, timestampsMs: [60000], analysis: 'What is he wearing?' } }, + { name: 'complete_research', input: {} }, + ]; + const usage = { inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 1, text: 1, reasoning: 0 } }; + let step = 0; + const model = new MockLanguageModelV4({ doGenerate: async () => { + const call = calls[step++]!; + return { content: [{ type: 'tool-call', toolCallId: `step-${step}`, toolName: call.name, input: JSON.stringify(call.input) }], + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, usage, warnings: [] }; + } }); + const finalizationModel = new MockLanguageModelV4({ doGenerate: async () => ({ + content: [{ type: 'text', text: JSON.stringify({ confidence: 'medium', warnings: [], + blocks: [{ text: 'Both wear shirts.', evidenceIds: ['ref_1'] }] }) }], + finishReason: { unified: 'stop', raw: 'stop' }, usage, warnings: [], + }) }); + await runResearchAgentWithModel({ model, finalizationModel, context, + toolNames: ['analyze_video_frames', 'get_video_frames', 'finalize_answer'], + message: 'Re-inspect the saved 0:30 frame without recapturing it, and capture only 1:00 of the new video.', + decision: { route: 'topic_research', comparisonVideoIds: [videoId, other], researchVideoCount: 2, useStoryboard: true, + refreshFramesOnly: true, visualEvidence: 'required', frameAccess: [ + { videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, + { videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }, + ] }, + }); + expect(provider.frames).toHaveBeenCalledOnce(); + expect(provider.frames).toHaveBeenCalledWith(expect.objectContaining({ videoId: other, timestampsMs: [60000] }), expect.anything(), + expect.anything(), expect.anything(), expect.anything()); + expect(vi.mocked(context.analyzeFrames).mock.calls.map(([input]) => [input.frames.videoId, input.frames.frames[0]!.timestampMs])) + .toEqual([[videoId, 30000], [other, 60000]]); + const answer = vi.mocked(context.finalize).mock.calls.at(-1)![1].answer; + expect(answer).toMatch(/\n\nI re-inspected the saved 0:30 frame of "[^"]+"\. I retrieved the 1:00 frame of "[^"]+" and inspected it\.$/); +}); + +it('keeps research open until a required capture is analyzed, without an exact-frame question', async () => { + const { context, provider, packets, frames } = setup(); + provider.frames.mockResolvedValue({ value: { ...frames, frames: [{ ...frames.frames[0]!, timestampMs: 45000 }] }, cacheStatus: 'miss' }); + context.analyzeFrames = vi.fn(async ({ frames }) => ({ + findings: [{ observation: 'A blue chart.', timestampsMs: [frames.frames[0]!.timestampMs] }], warnings: [] })); + const usage = { inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 1, text: 1, reasoning: 0 } }; + const calls = [ + () => ({ name: 'get_video_frames', input: { videoId, timestampsMs: [45000] } }), + () => ({ name: 'complete_research', input: {} }), + () => ({ name: 'analyze_video_frames', input: { assetVersions: packets.at(-1)!.assetVersions, focus: 'Describe the scene.' } }), + () => ({ name: 'complete_research', input: {} }), + ]; + let step = 0; + const model = new MockLanguageModelV4({ doGenerate: async () => { + const call = calls[step++]!(); + return { content: [{ type: 'tool-call', toolCallId: `step-${step}`, toolName: call.name, input: JSON.stringify(call.input) }], + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, usage, warnings: [] }; + } }); + const finalizationModel = new MockLanguageModelV4({ doGenerate: async () => ({ + content: [{ type: 'text', text: JSON.stringify({ confidence: 'medium', warnings: [], blocks: [{ text: 'A blue chart.', evidenceIds: ['ref_1'] }] }) }], + finishReason: { unified: 'stop', raw: 'stop' }, usage, warnings: [], + }) }); + const warn = vi.spyOn(console, 'warn'); + try { + await runResearchAgentWithModel({ model, finalizationModel, context, + toolNames: ['analyze_video_frames', 'get_video_frames', 'finalize_answer'], message: 'Capture only 0:45 and describe the scene.', + decision: { route: 'inspect_video', videoId, useStoryboard: true, refreshFramesOnly: true, visualEvidence: 'required', + frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [45000] }] }, + }); + expect(warn.mock.calls.filter(([line]) => String(line).includes('complete_research did not execute'))).toHaveLength(1); + } finally { warn.mockRestore(); } + expect(vi.mocked(context.analyzeFrames).mock.calls.map(([input]) => input.frames.frames[0]!.timestampMs)).toEqual([45000]); +}); + it('rejects forbidden positions and other videos before metadata, billing or extraction', () => { const { context, provider } = setup(); - context.frameAccess = { videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; + context.frameAccess = [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]; context.maxVideoSeconds = 7200; context.executeEvidenceTool = vi.fn(); for (const request of [ @@ -174,6 +253,14 @@ it('rejects forbidden positions and other videos before metadata, billing or ext { videoId, timestampsMs: [646000] }, { videoId: 'lmnopqrstuv', timestampsMs: [645000] }, ]) expect(() => executeGetVideoFrames(request, context, 'forbidden')).toThrow('Capture is limited'); + // With several videos, each position is authorized only on its own video. + context.frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [645000] }, + { videoId: 'lmnopqrstuv', savedTimestampsMs: [30000], captureTimestampsMs: [60000] }]; + for (const request of [ + { videoId, timestampsMs: [60000] }, + { videoId: 'lmnopqrstuv', timestampsMs: [645000] }, + { videoId: 'lmnopqrstuv', timestampsMs: [30000] }, + ]) expect(() => executeGetVideoFrames(request, context, 'forbidden')).toThrow('Capture is limited'); expect(context.executeEvidenceTool).not.toHaveBeenCalled(); expect(provider.frames).not.toHaveBeenCalled(); }); @@ -181,7 +268,7 @@ it('rejects forbidden positions and other videos before metadata, billing or ext it('keeps the freshness requirement for a previously saved version at the new capture position', async () => { const { context, store, frames } = setup(); context.refreshFramesOnly = true; - context.frameAccess = { videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; + context.frameAccess = [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]; const stale = store.put('frame', videoId, { ...frames, frames: [{ ...frames.frames[0]!, timestampMs: 645000 }] }); await expect(analyze('frame', stale, context)).rejects.toThrow('requires a fresh capture'); expect(context.analyzeFrames).not.toHaveBeenCalled(); From 2e577a6098f21eacfeb19b829ef6e3cda32efd73 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sat, 7 Nov 2026 20:46:16 +0530 Subject: [PATCH 02/12] Combine frame-scope follow-ups: per-version completion, app-owned activity notes Folds in the per-version completion tracking, finalizer-owned activity report and resume guard from #261, and addresses the review of the previous commit: - A refresh that names positions or forbids recapture now needs a capture list; a fallback asks instead of widening it. Open refreshes no longer need one, so agent_selected is gone. - Completion tracks each frame version: one video's inspection no longer stands in for another's, and saved frames whose inventory omits the position are tried. - buildAgentTurnResult appends the activity report and states when a cited frame observation comes from an earlier turn and was not re-inspected, which holds even when the route omitted that video. --- platform/src/agents/agent-runtime-do.ts | 3 +- platform/src/agents/finalizer.ts | 14 +- .../src/agents/research/capability-router.ts | 50 ++++--- .../src/agents/research/research-agent.ts | 51 ++++--- platform/src/agents/runtime/frame-access.ts | 29 +++- .../youtube-agent-frame-deadlines.test.ts | 5 +- platform/test/youtube-agent-router.test.ts | 37 +++-- platform/test/youtube-stored-analysis.test.ts | 129 +++++++++++++++--- 8 files changed, 241 insertions(+), 77 deletions(-) diff --git a/platform/src/agents/agent-runtime-do.ts b/platform/src/agents/agent-runtime-do.ts index 9e7bac7b..7d7f4bee 100644 --- a/platform/src/agents/agent-runtime-do.ts +++ b/platform/src/agents/agent-runtime-do.ts @@ -1050,7 +1050,8 @@ export class AgentRuntimeDO extends Agent { requestedVideoIds: decision.route === 'inspect_video' ? [decision.videoId] : decision.comparisonVideoIds ?? [], // An application fallback is checked against what this run had, not earlier turns' cited content. runPackets: this.admittedRunEvidence(runId, history), - } : undefined); + } : undefined, (decision.route === 'inspect_video' || decision.route === 'topic_research') + && (decision.frameAccess || decision.refreshFramesOnly) ? { access: decision.frameAccess } : undefined); // Hash the accepted text: unmatched citations may have been removed from the input. const answerHash = createHash('sha256').update(result.answer).digest('hex'); const serialized = JSON.stringify(result); diff --git a/platform/src/agents/finalizer.ts b/platform/src/agents/finalizer.ts index c778af4b..92e10bf0 100644 --- a/platform/src/agents/finalizer.ts +++ b/platform/src/agents/finalizer.ts @@ -8,7 +8,9 @@ import { type AgentTurnResult, type EvidencePacket, type FinalizeAnswerInput, + type FrameAccess, } from './contracts'; +import { earlierFrameNotes, frameActivity } from './runtime/frame-access'; import { appendNotices, playlistCoverageDisclosures } from './playlist-coverage'; import { CITED_FRAME_PREVIEWS, citedPacketFramePreviews } from './runtime/frame-previews'; @@ -28,6 +30,8 @@ export function buildAgentTurnResult( packets: EvidencePacket[], creditsCharged: number, durationLimitContext?: DurationLimitAnswerContext, + /** Present when the route asked for fresh frame work, with any exact access it granted. */ + frameTurn?: { access?: FrameAccess }, ): AgentTurnResult { const markers = answerCitationIds(input.answer); const citations: AgentCitation[] = []; @@ -35,6 +39,7 @@ export function buildAgentTurnResult( // Which saved previews each cited frame excerpt used, from its own packet. A later answer can // hold other captures of the same moment, so time alone never identifies the pixels. const citedFrames: Array<{ citationId: string; frames: ReturnType }> = []; + const citedObservations: Parameters[1][number][] = []; for (const marker of markers) { const matches = packets.flatMap((packet) => packet.excerpts .filter((excerpt) => excerpt.id === marker) @@ -57,6 +62,7 @@ export function buildAgentTurnResult( ? citedPacketFramePreviews(packet, excerpt).slice(0, 6) : []; // Record a frame citation even without a preview, so it never falls back to matching by time. if (source.kind === 'frames') citedFrames.push({ citationId: excerpt.id, frames }); + if (source.kind === 'frames' && excerpt.startMs !== undefined) citedObservations.push({ packet, startMs: excerpt.startMs, videoId: source.videoId, title: source.title }); citations.push({ id: excerpt.id, sourceId: source.id, provider: source.provider, videoId: source.videoId, channelId: source.channelId, playlistId: source.playlistId, @@ -75,7 +81,13 @@ export function buildAgentTurnResult( // The application states playlist totals, the listing limit and reviewed content itself. const playlistCoverage = playlistCoverageDisclosures(packets, citations.map(citation => citation.id)); - const disclosedAnswer = appendNotices(answer, playlistCoverage.notes); + const notes = [...playlistCoverage.notes]; + // The application, not the model, states which frames this turn actually inspected. + if (frameTurn && input.intent !== 'clarification' && input.intent !== 'rejected') { + if (frameTurn.access?.some(entry => entry.savedTimestampsMs.length)) notes.push(frameActivity(frameTurn.access, identity.runId, packets).report); + notes.push(...earlierFrameNotes(identity.runId, citedObservations, packets)); + } + const disclosedAnswer = appendNotices(answer, notes); const warnings = answerWarnings(packets, [ ...input.warnings, diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index c595e0f8..6a18af1a 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -30,7 +30,7 @@ const classifierDecisionSchema = z.object({ skill: z.enum(['verify_saved_evidence']).optional().describe('Verify saved evidence: use for follow-ups that read saved original captions/comments, check exact quotations or precise passage links, or correct source claims. It does not by itself prohibit fetching; set savedOnly separately. Omit for broad summaries and visual analysis.'), requestedEvidence: requestedEvidenceSchema.optional().describe('Evidence the user explicitly asks to be read before answering: original_quotes for exact quotations, verbatim wording or precise passage links from captions; comments for what viewers or commenters say. Research must read each or report it unavailable. Omit when not explicitly requested.'), savedOnly: z.boolean().optional().describe('True when the user requires existing saved evidence only or prohibits fresh retrieval. All provider access is disabled for this run; use saved evidence reads and report missing assets.'), - frameAccess: z.union([frameAccessListSchema, z.literal('agent_selected')]).optional().describe('For exact captures, one entry per video the turn touches. captureTimestampsMs lists only the permitted NEW capture positions in milliseconds; savedTimestampsMs lists saved positions to re-inspect without recapture, even if their assets are missing. A video may have only saved positions. Use agent_selected only for open-ended capture positions.'), + frameAccess: frameAccessListSchema.optional().describe('For exact captures, one entry per video the turn touches. captureTimestampsMs lists only the permitted NEW capture positions in milliseconds; savedTimestampsMs lists saved positions to re-inspect without recapture, even if their assets are missing. A video may have only saved positions. Omit for an open-ended frame refresh where the agent chooses positions.'), timestampedFrameQuestion: z.string().trim().min(1).max(500).optional().describe('For inspect_video: when the user supplies exact timestamps and asks what is visible, copy the focused visual question here. Include this for a request to describe each specified frame. Omit for image retrieval only or saved-only requests.'), refreshFramesOnly: z.boolean().optional().describe('For inspection or research, including multi-video comparisons: true when the user asks to freshly capture or refresh frames only. Reuse verified video metadata. Leave refreshEvidence unset unless other source evidence must also be refreshed.'), refreshEvidence: z.boolean().optional().describe('Except for a frames-only refresh, true only when the user explicitly asks to fetch again, refresh or get fresh source data. Choose an executable route in that case.'), @@ -173,7 +173,7 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio } if (input.refreshFramesOnly === true && input.visualEvidence === 'none') issue('visualEvidence', 'Refreshing frames requires visual tool access.'); const scope = parsed.success && parsed.data.frameAccess; - if (scope && scope !== 'agent_selected') { + if (scope) { const scoped = scope.map(entry => entry.videoId); // Every scoped video must be a subject of the route, or its frames cannot be read or captured. const subjects = input.route === 'inspect_video' ? [input.videoId] @@ -298,7 +298,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline 'Select skill verify_saved_evidence for follow-ups that recover saved original captions/comments, verify exact quotations or passage links, or correct source claims. This replaces general discovery and analysis with a verification workflow. The skill does not prohibit fetching: a request that also needs evidence not yet saved, such as comments, still gets it unless savedOnly is set. Omit the skill for broad summaries or visual analysis.', 'Set requestedEvidence for each kind of evidence the user explicitly asks for: original_quotes for exact quotations, verbatim wording or precise passage links; comments for what viewers or commenters say. Include every requested kind in one decision, alongside any explanation. Research cannot complete until each is read or reported unavailable, so choose an executable route.', 'Set savedOnly true whenever the user prohibits fresh retrieval or asks to use existing saved sources only. Keep this restriction on follow-ups when it remains in force. Missing saved evidence is a gap, never permission to fetch. Use the saved reader instead of transcript retrieval or discovery.', - 'When a turn mixes saved and new frames, omit savedOnly and supply frameAccess with one entry per video. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. A prohibition on recapturing 10:30 still allows the requested 10:45. When the saved frame belongs to one video and the new capture to another, list both videos and choose topic_research with both in comparisonVideoIds, even if the message links only one. Times mentioned for comparison, narration or history do not authorize capture.', + 'When the user names frame positions to capture, or mixes saved and new frames, omit savedOnly and supply frameAccess with one entry per video. Omit it only when the user leaves the capture positions to you. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. A prohibition on recapturing 10:30 still allows the requested 10:45. When the saved frame belongs to one video and the new capture to another, list both videos and choose topic_research with both in comparisonVideoIds, even if the message links only one. Times mentioned for comparison, narration or history do not authorize capture.', 'Requests for exact quotations from original saved captions/comments, precise passage-start links, or explicit original-asset reads require inspect_video or topic_research with the resolved subjects, even if prior summaries discuss the topic. Research can call search_originals without fetching YouTube. Quoting a source is different from quoting a prior conversation message. Do not route an original-source read directly to finalize.', 'For every context_answer set contextScope: history for listing, quoting, recalling or correcting conversation messages or preferences; video for claims about video content; mixed only when the requested answer needs both. A video URL inside a quoted earlier message does not require video evidence.', 'For finalize give a short routing reason, not a user-facing answer. Choose responseIntent clarification for missing scope, or rejected for unsupported requests. Do not use the legacy clarification or rejected routes for new decisions.', @@ -605,19 +605,20 @@ function candidateObject(value: unknown): Record | undefined { /** An explicit prohibition rules out an unrestricted fallback. Only the classifier * resolves which video and positions the user authorized. */ const prohibitsRecapture = (message: string) => /\b(?:without|never|do not|don['’]t)\s+re-?captur(?:e|ing)\b/i.test(message); +const namesFramePositions = (message: string) => /\b(?:\d+:\d{2}|\d+\s*ms\b)/i.test(message); function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[]): ClassificationIssue[] { - if (decision.frameAccess === 'agent_selected' && prohibitsRecapture(input.message)) { - return [{ path: 'frameAccess', code: 'prohibited_recapture', message: 'The request prohibits recapture. Supply the fixed fresh positions and keep the saved inspection positions separate.' }]; - } // A single linked video pins inspection to itself, which would leave another video's frames out of reach. const linked = extractYouTubeVideoIds(input.message); - if (Array.isArray(decision.frameAccess) && decision.route === 'inspect_video' && linked.length === 1 && linked[0] !== decision.videoId) { + if (decision.frameAccess && decision.route === 'inspect_video' && linked.length === 1 && linked[0] !== decision.videoId) { return [{ path: 'frameAccess', code: 'frame_scope_outside_link', message: `The message links ${linked[0]}. Inspect that video, or use topic_research with comparisonVideoIds for every video whose frames this turn uses.` }]; } - if (decision.refreshFramesOnly && !decision.frameAccess && !decision.comparisonVideoIds?.length - && (decision.route === 'inspect_video' || videoIds.length === 1)) { - return [{ path: 'frameAccess', code: 'missing_capture_scope', message: 'Specify saved and authorized new timestamps separately, or agent_selected if the user delegated the capture positions.' }]; + // An open refresh may choose its own positions. Named positions or a recapture prohibition + // need a bounded list, or every fallback would quietly widen what the user allowed. + const unrestricted = decision.route !== 'finalize' && !decision.savedOnly && !decision.frameAccess; + const fixedVideos = decision.route === 'inspect_video' || !!decision.comparisonVideoIds?.length || videoIds.length === 1; + if (unrestricted && (prohibitsRecapture(input.message) || (decision.refreshFramesOnly && fixedVideos && namesFramePositions(input.message)))) { + return [{ path: 'frameAccess', code: 'missing_capture_scope', message: 'The user fixed the frame positions. List the saved inspection positions and the permitted capture positions in frameAccess, one entry per video.' }]; } const scope = comparisonScopeIssues(decision, input, videoIds); return scope.length ? scope : searchQueryNumberIssues(decision, input); @@ -651,10 +652,15 @@ function lastResortDecision( // Requirements any candidate stated, or the request implies, survive every fallback: // fresh data needs an executable route, required visuals stay required, and a // follow-up comparison keeps both of its subjects. + // A bounded list that no attempt stated validly is not widened to an open refresh. + const frameAccess = firstValid('frameAccess'); + if (!frameAccess && candidates.some(candidate => Array.isArray(candidate.frameAccess))) { + const clarification = { route: 'finalize', responseIntent: 'clarification', reason: 'Clarify the saved and permitted fresh frame positions for each video.', answerDetail: 'standard' }; + return { decision: classifierDecisionSchema.parse(clarification), method: 'defaults', + defaultedFields: Object.keys(clarification) as DefaultedClassificationField[] }; + } const constraints = { - // Preserve a bounded list even when a later failed attempt proposed open-ended selection. - frameAccess: firstValid('frameAccess', candidates.filter(candidate => candidate.frameAccess !== 'agent_selected')) - ?? firstValid('frameAccess'), + frameAccess, savedOnly: candidates.some(candidate => candidate.savedOnly === true), requestedEvidence: [...new Set(candidates.flatMap(candidate => requestedEvidenceSchema.safeParse(candidate.requestedEvidence).data ?? []))], refreshFramesOnly: candidates.some(candidate => candidate.refreshFramesOnly === true), @@ -720,9 +726,6 @@ function lastResortDecision( } if (constraints.refreshFramesOnly && !constraints.savedOnly) { fill('refreshFramesOnly', true); - // No attempt fixed the positions. Without a prohibition the refresh may choose them, - // as it did before scoped access; with one, the checks below ask the user instead. - if (decision.frameAccess === undefined && !prohibitsRecapture(input.message)) fill('frameAccess', 'agent_selected'); if (!decision.visualEvidence || decision.visualEvidence === 'none') fill('visualEvidence', 'helpful'); } if (constraints.refreshEvidence && !constraints.savedOnly) fill('refreshEvidence', true); @@ -836,8 +839,7 @@ function expectedComparisonSubjects(input: CapabilityClassifierInput): string[] function finishClassification( decision: z.infer, videoIds: string[], channelIds: string[], explicitVideoIds: string[], message: string, ): CapabilityRouteDecision { - const { explicitSourceCount, frameAccess, ...fields } = withVisualAccess(decision); - const classified = { ...fields, ...(frameAccess && frameAccess !== 'agent_selected' ? { frameAccess } : {}) }; + const { explicitSourceCount, ...classified } = withVisualAccess(decision); const pinned = !(decision.route === 'topic_research' && decision.playlistListing) && explicitVideoIds.length === 1 && !decision.comparisonVideoIds?.length && (decision.route !== 'finalize' || (decision.responseIntent === 'context_answer' && decision.contextScope !== 'history')); const route = pinned ? { ...classified, route: 'inspect_video' as const, videoId: explicitVideoIds[0]!, @@ -975,7 +977,17 @@ export async function resolveCapabilityRoute(options: { classify: () => Promise; persist: (decision: CapabilityRouteDecision) => void | Promise; }): Promise { - if (options.persisted !== undefined) return capabilityRouteDecisionSchema.parse(options.persisted); + if (options.persisted !== undefined) { + const decision = capabilityRouteDecisionSchema.parse(options.persisted); + // Routes persisted before every scoped video had to be a route subject could pin another + // video after validation. Resuming must not run them without the access the user gave. + const subjects = decision.route === 'inspect_video' ? [decision.videoId] + : decision.route === 'topic_research' ? decision.comparisonVideoIds ?? [] : []; + if ('frameAccess' in decision && decision.frameAccess?.some(entry => !subjects.includes(entry.videoId))) { + return { route: 'finalize', responseIntent: 'clarification', reason: 'Clarify which video the saved and fresh frame positions refer to.' }; + } + return decision; + } const decision = capabilityRouteDecisionSchema.parse(await options.classify()); await options.persist(decision); return decision; diff --git a/platform/src/agents/research/research-agent.ts b/platform/src/agents/research/research-agent.ts index 93d45ab8..0835e5be 100644 --- a/platform/src/agents/research/research-agent.ts +++ b/platform/src/agents/research/research-agent.ts @@ -11,7 +11,7 @@ import { durationLimitNotice, withDurationLimitNotice } from './duration-limit-a import { canAnalyzeStoryboard, storyboardRetrievalBudget, STORYBOARD_RETRIEVAL_MIN_MS } from '../runtime/storyboard-budget'; import { transcriptFailureCode, YOUTUBE_UNAVAILABLE_MESSAGE } from '../providers/youtube/tools/transcript-tool-errors'; import { traceToolCallRepair, traceToolSet, type TraceToolCall } from '../runtime/tool-call-trace'; -import { sessionBriefForModel, type SessionEvidenceStore } from '../runtime/session-evidence'; +import { sessionBriefForModel, type SessionBrief, type SessionEvidenceStore } from '../runtime/session-evidence'; import { sessionProvider } from '../runtime/session-provider'; import { createReadPriorEvidenceTool, isHistoryOnlyRoute, preparePriorEvidence, PRIOR_EVIDENCE_GUIDANCE, READ_PRIOR_EVIDENCE_TOOL_NAME, type DeliverEvidence, type PriorEvidenceAccess } from '../runtime/prior-evidence'; @@ -626,6 +626,8 @@ async function runResearchAgentWithModelWithinDeadline(options: { } return pending; }; + const frameScope = options.decision.frameAccess; + const failedFrameVersions = new Set(); const trackedContext: AgentToolContext = { ...options.context, researchDeadlineAt: options.researchDeadlineAt, @@ -668,11 +670,6 @@ async function runResearchAgentWithModelWithinDeadline(options: { youtubeAvailabilityWarnings([...toolFailures.values()])); // Application notes lead the list so the warning cap never drops them. const notes: AgentWarning[] = []; - if (frameScope?.some(entry => entry.savedTimestampsMs.length)) { - // The unified finalizer renders this in prose. Direct and fallback answers retain it as a note. - const { report } = frameActivity(frameScope, options.context.runId, evidence.values()); - if (!input.answer.includes(report)) notes.push({ code: 'FRAME_ACTIVITY', message: report.slice(0, 1000) }); - } if (visualRequired && !hasVisualObservations()) { notes.push({ code: 'VISUAL_EVIDENCE_INCOMPLETE', message: `The request required visual evidence${visualRequirements.length ? ` (${visualRequirements.join('; ')})` : ''}, but no analyzed visual observations were collected. Visual portions of the answer remain unverified.` }); @@ -745,6 +742,10 @@ async function runResearchAgentWithModelWithinDeadline(options: { for (const id of evidence.keys()) if (!retained.has(id)) evidence.delete(id); return packet; } catch (error) { + // A failed saved-frame read is a reported gap, not work to repeat until the deadline. + if (frameScope && execution.toolName === 'analyze_video_frames') { + for (const version of (execution.input as { assetVersions?: string[] }).assetVersions ?? []) failedFrameVersions.add(version); + } toolFailures.set(execution.toolCallId, { toolCallId: execution.toolCallId, toolName: execution.toolName, @@ -761,8 +762,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { }; const visualRequired = visualEvidenceLevel(options.decision) === 'required'; const visualRequirements = options.decision.visualRequirements ?? []; - const frameScope = options.decision.frameAccess; - let savedFramePositions: Set | undefined; + let savedFramesAtStart: SessionBrief['assets'] | undefined; const hasVisualObservations = () => [...evidence.values()].some(packet => packet.excerpts.length > 0 && packet.artifacts.some(artifact => ['youtube_frame_analysis', 'youtube_storyboard_analysis'].includes(artifact.type))); @@ -771,16 +771,19 @@ async function runResearchAgentWithModelWithinDeadline(options: { const needsVisualWork = () => { const failed = new Set([...toolFailures.values()].map(failure => failure.toolName)); if (frameScope) { - const { saved, captures } = frameActivity(frameScope, options.context.runId, evidence.values()); - // Saved positions predate the run, so read the inventory once rather than on every step. - savedFramePositions ??= new Set((options.context.session?.brief().assets ?? []).flatMap(asset => - asset.kind === 'frame' && typeof asset.details.timestampMs === 'number' ? [`${asset.videoId}:${asset.details.timestampMs}`] : [])); - const needsSavedAnalysis = saved.some(item => !item.inspected && savedFramePositions!.has(`${item.videoId}:${item.time}`)); - // Exact-frame questions, and visual requests with nothing analyzed yet, need the new images read. - const needsCaptureAnalysis = ((options.decision.route === 'inspect_video' && options.decision.timestampedFrameQuestion) - || (visualRequired && !hasVisualObservations())) && captures.some(item => item.retrieved && !item.inspected); - return (toolNames.includes('analyze_video_frames') && !!options.context.analyzeFrames && !failed.has('analyze_video_frames') - && (needsSavedAnalysis || needsCaptureAnalysis)) + const { saved, captures, inspectedVersions, pendingCaptureVersions } = frameActivity(frameScope, options.context.runId, evidence.values()); + // Saved frames predate the run, so read the inventory once rather than on every step. + savedFramesAtStart ??= (options.context.session?.brief().assets ?? []).filter(asset => asset.kind === 'frame' + && frameScope.some(entry => entry.videoId === asset.videoId && entry.savedTimestampsMs.length)); + // Each image is tracked by version: one video's inspection never stands in for another's. + const open = (version: string) => !failedFrameVersions.has(version) && options.context.session?.has?.(version) !== false; + // Older inventory can omit the position the payload holds, so such a frame is tried once. + const needsSavedAnalysis = saved.some(item => !item.inspected && savedFramesAtStart!.some(asset => asset.videoId === item.videoId + && (asset.details.timestampMs === item.time || typeof asset.details.timestampMs !== 'number') + && !inspectedVersions.has(asset.version) && open(asset.version))); + const needsCaptureAnalysis = (visualRequired || (options.decision.route === 'inspect_video' && !!options.decision.timestampedFrameQuestion)) + && pendingCaptureVersions.some(open); + return (toolNames.includes('analyze_video_frames') && !!options.context.analyzeFrames && (needsSavedAnalysis || needsCaptureAnalysis)) || (toolNames.includes('get_video_frames') && !!options.context.provider.frames && !failed.has('get_video_frames') && frameExtractionBudget(options.researchDeadlineAt) >= FRAME_EXTRACTION_MIN_MS && captures.some(item => !item.requested)); } @@ -1102,8 +1105,7 @@ async function runUnifiedFinalizer(options: { .filter(packet => packet.sources.some(source => source.videoId && subjects.has(source.videoId))); const inspectionRecords = inspected.length ? inspectionRecordsForModel(inspected, options.context.evidenceRunRecords?.(inspected.map(packet => packet.packetId))) : undefined; - const frameReport = 'frameAccess' in options.decision && options.decision.frameAccess?.some(entry => entry.savedTimestampsMs.length) - ? frameActivity(options.decision.frameAccess, options.context.runId, options.evidence).report : undefined; + const frameReport = 'frameAccess' in options.decision && !!options.decision.frameAccess?.some(entry => entry.savedTimestampsMs.length); const baseOutputSchema = conversational ? conversationalFinalizationOutputSchema : intent === 'context_answer' ? contextFinalizationOutputSchema : finalizationOutputSchema; const numberedItemCount = 'numberedItemCount' in options.decision ? options.decision.numberedItemCount : undefined; @@ -1206,7 +1208,7 @@ async function runUnifiedFinalizer(options: { 'Keep JSON compact. Use short ref_N citations rather than full evidence IDs. For specific-video comparisons cite every subject, or explicitly state the missing side and add ANSWER_SCOPE_SHORTFALL. If contextIncomplete is true, do not claim exhaustive coverage unless the supplied evidence establishes it.', EVIDENCE_DELIVERY_GUIDANCE, INSPECTION_RECORD_GUIDANCE, - ...(frameReport ? ['The application appends frameActivityReport to the answer as its own closing paragraph. Write the findings and their limitations. Use only the inspections confirmed in that report as current visual evidence. If a saved inspection is missing, disclose what could not be verified. Leave the account of tool actions to the application.'] : []), + ...(frameReport ? ['Base visual claims on inspections made in this turn and explain any gap. The application appends the frame activity, and notes any cited observation from an earlier turn, from the recorded operations. Do not describe tool actions yourself.'] : []), 'Recovery has a limited token budget. Preserve the requested count where evidence permits by shortening each item before reducing the count. If scope remains incomplete, state the shortfall and add ANSWER_SCOPE_SHORTFALL. Do not pad or invent findings.', 'State important evidence gaps plainly. Do not claim that a failed provider operation succeeded.', 'For list evidence such as playlists, take counts from collection: entriesShown are the entries supplied here, and collection.coverage states what was not retained, not shown, or not stated by the provider. Never report fewer available entries than are shown. Describe unshown or unretained entries as a context limit, not as absent from the source. When the answer cites a playlist entry, or reviewed transcript or visual content for a listed video, the application appends a playlist coverage note stating the provider total, the 12-video listing limit and which videos had content reviewed; do not contradict it. Otherwise no note is appended, so state any material playlist coverage limit yourself. Never present listing details as reviewed video content. evidenceOmittedFromContext counts saved list packets that did not fit at all; notListed of them are counted but not named.', @@ -1231,7 +1233,6 @@ async function runUnifiedFinalizer(options: { providerFailures: groupedToolFailures(options.toolFailures).map(failure => failure.durationLimit ? { ...failure, message: 'VIDEO_TOO_LONG: The video exceeded the configured Agent duration limit.' } : failure), inspectionRecords, - frameActivityReport: frameReport, validationFeedback: feedback, })}, ...(usesSavedEvidenceSkill(options.decision) ? (options.conversationHistory ?? []).flatMap(turn => [ @@ -1339,12 +1340,6 @@ async function runUnifiedFinalizer(options: { for (const [index, block] of output.blocks.entries()) block.text = markUnitMismatches(block.text, figures.filter(figure => figure.blockIndex === index)); validationStage = 'rendered_answer'; const input = renderStructuredAnswer({ ...output, intent, artifacts: [] }, prepared.fullIds); - // Its own uncited paragraph, so no finding's citation appears to support the account of tool actions. - if (frameReport) { - const answer = `${input.answer}\n\n${frameReport}`; - if (answer.length <= 20_000) input.answer = answer; - else notes.push({ code: 'FRAME_ACTIVITY', message: frameReport.slice(0, 1000) }); - } input.warnings = mergeWarnings(input.warnings, [...failureWarnings, ...prepared.evidence.flatMap(packet => packet.warnings.filter(warning => warning.code === 'TRANSCRIPT_CONTEXT_TRUNCATED'))]); // A history-only answer carries no source content, so incidental subjects need no citations. diff --git a/platform/src/agents/runtime/frame-access.ts b/platform/src/agents/runtime/frame-access.ts index 97b9fe17..4488f1b7 100644 --- a/platform/src/agents/runtime/frame-access.ts +++ b/platform/src/agents/runtime/frame-access.ts @@ -9,6 +9,10 @@ export function frameActivity(scope: FrameAccess, runId: string, evidence: Itera const named = scope.length > 1; const saved: { videoId: string; time: number; inspected: boolean }[] = []; const captures: { videoId: string; time: number; requested: boolean; retrieved: boolean; inspected: boolean }[] = []; + // Versions are tracked separately from positions: inspecting one video's frame + // must never count as inspecting another video's frame at the same time. + const inspectedVersions = new Set(); + const retrievedVersions = new Set(); const sentences: string[] = []; for (const { videoId, savedTimestampsMs, captureTimestampsMs } of scope) { const inspected = new Set(); @@ -23,6 +27,7 @@ export function frameActivity(scope: FrameAccess, runId: string, evidence: Itera const coverage = frameCoverageSchema.safeParse(artifact.data); if (!coverage.success) continue; const analyzing = artifact.type === 'youtube_frame_analysis'; + for (const version of packet.assetVersions ?? []) (analyzing ? inspectedVersions : retrievedVersions).add(version); if (!analyzing) for (const time of coverage.data.requestedTimestampsMs) requested.add(time); for (const frame of coverage.data.frames) (analyzing ? inspected : retrieved).add(frame.timestampMs); } @@ -44,5 +49,27 @@ export function frameActivity(scope: FrameAccess, runId: string, evidence: Itera : `I have no confirmed retrieval of the ${frame(time)} in this turn.`); } } - return { saved, captures, report: sentences.join(' ') }; + return { saved, captures, inspectedVersions, + pendingCaptureVersions: [...retrievedVersions].filter(version => !inspectedVersions.has(version)), report: sentences.join(' ') }; +} + +/** Cited frame observations recorded by an earlier turn, which this turn did not inspect again. + * The route cannot be trusted to list every video the user meant, so the answer states it. */ +export function earlierFrameNotes(runId: string, cited: ReadonlyArray<{ packet: EvidencePacket; startMs: number; videoId?: string; title?: string }>, + packets: readonly EvidencePacket[]): string[] { + const own = (packet: EvidencePacket) => packet.packetId.startsWith(`packet:${runId}:`); + const inspectedNow = new Set(packets.filter(packet => own(packet) && packet.kind === 'youtube_frames').flatMap(packet => + packet.artifacts.flatMap(artifact => { + if (artifact.type !== 'youtube_frame_analysis') return []; + const coverage = frameCoverageSchema.safeParse(artifact.data); + return coverage.success ? packet.sources.flatMap(source => coverage.data.frames.map(frame => `${source.videoId}:${frame.timestampMs}`)) : []; + }))); + const notes = new Map(); + for (const { packet, startMs, videoId, title } of cited) { + const key = `${videoId}:${startMs}`; + if (own(packet) || packet.kind !== 'youtube_frames' || inspectedNow.has(key) || notes.has(key) + || !packet.artifacts.some(artifact => artifact.type === 'youtube_frame_analysis')) continue; + notes.set(key, `The ${clockTime(startMs)} observation${title ? ` of "${title}"` : ''} comes from an earlier turn and was not re-inspected in this one.`); + } + return [...notes.values()]; } diff --git a/platform/test/youtube-agent-frame-deadlines.test.ts b/platform/test/youtube-agent-frame-deadlines.test.ts index 7a3bcccd..f684d7c5 100644 --- a/platform/test/youtube-agent-frame-deadlines.test.ts +++ b/platform/test/youtube-agent-frame-deadlines.test.ts @@ -243,8 +243,11 @@ test('names the required visual facts when finalizing without visual observation const model = new MockLanguageModelV4({ doGenerate: async () => ({ content: [{ type: 'tool-call', toolCallId: 'finish', toolName: 'finalize_answer', input: JSON.stringify({ intent: 'inspect_video', confidence: 'low', artifacts: [], warnings: [], blocks: [{ text: 'Unverified.', evidenceIds: ['ref_1'] }] }) }], finishReason: { unified: 'tool-calls', raw: undefined }, usage, warnings: [] }) }); - await runResearchAgentWithModel({ ...options, model, recoveredEvidence: [metadata], + // A full warning list must not crowd out the application's own visual-evidence note. + const warnings = Array.from({ length: 50 }, (_, index) => ({ code: 'TRANSCRIPT_CONTEXT_TRUNCATED', message: `Omitted passage ${index}.` })); + await runResearchAgentWithModel({ ...options, model, recoveredEvidence: [{ ...metadata, warnings }], recoveredToolFailures: [{ toolCallId: 'frames', toolName: 'get_video_frames', operation: 'frames', message: 'Frame extraction failed' }] }); + expect(vi.mocked(context.finalize).mock.calls[0]![1].warnings).toHaveLength(50); expect(JSON.stringify(model.doGenerateCalls[0]?.prompt)).toContain('Required visual evidence: player names on shirts'); expect(context.finalize).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ warnings: expect.arrayContaining([expect.objectContaining({ code: 'VISUAL_EVIDENCE_INCOMPLETE', message: expect.stringContaining('player names on shirts') })]) })); diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index e2834c14..4744c360 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -1092,12 +1092,12 @@ describe('YouTube agent capability router', () => { expect(decision).toMatchObject({ route: 'inspect_video', videoId, timestampedFrameQuestion: question }); }); - it.each(['overlap', 'unrestricted'])('repairs an unscoped refresh and rejects %s capture permission', async invalid => { + it.each(['overlap', 'omitted'])('repairs an %s capture list before authorizing a bounded refresh', async invalid => { const videoId = 'abcdefghijk'; const frameAccess = [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]; const base = { route: 'inspect_video', videoId, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['dough shape'] }; const model = sequenceClassifier([base, { ...base, frameAccess: invalid === 'overlap' - ? [{ ...frameAccess[0]!, captureTimestampsMs: [630000, 645000] }] : 'agent_selected' }]); + ? [{ ...frameAccess[0]!, captureTimestampsMs: [630000, 645000] }] : undefined }]); const fallbackModel = sequenceClassifier([{ ...base, frameAccess }]); const route = await classifyCapabilityWithModel({ model, fallbackModel, message: 'Reuse saved 10:30 without recapturing it and capture only 10:45.', sessionBrief: { memories: [], assets: [{ videoId, version: 'a'.repeat(64), kind: 'frame', current: true, @@ -1146,15 +1146,36 @@ describe('YouTube agent capability router', () => { useStoryboard: true, answerDetail: 'standard', frameAccess: legacy })).toMatchObject({ frameAccess: [legacy] }); }); + it('runs an open refresh without a capture list on the first attempt', async () => { + const model = sequenceClassifier([{ route: 'inspect_video', videoId: 'abcdefghijk', refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'] }]); + const route = await classifyCapabilityWithModel({ model, message: 'Refresh the frames of https://youtu.be/abcdefghijk', signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'inspect_video', refreshFramesOnly: true }); + expect(route).not.toHaveProperty('frameAccess'); + expect(model.doGenerateCalls).toHaveLength(1); + }); + it.each([ - ['runs an open refresh', 'Refresh the frames of https://youtu.be/abcdefghijk', { route: 'inspect_video', refreshFramesOnly: true }], - ['asks before recapturing against a prohibition', 'Look at https://youtu.be/abcdefghijk again without recapturing 0:30, refresh the rest', - { route: 'finalize', responseIntent: 'clarification' }], - ])('%s when no attempt fixes capture positions', async (_name, message, expected) => { + ['a named position', 'Capture only 1:00 of https://youtu.be/abcdefghijk. No other timestamps.'], + ['a recapture prohibition', 'Look at https://youtu.be/abcdefghijk again without recapturing the saved frame, refresh the rest'], + ])('asks rather than widen %s when no attempt lists the positions', async (_name, message) => { const model = sequenceClassifier([{ route: 'inspect_video', videoId: 'abcdefghijk', refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'] }]); const route = await expectLastResort(classifyCapabilityWithModel({ model, message, signal: new AbortController().signal })); - expect(route).toMatchObject(expected); - expect(route).not.toHaveProperty('frameAccess'); + expect(route).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it('asks rather than widen a capture list that no attempt stated validly', async () => { + const videoId = 'abcdefghijk'; + const model = sequenceClassifier([{ route: 'inspect_video', videoId, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'], + frameAccess: [{ videoId, savedTimestampsMs: [60000], captureTimestampsMs: [60000] }] }]); + const route = await expectLastResort(classifyCapabilityWithModel({ model, message: `Refresh the frames of https://youtu.be/${videoId}`, signal: new AbortController().signal })); + expect(route).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it('keeps frame access outside the persisted route closed on resume', async () => { + const frameAccess = { videoId: 'lmnopqrstuv', savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; + const decision = await resolveCapabilityRoute({ persisted: { route: 'inspect_video', videoId: 'abcdefghijk', researchVideoCount: 1, + useStoryboard: true, answerDetail: 'standard', frameAccess }, classify: vi.fn(), persist: vi.fn() }); + expect(decision).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); }); it('names frameAccess when a saved-only request also asks for captures', async () => { diff --git a/platform/test/youtube-stored-analysis.test.ts b/platform/test/youtube-stored-analysis.test.ts index fa2da511..b1296c17 100644 --- a/platform/test/youtube-stored-analysis.test.ts +++ b/platform/test/youtube-stored-analysis.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it, vi } from 'vitest'; +import { buildAgentTurnResult } from '../src/agents/finalizer'; import { MockLanguageModelV4 } from 'ai/test'; import { runResearchAgentWithModel } from '../src/agents/research/research-agent'; import { z } from 'zod'; import type { AgentToolContext } from '../src/agents/providers/youtube/tool-context'; -import { evidencePacketSchema, type EvidencePacket } from '../src/agents/contracts'; +import { evidencePacketSchema, type EvidencePacket, type FinalizeAnswerInput, type FrameAccess } from '../src/agents/contracts'; import { createCapabilityToolSet } from '../src/agents/providers/youtube/tool-library'; import { executeGetVideoTranscriptForModel } from '../src/agents/providers/youtube/tools/get-video-transcript'; import { executeGetVideoFrames } from '../src/agents/providers/youtube/tools/get-video-frames'; @@ -114,8 +115,20 @@ function setup() { } const kinds = ['transcript', 'frame', 'storyboard_sheet'] as const; +/** Finalizes through the application's answer builder, which owns the frame activity notes. */ +function finalizeThroughApplication(context: AgentToolContext, packets: EvidencePacket[], access: FrameAccess, earlier: EvidencePacket[] = []) { + context.runId = crypto.randomUUID(); + const identity = { runId: context.runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }; + const finalize = vi.fn(async (_id: string, input: FinalizeAnswerInput) => buildAgentTurnResult(identity, { userId: 'user', creditsRemaining: 100 }, + input, [...earlier, ...packets], 0, undefined, { access })); + context.finalize = finalize; + return async () => (await finalize.mock.results.at(-1)!.value).answer as string; +} + it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and only fetches 10:45 (saved image %s)', async savedState => { - const { context, provider, store, frames } = setup(); + const { context, provider, store, frames, packets } = setup(); + const frameAccess = [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]; + const answer = finalizeThroughApplication(context, packets, frameAccess); const at = (timestampMs: number) => ({ ...frames, frames: [{ ...frames.frames[0]!, timestampMs }] }); const saved = store.put('frame', videoId, at(630000), { timestampMs: 630000 }); if (savedState === 'missing') store.remove(saved); @@ -146,8 +159,7 @@ it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and on await runResearchAgentWithModel({ model, finalizationModel, context, toolNames: ['analyze_video_frames', 'get_video_frames', 'finalize_answer'], message: 'Re-inspect the saved 10:30 image and capture only 10:45. Report the gap if reuse fails.', - decision: { route: 'inspect_video', videoId, useStoryboard: true, refreshFramesOnly: true, visualEvidence: 'required', - frameAccess: [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }] }, + decision: { route: 'inspect_video', videoId, useStoryboard: true, refreshFramesOnly: true, visualEvidence: 'required', frameAccess }, }); expect(provider.frames).toHaveBeenCalledOnce(); expect(provider.frames).toHaveBeenCalledWith(expect.objectContaining({ timestampsMs: [645000] }), expect.anything(), @@ -155,18 +167,19 @@ it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and on expect(vi.mocked(context.analyzeFrames).mock.calls.map(([input]) => input.frames.frames[0]!.timestampMs)) .toEqual(savedState === 'missing' ? [645000] : [630000, 645000]); const savedReport = savedState === 'available' ? 'I re-inspected the saved 10:30 frame' : 'I could not re-inspect the saved 10:30 frame'; - expect(JSON.stringify(finalizationModel.doGenerateCalls[0]!.prompt)).toContain(savedReport); - expect(context.finalize).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ - answer: expect.stringContaining(savedReport), - })); - expect(context.finalize).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ - answer: expect.stringContaining('I retrieved the 10:45 frame and inspected it.'), - })); + // Findings keep their citations; the account of tool actions is a separate, uncited paragraph. + const [findings, activity] = (await answer()).split('\n\n'); + expect(findings).toContain('[cite:'); + expect(activity).toContain(savedReport); + expect(activity).toContain('I retrieved the 10:45 frame and inspected it.'); + expect(activity).not.toContain('[cite:'); }); it('re-inspects a saved frame of one video while capturing only the other video', async () => { - const { context, provider, store, frames } = setup(); + const { context, provider, store, frames, packets } = setup(); const other = 'lmnopqrstuv'; + const frameAccess = [{ videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, { videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + const answer = finalizeThroughApplication(context, packets, frameAccess); const at = (id: string, timestampMs: number) => ({ ...frames, videoId: id, frames: [{ ...frames.frames[0]!, timestampMs }] }); const saved = store.put('frame', videoId, at(videoId, 30000), { timestampMs: 30000 }); provider.frames.mockResolvedValue({ value: at(other, 60000), cacheStatus: 'miss' }); @@ -194,18 +207,14 @@ it('re-inspects a saved frame of one video while capturing only the other video' toolNames: ['analyze_video_frames', 'get_video_frames', 'finalize_answer'], message: 'Re-inspect the saved 0:30 frame without recapturing it, and capture only 1:00 of the new video.', decision: { route: 'topic_research', comparisonVideoIds: [videoId, other], researchVideoCount: 2, useStoryboard: true, - refreshFramesOnly: true, visualEvidence: 'required', frameAccess: [ - { videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, - { videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }, - ] }, + refreshFramesOnly: true, visualEvidence: 'required', frameAccess }, }); expect(provider.frames).toHaveBeenCalledOnce(); expect(provider.frames).toHaveBeenCalledWith(expect.objectContaining({ videoId: other, timestampsMs: [60000] }), expect.anything(), expect.anything(), expect.anything(), expect.anything()); expect(vi.mocked(context.analyzeFrames).mock.calls.map(([input]) => [input.frames.videoId, input.frames.frames[0]!.timestampMs])) .toEqual([[videoId, 30000], [other, 60000]]); - const answer = vi.mocked(context.finalize).mock.calls.at(-1)![1].answer; - expect(answer).toMatch(/\n\nI re-inspected the saved 0:30 frame of "[^"]+"\. I retrieved the 1:00 frame of "[^"]+" and inspected it\.$/); + expect(await answer()).toMatch(/\n\nI re-inspected the saved 0:30 frame of "[^"]+"\. I retrieved the 1:00 frame of "[^"]+" and inspected it\.$/); }); it('keeps research open until a required capture is analyzed, without an exact-frame question', async () => { @@ -243,6 +252,90 @@ it('keeps research open until a required capture is analyzed, without an exact-f expect(vi.mocked(context.analyzeFrames).mock.calls.map(([input]) => input.frames.frames[0]!.timestampMs)).toEqual([45000]); }); +/** Runs scripted research steps, then keeps asking to complete. Returns how often completion was refused. */ +async function refusedCompletions(context: AgentToolContext, steps: Array<() => { name: string; input: unknown }>, + decision: Parameters[0]['decision']) { + const usage = { inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, outputTokens: { total: 1, text: 1, reasoning: 0 } }; + let step = 0; + const model = new MockLanguageModelV4({ doGenerate: async () => { + const call = steps[step++]?.() ?? { name: 'complete_research', input: {} }; + return { content: [{ type: 'tool-call', toolCallId: `step-${step}`, toolName: call.name, input: JSON.stringify(call.input) }], + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, usage, warnings: [] }; + } }); + const finalizationModel = new MockLanguageModelV4({ doGenerate: async () => ({ + content: [{ type: 'text', text: JSON.stringify({ confidence: 'medium', warnings: [], blocks: [{ text: 'Both wear shirts.', evidenceIds: ['ref_1'] }] }) }], + finishReason: { unified: 'stop', raw: 'stop' }, usage, warnings: [] }) }); + const warn = vi.spyOn(console, 'warn'); + try { + await runResearchAgentWithModel({ model, finalizationModel, context, decision, + toolNames: ['analyze_video_frames', 'get_video_frames', 'finalize_answer'], message: 'Compare the saved 0:30 frame with a new 1:00 capture.' }); + return warn.mock.calls.filter(([line]) => String(line).includes('complete_research did not execute')).length; + } finally { warn.mockRestore(); } +} + +describe('cross-video frame completion', () => { + const other = 'lmnopqrstuv'; + const frameAccess = [{ videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, { videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + const decision = { route: 'topic_research' as const, comparisonVideoIds: [videoId, other], researchVideoCount: 2, useStoryboard: true, + refreshFramesOnly: true, visualEvidence: 'required' as const, frameAccess }; + function scene(legacy = false) { + const fixture = setup(); + const at = (id: string, timestampMs: number) => ({ ...fixture.frames, videoId: id, frames: [{ ...fixture.frames.frames[0]!, timestampMs }] }); + // Older inventory rows can omit the position that the stored payload still holds. + const saved = fixture.store.put('frame', videoId, at(videoId, 30000), legacy ? undefined : { timestampMs: 30000 }); + fixture.provider.frames.mockResolvedValue({ value: at(other, 60000), cacheStatus: 'miss' }); + fixture.context.analyzeFrames = vi.fn(async ({ frames }) => ({ + findings: [{ observation: 'A shirt.', timestampsMs: [frames.frames[0]!.timestampMs] }], warnings: [] })); + const analyzed = () => vi.mocked(fixture.context.analyzeFrames!).mock.calls.map(([input]) => input.frames.videoId); + return { ...fixture, saved, analyzed }; + } + + it('keeps research open for the other video after one video is inspected', async () => { + const { context, packets, saved, analyzed } = scene(); + const refused = await refusedCompletions(context, [ + () => ({ name: 'analyze_video_frames', input: { assetVersions: [saved], focus: 'What is he wearing?' } }), + () => ({ name: 'get_video_frames', input: { videoId: other, timestampsMs: [60000] } }), + () => ({ name: 'complete_research', input: {} }), + () => ({ name: 'analyze_video_frames', input: { assetVersions: packets.at(-1)!.assetVersions, focus: 'What is he wearing?' } }), + ], decision); + expect(refused).toBe(1); + expect(analyzed()).toEqual([videoId, other]); + }); + + it('tries a saved frame whose inventory omits its position before completing', async () => { + const { context, saved, analyzed } = scene(true); + const refused = await refusedCompletions(context, [ + () => ({ name: 'get_video_frames', input: { videoId: other, timestampsMs: [60000], analysis: 'What is he wearing?' } }), + () => ({ name: 'complete_research', input: {} }), + () => ({ name: 'analyze_video_frames', input: { assetVersions: [saved], focus: 'What is he wearing?' } }), + ], decision); + expect(refused).toBe(1); + expect(analyzed()).toEqual([other, videoId]); + }); +}); + +it('notes a cited observation from an earlier turn when the route omitted its video', async () => { + const { context, store, frames } = setup(); + const other = 'lmnopqrstuv'; + const at = (id: string, timestampMs: number) => ({ ...frames, videoId: id, frames: [{ ...frames.frames[0]!, timestampMs }] }); + context.analyzeFrames = vi.fn(async ({ frames }) => ({ + findings: [{ observation: 'A shirt.', timestampsMs: [frames.frames[0]!.timestampMs] }], warnings: [] })); + // An earlier turn inspected the first video at 0:30. + context.runId = crypto.randomUUID(); + const earlier = await analyze('frame', store.put('frame', videoId, at(videoId, 30000), { timestampMs: 30000 }), context, 'What is he wearing?'); + // This turn's route listed only the second video, as in the production incident. + context.runId = crypto.randomUUID(); + context.frameAccess = [{ videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + const current = await analyze('frame', store.put('frame', other, at(other, 60000), { timestampMs: 60000 }), context, 'What is he wearing?'); + const result = buildAgentTurnResult({ runId: context.runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), + agentMessageId: crypto.randomUUID() }, { userId: 'user', creditsRemaining: 100 }, { + answer: `He re-checked the saved frame: a shirt [cite:${earlier.excerpts[0]!.id}]. The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, + intent: 'topic_research', confidence: 'medium', citations: [], artifacts: [], warnings: [], + }, [earlier, current], 0, undefined, { access: context.frameAccess }); + expect(result.answer).toMatch(/\n\nThe 0:30 observation( of "[^"]+")? comes from an earlier turn and was not re-inspected in this one\.$/); + expect(result.answer).not.toContain('The 1:00 observation'); +}); + it('rejects forbidden positions and other videos before metadata, billing or extraction', () => { const { context, provider } = setup(); context.frameAccess = [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]; From 55958eab4fec6010e77e7c39f966e6d66e70e93d Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sat, 7 Nov 2026 21:15:07 +0530 Subject: [PATCH 03/12] Close remaining frame-scope gaps from review - Exact-frame questions need a capture list, not only refreshes, and positions given as 'at/only N seconds' count as named positions. - A saved frame at a time the user names must appear in frameAccess; saved-only access lists are allowed and offer no capture tool. - The answer builder corrects an uncited claim of re-inspection when this turn re-inspected no saved frame, and matches earlier observations by asset version. - A failed capture closes only its own positions; other authorized captures stay required before completion. --- platform/src/agents/contracts.ts | 3 +- platform/src/agents/finalizer.ts | 4 +- .../src/agents/research/capability-router.ts | 40 ++++++--- .../src/agents/research/research-agent.ts | 14 ++- platform/src/agents/runtime/frame-access.ts | 39 +++++--- platform/test/youtube-agent-router.test.ts | 37 +++++++- platform/test/youtube-stored-analysis.test.ts | 88 +++++++++++++++---- 7 files changed, 177 insertions(+), 48 deletions(-) diff --git a/platform/src/agents/contracts.ts b/platform/src/agents/contracts.ts index f92f26ef..9fd895c8 100644 --- a/platform/src/agents/contracts.ts +++ b/platform/src/agents/contracts.ts @@ -25,8 +25,7 @@ export const frameAccessListSchema = z.array(frameAccessEntrySchema).min(1).max( * Routes persisted before multi-video access stored one entry as a bare object. */ export const frameAccessSchema = z.preprocess( value => value && typeof value === 'object' && !Array.isArray(value) ? [value] : value, - frameAccessListSchema.refine(entries => entries.some(entry => entry.captureTimestampsMs.length > 0), - 'Frame access authorizes at least one new capture.'), + frameAccessListSchema, ); export type FrameAccess = z.infer; diff --git a/platform/src/agents/finalizer.ts b/platform/src/agents/finalizer.ts index 92e10bf0..e7733405 100644 --- a/platform/src/agents/finalizer.ts +++ b/platform/src/agents/finalizer.ts @@ -10,7 +10,7 @@ import { type FinalizeAnswerInput, type FrameAccess, } from './contracts'; -import { earlierFrameNotes, frameActivity } from './runtime/frame-access'; +import { earlierFrameNotes, frameActivity, reinspectionClaimNote } from './runtime/frame-access'; import { appendNotices, playlistCoverageDisclosures } from './playlist-coverage'; import { CITED_FRAME_PREVIEWS, citedPacketFramePreviews } from './runtime/frame-previews'; @@ -86,6 +86,8 @@ export function buildAgentTurnResult( if (frameTurn && input.intent !== 'clarification' && input.intent !== 'rejected') { if (frameTurn.access?.some(entry => entry.savedTimestampsMs.length)) notes.push(frameActivity(frameTurn.access, identity.runId, packets).report); notes.push(...earlierFrameNotes(identity.runId, citedObservations, packets)); + const claim = reinspectionClaimNote(identity.runId, input.answer, packets); + if (claim) notes.push(claim); } const disclosedAnswer = appendNotices(answer, notes); diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index 6a18af1a..6de9a6e1 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -18,6 +18,7 @@ import { type FinalizeAnswerInput, } from '../contracts'; import { sessionBriefForModel, type SessionBrief } from '../runtime/session-evidence'; +import { clockTime } from '../runtime/clock-time'; import { conversationAssistantMessage, type ConversationTurn } from '../runtime/conversation-memory'; import { assertModelCostAvailable, type AgentModelCostBudget } from '../runtime/model-budget'; import { AGENT_CLASSIFICATION_TIMEOUT_MS, withRunDeadline } from '../runtime/deadline'; @@ -30,7 +31,7 @@ const classifierDecisionSchema = z.object({ skill: z.enum(['verify_saved_evidence']).optional().describe('Verify saved evidence: use for follow-ups that read saved original captions/comments, check exact quotations or precise passage links, or correct source claims. It does not by itself prohibit fetching; set savedOnly separately. Omit for broad summaries and visual analysis.'), requestedEvidence: requestedEvidenceSchema.optional().describe('Evidence the user explicitly asks to be read before answering: original_quotes for exact quotations, verbatim wording or precise passage links from captions; comments for what viewers or commenters say. Research must read each or report it unavailable. Omit when not explicitly requested.'), savedOnly: z.boolean().optional().describe('True when the user requires existing saved evidence only or prohibits fresh retrieval. All provider access is disabled for this run; use saved evidence reads and report missing assets.'), - frameAccess: frameAccessListSchema.optional().describe('For exact captures, one entry per video the turn touches. captureTimestampsMs lists only the permitted NEW capture positions in milliseconds; savedTimestampsMs lists saved positions to re-inspect without recapture, even if their assets are missing. A video may have only saved positions. Omit for an open-ended frame refresh where the agent chooses positions.'), + frameAccess: frameAccessListSchema.optional().describe('For frame positions the user fixes, saved or new: one entry per video the turn touches. captureTimestampsMs lists only the permitted NEW capture positions in milliseconds, empty when nothing may be captured; savedTimestampsMs lists saved positions to re-inspect without recapture, even if their assets are missing. Required for exact-timestamp questions. Omit only for an open-ended frame refresh where the agent chooses positions.'), timestampedFrameQuestion: z.string().trim().min(1).max(500).optional().describe('For inspect_video: when the user supplies exact timestamps and asks what is visible, copy the focused visual question here. Include this for a request to describe each specified frame. Omit for image retrieval only or saved-only requests.'), refreshFramesOnly: z.boolean().optional().describe('For inspection or research, including multi-video comparisons: true when the user asks to freshly capture or refresh frames only. Reuse verified video metadata. Leave refreshEvidence unset unless other source evidence must also be refreshed.'), refreshEvidence: z.boolean().optional().describe('Except for a frames-only refresh, true only when the user explicitly asks to fetch again, refresh or get fresh source data. Choose an executable route in that case.'), @@ -131,7 +132,9 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio // boundary, retaining which fields needed a fallback before the SDK parses them. const input = { ...value } as Record; // Fixed captures imply a frames refresh. A saved-only conflict is reported against frameAccess below. - if (input.frameAccess && typeof input.frameAccess === 'object' && input.refreshFramesOnly !== true && input.savedOnly !== true) { + const capturesFrames = Array.isArray(input.frameAccess) && input.frameAccess.some(entry => + Array.isArray((entry as { captureTimestampsMs?: unknown })?.captureTimestampsMs) && (entry as { captureTimestampsMs: unknown[] }).captureTimestampsMs.length > 0); + if (capturesFrames && input.refreshFramesOnly !== true && input.savedOnly !== true) { input.refreshFramesOnly = true; defaultedFields.push('refreshFramesOnly'); } @@ -178,14 +181,12 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio // Every scoped video must be a subject of the route, or its frames cannot be read or captured. const subjects = input.route === 'inspect_video' ? [input.videoId] : input.route === 'topic_research' && Array.isArray(input.comparisonVideoIds) ? input.comparisonVideoIds : []; - if (input.savedOnly === true) - issue('frameAccess', 'Saved-only requests cannot capture frames. Omit frameAccess, or omit savedOnly if the user authorized new captures.'); + if (input.savedOnly === true && capturesFrames) + issue('frameAccess', 'Saved-only requests cannot capture frames. List only savedTimestampsMs, or omit savedOnly if the user authorized new captures.'); else if (scoped.some(id => !suppliedVideoIds.includes(id) || !subjects.includes(id)) || new Set(scoped).size !== scoped.length) issue('frameAccess', 'List one frameAccess entry per video, only for the route\'s subjects. When saved or new frames span several videos, use topic_research with every one in comparisonVideoIds.'); else if (scope.some(entry => new Set([...entry.savedTimestampsMs, ...entry.captureTimestampsMs]).size !== entry.savedTimestampsMs.length + entry.captureTimestampsMs.length)) issue('frameAccess', 'Give each timestamp exactly one role, saved inspection or fresh capture.'); - else if (!scope.some(entry => entry.captureTimestampsMs.length)) - issue('frameAccess', 'frameAccess authorizes new captures. Omit it when no new frame is requested, and analyze saved frames directly.'); } if (input.timestampedFrameQuestion && input.route !== 'inspect_video') issue('route', 'Exact frame requests require single-video inspection.'); @@ -298,7 +299,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline 'Select skill verify_saved_evidence for follow-ups that recover saved original captions/comments, verify exact quotations or passage links, or correct source claims. This replaces general discovery and analysis with a verification workflow. The skill does not prohibit fetching: a request that also needs evidence not yet saved, such as comments, still gets it unless savedOnly is set. Omit the skill for broad summaries or visual analysis.', 'Set requestedEvidence for each kind of evidence the user explicitly asks for: original_quotes for exact quotations, verbatim wording or precise passage links; comments for what viewers or commenters say. Include every requested kind in one decision, alongside any explanation. Research cannot complete until each is read or reported unavailable, so choose an executable route.', 'Set savedOnly true whenever the user prohibits fresh retrieval or asks to use existing saved sources only. Keep this restriction on follow-ups when it remains in force. Missing saved evidence is a gap, never permission to fetch. Use the saved reader instead of transcript retrieval or discovery.', - 'When the user names frame positions to capture, or mixes saved and new frames, omit savedOnly and supply frameAccess with one entry per video. Omit it only when the user leaves the capture positions to you. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. A prohibition on recapturing 10:30 still allows the requested 10:45. When the saved frame belongs to one video and the new capture to another, list both videos and choose topic_research with both in comparisonVideoIds, even if the message links only one. Times mentioned for comparison, narration or history do not authorize capture.', + 'When the user names frame positions, asks about exact timestamps, or re-inspects saved frames, supply frameAccess with one entry per video. Omit savedOnly when new captures are allowed. Omit frameAccess only when the user leaves the capture positions to you. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. A prohibition on recapturing 10:30 still allows the requested 10:45. When the saved frame belongs to one video and the new capture to another, list both videos and choose topic_research with both in comparisonVideoIds, even if the message links only one. Times mentioned for comparison, narration or history do not authorize capture.', 'Requests for exact quotations from original saved captions/comments, precise passage-start links, or explicit original-asset reads require inspect_video or topic_research with the resolved subjects, even if prior summaries discuss the topic. Research can call search_originals without fetching YouTube. Quoting a source is different from quoting a prior conversation message. Do not route an original-source read directly to finalize.', 'For every context_answer set contextScope: history for listing, quoting, recalling or correcting conversation messages or preferences; video for claims about video content; mixed only when the requested answer needs both. A video URL inside a quoted earlier message does not require video evidence.', 'For finalize give a short routing reason, not a user-facing answer. Choose responseIntent clarification for missing scope, or rejected for unsupported requests. Do not use the legacy clarification or rejected routes for new decisions.', @@ -605,7 +606,12 @@ function candidateObject(value: unknown): Record | undefined { /** An explicit prohibition rules out an unrestricted fallback. Only the classifier * resolves which video and positions the user authorized. */ const prohibitsRecapture = (message: string) => /\b(?:without|never|do not|don['’]t)\s+re-?captur(?:e|ing)\b/i.test(message); -const namesFramePositions = (message: string) => /\b(?:\d+:\d{2}|\d+\s*ms\b)/i.test(message); +/** Clock times, and durations introduced as a position ("at 90 seconds", "only 60 seconds"), not lengths ("a 10 minute video"). */ +const FRAME_POSITION = /\b\d+:\d{2}\b|\b\d+\s*ms\b|(?:\b(?:at|from|around|only|to|until)|@)\s*\d+(?:\.\d+)?\s*(?:milliseconds?|s|secs?|seconds?|mins?|minutes?)\b/i; +const namesFramePositions = (message: string) => FRAME_POSITION.test(message); +/** Clock times named in the message, in milliseconds. */ +const namedClockTimes = (message: string) => [...message.matchAll(/\b(?:(\d+):)?(\d{1,2}):(\d{2})\b/g)] + .map(([, hours, minutes, seconds]) => ((Number(hours ?? 0) * 60 + Number(minutes)) * 60 + Number(seconds)) * 1000); function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[]): ClassificationIssue[] { // A single linked video pins inspection to itself, which would leave another video's frames out of reach. @@ -615,11 +621,25 @@ function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifie } // An open refresh may choose its own positions. Named positions or a recapture prohibition // need a bounded list, or every fallback would quietly widen what the user allowed. + // An exact-frame question is the classifier's own statement that the user fixed the positions. const unrestricted = decision.route !== 'finalize' && !decision.savedOnly && !decision.frameAccess; const fixedVideos = decision.route === 'inspect_video' || !!decision.comparisonVideoIds?.length || videoIds.length === 1; - if (unrestricted && (prohibitsRecapture(input.message) || (decision.refreshFramesOnly && fixedVideos && namesFramePositions(input.message)))) { + if (unrestricted && (prohibitsRecapture(input.message) || !!decision.timestampedFrameQuestion + || (decision.refreshFramesOnly && fixedVideos && namesFramePositions(input.message)))) { return [{ path: 'frameAccess', code: 'missing_capture_scope', message: 'The user fixed the frame positions. List the saved inspection positions and the permitted capture positions in frameAccess, one entry per video.' }]; } + // A saved frame at a time the user names is part of the request. Unlisted, nothing would + // hold research open for it or report whether this turn actually inspected it. + if (decision.route !== 'finalize' && (decision.frameAccess || decision.refreshFramesOnly || decision.timestampedFrameQuestion)) { + // A time the route already lists on any video is accounted for, so another video's saved frame there is incidental. + const named = new Set(namedClockTimes(input.message)); + const listed = new Set(decision.frameAccess?.flatMap(entry => [...entry.savedTimestampsMs, ...entry.captureTimestampsMs])); + const omitted = input.sessionBrief?.assets.find(asset => asset.kind === 'frame' && typeof asset.details.timestampMs === 'number' + && named.has(asset.details.timestampMs) && !listed.has(asset.details.timestampMs)); + if (omitted) { + return [{ path: 'frameAccess', code: 'saved_frame_omitted', message: `The session holds a saved ${clockTime(omitted.details.timestampMs as number)} frame of ${omitted.videoId} at a time the user named. List it in that video's savedTimestampsMs, or captureTimestampsMs if the user asked to capture it again, and include that video in the route.` }]; + } + } const scope = comparisonScopeIssues(decision, input, videoIds); return scope.length ? scope : searchQueryNumberIssues(decision, input); } @@ -675,7 +695,7 @@ function lastResortDecision( return kept.length >= 2 ? [kept] : []; })[0] ?? []])].slice(0, 8), }; - const needsFreshData = constraints.frameAccess || constraints.refreshFramesOnly || constraints.refreshEvidence || constraints.refreshDynamicData; + const needsFreshData = constraints.frameAccess?.some(entry => entry.captureTimestampsMs.length) || constraints.refreshFramesOnly || constraints.refreshEvidence || constraints.refreshDynamicData; if (needsFreshData && constraints.savedOnly) { const clarification = { route: 'finalize', responseIntent: 'clarification', reason: 'Routing attempts disagree about saved-only access and fresh retrieval. Ask which sources may be refreshed before retrieving or answering from stale data.', answerDetail: 'standard' }; diff --git a/platform/src/agents/research/research-agent.ts b/platform/src/agents/research/research-agent.ts index 0835e5be..eab936f5 100644 --- a/platform/src/agents/research/research-agent.ts +++ b/platform/src/agents/research/research-agent.ts @@ -463,6 +463,9 @@ async function runResearchAgentWithModelWithinDeadline(options: { // Missing flags belong to legacy persisted routes, which retain their tool set. const toolNames = (options.toolNames ?? capability.toolNames) .filter(name => !options.decision.frameAccess || name !== 'get_video_storyboard') + // A saved-only access list authorizes no capture, so the capture tool is not offered. + .filter(name => name !== 'get_video_frames' || !options.decision.frameAccess + || options.decision.frameAccess.some(entry => entry.captureTimestampsMs.length)) .filter(name => !playlistListing || (!playlistListing.playlistId && name === 'get_playlist') || name === FINALIZE_ANSWER_TOOL_NAME) .filter(name => !verifySaved || name === FINALIZE_ANSWER_TOOL_NAME || (!options.decision.savedOnly && (SAVED_VERIFICATION_RETRIEVAL_TOOLS as readonly string[]).includes(name))) @@ -628,6 +631,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { }; const frameScope = options.decision.frameAccess; const failedFrameVersions = new Set(); + const failedCaptures = new Set(); const trackedContext: AgentToolContext = { ...options.context, researchDeadlineAt: options.researchDeadlineAt, @@ -746,6 +750,11 @@ async function runResearchAgentWithModelWithinDeadline(options: { if (frameScope && execution.toolName === 'analyze_video_frames') { for (const version of (execution.input as { assetVersions?: string[] }).assetVersions ?? []) failedFrameVersions.add(version); } + // A failed capture closes only its own positions; other authorized captures stay required. + if (frameScope && execution.toolName === 'get_video_frames') { + const { videoId, timestampsMs } = execution.input as { videoId?: string; timestampsMs?: number[] }; + for (const time of timestampsMs ?? []) failedCaptures.add(`${videoId}:${time}`); + } toolFailures.set(execution.toolCallId, { toolCallId: execution.toolCallId, toolName: execution.toolName, @@ -784,8 +793,9 @@ async function runResearchAgentWithModelWithinDeadline(options: { const needsCaptureAnalysis = (visualRequired || (options.decision.route === 'inspect_video' && !!options.decision.timestampedFrameQuestion)) && pendingCaptureVersions.some(open); return (toolNames.includes('analyze_video_frames') && !!options.context.analyzeFrames && (needsSavedAnalysis || needsCaptureAnalysis)) - || (toolNames.includes('get_video_frames') && !!options.context.provider.frames && !failed.has('get_video_frames') - && frameExtractionBudget(options.researchDeadlineAt) >= FRAME_EXTRACTION_MIN_MS && captures.some(item => !item.requested)); + || (toolNames.includes('get_video_frames') && !!options.context.provider.frames + && frameExtractionBudget(options.researchDeadlineAt) >= FRAME_EXTRACTION_MIN_MS + && captures.some(item => !item.requested && !failedCaptures.has(`${item.videoId}:${item.time}`))); } if (!visualRequired || hasVisualObservations()) return false; const savedFrames = [...evidence.values()].some(packet => packet.kind === 'youtube_frames' && packet.assetVersions?.length); diff --git a/platform/src/agents/runtime/frame-access.ts b/platform/src/agents/runtime/frame-access.ts index 4488f1b7..03344e3a 100644 --- a/platform/src/agents/runtime/frame-access.ts +++ b/platform/src/agents/runtime/frame-access.ts @@ -53,23 +53,38 @@ export function frameActivity(scope: FrameAccess, runId: string, evidence: Itera pendingCaptureVersions: [...retrievedVersions].filter(version => !inspectedVersions.has(version)), report: sentences.join(' ') }; } -/** Cited frame observations recorded by an earlier turn, which this turn did not inspect again. - * The route cannot be trusted to list every video the user meant, so the answer states it. */ +/** Versions this run analyzed, and which of them it did not itself retrieve (saved images it re-inspected). */ +function runInspections(runId: string, packets: readonly EvidencePacket[]) { + const own = packets.filter(packet => packet.packetId.startsWith(`packet:${runId}:`) && packet.kind === 'youtube_frames'); + const versions = (type: string) => new Set(own.filter(packet => packet.artifacts.some(artifact => artifact.type === type)) + .flatMap(packet => packet.assetVersions ?? [])); + const analyzed = versions('youtube_frame_analysis'); + const retrieved = versions('youtube_frame_retrieval'); + return { analyzed, savedInspected: [...analyzed].some(version => !retrieved.has(version)) }; +} + +/** Cited frame observations recorded by an earlier turn whose image this turn did not inspect again. + * The route cannot be trusted to list every video the user meant, so the answer states it. + * Matching is by immutable asset version: a newer image at the same moment is a different image. */ export function earlierFrameNotes(runId: string, cited: ReadonlyArray<{ packet: EvidencePacket; startMs: number; videoId?: string; title?: string }>, packets: readonly EvidencePacket[]): string[] { - const own = (packet: EvidencePacket) => packet.packetId.startsWith(`packet:${runId}:`); - const inspectedNow = new Set(packets.filter(packet => own(packet) && packet.kind === 'youtube_frames').flatMap(packet => - packet.artifacts.flatMap(artifact => { - if (artifact.type !== 'youtube_frame_analysis') return []; - const coverage = frameCoverageSchema.safeParse(artifact.data); - return coverage.success ? packet.sources.flatMap(source => coverage.data.frames.map(frame => `${source.videoId}:${frame.timestampMs}`)) : []; - }))); + const { analyzed } = runInspections(runId, packets); const notes = new Map(); for (const { packet, startMs, videoId, title } of cited) { const key = `${videoId}:${startMs}`; - if (own(packet) || packet.kind !== 'youtube_frames' || inspectedNow.has(key) || notes.has(key) - || !packet.artifacts.some(artifact => artifact.type === 'youtube_frame_analysis')) continue; - notes.set(key, `The ${clockTime(startMs)} observation${title ? ` of "${title}"` : ''} comes from an earlier turn and was not re-inspected in this one.`); + const versions = packet.assetVersions ?? []; + if (packet.packetId.startsWith(`packet:${runId}:`) || packet.kind !== 'youtube_frames' || notes.has(key) + || !packet.artifacts.some(artifact => artifact.type === 'youtube_frame_analysis') + || (versions.length > 0 && versions.every(version => analyzed.has(version)))) continue; + notes.set(key, `The ${clockTime(startMs)} observation${title ? ` of "${title}"` : ''} comes from an image inspected in an earlier turn, not in this one.`); } return [...notes.values()]; } + +const REINSPECTION_CLAIM = /\b(?:re-?inspect\w*|re-?examin\w*|re-?analy[sz]\w*|re-?check\w*|looked (?:at \w+ )?again|(?:reused?|reusing) (?:the |that |this )?saved)\b/i; + +/** A claim of re-inspection needs no citation, so it is checked against what this run did. */ +export function reinspectionClaimNote(runId: string, answer: string, packets: readonly EvidencePacket[]): string | undefined { + return REINSPECTION_CLAIM.test(answer) && !runInspections(runId, packets).savedInspected + ? 'No frame saved before this turn was re-inspected in this turn.' : undefined; +} diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index 4744c360..dddd250a 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -1083,13 +1083,14 @@ describe('YouTube agent capability router', () => { it('carries the timestamp question across recovery candidates for the same video', async () => { const videoId = 'abcdefghijk', question = 'What does the scoreboard show at 1:00?'; + const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; const model = sequenceClassifier([ - { route: 'inspect_video', videoId, timestampedFrameQuestion: question, visualEvidence: 'invalid' }, + { route: 'inspect_video', videoId, timestampedFrameQuestion: question, visualEvidence: 'invalid', frameAccess }, { videoId, visualEvidence: 'required', visualRequirements: ['scoreboard'] }, ]); const decision = await expectLastResort(classifyCapabilityWithModel({ message: `At 1:00 in https://youtu.be/${videoId}, read the scoreboard`, model, signal: new AbortController().signal })); - expect(decision).toMatchObject({ route: 'inspect_video', videoId, timestampedFrameQuestion: question }); + expect(decision).toMatchObject({ route: 'inspect_video', videoId, timestampedFrameQuestion: question, frameAccess }); }); it.each(['overlap', 'omitted'])('repairs an %s capture list before authorizing a bounded refresh', async invalid => { @@ -1190,6 +1191,38 @@ describe('YouTube agent capability router', () => { expect(JSON.stringify(model.doGenerateCalls[1]!.prompt)).toContain('Saved-only requests cannot capture frames'); }); + it.each([ + ['an exact-frame question without a refresh', 'Capture only 1:00 of https://youtu.be/abcdefghijk. No other timestamps.', 60000, + { timestampedFrameQuestion: 'What is on screen at 1:00?' }], + ['a refresh at a position given in seconds', 'Refresh the frame of https://youtu.be/abcdefghijk at only 60 seconds.', 60000, + { refreshFramesOnly: true }], + ])('requires a capture list for %s', async (_name, message, time, fields) => { + const videoId = 'abcdefghijk'; + const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['screen'], ...fields }; + const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [time] }]; + const model = sequenceClassifier([base, { ...base, frameAccess }]); + const route = await classifyCapabilityWithModel({ model, message, signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'inspect_video', frameAccess }); + expect(model.doGenerateCalls).toHaveLength(2); + }); + + it('records a saved frame at a named time that the route left out', async () => { + const [saved, linked] = ['abcdefghijk', 'lmnopqrstuv']; + const visual = { visualEvidence: 'required', visualRequirements: ['outfits'] }; + const both = [{ videoId: saved, savedTimestampsMs: [30000], captureTimestampsMs: [] }, { videoId: linked, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + // The production decision: only the linked video's capture, with the saved frame silently dropped. + const model = sequenceClassifier([ + { route: 'inspect_video', videoId: linked, ...visual, frameAccess: [both[1]] }, + { route: 'topic_research', comparisonVideoIds: [saved, linked], ...visual, frameAccess: both }, + ]); + const route = await classifyCapabilityWithModel({ model, signal: new AbortController().signal, + message: `Reuse the saved 0:30 frame and capture 1:00 of https://youtu.be/${linked}. Compare the outfits.`, + sessionBrief: { memories: [], assets: [{ videoId: saved, version: 'a'.repeat(64), kind: 'frame', current: true, collectedAt: 1, details: { timestampMs: 30000 } }] } }); + expect(route).toMatchObject({ route: 'topic_research', frameAccess: both }); + expect(model.doGenerateCalls).toHaveLength(2); + expect(JSON.stringify(model.doGenerateCalls[1]!.prompt)).toContain('saved 0:30 frame of abcdefghijk'); + }); + it('pins inspect_video provider calls to the classified video', async () => { const video = vi.fn(async () => ({ cacheStatus: 'miss' as const, value: {} as never })); const provider = createCapabilityProvider(providerWith({ video }), { diff --git a/platform/test/youtube-stored-analysis.test.ts b/platform/test/youtube-stored-analysis.test.ts index b1296c17..9938e8a4 100644 --- a/platform/test/youtube-stored-analysis.test.ts +++ b/platform/test/youtube-stored-analysis.test.ts @@ -302,6 +302,23 @@ describe('cross-video frame completion', () => { expect(analyzed()).toEqual([videoId, other]); }); + it('keeps other authorized captures required after one capture fails', async () => { + const { context, provider, frames } = scene(); + const at = (id: string, timestampMs: number) => ({ ...frames, videoId: id, frames: [{ ...frames.frames[0]!, timestampMs }] }); + provider.frames.mockImplementation((async (request: { videoId: string }) => { + if (request.videoId === videoId) throw new Error('Extraction failed.'); + return { value: at(other, 60000), cacheStatus: 'miss' }; + }) as never); + const refused = await refusedCompletions(context, [ + () => ({ name: 'get_video_frames', input: { videoId, timestampsMs: [35000] } }), + () => ({ name: 'complete_research', input: {} }), + () => ({ name: 'get_video_frames', input: { videoId: other, timestampsMs: [60000], analysis: 'What is he wearing?' } }), + ], { ...decision, frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [35000] }, + { videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }] }); + expect(refused).toBe(1); + expect((provider.frames.mock.calls as unknown as [{ videoId: string }][]).map(([request]) => request.videoId)).toEqual([videoId, other]); + }); + it('tries a saved frame whose inventory omits its position before completing', async () => { const { context, saved, analyzed } = scene(true); const refused = await refusedCompletions(context, [ @@ -314,26 +331,59 @@ describe('cross-video frame completion', () => { }); }); -it('notes a cited observation from an earlier turn when the route omitted its video', async () => { - const { context, store, frames } = setup(); +describe('application notes on frame turns', () => { const other = 'lmnopqrstuv'; - const at = (id: string, timestampMs: number) => ({ ...frames, videoId: id, frames: [{ ...frames.frames[0]!, timestampMs }] }); - context.analyzeFrames = vi.fn(async ({ frames }) => ({ - findings: [{ observation: 'A shirt.', timestampsMs: [frames.frames[0]!.timestampMs] }], warnings: [] })); - // An earlier turn inspected the first video at 0:30. - context.runId = crypto.randomUUID(); - const earlier = await analyze('frame', store.put('frame', videoId, at(videoId, 30000), { timestampMs: 30000 }), context, 'What is he wearing?'); - // This turn's route listed only the second video, as in the production incident. - context.runId = crypto.randomUUID(); - context.frameAccess = [{ videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; - const current = await analyze('frame', store.put('frame', other, at(other, 60000), { timestampMs: 60000 }), context, 'What is he wearing?'); - const result = buildAgentTurnResult({ runId: context.runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), - agentMessageId: crypto.randomUUID() }, { userId: 'user', creditsRemaining: 100 }, { - answer: `He re-checked the saved frame: a shirt [cite:${earlier.excerpts[0]!.id}]. The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, - intent: 'topic_research', confidence: 'medium', citations: [], artifacts: [], warnings: [], - }, [earlier, current], 0, undefined, { access: context.frameAccess }); - expect(result.answer).toMatch(/\n\nThe 0:30 observation( of "[^"]+")? comes from an earlier turn and was not re-inspected in this one\.$/); - expect(result.answer).not.toContain('The 1:00 observation'); + /** An earlier turn inspected the first video at 0:30; this turn's route listed only the second video. */ + async function turns() { + const { context, store, frames, provider } = setup(); + const at = (id: string, timestampMs: number, width = 640) => ({ ...frames, videoId: id, frames: [{ ...frames.frames[0]!, timestampMs, width }] }); + context.analyzeFrames = vi.fn(async ({ frames }) => ({ + findings: [{ observation: 'A shirt.', timestampsMs: [frames.frames[0]!.timestampMs] }], warnings: [] })); + context.runId = crypto.randomUUID(); + const savedVersion = store.put('frame', videoId, at(videoId, 30000), { timestampMs: 30000 }); + const earlier = await analyze('frame', savedVersion, context, 'What is he wearing?'); + context.runId = crypto.randomUUID(); + context.frameAccess = [{ videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + provider.frames.mockResolvedValue({ value: at(other, 60000), cacheStatus: 'miss' }); + const current = await executeGetVideoFrames({ videoId: other, timestampsMs: [60000], analysis: 'What is he wearing?' }, context, 'capture'); + const answer = (text: string, packets: EvidencePacket[]) => buildAgentTurnResult({ runId: context.runId, conversationId: crypto.randomUUID(), + userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }, { userId: 'user', creditsRemaining: 100 }, + { answer: text, intent: 'topic_research', confidence: 'medium', citations: [], artifacts: [], warnings: [] }, + [earlier, ...packets], 0, undefined, { access: context.frameAccess }).answer; + return { context, store, at, savedVersion, earlier, current, answer }; + } + const earlierNote = /The 0:30 observation( of "[^"]+")? comes from an image inspected in an earlier turn, not in this one\./; + const claimNote = 'No frame saved before this turn was re-inspected in this turn.'; + + it('states that a cited observation came from an earlier turn', async () => { + const { earlier, current, answer } = await turns(); + const text = answer(`The saved frame shows a shirt [cite:${earlier.excerpts[0]!.id}]. The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current]); + expect(text).toMatch(earlierNote); + expect(text).not.toContain('The 1:00 observation'); + }); + + it('corrects an uncited claim of re-inspection when no saved frame was inspected', async () => { + const { current, answer } = await turns(); + const text = answer(`I re-inspected the saved 0:30 frame. The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current]); + expect(text.endsWith(claimNote)).toBe(true); + }); + + it('keeps the earlier-turn note when this turn inspected a different image at the same moment', async () => { + const { context, store, at, earlier, current, answer } = await turns(); + context.frameAccess = [{ videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, ...context.frameAccess!]; + const replacement = await analyze('frame', store.put('frame', videoId, at(videoId, 30000, 320), { timestampMs: 30000 }), context, 'What is he wearing?'); + const text = answer(`The saved frame shows a shirt [cite:${earlier.excerpts[0]!.id}] [cite:${current.excerpts[0]!.id}].`, [current, replacement]); + expect(text).toMatch(earlierNote); + }); + + it('adds neither note when this turn re-inspected the cited image', async () => { + const { context, savedVersion, earlier, current, answer } = await turns(); + context.frameAccess = [{ videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, ...context.frameAccess!]; + const again = await analyze('frame', savedVersion, context, 'What is he wearing?'); + const text = answer(`I re-inspected the saved frame: a shirt [cite:${earlier.excerpts[0]!.id}] [cite:${current.excerpts[0]!.id}].`, [current, again]); + expect(text).not.toMatch(earlierNote); + expect(text).not.toContain(claimNote); + }); }); it('rejects forbidden positions and other videos before metadata, billing or extraction', () => { From 4f38472669a79111ef8d325113bb7753aa01e90a Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sat, 7 Nov 2026 21:52:22 +0530 Subject: [PATCH 04/12] Close frame capture by default and state frame activity from recorded operations Review of 55958ea showed that wording-based checks keep leaking: - New captures are now closed unless the route lists their positions or delegates them with agent_selected, so an omitted grant can never widen a limit such as 'capture only 1:00'. Refreshes must state a grant; exact-frame questions, recapture prohibitions and named positions need a list. - The time-only saved-frame router check is removed: it lost which video a time belonged to and rejected plans that excluded a frame on purpose. - Every frame turn's answer now carries the application's account built from the requested frames, the session inventory and this run's operations: listed positions, saved frames re-inspected, or that none was. It no longer depends on the answer's wording or citations. - Earlier-turn notes judge each citation by the exact image behind it, so an untouched frame from the same batch no longer changes the note. --- docs/api-reference/openapi.json | 312 ++++++++++-------- platform/src/agents/agent-runtime-do.ts | 6 +- platform/src/agents/contracts.ts | 7 +- platform/src/agents/finalizer.ts | 13 +- .../src/agents/research/capability-router.ts | 50 ++- .../src/agents/research/research-agent.ts | 24 +- platform/src/agents/runtime/frame-access.ts | 78 +++-- .../youtube-agent-analysis-context.test.ts | 6 +- .../youtube-agent-frame-deadlines.test.ts | 6 +- platform/test/youtube-agent-router.test.ts | 58 ++-- platform/test/youtube-stored-analysis.test.ts | 37 ++- 11 files changed, 349 insertions(+), 248 deletions(-) diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json index c5fc4da7..a7747442 100644 --- a/docs/api-reference/openapi.json +++ b/docs/api-reference/openapi.json @@ -7661,42 +7661,52 @@ } }, "frameAccess": { - "minItems": 1, - "maxItems": 8, - "type": "array", - "items": { - "type": "object", - "properties": { - "videoId": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]{11}$" - }, - "savedTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "captureTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } + "anyOf": [ + { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "savedTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "captureTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "videoId", + "savedTimestampsMs", + "captureTimestampsMs" + ], + "additionalProperties": false } }, - "required": [ - "videoId", - "savedTimestampsMs", - "captureTimestampsMs" - ], - "additionalProperties": false - } + { + "type": "string", + "enum": [ + "agent_selected" + ] + } + ] }, "refreshFramesOnly": { "type": "boolean" @@ -7796,42 +7806,52 @@ } }, "frameAccess": { - "minItems": 1, - "maxItems": 8, - "type": "array", - "items": { - "type": "object", - "properties": { - "videoId": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]{11}$" - }, - "savedTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "captureTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } + "anyOf": [ + { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "savedTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "captureTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "videoId", + "savedTimestampsMs", + "captureTimestampsMs" + ], + "additionalProperties": false } }, - "required": [ - "videoId", - "savedTimestampsMs", - "captureTimestampsMs" - ], - "additionalProperties": false - } + { + "type": "string", + "enum": [ + "agent_selected" + ] + } + ] }, "refreshFramesOnly": { "type": "boolean" @@ -9529,42 +9549,52 @@ } }, "frameAccess": { - "minItems": 1, - "maxItems": 8, - "type": "array", - "items": { - "type": "object", - "properties": { - "videoId": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]{11}$" - }, - "savedTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "captureTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } + "anyOf": [ + { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "savedTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "captureTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "videoId", + "savedTimestampsMs", + "captureTimestampsMs" + ], + "additionalProperties": false } }, - "required": [ - "videoId", - "savedTimestampsMs", - "captureTimestampsMs" - ], - "additionalProperties": false - } + { + "type": "string", + "enum": [ + "agent_selected" + ] + } + ] }, "refreshFramesOnly": { "type": "boolean" @@ -9664,42 +9694,52 @@ } }, "frameAccess": { - "minItems": 1, - "maxItems": 8, - "type": "array", - "items": { - "type": "object", - "properties": { - "videoId": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]{11}$" - }, - "savedTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "captureTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } + "anyOf": [ + { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "savedTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "captureTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "videoId", + "savedTimestampsMs", + "captureTimestampsMs" + ], + "additionalProperties": false } }, - "required": [ - "videoId", - "savedTimestampsMs", - "captureTimestampsMs" - ], - "additionalProperties": false - } + { + "type": "string", + "enum": [ + "agent_selected" + ] + } + ] }, "refreshFramesOnly": { "type": "boolean" diff --git a/platform/src/agents/agent-runtime-do.ts b/platform/src/agents/agent-runtime-do.ts index 7d7f4bee..5668635c 100644 --- a/platform/src/agents/agent-runtime-do.ts +++ b/platform/src/agents/agent-runtime-do.ts @@ -14,6 +14,7 @@ import { storedExtractionDiagnosticSchema, type StoredExtractionDiagnostic } fro import { transcriptDiagnosticSchema, type TranscriptDiagnostic } from './runtime/transcript-diagnostics'; import { agentDraftSchema, agentRunProgressSchema, toolTrace, type AgentDraft } from './runtime/run-progress'; import { framePreviewKey, recordedFramePreviews, saveFramePreviews, type SavedFrame } from './runtime/frame-previews'; +import { frameList, isFrameTurn } from './runtime/frame-access'; import { saveStoryboardPreviews } from './runtime/storyboard-previews'; import { compactAgentRun, compactAgentResult } from './response'; import { queuedRunIdentitySchema, type QueuedRunIdentity } from './runtime/admission-queue'; @@ -1050,8 +1051,9 @@ export class AgentRuntimeDO extends Agent { requestedVideoIds: decision.route === 'inspect_video' ? [decision.videoId] : decision.comparisonVideoIds ?? [], // An application fallback is checked against what this run had, not earlier turns' cited content. runPackets: this.admittedRunEvidence(runId, history), - } : undefined, (decision.route === 'inspect_video' || decision.route === 'topic_research') - && (decision.frameAccess || decision.refreshFramesOnly) ? { access: decision.frameAccess } : undefined); + } : undefined, isFrameTurn(decision) ? { access: frameList(decision), inventory: this.sessionStore.brief().assets.flatMap(asset => + asset.kind === 'frame' ? [{ version: asset.version, videoId: asset.videoId, + ...(typeof asset.details.timestampMs === 'number' ? { timestampMs: asset.details.timestampMs } : {}) }] : []) } : undefined); // Hash the accepted text: unmatched citations may have been removed from the input. const answerHash = createHash('sha256').update(result.answer).digest('hex'); const serialized = JSON.stringify(result); diff --git a/platform/src/agents/contracts.ts b/platform/src/agents/contracts.ts index 9fd895c8..11e9b2c1 100644 --- a/platform/src/agents/contracts.ts +++ b/platform/src/agents/contracts.ts @@ -28,6 +28,9 @@ export const frameAccessSchema = z.preprocess( frameAccessListSchema, ); export type FrameAccess = z.infer; +/** New captures are closed unless the route lists their positions or delegates them with agent_selected. */ +export const routeFrameAccessSchema = z.union([frameAccessSchema, z.literal('agent_selected')]); +export type RouteFrameAccess = z.infer; /** Evidence the user explicitly asked to be read before answering. Research cannot complete until each is read or unavailable. */ export const requestedEvidenceSchema = z.array(z.enum(['original_quotes', 'comments'])).min(1).max(2); @@ -49,7 +52,7 @@ export const capabilityRouteDecisionSchema = z.discriminatedUnion('route', [ savedOnly: z.boolean().optional(), skill: z.enum(['verify_saved_evidence']).optional(), requestedEvidence: requestedEvidenceSchema.optional(), - frameAccess: frameAccessSchema.optional(), + frameAccess: routeFrameAccessSchema.optional(), refreshFramesOnly: z.boolean().optional(), refreshEvidence: z.boolean().optional(), refreshDynamicData: z.boolean().optional(), @@ -67,7 +70,7 @@ export const capabilityRouteDecisionSchema = z.discriminatedUnion('route', [ savedOnly: z.boolean().optional(), skill: z.enum(['verify_saved_evidence']).optional(), requestedEvidence: requestedEvidenceSchema.optional(), - frameAccess: frameAccessSchema.optional(), + frameAccess: routeFrameAccessSchema.optional(), refreshFramesOnly: z.boolean().optional(), timestampedFrameQuestion: z.string().trim().min(1).max(500).optional(), refreshEvidence: z.boolean().optional(), diff --git a/platform/src/agents/finalizer.ts b/platform/src/agents/finalizer.ts index e7733405..7cc11de4 100644 --- a/platform/src/agents/finalizer.ts +++ b/platform/src/agents/finalizer.ts @@ -10,7 +10,7 @@ import { type FinalizeAnswerInput, type FrameAccess, } from './contracts'; -import { earlierFrameNotes, frameActivity, reinspectionClaimNote } from './runtime/frame-access'; +import { frameTurnNotes, type InventoryFrame } from './runtime/frame-access'; import { appendNotices, playlistCoverageDisclosures } from './playlist-coverage'; import { CITED_FRAME_PREVIEWS, citedPacketFramePreviews } from './runtime/frame-previews'; @@ -30,8 +30,8 @@ export function buildAgentTurnResult( packets: EvidencePacket[], creditsCharged: number, durationLimitContext?: DurationLimitAnswerContext, - /** Present when the route asked for fresh frame work, with any exact access it granted. */ - frameTurn?: { access?: FrameAccess }, + /** Present when the route asked for frame work: its exact access, and the session's saved frames. */ + frameTurn?: { access?: FrameAccess; inventory: readonly InventoryFrame[] }, ): AgentTurnResult { const markers = answerCitationIds(input.answer); const citations: AgentCitation[] = []; @@ -39,7 +39,7 @@ export function buildAgentTurnResult( // Which saved previews each cited frame excerpt used, from its own packet. A later answer can // hold other captures of the same moment, so time alone never identifies the pixels. const citedFrames: Array<{ citationId: string; frames: ReturnType }> = []; - const citedObservations: Parameters[1][number][] = []; + const citedObservations: Parameters[0]['cited'][number][] = []; for (const marker of markers) { const matches = packets.flatMap((packet) => packet.excerpts .filter((excerpt) => excerpt.id === marker) @@ -84,10 +84,7 @@ export function buildAgentTurnResult( const notes = [...playlistCoverage.notes]; // The application, not the model, states which frames this turn actually inspected. if (frameTurn && input.intent !== 'clarification' && input.intent !== 'rejected') { - if (frameTurn.access?.some(entry => entry.savedTimestampsMs.length)) notes.push(frameActivity(frameTurn.access, identity.runId, packets).report); - notes.push(...earlierFrameNotes(identity.runId, citedObservations, packets)); - const claim = reinspectionClaimNote(identity.runId, input.answer, packets); - if (claim) notes.push(claim); + notes.push(...frameTurnNotes({ runId: identity.runId, access: frameTurn.access, inventory: frameTurn.inventory, packets, cited: citedObservations })); } const disclosedAnswer = appendNotices(answer, notes); diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index 6de9a6e1..d8ea02c4 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -18,7 +18,6 @@ import { type FinalizeAnswerInput, } from '../contracts'; import { sessionBriefForModel, type SessionBrief } from '../runtime/session-evidence'; -import { clockTime } from '../runtime/clock-time'; import { conversationAssistantMessage, type ConversationTurn } from '../runtime/conversation-memory'; import { assertModelCostAvailable, type AgentModelCostBudget } from '../runtime/model-budget'; import { AGENT_CLASSIFICATION_TIMEOUT_MS, withRunDeadline } from '../runtime/deadline'; @@ -31,7 +30,7 @@ const classifierDecisionSchema = z.object({ skill: z.enum(['verify_saved_evidence']).optional().describe('Verify saved evidence: use for follow-ups that read saved original captions/comments, check exact quotations or precise passage links, or correct source claims. It does not by itself prohibit fetching; set savedOnly separately. Omit for broad summaries and visual analysis.'), requestedEvidence: requestedEvidenceSchema.optional().describe('Evidence the user explicitly asks to be read before answering: original_quotes for exact quotations, verbatim wording or precise passage links from captions; comments for what viewers or commenters say. Research must read each or report it unavailable. Omit when not explicitly requested.'), savedOnly: z.boolean().optional().describe('True when the user requires existing saved evidence only or prohibits fresh retrieval. All provider access is disabled for this run; use saved evidence reads and report missing assets.'), - frameAccess: frameAccessListSchema.optional().describe('For frame positions the user fixes, saved or new: one entry per video the turn touches. captureTimestampsMs lists only the permitted NEW capture positions in milliseconds, empty when nothing may be captured; savedTimestampsMs lists saved positions to re-inspect without recapture, even if their assets are missing. Required for exact-timestamp questions. Omit only for an open-ended frame refresh where the agent chooses positions.'), + frameAccess: z.union([frameAccessListSchema, z.literal('agent_selected')]).optional().describe('Permission to capture new video frames; without it no new frame is captured. When the user fixes frame positions, saved or new, list one entry per video the turn touches: captureTimestampsMs holds only the permitted NEW capture positions in milliseconds, empty when nothing may be captured; savedTimestampsMs holds saved positions to re-inspect without recapture, even if their assets are missing. Required for exact-timestamp questions and for any limit on what may be captured. Use agent_selected only when new frames may help and the user leaves every position to you.'), timestampedFrameQuestion: z.string().trim().min(1).max(500).optional().describe('For inspect_video: when the user supplies exact timestamps and asks what is visible, copy the focused visual question here. Include this for a request to describe each specified frame. Omit for image retrieval only or saved-only requests.'), refreshFramesOnly: z.boolean().optional().describe('For inspection or research, including multi-video comparisons: true when the user asks to freshly capture or refresh frames only. Reuse verified video metadata. Leave refreshEvidence unset unless other source evidence must also be refreshed.'), refreshEvidence: z.boolean().optional().describe('Except for a frames-only refresh, true only when the user explicitly asks to fetch again, refresh or get fresh source data. Choose an executable route in that case.'), @@ -176,7 +175,9 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio } if (input.refreshFramesOnly === true && input.visualEvidence === 'none') issue('visualEvidence', 'Refreshing frames requires visual tool access.'); const scope = parsed.success && parsed.data.frameAccess; - if (scope) { + if (scope === 'agent_selected' && input.savedOnly === true) + issue('frameAccess', 'Saved-only requests cannot capture frames. Omit frameAccess, or list only savedTimestampsMs.'); + if (Array.isArray(scope)) { const scoped = scope.map(entry => entry.videoId); // Every scoped video must be a subject of the route, or its frames cannot be read or captured. const subjects = input.route === 'inspect_video' ? [input.videoId] @@ -299,7 +300,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline 'Select skill verify_saved_evidence for follow-ups that recover saved original captions/comments, verify exact quotations or passage links, or correct source claims. This replaces general discovery and analysis with a verification workflow. The skill does not prohibit fetching: a request that also needs evidence not yet saved, such as comments, still gets it unless savedOnly is set. Omit the skill for broad summaries or visual analysis.', 'Set requestedEvidence for each kind of evidence the user explicitly asks for: original_quotes for exact quotations, verbatim wording or precise passage links; comments for what viewers or commenters say. Include every requested kind in one decision, alongside any explanation. Research cannot complete until each is read or reported unavailable, so choose an executable route.', 'Set savedOnly true whenever the user prohibits fresh retrieval or asks to use existing saved sources only. Keep this restriction on follow-ups when it remains in force. Missing saved evidence is a gap, never permission to fetch. Use the saved reader instead of transcript retrieval or discovery.', - 'When the user names frame positions, asks about exact timestamps, or re-inspects saved frames, supply frameAccess with one entry per video. Omit savedOnly when new captures are allowed. Omit frameAccess only when the user leaves the capture positions to you. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. A prohibition on recapturing 10:30 still allows the requested 10:45. When the saved frame belongs to one video and the new capture to another, list both videos and choose topic_research with both in comparisonVideoIds, even if the message links only one. Times mentioned for comparison, narration or history do not authorize capture.', + 'New frames are captured only with frameAccess. When the user names frame positions, limits what may be captured, asks about exact timestamps, or re-inspects saved frames, list them in frameAccess with one entry per video. Set agent_selected only when new frames may help and the user leaves every position to you, including a plain frame refresh. Omit savedOnly when new captures are allowed. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. A prohibition on recapturing 10:30 still allows the requested 10:45. When the saved frame belongs to one video and the new capture to another, list both videos and choose topic_research with both in comparisonVideoIds, even if the message links only one. Times mentioned for comparison, narration or history do not authorize capture.', 'Requests for exact quotations from original saved captions/comments, precise passage-start links, or explicit original-asset reads require inspect_video or topic_research with the resolved subjects, even if prior summaries discuss the topic. Research can call search_originals without fetching YouTube. Quoting a source is different from quoting a prior conversation message. Do not route an original-source read directly to finalize.', 'For every context_answer set contextScope: history for listing, quoting, recalling or correcting conversation messages or preferences; video for claims about video content; mixed only when the requested answer needs both. A video URL inside a quoted earlier message does not require video evidence.', 'For finalize give a short routing reason, not a user-facing answer. Choose responseIntent clarification for missing scope, or rejected for unsupported requests. Do not use the legacy clarification or rejected routes for new decisions.', @@ -609,9 +610,6 @@ const prohibitsRecapture = (message: string) => /\b(?:without|never|do not|don[' /** Clock times, and durations introduced as a position ("at 90 seconds", "only 60 seconds"), not lengths ("a 10 minute video"). */ const FRAME_POSITION = /\b\d+:\d{2}\b|\b\d+\s*ms\b|(?:\b(?:at|from|around|only|to|until)|@)\s*\d+(?:\.\d+)?\s*(?:milliseconds?|s|secs?|seconds?|mins?|minutes?)\b/i; const namesFramePositions = (message: string) => FRAME_POSITION.test(message); -/** Clock times named in the message, in milliseconds. */ -const namedClockTimes = (message: string) => [...message.matchAll(/\b(?:(\d+):)?(\d{1,2}):(\d{2})\b/g)] - .map(([, hours, minutes, seconds]) => ((Number(hours ?? 0) * 60 + Number(minutes)) * 60 + Number(seconds)) * 1000); function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[]): ClassificationIssue[] { // A single linked video pins inspection to itself, which would leave another video's frames out of reach. @@ -621,23 +619,16 @@ function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifie } // An open refresh may choose its own positions. Named positions or a recapture prohibition // need a bounded list, or every fallback would quietly widen what the user allowed. - // An exact-frame question is the classifier's own statement that the user fixed the positions. - const unrestricted = decision.route !== 'finalize' && !decision.savedOnly && !decision.frameAccess; - const fixedVideos = decision.route === 'inspect_video' || !!decision.comparisonVideoIds?.length || videoIds.length === 1; - if (unrestricted && (prohibitsRecapture(input.message) || !!decision.timestampedFrameQuestion - || (decision.refreshFramesOnly && fixedVideos && namesFramePositions(input.message)))) { - return [{ path: 'frameAccess', code: 'missing_capture_scope', message: 'The user fixed the frame positions. List the saved inspection positions and the permitted capture positions in frameAccess, one entry per video.' }]; - } - // A saved frame at a time the user names is part of the request. Unlisted, nothing would - // hold research open for it or report whether this turn actually inspected it. - if (decision.route !== 'finalize' && (decision.frameAccess || decision.refreshFramesOnly || decision.timestampedFrameQuestion)) { - // A time the route already lists on any video is accounted for, so another video's saved frame there is incidental. - const named = new Set(namedClockTimes(input.message)); - const listed = new Set(decision.frameAccess?.flatMap(entry => [...entry.savedTimestampsMs, ...entry.captureTimestampsMs])); - const omitted = input.sessionBrief?.assets.find(asset => asset.kind === 'frame' && typeof asset.details.timestampMs === 'number' - && named.has(asset.details.timestampMs) && !listed.has(asset.details.timestampMs)); - if (omitted) { - return [{ path: 'frameAccess', code: 'saved_frame_omitted', message: `The session holds a saved ${clockTime(omitted.details.timestampMs as number)} frame of ${omitted.videoId} at a time the user named. List it in that video's savedTimestampsMs, or captureTimestampsMs if the user asked to capture it again, and include that video in the route.` }]; + // Capture is closed without frameAccess, so omission never widens it. These checks only ask + // for a list where the classifier itself, or an explicit limit in the message, implies one. + if (decision.route !== 'finalize' && !decision.savedOnly && !Array.isArray(decision.frameAccess)) { + const fixedVideos = decision.route === 'inspect_video' || !!decision.comparisonVideoIds?.length || videoIds.length === 1; + const limited = prohibitsRecapture(input.message) || (fixedVideos && namesFramePositions(input.message)); + if (decision.timestampedFrameQuestion || (decision.frameAccess === 'agent_selected' && limited)) { + return [{ path: 'frameAccess', code: 'missing_capture_scope', message: 'The user fixed the frame positions. List the saved inspection positions and the permitted capture positions in frameAccess, one entry per video.' }]; + } + if (decision.refreshFramesOnly && !decision.frameAccess) { + return [{ path: 'frameAccess', code: 'missing_capture_permission', message: 'A frame refresh needs capture permission. List the positions in frameAccess, or set agent_selected if the user left them to you.' }]; } } const scope = comparisonScopeIssues(decision, input, videoIds); @@ -672,9 +663,10 @@ function lastResortDecision( // Requirements any candidate stated, or the request implies, survive every fallback: // fresh data needs an executable route, required visuals stay required, and a // follow-up comparison keeps both of its subjects. - // A bounded list that no attempt stated validly is not widened to an open refresh. - const frameAccess = firstValid('frameAccess'); - if (!frameAccess && candidates.some(candidate => Array.isArray(candidate.frameAccess))) { + // A bounded list that no attempt stated validly is never widened, not even to another attempt's agent_selected. + const listing = candidates.filter(candidate => Array.isArray(candidate.frameAccess)); + const frameAccess = firstValid('frameAccess', listing.length ? listing : candidates); + if (listing.length && !frameAccess) { const clarification = { route: 'finalize', responseIntent: 'clarification', reason: 'Clarify the saved and permitted fresh frame positions for each video.', answerDetail: 'standard' }; return { decision: classifierDecisionSchema.parse(clarification), method: 'defaults', defaultedFields: Object.keys(clarification) as DefaultedClassificationField[] }; @@ -695,7 +687,7 @@ function lastResortDecision( return kept.length >= 2 ? [kept] : []; })[0] ?? []])].slice(0, 8), }; - const needsFreshData = constraints.frameAccess?.some(entry => entry.captureTimestampsMs.length) || constraints.refreshFramesOnly || constraints.refreshEvidence || constraints.refreshDynamicData; + const needsFreshData = (Array.isArray(constraints.frameAccess) && constraints.frameAccess.some(entry => entry.captureTimestampsMs.length)) || constraints.refreshFramesOnly || constraints.refreshEvidence || constraints.refreshDynamicData; if (needsFreshData && constraints.savedOnly) { const clarification = { route: 'finalize', responseIntent: 'clarification', reason: 'Routing attempts disagree about saved-only access and fresh retrieval. Ask which sources may be refreshed before retrieving or answering from stale data.', answerDetail: 'standard' }; @@ -1003,7 +995,7 @@ export async function resolveCapabilityRoute(options: { // video after validation. Resuming must not run them without the access the user gave. const subjects = decision.route === 'inspect_video' ? [decision.videoId] : decision.route === 'topic_research' ? decision.comparisonVideoIds ?? [] : []; - if ('frameAccess' in decision && decision.frameAccess?.some(entry => !subjects.includes(entry.videoId))) { + if ('frameAccess' in decision && Array.isArray(decision.frameAccess) && decision.frameAccess.some(entry => !subjects.includes(entry.videoId))) { return { route: 'finalize', responseIntent: 'clarification', reason: 'Clarify which video the saved and fresh frame positions refer to.' }; } return decision; diff --git a/platform/src/agents/research/research-agent.ts b/platform/src/agents/research/research-agent.ts index eab936f5..741a4b8c 100644 --- a/platform/src/agents/research/research-agent.ts +++ b/platform/src/agents/research/research-agent.ts @@ -1,5 +1,5 @@ import { inspectionRecordsForModel, INSPECTION_RECORD_GUIDANCE } from '../runtime/inspection-records'; -import { frameActivity } from '../runtime/frame-access'; +import { frameActivity, frameList, isFrameTurn } from '../runtime/frame-access'; import { VISUAL_INSPECTION_SKILL } from './skills/visual-inspection.generated'; import { scheduleEvidenceTools } from '../runtime/evidence-scheduler'; import { withRunAssetCache, type RunAssetCache } from '../runtime/run-asset-cache'; @@ -41,6 +41,7 @@ import { type AgentWarning, type AgentTurnResult, type CapabilityRouteDecision, + type FrameAccess, type EvidenceOperation, type EvidencePacket, type FinalizeAnswerInput, @@ -462,10 +463,9 @@ async function runResearchAgentWithModelWithinDeadline(options: { ? options.decision : undefined; // Missing flags belong to legacy persisted routes, which retain their tool set. const toolNames = (options.toolNames ?? capability.toolNames) - .filter(name => !options.decision.frameAccess || name !== 'get_video_storyboard') - // A saved-only access list authorizes no capture, so the capture tool is not offered. - .filter(name => name !== 'get_video_frames' || !options.decision.frameAccess - || options.decision.frameAccess.some(entry => entry.captureTimestampsMs.length)) + .filter(name => !frameList(options.decision) || name !== 'get_video_storyboard') + // New captures are closed unless the route lists their positions or delegates them. + .filter(name => name !== 'get_video_frames' || capturesAllowed(options.decision)) .filter(name => !playlistListing || (!playlistListing.playlistId && name === 'get_playlist') || name === FINALIZE_ANSWER_TOOL_NAME) .filter(name => !verifySaved || name === FINALIZE_ANSWER_TOOL_NAME || (!options.decision.savedOnly && (SAVED_VERIFICATION_RETRIEVAL_TOOLS as readonly string[]).includes(name))) @@ -629,7 +629,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { } return pending; }; - const frameScope = options.decision.frameAccess; + const frameScope = frameList(options.decision); const failedFrameVersions = new Set(); const failedCaptures = new Set(); const trackedContext: AgentToolContext = { @@ -638,7 +638,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { researchQuestion: options.message, refreshEvidence: options.decision.refreshEvidence, refreshFramesOnly: options.decision.refreshFramesOnly, - frameAccess: options.decision.frameAccess, + frameAccess: frameScope, timestampedFrameQuestion: options.decision.route === 'inspect_video' && !options.decision.savedOnly ? options.decision.timestampedFrameQuestion : undefined, pinnedVideoId: options.decision.route === 'inspect_video' ? options.decision.videoId : undefined, @@ -883,7 +883,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { ...(savedEvidenceSkill && !verifySaved ? ['Reuse supplied visual observations. Saved visual analysis tools remain available only for requested visual facts that are still missing; they do not replace required original caption/comment reads.'] : []), ...(!verifySaved ? ['For exact quotations or source corrections, use search_originals: omit assetVersion to find which saved source says it, then read that source by its version. Analyst summaries are not verbatim evidence. Read original passages before completing research.'] : []), ...(options.decision.savedOnly ? ['This run is saved-only. Retrieval and discovery are disabled, including incidental metadata calls. Missing saved assets are gaps, not permission to fetch.'] : []), - ...(options.decision.frameAccess ? [`Exact frame access for this turn: ${JSON.stringify(options.decision.frameAccess)}. Analyze the saved positions by their session asset versions. If saved inspection fails, stop that inspection and report the gap. Independently retrieve the authorized new positions. Earlier observations cannot stand in for a successful re-inspection.`] : []), + ...(frameScope ? [`Exact frame access for this turn: ${JSON.stringify(frameScope)}. Analyze the saved positions by their session asset versions. If saved inspection fails, stop that inspection and report the gap. Independently retrieve the authorized new positions. Earlier observations cannot stand in for a successful re-inspection.`] : []), ...(requestedKinds.length ? [requestedEvidenceGuidance(requestedKinds)] : []), // The skill accompanies exactly the visual tools this model can call. ...(providerToolNames.some(name => VISUAL_TOOL_NAMES.includes(name)) ? [VISUAL_INSPECTION_SKILL] : []), @@ -1115,7 +1115,7 @@ async function runUnifiedFinalizer(options: { .filter(packet => packet.sources.some(source => source.videoId && subjects.has(source.videoId))); const inspectionRecords = inspected.length ? inspectionRecordsForModel(inspected, options.context.evidenceRunRecords?.(inspected.map(packet => packet.packetId))) : undefined; - const frameReport = 'frameAccess' in options.decision && !!options.decision.frameAccess?.some(entry => entry.savedTimestampsMs.length); + const frameReport = isFrameTurn(options.decision); const baseOutputSchema = conversational ? conversationalFinalizationOutputSchema : intent === 'context_answer' ? contextFinalizationOutputSchema : finalizationOutputSchema; const numberedItemCount = 'numberedItemCount' in options.decision ? options.decision.numberedItemCount : undefined; @@ -1500,6 +1500,12 @@ function toolFailureWarnings(failures: EvidenceToolFailure[]): AgentWarning[] { }))]; } +/** Whether this route may capture new frames: at listed positions, or anywhere when delegated. */ +function capturesAllowed(decision: CapabilityRouteDecision): boolean { + return 'frameAccess' in decision && (decision.frameAccess === 'agent_selected' + || !!frameList(decision)?.some(entry => entry.captureTimestampsMs.length)); +} + function mergeWarnings(modelWarnings: AgentWarning[], failureWarnings: AgentWarning[]): AgentWarning[] { const warnings = new Map(); for (const warning of [...modelWarnings, ...failureWarnings]) { diff --git a/platform/src/agents/runtime/frame-access.ts b/platform/src/agents/runtime/frame-access.ts index 03344e3a..e8b9889f 100644 --- a/platform/src/agents/runtime/frame-access.ts +++ b/platform/src/agents/runtime/frame-access.ts @@ -1,4 +1,4 @@ -import type { EvidencePacket, FrameAccess } from '../contracts'; +import type { CapabilityRouteDecision, EvidencePacket, FrameAccess } from '../contracts'; import { clockTime } from './clock-time'; import { frameCoverageSchema } from './model-evidence'; @@ -53,38 +53,62 @@ export function frameActivity(scope: FrameAccess, runId: string, evidence: Itera pendingCaptureVersions: [...retrievedVersions].filter(version => !inspectedVersions.has(version)), report: sentences.join(' ') }; } -/** Versions this run analyzed, and which of them it did not itself retrieve (saved images it re-inspected). */ -function runInspections(runId: string, packets: readonly EvidencePacket[]) { +/** The exact frame positions a route grants, if it fixed them. */ +export function frameList(decision: CapabilityRouteDecision): FrameAccess | undefined { + return 'frameAccess' in decision && Array.isArray(decision.frameAccess) ? decision.frameAccess : undefined; +} + +/** A turn that asked for frame work, whose answer carries the application's account of it. */ +export function isFrameTurn(decision: CapabilityRouteDecision): boolean { + return (decision.route === 'inspect_video' || decision.route === 'topic_research') + && (!!decision.frameAccess || !!decision.refreshFramesOnly || (decision.route === 'inspect_video' && !!decision.timestampedFrameQuestion)); +} + +/** A saved frame as the session inventory records it. */ +export interface InventoryFrame { version: string; videoId: string; timestampMs?: number } + +/** + * The application's account of a frame turn, built from what the route requested and what + * this run recorded, never from the answer's wording or citations: + * - each listed position: re-inspected, retrieved, inspected, or a gap; + * - saved frames re-inspected outside the list, or that none was; + * - cited observations whose exact image this turn did not inspect again. + */ +export function frameTurnNotes(options: { + runId: string; access?: FrameAccess; inventory: readonly InventoryFrame[]; packets: readonly EvidencePacket[]; + cited: ReadonlyArray<{ packet: EvidencePacket; startMs: number; videoId?: string; title?: string }>; +}): string[] { + const { runId, access, inventory, packets } = options; const own = packets.filter(packet => packet.packetId.startsWith(`packet:${runId}:`) && packet.kind === 'youtube_frames'); const versions = (type: string) => new Set(own.filter(packet => packet.artifacts.some(artifact => artifact.type === type)) .flatMap(packet => packet.assetVersions ?? [])); const analyzed = versions('youtube_frame_analysis'); const retrieved = versions('youtube_frame_retrieval'); - return { analyzed, savedInspected: [...analyzed].some(version => !retrieved.has(version)) }; -} + const frames = new Map(inventory.map(frame => [frame.version, frame])); + const title = (videoId: string) => packets.flatMap(packet => packet.sources).find(source => source.videoId === videoId && source.title)?.title; + const named = (videoId: string, time: number) => `${clockTime(time)} frame${title(videoId) ? ` of "${title(videoId)}"` : ''}`; + const notes: string[] = []; + if (access?.length) notes.push(frameActivity(access, runId, packets).report); -/** Cited frame observations recorded by an earlier turn whose image this turn did not inspect again. - * The route cannot be trusted to list every video the user meant, so the answer states it. - * Matching is by immutable asset version: a newer image at the same moment is a different image. */ -export function earlierFrameNotes(runId: string, cited: ReadonlyArray<{ packet: EvidencePacket; startMs: number; videoId?: string; title?: string }>, - packets: readonly EvidencePacket[]): string[] { - const { analyzed } = runInspections(runId, packets); - const notes = new Map(); - for (const { packet, startMs, videoId, title } of cited) { - const key = `${videoId}:${startMs}`; - const versions = packet.assetVersions ?? []; - if (packet.packetId.startsWith(`packet:${runId}:`) || packet.kind !== 'youtube_frames' || notes.has(key) - || !packet.artifacts.some(artifact => artifact.type === 'youtube_frame_analysis') - || (versions.length > 0 && versions.every(version => analyzed.has(version)))) continue; - notes.set(key, `The ${clockTime(startMs)} observation${title ? ` of "${title}"` : ''} comes from an image inspected in an earlier turn, not in this one.`); + const listedSaved = new Set(access?.flatMap(entry => entry.savedTimestampsMs.map(time => `${entry.videoId}:${time}`))); + const reinspected = [...analyzed].filter(version => !retrieved.has(version)); + for (const frame of reinspected.flatMap(version => frames.get(version) ?? [])) { + if (frame.timestampMs !== undefined && !listedSaved.has(`${frame.videoId}:${frame.timestampMs}`)) + notes.push(`I re-inspected the saved ${named(frame.videoId, frame.timestampMs)}.`); } - return [...notes.values()]; -} - -const REINSPECTION_CLAIM = /\b(?:re-?inspect\w*|re-?examin\w*|re-?analy[sz]\w*|re-?check\w*|looked (?:at \w+ )?again|(?:reused?|reusing) (?:the |that |this )?saved)\b/i; + if (!reinspected.length && inventory.some(frame => !retrieved.has(frame.version))) + notes.push('No frame saved before this turn was re-inspected in this turn.'); -/** A claim of re-inspection needs no citation, so it is checked against what this run did. */ -export function reinspectionClaimNote(runId: string, answer: string, packets: readonly EvidencePacket[]): string | undefined { - return REINSPECTION_CLAIM.test(answer) && !runInspections(runId, packets).savedInspected - ? 'No frame saved before this turn was re-inspected in this turn.' : undefined; + // Each citation is judged by the image behind it, not by its batch or its moment. + const earlier = new Map(); + for (const { packet, startMs, videoId, title: cited } of options.cited) { + const key = `${videoId}:${startMs}`; + if (packet.packetId.startsWith(`packet:${runId}:`) || packet.kind !== 'youtube_frames' || earlier.has(key) + || !packet.artifacts.some(artifact => artifact.type === 'youtube_frame_analysis')) continue; + const all = packet.assetVersions ?? []; + const exact = all.filter(version => frames.get(version)?.videoId === videoId && frames.get(version)?.timestampMs === startMs); + if (exact.length ? exact.some(version => analyzed.has(version)) : all.length > 0 && all.every(version => analyzed.has(version))) continue; + earlier.set(key, `The ${clockTime(startMs)} observation${cited ? ` of "${cited}"` : ''} comes from an image inspected in an earlier turn, not in this one.`); + } + return [...notes, ...earlier.values()]; } diff --git a/platform/test/youtube-agent-analysis-context.test.ts b/platform/test/youtube-agent-analysis-context.test.ts index eadd2579..860494e5 100644 --- a/platform/test/youtube-agent-analysis-context.test.ts +++ b/platform/test/youtube-agent-analysis-context.test.ts @@ -166,7 +166,7 @@ it.each(['required', 'helpful'] as const)('keeps provider calls for exact frames usage: { inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 }, outputTokens: { total: 10, text: 10, reasoning: 0 } }, }) }); await runResearchAgentWithModel({ model, message: 'Compare the camera angles', context, decision: { - route: 'topic_research', researchBreadth: 'comparative', researchVideoCount: 4, useStoryboard: true, visualEvidence, + route: 'topic_research', researchBreadth: 'comparative', researchVideoCount: 4, useStoryboard: true, visualEvidence, frameAccess: 'agent_selected', ...(visualEvidence === 'required' ? { visualRequirements: ['camera angle'] } : {}), } }); const reserve = visualEvidence === 'required' ? 2 : undefined; @@ -210,7 +210,7 @@ it.each([ usage: { inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 }, outputTokens: { total: 10, text: 10, reasoning: 0 } }, }) }); await runResearchAgentWithModel({ model, message: 'Compare how the presenters look and what they say', context, decision: { - route: 'topic_research', researchBreadth: 'comparative', researchVideoCount: 2, useStoryboard: true, visualEvidence: 'required', + route: 'topic_research', researchBreadth: 'comparative', researchVideoCount: 2, useStoryboard: true, visualEvidence: 'required', frameAccess: 'agent_selected', visualRequirements: ['presenter appearance'], ...(subjects ? { comparisonVideoIds: subjects } : {}), } }); expect(executions.filter(execution => execution.toolName === 'get_video_transcript').map(execution => execution.frameReserveCalls)).toEqual(reserves); @@ -262,7 +262,7 @@ it.each([ usage: { inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 }, outputTokens: { total: 10, text: 10, reasoning: 0 } }, }) }); await runResearchAgentWithModel({ model, message: 'Compare how the presenters look and what they say', context, recoveredEvidence: recovered, - decision: { route: 'topic_research', comparisonVideoIds: [first, second], useStoryboard: true, visualEvidence: 'required', + decision: { route: 'topic_research', comparisonVideoIds: [first, second], useStoryboard: true, visualEvidence: 'required', frameAccess: 'agent_selected', visualRequirements: ['presenter appearance'] } }); expect(executions.find(execution => execution.toolName === 'get_video_transcript')!.frameReserveCalls).toBe(reserve); }); diff --git a/platform/test/youtube-agent-frame-deadlines.test.ts b/platform/test/youtube-agent-frame-deadlines.test.ts index f684d7c5..17060ded 100644 --- a/platform/test/youtube-agent-frame-deadlines.test.ts +++ b/platform/test/youtube-agent-frame-deadlines.test.ts @@ -47,7 +47,7 @@ function setup(delayMs: number) { confidence: 'low', warnings: [], blocks: [{ text: 'The available evidence is limited.', evidenceIds: ['ref_1'] }], }) }], finishReason: { unified: 'stop', raw: undefined }, usage, warnings: [] }) }); const options = { model, finalizationModel: finalizer, context, message: 'Look at frames to confirm player names.', - decision: { route: 'inspect_video' as const, videoId, useStoryboard: true, visualEvidence: 'required' as const, visualRequirements: ['player names on shirts'] }, + decision: { route: 'inspect_video' as const, videoId, useStoryboard: true, visualEvidence: 'required' as const, visualRequirements: ['player names on shirts'], frameAccess: 'agent_selected' as const }, toolNames: ['get_video_frames', 'analyze_video_frames', 'finalize_answer'] as const }; return { options, context, analyzed, finalizer }; } @@ -123,7 +123,7 @@ test('continues after the transcript target and a failed storyboard to retrieve } }); const run = runResearchAgentWithModel({ ...options, model, message: 'What were the names of the presenters and what were they wearing on stage?', - decision: { route: 'topic_research', useStoryboard: true, visualEvidence: 'required', visualRequirements: ['presenter clothing'], researchVideoCount: 1 }, + decision: { route: 'topic_research', useStoryboard: true, visualEvidence: 'required', visualRequirements: ['presenter clothing'], researchVideoCount: 1, frameAccess: 'agent_selected' }, recoveredSearchUsed: true, recoveredEvidence: [transcript], toolNames: ['get_video_storyboard', 'get_video_frames', 'analyze_video_frames', 'finalize_answer'] }); await vi.advanceTimersByTimeAsync(0); @@ -231,7 +231,7 @@ test('does not withhold finalization when visual evidence is only helpful', asyn }) }], finishReason: { unified: 'tool-calls', raw: undefined }, usage, warnings: [] }; } }); await runResearchAgentWithModel({ ...options, model, recoveredEvidence: [metadata], - decision: { route: 'inspect_video', videoId, useStoryboard: true, visualEvidence: 'helpful' } }); + decision: { route: 'inspect_video', videoId, useStoryboard: true, visualEvidence: 'helpful', frameAccess: 'agent_selected' } }); expect(model.doGenerateCalls).toHaveLength(1); expect(context.provider.frames).not.toHaveBeenCalled(); expect(context.finalize).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ warnings: diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index dddd250a..45d02267 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -1072,12 +1072,12 @@ describe('YouTube agent capability router', () => { it.each(['direct', 'recovered'])('keeps a frame-only refresh on a %s two-video comparison', async mode => { const ids = ['abcdefghijk', 'lmnopqrstuv']; const output = mode === 'direct' - ? { route: 'topic_research', comparisonVideoIds: ids, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['read the clocks'] } - : { route: 'inspect_video', videoId: ids[0], comparisonVideoIds: ids, refreshFramesOnly: true, visualEvidence: 'invalid' }; + ? { route: 'topic_research', comparisonVideoIds: ids, refreshFramesOnly: true, frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['read the clocks'] } + : { route: 'inspect_video', videoId: ids[0], comparisonVideoIds: ids, refreshFramesOnly: true, frameAccess: 'agent_selected', visualEvidence: 'invalid' }; const pending = classifyCapabilityWithModel({ message: 'Compare fresh frames from https://youtu.be/abcdefghijk and https://youtu.be/lmnopqrstuv', model: sequenceClassifier([output]), signal: new AbortController().signal }); const route = mode === 'direct' ? await pending : await expectLastResort(pending); - expect(route).toMatchObject({ route: 'topic_research', comparisonVideoIds: ids, refreshFramesOnly: true, useStoryboard: true }); + expect(route).toMatchObject({ route: 'topic_research', comparisonVideoIds: ids, refreshFramesOnly: true, frameAccess: 'agent_selected', useStoryboard: true }); expect(route).not.toHaveProperty('refreshEvidence', true); }); @@ -1147,11 +1147,37 @@ describe('YouTube agent capability router', () => { useStoryboard: true, answerDetail: 'standard', frameAccess: legacy })).toMatchObject({ frameAccess: [legacy] }); }); - it('runs an open refresh without a capture list on the first attempt', async () => { - const model = sequenceClassifier([{ route: 'inspect_video', videoId: 'abcdefghijk', refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'] }]); + it('runs a delegated refresh, and asks an unstated one for capture permission', async () => { + const base = { route: 'inspect_video', videoId: 'abcdefghijk', refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'] }; + const model = sequenceClassifier([base, { ...base, frameAccess: 'agent_selected' }]); const route = await classifyCapabilityWithModel({ model, message: 'Refresh the frames of https://youtu.be/abcdefghijk', signal: new AbortController().signal }); - expect(route).toMatchObject({ route: 'inspect_video', refreshFramesOnly: true }); + expect(route).toMatchObject({ route: 'inspect_video', refreshFramesOnly: true, frameAccess: 'agent_selected' }); + expect(model.doGenerateCalls).toHaveLength(2); + expect(JSON.stringify(model.doGenerateCalls[1]!.prompt)).toContain('needs capture permission'); + }); + + it('leaves capture closed when the classifier grants nothing for an exact capture request', async () => { + const videoId = 'abcdefghijk'; + // The classifier omitted every frame field, so no attempt is retried and nothing may be captured. + const route = await classifyCapabilityWithModel({ model: sequenceClassifier([{ route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['the 1:00 still'] }]), + message: `Capture only 1:00 of https://youtu.be/${videoId}. Just the still, no analysis.`, signal: new AbortController().signal }); expect(route).not.toHaveProperty('frameAccess'); + if (route.route !== 'inspect_video') throw new Error('Expected inspection.'); + const model = finalizingModel(); + await runResearchAgentWithModel({ model, message: 'Capture only 1:00.', decision: route, context: inspectContext() }); + const tools = model.doGenerateCalls[0]?.tools?.map(tool => tool.name); + expect(tools).toContain('analyze_video_frames'); + expect(tools).not.toContain('get_video_frames'); + }); + + it('accepts a capture list that excludes a saved frame the user set aside', async () => { + const videoId = 'abcdefghijk'; + const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + const model = sequenceClassifier([{ route: 'inspect_video', videoId, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'], frameAccess }]); + const route = await classifyCapabilityWithModel({ model, message: `Ignore the old 0:30 frame entirely. Capture only 1:00 of https://youtu.be/${videoId}.`, + sessionBrief: { memories: [], assets: [{ videoId, version: 'a'.repeat(64), kind: 'frame', current: true, collectedAt: 1, details: { timestampMs: 30000 } }] }, + signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'inspect_video', frameAccess }); expect(model.doGenerateCalls).toHaveLength(1); }); @@ -1206,23 +1232,6 @@ describe('YouTube agent capability router', () => { expect(model.doGenerateCalls).toHaveLength(2); }); - it('records a saved frame at a named time that the route left out', async () => { - const [saved, linked] = ['abcdefghijk', 'lmnopqrstuv']; - const visual = { visualEvidence: 'required', visualRequirements: ['outfits'] }; - const both = [{ videoId: saved, savedTimestampsMs: [30000], captureTimestampsMs: [] }, { videoId: linked, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; - // The production decision: only the linked video's capture, with the saved frame silently dropped. - const model = sequenceClassifier([ - { route: 'inspect_video', videoId: linked, ...visual, frameAccess: [both[1]] }, - { route: 'topic_research', comparisonVideoIds: [saved, linked], ...visual, frameAccess: both }, - ]); - const route = await classifyCapabilityWithModel({ model, signal: new AbortController().signal, - message: `Reuse the saved 0:30 frame and capture 1:00 of https://youtu.be/${linked}. Compare the outfits.`, - sessionBrief: { memories: [], assets: [{ videoId: saved, version: 'a'.repeat(64), kind: 'frame', current: true, collectedAt: 1, details: { timestampMs: 30000 } }] } }); - expect(route).toMatchObject({ route: 'topic_research', frameAccess: both }); - expect(model.doGenerateCalls).toHaveLength(2); - expect(JSON.stringify(model.doGenerateCalls[1]!.prompt)).toContain('saved 0:30 frame of abcdefghijk'); - }); - it('pins inspect_video provider calls to the classified video', async () => { const video = vi.fn(async () => ({ cacheStatus: 'miss' as const, value: {} as never })); const provider = createCapabilityProvider(providerWith({ video }), { @@ -1252,7 +1261,8 @@ describe('YouTube agent capability router', () => { }); const toolNames = model.doGenerateCalls[0]?.tools?.map((candidate) => candidate.name); - expect(toolNames).toEqual([...INSPECT_VIDEO_TOOL_NAMES]); + // The route granted no frame capture, so the capture tool is closed. + expect(toolNames).toEqual(INSPECT_VIDEO_TOOL_NAMES.filter(name => name !== 'get_video_frames')); expect(JSON.stringify(model.doGenerateCalls[0]?.prompt)).toContain('Pinned video ID: abcdefghijk'); expect(JSON.stringify(model.doGenerateCalls[0]?.prompt)).toContain('Start with this video.'); expect(JSON.stringify(model.doGenerateCalls[0]?.prompt)).toContain('I inspected its main argument.'); diff --git a/platform/test/youtube-stored-analysis.test.ts b/platform/test/youtube-stored-analysis.test.ts index 9938e8a4..05cdc238 100644 --- a/platform/test/youtube-stored-analysis.test.ts +++ b/platform/test/youtube-stored-analysis.test.ts @@ -115,12 +115,18 @@ function setup() { } const kinds = ['transcript', 'frame', 'storyboard_sheet'] as const; +/** The session's saved frames as the runtime passes them to the answer builder. */ +function inventoryOf(context: AgentToolContext) { + return context.session!.brief().assets.flatMap(asset => asset.kind === 'frame' + ? [{ version: asset.version, videoId: asset.videoId, ...(typeof asset.details.timestampMs === 'number' ? { timestampMs: asset.details.timestampMs } : {}) }] : []); +} + /** Finalizes through the application's answer builder, which owns the frame activity notes. */ function finalizeThroughApplication(context: AgentToolContext, packets: EvidencePacket[], access: FrameAccess, earlier: EvidencePacket[] = []) { context.runId = crypto.randomUUID(); const identity = { runId: context.runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }; const finalize = vi.fn(async (_id: string, input: FinalizeAnswerInput) => buildAgentTurnResult(identity, { userId: 'user', creditsRemaining: 100 }, - input, [...earlier, ...packets], 0, undefined, { access })); + input, [...earlier, ...packets], 0, undefined, { access, inventory: inventoryOf(context) })); context.finalize = finalize; return async () => (await finalize.mock.results.at(-1)!.value).answer as string; } @@ -349,7 +355,7 @@ describe('application notes on frame turns', () => { const answer = (text: string, packets: EvidencePacket[]) => buildAgentTurnResult({ runId: context.runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }, { userId: 'user', creditsRemaining: 100 }, { answer: text, intent: 'topic_research', confidence: 'medium', citations: [], artifacts: [], warnings: [] }, - [earlier, ...packets], 0, undefined, { access: context.frameAccess }).answer; + [earlier, ...packets], 0, undefined, { access: context.frameAccess, inventory: inventoryOf(context) }).answer; return { context, store, at, savedVersion, earlier, current, answer }; } const earlierNote = /The 0:30 observation( of "[^"]+")? comes from an image inspected in an earlier turn, not in this one\./; @@ -362,10 +368,31 @@ describe('application notes on frame turns', () => { expect(text).not.toContain('The 1:00 observation'); }); - it('corrects an uncited claim of re-inspection when no saved frame was inspected', async () => { + it.each([ + 'I inspected the saved 0:30 frame again.', + 'Both frames show shirts.', + ])('states that no saved frame was re-inspected, whatever the answer says: %s', async claim => { const { current, answer } = await turns(); - const text = answer(`I re-inspected the saved 0:30 frame. The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current]); - expect(text.endsWith(claimNote)).toBe(true); + const text = answer(`${claim} The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current]); + expect(text).toContain(claimNote); + expect(text).toContain('I retrieved the 1:00 frame and inspected it.'); + }); + + it('judges each citation by its own image when an earlier batch held several', async () => { + const { context, store, at, current, answer } = await turns(); + // An earlier turn analyzed 0:30 and 0:45 together; this turn re-inspects only the exact 0:30 image. + context.runId = crypto.randomUUID(); + context.frameAccess = undefined; + const [thirty, fortyFive] = [30000, 45000].map(time => store.put('frame', videoId, at(videoId, time, 480), { timestampMs: time })); + const batch = await executeAnalyzeVideoFrames({ assetVersions: [thirty!, fortyFive!], focus: 'What is he wearing?' }, context, 'batch'); + context.runId = current.packetId.split(':')[1]!; + context.frameAccess = [{ videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, { videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + const again = await analyze('frame', thirty!, context, 'What is he wearing?'); + const cited = batch.excerpts.find(excerpt => excerpt.startMs === 30000)!; + const text = answer(`The saved frame shows a shirt [cite:${cited.id}] [cite:${current.excerpts[0]!.id}].`, [batch, current, again]); + expect(text).toContain('I re-inspected the saved 0:30 frame'); + expect(text).not.toMatch(earlierNote); + expect(text).not.toContain(claimNote); }); it('keeps the earlier-turn note when this turn inspected a different image at the same moment', async () => { From 7f8cc6b9f862f0a42117e0780677e07c282cf863 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sat, 7 Nov 2026 22:59:47 +0530 Subject: [PATCH 05/12] Keep fixed times through discovery, account for named saved frames, date saved images Review of 4f38472: - A frameAccess entry without videoId fixes capture times for videos discovery will choose, and agent_selected is rejected whenever the message names a position, so delegating the video no longer delegates the time. - Every saved frame at a time the user names must be listed on its own video or in setAsideFrames, so a plan that drops the saved half of a comparison is retried, and an explicit exclusion is still accepted. - An image counts as saved when the session stored it before the run started, so a capture served from session storage is a re-inspection. - Citations are placed by the inventory or by single-image packets; when neither places one, a partial re-inspection is reported as unrecorded, not as missed. --- docs/api-reference/openapi.json | 4 - platform/src/agents/agent-runtime-do.ts | 6 +- platform/src/agents/contracts.ts | 3 +- platform/src/agents/finalizer.ts | 4 +- .../youtube/tools/get-video-frames.ts | 12 +-- .../youtube/tools/stored-analysis.ts | 2 +- .../src/agents/research/capability-router.ts | 40 +++++-- .../src/agents/research/research-agent.ts | 3 +- platform/src/agents/runtime/frame-access.ts | 101 +++++++++++------- platform/test/fixtures/analysis-session.ts | 7 +- platform/test/youtube-agent-router.test.ts | 31 +++++- platform/test/youtube-stored-analysis.test.ts | 58 ++++++++-- 12 files changed, 195 insertions(+), 76 deletions(-) diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json index a7747442..f9d1b098 100644 --- a/docs/api-reference/openapi.json +++ b/docs/api-reference/openapi.json @@ -7693,7 +7693,6 @@ } }, "required": [ - "videoId", "savedTimestampsMs", "captureTimestampsMs" ], @@ -7838,7 +7837,6 @@ } }, "required": [ - "videoId", "savedTimestampsMs", "captureTimestampsMs" ], @@ -9581,7 +9579,6 @@ } }, "required": [ - "videoId", "savedTimestampsMs", "captureTimestampsMs" ], @@ -9726,7 +9723,6 @@ } }, "required": [ - "videoId", "savedTimestampsMs", "captureTimestampsMs" ], diff --git a/platform/src/agents/agent-runtime-do.ts b/platform/src/agents/agent-runtime-do.ts index 5668635c..402bd98f 100644 --- a/platform/src/agents/agent-runtime-do.ts +++ b/platform/src/agents/agent-runtime-do.ts @@ -1051,9 +1051,9 @@ export class AgentRuntimeDO extends Agent { requestedVideoIds: decision.route === 'inspect_video' ? [decision.videoId] : decision.comparisonVideoIds ?? [], // An application fallback is checked against what this run had, not earlier turns' cited content. runPackets: this.admittedRunEvidence(runId, history), - } : undefined, isFrameTurn(decision) ? { access: frameList(decision), inventory: this.sessionStore.brief().assets.flatMap(asset => - asset.kind === 'frame' ? [{ version: asset.version, videoId: asset.videoId, - ...(typeof asset.details.timestampMs === 'number' ? { timestampMs: asset.details.timestampMs } : {}) }] : []) } : undefined); + } : undefined, isFrameTurn(decision) ? { access: frameList(decision), startedAt: run.created_at, + inventory: this.sessionStore.brief().assets.flatMap(asset => asset.kind === 'frame' ? [{ version: asset.version, videoId: asset.videoId, + collectedAt: asset.collectedAt, ...(typeof asset.details.timestampMs === 'number' ? { timestampMs: asset.details.timestampMs } : {}) }] : []) } : undefined); // Hash the accepted text: unmatched citations may have been removed from the input. const answerHash = createHash('sha256').update(result.answer).digest('hex'); const serialized = JSON.stringify(result); diff --git a/platform/src/agents/contracts.ts b/platform/src/agents/contracts.ts index 11e9b2c1..04812823 100644 --- a/platform/src/agents/contracts.ts +++ b/platform/src/agents/contracts.ts @@ -15,7 +15,8 @@ export const visualRequirementsSchema = z.array(z.string().trim().min(1).max(200 const frameTimes = z.array(z.number().int().nonnegative()).max(24); const frameAccessEntrySchema = z.object({ - videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), + /** Absent: capture times for whichever videos discovery selects. */ + videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/).optional(), savedTimestampsMs: frameTimes, captureTimestampsMs: frameTimes, }); diff --git a/platform/src/agents/finalizer.ts b/platform/src/agents/finalizer.ts index 7cc11de4..fc26048f 100644 --- a/platform/src/agents/finalizer.ts +++ b/platform/src/agents/finalizer.ts @@ -31,7 +31,7 @@ export function buildAgentTurnResult( creditsCharged: number, durationLimitContext?: DurationLimitAnswerContext, /** Present when the route asked for frame work: its exact access, and the session's saved frames. */ - frameTurn?: { access?: FrameAccess; inventory: readonly InventoryFrame[] }, + frameTurn?: { access?: FrameAccess; inventory: readonly InventoryFrame[]; startedAt: number }, ): AgentTurnResult { const markers = answerCitationIds(input.answer); const citations: AgentCitation[] = []; @@ -84,7 +84,7 @@ export function buildAgentTurnResult( const notes = [...playlistCoverage.notes]; // The application, not the model, states which frames this turn actually inspected. if (frameTurn && input.intent !== 'clarification' && input.intent !== 'rejected') { - notes.push(...frameTurnNotes({ runId: identity.runId, access: frameTurn.access, inventory: frameTurn.inventory, packets, cited: citedObservations })); + notes.push(...frameTurnNotes({ runId: identity.runId, startedAt: frameTurn.startedAt, access: frameTurn.access, inventory: frameTurn.inventory, packets, cited: citedObservations })); } const disclosedAnswer = appendNotices(answer, notes); diff --git a/platform/src/agents/providers/youtube/tools/get-video-frames.ts b/platform/src/agents/providers/youtube/tools/get-video-frames.ts index cdd939d2..fd9c1d05 100644 --- a/platform/src/agents/providers/youtube/tools/get-video-frames.ts +++ b/platform/src/agents/providers/youtube/tools/get-video-frames.ts @@ -27,13 +27,13 @@ export const getVideoFramesInputSchema = frameRequestSchema.extend({ export function createGetVideoFramesTool(context: AgentToolContext) { const schema = getVideoFramesInputSchema.omit({ focus: true }); const captures = context.frameAccess?.filter(entry => entry.captureTimestampsMs.length); + const times = captures?.length ? z.array(z.literal([...new Set(captures.flatMap(entry => entry.captureTimestampsMs))])).min(1).max(6) : undefined; + const videos = captures?.every(entry => entry.videoId) ? captures.map(entry => entry.videoId!) : undefined; return tool({ description: 'Retrieve or reuse up to six video frames at millisecond timestamps. Supply analysis with your question to analyze immediately while images are saved. Omit analysis for retrieval only. Returns saved assetVersions, dimensions and failures, without image bytes. Use analyze_video_frames for questions about versions already saved. If suitable frames already exist in session inventory, analyze them directly. Still images cannot establish motion or speech.', // Per-video pairing is enforced at execution; the schema narrows what the model can propose. - inputSchema: captures?.length ? schema.extend({ - videoId: z.literal(captures.map(entry => entry.videoId)), - timestampsMs: z.array(z.literal([...new Set(captures.flatMap(entry => entry.captureTimestampsMs))])).min(1).max(6), - }) : schema, + // An entry without a video fixes only the times, for whichever video discovery selects. + inputSchema: !times ? schema : videos ? schema.extend({ videoId: z.literal(videos), timestampsMs: times }) : schema.extend({ timestampsMs: times }), outputSchema: evidencePacketSchema, // Previews are dashboard images with random IDs; the model analyzes assetVersions instead. toModelOutput: ({ output }) => ({ type: 'text', value: JSON.stringify({ @@ -49,10 +49,10 @@ export function createGetVideoFramesTool(context: AgentToolContext) { export function executeGetVideoFrames(input: z.input, context: AgentToolContext, toolCallId: string) { const parsed = getVideoFramesInputSchema.parse(input); const scope = context.frameAccess; - const allowed = scope?.find(entry => entry.videoId === parsed.videoId)?.captureTimestampsMs ?? []; + const allowed = (scope?.find(entry => entry.videoId === parsed.videoId) ?? scope?.find(entry => !entry.videoId))?.captureTimestampsMs ?? []; if (scope && parsed.timestampsMs.some(time => !allowed.includes(time))) { const permitted = scope.filter(entry => entry.captureTimestampsMs.length) - .map(entry => `${entry.captureTimestampsMs.join(', ')} ms on ${entry.videoId}`).join('; '); + .map(entry => `${entry.captureTimestampsMs.join(', ')} ms on ${entry.videoId ?? 'videos found by discovery'}`).join('; '); throw new Error(`Capture is limited to ${permitted}. Use saved analysis for saved positions. If it fails, report the gap.`); } if (!parsed.analysis && parsed.videoId === context.pinnedVideoId && diff --git a/platform/src/agents/providers/youtube/tools/stored-analysis.ts b/platform/src/agents/providers/youtube/tools/stored-analysis.ts index a9cc4d61..d0cd5250 100644 --- a/platform/src/agents/providers/youtube/tools/stored-analysis.ts +++ b/platform/src/agents/providers/youtube/tools/stored-analysis.ts @@ -73,7 +73,7 @@ export async function readAnalysisAssets( if (scope) { // Read the position from the immutable payload, even when older inventory metadata omits it. const time = storedFramePosition.parse(stored.value).frames[0]!.timestampMs; - const entry = scope.find(item => item.videoId === stored.asset.videoId); + const entry = scope.find(item => item.videoId === stored.asset.videoId) ?? scope.find(item => !item.videoId); const allowed = entry && (entry.savedTimestampsMs.includes(time) || (!requiresRetrieval && entry.captureTimestampsMs.includes(time))); if (!allowed) throw new Error('This saved version is outside the requested inspection, or its position still requires a fresh capture.'); diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index d8ea02c4..dfcfdf34 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -18,6 +18,7 @@ import { type FinalizeAnswerInput, } from '../contracts'; import { sessionBriefForModel, type SessionBrief } from '../runtime/session-evidence'; +import { clockTime } from '../runtime/clock-time'; import { conversationAssistantMessage, type ConversationTurn } from '../runtime/conversation-memory'; import { assertModelCostAvailable, type AgentModelCostBudget } from '../runtime/model-budget'; import { AGENT_CLASSIFICATION_TIMEOUT_MS, withRunDeadline } from '../runtime/deadline'; @@ -30,7 +31,8 @@ const classifierDecisionSchema = z.object({ skill: z.enum(['verify_saved_evidence']).optional().describe('Verify saved evidence: use for follow-ups that read saved original captions/comments, check exact quotations or precise passage links, or correct source claims. It does not by itself prohibit fetching; set savedOnly separately. Omit for broad summaries and visual analysis.'), requestedEvidence: requestedEvidenceSchema.optional().describe('Evidence the user explicitly asks to be read before answering: original_quotes for exact quotations, verbatim wording or precise passage links from captions; comments for what viewers or commenters say. Research must read each or report it unavailable. Omit when not explicitly requested.'), savedOnly: z.boolean().optional().describe('True when the user requires existing saved evidence only or prohibits fresh retrieval. All provider access is disabled for this run; use saved evidence reads and report missing assets.'), - frameAccess: z.union([frameAccessListSchema, z.literal('agent_selected')]).optional().describe('Permission to capture new video frames; without it no new frame is captured. When the user fixes frame positions, saved or new, list one entry per video the turn touches: captureTimestampsMs holds only the permitted NEW capture positions in milliseconds, empty when nothing may be captured; savedTimestampsMs holds saved positions to re-inspect without recapture, even if their assets are missing. Required for exact-timestamp questions and for any limit on what may be captured. Use agent_selected only when new frames may help and the user leaves every position to you.'), + frameAccess: z.union([frameAccessListSchema, z.literal('agent_selected')]).optional().describe('Permission to capture new video frames; without it no new frame is captured. When the user fixes frame positions, saved or new, list one entry per video the turn touches: captureTimestampsMs holds only the permitted NEW capture positions in milliseconds, empty when nothing may be captured; savedTimestampsMs holds saved positions to re-inspect without recapture, even if their assets are missing. When the videos are still to be found by discovery, give one entry without videoId holding the capture times. Required for exact-timestamp questions and for any limit on what may be captured. Use agent_selected only when new frames may help and the user leaves every position to you.'), + setAsideFrames: z.array(z.object({ videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), timestampMs: z.number().int().nonnegative() })).max(24).optional().describe('Saved frames from the session inventory at a time the user named that this turn must not use: the user excluded them, or the time refers to another video. Every saved frame at a named time belongs either in its video\'s frameAccess entry or here.'), timestampedFrameQuestion: z.string().trim().min(1).max(500).optional().describe('For inspect_video: when the user supplies exact timestamps and asks what is visible, copy the focused visual question here. Include this for a request to describe each specified frame. Omit for image retrieval only or saved-only requests.'), refreshFramesOnly: z.boolean().optional().describe('For inspection or research, including multi-video comparisons: true when the user asks to freshly capture or refresh frames only. Reuse verified video metadata. Leave refreshEvidence unset unless other source evidence must also be refreshed.'), refreshEvidence: z.boolean().optional().describe('Except for a frames-only refresh, true only when the user explicitly asks to fetch again, refresh or get fresh source data. Choose an executable route in that case.'), @@ -178,7 +180,8 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio if (scope === 'agent_selected' && input.savedOnly === true) issue('frameAccess', 'Saved-only requests cannot capture frames. Omit frameAccess, or list only savedTimestampsMs.'); if (Array.isArray(scope)) { - const scoped = scope.map(entry => entry.videoId); + const scoped = scope.flatMap(entry => entry.videoId ? [entry.videoId] : []); + const open = scope.filter(entry => !entry.videoId); // Every scoped video must be a subject of the route, or its frames cannot be read or captured. const subjects = input.route === 'inspect_video' ? [input.videoId] : input.route === 'topic_research' && Array.isArray(input.comparisonVideoIds) ? input.comparisonVideoIds : []; @@ -186,6 +189,9 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio issue('frameAccess', 'Saved-only requests cannot capture frames. List only savedTimestampsMs, or omit savedOnly if the user authorized new captures.'); else if (scoped.some(id => !suppliedVideoIds.includes(id) || !subjects.includes(id)) || new Set(scoped).size !== scoped.length) issue('frameAccess', 'List one frameAccess entry per video, only for the route\'s subjects. When saved or new frames span several videos, use topic_research with every one in comparisonVideoIds.'); + else if (open.length && (open.length > 1 || scope.length > 1 || open.some(entry => entry.savedTimestampsMs.length) + || input.route !== 'topic_research' || input.comparisonVideoIds !== undefined)) + issue('frameAccess', 'An entry without videoId fixes capture times for videos that discovery will select. Use it alone, once, without saved positions, and only for topic_research without comparisonVideoIds.'); else if (scope.some(entry => new Set([...entry.savedTimestampsMs, ...entry.captureTimestampsMs]).size !== entry.savedTimestampsMs.length + entry.captureTimestampsMs.length)) issue('frameAccess', 'Give each timestamp exactly one role, saved inspection or fresh capture.'); } @@ -300,7 +306,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline 'Select skill verify_saved_evidence for follow-ups that recover saved original captions/comments, verify exact quotations or passage links, or correct source claims. This replaces general discovery and analysis with a verification workflow. The skill does not prohibit fetching: a request that also needs evidence not yet saved, such as comments, still gets it unless savedOnly is set. Omit the skill for broad summaries or visual analysis.', 'Set requestedEvidence for each kind of evidence the user explicitly asks for: original_quotes for exact quotations, verbatim wording or precise passage links; comments for what viewers or commenters say. Include every requested kind in one decision, alongside any explanation. Research cannot complete until each is read or reported unavailable, so choose an executable route.', 'Set savedOnly true whenever the user prohibits fresh retrieval or asks to use existing saved sources only. Keep this restriction on follow-ups when it remains in force. Missing saved evidence is a gap, never permission to fetch. Use the saved reader instead of transcript retrieval or discovery.', - 'New frames are captured only with frameAccess. When the user names frame positions, limits what may be captured, asks about exact timestamps, or re-inspects saved frames, list them in frameAccess with one entry per video. Set agent_selected only when new frames may help and the user leaves every position to you, including a plain frame refresh. Omit savedOnly when new captures are allowed. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. A prohibition on recapturing 10:30 still allows the requested 10:45. When the saved frame belongs to one video and the new capture to another, list both videos and choose topic_research with both in comparisonVideoIds, even if the message links only one. Times mentioned for comparison, narration or history do not authorize capture.', + 'New frames are captured only with frameAccess. When discovery will choose the videos but the user fixes the time, use one entry without videoId. When the session holds a saved frame at a time the user names, list it on its own video, or in setAsideFrames if the user excluded it or meant another video. When the user names frame positions, limits what may be captured, asks about exact timestamps, or re-inspects saved frames, list them in frameAccess with one entry per video. Set agent_selected only when new frames may help and the user leaves every position to you, including a plain frame refresh. Omit savedOnly when new captures are allowed. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. A prohibition on recapturing 10:30 still allows the requested 10:45. When the saved frame belongs to one video and the new capture to another, list both videos and choose topic_research with both in comparisonVideoIds, even if the message links only one. Times mentioned for comparison, narration or history do not authorize capture.', 'Requests for exact quotations from original saved captions/comments, precise passage-start links, or explicit original-asset reads require inspect_video or topic_research with the resolved subjects, even if prior summaries discuss the topic. Research can call search_originals without fetching YouTube. Quoting a source is different from quoting a prior conversation message. Do not route an original-source read directly to finalize.', 'For every context_answer set contextScope: history for listing, quoting, recalling or correcting conversation messages or preferences; video for claims about video content; mixed only when the requested answer needs both. A video URL inside a quoted earlier message does not require video evidence.', 'For finalize give a short routing reason, not a user-facing answer. Choose responseIntent clarification for missing scope, or rejected for unsupported requests. Do not use the legacy clarification or rejected routes for new decisions.', @@ -595,7 +601,7 @@ const ROUTE_FIELDS = { finalize: ['responseIntent', 'contextScope', 'historySelection', 'reason', 'comparisonVideoIds'], } as const satisfies Record; const SHARED_FIELDS = ['skill', 'savedOnly', 'requestedEvidence', 'answerDetail', 'numberedItemCount', 'refreshEvidence', 'refreshDynamicData', - 'visualEvidence', 'visualRequirements'] as const satisfies readonly DefaultedClassificationField[]; + 'visualEvidence', 'visualRequirements', 'setAsideFrames'] as const satisfies readonly DefaultedClassificationField[]; function candidateObject(value: unknown): Record | undefined { if (typeof value === 'string') { @@ -610,6 +616,9 @@ const prohibitsRecapture = (message: string) => /\b(?:without|never|do not|don[' /** Clock times, and durations introduced as a position ("at 90 seconds", "only 60 seconds"), not lengths ("a 10 minute video"). */ const FRAME_POSITION = /\b\d+:\d{2}\b|\b\d+\s*ms\b|(?:\b(?:at|from|around|only|to|until)|@)\s*\d+(?:\.\d+)?\s*(?:milliseconds?|s|secs?|seconds?|mins?|minutes?)\b/i; const namesFramePositions = (message: string) => FRAME_POSITION.test(message); +/** Clock times named in the message, in milliseconds. */ +const namedClockTimes = (message: string) => [...message.matchAll(/\b(?:(\d+):)?(\d{1,2}):(\d{2})\b/g)] + .map(([, hours, minutes, seconds]) => ((Number(hours ?? 0) * 60 + Number(minutes)) * 60 + Number(seconds)) * 1000); function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[]): ClassificationIssue[] { // A single linked video pins inspection to itself, which would leave another video's frames out of reach. @@ -622,8 +631,8 @@ function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifie // Capture is closed without frameAccess, so omission never widens it. These checks only ask // for a list where the classifier itself, or an explicit limit in the message, implies one. if (decision.route !== 'finalize' && !decision.savedOnly && !Array.isArray(decision.frameAccess)) { - const fixedVideos = decision.route === 'inspect_video' || !!decision.comparisonVideoIds?.length || videoIds.length === 1; - const limited = prohibitsRecapture(input.message) || (fixedVideos && namesFramePositions(input.message)); + // Delegating which video is chosen is not delegating when it is captured. + const limited = prohibitsRecapture(input.message) || namesFramePositions(input.message); if (decision.timestampedFrameQuestion || (decision.frameAccess === 'agent_selected' && limited)) { return [{ path: 'frameAccess', code: 'missing_capture_scope', message: 'The user fixed the frame positions. List the saved inspection positions and the permitted capture positions in frameAccess, one entry per video.' }]; } @@ -631,6 +640,20 @@ function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifie return [{ path: 'frameAccess', code: 'missing_capture_permission', message: 'A frame refresh needs capture permission. List the positions in frameAccess, or set agent_selected if the user left them to you.' }]; } } + // Each saved frame at a time the user named is accounted for on its own video: inspected, + // recaptured, or set aside. Otherwise a plan can drop it without the research ever being able to reach it. + if (decision.route !== 'finalize' && (decision.frameAccess || decision.refreshFramesOnly || decision.timestampedFrameQuestion || decision.visualEvidence !== 'none')) { + const named = new Set(namedClockTimes(input.message)); + const accounted = (videoId: string, time: number) => (Array.isArray(decision.frameAccess) && decision.frameAccess.some(entry => + entry.videoId === videoId && (entry.savedTimestampsMs.includes(time) || entry.captureTimestampsMs.includes(time)))) + || !!decision.setAsideFrames?.some(frame => frame.videoId === videoId && frame.timestampMs === time); + const omitted = input.sessionBrief?.assets.find(asset => asset.kind === 'frame' && typeof asset.details.timestampMs === 'number' + && named.has(asset.details.timestampMs) && !accounted(asset.videoId, asset.details.timestampMs)); + if (omitted) { + const time = clockTime(omitted.details.timestampMs as number); + return [{ path: 'frameAccess', code: 'saved_frame_unaccounted', message: `The session holds a saved ${time} frame of ${omitted.videoId}, and the user named ${time}. List it in that video's frameAccess entry (savedTimestampsMs to re-inspect, captureTimestampsMs to recapture), with that video in the route, or in setAsideFrames if the user excluded it or meant another video.` }]; + } + } const scope = comparisonScopeIssues(decision, input, videoIds); return scope.length ? scope : searchQueryNumberIssues(decision, input); } @@ -851,7 +874,8 @@ function expectedComparisonSubjects(input: CapabilityClassifierInput): string[] function finishClassification( decision: z.infer, videoIds: string[], channelIds: string[], explicitVideoIds: string[], message: string, ): CapabilityRouteDecision { - const { explicitSourceCount, ...classified } = withVisualAccess(decision); + // Set-aside frames only check the classification; the route carries what research may use. + const { explicitSourceCount, setAsideFrames: _setAside, ...classified } = withVisualAccess(decision); const pinned = !(decision.route === 'topic_research' && decision.playlistListing) && explicitVideoIds.length === 1 && !decision.comparisonVideoIds?.length && (decision.route !== 'finalize' || (decision.responseIntent === 'context_answer' && decision.contextScope !== 'history')); const route = pinned ? { ...classified, route: 'inspect_video' as const, videoId: explicitVideoIds[0]!, @@ -995,7 +1019,7 @@ export async function resolveCapabilityRoute(options: { // video after validation. Resuming must not run them without the access the user gave. const subjects = decision.route === 'inspect_video' ? [decision.videoId] : decision.route === 'topic_research' ? decision.comparisonVideoIds ?? [] : []; - if ('frameAccess' in decision && Array.isArray(decision.frameAccess) && decision.frameAccess.some(entry => !subjects.includes(entry.videoId))) { + if ('frameAccess' in decision && Array.isArray(decision.frameAccess) && decision.frameAccess.some(entry => entry.videoId && !subjects.includes(entry.videoId))) { return { route: 'finalize', responseIntent: 'clarification', reason: 'Clarify which video the saved and fresh frame positions refer to.' }; } return decision; diff --git a/platform/src/agents/research/research-agent.ts b/platform/src/agents/research/research-agent.ts index 741a4b8c..1c9546fc 100644 --- a/platform/src/agents/research/research-agent.ts +++ b/platform/src/agents/research/research-agent.ts @@ -753,7 +753,8 @@ async function runResearchAgentWithModelWithinDeadline(options: { // A failed capture closes only its own positions; other authorized captures stay required. if (frameScope && execution.toolName === 'get_video_frames') { const { videoId, timestampsMs } = execution.input as { videoId?: string; timestampsMs?: number[] }; - for (const time of timestampsMs ?? []) failedCaptures.add(`${videoId}:${time}`); + // A time fixed before discovery chose the video is closed by a failure on any video. + for (const time of timestampsMs ?? []) { failedCaptures.add(`${videoId}:${time}`); failedCaptures.add(`*:${time}`); } } toolFailures.set(execution.toolCallId, { toolCallId: execution.toolCallId, diff --git a/platform/src/agents/runtime/frame-access.ts b/platform/src/agents/runtime/frame-access.ts index e8b9889f..92b4d260 100644 --- a/platform/src/agents/runtime/frame-access.ts +++ b/platform/src/agents/runtime/frame-access.ts @@ -6,43 +6,54 @@ import { frameCoverageSchema } from './model-evidence'; * Only packets produced by this run can establish that it did the work. */ export function frameActivity(scope: FrameAccess, runId: string, evidence: Iterable) { const packets = [...evidence]; - const named = scope.length > 1; - const saved: { videoId: string; time: number; inspected: boolean }[] = []; - const captures: { videoId: string; time: number; requested: boolean; retrieved: boolean; inspected: boolean }[] = []; + const own = packets.filter(packet => packet.packetId.startsWith(`packet:${runId}:`) && packet.kind === 'youtube_frames'); // Versions are tracked separately from positions: inspecting one video's frame // must never count as inspecting another video's frame at the same time. const inspectedVersions = new Set(); const retrievedVersions = new Set(); + const observed = new Map; retrieved: Set; inspected: Set }>(); + for (const packet of own) { + for (const artifact of packet.artifacts) { + if (artifact.type !== 'youtube_frame_analysis' && artifact.type !== 'youtube_frame_retrieval') continue; + const coverage = frameCoverageSchema.safeParse(artifact.data); + if (!coverage.success) continue; + const analyzing = artifact.type === 'youtube_frame_analysis'; + for (const version of packet.assetVersions ?? []) (analyzing ? inspectedVersions : retrievedVersions).add(version); + // A partial inline analysis may cover fewer positions than its paired retrieval. + for (const videoId of new Set(packet.sources.flatMap(source => source.videoId ? [source.videoId] : []))) { + const sets = observed.get(videoId) ?? { requested: new Set(), retrieved: new Set(), inspected: new Set() }; + observed.set(videoId, sets); + if (!analyzing) for (const time of coverage.data.requestedTimestampsMs) sets.requested.add(time); + for (const frame of coverage.data.frames) (analyzing ? sets.inspected : sets.retrieved).add(frame.timestampMs); + } + } + } + const has = (videoId: string, kind: 'requested' | 'retrieved' | 'inspected', time: number) => !!observed.get(videoId)?.[kind].has(time); + const titleOf = (videoId: string) => packets.flatMap(packet => packet.sources).find(source => source.videoId === videoId && source.title)?.title ?? videoId; + const saved: { videoId: string; time: number; inspected: boolean }[] = []; + const captures: { videoId: string; time: number; requested: boolean; retrieved: boolean; inspected: boolean }[] = []; const sentences: string[] = []; for (const { videoId, savedTimestampsMs, captureTimestampsMs } of scope) { - const inspected = new Set(); - const retrieved = new Set(); - const requested = new Set(); - for (const packet of packets) { - if (!packet.packetId.startsWith(`packet:${runId}:`) || packet.kind !== 'youtube_frames' - || !packet.sources.some(source => source.videoId === videoId)) continue; - // A partial inline analysis may cover fewer positions than its paired retrieval. - for (const artifact of packet.artifacts) { - if (artifact.type !== 'youtube_frame_analysis' && artifact.type !== 'youtube_frame_retrieval') continue; - const coverage = frameCoverageSchema.safeParse(artifact.data); - if (!coverage.success) continue; - const analyzing = artifact.type === 'youtube_frame_analysis'; - for (const version of packet.assetVersions ?? []) (analyzing ? inspectedVersions : retrievedVersions).add(version); - if (!analyzing) for (const time of coverage.data.requestedTimestampsMs) requested.add(time); - for (const frame of coverage.data.frames) (analyzing ? inspected : retrieved).add(frame.timestampMs); + if (!videoId) { + // Times fixed before discovery chose the videos: report each video that was captured there. + for (const time of captureTimestampsMs) { + const videos = [...observed.keys()].filter(id => has(id, 'retrieved', time)); + const requested = [...observed.keys()].some(id => has(id, 'requested', time)); + captures.push({ videoId: '*', time, requested, retrieved: videos.length > 0, inspected: videos.some(id => has(id, 'inspected', time)) }); + sentences.push(...(videos.length ? videos.map(id => `I retrieved the ${clockTime(time)} frame of "${titleOf(id)}"${has(id, 'inspected', time) ? ' and inspected it' : ''}.`) + : [requested ? `I requested ${clockTime(time)} frames, but none was returned.` : `I have no confirmed retrieval of a ${clockTime(time)} frame in this turn.`])); } + continue; } // With several videos, each position names its video so a gap cannot be read as another video's. - const title = named ? packets.flatMap(packet => packet.sources) - .find(source => source.videoId === videoId && source.title)?.title ?? videoId : undefined; - const frame = (time: number) => `${clockTime(time)} frame${title ? ` of "${title}"` : ''}`; + const frame = (time: number) => `${clockTime(time)} frame${scope.length > 1 ? ` of "${titleOf(videoId)}"` : ''}`; for (const time of savedTimestampsMs) { - saved.push({ videoId, time, inspected: inspected.has(time) }); - sentences.push(inspected.has(time) ? `I re-inspected the saved ${frame(time)}.` + saved.push({ videoId, time, inspected: has(videoId, 'inspected', time) }); + sentences.push(has(videoId, 'inspected', time) ? `I re-inspected the saved ${frame(time)}.` : `I could not re-inspect the saved ${frame(time)} in this turn.`); } for (const time of captureTimestampsMs) { - const item = { videoId, time, requested: requested.has(time), retrieved: retrieved.has(time), inspected: inspected.has(time) }; + const item = { videoId, time, requested: has(videoId, 'requested', time), retrieved: has(videoId, 'retrieved', time), inspected: has(videoId, 'inspected', time) }; captures.push(item); sentences.push(item.retrieved ? `I retrieved the ${frame(time)}${item.inspected ? ' and inspected it' : ''}.` : item.requested ? `I requested the ${frame(time)}, but no frame was returned.` @@ -65,7 +76,7 @@ export function isFrameTurn(decision: CapabilityRouteDecision): boolean { } /** A saved frame as the session inventory records it. */ -export interface InventoryFrame { version: string; videoId: string; timestampMs?: number } +export interface InventoryFrame { version: string; videoId: string; timestampMs?: number; collectedAt: number } /** * The application's account of a frame turn, built from what the route requested and what @@ -73,42 +84,54 @@ export interface InventoryFrame { version: string; videoId: string; timestampMs? * - each listed position: re-inspected, retrieved, inspected, or a gap; * - saved frames re-inspected outside the list, or that none was; * - cited observations whose exact image this turn did not inspect again. + * An image is saved when the session stored it before this run started, however this run reached it. */ export function frameTurnNotes(options: { - runId: string; access?: FrameAccess; inventory: readonly InventoryFrame[]; packets: readonly EvidencePacket[]; + runId: string; startedAt: number; access?: FrameAccess; inventory: readonly InventoryFrame[]; packets: readonly EvidencePacket[]; cited: ReadonlyArray<{ packet: EvidencePacket; startMs: number; videoId?: string; title?: string }>; }): string[] { const { runId, access, inventory, packets } = options; const own = packets.filter(packet => packet.packetId.startsWith(`packet:${runId}:`) && packet.kind === 'youtube_frames'); - const versions = (type: string) => new Set(own.filter(packet => packet.artifacts.some(artifact => artifact.type === type)) + const analyzed = new Set(own.filter(packet => packet.artifacts.some(artifact => artifact.type === 'youtube_frame_analysis')) .flatMap(packet => packet.assetVersions ?? [])); - const analyzed = versions('youtube_frame_analysis'); - const retrieved = versions('youtube_frame_retrieval'); - const frames = new Map(inventory.map(frame => [frame.version, frame])); + const savedBefore = inventory.filter(frame => frame.collectedAt < options.startedAt); + // Where each image sits: from the inventory, or from any packet that holds exactly one image. + const positions = new Map(inventory.flatMap(frame => frame.timestampMs === undefined ? [] : [[frame.version, `${frame.videoId}:${frame.timestampMs}`] as const])); + for (const packet of packets) { + const coverage = packet.kind === 'youtube_frames' && packet.assetVersions?.length === 1 + ? packet.artifacts.map(artifact => frameCoverageSchema.safeParse(artifact.data)).find(result => result.success && result.data.frames.length === 1) : undefined; + const videoId = packet.sources.find(source => source.videoId)?.videoId; + if (coverage?.success && videoId && !positions.has(packet.assetVersions![0]!)) + positions.set(packet.assetVersions![0]!, `${videoId}:${coverage.data.frames[0]!.timestampMs}`); + } const title = (videoId: string) => packets.flatMap(packet => packet.sources).find(source => source.videoId === videoId && source.title)?.title; const named = (videoId: string, time: number) => `${clockTime(time)} frame${title(videoId) ? ` of "${title(videoId)}"` : ''}`; const notes: string[] = []; if (access?.length) notes.push(frameActivity(access, runId, packets).report); - const listedSaved = new Set(access?.flatMap(entry => entry.savedTimestampsMs.map(time => `${entry.videoId}:${time}`))); - const reinspected = [...analyzed].filter(version => !retrieved.has(version)); - for (const frame of reinspected.flatMap(version => frames.get(version) ?? [])) { - if (frame.timestampMs !== undefined && !listedSaved.has(`${frame.videoId}:${frame.timestampMs}`)) + const listed = new Set(access?.flatMap(entry => entry.videoId + ? [...entry.savedTimestampsMs, ...entry.captureTimestampsMs].map(time => `${entry.videoId}:${time}`) : [])); + const reinspected = savedBefore.filter(frame => analyzed.has(frame.version)); + for (const frame of reinspected) { + if (frame.timestampMs !== undefined && !listed.has(`${frame.videoId}:${frame.timestampMs}`)) notes.push(`I re-inspected the saved ${named(frame.videoId, frame.timestampMs)}.`); } - if (!reinspected.length && inventory.some(frame => !retrieved.has(frame.version))) - notes.push('No frame saved before this turn was re-inspected in this turn.'); + if (savedBefore.length && !reinspected.length) notes.push('No frame saved before this turn was re-inspected in this turn.'); - // Each citation is judged by the image behind it, not by its batch or its moment. + // Each citation is judged by the image behind it. When the record cannot tie the citation + // to one image, the answer says so rather than guessing from the rest of the batch. const earlier = new Map(); for (const { packet, startMs, videoId, title: cited } of options.cited) { const key = `${videoId}:${startMs}`; if (packet.packetId.startsWith(`packet:${runId}:`) || packet.kind !== 'youtube_frames' || earlier.has(key) || !packet.artifacts.some(artifact => artifact.type === 'youtube_frame_analysis')) continue; const all = packet.assetVersions ?? []; - const exact = all.filter(version => frames.get(version)?.videoId === videoId && frames.get(version)?.timestampMs === startMs); + const exact = all.filter(version => positions.get(version) === key); + const of = cited ? ` of "${cited}"` : ''; if (exact.length ? exact.some(version => analyzed.has(version)) : all.length > 0 && all.every(version => analyzed.has(version))) continue; - earlier.set(key, `The ${clockTime(startMs)} observation${cited ? ` of "${cited}"` : ''} comes from an image inspected in an earlier turn, not in this one.`); + earlier.set(key, !exact.length && all.some(version => analyzed.has(version)) + ? `It is not recorded whether the cited ${clockTime(startMs)} observation${of} was re-inspected in this turn.` + : `The ${clockTime(startMs)} observation${of} comes from an image inspected in an earlier turn, not in this one.`); } return [...notes, ...earlier.values()]; } diff --git a/platform/test/fixtures/analysis-session.ts b/platform/test/fixtures/analysis-session.ts index a5be6545..ddbd69dd 100644 --- a/platform/test/fixtures/analysis-session.ts +++ b/platform/test/fixtures/analysis-session.ts @@ -14,7 +14,8 @@ export function attachTestAssetStore(context: AgentToolContext) { details: SessionAsset['details'] = {}, ) => { const version = (++ordinal).toString(16).padStart(64, '0'); - saved.set(version, { asset: { version, kind, videoId, current: true, collectedAt: 1, details }, value }); + // Collection order stands in for time, so a test can place a run start between saves. + saved.set(version, { asset: { version, kind, videoId, current: true, collectedAt: ordinal, details }, value }); return version; }; context.session = { @@ -87,5 +88,7 @@ export function attachTestAssetStore(context: AgentToolContext) { generation++; if (version) saved.delete(version); else saved.clear(); }; - return { saved, put, remove }; + /** A run start after every save so far. */ + const runStart = () => ordinal + 0.5; + return { saved, put, remove, runStart }; } diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index 45d02267..ef6aa8e5 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -1173,14 +1173,43 @@ describe('YouTube agent capability router', () => { it('accepts a capture list that excludes a saved frame the user set aside', async () => { const videoId = 'abcdefghijk'; const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; - const model = sequenceClassifier([{ route: 'inspect_video', videoId, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'], frameAccess }]); + const model = sequenceClassifier([{ route: 'inspect_video', videoId, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'], frameAccess, + setAsideFrames: [{ videoId, timestampMs: 30000 }] }]); const route = await classifyCapabilityWithModel({ model, message: `Ignore the old 0:30 frame entirely. Capture only 1:00 of https://youtu.be/${videoId}.`, sessionBrief: { memories: [], assets: [{ videoId, version: 'a'.repeat(64), kind: 'frame', current: true, collectedAt: 1, details: { timestampMs: 30000 } }] }, signal: new AbortController().signal }); expect(route).toMatchObject({ route: 'inspect_video', frameAccess }); + expect(route).not.toHaveProperty('setAsideFrames'); expect(model.doGenerateCalls).toHaveLength(1); }); + it('keeps a fixed capture time when discovery chooses the video', async () => { + const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', visualEvidence: 'required', visualRequirements: ['dough'] }; + const frameAccess = [{ savedTimestampsMs: [], captureTimestampsMs: [30000] }]; + const model = sequenceClassifier([{ ...discovery, frameAccess: 'agent_selected' }, { ...discovery, frameAccess }]); + const route = await classifyCapabilityWithModel({ model, message: 'Find a sourdough tutorial and capture only 0:30.', signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'topic_research', frameAccess }); + expect(model.doGenerateCalls).toHaveLength(2); + }); + + it.each([ + ['different times', 30000, 60000, 'Reuse the saved 0:30 frame and capture 1:00 of https://youtu.be/lmnopqrstuv. Compare the outfits.'], + ['the same time', 30000, 30000, 'Re-inspect the saved 0:30 frame and capture a new 0:30 frame of https://youtu.be/lmnopqrstuv. Compare the outfits.'], + ])('recovers both requested operations when a plan drops the saved video (%s)', async (_name, savedTime, captureTime, message) => { + const [saved, linked] = ['abcdefghijk', 'lmnopqrstuv']; + const visual = { visualEvidence: 'required', visualRequirements: ['outfits'] }; + const both = [{ videoId: saved, savedTimestampsMs: [savedTime], captureTimestampsMs: [] }, { videoId: linked, savedTimestampsMs: [], captureTimestampsMs: [captureTime] }]; + const model = sequenceClassifier([ + { route: 'inspect_video', videoId: linked, ...visual, frameAccess: [both[1]] }, + { route: 'topic_research', comparisonVideoIds: [saved, linked], ...visual, frameAccess: both }, + ]); + const route = await classifyCapabilityWithModel({ model, message, signal: new AbortController().signal, + sessionBrief: { memories: [], assets: [{ videoId: saved, version: 'a'.repeat(64), kind: 'frame', current: true, collectedAt: 1, details: { timestampMs: savedTime } }] } }); + expect(route).toMatchObject({ route: 'topic_research', comparisonVideoIds: [saved, linked], frameAccess: both }); + expect(model.doGenerateCalls).toHaveLength(2); + expect(JSON.stringify(model.doGenerateCalls[1]!.prompt)).toContain(`saved 0:30 frame of ${saved}`); + }); + it.each([ ['a named position', 'Capture only 1:00 of https://youtu.be/abcdefghijk. No other timestamps.'], ['a recapture prohibition', 'Look at https://youtu.be/abcdefghijk again without recapturing the saved frame, refresh the rest'], diff --git a/platform/test/youtube-stored-analysis.test.ts b/platform/test/youtube-stored-analysis.test.ts index 05cdc238..5ca027f6 100644 --- a/platform/test/youtube-stored-analysis.test.ts +++ b/platform/test/youtube-stored-analysis.test.ts @@ -118,15 +118,16 @@ const kinds = ['transcript', 'frame', 'storyboard_sheet'] as const; /** The session's saved frames as the runtime passes them to the answer builder. */ function inventoryOf(context: AgentToolContext) { return context.session!.brief().assets.flatMap(asset => asset.kind === 'frame' - ? [{ version: asset.version, videoId: asset.videoId, ...(typeof asset.details.timestampMs === 'number' ? { timestampMs: asset.details.timestampMs } : {}) }] : []); + ? [{ version: asset.version, videoId: asset.videoId, collectedAt: asset.collectedAt, + ...(typeof asset.details.timestampMs === 'number' ? { timestampMs: asset.details.timestampMs } : {}) }] : []); } /** Finalizes through the application's answer builder, which owns the frame activity notes. */ -function finalizeThroughApplication(context: AgentToolContext, packets: EvidencePacket[], access: FrameAccess, earlier: EvidencePacket[] = []) { +function finalizeThroughApplication(context: AgentToolContext, packets: EvidencePacket[], access: FrameAccess, startedAt: number) { context.runId = crypto.randomUUID(); const identity = { runId: context.runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }; const finalize = vi.fn(async (_id: string, input: FinalizeAnswerInput) => buildAgentTurnResult(identity, { userId: 'user', creditsRemaining: 100 }, - input, [...earlier, ...packets], 0, undefined, { access, inventory: inventoryOf(context) })); + input, packets, 0, undefined, { access, inventory: inventoryOf(context), startedAt })); context.finalize = finalize; return async () => (await finalize.mock.results.at(-1)!.value).answer as string; } @@ -134,9 +135,9 @@ function finalizeThroughApplication(context: AgentToolContext, packets: Evidence it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and only fetches 10:45 (saved image %s)', async savedState => { const { context, provider, store, frames, packets } = setup(); const frameAccess = [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]; - const answer = finalizeThroughApplication(context, packets, frameAccess); const at = (timestampMs: number) => ({ ...frames, frames: [{ ...frames.frames[0]!, timestampMs }] }); const saved = store.put('frame', videoId, at(630000), { timestampMs: 630000 }); + const answer = finalizeThroughApplication(context, packets, frameAccess, store.runStart()); if (savedState === 'missing') store.remove(saved); provider.frames.mockResolvedValue({ value: at(645000), cacheStatus: 'miss' }); context.analyzeFrames = vi.fn(async ({ frames }) => { @@ -185,9 +186,9 @@ it('re-inspects a saved frame of one video while capturing only the other video' const { context, provider, store, frames, packets } = setup(); const other = 'lmnopqrstuv'; const frameAccess = [{ videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, { videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; - const answer = finalizeThroughApplication(context, packets, frameAccess); const at = (id: string, timestampMs: number) => ({ ...frames, videoId: id, frames: [{ ...frames.frames[0]!, timestampMs }] }); const saved = store.put('frame', videoId, at(videoId, 30000), { timestampMs: 30000 }); + const answer = finalizeThroughApplication(context, packets, frameAccess, store.runStart()); provider.frames.mockResolvedValue({ value: at(other, 60000), cacheStatus: 'miss' }); context.analyzeFrames = vi.fn(async ({ frames }) => ({ findings: [{ observation: 'A blue shirt.', timestampsMs: [frames.frames[0]!.timestampMs] }], warnings: [] })); @@ -349,14 +350,17 @@ describe('application notes on frame turns', () => { const savedVersion = store.put('frame', videoId, at(videoId, 30000), { timestampMs: 30000 }); const earlier = await analyze('frame', savedVersion, context, 'What is he wearing?'); context.runId = crypto.randomUUID(); + let startedAt = store.runStart(); context.frameAccess = [{ videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; provider.frames.mockResolvedValue({ value: at(other, 60000), cacheStatus: 'miss' }); const current = await executeGetVideoFrames({ videoId: other, timestampsMs: [60000], analysis: 'What is he wearing?' }, context, 'capture'); const answer = (text: string, packets: EvidencePacket[]) => buildAgentTurnResult({ runId: context.runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }, { userId: 'user', creditsRemaining: 100 }, { answer: text, intent: 'topic_research', confidence: 'medium', citations: [], artifacts: [], warnings: [] }, - [earlier, ...packets], 0, undefined, { access: context.frameAccess, inventory: inventoryOf(context) }).answer; - return { context, store, at, savedVersion, earlier, current, answer }; + [earlier, ...packets], 0, undefined, { access: context.frameAccess, inventory: inventoryOf(context), startedAt }).answer; + /** Moves this turn's start after saves made by a test's own earlier turn. */ + const restart = () => { startedAt = store.runStart(); }; + return { context, store, at, savedVersion, earlier, current, answer, restart }; } const earlierNote = /The 0:30 observation( of "[^"]+")? comes from an image inspected in an earlier turn, not in this one\./; const claimNote = 'No frame saved before this turn was re-inspected in this turn.'; @@ -378,13 +382,41 @@ describe('application notes on frame turns', () => { expect(text).toContain('I retrieved the 1:00 frame and inspected it.'); }); + it('counts an image served from session storage as a re-inspected saved frame', async () => { + const { savedVersion, current, answer } = await turns(); + // A capture request answered from the session returns the image saved by the earlier turn. + const reused = { ...current, assetVersions: [savedVersion] }; + const text = answer(`The frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [reused]); + expect(text).not.toContain(claimNote); + }); + + it.each([ + ['re-inspected alone', 'single', false], + ['re-inspected with another image', 'batch', true], + ])('handles a cited batch image without inventory positions when it was %s', async (_name, mode, uncertain) => { + const { context, store, at, current, answer, restart } = await turns(); + context.runId = crypto.randomUUID(); + context.frameAccess = undefined; + // Older inventory rows omit positions, so only the recorded analyses can place each image. + const [thirty, fortyFive, other] = [30000, 45000, 50000].map(time => store.put('frame', videoId, at(videoId, time, 480))); + const batch = await executeAnalyzeVideoFrames({ assetVersions: [thirty!, fortyFive!], focus: 'What is he wearing?' }, context, 'batch'); + restart(); + context.runId = current.packetId.split(':')[1]!; + const again = await executeAnalyzeVideoFrames({ assetVersions: mode === 'single' ? [thirty!] : [fortyFive!, other!], focus: 'What is he wearing?' }, context, 'again'); + const cited = batch.excerpts.find(excerpt => excerpt.startMs === 30000)!; + const text = answer(`The saved frame shows a shirt [cite:${cited.id}] [cite:${current.excerpts[0]!.id}].`, [batch, current, again]); + expect(text).not.toMatch(earlierNote); + expect(text.includes('It is not recorded whether the cited 0:30 observation')).toBe(uncertain); + }); + it('judges each citation by its own image when an earlier batch held several', async () => { - const { context, store, at, current, answer } = await turns(); + const { context, store, at, current, answer, restart } = await turns(); // An earlier turn analyzed 0:30 and 0:45 together; this turn re-inspects only the exact 0:30 image. context.runId = crypto.randomUUID(); context.frameAccess = undefined; const [thirty, fortyFive] = [30000, 45000].map(time => store.put('frame', videoId, at(videoId, time, 480), { timestampMs: time })); const batch = await executeAnalyzeVideoFrames({ assetVersions: [thirty!, fortyFive!], focus: 'What is he wearing?' }, context, 'batch'); + restart(); context.runId = current.packetId.split(':')[1]!; context.frameAccess = [{ videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, { videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; const again = await analyze('frame', thirty!, context, 'What is he wearing?'); @@ -413,6 +445,16 @@ describe('application notes on frame turns', () => { }); }); +it('enforces a capture time fixed before discovery on whichever video is chosen', async () => { + const { context, provider, frames } = setup(); + context.frameAccess = [{ savedTimestampsMs: [], captureTimestampsMs: [30000] }]; + provider.frames.mockResolvedValue({ value: { ...frames, videoId: 'lmnopqrstuv', frames: [{ ...frames.frames[0]!, timestampMs: 30000 }] }, cacheStatus: 'miss' }); + expect(() => executeGetVideoFrames({ videoId: 'lmnopqrstuv', timestampsMs: [60000] }, context, 'other-time')).toThrow('Capture is limited'); + await executeGetVideoFrames({ videoId: 'lmnopqrstuv', timestampsMs: [30000] }, context, 'fixed-time'); + expect((provider.frames.mock.calls as unknown as [{ videoId: string; timestampsMs: number[] }][])[0]![0]) + .toMatchObject({ videoId: 'lmnopqrstuv', timestampsMs: [30000] }); +}); + it('rejects forbidden positions and other videos before metadata, billing or extraction', () => { const { context, provider } = setup(); context.frameAccess = [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]; From 2257506362d0a9a709c9e4c2e680b8162d49ed5a Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sun, 8 Nov 2026 09:29:24 +0530 Subject: [PATCH 06/12] Attempt or explain every named saved frame Review of 7f8cc6b found two ways the saved half of a comparison could still be skipped: - Times written as quantities ('at 30 seconds', '1m30s', '1 minute 30 seconds') now count as named times for the saved-frame check; lengths such as 'a 30 second clip' do not. - The session inventory recovers a frame's position from its resource key when older rows omitted it, so the router check and completion gate can place it. When a saved frame still goes unused, the activity note says why: the plan included none, or the frame has no recoverable position. --- .../src/agents/research/capability-router.ts | 18 ++++++++++++++---- platform/src/agents/runtime/frame-access.ts | 8 +++++++- .../src/agents/runtime/session-evidence.ts | 9 ++++++++- .../test/session-evidence.integration.test.ts | 8 ++++++++ platform/test/youtube-agent-router.test.ts | 1 + platform/test/youtube-stored-analysis.test.ts | 15 ++++++++++++--- 6 files changed, 50 insertions(+), 9 deletions(-) diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index dfcfdf34..62f99877 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -616,9 +616,19 @@ const prohibitsRecapture = (message: string) => /\b(?:without|never|do not|don[' /** Clock times, and durations introduced as a position ("at 90 seconds", "only 60 seconds"), not lengths ("a 10 minute video"). */ const FRAME_POSITION = /\b\d+:\d{2}\b|\b\d+\s*ms\b|(?:\b(?:at|from|around|only|to|until)|@)\s*\d+(?:\.\d+)?\s*(?:milliseconds?|s|secs?|seconds?|mins?|minutes?)\b/i; const namesFramePositions = (message: string) => FRAME_POSITION.test(message); -/** Clock times named in the message, in milliseconds. */ -const namedClockTimes = (message: string) => [...message.matchAll(/\b(?:(\d+):)?(\d{1,2}):(\d{2})\b/g)] - .map(([, hours, minutes, seconds]) => ((Number(hours ?? 0) * 60 + Number(minutes)) * 60 + Number(seconds)) * 1000); +const UNIT_MS: ReadonlyArray<[RegExp, number]> = [[/^(?:ms|milliseconds?)$/i, 1], [/^(?:s|secs?|seconds?)$/i, 1_000], + [/^(?:m|mins?|minutes?)$/i, 60_000], [/^(?:h|hrs?|hours?)$/i, 3_600_000]]; +const unitMs = (unit: string) => UNIT_MS.find(([pattern]) => pattern.test(unit))?.[1] ?? 0; +/** A quantity of time, optionally followed by seconds ("1 minute 30 seconds", "1m30s"), not a length ("a 30 second clip"). */ +const TIME_QUANTITY = /\b(\d+(?:\.\d+)?)\s*-?\s*(milliseconds?|ms|hours?|hrs?|h|minutes?|mins?|m|seconds?|secs?|s)(?![a-z])(?:\s*(?:and\s*)?(\d+(?:\.\d+)?)\s*(seconds?|secs?|s)\b)?(?!\s*-?\s*(?:long|videos?|clips?|ads?|intros?|segments?|versions?|cuts?|shorts?|views?)\b)/gi; +/** Times named in the message, in milliseconds: clock times and spoken quantities. The saved-frame + * check uses this generously, since a false match costs one more classification attempt. */ +const namedTimes = (message: string) => [ + ...[...message.matchAll(/\b(?:(\d+):)?(\d{1,2}):(\d{2})\b/g)] + .map(([, hours, minutes, seconds]) => ((Number(hours ?? 0) * 60 + Number(minutes)) * 60 + Number(seconds)) * 1000), + ...[...message.matchAll(TIME_QUANTITY)].map(([, amount, unit, extra]) => + Math.round(Number(amount) * unitMs(unit!) + (extra ? Number(extra) * 1_000 : 0))), +]; function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[]): ClassificationIssue[] { // A single linked video pins inspection to itself, which would leave another video's frames out of reach. @@ -643,7 +653,7 @@ function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifie // Each saved frame at a time the user named is accounted for on its own video: inspected, // recaptured, or set aside. Otherwise a plan can drop it without the research ever being able to reach it. if (decision.route !== 'finalize' && (decision.frameAccess || decision.refreshFramesOnly || decision.timestampedFrameQuestion || decision.visualEvidence !== 'none')) { - const named = new Set(namedClockTimes(input.message)); + const named = new Set(namedTimes(input.message)); const accounted = (videoId: string, time: number) => (Array.isArray(decision.frameAccess) && decision.frameAccess.some(entry => entry.videoId === videoId && (entry.savedTimestampsMs.includes(time) || entry.captureTimestampsMs.includes(time)))) || !!decision.setAsideFrames?.some(frame => frame.videoId === videoId && frame.timestampMs === time); diff --git a/platform/src/agents/runtime/frame-access.ts b/platform/src/agents/runtime/frame-access.ts index 92b4d260..fe16135f 100644 --- a/platform/src/agents/runtime/frame-access.ts +++ b/platform/src/agents/runtime/frame-access.ts @@ -116,7 +116,13 @@ export function frameTurnNotes(options: { if (frame.timestampMs !== undefined && !listed.has(`${frame.videoId}:${frame.timestampMs}`)) notes.push(`I re-inspected the saved ${named(frame.videoId, frame.timestampMs)}.`); } - if (savedBefore.length && !reinspected.length) notes.push('No frame saved before this turn was re-inspected in this turn.'); + // Say why a saved frame went unused, so a skipped inspection is never silent. + const plannedSaved = !!access?.some(entry => entry.savedTimestampsMs.length); + if (savedBefore.length && !reinspected.length) notes.push(plannedSaved + ? 'No frame saved before this turn was re-inspected in this turn.' + : 'No frame saved before this turn was re-inspected in this turn, because its plan included none.'); + const unplaced = [...new Set(savedBefore.filter(frame => frame.timestampMs === undefined && !analyzed.has(frame.version)).map(frame => frame.videoId))]; + for (const videoId of unplaced) notes.push(`Saved frames of "${title(videoId) ?? videoId}" have no recorded position, so this turn could not match them to requested times or re-inspect them.`); // Each citation is judged by the image behind it. When the record cannot tie the citation // to one image, the answer says so rather than guessing from the rest of the batch. diff --git a/platform/src/agents/runtime/session-evidence.ts b/platform/src/agents/runtime/session-evidence.ts index 603ca3d1..9a32ea4f 100644 --- a/platform/src/agents/runtime/session-evidence.ts +++ b/platform/src/agents/runtime/session-evidence.ts @@ -42,6 +42,13 @@ export interface SessionAsset { current: boolean; details: Record; } +/** Stored details, with a frame's position recovered from its resource key when older rows omitted it. */ +function assetDetails(row: AssetRow): SessionAsset['details'] { + const details = JSON.parse(row.details_json) as SessionAsset['details']; + const position = row.kind === 'frame' && typeof details.timestampMs !== 'number' + ? /^frame:[\w-]{11}:\d+:(\d+)$/.exec(row.resource_key)?.[1] : undefined; + return position === undefined ? details : { ...details, timestampMs: Number(position) }; +} type AssetRow = { version: string; resource_key: string; @@ -202,7 +209,7 @@ export class SessionEvidenceStore implements SessionAccess { videoId: row.video_id, collectedAt: row.created_at, current: current.has(row.version), - details: JSON.parse(row.details_json), + details: assetDetails(row), })), memories: this.sql .exec<{ memory_json: string }>('SELECT memory_json FROM session_memories ORDER BY updated_at DESC') diff --git a/platform/test/session-evidence.integration.test.ts b/platform/test/session-evidence.integration.test.ts index 849e918c..11706dd3 100644 --- a/platform/test/session-evidence.integration.test.ts +++ b/platform/test/session-evidence.integration.test.ts @@ -22,6 +22,14 @@ import { bicycleHistory, gcnCaptions, officialCaptions, gcnAnalysis, officialAna import { readSavedEvidence, searchSavedEvidence, SAVED_EVIDENCE_READ_LIMIT } from '../src/agents/runtime/saved-evidence-reader'; import { annaId, kelogId, watercolorTranscript, watercolorCorrection, watercolorFollowup } from './fixtures/watercolor-session'; +test('the inventory recovers a saved frame position that older details omitted', async () => + within('legacy-frame-position', async (store, reopen) => { + // Older rows stored no timestampMs; the frame's resource key still records it. + await store.retrieve('frame:abcdefghijk:640:1000', 'frame', id, false, async () => ({ value: oneFrame(id), cacheStatus: 'miss' as const }), () => ({})); + const frame = reopen().brief().assets.find(asset => asset.kind === 'frame')!; + expect(frame.details.timestampMs).toBe(1000); + })); + test('saved original comments include positions 13 and 17 and long tails, with bounded pagination', async () => within('original-comments', async (store, reopen) => { const value = { videoId: id, replyContinuations: [], meta: transcript().meta, diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index ef6aa8e5..165637b3 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -1195,6 +1195,7 @@ describe('YouTube agent capability router', () => { it.each([ ['different times', 30000, 60000, 'Reuse the saved 0:30 frame and capture 1:00 of https://youtu.be/lmnopqrstuv. Compare the outfits.'], ['the same time', 30000, 30000, 'Re-inspect the saved 0:30 frame and capture a new 0:30 frame of https://youtu.be/lmnopqrstuv. Compare the outfits.'], + ['a time in seconds', 30000, 60000, 'Reuse the saved frame at 30 seconds and capture 1:00 of https://youtu.be/lmnopqrstuv. Compare the outfits.'], ])('recovers both requested operations when a plan drops the saved video (%s)', async (_name, savedTime, captureTime, message) => { const [saved, linked] = ['abcdefghijk', 'lmnopqrstuv']; const visual = { visualEvidence: 'required', visualRequirements: ['outfits'] }; diff --git a/platform/test/youtube-stored-analysis.test.ts b/platform/test/youtube-stored-analysis.test.ts index 5ca027f6..6e58f8b5 100644 --- a/platform/test/youtube-stored-analysis.test.ts +++ b/platform/test/youtube-stored-analysis.test.ts @@ -221,7 +221,7 @@ it('re-inspects a saved frame of one video while capturing only the other video' expect.anything(), expect.anything(), expect.anything()); expect(vi.mocked(context.analyzeFrames).mock.calls.map(([input]) => [input.frames.videoId, input.frames.frames[0]!.timestampMs])) .toEqual([[videoId, 30000], [other, 60000]]); - expect(await answer()).toMatch(/\n\nI re-inspected the saved 0:30 frame of "[^"]+"\. I retrieved the 1:00 frame of "[^"]+" and inspected it\.$/); + expect(await answer()).toMatch(/\n\nI re-inspected the saved 0:30 frame of "[^"]+"\. I retrieved the 1:00 frame of "[^"]+" and inspected it\./); }); it('keeps research open until a required capture is analyzed, without an exact-frame question', async () => { @@ -341,7 +341,7 @@ describe('cross-video frame completion', () => { describe('application notes on frame turns', () => { const other = 'lmnopqrstuv'; /** An earlier turn inspected the first video at 0:30; this turn's route listed only the second video. */ - async function turns() { + async function turns(beforeTurn?: (put: (id: string, timestampMs: number) => void) => void) { const { context, store, frames, provider } = setup(); const at = (id: string, timestampMs: number, width = 640) => ({ ...frames, videoId: id, frames: [{ ...frames.frames[0]!, timestampMs, width }] }); context.analyzeFrames = vi.fn(async ({ frames }) => ({ @@ -349,6 +349,8 @@ describe('application notes on frame turns', () => { context.runId = crypto.randomUUID(); const savedVersion = store.put('frame', videoId, at(videoId, 30000), { timestampMs: 30000 }); const earlier = await analyze('frame', savedVersion, context, 'What is he wearing?'); + // Older frames saved without a recorded position, before this turn starts. + beforeTurn?.((id, timestampMs) => { store.put('frame', id, at(id, timestampMs, 480)); }); context.runId = crypto.randomUUID(); let startedAt = store.runStart(); context.frameAccess = [{ videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; @@ -363,7 +365,7 @@ describe('application notes on frame turns', () => { return { context, store, at, savedVersion, earlier, current, answer, restart }; } const earlierNote = /The 0:30 observation( of "[^"]+")? comes from an image inspected in an earlier turn, not in this one\./; - const claimNote = 'No frame saved before this turn was re-inspected in this turn.'; + const claimNote = 'No frame saved before this turn was re-inspected in this turn'; it('states that a cited observation came from an earlier turn', async () => { const { earlier, current, answer } = await turns(); @@ -409,6 +411,13 @@ describe('application notes on frame turns', () => { expect(text.includes('It is not recorded whether the cited 0:30 observation')).toBe(uncertain); }); + it('names saved frames whose position cannot be recovered instead of skipping them silently', async () => { + const { current, answer } = await turns(put => put(other, 30000)); + const text = answer(`The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current]); + expect(text).toContain(`Saved frames of "${other}" have no recorded position, so this turn could not match them to requested times or re-inspect them.`); + expect(text).toContain(`${claimNote}, because its plan included none.`); + }); + it('judges each citation by its own image when an earlier batch held several', async () => { const { context, store, at, current, answer, restart } = await turns(); // An earlier turn analyzed 0:30 and 0:45 together; this turn re-inspects only the exact 0:30 image. From 5b90df434d75c7e2c2b622a73da0d015d15eba2e Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sun, 8 Nov 2026 09:53:03 +0530 Subject: [PATCH 07/12] Use one time parser for capture permission and saved-frame checks The capture-permission check used a narrower pattern than the saved-frame check, so 'capture only 1m30s' during discovery still accepted agent_selected. Both checks now share one parser of clock times and spoken quantities, which skips length bounds such as 'under 5 minutes'. A parity test runs every format through each check separately. --- .../src/agents/research/capability-router.ts | 16 ++++---- platform/test/youtube-agent-router.test.ts | 38 +++++++++++++++++++ 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index 62f99877..9d261483 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -613,22 +613,24 @@ function candidateObject(value: unknown): Record | undefined { /** An explicit prohibition rules out an unrestricted fallback. Only the classifier * resolves which video and positions the user authorized. */ const prohibitsRecapture = (message: string) => /\b(?:without|never|do not|don['’]t)\s+re-?captur(?:e|ing)\b/i.test(message); -/** Clock times, and durations introduced as a position ("at 90 seconds", "only 60 seconds"), not lengths ("a 10 minute video"). */ -const FRAME_POSITION = /\b\d+:\d{2}\b|\b\d+\s*ms\b|(?:\b(?:at|from|around|only|to|until)|@)\s*\d+(?:\.\d+)?\s*(?:milliseconds?|s|secs?|seconds?|mins?|minutes?)\b/i; -const namesFramePositions = (message: string) => FRAME_POSITION.test(message); const UNIT_MS: ReadonlyArray<[RegExp, number]> = [[/^(?:ms|milliseconds?)$/i, 1], [/^(?:s|secs?|seconds?)$/i, 1_000], [/^(?:m|mins?|minutes?)$/i, 60_000], [/^(?:h|hrs?|hours?)$/i, 3_600_000]]; const unitMs = (unit: string) => UNIT_MS.find(([pattern]) => pattern.test(unit))?.[1] ?? 0; +/** A bound on length ("under 5 minutes", "longer than 10:00") is a filter, not a position. */ +const NOT_LENGTH_BOUND = /(? [ - ...[...message.matchAll(/\b(?:(\d+):)?(\d{1,2}):(\d{2})\b/g)] + ...[...message.matchAll(CLOCK_TIME)] .map(([, hours, minutes, seconds]) => ((Number(hours ?? 0) * 60 + Number(minutes)) * 60 + Number(seconds)) * 1000), ...[...message.matchAll(TIME_QUANTITY)].map(([, amount, unit, extra]) => Math.round(Number(amount) * unitMs(unit!) + (extra ? Number(extra) * 1_000 : 0))), ]; +const namesFramePositions = (message: string) => namedTimes(message).length > 0; function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[]): ClassificationIssue[] { // A single linked video pins inspection to itself, which would leave another video's frames out of reach. diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index 165637b3..5490dfc3 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -1183,6 +1183,44 @@ describe('YouTube agent capability router', () => { expect(model.doGenerateCalls).toHaveLength(1); }); + it.each([[false], [true]])('limits a discovery capture to 1m30s or asks (classifier keeps agent_selected: %s)', async persists => { + const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', visualEvidence: 'required', visualRequirements: ['dough'] }; + const frameAccess = [{ savedTimestampsMs: [], captureTimestampsMs: [90000] }]; + const model = sequenceClassifier([{ ...discovery, frameAccess: 'agent_selected' }, ...(persists ? [] : [{ ...discovery, frameAccess }])]); + const route = await classifyCapabilityWithModel({ model, signal: new AbortController().signal, + message: 'Find a sourdough tutorial and capture only 1m30s of that video. No other timestamps.' }); + if (persists) expect(route).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + else expect(route).toMatchObject({ route: 'topic_research', frameAccess }); + expect(route).not.toHaveProperty('frameAccess', 'agent_selected'); + }); + + // One parser serves both checks, so every format limits capture and requires the saved frame alike. + it.each(['1:30', '1m30s', '90 seconds', '1 minute 30 seconds', 'the 90-second mark'])('treats %s as a named time in both checks', async time => { + const videoId = 'abcdefghijk'; + const sessionBrief = { memories: [], assets: [{ videoId, version: 'a'.repeat(64), kind: 'frame' as const, current: true, collectedAt: 1, details: { timestampMs: 90000 } }] }; + const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'] }; + const permission = sequenceClassifier([{ ...base, frameAccess: 'agent_selected' }, + { ...base, frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [90000] }] }]); + // No saved frame here, so only the capture-permission check can reject agent_selected. + await classifyCapabilityWithModel({ model: permission, signal: new AbortController().signal, + message: `Capture only ${time} of https://youtu.be/${videoId}.` }); + expect(permission.doGenerateCalls).toHaveLength(2); + const saved = sequenceClassifier([{ ...base, frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [120000] }] }, + { ...base, frameAccess: [{ videoId, savedTimestampsMs: [90000], captureTimestampsMs: [120000] }] }]); + await classifyCapabilityWithModel({ model: saved, sessionBrief, signal: new AbortController().signal, + message: `Re-inspect the saved frame at ${time} and capture 2:00 of https://youtu.be/${videoId}.` }); + expect(saved.doGenerateCalls).toHaveLength(2); + }); + + it('keeps delegated capture when a length bound is the only time in the message', async () => { + const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough under 5 minutes', visualEvidence: 'required', visualRequirements: ['dough'] }; + const model = sequenceClassifier([{ ...discovery, frameAccess: 'agent_selected' }]); + const route = await classifyCapabilityWithModel({ model, signal: new AbortController().signal, + message: 'Find sourdough videos under 5 minutes and show what the dough looks like.' }); + expect(route).toMatchObject({ frameAccess: 'agent_selected' }); + expect(model.doGenerateCalls).toHaveLength(1); + }); + it('keeps a fixed capture time when discovery chooses the video', async () => { const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', visualEvidence: 'required', visualRequirements: ['dough'] }; const frameAccess = [{ savedTimestampsMs: [], captureTimestampsMs: [30000] }]; From 179f00aad12f3feab2a27c930e7e68c683e2c878 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sun, 8 Nov 2026 12:21:47 +0530 Subject: [PATCH 08/12] fix: enforce capture windows separately from video length filters --- .../src/agents/research/capability-router.ts | 77 ++++++++++++++----- platform/test/youtube-agent-router.test.ts | 28 ++++++- 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index 9d261483..e9e7ba4a 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -306,6 +306,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline 'Select skill verify_saved_evidence for follow-ups that recover saved original captions/comments, verify exact quotations or passage links, or correct source claims. This replaces general discovery and analysis with a verification workflow. The skill does not prohibit fetching: a request that also needs evidence not yet saved, such as comments, still gets it unless savedOnly is set. Omit the skill for broad summaries or visual analysis.', 'Set requestedEvidence for each kind of evidence the user explicitly asks for: original_quotes for exact quotations, verbatim wording or precise passage links; comments for what viewers or commenters say. Include every requested kind in one decision, alongside any explanation. Research cannot complete until each is read or reported unavailable, so choose an executable route.', 'Set savedOnly true whenever the user prohibits fresh retrieval or asks to use existing saved sources only. Keep this restriction on follow-ups when it remains in force. Missing saved evidence is a gap, never permission to fetch. Use the saved reader instead of transcript retrieval or discovery.', + 'A video length limit filters the source. A capture window limits the seek positions. For a tutorial under five minutes with a capture within the first 30 seconds, choose a captureTimestampsMs value between 0 and 30000 inclusive. Use an exact list even when the user lets you choose a position inside the window.', 'New frames are captured only with frameAccess. When discovery will choose the videos but the user fixes the time, use one entry without videoId. When the session holds a saved frame at a time the user names, list it on its own video, or in setAsideFrames if the user excluded it or meant another video. When the user names frame positions, limits what may be captured, asks about exact timestamps, or re-inspects saved frames, list them in frameAccess with one entry per video. Set agent_selected only when new frames may help and the user leaves every position to you, including a plain frame refresh. Omit savedOnly when new captures are allowed. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. A prohibition on recapturing 10:30 still allows the requested 10:45. When the saved frame belongs to one video and the new capture to another, list both videos and choose topic_research with both in comparisonVideoIds, even if the message links only one. Times mentioned for comparison, narration or history do not authorize capture.', 'Requests for exact quotations from original saved captions/comments, precise passage-start links, or explicit original-asset reads require inspect_video or topic_research with the resolved subjects, even if prior summaries discuss the topic. Research can call search_originals without fetching YouTube. Quoting a source is different from quoting a prior conversation message. Do not route an original-source read directly to finalize.', 'For every context_answer set contextScope: history for listing, quoting, recalling or correcting conversation messages or preferences; video for claims about video content; mixed only when the requested answer needs both. A video URL inside a quoted earlier message does not require video evidence.', @@ -616,21 +617,56 @@ const prohibitsRecapture = (message: string) => /\b(?:without|never|do not|don[' const UNIT_MS: ReadonlyArray<[RegExp, number]> = [[/^(?:ms|milliseconds?)$/i, 1], [/^(?:s|secs?|seconds?)$/i, 1_000], [/^(?:m|mins?|minutes?)$/i, 60_000], [/^(?:h|hrs?|hours?)$/i, 3_600_000]]; const unitMs = (unit: string) => UNIT_MS.find(([pattern]) => pattern.test(unit))?.[1] ?? 0; -/** A bound on length ("under 5 minutes", "longer than 10:00") is a filter, not a position. */ -const NOT_LENGTH_BOUND = /(? [ - ...[...message.matchAll(CLOCK_TIME)] - .map(([, hours, minutes, seconds]) => ((Number(hours ?? 0) * 60 + Number(minutes)) * 60 + Number(seconds)) * 1000), - ...[...message.matchAll(TIME_QUANTITY)].map(([, amount, unit, extra]) => - Math.round(Number(amount) * unitMs(unit!) + (extra ? Number(extra) * 1_000 : 0))), -]; -const namesFramePositions = (message: string) => namedTimes(message).length > 0; +const CLOCK_TIME = /\b(?:(\d+):)?(\d{1,2}):(\d{2})\b/gi; +const TIME_QUANTITY = /\b(\d+(?:\.\d+)?)\s*-?\s*(milliseconds?|ms|hours?|hrs?|h|minutes?|mins?|m|seconds?|secs?|s)(?![a-z])(?:\s*(?:and\s*)?(\d+(?:\.\d+)?)\s*(seconds?|secs?|s)\b)?/gi; + +// A comparison such as "within" or "at most" can limit either video length or capture position. +// Exclude a time when the surrounding words describe duration ("videos under 5 minutes", "5 minutes long"). +const COMPARISON = /\b(?:under|over|within|up to|at (?:least|most)|(?:no )?(?:longer|shorter|less|more) than)\s*$/i; +const DURATION_SUBJECT = /\b(?:duration|length|runtime)(?:\s+(?:of|is|must be|should be))?\s*$/i; +const SOURCE_SELECTION = /\b(?:find|search|recommend|list|videos?|clips?|tutorials?|lectures?|recordings?|episodes?)\b/i; +const FRAME_ACTION = /\b(?:frames?|stills?|timestamps?|capture|capturing|inspect|sample|extract)\b/i; +const VIDEO_LENGTH_SUFFIX = /^\s*-?\s*(?:long|in (?:length|duration)|videos?|clips?|ads?|intros?|segments?|versions?|cuts?|shorts?|views?)\b/i; +const START_WINDOW = /\b(?:within(?: the first)?|up to|no later than|in the first|during the first)\s*$/i; +const START_ANCHOR = /^\s+(?:of|from)\s+(?:the\s+)?(?:(?:video|clip)(?:'s|’s)\s+)?(?:start|beginning)\b/i; +const RELATIVE_ANCHOR = /^\s+(?:of|from|after|before)\b/i; + +function isVideoLength(before: string, after: string): boolean { + if (VIDEO_LENGTH_SUFFIX.test(after)) return true; + const comparison = COMPARISON.exec(before); + if (!comparison) return false; + const subject = before.slice(0, comparison.index); + if (DURATION_SUBJECT.test(subject)) return true; + if (!START_WINDOW.test(before) && /\b(?:video|clip|tutorial|lecture|recording|episode)s?\s*$/i.test(subject)) return true; + const clause = subject.split(/[.!?;,\n]|\b(?:and|but|then)\b/i).at(-1) ?? ''; + return SOURCE_SELECTION.test(clause) && !FRAME_ACTION.test(clause); +} + +function frameTimes(message: string) { + const positions = new Set(); + let captureEndMs: number | undefined; + function add(match: RegExpExecArray, time: number) { + const before = message.slice(0, match.index); + const after = message.slice(match.index + match[0].length); + const fromStart = START_ANCHOR.test(after); + if (!fromStart && isVideoLength(before, after)) return; + // A window end limits new captures. It does not request inspection of a saved frame at the endpoint. + if (START_WINDOW.test(before) && (fromStart || !RELATIVE_ANCHOR.test(after) || /\bthe first\s*$/i.test(before))) { + captureEndMs = Math.min(captureEndMs ?? time, time); + } else { + positions.add(time); + } + } + for (const match of message.matchAll(CLOCK_TIME)) { + const [, hours, minutes, seconds] = match; + add(match, ((Number(hours ?? 0) * 60 + Number(minutes)) * 60 + Number(seconds)) * 1_000); + } + for (const match of message.matchAll(TIME_QUANTITY)) { + const [, amount, unit, extra] = match; + add(match, Math.round(Number(amount) * unitMs(unit!) + Number(extra ?? 0) * 1_000)); + } + return { positions, captureEndMs }; +} function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[]): ClassificationIssue[] { // A single linked video pins inspection to itself, which would leave another video's frames out of reach. @@ -640,11 +676,12 @@ function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifie } // An open refresh may choose its own positions. Named positions or a recapture prohibition // need a bounded list, or every fallback would quietly widen what the user allowed. + const { positions, captureEndMs } = frameTimes(input.message); // Capture is closed without frameAccess, so omission never widens it. These checks only ask // for a list where the classifier itself, or an explicit limit in the message, implies one. if (decision.route !== 'finalize' && !decision.savedOnly && !Array.isArray(decision.frameAccess)) { // Delegating which video is chosen is not delegating when it is captured. - const limited = prohibitsRecapture(input.message) || namesFramePositions(input.message); + const limited = prohibitsRecapture(input.message) || positions.size > 0 || captureEndMs !== undefined; if (decision.timestampedFrameQuestion || (decision.frameAccess === 'agent_selected' && limited)) { return [{ path: 'frameAccess', code: 'missing_capture_scope', message: 'The user fixed the frame positions. List the saved inspection positions and the permitted capture positions in frameAccess, one entry per video.' }]; } @@ -652,15 +689,19 @@ function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifie return [{ path: 'frameAccess', code: 'missing_capture_permission', message: 'A frame refresh needs capture permission. List the positions in frameAccess, or set agent_selected if the user left them to you.' }]; } } + // Apply a start-relative window to captures only. A saved inspection can be anywhere in the video. + if (captureEndMs !== undefined && Array.isArray(decision.frameAccess) + && decision.frameAccess.some(entry => entry.captureTimestampsMs.some(time => time > captureEndMs))) { + return [{ path: 'frameAccess', code: 'capture_outside_window', message: `Select capture positions within the requested window ending at ${captureEndMs} ms, or clarify the requested positions.` }]; + } // Each saved frame at a time the user named is accounted for on its own video: inspected, // recaptured, or set aside. Otherwise a plan can drop it without the research ever being able to reach it. if (decision.route !== 'finalize' && (decision.frameAccess || decision.refreshFramesOnly || decision.timestampedFrameQuestion || decision.visualEvidence !== 'none')) { - const named = new Set(namedTimes(input.message)); const accounted = (videoId: string, time: number) => (Array.isArray(decision.frameAccess) && decision.frameAccess.some(entry => entry.videoId === videoId && (entry.savedTimestampsMs.includes(time) || entry.captureTimestampsMs.includes(time)))) || !!decision.setAsideFrames?.some(frame => frame.videoId === videoId && frame.timestampMs === time); const omitted = input.sessionBrief?.assets.find(asset => asset.kind === 'frame' && typeof asset.details.timestampMs === 'number' - && named.has(asset.details.timestampMs) && !accounted(asset.videoId, asset.details.timestampMs)); + && positions.has(asset.details.timestampMs) && !accounted(asset.videoId, asset.details.timestampMs)); if (omitted) { const time = clockTime(omitted.details.timestampMs as number); return [{ path: 'frameAccess', code: 'saved_frame_unaccounted', message: `The session holds a saved ${time} frame of ${omitted.videoId}, and the user named ${time}. List it in that video's frameAccess entry (savedTimestampsMs to re-inspect, captureTimestampsMs to recapture), with that video in the route, or in setAsideFrames if the user excluded it or meant another video.` }]; diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index 5490dfc3..3cbb16c1 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -1212,15 +1212,39 @@ describe('YouTube agent capability router', () => { expect(saved.doGenerateCalls).toHaveLength(2); }); - it('keeps delegated capture when a length bound is the only time in the message', async () => { + it.each([ + 'Find sourdough videos under 5 minutes and show what the dough looks like.', + 'Find tutorials about shaping dough that are shorter than 5:00 and choose a frame.', + 'Choose a frame from a sourdough tutorial under 5 minutes.', + ])('keeps delegated capture when a length bound is the only time in the message: %s', async message => { const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough under 5 minutes', visualEvidence: 'required', visualRequirements: ['dough'] }; const model = sequenceClassifier([{ ...discovery, frameAccess: 'agent_selected' }]); const route = await classifyCapabilityWithModel({ model, signal: new AbortController().signal, - message: 'Find sourdough videos under 5 minutes and show what the dough looks like.' }); + message }); expect(route).toMatchObject({ frameAccess: 'agent_selected' }); expect(model.doGenerateCalls).toHaveLength(1); }); + it.each(['0:30', '30 seconds', '0m30s'])('enforces a capture window within %s of the start during discovery', async limit => { + const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', + refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['dough'] }; + const captureAt = (time: number) => [{ savedTimestampsMs: [], captureTimestampsMs: [time] }]; + const model = sequenceClassifier([{ ...discovery, frameAccess: 'agent_selected' }, { ...discovery, frameAccess: captureAt(30001) }]); + const fallbackModel = sequenceClassifier([{ ...discovery, frameAccess: captureAt(30000) }]); + const route = await classifyCapabilityWithModel({ model, fallbackModel, signal: new AbortController().signal, + message: `Find a sourdough tutorial under 5 minutes and capture a frame only within ${limit} of the start. Do not capture anything later.` }); + expect(route).toMatchObject({ route: 'topic_research', frameAccess: captureAt(30000) }); + expect(fallbackModel.doGenerateCalls).toHaveLength(1); + }); + + it('asks for clarification if every attempt leaves the capture window unrestricted', async () => { + const model = sequenceClassifier([{ route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', + frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['dough'] }]); + const route = await classifyCapabilityWithModel({ model, signal: new AbortController().signal, + message: 'Find a sourdough tutorial and capture a frame only within 0:30 of the start. Do not capture anything later.' }); + expect(route).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + it('keeps a fixed capture time when discovery chooses the video', async () => { const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', visualEvidence: 'required', visualRequirements: ['dough'] }; const frameAccess = [{ savedTimestampsMs: [], captureTimestampsMs: [30000] }]; From c3709634281c53a429f1677e2ee53546b4e06eeb Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sun, 8 Nov 2026 12:37:10 +0530 Subject: [PATCH 09/12] require permission to retrieve storyboard images --- docs/api-reference/openapi.json | 4 ++ platform/src/agents/contracts.ts | 5 +- .../src/agents/research/capability-router.ts | 15 +++--- .../src/agents/research/research-agent.ts | 5 +- platform/src/agents/runtime/frame-access.ts | 3 +- platform/test/youtube-agent-router.test.ts | 49 +++++++++++++++++-- 6 files changed, 66 insertions(+), 15 deletions(-) diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json index f9d1b098..e313a254 100644 --- a/docs/api-reference/openapi.json +++ b/docs/api-reference/openapi.json @@ -7702,6 +7702,7 @@ { "type": "string", "enum": [ + "storyboard_only", "agent_selected" ] } @@ -7846,6 +7847,7 @@ { "type": "string", "enum": [ + "storyboard_only", "agent_selected" ] } @@ -9588,6 +9590,7 @@ { "type": "string", "enum": [ + "storyboard_only", "agent_selected" ] } @@ -9732,6 +9735,7 @@ { "type": "string", "enum": [ + "storyboard_only", "agent_selected" ] } diff --git a/platform/src/agents/contracts.ts b/platform/src/agents/contracts.ts index 04812823..7df476af 100644 --- a/platform/src/agents/contracts.ts +++ b/platform/src/agents/contracts.ts @@ -29,8 +29,9 @@ export const frameAccessSchema = z.preprocess( frameAccessListSchema, ); export type FrameAccess = z.infer; -/** New captures are closed unless the route lists their positions or delegates them with agent_selected. */ -export const routeFrameAccessSchema = z.union([frameAccessSchema, z.literal('agent_selected')]); +/** Storyboard sheets contain multiple positions. Only unrestricted image selection may retrieve them. */ +export const unscopedImageAccessSchema = z.enum(['storyboard_only', 'agent_selected']); +export const routeFrameAccessSchema = z.union([frameAccessSchema, unscopedImageAccessSchema]); export type RouteFrameAccess = z.infer; /** Evidence the user explicitly asked to be read before answering. Research cannot complete until each is read or unavailable. */ diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index e9e7ba4a..1d75748d 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -9,6 +9,7 @@ import { answerDetailSchema, comparisonVideoIdsSchema, frameAccessListSchema, + unscopedImageAccessSchema, requestedEvidenceSchema, numberedItemCountSchema, visualEvidenceSchema, @@ -31,7 +32,7 @@ const classifierDecisionSchema = z.object({ skill: z.enum(['verify_saved_evidence']).optional().describe('Verify saved evidence: use for follow-ups that read saved original captions/comments, check exact quotations or precise passage links, or correct source claims. It does not by itself prohibit fetching; set savedOnly separately. Omit for broad summaries and visual analysis.'), requestedEvidence: requestedEvidenceSchema.optional().describe('Evidence the user explicitly asks to be read before answering: original_quotes for exact quotations, verbatim wording or precise passage links from captions; comments for what viewers or commenters say. Research must read each or report it unavailable. Omit when not explicitly requested.'), savedOnly: z.boolean().optional().describe('True when the user requires existing saved evidence only or prohibits fresh retrieval. All provider access is disabled for this run; use saved evidence reads and report missing assets.'), - frameAccess: z.union([frameAccessListSchema, z.literal('agent_selected')]).optional().describe('Permission to capture new video frames; without it no new frame is captured. When the user fixes frame positions, saved or new, list one entry per video the turn touches: captureTimestampsMs holds only the permitted NEW capture positions in milliseconds, empty when nothing may be captured; savedTimestampsMs holds saved positions to re-inspect without recapture, even if their assets are missing. When the videos are still to be found by discovery, give one entry without videoId holding the capture times. Required for exact-timestamp questions and for any limit on what may be captured. Use agent_selected only when new frames may help and the user leaves every position to you.'), + frameAccess: z.union([frameAccessListSchema, unscopedImageAccessSchema]).optional().describe('Permission to retrieve new images. Omit to disable both exact captures and storyboard retrieval. Use storyboard_only for storyboard overviews when exact captures are not authorized. When the user fixes frame positions, saved or new, list one entry per video the turn touches: captureTimestampsMs holds only the permitted NEW capture positions in milliseconds, empty when nothing may be captured; savedTimestampsMs holds saved positions to re-inspect without recapture, even if their assets are missing. When the videos are still to be found by discovery, give one entry without videoId holding the capture times. Required for exact-timestamp questions and for any limit on what may be captured. Use agent_selected only when new frames may help and the user leaves every position to you.'), setAsideFrames: z.array(z.object({ videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), timestampMs: z.number().int().nonnegative() })).max(24).optional().describe('Saved frames from the session inventory at a time the user named that this turn must not use: the user excluded them, or the time refers to another video. Every saved frame at a named time belongs either in its video\'s frameAccess entry or here.'), timestampedFrameQuestion: z.string().trim().min(1).max(500).optional().describe('For inspect_video: when the user supplies exact timestamps and asks what is visible, copy the focused visual question here. Include this for a request to describe each specified frame. Omit for image retrieval only or saved-only requests.'), refreshFramesOnly: z.boolean().optional().describe('For inspection or research, including multi-video comparisons: true when the user asks to freshly capture or refresh frames only. Reuse verified video metadata. Leave refreshEvidence unset unless other source evidence must also be refreshed.'), @@ -50,7 +51,7 @@ const classifierDecisionSchema = z.object({ channelId: capabilityRouteDecisionSchema.options[0].shape.channelId.describe('For research restricted to one supplied channel, copy its channel ID or handle from suppliedChannelIds. Never invent a channel identifier.'), videoId: capabilityRouteDecisionSchema.options[1].shape.videoId.optional(), reason: capabilityRouteDecisionSchema.options[3].shape.reason.optional().describe('Required for finalize: explain why existing context suffices, what scope is missing, or why the request is unsupported. The finalizer writes the response.'), - visualEvidence: visualEvidenceSchema.optional().describe('Required for executable routes. Whether answering needs images: none, helpful or required. helpful and required enable storyboard and frame tools.'), + visualEvidence: visualEvidenceSchema.optional().describe('Required for executable routes. Whether answering needs images: none, helpful or required. helpful and required enable visual analysis. New image retrieval also needs frameAccess.'), visualRequirements: visualRequirementsSchema.optional().describe('Only when visualEvidence is required: each requested fact that needs images, such as "presenter clothing".'), }); @@ -177,8 +178,8 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio } if (input.refreshFramesOnly === true && input.visualEvidence === 'none') issue('visualEvidence', 'Refreshing frames requires visual tool access.'); const scope = parsed.success && parsed.data.frameAccess; - if (scope === 'agent_selected' && input.savedOnly === true) - issue('frameAccess', 'Saved-only requests cannot capture frames. Omit frameAccess, or list only savedTimestampsMs.'); + if (typeof scope === 'string' && input.savedOnly === true) + issue('frameAccess', 'Saved-only requests cannot retrieve new images. Omit frameAccess, or list only savedTimestampsMs.'); if (Array.isArray(scope)) { const scoped = scope.flatMap(entry => entry.videoId ? [entry.videoId] : []); const open = scope.filter(entry => !entry.videoId); @@ -307,7 +308,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline 'Set requestedEvidence for each kind of evidence the user explicitly asks for: original_quotes for exact quotations, verbatim wording or precise passage links; comments for what viewers or commenters say. Include every requested kind in one decision, alongside any explanation. Research cannot complete until each is read or reported unavailable, so choose an executable route.', 'Set savedOnly true whenever the user prohibits fresh retrieval or asks to use existing saved sources only. Keep this restriction on follow-ups when it remains in force. Missing saved evidence is a gap, never permission to fetch. Use the saved reader instead of transcript retrieval or discovery.', 'A video length limit filters the source. A capture window limits the seek positions. For a tutorial under five minutes with a capture within the first 30 seconds, choose a captureTimestampsMs value between 0 and 30000 inclusive. Use an exact list even when the user lets you choose a position inside the window.', - 'New frames are captured only with frameAccess. When discovery will choose the videos but the user fixes the time, use one entry without videoId. When the session holds a saved frame at a time the user names, list it on its own video, or in setAsideFrames if the user excluded it or meant another video. When the user names frame positions, limits what may be captured, asks about exact timestamps, or re-inspects saved frames, list them in frameAccess with one entry per video. Set agent_selected only when new frames may help and the user leaves every position to you, including a plain frame refresh. Omit savedOnly when new captures are allowed. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. A prohibition on recapturing 10:30 still allows the requested 10:45. When the saved frame belongs to one video and the new capture to another, list both videos and choose topic_research with both in comparisonVideoIds, even if the message links only one. Times mentioned for comparison, narration or history do not authorize capture.', + 'New image retrieval requires frameAccess. Use storyboard_only for a storyboard overview when the turn permits sheets but not exact captures. A sheet includes neighboring images, so neither storyboard_only nor agent_selected is suitable for a request limited to exact frames. When discovery will choose the videos but the user fixes the time, use one entry without videoId. When the session holds a saved frame at a time the user names, list it on its own video, or in setAsideFrames if the user excluded it or meant another video. When the user names frame positions, limits what may be captured, asks about exact timestamps, or re-inspects saved frames, list them in frameAccess with one entry per video. Set agent_selected only when new frames may help and the user leaves every position to you, including a plain frame refresh. Omit savedOnly when new captures are allowed. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. A prohibition on recapturing 10:30 still allows the requested 10:45. When the saved frame belongs to one video and the new capture to another, list both videos and choose topic_research with both in comparisonVideoIds, even if the message links only one. Times mentioned for comparison, narration or history do not authorize capture.', 'Requests for exact quotations from original saved captions/comments, precise passage-start links, or explicit original-asset reads require inspect_video or topic_research with the resolved subjects, even if prior summaries discuss the topic. Research can call search_originals without fetching YouTube. Quoting a source is different from quoting a prior conversation message. Do not route an original-source read directly to finalize.', 'For every context_answer set contextScope: history for listing, quoting, recalling or correcting conversation messages or preferences; video for claims about video content; mixed only when the requested answer needs both. A video URL inside a quoted earlier message does not require video evidence.', 'For finalize give a short routing reason, not a user-facing answer. Choose responseIntent clarification for missing scope, or rejected for unsupported requests. Do not use the legacy clarification or rejected routes for new decisions.', @@ -682,10 +683,10 @@ function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifie if (decision.route !== 'finalize' && !decision.savedOnly && !Array.isArray(decision.frameAccess)) { // Delegating which video is chosen is not delegating when it is captured. const limited = prohibitsRecapture(input.message) || positions.size > 0 || captureEndMs !== undefined; - if (decision.timestampedFrameQuestion || (decision.frameAccess === 'agent_selected' && limited)) { + if (decision.timestampedFrameQuestion || (decision.frameAccess && limited)) { return [{ path: 'frameAccess', code: 'missing_capture_scope', message: 'The user fixed the frame positions. List the saved inspection positions and the permitted capture positions in frameAccess, one entry per video.' }]; } - if (decision.refreshFramesOnly && !decision.frameAccess) { + if (decision.refreshFramesOnly && decision.frameAccess !== 'agent_selected') { return [{ path: 'frameAccess', code: 'missing_capture_permission', message: 'A frame refresh needs capture permission. List the positions in frameAccess, or set agent_selected if the user left them to you.' }]; } } diff --git a/platform/src/agents/research/research-agent.ts b/platform/src/agents/research/research-agent.ts index 1c9546fc..7340a4fe 100644 --- a/platform/src/agents/research/research-agent.ts +++ b/platform/src/agents/research/research-agent.ts @@ -461,9 +461,10 @@ async function runResearchAgentWithModelWithinDeadline(options: { const savedEvidenceSkill = usesSavedEvidenceSkill(options.decision); const playlistListing = options.decision.route === 'topic_research' && options.decision.playlistListing && !options.decision.savedOnly ? options.decision : undefined; - // Missing flags belong to legacy persisted routes, which retain their tool set. + // A storyboard sheet includes neighboring images, even when requested at one timestamp. + const storyboardAllowed = options.decision.frameAccess === 'storyboard_only' || options.decision.frameAccess === 'agent_selected'; const toolNames = (options.toolNames ?? capability.toolNames) - .filter(name => !frameList(options.decision) || name !== 'get_video_storyboard') + .filter(name => name !== 'get_video_storyboard' || storyboardAllowed) // New captures are closed unless the route lists their positions or delegates them. .filter(name => name !== 'get_video_frames' || capturesAllowed(options.decision)) .filter(name => !playlistListing || (!playlistListing.playlistId && name === 'get_playlist') || name === FINALIZE_ANSWER_TOOL_NAME) diff --git a/platform/src/agents/runtime/frame-access.ts b/platform/src/agents/runtime/frame-access.ts index fe16135f..9f6300f4 100644 --- a/platform/src/agents/runtime/frame-access.ts +++ b/platform/src/agents/runtime/frame-access.ts @@ -72,7 +72,8 @@ export function frameList(decision: CapabilityRouteDecision): FrameAccess | unde /** A turn that asked for frame work, whose answer carries the application's account of it. */ export function isFrameTurn(decision: CapabilityRouteDecision): boolean { return (decision.route === 'inspect_video' || decision.route === 'topic_research') - && (!!decision.frameAccess || !!decision.refreshFramesOnly || (decision.route === 'inspect_video' && !!decision.timestampedFrameQuestion)); + && (Array.isArray(decision.frameAccess) || decision.frameAccess === 'agent_selected' || !!decision.refreshFramesOnly + || (decision.route === 'inspect_video' && !!decision.timestampedFrameQuestion)); } /** A saved frame as the session inventory records it. */ diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index 3cbb16c1..905fd66c 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -1170,6 +1170,49 @@ describe('YouTube agent capability router', () => { expect(tools).not.toContain('get_video_frames'); }); + it.each([ + ['Capture only 0:45 of https://youtu.be/abcdefghijk and describe it. Do not retrieve any other images.', undefined, 0], + ['Use a storyboard overview of https://youtu.be/abcdefghijk to describe the scenery.', 'storyboard_only', 1], + ] as const)('enforces storyboard access through the provider for "%s"', async (message, frameAccess, expectedCalls) => { + const videoId = 'abcdefghijk'; + const decision = await classifyCapabilityWithModel({ message, signal: new AbortController().signal, + model: classifierModel({ route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'], frameAccess }) }); + if (decision.route !== 'inspect_video') throw new Error('Expected inspection.'); + const context = inspectContext(); + const storyboard = vi.fn(async () => { throw new Error('Storyboard unavailable'); }); + context.provider.storyboard = storyboard; + const finalizer = finalizingModel(); + let attempted = false; + const model = new MockLanguageModelV4({ doGenerate: async call => { + const result = await finalizer.doGenerate(call); + if (attempted) return result; + attempted = true; + return { ...result, content: [{ type: 'tool-call', toolName: 'get_video_storyboard', toolCallId: 'storyboard-at-90s', + input: JSON.stringify({ videoId, timestampsMs: [90000] }) }] }; + } }); + await runResearchAgentWithModel({ model, message, decision, context }); + expect(storyboard).toHaveBeenCalledTimes(expectedCalls); + expect(model.doGenerateCalls[0]?.tools?.some(tool => tool.name === 'get_video_frames')).toBe(false); + expect(model.doGenerateCalls[0]?.tools?.some(tool => tool.name === 'get_video_storyboard')).toBe(expectedCalls > 0); + }); + + it('repairs a storyboard grant into the requested exact capture', async () => { + const videoId = 'abcdefghijk'; + const route = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene at 0:45'] }; + const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [45000] }]; + const classifier = sequenceClassifier([{ ...route, frameAccess: 'storyboard_only' }, { ...route, frameAccess }]); + const message = `Capture only 0:45 of https://youtu.be/${videoId} and describe it. Do not retrieve any other images.`; + const decision = await classifyCapabilityWithModel({ model: classifier, message, signal: new AbortController().signal }); + expect(classifier.doGenerateCalls).toHaveLength(2); + expect(decision).toMatchObject({ frameAccess }); + if (decision.route !== 'inspect_video') throw new Error('Expected inspection.'); + const model = finalizingModel(); + await runResearchAgentWithModel({ model, message, decision, context: inspectContext() }); + const names = model.doGenerateCalls[0]?.tools?.map(tool => tool.name); + expect(names).toContain('get_video_frames'); + expect(names).not.toContain('get_video_storyboard'); + }); + it('accepts a capture list that excludes a saved frame the user set aside', async () => { const videoId = 'abcdefghijk'; const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; @@ -1353,8 +1396,8 @@ describe('YouTube agent capability router', () => { }); const toolNames = model.doGenerateCalls[0]?.tools?.map((candidate) => candidate.name); - // The route granted no frame capture, so the capture tool is closed. - expect(toolNames).toEqual(INSPECT_VIDEO_TOOL_NAMES.filter(name => name !== 'get_video_frames')); + // Without a retrieval grant, only stored images can be analyzed. + expect(toolNames).toEqual(INSPECT_VIDEO_TOOL_NAMES.filter(name => name !== 'get_video_frames' && name !== 'get_video_storyboard')); expect(JSON.stringify(model.doGenerateCalls[0]?.prompt)).toContain('Pinned video ID: abcdefghijk'); expect(JSON.stringify(model.doGenerateCalls[0]?.prompt)).toContain('Start with this video.'); expect(JSON.stringify(model.doGenerateCalls[0]?.prompt)).toContain('I inspected its main argument.'); @@ -1397,7 +1440,7 @@ describe('YouTube agent capability router', () => { ] as const)('gates the %s storyboard tool using classifier choice %s', async (route, useStoryboard) => { const model = finalizingModel(); await runResearchAgentWithModel({ model, message: 'Inspect the video', - decision: route === 'inspect_video' ? { route, videoId: 'abcdefghijk', useStoryboard } : { route, useStoryboard }, + decision: route === 'inspect_video' ? { route, videoId: 'abcdefghijk', useStoryboard, frameAccess: 'storyboard_only' } : { route, useStoryboard, frameAccess: 'storyboard_only' }, context: inspectContext(), // Even an explicit caller tool list cannot override the classifier's decision. toolNames: ['get_video_storyboard', 'finalize_answer'], From 49b7b36c5abc065eb9581bf684cf72cd158b603b Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sun, 8 Nov 2026 13:07:12 +0530 Subject: [PATCH 10/12] bind image retrieval to the requested operation and timestamp --- docs/api-reference/openapi.json | 76 ++++++++ platform/src/agents/agent-runtime-do.ts | 1 + platform/src/agents/contracts.ts | 5 +- .../agents/providers/youtube/tool-context.ts | 1 + .../youtube/tools/get-video-storyboard.ts | 13 +- .../src/agents/research/capability-router.ts | 163 +++--------------- platform/src/agents/research/frame-request.ts | 161 +++++++++++++++++ .../src/agents/research/research-agent.ts | 21 ++- .../src/agents/research/youtube-references.ts | 40 +++++ platform/test/youtube-agent-router.test.ts | 131 ++++++++++---- .../test/youtube-agent-storyboard.test.ts | 13 ++ 11 files changed, 445 insertions(+), 180 deletions(-) create mode 100644 platform/src/agents/research/frame-request.ts create mode 100644 platform/src/agents/research/youtube-references.ts diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json index e313a254..3a23d5ce 100644 --- a/docs/api-reference/openapi.json +++ b/docs/api-reference/openapi.json @@ -7705,6 +7705,25 @@ "storyboard_only", "agent_selected" ] + }, + { + "type": "object", + "properties": { + "storyboardTimestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "storyboardTimestampsMs" + ], + "additionalProperties": false } ] }, @@ -7850,6 +7869,25 @@ "storyboard_only", "agent_selected" ] + }, + { + "type": "object", + "properties": { + "storyboardTimestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "storyboardTimestampsMs" + ], + "additionalProperties": false } ] }, @@ -9593,6 +9631,25 @@ "storyboard_only", "agent_selected" ] + }, + { + "type": "object", + "properties": { + "storyboardTimestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "storyboardTimestampsMs" + ], + "additionalProperties": false } ] }, @@ -9738,6 +9795,25 @@ "storyboard_only", "agent_selected" ] + }, + { + "type": "object", + "properties": { + "storyboardTimestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "storyboardTimestampsMs" + ], + "additionalProperties": false } ] }, diff --git a/platform/src/agents/agent-runtime-do.ts b/platform/src/agents/agent-runtime-do.ts index 402bd98f..236f0ead 100644 --- a/platform/src/agents/agent-runtime-do.ts +++ b/platform/src/agents/agent-runtime-do.ts @@ -709,6 +709,7 @@ export class AgentRuntimeDO extends Agent { }, onExtractionDiagnostic: event => this.recordExtractionDiagnostic(runId, event), onTranscriptDiagnostic: event => this.recordTranscriptDiagnostic(runId, event), + startedAt: row.created_at, persistedRoute: this.readRoute(runId), persistRoute: (selected) => { this.persistRoute(runId, selected); diff --git a/platform/src/agents/contracts.ts b/platform/src/agents/contracts.ts index 7df476af..428052f4 100644 --- a/platform/src/agents/contracts.ts +++ b/platform/src/agents/contracts.ts @@ -29,9 +29,10 @@ export const frameAccessSchema = z.preprocess( frameAccessListSchema, ); export type FrameAccess = z.infer; -/** Storyboard sheets contain multiple positions. Only unrestricted image selection may retrieve them. */ +/** Model choices for visual retrieval without a per-video exact-frame list. */ export const unscopedImageAccessSchema = z.enum(['storyboard_only', 'agent_selected']); -export const routeFrameAccessSchema = z.union([frameAccessSchema, unscopedImageAccessSchema]); +export const routeFrameAccessSchema = z.union([frameAccessSchema, unscopedImageAccessSchema, + z.object({ storyboardTimestampsMs: frameTimes.min(1) })]); export type RouteFrameAccess = z.infer; /** Evidence the user explicitly asked to be read before answering. Research cannot complete until each is read or unavailable. */ diff --git a/platform/src/agents/providers/youtube/tool-context.ts b/platform/src/agents/providers/youtube/tool-context.ts index d528d028..32dd28d6 100644 --- a/platform/src/agents/providers/youtube/tool-context.ts +++ b/platform/src/agents/providers/youtube/tool-context.ts @@ -58,6 +58,7 @@ export interface AgentToolContext { refreshEvidence?: boolean; refreshFramesOnly?: boolean; frameAccess?: FrameAccess; + storyboardSelection?: { videoId: string; timestampsMs: number[] }; timestampedFrameQuestion?: string; session?: import('../../runtime/session-evidence').SessionAccess; assetCache?: RunAssetCache; diff --git a/platform/src/agents/providers/youtube/tools/get-video-storyboard.ts b/platform/src/agents/providers/youtube/tools/get-video-storyboard.ts index 4389a85c..04ed56f7 100644 --- a/platform/src/agents/providers/youtube/tools/get-video-storyboard.ts +++ b/platform/src/agents/providers/youtube/tools/get-video-storyboard.ts @@ -36,10 +36,19 @@ function validateInput(input: z.infer, ctx: z.RefinementC if (input.sheetIndexes && input.timestampsMs) ctx.addIssue({ code: 'custom', message: 'Choose sheetIndexes or timestampsMs, not both.' }); } export const getVideoStoryboardInputSchema = retrievalInput.superRefine(validateInput); +// Apply the same selection before advertising the tool and before any evidence lookup or billing. +function storyboardInput(context: AgentToolContext) { + const selection = context.storyboardSelection; + return selection ? retrievalInput.extend({ + videoId: z.literal(selection.videoId), + timestampsMs: z.array(z.literal(selection.timestampsMs)).min(1).max(MAX_STORYBOARD_SHEETS), + sheetIndexes: z.never().optional(), + }) : retrievalInput; +} export function createGetVideoStoryboardTool(context: AgentToolContext) { return tool({ description: 'Retrieve storyboard images and metadata. Supply analysis with a visual question to analyze immediately while saving; omit it for retrieval only. With videoId alone, retrieve up to 3 sheets spread across the video. Use maxSheets, sheetIndexes or timestampsMs to select other coverage. No preliminary metadata call is needed. maxSheets gives a spread overview; use transcript findings to choose timestamps for targeted inspection. Returns saved assetVersions and sampled coverage, without image bytes. Use analyze_video_storyboard for questions about versions already in session inventory. Up to 20 sheets and 8 MiB per selection. Metadata alone is not visual evidence.', - inputSchema: retrievalInput.omit({focus:true}).superRefine(validateInput), + inputSchema: storyboardInput(context).omit({focus:true}).superRefine(validateInput), outputSchema: evidencePacketSchema, // Previews are dashboard images with random IDs. Hidden from the model, which otherwise // mistakes their per-sheet assetId for the analysisAssetVersions analysis needs. @@ -53,7 +62,7 @@ export function createGetVideoStoryboardTool(context: AgentToolContext) { }); } export function executeGetVideoStoryboard(input: z.infer, context: AgentToolContext, toolCallId: string) { - const requested = getVideoStoryboardInputSchema.parse(input); + const requested = storyboardInput(context).superRefine(validateInput).parse(input); // Only the old metadata-only shape needs a new key. Preserve existing keys // for explicit selections, and use schema order for the default overview. const parsed = requested.maxSheets === undefined && !requested.sheetIndexes && !requested.timestampsMs diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index 1d75748d..a4c6db75 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -19,7 +19,9 @@ import { type FinalizeAnswerInput, } from '../contracts'; import { sessionBriefForModel, type SessionBrief } from '../runtime/session-evidence'; -import { clockTime } from '../runtime/clock-time'; +import { extractYouTubeVideoIds, youTubeUrls } from './youtube-references'; +export { extractYouTubeVideoIds } from './youtube-references'; +import { frameRequestIssue, readFrameRequest, type FrameRequest } from './frame-request'; import { conversationAssistantMessage, type ConversationTurn } from '../runtime/conversation-memory'; import { assertModelCostAvailable, type AgentModelCostBudget } from '../runtime/model-budget'; import { AGENT_CLASSIFICATION_TIMEOUT_MS, withRunDeadline } from '../runtime/deadline'; @@ -32,7 +34,7 @@ const classifierDecisionSchema = z.object({ skill: z.enum(['verify_saved_evidence']).optional().describe('Verify saved evidence: use for follow-ups that read saved original captions/comments, check exact quotations or precise passage links, or correct source claims. It does not by itself prohibit fetching; set savedOnly separately. Omit for broad summaries and visual analysis.'), requestedEvidence: requestedEvidenceSchema.optional().describe('Evidence the user explicitly asks to be read before answering: original_quotes for exact quotations, verbatim wording or precise passage links from captions; comments for what viewers or commenters say. Research must read each or report it unavailable. Omit when not explicitly requested.'), savedOnly: z.boolean().optional().describe('True when the user requires existing saved evidence only or prohibits fresh retrieval. All provider access is disabled for this run; use saved evidence reads and report missing assets.'), - frameAccess: z.union([frameAccessListSchema, unscopedImageAccessSchema]).optional().describe('Permission to retrieve new images. Omit to disable both exact captures and storyboard retrieval. Use storyboard_only for storyboard overviews when exact captures are not authorized. When the user fixes frame positions, saved or new, list one entry per video the turn touches: captureTimestampsMs holds only the permitted NEW capture positions in milliseconds, empty when nothing may be captured; savedTimestampsMs holds saved positions to re-inspect without recapture, even if their assets are missing. When the videos are still to be found by discovery, give one entry without videoId holding the capture times. Required for exact-timestamp questions and for any limit on what may be captured. Use agent_selected only when new frames may help and the user leaves every position to you.'), + frameAccess: z.union([frameAccessListSchema, unscopedImageAccessSchema]).optional().describe('Permission to retrieve new images. Omit to disable both exact captures and storyboard retrieval. Use storyboard_only for requested storyboard sheets or an overview. A named sheet time limits the retrieval to the sheet containing that time. When the user fixes frame positions, saved or new, list one entry per video the turn touches: captureTimestampsMs holds only the permitted NEW capture positions in milliseconds, empty when nothing may be captured; savedTimestampsMs holds saved positions to re-inspect without recapture, even if their assets are missing. When the videos are still to be found by discovery, give one entry without videoId holding the capture times. Required for exact-timestamp questions and for any limit on what may be captured. Use agent_selected only when new frames may help and the user leaves every position to you.'), setAsideFrames: z.array(z.object({ videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), timestampMs: z.number().int().nonnegative() })).max(24).optional().describe('Saved frames from the session inventory at a time the user named that this turn must not use: the user excluded them, or the time refers to another video. Every saved frame at a named time belongs either in its video\'s frameAccess entry or here.'), timestampedFrameQuestion: z.string().trim().min(1).max(500).optional().describe('For inspect_video: when the user supplies exact timestamps and asks what is visible, copy the focused visual question here. Include this for a request to describe each specified frame. Omit for image retrieval only or saved-only requests.'), refreshFramesOnly: z.boolean().optional().describe('For inspection or research, including multi-video comparisons: true when the user asks to freshly capture or refresh frames only. Reuse verified video metadata. Leave refreshEvidence unset unless other source evidence must also be refreshed.'), @@ -178,6 +180,7 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio } if (input.refreshFramesOnly === true && input.visualEvidence === 'none') issue('visualEvidence', 'Refreshing frames requires visual tool access.'); const scope = parsed.success && parsed.data.frameAccess; + if (scope && input.route === 'finalize') issue('route', 'Image retrieval or saved-frame inspection requires an executable route.'); if (typeof scope === 'string' && input.savedOnly === true) issue('frameAccess', 'Saved-only requests cannot retrieve new images. Omit frameAccess, or list only savedTimestampsMs.'); if (Array.isArray(scope)) { @@ -224,7 +227,6 @@ function classifierToolSchema(options: ClassifierValidationOptions, defaultedFie } }); } -const VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{11}$/; export interface ClassificationDiagnostic { attempt: number; @@ -286,6 +288,7 @@ const RECONSIDERATION_MIN_MS = 1_500; async function classifyWithinDeadline(input: CapabilityClassifierInput, deadlineAt: number): Promise { assertModelCostAvailable(input.modelBudget); + const request = readFrameRequest(input.message); const conversationHistory = input.conversationHistory ?? []; const videoIds = [...new Set([ ...extractYouTubeVideoIds(input.message), @@ -308,7 +311,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline 'Set requestedEvidence for each kind of evidence the user explicitly asks for: original_quotes for exact quotations, verbatim wording or precise passage links; comments for what viewers or commenters say. Include every requested kind in one decision, alongside any explanation. Research cannot complete until each is read or reported unavailable, so choose an executable route.', 'Set savedOnly true whenever the user prohibits fresh retrieval or asks to use existing saved sources only. Keep this restriction on follow-ups when it remains in force. Missing saved evidence is a gap, never permission to fetch. Use the saved reader instead of transcript retrieval or discovery.', 'A video length limit filters the source. A capture window limits the seek positions. For a tutorial under five minutes with a capture within the first 30 seconds, choose a captureTimestampsMs value between 0 and 30000 inclusive. Use an exact list even when the user lets you choose a position inside the window.', - 'New image retrieval requires frameAccess. Use storyboard_only for a storyboard overview when the turn permits sheets but not exact captures. A sheet includes neighboring images, so neither storyboard_only nor agent_selected is suitable for a request limited to exact frames. When discovery will choose the videos but the user fixes the time, use one entry without videoId. When the session holds a saved frame at a time the user names, list it on its own video, or in setAsideFrames if the user excluded it or meant another video. When the user names frame positions, limits what may be captured, asks about exact timestamps, or re-inspects saved frames, list them in frameAccess with one entry per video. Set agent_selected only when new frames may help and the user leaves every position to you, including a plain frame refresh. Omit savedOnly when new captures are allowed. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. A prohibition on recapturing 10:30 still allows the requested 10:45. When the saved frame belongs to one video and the new capture to another, list both videos and choose topic_research with both in comparisonVideoIds, even if the message links only one. Times mentioned for comparison, narration or history do not authorize capture.', + 'Set frameAccess from the user request. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. If the images belong to different videos, include both in comparisonVideoIds. A saved image that cannot be read stays a saved inspection. Times mentioned for comparison, narration or history do not authorize capture.', 'Requests for exact quotations from original saved captions/comments, precise passage-start links, or explicit original-asset reads require inspect_video or topic_research with the resolved subjects, even if prior summaries discuss the topic. Research can call search_originals without fetching YouTube. Quoting a source is different from quoting a prior conversation message. Do not route an original-source read directly to finalize.', 'For every context_answer set contextScope: history for listing, quoting, recalling or correcting conversation messages or preferences; video for claims about video content; mixed only when the requested answer needs both. A video URL inside a quoted earlier message does not require video evidence.', 'For finalize give a short routing reason, not a user-facing answer. Choose responseIntent clarification for missing scope, or rejected for unsupported requests. Do not use the legacy clarification or rejected routes for new decisions.', @@ -389,7 +392,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline }] : parsed.success ? [] : parsed.error.issues.map(issue => ({ path: issue.path.map(String).join('.'), code: issue.code, message: issue.message, })); - if (parsed.success && feedback.length === 0) feedback = semanticIssues(parsed.data, input, videoIds); + if (parsed.success && feedback.length === 0) feedback = semanticIssues(parsed.data, input, videoIds, request); // Advisory only: the reconsideration may keep its choice, so a keyword match never fails classification. let advisory = false; if (parsed.success && feedback.length === 0 && context.attempt === 1) { @@ -423,7 +426,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline const lastResort = (evaluations: Evaluation[]) => { // The fallback uses no model, but an exhausted budget still ends the run. assertModelCostAvailable(input.modelBudget); - const built = lastResortDecision(evaluations, input, videoIds, channelIds, explicitVideoIds); + const built = lastResortDecision(evaluations, input, videoIds, channelIds, explicitVideoIds, request); console.warn(JSON.stringify({ event: 'agent_classification_last_resort', modelCallId: callId, method: built.method, candidates: evaluations.length, defaultedFields: built.defaultedFields })); input.onDiagnostic?.({ attempt: 0, stage: 'last_resort', lastResort: built.method, outcome: 'valid', modelId: '', @@ -433,7 +436,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline const decision = await coordinateClassification(call, input.signal, deadlineAt, callId, { fallbackModel: Boolean(input.fallbackModel), managedTimeouts: hasModelTimeouts(input.model), lastResort }); - return finishClassification(decision, videoIds, channelIds, explicitVideoIds, input.message); + return bindStoryboardSelection(finishClassification(decision, videoIds, channelIds, explicitVideoIds, input.message), request); } type ClassificationIssue = { path: string; code: string; message: string }; @@ -612,101 +615,15 @@ function candidateObject(value: unknown): Record | undefined { return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : undefined; } -/** An explicit prohibition rules out an unrestricted fallback. Only the classifier - * resolves which video and positions the user authorized. */ -const prohibitsRecapture = (message: string) => /\b(?:without|never|do not|don['’]t)\s+re-?captur(?:e|ing)\b/i.test(message); -const UNIT_MS: ReadonlyArray<[RegExp, number]> = [[/^(?:ms|milliseconds?)$/i, 1], [/^(?:s|secs?|seconds?)$/i, 1_000], - [/^(?:m|mins?|minutes?)$/i, 60_000], [/^(?:h|hrs?|hours?)$/i, 3_600_000]]; -const unitMs = (unit: string) => UNIT_MS.find(([pattern]) => pattern.test(unit))?.[1] ?? 0; -const CLOCK_TIME = /\b(?:(\d+):)?(\d{1,2}):(\d{2})\b/gi; -const TIME_QUANTITY = /\b(\d+(?:\.\d+)?)\s*-?\s*(milliseconds?|ms|hours?|hrs?|h|minutes?|mins?|m|seconds?|secs?|s)(?![a-z])(?:\s*(?:and\s*)?(\d+(?:\.\d+)?)\s*(seconds?|secs?|s)\b)?/gi; - -// A comparison such as "within" or "at most" can limit either video length or capture position. -// Exclude a time when the surrounding words describe duration ("videos under 5 minutes", "5 minutes long"). -const COMPARISON = /\b(?:under|over|within|up to|at (?:least|most)|(?:no )?(?:longer|shorter|less|more) than)\s*$/i; -const DURATION_SUBJECT = /\b(?:duration|length|runtime)(?:\s+(?:of|is|must be|should be))?\s*$/i; -const SOURCE_SELECTION = /\b(?:find|search|recommend|list|videos?|clips?|tutorials?|lectures?|recordings?|episodes?)\b/i; -const FRAME_ACTION = /\b(?:frames?|stills?|timestamps?|capture|capturing|inspect|sample|extract)\b/i; -const VIDEO_LENGTH_SUFFIX = /^\s*-?\s*(?:long|in (?:length|duration)|videos?|clips?|ads?|intros?|segments?|versions?|cuts?|shorts?|views?)\b/i; -const START_WINDOW = /\b(?:within(?: the first)?|up to|no later than|in the first|during the first)\s*$/i; -const START_ANCHOR = /^\s+(?:of|from)\s+(?:the\s+)?(?:(?:video|clip)(?:'s|’s)\s+)?(?:start|beginning)\b/i; -const RELATIVE_ANCHOR = /^\s+(?:of|from|after|before)\b/i; - -function isVideoLength(before: string, after: string): boolean { - if (VIDEO_LENGTH_SUFFIX.test(after)) return true; - const comparison = COMPARISON.exec(before); - if (!comparison) return false; - const subject = before.slice(0, comparison.index); - if (DURATION_SUBJECT.test(subject)) return true; - if (!START_WINDOW.test(before) && /\b(?:video|clip|tutorial|lecture|recording|episode)s?\s*$/i.test(subject)) return true; - const clause = subject.split(/[.!?;,\n]|\b(?:and|but|then)\b/i).at(-1) ?? ''; - return SOURCE_SELECTION.test(clause) && !FRAME_ACTION.test(clause); -} - -function frameTimes(message: string) { - const positions = new Set(); - let captureEndMs: number | undefined; - function add(match: RegExpExecArray, time: number) { - const before = message.slice(0, match.index); - const after = message.slice(match.index + match[0].length); - const fromStart = START_ANCHOR.test(after); - if (!fromStart && isVideoLength(before, after)) return; - // A window end limits new captures. It does not request inspection of a saved frame at the endpoint. - if (START_WINDOW.test(before) && (fromStart || !RELATIVE_ANCHOR.test(after) || /\bthe first\s*$/i.test(before))) { - captureEndMs = Math.min(captureEndMs ?? time, time); - } else { - positions.add(time); - } - } - for (const match of message.matchAll(CLOCK_TIME)) { - const [, hours, minutes, seconds] = match; - add(match, ((Number(hours ?? 0) * 60 + Number(minutes)) * 60 + Number(seconds)) * 1_000); - } - for (const match of message.matchAll(TIME_QUANTITY)) { - const [, amount, unit, extra] = match; - add(match, Math.round(Number(amount) * unitMs(unit!) + Number(extra ?? 0) * 1_000)); - } - return { positions, captureEndMs }; -} - -function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[]): ClassificationIssue[] { +function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[], request: FrameRequest): ClassificationIssue[] { // A single linked video pins inspection to itself, which would leave another video's frames out of reach. const linked = extractYouTubeVideoIds(input.message); if (decision.frameAccess && decision.route === 'inspect_video' && linked.length === 1 && linked[0] !== decision.videoId) { return [{ path: 'frameAccess', code: 'frame_scope_outside_link', message: `The message links ${linked[0]}. Inspect that video, or use topic_research with comparisonVideoIds for every video whose frames this turn uses.` }]; } - // An open refresh may choose its own positions. Named positions or a recapture prohibition - // need a bounded list, or every fallback would quietly widen what the user allowed. - const { positions, captureEndMs } = frameTimes(input.message); - // Capture is closed without frameAccess, so omission never widens it. These checks only ask - // for a list where the classifier itself, or an explicit limit in the message, implies one. - if (decision.route !== 'finalize' && !decision.savedOnly && !Array.isArray(decision.frameAccess)) { - // Delegating which video is chosen is not delegating when it is captured. - const limited = prohibitsRecapture(input.message) || positions.size > 0 || captureEndMs !== undefined; - if (decision.timestampedFrameQuestion || (decision.frameAccess && limited)) { - return [{ path: 'frameAccess', code: 'missing_capture_scope', message: 'The user fixed the frame positions. List the saved inspection positions and the permitted capture positions in frameAccess, one entry per video.' }]; - } - if (decision.refreshFramesOnly && decision.frameAccess !== 'agent_selected') { - return [{ path: 'frameAccess', code: 'missing_capture_permission', message: 'A frame refresh needs capture permission. List the positions in frameAccess, or set agent_selected if the user left them to you.' }]; - } - } - // Apply a start-relative window to captures only. A saved inspection can be anywhere in the video. - if (captureEndMs !== undefined && Array.isArray(decision.frameAccess) - && decision.frameAccess.some(entry => entry.captureTimestampsMs.some(time => time > captureEndMs))) { - return [{ path: 'frameAccess', code: 'capture_outside_window', message: `Select capture positions within the requested window ending at ${captureEndMs} ms, or clarify the requested positions.` }]; - } - // Each saved frame at a time the user named is accounted for on its own video: inspected, - // recaptured, or set aside. Otherwise a plan can drop it without the research ever being able to reach it. - if (decision.route !== 'finalize' && (decision.frameAccess || decision.refreshFramesOnly || decision.timestampedFrameQuestion || decision.visualEvidence !== 'none')) { - const accounted = (videoId: string, time: number) => (Array.isArray(decision.frameAccess) && decision.frameAccess.some(entry => - entry.videoId === videoId && (entry.savedTimestampsMs.includes(time) || entry.captureTimestampsMs.includes(time)))) - || !!decision.setAsideFrames?.some(frame => frame.videoId === videoId && frame.timestampMs === time); - const omitted = input.sessionBrief?.assets.find(asset => asset.kind === 'frame' && typeof asset.details.timestampMs === 'number' - && positions.has(asset.details.timestampMs) && !accounted(asset.videoId, asset.details.timestampMs)); - if (omitted) { - const time = clockTime(omitted.details.timestampMs as number); - return [{ path: 'frameAccess', code: 'saved_frame_unaccounted', message: `The session holds a saved ${time} frame of ${omitted.videoId}, and the user named ${time}. List it in that video's frameAccess entry (savedTimestampsMs to re-inspect, captureTimestampsMs to recapture), with that video in the route, or in setAsideFrames if the user excluded it or meant another video.` }]; - } + if (decision.route !== 'finalize') { + const issue = frameRequestIssue(decision, request, input.sessionBrief?.assets); + if (issue) return [{ path: 'frameAccess', code: 'invalid_frame_access', message: issue }]; } const scope = comparisonScopeIssues(decision, input, videoIds); return scope.length ? scope : searchQueryNumberIssues(decision, input); @@ -719,7 +636,7 @@ function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifie * request defaults alone. Candidates from later attempts are preferred. */ function lastResortDecision( - evaluations: Evaluation[], input: CapabilityClassifierInput, videoIds: string[], channelIds: string[], explicitVideoIds: string[], + evaluations: Evaluation[], input: CapabilityClassifierInput, videoIds: string[], channelIds: string[], explicitVideoIds: string[], request: FrameRequest, ): { decision: ClassifierDecision; method: LastResortMethod; defaultedFields: DefaultedClassificationField[] } { const candidates = [...evaluations].sort((a, b) => b.context.attempt - a.context.attempt) .flatMap(item => { const candidate = candidateObject(item.candidate); return candidate ? [candidate] : []; }); @@ -846,7 +763,7 @@ function lastResortDecision( }; const accept = (decision: Record) => { const parsed = validateClassifierDecision(decision, validation); - return parsed.success ? { parsed: parsed.data, issues: semanticIssues(parsed.data, input, videoIds) } : undefined; + return parsed.success ? { parsed: parsed.data, issues: semanticIssues(parsed.data, input, videoIds, request) } : undefined; }; // Recover a missing route from one candidate as returned. The recovered decision @@ -1027,42 +944,14 @@ export function extractYouTubeChannelIds(message: string): string[] { return [...ids]; } -// Consume the whole URL before checking its host. Embedded links and v= parameters belong to that URL. -const urlCandidates = /(?:https?:\/\/|(?"']+/gi; -function* youTubeUrls(message: string): Generator { - for (const candidate of message.match(urlCandidates) ?? []) { - const normalized = candidate.replace(/[\])},.!?;:'"]+$/g, ''); - let url: URL; - try { - url = new URL(/^https?:\/\//i.test(normalized) ? normalized : `https://${normalized}`); - } catch { - continue; - } - const host = url.hostname.toLowerCase(); - if (['youtube.com', 'www.youtube.com', 'm.youtube.com', 'music.youtube.com', 'youtu.be', 'www.youtu.be', 'm.youtu.be'].includes(host)) yield url; - } -} - -export function extractYouTubeVideoIds(message: string): string[] { - const ids = new Set(); - for (const url of youTubeUrls(message)) { - if (url.hostname.endsWith('youtu.be')) { - addVideoId(ids, url.pathname.split('/').filter(Boolean)[0]); - continue; - } - addVideoId(ids, url.searchParams.get('v') ?? undefined); - const [kind, pathId] = url.pathname.split('/').filter(Boolean); - if (kind === 'shorts' || kind === 'live' || kind === 'embed') addVideoId(ids, pathId); - } - - const labelledIdPattern = /(?:^|[\s(])(?:video(?:\s+id)?|v)\s*[:=]\s*([A-Za-z0-9_-]{11})(?=$|\s|[),.!?;])/gi; - for (const match of message.replace(urlCandidates, ' ').matchAll(labelledIdPattern)) addVideoId(ids, match[1]); - const trimmed = message.trim(); - if (VIDEO_ID_PATTERN.test(trimmed)) ids.add(trimmed); - return [...ids]; +function bindStoryboardSelection(decision: CapabilityRouteDecision, request: FrameRequest): CapabilityRouteDecision { + return decision.route === 'inspect_video' && decision.frameAccess === 'storyboard_only' && request.sheetTimes.length + ? { ...decision, frameAccess: { storyboardTimestampsMs: request.sheetTimes } } : decision; } export async function resolveCapabilityRoute(options: { + message?: string; + priorAssets?: SessionBrief['assets']; persisted?: unknown; classify: () => Promise; persist: (decision: CapabilityRouteDecision) => void | Promise; @@ -1076,6 +965,12 @@ export async function resolveCapabilityRoute(options: { if ('frameAccess' in decision && Array.isArray(decision.frameAccess) && decision.frameAccess.some(entry => entry.videoId && !subjects.includes(entry.videoId))) { return { route: 'finalize', responseIntent: 'clarification', reason: 'Clarify which video the saved and fresh frame positions refer to.' }; } + if (options.message && (decision.route === 'inspect_video' || decision.route === 'topic_research')) { + const request = readFrameRequest(options.message); + const issue = frameRequestIssue(decision, request, options.priorAssets, true); + if (issue) return { route: 'finalize', responseIntent: 'clarification', reason: issue }; + return bindStoryboardSelection(decision, request); + } return decision; } const decision = capabilityRouteDecisionSchema.parse(await options.classify()); @@ -1105,7 +1000,3 @@ function resolveClassification( : 'Which supplied YouTube video would you like me to inspect?', }; } - -function addVideoId(ids: Set, candidate: string | undefined): void { - if (candidate && VIDEO_ID_PATTERN.test(candidate)) ids.add(candidate); -} diff --git a/platform/src/agents/research/frame-request.ts b/platform/src/agents/research/frame-request.ts new file mode 100644 index 00000000..de1ae769 --- /dev/null +++ b/platform/src/agents/research/frame-request.ts @@ -0,0 +1,161 @@ +import type { RouteFrameAccess } from '../contracts'; +import { clockTime } from '../runtime/clock-time'; +import { extractYouTubeVideoIds, urlCandidates } from './youtube-references'; +import type { SessionBrief } from '../runtime/session-evidence'; + +const UNIT_MS: ReadonlyArray<[RegExp, number]> = [[/^(?:ms|milliseconds?)$/i, 1], [/^(?:s|secs?|seconds?)$/i, 1_000], + [/^(?:m|mins?|minutes?)$/i, 60_000], [/^(?:h|hrs?|hours?)$/i, 3_600_000]]; +const unitMs = (unit: string) => UNIT_MS.find(([pattern]) => pattern.test(unit))?.[1] ?? 0; +const CLOCK_TIME = /\b(?:(\d+):)?(\d{1,2}):(\d{2})\b/gi; +const TIME_QUANTITY = /\b(\d+(?:\.\d+)?)\s*-?\s*(milliseconds?|ms|hours?|hrs?|h|minutes?|mins?|m|seconds?|secs?|s)(?![a-z])(?:\s*(?:and\s*)?(\d+(?:\.\d+)?)\s*(seconds?|secs?|s)\b)?/gi; + +// A comparison such as "within" or "at most" can limit either video length or capture position. +// Exclude a time when the surrounding words describe duration ("videos under 5 minutes", "5 minutes long"). +const COMPARISON = /\b(?:under|over|within|up to|at (?:least|most)|(?:no )?(?:longer|shorter|less|more) than)\s*$/i; +const DURATION_SUBJECT = /\b(?:duration|length|runtime)(?:\s+(?:of|is|must be|should be))?\s*$/i; +const SOURCE_SELECTION = /\b(?:find|search|recommend|list|videos?|clips?|tutorials?|lectures?|recordings?|episodes?)\b/i; +const FRAME_ACTION = /\b(?:frames?|stills?|timestamps?|capture|capturing|inspect|sample|extract)\b/i; +const VIDEO_LENGTH_SUFFIX = /^\s*-?\s*(?:long|in (?:length|duration)|videos?|clips?|ads?|intros?|segments?|versions?|cuts?|shorts?|views?)\b/i; +const START_WINDOW = /\b(?:within(?: the first)?|up to|no later than|in the first|during the first)\s*$/i; +const START_ANCHOR = /^\s+(?:of|from)\s+(?:the\s+)?(?:(?:video|clip)(?:'s|’s)\s+)?(?:start|beginning)\b/i; +const RELATIVE_ANCHOR = /^\s+(?:of|from|after|before)\b/i; + +function isVideoLength(before: string, after: string): boolean { + if (VIDEO_LENGTH_SUFFIX.test(after)) return true; + const comparison = COMPARISON.exec(before); + if (!comparison) return false; + const subject = before.slice(0, comparison.index); + if (DURATION_SUBJECT.test(subject)) return true; + if (!START_WINDOW.test(before) && /\b(?:video|clip|tutorial|lecture|recording|episode)s?\s*$/i.test(subject)) return true; + const clause = subject.split(/[.!?;,\n]|\b(?:and|but|then)\b/i).at(-1) ?? ''; + return SOURCE_SELECTION.test(clause) && !FRAME_ACTION.test(clause); +} + +// A list continues the preceding operation ("capture 0:30, 0:45 and 1:00"). +// A new clause can change it ("reuse 0:30, then capture 1:00"). +const CLAUSE = /[:;.!?]\s+|\n|(?:,\s*|\b(?:then|and|but)\s+)(?=[a-z])/i; +function positionRole(clause: string, after: string) { + if (/\b(?:no|not|never|without|don['’]t|except|instead of|rather than|exclude|ignore|skip)\b/i.test(clause)) return 'excluded'; + if (/\bstoryboards?\b/i.test(clause) || /^\s+storyboard sheet\b/i.test(after)) return 'sheet'; + if (/\b(?:re-?capture|capture|refresh|new (?:image|still|frame))\b/i.test(clause)) return 'capture'; + if (/\b(?:transcript|caption|comment)s?\b/i.test(clause)) return; + if (/\b(?:reuse|re-?inspect|saved|previous|prior)\b/i.test(clause) || /^\s+saved (?:frame|image)\b/i.test(after)) return 'saved'; +} + +// The classifier resolves the subject. The user's named positions bound what the classifier may grant. +export function readFrameRequest(message: string) { + const positions: { time: number; role: ReturnType; videoIds: string[] }[] = []; + let captureEndMs: number | undefined; + function add(match: RegExpExecArray, time: number) { + const before = message.slice(0, match.index); + const after = message.slice(match.index + match[0].length); + const fromStart = START_ANCHOR.test(after); + if (!fromStart && isVideoLength(before, after)) return; + // A window end limits new captures. It does not request inspection of a saved frame at the endpoint. + if (START_WINDOW.test(before) && (fromStart || !RELATIVE_ANCHOR.test(after) || /\bthe first\s*$/i.test(before))) { + captureEndMs = Math.min(captureEndMs ?? time, time); + } else { + const prefix = before.split(CLAUSE).at(-1) ?? ''; + const suffix = after.split(CLAUSE, 1)[0] ?? ''; + positions.push({ time, role: positionRole(prefix, suffix), videoIds: extractYouTubeVideoIds(prefix + match[0] + suffix) }); + } + } + // A link can open at a different time than the requested capture. Its seek position is + // relevant only when the user supplies no position or capture window in the surrounding text. + const urls = [...message.matchAll(urlCandidates)]; + const linked: [RegExpExecArray, number][] = []; + function record(match: RegExpExecArray, time: number) { + if (urls.some(url => match.index >= url.index && match.index < url.index + url[0].length)) linked.push([match, time]); + else add(match, time); + } + for (const match of message.matchAll(CLOCK_TIME)) { + const [, hours, minutes, seconds] = match; + record(match, ((Number(hours ?? 0) * 60 + Number(minutes)) * 60 + Number(seconds)) * 1_000); + } + for (const match of message.matchAll(TIME_QUANTITY)) { + const [, amount, unit, extra] = match; + record(match, Math.round(Number(amount) * unitMs(unit!) + Number(extra ?? 0) * 1_000)); + } + if (!positions.length && captureEndMs === undefined) for (const [match, time] of linked) add(match, time); + const noRecapture = /\b(?:without|never|do not|don['’]t)\s+re-?captur(?:e|ing)\b/i.test(message); + const noCapture = /\b(?:never|do not|don['’]t)\s+capture\s+(?:any\s+)?(?:new |exact )?(?:frames?|images?|stills?)\b/i.test(message); + const sheetTimes = positions.every(position => position.role === 'sheet') && captureEndMs === undefined + ? [...new Set(positions.map(position => position.time))] : []; + return { positions, sheetTimes, captureEndMs, noRecapture, noCapture }; +} + +export type FrameRequest = ReturnType; +interface FramePlan { + route: string; + videoId?: string; + frameAccess?: RouteFrameAccess; + savedOnly?: boolean; + timestampedFrameQuestion?: string; + refreshFramesOnly?: boolean; + visualEvidence?: string; + setAsideFrames?: { videoId: string; timestampMs: number }[]; +} + +/** Bound the classifier's grant to operations named in the request, including on resume. */ +export function frameRequestIssue(plan: FramePlan, request: FrameRequest, assets: SessionBrief['assets'] = [], resumed = false): string | undefined { + const access = plan.frameAccess; + const { positions, sheetTimes, captureEndMs, noRecapture, noCapture } = request; + const limited = positions.length > 0 || captureEndMs !== undefined || noRecapture || noCapture; + const sameSubject = (ids: string[], videoId?: string) => !ids.length || (videoId !== undefined && ids.includes(videoId)); + const targetedSheets = sheetTimes.length > 0 && sheetTimes.length <= 24 && plan.route === 'inspect_video' + && positions.every(position => sameSubject(position.videoIds, plan.videoId)); + if (access && typeof access === 'object' && !Array.isArray(access)) { + // A previously persisted targeted sheet cannot widen during recovery. + if (!targetedSheets || new Set(access.storyboardTimestampsMs).size !== sheetTimes.length + || access.storyboardTimestampsMs.some(time => !sheetTimes.includes(time))) return 'Clarify the requested storyboard sheet positions.'; + return; + } + + const captures = positions.filter(position => position.role === 'capture'); + const matches = (time: number, videoId?: string) => positions.filter(pos => pos.time === time && sameSubject(pos.videoIds, videoId)); + if (Array.isArray(access)) { + for (const entry of access) { + for (const time of entry.captureTimestampsMs) { + const named = matches(time, entry.videoId); + if (captureEndMs !== undefined && time > captureEndMs) + return `Select capture positions within the requested window ending at ${captureEndMs} ms.`; + const explicit = named.find(pos => pos.role === 'capture'); + const inRequestedRange = captureEndMs !== undefined || !positions.length || named.length > 0; + const authorized = captures.length ? !!explicit : inRequestedRange; + const stored = assets.some(asset => asset.kind === 'frame' && asset.videoId === entry.videoId && asset.details.timestampMs === time); + // Two videos may share a timestamp. An explicit capture on B can coexist with a saved inspection on A. + const savedHere = named.some(pos => pos.role === 'saved') && (!explicit || (stored && !explicit.videoIds.length)); + const forbidden = noCapture || savedHere || named.some(pos => pos.role === 'excluded' || pos.role === 'sheet'); + if (!authorized || forbidden || (noRecapture && !explicit && (stored || captureEndMs === undefined))) + return `The user did not authorize a new exact capture at ${clockTime(time)} on this video. List only the authorized capture positions.`; + } + } + for (const pos of captures) { + if (!access.some(entry => sameSubject(pos.videoIds, entry.videoId) && entry.captureTimestampsMs.includes(pos.time))) + return `Include the requested new capture at ${clockTime(pos.time)} on its video.`; + } + } else { + if (access === 'storyboard_only' && !plan.refreshFramesOnly && (targetedSheets || (!positions.length && captureEndMs === undefined && !noRecapture))) return; + if (!plan.savedOnly && (plan.timestampedFrameQuestion || (access && limited))) + return 'The user fixed the frame positions. List the saved inspection positions and the permitted capture positions in frameAccess, one entry per video.'; + if (plan.refreshFramesOnly && access !== 'agent_selected') return 'A frame refresh needs capture permission. List the positions in frameAccess, or set agent_selected if the user left them to you.'; + } + + // Classification can set aside named inventory from an unrelated video. That choice + // has already been validated and is not part of the persisted execution grant. + if (!access && !plan.refreshFramesOnly && !plan.timestampedFrameQuestion && plan.visualEvidence === 'none') return; + if (!resumed) { + for (const asset of assets) { + const time = asset.details.timestampMs; + if (asset.kind !== 'frame' || typeof time !== 'number' || !positions.some(pos => pos.time === time)) continue; + const entry = Array.isArray(access) && access.find(entry => entry.videoId === asset.videoId); + if (entry && (entry.savedTimestampsMs.includes(time) || entry.captureTimestampsMs.includes(time))) continue; + if (plan.setAsideFrames?.some(frame => frame.videoId === asset.videoId && frame.timestampMs === time)) continue; + return `The session holds a saved ${clockTime(time)} frame of ${asset.videoId}. Include its requested operation in frameAccess, or in setAsideFrames if the user excluded it or meant another video.`; + } + } + for (const pos of positions.filter(pos => pos.role === 'saved')) { + if (!Array.isArray(access) || !access.some(entry => sameSubject(pos.videoIds, entry.videoId) && entry.savedTimestampsMs.includes(pos.time))) + return `List the requested saved ${clockTime(pos.time)} inspection even if its image is unavailable.`; + } +} diff --git a/platform/src/agents/research/research-agent.ts b/platform/src/agents/research/research-agent.ts index 7340a4fe..0f5ab43b 100644 --- a/platform/src/agents/research/research-agent.ts +++ b/platform/src/agents/research/research-agent.ts @@ -140,6 +140,7 @@ async function executeResearchRunWithAssets(options: { env: Env; modelFailover?: ModelFailoverState; runId: string; + startedAt?: number; message: string; /** Trusted line naming the run's date, from its admission time and the user's time zone. */ currentDate?: string; @@ -188,6 +189,9 @@ async function executeResearchRunWithAssets(options: { if (!options.persistedRoute) await options.onClassifying?.(classificationDeadlineAt); const decision = await resolveCapabilityRoute({ persisted: options.persistedRoute, + message: options.message, + priorAssets: options.persistedRoute && 'frameAccess' in options.persistedRoute + ? options.session?.brief().assets.filter(asset => asset.collectedAt < (options.startedAt ?? Infinity)) : undefined, classify: () => withRunDeadline(classificationDeadlineAt, options.signal, signal => classifyCapabilityWithModel({ message: options.message, conversationHistory: options.conversationHistory, @@ -462,11 +466,16 @@ async function runResearchAgentWithModelWithinDeadline(options: { const playlistListing = options.decision.route === 'topic_research' && options.decision.playlistListing && !options.decision.savedOnly ? options.decision : undefined; // A storyboard sheet includes neighboring images, even when requested at one timestamp. - const storyboardAllowed = options.decision.frameAccess === 'storyboard_only' || options.decision.frameAccess === 'agent_selected'; + const frameScope = frameList(options.decision); + const access = options.decision.frameAccess; + const storyboardSelection = options.decision.route === 'inspect_video' && access && typeof access === 'object' && 'storyboardTimestampsMs' in access + ? { videoId: options.decision.videoId, timestampsMs: access.storyboardTimestampsMs } : undefined; + const canCapture = access === 'agent_selected' || frameScope?.some(entry => entry.captureTimestampsMs.length); + const storyboardAllowed = access === 'storyboard_only' || access === 'agent_selected' || !!storyboardSelection; const toolNames = (options.toolNames ?? capability.toolNames) .filter(name => name !== 'get_video_storyboard' || storyboardAllowed) // New captures are closed unless the route lists their positions or delegates them. - .filter(name => name !== 'get_video_frames' || capturesAllowed(options.decision)) + .filter(name => name !== 'get_video_frames' || canCapture) .filter(name => !playlistListing || (!playlistListing.playlistId && name === 'get_playlist') || name === FINALIZE_ANSWER_TOOL_NAME) .filter(name => !verifySaved || name === FINALIZE_ANSWER_TOOL_NAME || (!options.decision.savedOnly && (SAVED_VERIFICATION_RETRIEVAL_TOOLS as readonly string[]).includes(name))) @@ -630,7 +639,6 @@ async function runResearchAgentWithModelWithinDeadline(options: { } return pending; }; - const frameScope = frameList(options.decision); const failedFrameVersions = new Set(); const failedCaptures = new Set(); const trackedContext: AgentToolContext = { @@ -640,6 +648,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { refreshEvidence: options.decision.refreshEvidence, refreshFramesOnly: options.decision.refreshFramesOnly, frameAccess: frameScope, + storyboardSelection, timestampedFrameQuestion: options.decision.route === 'inspect_video' && !options.decision.savedOnly ? options.decision.timestampedFrameQuestion : undefined, pinnedVideoId: options.decision.route === 'inspect_video' ? options.decision.videoId : undefined, @@ -1502,12 +1511,6 @@ function toolFailureWarnings(failures: EvidenceToolFailure[]): AgentWarning[] { }))]; } -/** Whether this route may capture new frames: at listed positions, or anywhere when delegated. */ -function capturesAllowed(decision: CapabilityRouteDecision): boolean { - return 'frameAccess' in decision && (decision.frameAccess === 'agent_selected' - || !!frameList(decision)?.some(entry => entry.captureTimestampsMs.length)); -} - function mergeWarnings(modelWarnings: AgentWarning[], failureWarnings: AgentWarning[]): AgentWarning[] { const warnings = new Map(); for (const warning of [...modelWarnings, ...failureWarnings]) { diff --git a/platform/src/agents/research/youtube-references.ts b/platform/src/agents/research/youtube-references.ts new file mode 100644 index 00000000..d276f303 --- /dev/null +++ b/platform/src/agents/research/youtube-references.ts @@ -0,0 +1,40 @@ +const VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{11}$/; + +// Consume the whole URL before checking its host. Embedded links and v= parameters belong to that URL. +export const urlCandidates = /(?:https?:\/\/|(?"']+/gi; +export function* youTubeUrls(message: string): Generator { + for (const candidate of message.match(urlCandidates) ?? []) { + const normalized = candidate.replace(/[\])},.!?;:'"]+$/g, ''); + let url: URL; + try { + url = new URL(/^https?:\/\//i.test(normalized) ? normalized : `https://${normalized}`); + } catch { + continue; + } + const host = url.hostname.toLowerCase(); + if (['youtube.com', 'www.youtube.com', 'm.youtube.com', 'music.youtube.com', 'youtu.be', 'www.youtu.be', 'm.youtu.be'].includes(host)) yield url; + } +} + +export function extractYouTubeVideoIds(message: string): string[] { + const ids = new Set(); + for (const url of youTubeUrls(message)) { + if (url.hostname.endsWith('youtu.be')) { + addVideoId(ids, url.pathname.split('/').filter(Boolean)[0]); + continue; + } + addVideoId(ids, url.searchParams.get('v') ?? undefined); + const [kind, pathId] = url.pathname.split('/').filter(Boolean); + if (kind === 'shorts' || kind === 'live' || kind === 'embed') addVideoId(ids, pathId); + } + + const labelledIdPattern = /(?:^|[\s(])(?:video(?:\s+id)?|v)\s*[:=]\s*([A-Za-z0-9_-]{11})(?=$|\s|[),.!?;])/gi; + for (const match of message.replace(urlCandidates, ' ').matchAll(labelledIdPattern)) addVideoId(ids, match[1]); + const trimmed = message.trim(); + if (VIDEO_ID_PATTERN.test(trimmed)) ids.add(trimmed); + return [...ids]; +} + +function addVideoId(ids: Set, candidate: string | undefined): void { + if (candidate && VIDEO_ID_PATTERN.test(candidate)) ids.add(candidate); +} diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index 905fd66c..09c4c3b0 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -1181,15 +1181,7 @@ describe('YouTube agent capability router', () => { const context = inspectContext(); const storyboard = vi.fn(async () => { throw new Error('Storyboard unavailable'); }); context.provider.storyboard = storyboard; - const finalizer = finalizingModel(); - let attempted = false; - const model = new MockLanguageModelV4({ doGenerate: async call => { - const result = await finalizer.doGenerate(call); - if (attempted) return result; - attempted = true; - return { ...result, content: [{ type: 'tool-call', toolName: 'get_video_storyboard', toolCallId: 'storyboard-at-90s', - input: JSON.stringify({ videoId, timestampsMs: [90000] }) }] }; - } }); + const model = finalizingModel([{ name: 'get_video_storyboard', input: { videoId, timestampsMs: [90000] } }]); await runResearchAgentWithModel({ model, message, decision, context }); expect(storyboard).toHaveBeenCalledTimes(expectedCalls); expect(model.doGenerateCalls[0]?.tools?.some(tool => tool.name === 'get_video_frames')).toBe(false); @@ -1213,6 +1205,84 @@ describe('YouTube agent capability router', () => { expect(names).not.toContain('get_video_storyboard'); }); + it.each([ + ['Capture only 0:45 of https://youtu.be/abcdefghijk.', [], 45000, 90000], + ['Capture only 0:45 of https://youtu.be/abcdefghijk?t=90s.', [], 45000, 90000], + ['Re-inspect the saved 0:30 frame of https://youtu.be/abcdefghijk. Do not recapture it if reuse fails. Capture only 1:00.', [30000], 60000, 30000], + ] as const)('repairs an unauthorized capture before the provider can receive it: %s', async (message, saved, fresh, forbidden) => { + const videoId = 'abcdefghijk'; + const route = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'] }; + const valid = [{ videoId, savedTimestampsMs: [...saved], captureTimestampsMs: [fresh] }]; + const invalid = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [fresh, forbidden] }]; + const model = sequenceClassifier([{ ...route, frameAccess: invalid }, { ...route, frameAccess: valid }]); + const assets = saved.map(time => ({ kind: 'frame' as const, version: 'a'.repeat(64), videoId, current: true, collectedAt: 1, details: { timestampMs: time } })); + const decision = await classifyCapabilityWithModel({ model, message, sessionBrief: { assets, memories: [] }, signal: new AbortController().signal }); + expect(model.doGenerateCalls).toHaveLength(2); + expect(decision).toMatchObject({ frameAccess: valid }); + if (decision.route !== 'inspect_video') throw new Error('Expected inspection.'); + const context = inspectContext(); + const frames = vi.fn>().mockResolvedValue({ value: { videoId, + frames: [{ timestampMs: fresh, width: 1, height: 1, mimeType: 'image/jpeg', imageBase64: '/9j/2Q==' }], + failures: [], meta: { partial: false, warnings: [] } }, cacheStatus: 'miss' }); + context.provider.frames = frames; + await runResearchAgentWithModel({ message, decision, context, model: finalizingModel([ + { name: 'get_video_frames', input: { videoId, timestampsMs: [fresh, forbidden] } }, + { name: 'get_video_frames', input: { videoId, timestampsMs: [fresh] } }, + ]) }); + expect(frames.mock.calls.map(([request]) => request.timestampsMs)).toEqual([[fresh]]); + + // A pre-upgrade turn being resumed must also close this path. + await expect(resolveCapabilityRoute({ persisted: { ...decision, frameAccess: invalid }, message, priorAssets: assets, + classify: vi.fn(), persist: vi.fn() })).resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it('keeps exact capture permissions paired with each linked video', async () => { + const [a, b] = ['abcdefghijk', 'lmnopqrstuv'] as const; + const at = (videoId: string, time: number) => ({ videoId, savedTimestampsMs: [], captureTimestampsMs: [time] }); + const route = { route: 'topic_research', comparisonVideoIds: [a, b], visualEvidence: 'required', visualRequirements: ['compare scenes'] }; + const valid = [at(a, 30000), at(b, 60000)]; + const model = sequenceClassifier([{ ...route, frameAccess: [at(a, 60000), at(b, 30000)] }, { ...route, frameAccess: valid }]); + const message = `Capture only 0:30 of https://youtu.be/${a} and capture 1:00 of https://youtu.be/${b}. Compare the scenes.`; + await expect(classifyCapabilityWithModel({ model, message, signal: new AbortController().signal })).resolves.toMatchObject({ frameAccess: valid }); + expect(model.doGenerateCalls).toHaveLength(2); + }); + + it('closes a bad capture list when every classification attempt repeats it', async () => { + const videoId = 'abcdefghijk'; + const model = classifierModel({ route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'], + frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [45000, 90000] }] }); + await expect(classifyCapabilityWithModel({ message: `Capture only 0:45 of https://youtu.be/${videoId}.`, model, + signal: new AbortController().signal })).resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it('retrieves only the requested storyboard sheet while exact capture remains closed', async () => { + const videoId = 'abcdefghijk'; + const message = `Retrieve the storyboard sheet containing 1:30 of https://youtu.be/${videoId}. Do not capture any exact frames.`; + const classifier = classifierModel({ route: 'inspect_video', videoId, frameAccess: 'storyboard_only', + visualEvidence: 'required', visualRequirements: ['scene'] }); + const decision = await classifyCapabilityWithModel({ message, model: classifier, signal: new AbortController().signal }); + expect(classifier.doGenerateCalls).toHaveLength(1); + expect(decision).toMatchObject({ frameAccess: { storyboardTimestampsMs: [90000] } }); + if (decision.route !== 'inspect_video') throw new Error('Expected inspection.'); + const resume = (persisted: unknown) => resolveCapabilityRoute({ persisted, message, classify: vi.fn(), persist: vi.fn() }); + await expect(resume(JSON.parse(JSON.stringify(decision)))).resolves.toEqual(decision); + await expect(resume({ ...decision, frameAccess: { storyboardTimestampsMs: [90000, 120000] } })) + .resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + const context = inspectContext(); + const storyboard = vi.fn>().mockResolvedValue({ cacheStatus: 'miss', + value: { videoId, frameCount: 20, intervalMs: 5000, sheets: [{ imageBase64: '/9j/2Q==', tileWidth: 100, tileHeight: 100, + columns: 2, rows: 1, firstFrameIndex: 18, frameCount: 2, intervalMs: 5000 }], meta: { partial: false, warnings: [] } } }); + context.provider.storyboard = storyboard; + const model = finalizingModel([ + { name: 'get_video_storyboard', input: { videoId, sheetIndexes: [0] } }, + { name: 'get_video_storyboard', input: { videoId, timestampsMs: [90000, 120000] } }, + { name: 'get_video_storyboard', input: { videoId, timestampsMs: [90000] } }, + ]); + await runResearchAgentWithModel({ model, message, decision, context }); + expect(storyboard.mock.calls.map(([id, times]) => [id, times])).toEqual([[videoId, [90000]]]); + expect(model.doGenerateCalls[0]?.tools?.map(tool => tool.name)).not.toContain('get_video_frames'); + }); + it('accepts a capture list that excludes a saved frame the user set aside', async () => { const videoId = 'abcdefghijk'; const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; @@ -1509,29 +1579,28 @@ function conversationTurn(overrides: Pick ({ - content: [{ - type: 'tool-call', - toolCallId: 'finalize-inspect', - toolName: 'finalize_answer', - input: JSON.stringify({ - blocks: [{ text: 'A supported finding from the video.', evidenceIds: ['ref_1'] }], - intent: 'inspect_video', - confidence: 'low', - citations: [], - artifacts: [], - warnings: [], - }), - }], - finishReason: { unified: 'tool-calls', raw: undefined }, - usage: { - inputTokens: { total: 50, noCache: 50, cacheRead: undefined, cacheWrite: undefined }, - outputTokens: { total: 20, text: 20, reasoning: undefined }, - }, - warnings: [], - }), + doGenerate: async request => { + // Invalid calls stay invalid during argument repair. Research can then try the next call. + const repair = request.responseFormat?.type === 'json'; + const call = repair ? finish : calls[step++] ?? finish; + return { + content: repair ? [{ type: 'text', text: '{}' }] : [{ type: 'tool-call', toolCallId: `inspect-${step}`, + toolName: call.name, input: JSON.stringify(call.input) }], + finishReason: { unified: repair ? 'stop' : 'tool-calls', raw: undefined }, + usage: { + inputTokens: { total: 50, noCache: 50, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 20, text: 20, reasoning: undefined }, + }, + warnings: [], + }; + }, }); } diff --git a/platform/test/youtube-agent-storyboard.test.ts b/platform/test/youtube-agent-storyboard.test.ts index 5eb15776..5ad9c768 100644 --- a/platform/test/youtube-agent-storyboard.test.ts +++ b/platform/test/youtube-agent-storyboard.test.ts @@ -51,6 +51,19 @@ async function retrieveAndAnalyze(input: Parameters { + it.each([ + { videoId: storyboard.videoId }, { videoId: storyboard.videoId, maxSheets: 3 }, + { videoId: storyboard.videoId, sheetIndexes: [0] }, { videoId: storyboard.videoId, timestampsMs: [55000, 50000] }, + { videoId: 'abcdefghijk', timestampsMs: [55000] }, + ])('rejects out-of-scope sheet requests before evidence lookup or billing: %j', input => { + const ctx = context(); + ctx.storyboardSelection = { videoId: storyboard.videoId, timestampsMs: [55000] }; + ctx.executeEvidenceTool = vi.fn(); + expect(() => executeGetVideoStoryboard(input, ctx, 'denied')).toThrow(); + expect(ctx.executeEvidenceTool).not.toHaveBeenCalled(); + expect(ctx.provider.storyboard).not.toHaveBeenCalled(); + }); + it.each([ { sheetIndexes: [6] }, { sheetIndexes: [5, 6, 7] }, { timestampsMs: [60000] }, From eb593e42b038ea41403677752ad58fb9f651b642 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sun, 8 Nov 2026 13:53:33 +0530 Subject: [PATCH 11/12] respect linked seek times and pair comparison frames with their videos --- platform/src/agents/research/frame-request.ts | 34 ++++++----- .../src/agents/research/youtube-references.ts | 60 ++++++++++++------- platform/test/youtube-agent-router.test.ts | 32 ++++++++++ 3 files changed, 90 insertions(+), 36 deletions(-) diff --git a/platform/src/agents/research/frame-request.ts b/platform/src/agents/research/frame-request.ts index de1ae769..fdd777ea 100644 --- a/platform/src/agents/research/frame-request.ts +++ b/platform/src/agents/research/frame-request.ts @@ -1,6 +1,6 @@ import type { RouteFrameAccess } from '../contracts'; import { clockTime } from '../runtime/clock-time'; -import { extractYouTubeVideoIds, urlCandidates } from './youtube-references'; +import { extractYouTubeVideoIds, linkedSeek, urlCandidates } from './youtube-references'; import type { SessionBrief } from '../runtime/session-evidence'; const UNIT_MS: ReadonlyArray<[RegExp, number]> = [[/^(?:ms|milliseconds?)$/i, 1], [/^(?:s|secs?|seconds?)$/i, 1_000], @@ -33,11 +33,12 @@ function isVideoLength(before: string, after: string): boolean { // A list continues the preceding operation ("capture 0:30, 0:45 and 1:00"). // A new clause can change it ("reuse 0:30, then capture 1:00"). -const CLAUSE = /[:;.!?]\s+|\n|(?:,\s*|\b(?:then|and|but)\s+)(?=[a-z])/i; +const CLAUSE = /[:;.!?]\s+|\n|(?:,\s*|\b(?:then|and|but)\s+)(?=[a-z])|\b(?:with|versus|vs\.?)\s+(?=(?:(?:a|an|the)\s+)?(?:new|fresh|saved|previous|prior)\b)/i; function positionRole(clause: string, after: string) { if (/\b(?:no|not|never|without|don['’]t|except|instead of|rather than|exclude|ignore|skip)\b/i.test(clause)) return 'excluded'; if (/\bstoryboards?\b/i.test(clause) || /^\s+storyboard sheet\b/i.test(after)) return 'sheet'; - if (/\b(?:re-?capture|capture|refresh|new (?:image|still|frame))\b/i.test(clause)) return 'capture'; + if (/\b(?:re-?capture|capture|refresh|(?:new|fresh) (?:image|still|frame))\b/i.test(clause) + || (/\b(?:new|fresh)\s*$/i.test(clause) && /^\s+(?:image|still|frame)\b/i.test(after))) return 'capture'; if (/\b(?:transcript|caption|comment)s?\b/i.test(clause)) return; if (/\b(?:reuse|re-?inspect|saved|previous|prior)\b/i.test(clause) || /^\s+saved (?:frame|image)\b/i.test(after)) return 'saved'; } @@ -46,7 +47,9 @@ function positionRole(clause: string, after: string) { export function readFrameRequest(message: string) { const positions: { time: number; role: ReturnType; videoIds: string[] }[] = []; let captureEndMs: number | undefined; - function add(match: RegExpExecArray, time: number) { + const urls = [...message.matchAll(urlCandidates)]; + function addProseTime(match: RegExpExecArray, time: number) { + if (urls.some(url => match.index >= url.index && match.index < url.index + url[0].length)) return; const before = message.slice(0, match.index); const after = message.slice(match.index + match[0].length); const fromStart = START_ANCHOR.test(after); @@ -60,23 +63,24 @@ export function readFrameRequest(message: string) { positions.push({ time, role: positionRole(prefix, suffix), videoIds: extractYouTubeVideoIds(prefix + match[0] + suffix) }); } } - // A link can open at a different time than the requested capture. Its seek position is - // relevant only when the user supplies no position or capture window in the surrounding text. - const urls = [...message.matchAll(urlCandidates)]; - const linked: [RegExpExecArray, number][] = []; - function record(match: RegExpExecArray, time: number) { - if (urls.some(url => match.index >= url.index && match.index < url.index + url[0].length)) linked.push([match, time]); - else add(match, time); - } for (const match of message.matchAll(CLOCK_TIME)) { const [, hours, minutes, seconds] = match; - record(match, ((Number(hours ?? 0) * 60 + Number(minutes)) * 60 + Number(seconds)) * 1_000); + addProseTime(match, ((Number(hours ?? 0) * 60 + Number(minutes)) * 60 + Number(seconds)) * 1_000); } for (const match of message.matchAll(TIME_QUANTITY)) { const [, amount, unit, extra] = match; - record(match, Math.round(Number(amount) * unitMs(unit!) + Number(extra ?? 0) * 1_000)); + addProseTime(match, Math.round(Number(amount) * unitMs(unit!) + Number(extra ?? 0) * 1_000)); + } + // An explicit position or window in the request takes precedence over the link's seek time. + if (!positions.length && captureEndMs === undefined) { + for (const url of urls) { + const seek = linkedSeek(url[0]); + if (!seek) continue; + const prefix = message.slice(0, url.index).split(CLAUSE).at(-1) ?? ''; + const suffix = message.slice(url.index + url[0].length).split(CLAUSE, 1)[0] ?? ''; + positions.push({ ...seek, role: positionRole(prefix, suffix) }); + } } - if (!positions.length && captureEndMs === undefined) for (const [match, time] of linked) add(match, time); const noRecapture = /\b(?:without|never|do not|don['’]t)\s+re-?captur(?:e|ing)\b/i.test(message); const noCapture = /\b(?:never|do not|don['’]t)\s+capture\s+(?:any\s+)?(?:new |exact )?(?:frames?|images?|stills?)\b/i.test(message); const sheetTimes = positions.every(position => position.role === 'sheet') && captureEndMs === undefined diff --git a/platform/src/agents/research/youtube-references.ts b/platform/src/agents/research/youtube-references.ts index d276f303..e82179b3 100644 --- a/platform/src/agents/research/youtube-references.ts +++ b/platform/src/agents/research/youtube-references.ts @@ -2,39 +2,57 @@ const VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{11}$/; // Consume the whole URL before checking its host. Embedded links and v= parameters belong to that URL. export const urlCandidates = /(?:https?:\/\/|(?"']+/gi; +function youTubeUrl(candidate: string): URL | undefined { + const normalized = candidate.replace(/[\])},.!?;:'"]+$/g, ''); + try { + const url = new URL(/^https?:\/\//i.test(normalized) ? normalized : `https://${normalized}`); + if (['youtube.com', 'www.youtube.com', 'm.youtube.com', 'music.youtube.com', 'youtu.be', 'www.youtu.be', 'm.youtu.be'].includes(url.hostname)) return url; + } catch { /* Non-URL text can match the candidate pattern. */ } +} + export function* youTubeUrls(message: string): Generator { for (const candidate of message.match(urlCandidates) ?? []) { - const normalized = candidate.replace(/[\])},.!?;:'"]+$/g, ''); - let url: URL; - try { - url = new URL(/^https?:\/\//i.test(normalized) ? normalized : `https://${normalized}`); - } catch { - continue; - } - const host = url.hostname.toLowerCase(); - if (['youtube.com', 'www.youtube.com', 'm.youtube.com', 'music.youtube.com', 'youtu.be', 'www.youtu.be', 'm.youtu.be'].includes(host)) yield url; + const url = youTubeUrl(candidate); + if (url) yield url; } } +function videoIdsFromUrl(url: URL): string[] { + const [kind, id] = url.pathname.split('/').filter(Boolean); + const candidates = url.hostname.endsWith('youtu.be') ? [kind] + : [url.searchParams.get('v'), ...(['shorts', 'live', 'embed'].includes(kind ?? '') ? [id] : [])]; + return candidates.filter((value): value is string => !!value && VIDEO_ID_PATTERN.test(value)); +} + +/** Seek parameters belong to their linked video. Numeric values are seconds. */ +export function linkedSeek(candidate: string): { videoIds: string[]; time: number } | undefined { + const url = youTubeUrl(candidate); + if (!url) return; + const videoIds = videoIdsFromUrl(url); + const value = url.searchParams.get('t') ?? url.searchParams.get('start') ?? new URLSearchParams(url.hash.slice(1)).get('t'); + if (!videoIds.length || !value) return; + let seconds: number; + if (/^\d+(?:\.\d+)?$/.test(value)) seconds = Number(value); + else { + const units = /^(?:(\d+)h)?(?:(\d+)m)?(?:(\d+(?:\.\d+)?)s)?$/i.exec(value); + const clock = /^(?:(\d+):)?(\d{1,2}):(\d{2})$/.exec(value); + const parts = units ?? clock; + if (!parts) return; + seconds = Number(parts[1] ?? 0) * 3600 + Number(parts[2] ?? 0) * 60 + Number(parts[3] ?? 0); + } + const time = Math.round(seconds * 1000); + if (Number.isSafeInteger(time)) return { videoIds, time }; +} + export function extractYouTubeVideoIds(message: string): string[] { const ids = new Set(); for (const url of youTubeUrls(message)) { - if (url.hostname.endsWith('youtu.be')) { - addVideoId(ids, url.pathname.split('/').filter(Boolean)[0]); - continue; - } - addVideoId(ids, url.searchParams.get('v') ?? undefined); - const [kind, pathId] = url.pathname.split('/').filter(Boolean); - if (kind === 'shorts' || kind === 'live' || kind === 'embed') addVideoId(ids, pathId); + for (const videoId of videoIdsFromUrl(url)) ids.add(videoId); } const labelledIdPattern = /(?:^|[\s(])(?:video(?:\s+id)?|v)\s*[:=]\s*([A-Za-z0-9_-]{11})(?=$|\s|[),.!?;])/gi; - for (const match of message.replace(urlCandidates, ' ').matchAll(labelledIdPattern)) addVideoId(ids, match[1]); + for (const match of message.replace(urlCandidates, ' ').matchAll(labelledIdPattern)) ids.add(match[1]!); const trimmed = message.trim(); if (VIDEO_ID_PATTERN.test(trimmed)) ids.add(trimmed); return [...ids]; } - -function addVideoId(ids: Set, candidate: string | undefined): void { - if (candidate && VIDEO_ID_PATTERN.test(candidate)) ids.add(candidate); -} diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index 09c4c3b0..1d38c88a 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -18,6 +18,7 @@ import { createCapabilityProvider, savedOnlyProvider } from '../src/agents/resea import type { ConversationTurn } from '../src/agents/runtime/conversation-memory'; import type { YouTubeAgentProvider } from '../src/agents/providers/youtube/provider'; import type { AgentToolContext } from '../src/agents/providers/youtube/tool-context'; +import { readFrameRequest } from '../src/agents/research/frame-request'; import { currentDateGuidance } from '../src/agents/runtime/current-date'; describe('YouTube agent capability router', () => { @@ -1206,6 +1207,8 @@ describe('YouTube agent capability router', () => { }); it.each([ + ['Capture only the linked instant at https://youtu.be/abcdefghijk?t=45 and describe it.', [], 45000, 90000], + ['Capture only the linked instant at https://www.youtube.com/watch?v=abcdefghijk&t=45.', [], 45000, 90000], ['Capture only 0:45 of https://youtu.be/abcdefghijk.', [], 45000, 90000], ['Capture only 0:45 of https://youtu.be/abcdefghijk?t=90s.', [], 45000, 90000], ['Re-inspect the saved 0:30 frame of https://youtu.be/abcdefghijk. Do not recapture it if reuse fails. Capture only 1:00.', [30000], 60000, 30000], @@ -1236,6 +1239,35 @@ describe('YouTube agent capability router', () => { classify: vi.fn(), persist: vi.fn() })).resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); }); + it.each([ + ['Capture the linked instant at https://youtu.be/abcdefghijk?t=0', 0], + ['Capture the linked instant at https://www.youtube.com/embed/abcdefghijk?start=45', 45000], + ['Capture the linked instant at https://youtu.be/abcdefghijk#t=1m30s', 90000], + ['Capture only 0:30 of https://youtu.be/abcdefghijk?t=45', 30000], + ] as const)('resolves the exact capture position for %s', (message, time) => { + expect(readFrameRequest(message).positions).toEqual([{ videoIds: ['abcdefghijk'], role: 'capture', time }]); + }); + + it('ignores unrelated URL parameters and seek times on other hosts', () => { + expect(readFrameRequest('Inspect https://youtu.be/abcdefghijk?tracking=90s and https://example.org/watch?v=lmnopqrstuv&t=45s').positions).toEqual([]); + }); + + it.each([['0:30', 30000], ['1:00', 60000]] as const)('compares a saved frame with a new %s frame on another video', async (clock, time) => { + const a = 'abcdefghijk', b = 'lmnopqrstuv'; + const frameAccess = [{ videoId: a, savedTimestampsMs: [30000], captureTimestampsMs: [] }, + { videoId: b, savedTimestampsMs: [], captureTimestampsMs: [time] }]; + const route = { route: 'topic_research', comparisonVideoIds: [a, b], frameAccess, visualEvidence: 'required', visualRequirements: ['compare frames'] }; + const model = classifierModel(route); + const message = `Compare the saved 0:30 frame of https://youtu.be/${a} with a new ${clock} frame from https://youtu.be/${b}.`; + const assets = [{ videoId: a, kind: 'frame' as const, version: 'a'.repeat(64), current: true, collectedAt: 1, details: { timestampMs: 30000 } }]; + const decision = await classifyCapabilityWithModel({ model, message, sessionBrief: { assets, memories: [] }, signal: new AbortController().signal }); + expect(decision).toMatchObject({ route: 'topic_research', frameAccess }); + expect(model.doGenerateCalls).toHaveLength(1); + const swapped = frameAccess.map(entry => ({ ...entry, videoId: entry.videoId === a ? b : a })); + await expect(resolveCapabilityRoute({ persisted: { ...decision, frameAccess: swapped }, message, priorAssets: assets, + classify: vi.fn(), persist: vi.fn() })).resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + it('keeps exact capture permissions paired with each linked video', async () => { const [a, b] = ['abcdefghijk', 'lmnopqrstuv'] as const; const at = (videoId: string, time: number) => ({ videoId, savedTimestampsMs: [], captureTimestampsMs: [time] }); From a0d447c30f376cdca5c07ed876b508d908facd0c Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Sun, 8 Nov 2026 14:46:25 +0530 Subject: [PATCH 12/12] require usable visual access and report inspections from run evidence --- docs/api-reference/openapi.json | 152 ++++++++++++++++++ platform/src/agents/agent-runtime-do.ts | 5 +- platform/src/agents/contracts.ts | 3 +- platform/src/agents/finalizer.ts | 6 +- .../agents/providers/youtube/tool-context.ts | 2 +- .../youtube/tools/get-video-storyboard.ts | 15 +- .../src/agents/research/capability-router.ts | 27 +++- platform/src/agents/research/frame-request.ts | 68 ++++++-- .../src/agents/research/research-agent.ts | 66 +++++--- .../skills/visual-inspection.generated.ts | 2 +- .../skills/visual-inspection/SKILL.md | 14 +- platform/src/agents/runtime/frame-access.ts | 40 +++-- .../src/agents/runtime/session-provider.ts | 7 +- .../test/classifier-route-recovery.test.ts | 14 +- .../test/session-evidence.integration.test.ts | 6 + .../test/youtube-agent-loop-control.test.ts | 19 +++ platform/test/youtube-agent-router.test.ts | 129 ++++++++++++--- .../test/youtube-agent-storyboard.test.ts | 10 +- platform/test/youtube-stored-analysis.test.ts | 31 +++- 19 files changed, 507 insertions(+), 109 deletions(-) diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json index 3a23d5ce..166aa55a 100644 --- a/docs/api-reference/openapi.json +++ b/docs/api-reference/openapi.json @@ -7724,6 +7724,44 @@ "storyboardTimestampsMs" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "storyboardSelections": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "timestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "videoId", + "timestampsMs" + ], + "additionalProperties": false + } + } + }, + "required": [ + "storyboardSelections" + ], + "additionalProperties": false } ] }, @@ -7888,6 +7926,44 @@ "storyboardTimestampsMs" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "storyboardSelections": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "timestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "videoId", + "timestampsMs" + ], + "additionalProperties": false + } + } + }, + "required": [ + "storyboardSelections" + ], + "additionalProperties": false } ] }, @@ -9650,6 +9726,44 @@ "storyboardTimestampsMs" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "storyboardSelections": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "timestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "videoId", + "timestampsMs" + ], + "additionalProperties": false + } + } + }, + "required": [ + "storyboardSelections" + ], + "additionalProperties": false } ] }, @@ -9814,6 +9928,44 @@ "storyboardTimestampsMs" ], "additionalProperties": false + }, + { + "type": "object", + "properties": { + "storyboardSelections": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "timestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "videoId", + "timestampsMs" + ], + "additionalProperties": false + } + } + }, + "required": [ + "storyboardSelections" + ], + "additionalProperties": false } ] }, diff --git a/platform/src/agents/agent-runtime-do.ts b/platform/src/agents/agent-runtime-do.ts index 236f0ead..bc69f9a6 100644 --- a/platform/src/agents/agent-runtime-do.ts +++ b/platform/src/agents/agent-runtime-do.ts @@ -14,6 +14,7 @@ import { storedExtractionDiagnosticSchema, type StoredExtractionDiagnostic } fro import { transcriptDiagnosticSchema, type TranscriptDiagnostic } from './runtime/transcript-diagnostics'; import { agentDraftSchema, agentRunProgressSchema, toolTrace, type AgentDraft } from './runtime/run-progress'; import { framePreviewKey, recordedFramePreviews, saveFramePreviews, type SavedFrame } from './runtime/frame-previews'; +import { readFrameRequest } from './research/frame-request'; import { frameList, isFrameTurn } from './runtime/frame-access'; import { saveStoryboardPreviews } from './runtime/storyboard-previews'; import { compactAgentRun, compactAgentResult } from './response'; @@ -1040,6 +1041,8 @@ export class AgentRuntimeDO extends Agent { const history = this.readConversationHistory(run); const citedIds=answerCitationIds(parsedInput.answer); const citedSessionEvidence=this.sessionStore.evidenceForCitations(citedIds); + const requestedSaved = readFrameRequest(run.execution_message ?? run.message).requestsSaved; + const needsFrameInventory = requestedSaved || isFrameTurn(decision) || citedSessionEvidence.some(packet => packet.kind === 'youtube_frames'); const result = buildAgentTurnResult({ runId, conversationId: run.conversation_id, @@ -1052,7 +1055,7 @@ export class AgentRuntimeDO extends Agent { requestedVideoIds: decision.route === 'inspect_video' ? [decision.videoId] : decision.comparisonVideoIds ?? [], // An application fallback is checked against what this run had, not earlier turns' cited content. runPackets: this.admittedRunEvidence(runId, history), - } : undefined, isFrameTurn(decision) ? { access: frameList(decision), startedAt: run.created_at, + } : undefined, needsFrameInventory ? { requestedSaved, access: frameList(decision), startedAt: run.created_at, inventory: this.sessionStore.brief().assets.flatMap(asset => asset.kind === 'frame' ? [{ version: asset.version, videoId: asset.videoId, collectedAt: asset.collectedAt, ...(typeof asset.details.timestampMs === 'number' ? { timestampMs: asset.details.timestampMs } : {}) }] : []) } : undefined); // Hash the accepted text: unmatched citations may have been removed from the input. diff --git a/platform/src/agents/contracts.ts b/platform/src/agents/contracts.ts index 428052f4..91d0312c 100644 --- a/platform/src/agents/contracts.ts +++ b/platform/src/agents/contracts.ts @@ -32,7 +32,8 @@ export type FrameAccess = z.infer; /** Model choices for visual retrieval without a per-video exact-frame list. */ export const unscopedImageAccessSchema = z.enum(['storyboard_only', 'agent_selected']); export const routeFrameAccessSchema = z.union([frameAccessSchema, unscopedImageAccessSchema, - z.object({ storyboardTimestampsMs: frameTimes.min(1) })]); + z.object({ storyboardTimestampsMs: frameTimes.min(1) }), + z.object({ storyboardSelections: z.array(z.object({ videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), timestampsMs: frameTimes.min(1) })).min(1).max(8) })]); export type RouteFrameAccess = z.infer; /** Evidence the user explicitly asked to be read before answering. Research cannot complete until each is read or unavailable. */ diff --git a/platform/src/agents/finalizer.ts b/platform/src/agents/finalizer.ts index fc26048f..cfdf4e8d 100644 --- a/platform/src/agents/finalizer.ts +++ b/platform/src/agents/finalizer.ts @@ -31,7 +31,7 @@ export function buildAgentTurnResult( creditsCharged: number, durationLimitContext?: DurationLimitAnswerContext, /** Present when the route asked for frame work: its exact access, and the session's saved frames. */ - frameTurn?: { access?: FrameAccess; inventory: readonly InventoryFrame[]; startedAt: number }, + frameTurn?: { requestedSaved?: boolean; access?: FrameAccess; inventory: readonly InventoryFrame[]; startedAt: number }, ): AgentTurnResult { const markers = answerCitationIds(input.answer); const citations: AgentCitation[] = []; @@ -83,8 +83,8 @@ export function buildAgentTurnResult( const playlistCoverage = playlistCoverageDisclosures(packets, citations.map(citation => citation.id)); const notes = [...playlistCoverage.notes]; // The application, not the model, states which frames this turn actually inspected. - if (frameTurn && input.intent !== 'clarification' && input.intent !== 'rejected') { - notes.push(...frameTurnNotes({ runId: identity.runId, startedAt: frameTurn.startedAt, access: frameTurn.access, inventory: frameTurn.inventory, packets, cited: citedObservations })); + if (citedObservations.length || (frameTurn && input.intent !== 'clarification' && input.intent !== 'rejected')) { + notes.push(...frameTurnNotes({ runId: identity.runId, startedAt: frameTurn?.startedAt ?? 0, requestedSaved: frameTurn?.requestedSaved, access: frameTurn?.access, inventory: frameTurn?.inventory ?? [], packets, cited: citedObservations })); } const disclosedAnswer = appendNotices(answer, notes); diff --git a/platform/src/agents/providers/youtube/tool-context.ts b/platform/src/agents/providers/youtube/tool-context.ts index 32dd28d6..97cb095e 100644 --- a/platform/src/agents/providers/youtube/tool-context.ts +++ b/platform/src/agents/providers/youtube/tool-context.ts @@ -58,7 +58,7 @@ export interface AgentToolContext { refreshEvidence?: boolean; refreshFramesOnly?: boolean; frameAccess?: FrameAccess; - storyboardSelection?: { videoId: string; timestampsMs: number[] }; + storyboardSelections?: { videoId: string; timestampsMs: number[] }[]; timestampedFrameQuestion?: string; session?: import('../../runtime/session-evidence').SessionAccess; assetCache?: RunAssetCache; diff --git a/platform/src/agents/providers/youtube/tools/get-video-storyboard.ts b/platform/src/agents/providers/youtube/tools/get-video-storyboard.ts index 04ed56f7..6c3953f9 100644 --- a/platform/src/agents/providers/youtube/tools/get-video-storyboard.ts +++ b/platform/src/agents/providers/youtube/tools/get-video-storyboard.ts @@ -38,10 +38,10 @@ function validateInput(input: z.infer, ctx: z.RefinementC export const getVideoStoryboardInputSchema = retrievalInput.superRefine(validateInput); // Apply the same selection before advertising the tool and before any evidence lookup or billing. function storyboardInput(context: AgentToolContext) { - const selection = context.storyboardSelection; - return selection ? retrievalInput.extend({ - videoId: z.literal(selection.videoId), - timestampsMs: z.array(z.literal(selection.timestampsMs)).min(1).max(MAX_STORYBOARD_SHEETS), + const selections = context.storyboardSelections; + return selections ? retrievalInput.extend({ + videoId: z.literal(selections.map(item => item.videoId)), + timestampsMs: z.array(z.literal([...new Set(selections.flatMap(item => item.timestampsMs))])).min(1).max(MAX_STORYBOARD_SHEETS), sheetIndexes: z.never().optional(), }) : retrievalInput; } @@ -63,6 +63,10 @@ export function createGetVideoStoryboardTool(context: AgentToolContext) { } export function executeGetVideoStoryboard(input: z.infer, context: AgentToolContext, toolCallId: string) { const requested = storyboardInput(context).superRefine(validateInput).parse(input); + if (context.storyboardSelections) { + const allowed = context.storyboardSelections.find(item => item.videoId === requested.videoId)?.timestampsMs; + if (!allowed || !requested.timestampsMs?.every(time => allowed.includes(time))) throw new Error('Storyboard position is outside the grant for this video.'); + } // Only the old metadata-only shape needs a new key. Preserve existing keys // for explicit selections, and use schema order for the default overview. const parsed = requested.maxSheets === undefined && !requested.sheetIndexes && !requested.timestampsMs @@ -111,6 +115,7 @@ async function retrieveVideoStoryboard(parsed: z.infer withRunDeadline(deadlineAt, context.signal, signal => timeStoryboardStage(parsed.videoId, 'retrieval', () => context.provider.storyboard!(parsed.videoId, parsed.timestampsMs, { maxSheets: parsed.maxSheets ?? MAX_STORYBOARD_SHEETS, sheetIndexes: parsed.sheetIndexes, metadataOnly: false, resolution: requestedResolution(parsed), signal, deadlineAt, + ...(!context.session && (context.refreshEvidence || context.refreshFramesOnly) ? { refresh: true } : {}), }, event => context.onExtractionDiagnostic?.({ ...event, toolCallId }), ...(onReady ? [onReady] : [])), { runId: context.runId, toolCallId }), 'Storyboard retrieval exceeded its budget.'), async (response, signal) => { const storyboard = storyboardSchema.parse(response.value); @@ -140,7 +145,7 @@ async function retrieveVideoStoryboard(parsed: z.infer context.session?.brief().assets.some(asset => asset.version === version && asset.kind === 'storyboard_sheet')), - videoId: parsed.videoId, sessionReused: response.sessionReused === true, selection: storyboard.selection, manifest: storyboard.manifest, + videoId: parsed.videoId, requestedTimestampsMs: parsed.timestampsMs, sessionReused: response.sessionReused === true, selection: storyboard.selection, manifest: storyboard.manifest, resolution: servedResolution(storyboard), sampledRanges: storyboard.sheets.map(sheet => ({ startMs: sheet.firstFrameIndex * sheet.intervalMs, endMs: (sheet.firstFrameIndex + sheet.frameCount - 1) * sheet.intervalMs })), totalFrames: storyboard.frameCount, diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index a4c6db75..250c615b 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -21,7 +21,7 @@ import { import { sessionBriefForModel, type SessionBrief } from '../runtime/session-evidence'; import { extractYouTubeVideoIds, youTubeUrls } from './youtube-references'; export { extractYouTubeVideoIds } from './youtube-references'; -import { frameRequestIssue, readFrameRequest, type FrameRequest } from './frame-request'; +import { frameRequestIssue, readFrameRequest, requestedSheets, type FrameRequest } from './frame-request'; import { conversationAssistantMessage, type ConversationTurn } from '../runtime/conversation-memory'; import { assertModelCostAvailable, type AgentModelCostBudget } from '../runtime/model-budget'; import { AGENT_CLASSIFICATION_TIMEOUT_MS, withRunDeadline } from '../runtime/deadline'; @@ -53,7 +53,7 @@ const classifierDecisionSchema = z.object({ channelId: capabilityRouteDecisionSchema.options[0].shape.channelId.describe('For research restricted to one supplied channel, copy its channel ID or handle from suppliedChannelIds. Never invent a channel identifier.'), videoId: capabilityRouteDecisionSchema.options[1].shape.videoId.optional(), reason: capabilityRouteDecisionSchema.options[3].shape.reason.optional().describe('Required for finalize: explain why existing context suffices, what scope is missing, or why the request is unsupported. The finalizer writes the response.'), - visualEvidence: visualEvidenceSchema.optional().describe('Required for executable routes. Whether answering needs images: none, helpful or required. helpful and required enable visual analysis. New image retrieval also needs frameAccess.'), + visualEvidence: visualEvidenceSchema.optional().describe('Required for executable routes. Whether answering needs images: none, helpful or required. helpful and required enable visual analysis. Required visuals need an explicit image grant unless the turn is saved-only. Use agent_selected for an ordinary visual question with delegated positions, or storyboard_only when only sheets are permitted.'), visualRequirements: visualRequirementsSchema.optional().describe('Only when visualEvidence is required: each requested fact that needs images, such as "presenter clothing".'), }); @@ -615,6 +615,10 @@ function candidateObject(value: unknown): Record | undefined { return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : undefined; } +function pinnedFrameAccess(entries: T[], videoId: string) { + return entries.map(entry => ({ ...entry, videoId: entry.videoId ?? videoId })); +} + function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[], request: FrameRequest): ClassificationIssue[] { // A single linked video pins inspection to itself, which would leave another video's frames out of reach. const linked = extractYouTubeVideoIds(input.message); @@ -622,7 +626,10 @@ function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifie return [{ path: 'frameAccess', code: 'frame_scope_outside_link', message: `The message links ${linked[0]}. Inspect that video, or use topic_research with comparisonVideoIds for every video whose frames this turn uses.` }]; } if (decision.route !== 'finalize') { - const issue = frameRequestIssue(decision, request, input.sessionBrief?.assets); + const pin = decision.route === 'topic_research' && !decision.comparisonVideoIds?.length && !decision.playlistListing && linked.length === 1; + const plan = pin ? { ...decision, route: 'inspect_video', videoId: linked[0], + frameAccess: Array.isArray(decision.frameAccess) ? pinnedFrameAccess(decision.frameAccess, linked[0]!) : decision.frameAccess } : decision; + const issue = frameRequestIssue(plan, request, input.sessionBrief?.assets); if (issue) return [{ path: 'frameAccess', code: 'invalid_frame_access', message: issue }]; } const scope = comparisonScopeIssues(decision, input, videoIds); @@ -849,7 +856,9 @@ function finishClassification( const { explicitSourceCount, setAsideFrames: _setAside, ...classified } = withVisualAccess(decision); const pinned = !(decision.route === 'topic_research' && decision.playlistListing) && explicitVideoIds.length === 1 && !decision.comparisonVideoIds?.length && (decision.route !== 'finalize' || (decision.responseIntent === 'context_answer' && decision.contextScope !== 'history')); - const route = pinned ? { ...classified, route: 'inspect_video' as const, videoId: explicitVideoIds[0]!, + const pinnedAccess = pinned && Array.isArray(classified.frameAccess) + ? pinnedFrameAccess(classified.frameAccess, explicitVideoIds[0]!) : classified.frameAccess; + const route = pinned ? { ...classified, frameAccess: pinnedAccess, route: 'inspect_video' as const, videoId: explicitVideoIds[0]!, visualEvidence: classified.visualEvidence ?? 'helpful', useStoryboard: classified.visualEvidence !== 'none' } : classified; const researchVideoCount = route.route === 'inspect_video' ? 1 : route.route === 'topic_research' ? Math.min(8, route.comparisonVideoIds?.length ?? explicitSourceCount ?? (route.researchBreadth === 'comparative' ? 4 : 2)) : 0; @@ -945,8 +954,12 @@ export function extractYouTubeChannelIds(message: string): string[] { } function bindStoryboardSelection(decision: CapabilityRouteDecision, request: FrameRequest): CapabilityRouteDecision { - return decision.route === 'inspect_video' && decision.frameAccess === 'storyboard_only' && request.sheetTimes.length - ? { ...decision, frameAccess: { storyboardTimestampsMs: request.sheetTimes } } : decision; + if ((decision.route !== 'inspect_video' && decision.route !== 'topic_research') || decision.frameAccess !== 'storyboard_only') return decision; + const selections = requestedSheets(decision, request); + const scoped = request.refreshSheets ? { ...decision, refreshFramesOnly: true } : decision; + if (!selections?.length) return scoped; + return { ...scoped, frameAccess: decision.route === 'inspect_video' + ? { storyboardTimestampsMs: selections[0]!.timestampsMs } : { storyboardSelections: selections } }; } export async function resolveCapabilityRoute(options: { @@ -962,7 +975,7 @@ export async function resolveCapabilityRoute(options: { // video after validation. Resuming must not run them without the access the user gave. const subjects = decision.route === 'inspect_video' ? [decision.videoId] : decision.route === 'topic_research' ? decision.comparisonVideoIds ?? [] : []; - if ('frameAccess' in decision && Array.isArray(decision.frameAccess) && decision.frameAccess.some(entry => entry.videoId && !subjects.includes(entry.videoId))) { + if ('frameAccess' in decision && Array.isArray(decision.frameAccess) && decision.frameAccess.some(entry => entry.videoId ? !subjects.includes(entry.videoId) : decision.route === 'inspect_video' || subjects.length > 0)) { return { route: 'finalize', responseIntent: 'clarification', reason: 'Clarify which video the saved and fresh frame positions refer to.' }; } if (options.message && (decision.route === 'inspect_video' || decision.route === 'topic_research')) { diff --git a/platform/src/agents/research/frame-request.ts b/platform/src/agents/research/frame-request.ts index fdd777ea..08e08c31 100644 --- a/platform/src/agents/research/frame-request.ts +++ b/platform/src/agents/research/frame-request.ts @@ -6,8 +6,8 @@ import type { SessionBrief } from '../runtime/session-evidence'; const UNIT_MS: ReadonlyArray<[RegExp, number]> = [[/^(?:ms|milliseconds?)$/i, 1], [/^(?:s|secs?|seconds?)$/i, 1_000], [/^(?:m|mins?|minutes?)$/i, 60_000], [/^(?:h|hrs?|hours?)$/i, 3_600_000]]; const unitMs = (unit: string) => UNIT_MS.find(([pattern]) => pattern.test(unit))?.[1] ?? 0; -const CLOCK_TIME = /\b(?:(\d+):)?(\d{1,2}):(\d{2})\b/gi; -const TIME_QUANTITY = /\b(\d+(?:\.\d+)?)\s*-?\s*(milliseconds?|ms|hours?|hrs?|h|minutes?|mins?|m|seconds?|secs?|s)(?![a-z])(?:\s*(?:and\s*)?(\d+(?:\.\d+)?)\s*(seconds?|secs?|s)\b)?/gi; +const CLOCK_TIME = /\b(?:(\d+):)?(\d{1,2}):(\d{2})(?:s)?\b/gi; +const TIME_QUANTITY = /(? 1 && clause.split(operand).filter(part => + [...part.matchAll(CLOCK_TIME), ...part.matchAll(TIME_QUANTITY)].length).length > 1; + const localSubjects = separate ? extractYouTubeVideoIds((prefix.split(operand).at(-1) ?? '') + match[0] + suffix.split(operand)[0]) : subjects; + positions.push({ time, role, videoIds: localSubjects.length ? localSubjects : subjects }); } } for (const match of message.matchAll(CLOCK_TIME)) { @@ -71,6 +84,9 @@ export function readFrameRequest(message: string) { const [, amount, unit, extra] = match; addProseTime(match, Math.round(Number(amount) * unitMs(unit!) + Number(extra ?? 0) * 1_000)); } + for (const match of message.matchAll(/\b(?:opening|first) (?:exact )?(?:frame|still|image)\b/gi)) addProseTime(match, 0); + // An unspecified interval has no safe numeric end. Ask for it before retrieving images. + const vagueWindow = /\b(?:first|last|opening|final) (?:[a-z]+(?:[- ](?:of|[a-z]+))?) (?:seconds?|minutes?|frames?)\b|\b(?:last|closing|final) (?:frame|still)\b/i.test(message); // An explicit position or window in the request takes precedence over the link's seek time. if (!positions.length && captureEndMs === undefined) { for (const url of urls) { @@ -85,13 +101,18 @@ export function readFrameRequest(message: string) { const noCapture = /\b(?:never|do not|don['’]t)\s+capture\s+(?:any\s+)?(?:new |exact )?(?:frames?|images?|stills?)\b/i.test(message); const sheetTimes = positions.every(position => position.role === 'sheet') && captureEndMs === undefined ? [...new Set(positions.map(position => position.time))] : []; - return { positions, sheetTimes, captureEndMs, noRecapture, noCapture }; + const refreshSheets = message.split(CLAUSE).some(clause => + /\b(?:refresh|reload|re-?fetch|fresh)\s+(?:(?:only|just|the|these|both|saved|video)\s+)*storyboards?\b|\bstoryboard(?:-only)?\s+refresh\b/i.test(clause) + && !/\b(?:not|never|without|don['’]t)\s+(?:refresh|reload|re-?fetch)\b/i.test(clause)); + const requestsSaved = positions.some(pos => pos.role === 'saved') || /\b(?:reuse|re-?inspect|inspect|analy[sz]e)\s+(?:the |my |our |that )?saved (?:video |exact )?(?:frames?|images?|stills?)\b/i.test(message); + return { positions, requestsSaved, refreshSheets, sheetTimes, captureEndMs, noRecapture, noCapture, vagueWindow }; } export type FrameRequest = ReturnType; interface FramePlan { route: string; videoId?: string; + comparisonVideoIds?: string[]; frameAccess?: RouteFrameAccess; savedOnly?: boolean; timestampedFrameQuestion?: string; @@ -100,23 +121,44 @@ interface FramePlan { setAsideFrames?: { videoId: string; timestampMs: number }[]; } +/** Bind each sheet position to the named subjects before the model can call the provider. */ +export function requestedSheets(plan: FramePlan, request: FrameRequest) { + if (!request.sheetTimes.length || request.sheetTimes.length > 24) return; + const subjects = plan.route === 'inspect_video' && plan.videoId ? [plan.videoId] : plan.comparisonVideoIds ?? []; + if (!subjects.length || request.positions.some(pos => pos.videoIds.some(id => !subjects.includes(id)))) return; + return subjects.flatMap(videoId => { + const times = request.positions.filter(pos => !pos.videoIds.length || pos.videoIds.includes(videoId)).map(pos => pos.time); + return times.length ? [{ videoId, timestampsMs: [...new Set(times)] }] : []; + }); +} + /** Bound the classifier's grant to operations named in the request, including on resume. */ export function frameRequestIssue(plan: FramePlan, request: FrameRequest, assets: SessionBrief['assets'] = [], resumed = false): string | undefined { const access = plan.frameAccess; - const { positions, sheetTimes, captureEndMs, noRecapture, noCapture } = request; + const { positions, captureEndMs, noRecapture, noCapture } = request; const limited = positions.length > 0 || captureEndMs !== undefined || noRecapture || noCapture; const sameSubject = (ids: string[], videoId?: string) => !ids.length || (videoId !== undefined && ids.includes(videoId)); - const targetedSheets = sheetTimes.length > 0 && sheetTimes.length <= 24 && plan.route === 'inspect_video' - && positions.every(position => sameSubject(position.videoIds, plan.videoId)); + const targetedSheets = requestedSheets(plan, request); + if (request.vagueWindow && access && !(Array.isArray(access) && access.every(entry => !entry.captureTimestampsMs.length))) + return 'Ask for the exact capture interval before retrieving images.'; + if (plan.visualEvidence === 'required' && !plan.savedOnly && (!access || (Array.isArray(access) + && !access.some(entry => entry.savedTimestampsMs.length || entry.captureTimestampsMs.length)))) + return 'Required visual evidence needs capture permission or a saved inspection in frameAccess. Select permitted new images or specify the saved inspection.'; if (access && typeof access === 'object' && !Array.isArray(access)) { // A previously persisted targeted sheet cannot widen during recovery. - if (!targetedSheets || new Set(access.storyboardTimestampsMs).size !== sheetTimes.length - || access.storyboardTimestampsMs.some(time => !sheetTimes.includes(time))) return 'Clarify the requested storyboard sheet positions.'; + const actual = 'storyboardSelections' in access ? access.storyboardSelections + : plan.route === 'inspect_video' ? [{ videoId: plan.videoId, timestampsMs: access.storyboardTimestampsMs }] : []; + if (!targetedSheets || actual.length !== targetedSheets.length || targetedSheets.some(expected => { + const times = actual.find(item => item.videoId === expected.videoId)?.timestampsMs; + return !times || times.length !== expected.timestampsMs.length || expected.timestampsMs.some(time => !times.includes(time)); + })) return 'Clarify the requested storyboard sheet positions.'; return; } const captures = positions.filter(position => position.role === 'capture'); const matches = (time: number, videoId?: string) => positions.filter(pos => pos.time === time && sameSubject(pos.videoIds, videoId)); + if (request.refreshSheets && !captures.length && !positions.some(pos => pos.role === 'saved') && access !== 'storyboard_only') + return 'Refresh the requested storyboard sheets with storyboard_only access.'; if (Array.isArray(access)) { for (const entry of access) { for (const time of entry.captureTimestampsMs) { @@ -139,7 +181,7 @@ export function frameRequestIssue(plan: FramePlan, request: FrameRequest, assets return `Include the requested new capture at ${clockTime(pos.time)} on its video.`; } } else { - if (access === 'storyboard_only' && !plan.refreshFramesOnly && (targetedSheets || (!positions.length && captureEndMs === undefined && !noRecapture))) return; + if (access === 'storyboard_only' && (!plan.refreshFramesOnly || request.refreshSheets) && (targetedSheets || (!positions.length && captureEndMs === undefined && !noRecapture))) return; if (!plan.savedOnly && (plan.timestampedFrameQuestion || (access && limited))) return 'The user fixed the frame positions. List the saved inspection positions and the permitted capture positions in frameAccess, one entry per video.'; if (plan.refreshFramesOnly && access !== 'agent_selected') return 'A frame refresh needs capture permission. List the positions in frameAccess, or set agent_selected if the user left them to you.'; diff --git a/platform/src/agents/research/research-agent.ts b/platform/src/agents/research/research-agent.ts index 0f5ab43b..e5caf534 100644 --- a/platform/src/agents/research/research-agent.ts +++ b/platform/src/agents/research/research-agent.ts @@ -1,5 +1,5 @@ import { inspectionRecordsForModel, INSPECTION_RECORD_GUIDANCE } from '../runtime/inspection-records'; -import { frameActivity, frameList, isFrameTurn } from '../runtime/frame-access'; +import { frameActivity, imagePermissions, isFrameTurn } from '../runtime/frame-access'; import { VISUAL_INSPECTION_SKILL } from './skills/visual-inspection.generated'; import { scheduleEvidenceTools } from '../runtime/evidence-scheduler'; import { withRunAssetCache, type RunAssetCache } from '../runtime/run-asset-cache'; @@ -281,7 +281,10 @@ async function executeResearchRunWithAssets(options: { const upstream = decision.savedOnly ? savedOnlyProvider() : createYouTubeAgentProvider(options.env, undefined, researchDeadlineAt); const refreshFrames = decision.refreshEvidence || decision.refreshFramesOnly; - const provider = createCapabilityProvider(options.session ? sessionProvider(upstream, options.session, decision.refreshEvidence, options.registerRetrievedAsset, assetCache, refreshFrames) : upstream, decision); + const permissions = imagePermissions(decision); + const refreshSheets = decision.refreshEvidence || (decision.refreshFramesOnly && permissions.canRetrieveSheets && !permissions.canCapture); + const provider = createCapabilityProvider(options.session ? sessionProvider(upstream, options.session, decision.refreshEvidence, + options.registerRetrievedAsset, assetCache, refreshFrames, refreshSheets) : upstream, decision); const transcriptAnalyst = createTranscriptAnalyst( runModel(options.env, options.sessionAffinity, 'low', { ...modelMetadata, @@ -466,14 +469,9 @@ async function runResearchAgentWithModelWithinDeadline(options: { const playlistListing = options.decision.route === 'topic_research' && options.decision.playlistListing && !options.decision.savedOnly ? options.decision : undefined; // A storyboard sheet includes neighboring images, even when requested at one timestamp. - const frameScope = frameList(options.decision); - const access = options.decision.frameAccess; - const storyboardSelection = options.decision.route === 'inspect_video' && access && typeof access === 'object' && 'storyboardTimestampsMs' in access - ? { videoId: options.decision.videoId, timestampsMs: access.storyboardTimestampsMs } : undefined; - const canCapture = access === 'agent_selected' || frameScope?.some(entry => entry.captureTimestampsMs.length); - const storyboardAllowed = access === 'storyboard_only' || access === 'agent_selected' || !!storyboardSelection; + const { frameScope, sheetSelections, canCapture, canRetrieveSheets } = imagePermissions(options.decision); const toolNames = (options.toolNames ?? capability.toolNames) - .filter(name => name !== 'get_video_storyboard' || storyboardAllowed) + .filter(name => name !== 'get_video_storyboard' || canRetrieveSheets) // New captures are closed unless the route lists their positions or delegates them. .filter(name => name !== 'get_video_frames' || canCapture) .filter(name => !playlistListing || (!playlistListing.playlistId && name === 'get_playlist') || name === FINALIZE_ANSWER_TOOL_NAME) @@ -648,7 +646,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { refreshEvidence: options.decision.refreshEvidence, refreshFramesOnly: options.decision.refreshFramesOnly, frameAccess: frameScope, - storyboardSelection, + storyboardSelections: sheetSelections, timestampedFrameQuestion: options.decision.route === 'inspect_video' && !options.decision.savedOnly ? options.decision.timestampedFrameQuestion : undefined, pinnedVideoId: options.decision.route === 'inspect_video' ? options.decision.videoId : undefined, @@ -788,6 +786,10 @@ async function runResearchAgentWithModelWithinDeadline(options: { ['youtube_frame_analysis', 'youtube_storyboard_analysis'].includes(artifact.type))); // Transcript breadth is not completion when the requested facts need images. // Keep the existing time, total-call, step and cost ceilings authoritative. + const discoveryCaptureUnavailable = () => frameScope?.some(entry => !entry.videoId && entry.captureTimestampsMs.length) + && ![...evidence.values()].some(packet => packet.packetId.startsWith(`packet:${options.context.runId}:`) + && packet.sources.some(source => source.videoId && !trackedContext.transcriptSelection?.tooLong?.has(source.videoId) + && !trackedContext.transcriptSelection?.regionRestricted?.has(source.videoId))); const needsVisualWork = () => { const failed = new Set([...toolFailures.values()].map(failure => failure.toolName)); if (frameScope) { @@ -803,24 +805,46 @@ async function runResearchAgentWithModelWithinDeadline(options: { && !inspectedVersions.has(asset.version) && open(asset.version))); const needsCaptureAnalysis = (visualRequired || (options.decision.route === 'inspect_video' && !!options.decision.timestampedFrameQuestion)) && pendingCaptureVersions.some(open); + return (toolNames.includes('analyze_video_frames') && !!options.context.analyzeFrames && (needsSavedAnalysis || needsCaptureAnalysis)) || (toolNames.includes('get_video_frames') && !!options.context.provider.frames && frameExtractionBudget(options.researchDeadlineAt) >= FRAME_EXTRACTION_MIN_MS - && captures.some(item => !item.requested && !failedCaptures.has(`${item.videoId}:${item.time}`))); + && !discoveryCaptureUnavailable() && captures.some(item => !item.requested && !failedCaptures.has(`${item.videoId}:${item.time}`))); + } + if (sheetSelections) { + // A completed or failed sheet on one subject cannot stand in for another subject. + const own = [...evidence.values()].filter(packet => packet.kind === 'youtube_storyboard' + && packet.packetId.startsWith(`packet:${options.context.runId}:`)).flatMap(packet => packet.artifacts); + return sheetSelections.some(({ videoId, timestampsMs }) => { + const failedForVideo = (name: string) => [...toolFailures.values()].some(failure => failure.toolName === name + && (!failure.videoId || failure.videoId === videoId)); + return timestampsMs.some(time => { + const retrieved = own.find(artifact => artifact.type === 'youtube_storyboard_retrieval' && artifact.data.videoId === videoId + && Array.isArray(artifact.data.requestedTimestampsMs) && artifact.data.requestedTimestampsMs.includes(time)); + const inspected = own.some(artifact => artifact.type === 'youtube_storyboard_analysis' && artifact.data.videoId === videoId + && Array.isArray(artifact.data.sampledRanges) && artifact.data.sampledRanges.some(range => + range && typeof range.startMs === 'number' && typeof range.endMs === 'number' + && range.startMs <= time && time < range.endMs + Number(artifact.data.intervalMs))); + if (inspected) return false; + if (retrieved) return visualRequired && Array.isArray(retrieved.data.analysisAssetVersions) && retrieved.data.analysisAssetVersions.length > 0 + && toolNames.includes('analyze_video_storyboard') && !!options.context.analyzeStoryboard + && !failedForVideo('analyze_video_storyboard') && canAnalyzeStoryboard(options.researchDeadlineAt); + return toolNames.includes('get_video_storyboard') && !!options.context.provider.storyboard && !failedForVideo('get_video_storyboard') + && storyboardRetrievalBudget(options.researchDeadlineAt) >= STORYBOARD_RETRIEVAL_MIN_MS; + }); + }); } if (!visualRequired || hasVisualObservations()) return false; const savedFrames = [...evidence.values()].some(packet => packet.kind === 'youtube_frames' && packet.assetVersions?.length); - const framesPossible = toolNames.includes('get_video_frames') && toolNames.includes('analyze_video_frames') - && !!options.context.provider.frames && !!options.context.analyzeFrames - && !failed.has('get_video_frames') && !failed.has('analyze_video_frames') - && (savedFrames || frameExtractionBudget(options.researchDeadlineAt) >= FRAME_EXTRACTION_MIN_MS); + const framesPossible = toolNames.includes('analyze_video_frames') && !!options.context.analyzeFrames && !failed.has('analyze_video_frames') + && (savedFrames || (toolNames.includes('get_video_frames') && !!options.context.provider.frames + && !failed.has('get_video_frames') && frameExtractionBudget(options.researchDeadlineAt) >= FRAME_EXTRACTION_MIN_MS)); const savedStoryboards = [...evidence.values()].some(packet => packet.kind === 'youtube_storyboard' && packet.artifacts.some(artifact => artifact.type === 'youtube_storyboard_retrieval' && Number(artifact.data.sampledFrames) > 0)) || options.context.session?.brief().assets.some(asset => asset.kind === 'storyboard_sheet') === true; - const storyboardPossible = toolNames.includes('get_video_storyboard') && toolNames.includes('analyze_video_storyboard') - && !!options.context.provider.storyboard && !!options.context.analyzeStoryboard + const storyboardPossible = toolNames.includes('analyze_video_storyboard') && !!options.context.analyzeStoryboard && !failed.has('analyze_video_storyboard') && canAnalyzeStoryboard(options.researchDeadlineAt) - && (savedStoryboards || (!failed.has('get_video_storyboard') + && (savedStoryboards || (toolNames.includes('get_video_storyboard') && !!options.context.provider.storyboard && !failed.has('get_video_storyboard') && storyboardRetrievalBudget(options.researchDeadlineAt) >= STORYBOARD_RETRIEVAL_MIN_MS)); return framesPossible || storyboardPossible; }; @@ -918,7 +942,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { ...(options.finalizationModel ? { finalizationReserveMs: 1_000 } : {}) } : {}), unavailableTools: () => [ ...(searchUsed || !!options.decision.comparisonVideoIds?.length || (options.decision.route === 'topic_research' && !!options.decision.channelId) ? ['search_youtube'] : []), - ...(frameExtractionBudget(options.researchDeadlineAt) < FRAME_EXTRACTION_MIN_MS ? ['get_video_frames'] : []), + ...((discoveryCaptureUnavailable() || frameExtractionBudget(options.researchDeadlineAt) < FRAME_EXTRACTION_MIN_MS) ? ['get_video_frames'] : []), ...(storyboardRetrievalBudget(options.researchDeadlineAt) < STORYBOARD_RETRIEVAL_MIN_MS ? ['get_video_storyboard'] : []), ...(!canAnalyzeStoryboard(options.researchDeadlineAt) ? ['analyze_video_storyboard'] : []), ...(needsVisualWork() ? [completionTool] : []), @@ -1126,7 +1150,7 @@ async function runUnifiedFinalizer(options: { .filter(packet => packet.sources.some(source => source.videoId && subjects.has(source.videoId))); const inspectionRecords = inspected.length ? inspectionRecordsForModel(inspected, options.context.evidenceRunRecords?.(inspected.map(packet => packet.packetId))) : undefined; - const frameReport = isFrameTurn(options.decision); + const frameReport = isFrameTurn(options.decision) || options.evidence.some(packet => packet.kind === 'youtube_frames'); const baseOutputSchema = conversational ? conversationalFinalizationOutputSchema : intent === 'context_answer' ? contextFinalizationOutputSchema : finalizationOutputSchema; const numberedItemCount = 'numberedItemCount' in options.decision ? options.decision.numberedItemCount : undefined; @@ -1229,7 +1253,7 @@ async function runUnifiedFinalizer(options: { 'Keep JSON compact. Use short ref_N citations rather than full evidence IDs. For specific-video comparisons cite every subject, or explicitly state the missing side and add ANSWER_SCOPE_SHORTFALL. If contextIncomplete is true, do not claim exhaustive coverage unless the supplied evidence establishes it.', EVIDENCE_DELIVERY_GUIDANCE, INSPECTION_RECORD_GUIDANCE, - ...(frameReport ? ['Base visual claims on inspections made in this turn and explain any gap. The application appends the frame activity, and notes any cited observation from an earlier turn, from the recorded operations. Do not describe tool actions yourself.'] : []), + ...(frameReport ? ['Distinguish earlier visual observations from inspections made in this turn. Explain evidence gaps. The application appends frame activity and the provenance of earlier frame citations from the recorded operations.'] : []), 'Recovery has a limited token budget. Preserve the requested count where evidence permits by shortening each item before reducing the count. If scope remains incomplete, state the shortfall and add ANSWER_SCOPE_SHORTFALL. Do not pad or invent findings.', 'State important evidence gaps plainly. Do not claim that a failed provider operation succeeded.', 'For list evidence such as playlists, take counts from collection: entriesShown are the entries supplied here, and collection.coverage states what was not retained, not shown, or not stated by the provider. Never report fewer available entries than are shown. Describe unshown or unretained entries as a context limit, not as absent from the source. When the answer cites a playlist entry, or reviewed transcript or visual content for a listed video, the application appends a playlist coverage note stating the provider total, the 12-video listing limit and which videos had content reviewed; do not contradict it. Otherwise no note is appended, so state any material playlist coverage limit yourself. Never present listing details as reviewed video content. evidenceOmittedFromContext counts saved list packets that did not fit at all; notListed of them are counted but not named.', diff --git a/platform/src/agents/research/skills/visual-inspection.generated.ts b/platform/src/agents/research/skills/visual-inspection.generated.ts index cfe466c3..a22bb272 100644 --- a/platform/src/agents/research/skills/visual-inspection.generated.ts +++ b/platform/src/agents/research/skills/visual-inspection.generated.ts @@ -1,2 +1,2 @@ // Generated from visual-inspection/SKILL.md. Run npx tsx scripts/generate-agent-skills.ts to update. -export const VISUAL_INSPECTION_SKILL = "---\nname: visual-inspection\ndescription: Locate and inspect YouTube images for visible details, resolve conflicts with captions or metadata, and explain recorded frame choices on follow-up.\n---\n\n# Visual inspection\n\nAnswer visible questions from analyzed images. Metadata and captions locate moments and identify speakers; they do not establish clothing, scenery or readable on-screen text.\n\n## Locate\n\nReuse adequate saved analysis first. Choose how to locate by what the question needs:\n\n- The look of a whole video, such as camera style, colors, set or diagram style: a small spread of storyboard sheets. Captions add nothing.\n- A visual moment with no spoken cue: scan storyboard sheets with an analysis question. One call can include many sheets.\n- A moment defined by topic or speech, such as the slide for a named concept or a spoken introduction: use title, description or chapter timestamps first, then captions. Retrieve the transcript without an analysis question and find the passage with search_originals. A transcript analyst never answers visual questions.\n\nA chapter is a timestamped section listed by the publisher, not an inferred scene boundary. In topic research, fetch captions only for videos you will inspect. The last provider calls are kept for get_video_frames, so do not spend them on other retrieval. Respect the user's chosen inspection method.\n\nFor a presenter question, an introduction is a lead, not proof the presenter is on screen. A storyboard overview can locate her actual appearance. Prefer a few frames that answer the question over a six-frame quota or unexplained samples across the whole video. Questions about changes across the video need wider coverage.\n\n## Inspect\n\nUse get_video_frames for exact seek positions when storyboard images are too small, text is ambiguous, or captions conflict with visible labels. If storyboard retrieval or analysis fails, switch to get_video_frames with an analysis question while time remains; never repeat the failed storyboard request unchanged. Supply selections with one basis category and the locating evidenceIds for each timestamp. user_requested and exploratory selections cite no excerpts; describe an exploratory sample as exploratory. A basis records the kind of choice, not private reasoning or proof the desired scene exists.\n\nFor new images, supply an analysis question to get_video_frames or get_video_storyboard so the images are analyzed while they are saved. Retrieval without analysis provides no observations; a RETRIEVAL_ANALYSIS_FAILED warning means the images were saved but still need analyze_video_frames or analyze_video_storyboard. Inspect only analysisAssetVersions from storyboard results, never the manifest. The runtime supplies title, full description, channel and question to image analysts. Analyze saved images directly for new questions rather than fetching the same images again.\n\nCheck whether each requested detail is visible and readable. A transition or irrelevant shot does not answer a clothing question. When needed, try a nearby timestamp informed by the overview, within the remaining time and provider budget. When names conflict, inspect a readable on-screen label at an exact frame and distinguish the label from noisy captions. A name in metadata is context, not visual identification.\n\n## Finish and explain\n\nPresent the visible findings with citations and material uncertainty. Explain inspection steps only when the user explicitly asks about the workflow. Finishing transcript research does not finish a visual question. Before finishing, check every requested subject and attribute against analyzed images; stop when they are covered or a real retrieval, analysis or budget limit prevents it, not merely because captions omit visual details. Keep image quality and missing-image warnings. Cite each frame-specific observation at its own timestamp. Describe shared observations only over their supported sampled frames. State missing details and actual retrieval or analysis limits. Sampled stills do not establish continuous appearance or motion.\n\nFor a follow-up about how inspection happened, use frameCoverage, selection bases and delivery records. Separate requested or retrieved positions from analyzed frames and storyboard samples. A storyboard analyst receives every tile in the selected sheets; its reported finding times may cover only a subset. Sheet spread selection does not choose 3:20 over 3:30 within the same supplied sheet. A recorded basis can explain why a timestamp was chosen, such as a spoken introduction; what a cited excerpt says is evidence only when that excerpt is supplied. Without a basis, say the reason was not recorded. Do not reconstruct a past rationale from what now seems sensible. Treat user examples such as “why 3:20 instead of 3:30?” as alternatives, not captured frames. Claim transcript reading or fresh image analysis only when the supplied records establish it. Historical evidence and memory alone do not prove a current operation.\n"; +export const VISUAL_INSPECTION_SKILL = "---\nname: visual-inspection\ndescription: Locate and inspect YouTube images for visible details, resolve conflicts with captions or metadata, and explain recorded frame choices on follow-up.\n---\n\n# Visual inspection\n\nAnswer visible questions from analyzed images. Metadata and captions locate moments and identify speakers; they do not establish clothing, scenery or readable on-screen text.\n\n## Locate\n\nUse only the visual tools offered on this turn and the videos and positions the user permitted. If retrieval is unavailable, inspect permitted saved images or report the missing evidence. Reuse adequate saved analysis first. Choose how to locate by what the question needs:\n\n- The look of a whole video, such as camera style, colors, set or diagram style: use a small spread of storyboard sheets when that retrieval is available.\n- A visual moment with no spoken cue: scan permitted storyboard sheets with an analysis question when available. One call can include many sheets.\n- A moment defined by topic or speech, such as the slide for a named concept or a spoken introduction: use title, description or chapter timestamps first, then captions. Retrieve the transcript without an analysis question and find the passage with search_originals. A transcript analyst never answers visual questions.\n\nA chapter is a timestamped section listed by the publisher, not an inferred scene boundary. In topic research, fetch captions only for videos you will inspect. When exact capture is available, keep the reserved provider calls for it. Respect the user's chosen inspection method.\n\nFor a presenter question, an introduction is a lead, not proof the presenter is on screen. A storyboard overview can locate her actual appearance. Prefer a few frames that answer the question over a six-frame quota or unexplained samples across the whole video. Questions about changes across the video need wider coverage.\n\n## Inspect\n\nWhen get_video_frames is available, use it for permitted exact seek positions if storyboard images are too small, text is ambiguous, or captions conflict with visible labels. If a sheet fails, use an authorized exact capture with an analysis question when available. Otherwise record the gap. Supply selections with one basis category and the locating evidenceIds for each timestamp. user_requested and exploratory selections cite no excerpts; describe an exploratory sample as exploratory. A basis records the kind of choice, not private reasoning or proof the desired scene exists.\n\nWhen a retrieval tool is offered, include the analysis question so the analyst sees the images as they are saved. Retrieval without analysis provides no observations; a RETRIEVAL_ANALYSIS_FAILED warning means the images were saved but still need analyze_video_frames or analyze_video_storyboard. Inspect only analysisAssetVersions from storyboard results, never the manifest. The runtime supplies title, full description, channel and question to image analysts. Analyze saved images directly for new questions rather than fetching the same images again.\n\nCheck whether each requested detail is visible and readable. A transition or irrelevant shot does not answer a clothing question. When needed, try a permitted nearby timestamp informed by the overview, within the remaining time and provider budget. When names conflict, inspect a readable on-screen label at an exact frame and distinguish the label from noisy captions. A name in metadata is context, not visual identification.\n\n## Finish and explain\n\nPresent the visible findings with citations and material uncertainty. Explain inspection steps only when the user explicitly asks about the workflow. Finishing transcript research does not finish a visual question. Before finishing, check every requested subject and attribute against analyzed images; stop when they are covered or a real retrieval, analysis or budget limit prevents it, not merely because captions omit visual details. Keep image quality and missing-image warnings. Cite each frame-specific observation at its own timestamp. Describe shared observations only over their supported sampled frames. State missing details and actual retrieval or analysis limits. Sampled stills do not establish continuous appearance or motion.\n\nFor a follow-up about how inspection happened, use frameCoverage, selection bases and delivery records. Separate requested or retrieved positions from analyzed frames and storyboard samples. A storyboard analyst receives every tile in the selected sheets; its reported finding times may cover only a subset. Sheet spread selection does not choose 3:20 over 3:30 within the same supplied sheet. A recorded basis can explain why a timestamp was chosen, such as a spoken introduction; what a cited excerpt says is evidence only when that excerpt is supplied. Without a basis, say the reason was not recorded. Do not reconstruct a past rationale from what now seems sensible. Treat user examples such as “why 3:20 instead of 3:30?” as alternatives, not captured frames. Claim transcript reading or fresh image analysis only when the supplied records establish it. Historical evidence and memory alone do not prove a current operation.\n"; diff --git a/platform/src/agents/research/skills/visual-inspection/SKILL.md b/platform/src/agents/research/skills/visual-inspection/SKILL.md index aecb181e..ee2e8700 100644 --- a/platform/src/agents/research/skills/visual-inspection/SKILL.md +++ b/platform/src/agents/research/skills/visual-inspection/SKILL.md @@ -9,23 +9,23 @@ Answer visible questions from analyzed images. Metadata and captions locate mome ## Locate -Reuse adequate saved analysis first. Choose how to locate by what the question needs: +Use only the visual tools offered on this turn and the videos and positions the user permitted. If retrieval is unavailable, inspect permitted saved images or report the missing evidence. Reuse adequate saved analysis first. Choose how to locate by what the question needs: -- The look of a whole video, such as camera style, colors, set or diagram style: a small spread of storyboard sheets. Captions add nothing. -- A visual moment with no spoken cue: scan storyboard sheets with an analysis question. One call can include many sheets. +- The look of a whole video, such as camera style, colors, set or diagram style: use a small spread of storyboard sheets when that retrieval is available. +- A visual moment with no spoken cue: scan permitted storyboard sheets with an analysis question when available. One call can include many sheets. - A moment defined by topic or speech, such as the slide for a named concept or a spoken introduction: use title, description or chapter timestamps first, then captions. Retrieve the transcript without an analysis question and find the passage with search_originals. A transcript analyst never answers visual questions. -A chapter is a timestamped section listed by the publisher, not an inferred scene boundary. In topic research, fetch captions only for videos you will inspect. The last provider calls are kept for get_video_frames, so do not spend them on other retrieval. Respect the user's chosen inspection method. +A chapter is a timestamped section listed by the publisher, not an inferred scene boundary. In topic research, fetch captions only for videos you will inspect. When exact capture is available, keep the reserved provider calls for it. Respect the user's chosen inspection method. For a presenter question, an introduction is a lead, not proof the presenter is on screen. A storyboard overview can locate her actual appearance. Prefer a few frames that answer the question over a six-frame quota or unexplained samples across the whole video. Questions about changes across the video need wider coverage. ## Inspect -Use get_video_frames for exact seek positions when storyboard images are too small, text is ambiguous, or captions conflict with visible labels. If storyboard retrieval or analysis fails, switch to get_video_frames with an analysis question while time remains; never repeat the failed storyboard request unchanged. Supply selections with one basis category and the locating evidenceIds for each timestamp. user_requested and exploratory selections cite no excerpts; describe an exploratory sample as exploratory. A basis records the kind of choice, not private reasoning or proof the desired scene exists. +When get_video_frames is available, use it for permitted exact seek positions if storyboard images are too small, text is ambiguous, or captions conflict with visible labels. If a sheet fails, use an authorized exact capture with an analysis question when available. Otherwise record the gap. Supply selections with one basis category and the locating evidenceIds for each timestamp. user_requested and exploratory selections cite no excerpts; describe an exploratory sample as exploratory. A basis records the kind of choice, not private reasoning or proof the desired scene exists. -For new images, supply an analysis question to get_video_frames or get_video_storyboard so the images are analyzed while they are saved. Retrieval without analysis provides no observations; a RETRIEVAL_ANALYSIS_FAILED warning means the images were saved but still need analyze_video_frames or analyze_video_storyboard. Inspect only analysisAssetVersions from storyboard results, never the manifest. The runtime supplies title, full description, channel and question to image analysts. Analyze saved images directly for new questions rather than fetching the same images again. +When a retrieval tool is offered, include the analysis question so the analyst sees the images as they are saved. Retrieval without analysis provides no observations; a RETRIEVAL_ANALYSIS_FAILED warning means the images were saved but still need analyze_video_frames or analyze_video_storyboard. Inspect only analysisAssetVersions from storyboard results, never the manifest. The runtime supplies title, full description, channel and question to image analysts. Analyze saved images directly for new questions rather than fetching the same images again. -Check whether each requested detail is visible and readable. A transition or irrelevant shot does not answer a clothing question. When needed, try a nearby timestamp informed by the overview, within the remaining time and provider budget. When names conflict, inspect a readable on-screen label at an exact frame and distinguish the label from noisy captions. A name in metadata is context, not visual identification. +Check whether each requested detail is visible and readable. A transition or irrelevant shot does not answer a clothing question. When needed, try a permitted nearby timestamp informed by the overview, within the remaining time and provider budget. When names conflict, inspect a readable on-screen label at an exact frame and distinguish the label from noisy captions. A name in metadata is context, not visual identification. ## Finish and explain diff --git a/platform/src/agents/runtime/frame-access.ts b/platform/src/agents/runtime/frame-access.ts index 9f6300f4..b59497e6 100644 --- a/platform/src/agents/runtime/frame-access.ts +++ b/platform/src/agents/runtime/frame-access.ts @@ -69,11 +69,24 @@ export function frameList(decision: CapabilityRouteDecision): FrameAccess | unde return 'frameAccess' in decision && Array.isArray(decision.frameAccess) ? decision.frameAccess : undefined; } +export function imagePermissions(decision: Extract) { + const access = decision.frameAccess; + const frameScope = frameList(decision); + const object = access && typeof access === 'object' && !Array.isArray(access) ? access : undefined; + const sheetSelections = object && ('storyboardSelections' in object ? object.storyboardSelections + : decision.route === 'inspect_video' ? [{ videoId: decision.videoId, timestampsMs: object.storyboardTimestampsMs }] : undefined); + const retrieval = !decision.savedOnly && decision.useStoryboard !== false; + return { frameScope, sheetSelections, + canCapture: retrieval && (access === 'agent_selected' || !!frameScope?.some(entry => entry.captureTimestampsMs.length)), + canRetrieveSheets: retrieval && (access === 'storyboard_only' || access === 'agent_selected' || !!sheetSelections) }; +} + /** A turn that asked for frame work, whose answer carries the application's account of it. */ export function isFrameTurn(decision: CapabilityRouteDecision): boolean { - return (decision.route === 'inspect_video' || decision.route === 'topic_research') - && (Array.isArray(decision.frameAccess) || decision.frameAccess === 'agent_selected' || !!decision.refreshFramesOnly - || (decision.route === 'inspect_video' && !!decision.timestampedFrameQuestion)); + if (decision.route !== 'inspect_video' && decision.route !== 'topic_research') return false; + const { frameScope, canCapture, canRetrieveSheets } = imagePermissions(decision); + return !!frameScope || canCapture || (!!decision.refreshFramesOnly && !canRetrieveSheets) + || (decision.route === 'inspect_video' && !!decision.timestampedFrameQuestion); } /** A saved frame as the session inventory records it. */ @@ -88,7 +101,7 @@ export interface InventoryFrame { version: string; videoId: string; timestampMs? * An image is saved when the session stored it before this run started, however this run reached it. */ export function frameTurnNotes(options: { - runId: string; startedAt: number; access?: FrameAccess; inventory: readonly InventoryFrame[]; packets: readonly EvidencePacket[]; + runId: string; startedAt: number; access?: FrameAccess; requestedSaved?: boolean; inventory: readonly InventoryFrame[]; packets: readonly EvidencePacket[]; cited: ReadonlyArray<{ packet: EvidencePacket; startMs: number; videoId?: string; title?: string }>; }): string[] { const { runId, access, inventory, packets } = options; @@ -113,16 +126,15 @@ export function frameTurnNotes(options: { const listed = new Set(access?.flatMap(entry => entry.videoId ? [...entry.savedTimestampsMs, ...entry.captureTimestampsMs].map(time => `${entry.videoId}:${time}`) : [])); const reinspected = savedBefore.filter(frame => analyzed.has(frame.version)); - for (const frame of reinspected) { - if (frame.timestampMs !== undefined && !listed.has(`${frame.videoId}:${frame.timestampMs}`)) - notes.push(`I re-inspected the saved ${named(frame.videoId, frame.timestampMs)}.`); - } - // Say why a saved frame went unused, so a skipped inspection is never silent. - const plannedSaved = !!access?.some(entry => entry.savedTimestampsMs.length); - if (savedBefore.length && !reinspected.length) notes.push(plannedSaved - ? 'No frame saved before this turn was re-inspected in this turn.' - : 'No frame saved before this turn was re-inspected in this turn, because its plan included none.'); - const unplaced = [...new Set(savedBefore.filter(frame => frame.timestampMs === undefined && !analyzed.has(frame.version)).map(frame => frame.videoId))]; + const additional = reinspected.filter(frame => frame.timestampMs === undefined || !listed.has(`${frame.videoId}:${frame.timestampMs}`)); + if (additional.length === 1 && additional[0]!.timestampMs !== undefined) + notes.push(`I re-inspected the saved ${named(additional[0]!.videoId, additional[0]!.timestampMs)}.`); + else if (additional.length) notes.push(`I re-inspected ${additional.length} saved frame${additional.length === 1 ? '' : 's'} in this turn.`); + const plannedSaved = access?.filter(entry => entry.savedTimestampsMs.length) ?? []; + if (options.requestedSaved && !plannedSaved.length && !reinspected.length) + notes.push('No frame saved before this turn was re-inspected in this turn.'); + const unplaced = new Set(savedBefore.filter(frame => frame.timestampMs === undefined && !analyzed.has(frame.version) + && plannedSaved.some(entry => entry.videoId === frame.videoId)).map(frame => frame.videoId)); for (const videoId of unplaced) notes.push(`Saved frames of "${title(videoId) ?? videoId}" have no recorded position, so this turn could not match them to requested times or re-inspect them.`); // Each citation is judged by the image behind it. When the record cannot tie the citation diff --git a/platform/src/agents/runtime/session-provider.ts b/platform/src/agents/runtime/session-provider.ts index 4eaf0191..32d81f02 100644 --- a/platform/src/agents/runtime/session-provider.ts +++ b/platform/src/agents/runtime/session-provider.ts @@ -19,6 +19,7 @@ export function sessionProvider( onRetrieved?: (claim: string, version: string) => void, assetCache?: RunAssetCache, refreshFrames = refresh, + refreshSheets = refresh, ): YouTubeAgentProvider { const refreshed = new Set(); const remember = (result: CachedResult, generation: number) => { @@ -162,7 +163,7 @@ export function sessionProvider( // Sheet indexes are level-specific, so each served resolution has its own manifest. const requested = options.resolution ?? 'standard'; const manifestKey = (resolution: StoryboardResolution) => `storyboard:${id}:manifest${resolution === 'high' ? ':high' : ''}`; - const fresh = rediscovering || (refresh && !refreshed.has(manifestKey(requested))) || !!options.refresh; + const fresh = rediscovering || (refreshSheets && !refreshed.has(manifestKey(requested))) || !!options.refresh; let prefetched: CachedResult | undefined; let loaded = false; const retrieveManifest = (resolution: StoryboardResolution, reload: boolean, load: () => Promise>) => store.retrieve( @@ -216,7 +217,7 @@ export function sessionProvider( if (generation !== store.generation()) throw new Error('Session assets changed during retrieval.'); const key = `storyboard:${id}:${manifestVersion}:${index}`; const hit = - !prefetched && !(refresh && !refreshed.has(key)) && !options.refresh && (await store.lookup(key)); + !prefetched && !(refreshSheets && !refreshed.has(key)) && !options.refresh && (await store.lookup(key)); return { index, hit }; })); for (const { index, hit } of cached) { @@ -233,7 +234,7 @@ export function sessionProvider( const fetched = prefetched ?? await provider.storyboard!( id, undefined, - { sheetIndexes: missing, maxSheets: missing.length, ...(served === 'high' ? { resolution: served } : {}), ...(options.deadlineAt === undefined ? {} : { deadlineAt: options.deadlineAt }), ...(options.signal ? {signal:options.signal} : {}), ...((refresh || options.refresh) ? {refresh:true} : {}) }, + { sheetIndexes: missing, maxSheets: missing.length, ...(served === 'high' ? { resolution: served } : {}), ...(options.deadlineAt === undefined ? {} : { deadlineAt: options.deadlineAt }), ...(options.signal ? {signal:options.signal} : {}), ...((refreshSheets || options.refresh) ? {refresh:true} : {}) }, diagnostic, onReady ? value => { if (sameManifest(value)) onReady(storyboardSchema.parse({ ...value, sheets: [...results.flatMap(result => result.value.sheets), ...value.sheets].sort((a, b) => a.firstFrameIndex - b.firstFrameIndex), diff --git a/platform/test/classifier-route-recovery.test.ts b/platform/test/classifier-route-recovery.test.ts index 993ba207..a06e04e6 100644 --- a/platform/test/classifier-route-recovery.test.ts +++ b/platform/test/classifier-route-recovery.test.ts @@ -173,19 +173,19 @@ describe('classifier last resort', () => { it.each([ ['5d3f5302', 'dQw4w9WgXcQ', 'inspect_videovideoId'], ['794f5142', 'tXcT3OE7G1g', 'inspect_videovideoId'], - ])('recovers run %s from attempt 1 when the repair has corrupted keys and the fallback errors', async (_run, videoId, key) => { + ])('asks for missing frame permission for run %s when the repair has corrupted keys and the fallback errors', async (_run, videoId, key) => { const result = await classify(framesMessage(videoId), rawClassifier(production.inspectVideo(videoId), production.corruptedRepair(videoId, key)), { fallbackModel: rawClassifier({ error: 'fallback unavailable' }) }); - expect(result.decision).toMatchObject({ route: 'inspect_video', videoId, visualEvidence: 'required' }); - expect(result.lastResort).toMatchObject({ lastResort: 'route_recovery', defaultedFields: ['route'] }); + expect(result.decision).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + expect(result.decision).not.toHaveProperty('frameAccess'); }); it.each([ 'inspect_videovideoId', 'inspect_videovideoId', ])('never reads corrupted keys, and assembles the remaining valid fields: %s', async key => { - const corrupted = production.corruptedRepair('aaaaaaaaaaa', key); + const corrupted = { ...production.corruptedRepair('aaaaaaaaaaa', key), frameAccess: 'agent_selected' }; const result = await classify(framesMessage('dQw4w9WgXcQ'), rawClassifier(corrupted)); // The video comes from the request, not from the corrupted key's value. expect(result.decision).toMatchObject({ route: 'inspect_video', videoId: 'dQw4w9WgXcQ', visualEvidence: 'required' }); @@ -239,7 +239,7 @@ describe('classifier last resort', () => { it('keeps required visuals mandatory, taking the requirements from the request when the model gave none', async () => { const message = 'What did the keynote presenters wear?'; const result = await classify(message, rawClassifier({ answerDetail: 'standard', - researchBreadth: 'focused', searchQuery: 'keynote presenter outfits', visualEvidence: 'required' })); + researchBreadth: 'focused', searchQuery: 'keynote presenter outfits', frameAccess: 'agent_selected', visualEvidence: 'required' })); expect(result.decision).toMatchObject({ route: 'topic_research', visualEvidence: 'required', visualRequirements: [message], useStoryboard: true, searchQuery: 'keynote presenter outfits' }); expect(result.lastResort!.defaultedFields).toContain('visualRequirements'); @@ -248,7 +248,7 @@ describe('classifier last resort', () => { it('keeps required visuals even when the decision falls back to request defaults', async () => { // A corrupted key blocks recovery; the candidate still said the answer needs images. const result = await classify('What did the presenter wear in https://youtu.be/dQw4w9WgXcQ?', - rawClassifier({ answerDetail: 'standard', 'route': 'inspect_video', visualEvidence: 'required', + rawClassifier({ answerDetail: 'standard', 'route': 'inspect_video', frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['presenter clothing'] })); expect(result.decision).toMatchObject({ route: 'inspect_video', videoId: 'dQw4w9WgXcQ', visualEvidence: 'required', visualRequirements: ['presenter clothing'] }); @@ -286,7 +286,7 @@ describe('classifier last resort', () => { // its route, has no refresh flag, and says no visuals are needed. const first = { route: 'finalize', answerDetail: 'standard', responseIntent: 'context_answer', contextScope: 'video', reason: 'Saved data answers this.', refreshDynamicData: true, visualEvidence: 'required', visualRequirements: ['thumbnail text'] }; - const repair = { answerDetail: 'standard', researchBreadth: 'focused', searchQuery: 'MrBeast current views', visualEvidence: 'none' }; + const repair = { answerDetail: 'standard', researchBreadth: 'focused', searchQuery: 'MrBeast current views', frameAccess: 'agent_selected', visualEvidence: 'none' }; const result = await classify('What do the current MrBeast thumbnails say, and how many views do they have now?', rawClassifier(first, repair)); expect(result.lastResort).toMatchObject({ lastResort: 'route_recovery' }); expect(result.decision).toMatchObject({ route: 'topic_research', searchQuery: 'MrBeast current views', refreshDynamicData: true, diff --git a/platform/test/session-evidence.integration.test.ts b/platform/test/session-evidence.integration.test.ts index 11706dd3..d0bf9736 100644 --- a/platform/test/session-evidence.integration.test.ts +++ b/platform/test/session-evidence.integration.test.ts @@ -1473,6 +1473,12 @@ test('single-sheet storyboard reuses the processor middle sheet across restored expect(warm.value.value.sheets).toEqual(value.sheets); expect(warm.value.sessionReused).toBe(true); expect(storyboard).toHaveBeenCalledOnce(); + // A sheet-only refresh must reach the provider even when the requested sheet is already saved. + const refresh = sessionProvider(p, reopen(), false, undefined, undefined, false, true); + await refresh.storyboard!(id, undefined, { maxSheets: 1 }); + await refresh.storyboard!(id, undefined, { maxSheets: 1 }); + expect(storyboard).toHaveBeenCalledTimes(2); + expect(storyboard).toHaveBeenLastCalledWith(id, undefined, expect.objectContaining({ refresh: true }), undefined); for (const { diagnostics } of [cold, warm]) { for (const span of diagnostics.spans) { const parent = diagnostics.spans.find(candidate => candidate.id === span.parentId); diff --git a/platform/test/youtube-agent-loop-control.test.ts b/platform/test/youtube-agent-loop-control.test.ts index ea7cb839..f53d2ff6 100644 --- a/platform/test/youtube-agent-loop-control.test.ts +++ b/platform/test/youtube-agent-loop-control.test.ts @@ -497,6 +497,25 @@ describe('YouTube AgentCore loop control', () => { decision: { route: 'topic_research', researchBreadth: 'comparative', searchQuery: 'model practical use cases' }, context }); }); + it('allows completion on the first model step when there is no video for a discovery capture', async () => { + const context = inspectContext(); + context.provider.frames = vi.fn(); + context.provider.search = vi.fn(async () => ({ cacheStatus: 'miss' as const, value: { + query: 'exercise', results: [], videos: [], channels: [], playlists: [], + meta: { source: 'allthingsyoutube' as const, fetchedAt: new Date().toISOString(), partial: false, warnings: [] }, + } })); + const model = new MockLanguageModelV4({ doGenerate: async () => modelResult({ toolCallId: 'finish', toolName: 'complete_research', input: '{}' }) }); + const finalizationModel = new MockLanguageModelV4({ doGenerate: async () => finalizerModelResult({ confidence: 'low', warnings: [], + blocks: [{ text: 'No matching video was found.', evidenceIds: [] }] }) }); + await runResearchAgentWithModel({ model, finalizationModel, context, message: 'Find an exercise video and capture 0:30.', + decision: { route: 'topic_research', searchQuery: 'exercise', frameAccess: [{ captureTimestampsMs: [30000], savedTimestampsMs: [] }] } }); + expect(context.provider.search).toHaveBeenCalledOnce(); + expect(model.doGenerateCalls).toHaveLength(1); + expect(model.doGenerateCalls[0]?.tools?.map(tool => tool.name)).toContain('complete_research'); + expect(model.doGenerateCalls[0]?.tools?.map(tool => tool.name)).not.toContain('get_video_frames'); + expect(context.provider.frames).not.toHaveBeenCalled(); + }); + it('does not retry a failed classified search in the model loop', async () => { const context = inspectContext(); context.provider.search = vi.fn(async () => { throw new Error('Search unavailable'); }); diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index 1d38c88a..cfd7cf9e 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -166,7 +166,7 @@ describe('YouTube agent capability router', () => { it.each([false, true])('recovers the captured fixed-video follow-up with missing discovery fields (answerDetail present: %s)', async hasDetail => { const decision = { route: 'topic_research', comparisonVideoIds: ['abcdefghijk', 'lmnopqrstuv'], - visualEvidence: 'required', visualRequirements: ['exercise form'] }; + frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['exercise form'] }; const model = sequenceClassifier([{ ...decision, answerDetail: hasDetail ? 'standard' : undefined }, { ...decision, answerDetail: 'standard' }]); const diagnostics = vi.fn(); @@ -181,7 +181,7 @@ describe('YouTube agent capability router', () => { }); it('gives repair the previous candidate and all missing fields without losing valid choices', async () => { - const candidate = { route: 'topic_research', answerDetail: 'detailed', visualEvidence: 'required', + const candidate = { route: 'topic_research', answerDetail: 'detailed', frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['exercise form'] }; let calls = 0; const model = new MockLanguageModelV4({ doGenerate: async ({ prompt }) => { @@ -610,7 +610,7 @@ describe('YouTube agent capability router', () => { it.each([0, 1])('accepts the captured DevDay classification despite obsolete conflicting count fields (%s)', async requiredVideoCount => { const model = classifierModel({ answerDetail: 'standard', researchBreadth: 'focused', researchVideoCount: 3, requiredVideoCount, route: 'topic_research', searchQuery: 'OpenAI DevDay 2026 presenters on stage keynote', - visualEvidence: 'required', visualRequirements: ['presenter clothing'] }); + frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['presenter clothing'] }); const decision = await classifyCapabilityWithModel({ message: 'What were the names of presenters from OpenAI DevDay 2026 and what were they wearing?', model, signal: new AbortController().signal }); expect(model.doGenerateCalls).toHaveLength(1); expect(decision).toMatchObject({ route: 'topic_research', researchVideoCount: 2, visualEvidence: 'required' }); @@ -623,7 +623,7 @@ describe('YouTube agent capability router', () => { it.each(['https://youtu.be/abcdefghijk', 'https://www.youtube.com/watch?v=abcdefghijk', 'https://youtube.com/shorts/abcdefghijk'])('pins a classifier research decision to the supplied video: %s', async url => { const model = classifierModel({ route: 'topic_research', researchBreadth: 'focused', searchQuery: 'keynote presenters', - visualEvidence: 'required', visualRequirements: ['presenter clothing'] }); + frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['presenter clothing'] }); const decision = await classifyCapabilityWithModel({ message: `Research who presented and what they wore: ${url}`, model, conversationHistory: [{ userMessageId: 'previous-user', agentMessageId: 'previous-agent', user: 'An earlier video', assistant: 'Prior research', resourceIds: ['lmnopqrstuv'] }], signal: new AbortController().signal }); @@ -763,12 +763,12 @@ describe('YouTube agent capability router', () => { ['none', undefined, false], ['helpful', undefined, true], ['required', ['presenter clothing'], true], ] as const)('persists visual evidence %s with derived tool access', async (visualEvidence, visualRequirements, useStoryboard) => { const decision = await classifyCapabilityWithModel({ message: 'Inspect https://youtu.be/abcdefghijk', - model: classifierModel({ route: 'inspect_video', videoId: 'abcdefghijk', visualEvidence, visualRequirements }), + model: classifierModel({ route: 'inspect_video', videoId: 'abcdefghijk', visualEvidence, visualRequirements, ...(visualRequirements ? { frameAccess: 'agent_selected' } : {}) }), signal: new AbortController().signal, }); const recovered = await resolveCapabilityRoute({ persisted: decision, classify: vi.fn(), persist: vi.fn() }); expect(recovered).toEqual({ route: 'inspect_video', videoId: 'abcdefghijk', researchVideoCount: 1, useStoryboard, visualEvidence, - ...(visualRequirements ? { visualRequirements } : {}), answerDetail: 'standard' }); + ...(visualRequirements ? { visualRequirements, frameAccess: 'agent_selected' } : {}), answerDetail: 'standard' }); }); it('requires visual requirements when visual evidence is required, and drops them otherwise', async () => { @@ -784,7 +784,7 @@ describe('YouTube agent capability router', () => { it('asks once to reconsider visual cues, then accepts the reconsidered decision', async () => { const outputs = [ { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'OpenAI DevDay keynote presenters', visualEvidence: 'none' }, - { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'OpenAI DevDay keynote presenters', visualEvidence: 'required', visualRequirements: ['presenter clothing'] }, + { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'OpenAI DevDay keynote presenters', frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['presenter clothing'] }, ]; const model = sequenceClassifier(outputs); const decision = await classifyCapabilityWithModel({ message: 'Who presented at the DevDay keynote and what were they wearing?', @@ -794,7 +794,7 @@ describe('YouTube agent capability router', () => { expect(repair).toContain('possible_visual_requirement'); expect(repair).toContain('Reconsider the previous classification'); expect(repair).toMatch(/The request mentions \W+wearing\W+\. If any requested fact depends on what is visible/); - expect(decision).toMatchObject({ visualEvidence: 'required', visualRequirements: ['presenter clothing'], useStoryboard: true }); + expect(decision).toMatchObject({ frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['presenter clothing'], useStoryboard: true }); }); it('keeps the classifier choice when it declines a visual cue, without a third call', async () => { @@ -810,7 +810,7 @@ describe('YouTube agent capability router', () => { const message = 'Summarize the slide design tips in popular talks'; const first = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'slide design tips', visualEvidence: 'helpful' }; const full = (output: Record): Record => ({ answerDetail: 'standard', ...output }); - const required = full({ ...first, visualEvidence: 'required', visualRequirements: ['slide layouts'] }); + const required = full({ ...first, frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['slide layouts'] }); // Missing route and search query, so route recovery cannot apply. const { route: _omitted, searchQuery: _query, ...withoutRoute } = first; const missing = full(withoutRoute); @@ -939,7 +939,7 @@ describe('YouTube agent capability router', () => { await classifyCapabilityWithModel({ message: 'Help me understand reservoir computing', model: plain, signal: new AbortController().signal }); expect(plain.doGenerateCalls).toHaveLength(1); const required = sequenceClassifier([{ route: 'topic_research', researchBreadth: 'focused', searchQuery: 'keynote outfits', - visualEvidence: 'required', visualRequirements: ['presenter clothing'] }]); + frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['presenter clothing'] }]); await classifyCapabilityWithModel({ message: 'What were the keynote presenters wearing?', model: required, signal: new AbortController().signal }); expect(required.doGenerateCalls).toHaveLength(1); }); @@ -1157,18 +1157,107 @@ describe('YouTube agent capability router', () => { expect(JSON.stringify(model.doGenerateCalls[1]!.prompt)).toContain('needs capture permission'); }); + it('repairs missing image access for a required visual question', async () => { + const videoId = 'abcdefghijk'; + const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['car color'] }; + const model = sequenceClassifier([base, { ...base, frameAccess: 'storyboard_only' }]); + const message = `What color is the car in https://youtu.be/${videoId}?`; + const decision = await classifyCapabilityWithModel({ model, message, signal: new AbortController().signal }); + expect(decision).toMatchObject({ frameAccess: 'storyboard_only' }); + expect(model.doGenerateCalls).toHaveLength(2); + }); + + it.each([ + ['Capture only the opening frame', 0], ['Capture the frame at 1:30s', 90000], + ] as const)('binds "%s" to its actual timestamp', async (message, requested) => { + const videoId = 'abcdefghijk'; + const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['frame'] }; + const at = (time: number) => [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [time] }]; + const model = sequenceClassifier([{ ...base, frameAccess: 'agent_selected' }, { ...base, frameAccess: at(30000) }]); + const fallbackModel = classifierModel({ ...base, frameAccess: at(requested) }); + const decision = await classifyCapabilityWithModel({ model, fallbackModel, message: `${message} of https://youtu.be/${videoId}.`, signal: new AbortController().signal }); + expect(fallbackModel.doGenerateCalls).toHaveLength(1); + expect(decision).toMatchObject({ frameAccess: at(requested) }); + }); + + it('asks for an exact bound before capturing the first few seconds', async () => { + const videoId = 'abcdefghijk'; + const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'] }; + const model = sequenceClassifier([{ ...base, frameAccess: 'agent_selected' }, + { ...base, frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [120000] }] }]); + expect(await classifyCapabilityWithModel({ model, message: `Sample only the first few seconds of https://youtu.be/${videoId}`, + signal: new AbortController().signal })).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it.each(['Your previous answer described 0:30.', 'Find a 2 hour lecture and describe the cover.', + 'Find 5 hours of content about exercise.'])('does not authorize or require a position from "%s"', message => { + expect(readFrameRequest(message).positions).toEqual([]); + }); + + it('pins a discovery capture to the linked inspection subject', async () => { + const videoId = 'abcdefghijk'; + const model = classifierModel({ route: 'topic_research', researchBreadth: 'focused', searchQuery: 'exercise', + visualEvidence: 'required', visualRequirements: ['movement'], frameAccess: [{ captureTimestampsMs: [30000], savedTimestampsMs: [] }] }); + const message = `Capture only 0:30 of https://youtu.be/${videoId}.`; + const decision = await classifyCapabilityWithModel({ model, message, signal: new AbortController().signal }); + expect(decision).toMatchObject({ route: 'inspect_video', videoId, frameAccess: [{ videoId, captureTimestampsMs: [30000] }] }); + expect(await resolveCapabilityRoute({ message, persisted: { ...decision, frameAccess: [{ captureTimestampsMs: [30000], savedTimestampsMs: [] }] }, + classify: vi.fn(), persist: vi.fn() })).toMatchObject({ responseIntent: 'clarification' }); + }); + + it('assigns comparison sheet times joined by "with" to their own videos', () => { + const a = 'abcdefghijk', b = 'lmnopqrstuv'; + const message = `Compare the storyboard sheet at 0:30 of https://youtu.be/${a} with the one at 1:00 of https://youtu.be/${b}.`; + expect(readFrameRequest(message).positions.map(pos => [pos.time, pos.videoIds])).toEqual([[30000, [a]], [60000, [b]]]); + const common = `Compare the storyboard sheet at 0:30 in both https://youtu.be/${a} and https://youtu.be/${b}.`; + expect(readFrameRequest(common).positions.map(pos => [pos.time, pos.videoIds])).toEqual([[30000, [a, b]]]); + }); + + it('keeps paired storyboard positions on comparison routes, including resume', async () => { + const a = 'abcdefghijk', b = 'lmnopqrstuv'; + const model = classifierModel({ route: 'topic_research', comparisonVideoIds: [a, b], frameAccess: 'storyboard_only', + visualEvidence: 'required', visualRequirements: ['compare the scenes'] }); + const message = `Compare the storyboard sheets at 0:30 of https://youtu.be/${a} and 1:00 of https://youtu.be/${b}.`; + const decision = await classifyCapabilityWithModel({ message, model, signal: new AbortController().signal }); + const storyboardSelections = [{ videoId: a, timestampsMs: [30000] }, { videoId: b, timestampsMs: [60000] }]; + expect(decision).toMatchObject({ route: 'topic_research', frameAccess: { storyboardSelections } }); + const resume = (selections: typeof storyboardSelections) => resolveCapabilityRoute({ message, classify: vi.fn(), persist: vi.fn(), + persisted: { ...decision, frameAccess: { storyboardSelections: selections } } }); + await expect(resume(storyboardSelections)).resolves.toEqual(decision); + await expect(resume(storyboardSelections.map(item => ({ ...item, timestampsMs: [30000, 60000] })))).resolves.toMatchObject({ responseIntent: 'clarification' }); + if (decision.route !== 'topic_research') throw new Error('Expected comparison.'); + const context = inspectContext(); + const storyboard = vi.fn>(async (videoId, times) => ({ + cacheStatus: 'miss', value: { videoId, frameCount: 20, intervalMs: 5000, + sheets: [{ imageBase64: '/9j/2Q==', tileWidth: 100, tileHeight: 100, columns: 2, rows: 1, firstFrameIndex: times![0]! / 5000, + frameCount: 2, intervalMs: 5000 }], meta: { partial: false, warnings: [] } }, + })); + context.provider.storyboard = storyboard; + const research = finalizingModel([ + { name: 'get_video_storyboard', input: { videoId: a, timestampsMs: [60000] } }, + ...storyboardSelections.map(input => ({ name: 'get_video_storyboard', input })), + ]); + await runResearchAgentWithModel({ message, decision, context, model: research }); + expect(storyboard.mock.calls.map(([videoId, timestampsMs]) => ({ videoId, timestampsMs }))).toEqual(storyboardSelections); + expect(research.doGenerateCalls[0]?.tools?.map(tool => tool.name)).not.toContain('get_video_frames'); + }); + + it('repairs an unrestricted storyboard refresh without granting exact capture', async () => { + const videoId = 'abcdefghijk'; + const route = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['overview'] }; + const model = sequenceClassifier([{ ...route, frameAccess: 'agent_selected' }, { ...route, frameAccess: 'storyboard_only' }]); + const message = `Refresh only the storyboard for https://youtu.be/${videoId}.`; + const decision = await classifyCapabilityWithModel({ model, message, signal: new AbortController().signal }); + expect(model.doGenerateCalls).toHaveLength(2); + expect(decision).toMatchObject({ frameAccess: 'storyboard_only', refreshFramesOnly: true }); + }); + it('leaves capture closed when the classifier grants nothing for an exact capture request', async () => { const videoId = 'abcdefghijk'; - // The classifier omitted every frame field, so no attempt is retried and nothing may be captured. + // Even repeated omissions cannot open either retrieval method. const route = await classifyCapabilityWithModel({ model: sequenceClassifier([{ route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['the 1:00 still'] }]), message: `Capture only 1:00 of https://youtu.be/${videoId}. Just the still, no analysis.`, signal: new AbortController().signal }); - expect(route).not.toHaveProperty('frameAccess'); - if (route.route !== 'inspect_video') throw new Error('Expected inspection.'); - const model = finalizingModel(); - await runResearchAgentWithModel({ model, message: 'Capture only 1:00.', decision: route, context: inspectContext() }); - const tools = model.doGenerateCalls[0]?.tools?.map(tool => tool.name); - expect(tools).toContain('analyze_video_frames'); - expect(tools).not.toContain('get_video_frames'); + expect(route).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); }); it.each([ @@ -1178,6 +1267,10 @@ describe('YouTube agent capability router', () => { const videoId = 'abcdefghijk'; const decision = await classifyCapabilityWithModel({ message, signal: new AbortController().signal, model: classifierModel({ route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'], frameAccess }) }); + if (!frameAccess) { + expect(decision).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + return; + } if (decision.route !== 'inspect_video') throw new Error('Expected inspection.'); const context = inspectContext(); const storyboard = vi.fn(async () => { throw new Error('Storyboard unavailable'); }); diff --git a/platform/test/youtube-agent-storyboard.test.ts b/platform/test/youtube-agent-storyboard.test.ts index 5ad9c768..ec712788 100644 --- a/platform/test/youtube-agent-storyboard.test.ts +++ b/platform/test/youtube-agent-storyboard.test.ts @@ -51,13 +51,21 @@ async function retrieveAndAnalyze(input: Parameters { + it('refuses a timestamp permitted only for the other comparison video before lookup', () => { + const ctx = context(); + ctx.storyboardSelections = [{ videoId: storyboard.videoId, timestampsMs: [10000] }, { videoId: 'abcdefghijk', timestampsMs: [55000] }]; + ctx.executeEvidenceTool = vi.fn(); + expect(() => executeGetVideoStoryboard({ videoId: storyboard.videoId, timestampsMs: [55000] }, ctx, 'swapped')).toThrow('outside the grant'); + expect(ctx.executeEvidenceTool).not.toHaveBeenCalled(); + }); + it.each([ { videoId: storyboard.videoId }, { videoId: storyboard.videoId, maxSheets: 3 }, { videoId: storyboard.videoId, sheetIndexes: [0] }, { videoId: storyboard.videoId, timestampsMs: [55000, 50000] }, { videoId: 'abcdefghijk', timestampsMs: [55000] }, ])('rejects out-of-scope sheet requests before evidence lookup or billing: %j', input => { const ctx = context(); - ctx.storyboardSelection = { videoId: storyboard.videoId, timestampsMs: [55000] }; + ctx.storyboardSelections = [{ videoId: storyboard.videoId, timestampsMs: [55000] }]; ctx.executeEvidenceTool = vi.fn(); expect(() => executeGetVideoStoryboard(input, ctx, 'denied')).toThrow(); expect(ctx.executeEvidenceTool).not.toHaveBeenCalled(); diff --git a/platform/test/youtube-stored-analysis.test.ts b/platform/test/youtube-stored-analysis.test.ts index 6e58f8b5..3ed63a58 100644 --- a/platform/test/youtube-stored-analysis.test.ts +++ b/platform/test/youtube-stored-analysis.test.ts @@ -356,10 +356,10 @@ describe('application notes on frame turns', () => { context.frameAccess = [{ videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; provider.frames.mockResolvedValue({ value: at(other, 60000), cacheStatus: 'miss' }); const current = await executeGetVideoFrames({ videoId: other, timestampsMs: [60000], analysis: 'What is he wearing?' }, context, 'capture'); - const answer = (text: string, packets: EvidencePacket[]) => buildAgentTurnResult({ runId: context.runId, conversationId: crypto.randomUUID(), + const answer = (text: string, packets: EvidencePacket[], requestedSaved = false) => buildAgentTurnResult({ runId: context.runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }, { userId: 'user', creditsRemaining: 100 }, { answer: text, intent: 'topic_research', confidence: 'medium', citations: [], artifacts: [], warnings: [] }, - [earlier, ...packets], 0, undefined, { access: context.frameAccess, inventory: inventoryOf(context), startedAt }).answer; + [earlier, ...packets], 0, undefined, { requestedSaved, access: context.frameAccess, inventory: inventoryOf(context), startedAt }).answer; /** Moves this turn's start after saves made by a test's own earlier turn. */ const restart = () => { startedAt = store.runStart(); }; return { context, store, at, savedVersion, earlier, current, answer, restart }; @@ -367,6 +367,25 @@ describe('application notes on frame turns', () => { const earlierNote = /The 0:30 observation( of "[^"]+")? comes from an image inspected in an earlier turn, not in this one\./; const claimNote = 'No frame saved before this turn was re-inspected in this turn'; + it('labels an earlier frame citation on a history answer with no frame route or inventory', async () => { + const { context, earlier } = await turns(); + const result = buildAgentTurnResult({ runId: context.runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }, + { userId: 'user', creditsRemaining: 100 }, { answer: `I re-inspected it [cite:${earlier.excerpts[0]!.id}].`, + intent: 'context_answer', confidence: 'medium', citations: [], artifacts: [], warnings: [] }, [earlier], 0); + expect(result.answer).toMatch(earlierNote); + }); + + it('summarizes several saved versions inspected in an open capture', async () => { + const { context, store, at, current, answer, restart } = await turns(); + context.frameAccess = undefined; + const versions = [30000, 40000, 50000].map(time => store.put('frame', videoId, at(videoId, time, 480), { timestampMs: time })); + restart(); + const analysis = await executeAnalyzeVideoFrames({ assetVersions: versions, focus: 'Describe the shirt' }, context, 'batch'); + const text = answer(`A shirt [cite:${current.excerpts[0]!.id}].`, [analysis, current]); + expect(text).toMatch(/I re-inspected \d+ saved frames in this turn/); + expect(text.match(/I re-inspected/g)).toHaveLength(1); + }); + it('states that a cited observation came from an earlier turn', async () => { const { earlier, current, answer } = await turns(); const text = answer(`The saved frame shows a shirt [cite:${earlier.excerpts[0]!.id}]. The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current]); @@ -379,7 +398,7 @@ describe('application notes on frame turns', () => { 'Both frames show shirts.', ])('states that no saved frame was re-inspected, whatever the answer says: %s', async claim => { const { current, answer } = await turns(); - const text = answer(`${claim} The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current]); + const text = answer(`${claim} The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current], true); expect(text).toContain(claimNote); expect(text).toContain('I retrieved the 1:00 frame and inspected it.'); }); @@ -411,11 +430,11 @@ describe('application notes on frame turns', () => { expect(text.includes('It is not recorded whether the cited 0:30 observation')).toBe(uncertain); }); - it('names saved frames whose position cannot be recovered instead of skipping them silently', async () => { + it('omits unrelated saved inventory from a capture-only answer', async () => { const { current, answer } = await turns(put => put(other, 30000)); const text = answer(`The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current]); - expect(text).toContain(`Saved frames of "${other}" have no recorded position, so this turn could not match them to requested times or re-inspect them.`); - expect(text).toContain(`${claimNote}, because its plan included none.`); + expect(text).not.toContain('no recorded position'); + expect(text).not.toContain(claimNote); }); it('judges each citation by its own image when an earlier batch held several', async () => {