From ef33ddf82a5f749b54d7f90518103ab4ae541452 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Mon, 9 Nov 2026 15:05:19 +0530 Subject: [PATCH 1/2] enforce visual retrieval permissions from complete timestamps and report inspected frames --- docs/api-reference/openapi.json | 500 ++++++++++++---- platform/src/agents/agent-runtime-do.ts | 9 +- platform/src/agents/contracts.ts | 26 +- platform/src/agents/finalizer.ts | 13 +- .../agents/providers/youtube/tool-context.ts | 1 + .../youtube/tools/get-video-frames.ts | 18 +- .../youtube/tools/get-video-storyboard.ts | 20 +- .../youtube/tools/stored-analysis.ts | 5 +- .../src/agents/research/capability-router.ts | 166 +++--- platform/src/agents/research/frame-request.ts | 209 +++++++ .../src/agents/research/research-agent.ts | 130 ++-- .../skills/visual-inspection.generated.ts | 2 +- .../skills/visual-inspection/SKILL.md | 14 +- .../agents/research/timestamp-expression.ts | 65 ++ .../src/agents/research/youtube-references.ts | 51 ++ platform/src/agents/runtime/frame-access.ts | 165 +++++- .../src/agents/runtime/session-evidence.ts | 9 +- .../src/agents/runtime/session-provider.ts | 7 +- .../test/classifier-route-recovery.test.ts | 14 +- platform/test/fixtures/analysis-session.ts | 7 +- .../test/frame-timestamp-expression.test.ts | 53 ++ .../test/session-evidence.integration.test.ts | 14 + .../youtube-agent-analysis-context.test.ts | 6 +- .../youtube-agent-frame-deadlines.test.ts | 11 +- .../test/youtube-agent-loop-control.test.ts | 19 + platform/test/youtube-agent-router.test.ts | 559 ++++++++++++++++-- .../test/youtube-agent-storyboard.test.ts | 21 + platform/test/youtube-stored-analysis.test.ts | 353 ++++++++++- 28 files changed, 2122 insertions(+), 345 deletions(-) create mode 100644 platform/src/agents/research/frame-request.ts create mode 100644 platform/src/agents/research/timestamp-expression.ts create mode 100644 platform/src/agents/research/youtube-references.ts create mode 100644 platform/test/frame-timestamp-expression.test.ts diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json index 8cc88d27..166aa55a 100644 --- a/docs/api-reference/openapi.json +++ b/docs/api-reference/openapi.json @@ -7661,38 +7661,109 @@ } }, "frameAccess": { - "type": "object", - "properties": { - "videoId": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]{11}$" - }, - "savedTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "captureTimestampsMs": { + "anyOf": [ + { "minItems": 1, - "maxItems": 24, + "maxItems": 8, "type": "array", "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "savedTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "captureTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "savedTimestampsMs", + "captureTimestampsMs" + ], + "additionalProperties": false } + }, + { + "type": "string", + "enum": [ + "storyboard_only", + "agent_selected" + ] + }, + { + "type": "object", + "properties": { + "storyboardTimestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "storyboardTimestampsMs" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "storyboardSelections": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "timestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "videoId", + "timestampsMs" + ], + "additionalProperties": false + } + } + }, + "required": [ + "storyboardSelections" + ], + "additionalProperties": false } - }, - "required": [ - "videoId", - "savedTimestampsMs", - "captureTimestampsMs" - ], - "additionalProperties": false + ] }, "refreshFramesOnly": { "type": "boolean" @@ -7792,38 +7863,109 @@ } }, "frameAccess": { - "type": "object", - "properties": { - "videoId": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]{11}$" - }, - "savedTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "captureTimestampsMs": { + "anyOf": [ + { "minItems": 1, - "maxItems": 24, + "maxItems": 8, "type": "array", "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "savedTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "captureTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "savedTimestampsMs", + "captureTimestampsMs" + ], + "additionalProperties": false } + }, + { + "type": "string", + "enum": [ + "storyboard_only", + "agent_selected" + ] + }, + { + "type": "object", + "properties": { + "storyboardTimestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "storyboardTimestampsMs" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "storyboardSelections": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "timestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "videoId", + "timestampsMs" + ], + "additionalProperties": false + } + } + }, + "required": [ + "storyboardSelections" + ], + "additionalProperties": false } - }, - "required": [ - "videoId", - "savedTimestampsMs", - "captureTimestampsMs" - ], - "additionalProperties": false + ] }, "refreshFramesOnly": { "type": "boolean" @@ -9521,38 +9663,109 @@ } }, "frameAccess": { - "type": "object", - "properties": { - "videoId": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]{11}$" - }, - "savedTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "captureTimestampsMs": { + "anyOf": [ + { "minItems": 1, - "maxItems": 24, + "maxItems": 8, "type": "array", "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "savedTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "captureTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "savedTimestampsMs", + "captureTimestampsMs" + ], + "additionalProperties": false } + }, + { + "type": "string", + "enum": [ + "storyboard_only", + "agent_selected" + ] + }, + { + "type": "object", + "properties": { + "storyboardTimestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "storyboardTimestampsMs" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "storyboardSelections": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "timestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "videoId", + "timestampsMs" + ], + "additionalProperties": false + } + } + }, + "required": [ + "storyboardSelections" + ], + "additionalProperties": false } - }, - "required": [ - "videoId", - "savedTimestampsMs", - "captureTimestampsMs" - ], - "additionalProperties": false + ] }, "refreshFramesOnly": { "type": "boolean" @@ -9652,38 +9865,109 @@ } }, "frameAccess": { - "type": "object", - "properties": { - "videoId": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]{11}$" - }, - "savedTimestampsMs": { - "maxItems": 24, - "type": "array", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - } - }, - "captureTimestampsMs": { + "anyOf": [ + { "minItems": 1, - "maxItems": 24, + "maxItems": 8, "type": "array", "items": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "savedTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "captureTimestampsMs": { + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "savedTimestampsMs", + "captureTimestampsMs" + ], + "additionalProperties": false } + }, + { + "type": "string", + "enum": [ + "storyboard_only", + "agent_selected" + ] + }, + { + "type": "object", + "properties": { + "storyboardTimestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "storyboardTimestampsMs" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "storyboardSelections": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "videoId": { + "type": "string", + "pattern": "^[A-Za-z0-9_-]{11}$" + }, + "timestampsMs": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "required": [ + "videoId", + "timestampsMs" + ], + "additionalProperties": false + } + } + }, + "required": [ + "storyboardSelections" + ], + "additionalProperties": false } - }, - "required": [ - "videoId", - "savedTimestampsMs", - "captureTimestampsMs" - ], - "additionalProperties": false + ] }, "refreshFramesOnly": { "type": "boolean" diff --git a/platform/src/agents/agent-runtime-do.ts b/platform/src/agents/agent-runtime-do.ts index 9e7bac7b..bc69f9a6 100644 --- a/platform/src/agents/agent-runtime-do.ts +++ b/platform/src/agents/agent-runtime-do.ts @@ -14,6 +14,8 @@ import { storedExtractionDiagnosticSchema, type StoredExtractionDiagnostic } fro import { transcriptDiagnosticSchema, type TranscriptDiagnostic } from './runtime/transcript-diagnostics'; import { agentDraftSchema, agentRunProgressSchema, toolTrace, type AgentDraft } from './runtime/run-progress'; import { framePreviewKey, recordedFramePreviews, saveFramePreviews, type SavedFrame } from './runtime/frame-previews'; +import { readFrameRequest } from './research/frame-request'; +import { frameList, isFrameTurn } from './runtime/frame-access'; import { saveStoryboardPreviews } from './runtime/storyboard-previews'; import { compactAgentRun, compactAgentResult } from './response'; import { queuedRunIdentitySchema, type QueuedRunIdentity } from './runtime/admission-queue'; @@ -708,6 +710,7 @@ export class AgentRuntimeDO extends Agent { }, onExtractionDiagnostic: event => this.recordExtractionDiagnostic(runId, event), onTranscriptDiagnostic: event => this.recordTranscriptDiagnostic(runId, event), + startedAt: row.created_at, persistedRoute: this.readRoute(runId), persistRoute: (selected) => { this.persistRoute(runId, selected); @@ -1038,6 +1041,8 @@ export class AgentRuntimeDO extends Agent { const history = this.readConversationHistory(run); const citedIds=answerCitationIds(parsedInput.answer); const citedSessionEvidence=this.sessionStore.evidenceForCitations(citedIds); + const requestedSaved = readFrameRequest(run.execution_message ?? run.message).requestsSaved; + const needsFrameInventory = requestedSaved || isFrameTurn(decision) || citedSessionEvidence.some(packet => packet.kind === 'youtube_frames'); const result = buildAgentTurnResult({ runId, conversationId: run.conversation_id, @@ -1050,7 +1055,9 @@ export class AgentRuntimeDO extends Agent { requestedVideoIds: decision.route === 'inspect_video' ? [decision.videoId] : decision.comparisonVideoIds ?? [], // An application fallback is checked against what this run had, not earlier turns' cited content. runPackets: this.admittedRunEvidence(runId, history), - } : undefined); + } : undefined, needsFrameInventory ? { requestedSaved, access: frameList(decision), startedAt: run.created_at, + inventory: this.sessionStore.brief().assets.flatMap(asset => asset.kind === 'frame' ? [{ version: asset.version, videoId: asset.videoId, + collectedAt: asset.collectedAt, ...(typeof asset.details.timestampMs === 'number' ? { timestampMs: asset.details.timestampMs } : {}) }] : []) } : undefined); // Hash the accepted text: unmatched citations may have been removed from the input. const answerHash = createHash('sha256').update(result.answer).digest('hex'); const serialized = JSON.stringify(result); diff --git a/platform/src/agents/contracts.ts b/platform/src/agents/contracts.ts index fc752798..91d0312c 100644 --- a/platform/src/agents/contracts.ts +++ b/platform/src/agents/contracts.ts @@ -14,13 +14,27 @@ export type VisualEvidence = z.infer; export const visualRequirementsSchema = z.array(z.string().trim().min(1).max(200)).min(1).max(8); const frameTimes = z.array(z.number().int().nonnegative()).max(24); -/** A fixed capture list stays scoped to one video, including during route recovery. */ -export const frameAccessSchema = z.object({ - videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), +const frameAccessEntrySchema = z.object({ + /** Absent: capture times for whichever videos discovery selects. */ + videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/).optional(), savedTimestampsMs: frameTimes, - captureTimestampsMs: frameTimes.min(1), + captureTimestampsMs: frameTimes, }); +/** Exact saved and new frame positions for each video in the turn. Model-facing, so no transforms. */ +export const frameAccessListSchema = z.array(frameAccessEntrySchema).min(1).max(8); +/** A fixed capture list stays scoped to its videos, including during route recovery. + * Routes persisted before multi-video access stored one entry as a bare object. */ +export const frameAccessSchema = z.preprocess( + value => value && typeof value === 'object' && !Array.isArray(value) ? [value] : value, + frameAccessListSchema, +); export type FrameAccess = z.infer; +/** Model choices for visual retrieval without a per-video exact-frame list. */ +export const unscopedImageAccessSchema = z.enum(['storyboard_only', 'agent_selected']); +export const routeFrameAccessSchema = z.union([frameAccessSchema, unscopedImageAccessSchema, + z.object({ storyboardTimestampsMs: frameTimes.min(1) }), + z.object({ storyboardSelections: z.array(z.object({ videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), timestampsMs: frameTimes.min(1) })).min(1).max(8) })]); +export type RouteFrameAccess = z.infer; /** Evidence the user explicitly asked to be read before answering. Research cannot complete until each is read or unavailable. */ export const requestedEvidenceSchema = z.array(z.enum(['original_quotes', 'comments'])).min(1).max(2); @@ -42,7 +56,7 @@ export const capabilityRouteDecisionSchema = z.discriminatedUnion('route', [ savedOnly: z.boolean().optional(), skill: z.enum(['verify_saved_evidence']).optional(), requestedEvidence: requestedEvidenceSchema.optional(), - frameAccess: frameAccessSchema.optional(), + frameAccess: routeFrameAccessSchema.optional(), refreshFramesOnly: z.boolean().optional(), refreshEvidence: z.boolean().optional(), refreshDynamicData: z.boolean().optional(), @@ -60,7 +74,7 @@ export const capabilityRouteDecisionSchema = z.discriminatedUnion('route', [ savedOnly: z.boolean().optional(), skill: z.enum(['verify_saved_evidence']).optional(), requestedEvidence: requestedEvidenceSchema.optional(), - frameAccess: frameAccessSchema.optional(), + frameAccess: routeFrameAccessSchema.optional(), refreshFramesOnly: z.boolean().optional(), timestampedFrameQuestion: z.string().trim().min(1).max(500).optional(), refreshEvidence: z.boolean().optional(), diff --git a/platform/src/agents/finalizer.ts b/platform/src/agents/finalizer.ts index c778af4b..cfdf4e8d 100644 --- a/platform/src/agents/finalizer.ts +++ b/platform/src/agents/finalizer.ts @@ -8,7 +8,9 @@ import { type AgentTurnResult, type EvidencePacket, type FinalizeAnswerInput, + type FrameAccess, } from './contracts'; +import { frameTurnNotes, type InventoryFrame } from './runtime/frame-access'; import { appendNotices, playlistCoverageDisclosures } from './playlist-coverage'; import { CITED_FRAME_PREVIEWS, citedPacketFramePreviews } from './runtime/frame-previews'; @@ -28,6 +30,8 @@ export function buildAgentTurnResult( packets: EvidencePacket[], creditsCharged: number, durationLimitContext?: DurationLimitAnswerContext, + /** Present when the route asked for frame work: its exact access, and the session's saved frames. */ + frameTurn?: { requestedSaved?: boolean; access?: FrameAccess; inventory: readonly InventoryFrame[]; startedAt: number }, ): AgentTurnResult { const markers = answerCitationIds(input.answer); const citations: AgentCitation[] = []; @@ -35,6 +39,7 @@ export function buildAgentTurnResult( // Which saved previews each cited frame excerpt used, from its own packet. A later answer can // hold other captures of the same moment, so time alone never identifies the pixels. const citedFrames: Array<{ citationId: string; frames: ReturnType }> = []; + const citedObservations: Parameters[0]['cited'][number][] = []; for (const marker of markers) { const matches = packets.flatMap((packet) => packet.excerpts .filter((excerpt) => excerpt.id === marker) @@ -57,6 +62,7 @@ export function buildAgentTurnResult( ? citedPacketFramePreviews(packet, excerpt).slice(0, 6) : []; // Record a frame citation even without a preview, so it never falls back to matching by time. if (source.kind === 'frames') citedFrames.push({ citationId: excerpt.id, frames }); + if (source.kind === 'frames' && excerpt.startMs !== undefined) citedObservations.push({ packet, startMs: excerpt.startMs, videoId: source.videoId, title: source.title }); citations.push({ id: excerpt.id, sourceId: source.id, provider: source.provider, videoId: source.videoId, channelId: source.channelId, playlistId: source.playlistId, @@ -75,7 +81,12 @@ export function buildAgentTurnResult( // The application states playlist totals, the listing limit and reviewed content itself. const playlistCoverage = playlistCoverageDisclosures(packets, citations.map(citation => citation.id)); - const disclosedAnswer = appendNotices(answer, playlistCoverage.notes); + const notes = [...playlistCoverage.notes]; + // The application, not the model, states which frames this turn actually inspected. + if (citedObservations.length || (frameTurn && input.intent !== 'clarification' && input.intent !== 'rejected')) { + notes.push(...frameTurnNotes({ runId: identity.runId, startedAt: frameTurn?.startedAt ?? 0, requestedSaved: frameTurn?.requestedSaved, access: frameTurn?.access, inventory: frameTurn?.inventory ?? [], packets, cited: citedObservations })); + } + const disclosedAnswer = appendNotices(answer, notes); const warnings = answerWarnings(packets, [ ...input.warnings, diff --git a/platform/src/agents/providers/youtube/tool-context.ts b/platform/src/agents/providers/youtube/tool-context.ts index d528d028..97cb095e 100644 --- a/platform/src/agents/providers/youtube/tool-context.ts +++ b/platform/src/agents/providers/youtube/tool-context.ts @@ -58,6 +58,7 @@ export interface AgentToolContext { refreshEvidence?: boolean; refreshFramesOnly?: boolean; frameAccess?: FrameAccess; + storyboardSelections?: { videoId: string; timestampsMs: number[] }[]; timestampedFrameQuestion?: string; session?: import('../../runtime/session-evidence').SessionAccess; assetCache?: RunAssetCache; diff --git a/platform/src/agents/providers/youtube/tools/get-video-frames.ts b/platform/src/agents/providers/youtube/tools/get-video-frames.ts index a427ce9f..fd9c1d05 100644 --- a/platform/src/agents/providers/youtube/tools/get-video-frames.ts +++ b/platform/src/agents/providers/youtube/tools/get-video-frames.ts @@ -26,13 +26,14 @@ export const getVideoFramesInputSchema = frameRequestSchema.extend({ export function createGetVideoFramesTool(context: AgentToolContext) { const schema = getVideoFramesInputSchema.omit({ focus: true }); - const scope = context.frameAccess; + const captures = context.frameAccess?.filter(entry => entry.captureTimestampsMs.length); + const times = captures?.length ? z.array(z.literal([...new Set(captures.flatMap(entry => entry.captureTimestampsMs))])).min(1).max(6) : undefined; + const videos = captures?.every(entry => entry.videoId) ? captures.map(entry => entry.videoId!) : undefined; return tool({ description: 'Retrieve or reuse up to six video frames at millisecond timestamps. Supply analysis with your question to analyze immediately while images are saved. Omit analysis for retrieval only. Returns saved assetVersions, dimensions and failures, without image bytes. Use analyze_video_frames for questions about versions already saved. If suitable frames already exist in session inventory, analyze them directly. Still images cannot establish motion or speech.', - inputSchema: scope ? schema.extend({ - videoId: z.literal(scope.videoId), - timestampsMs: z.array(z.literal(scope.captureTimestampsMs)).min(1).max(6), - }) : schema, + // Per-video pairing is enforced at execution; the schema narrows what the model can propose. + // An entry without a video fixes only the times, for whichever video discovery selects. + inputSchema: !times ? schema : videos ? schema.extend({ videoId: z.literal(videos), timestampsMs: times }) : schema.extend({ timestampsMs: times }), outputSchema: evidencePacketSchema, // Previews are dashboard images with random IDs; the model analyzes assetVersions instead. toModelOutput: ({ output }) => ({ type: 'text', value: JSON.stringify({ @@ -48,8 +49,11 @@ export function createGetVideoFramesTool(context: AgentToolContext) { export function executeGetVideoFrames(input: z.input, context: AgentToolContext, toolCallId: string) { const parsed = getVideoFramesInputSchema.parse(input); const scope = context.frameAccess; - if (scope && (parsed.videoId !== scope.videoId || parsed.timestampsMs.some(time => !scope.captureTimestampsMs.includes(time)))) { - throw new Error(`Capture is limited to ${scope.captureTimestampsMs.join(', ')} ms on ${scope.videoId}. Use saved analysis for saved positions. If it fails, report the gap.`); + const allowed = (scope?.find(entry => entry.videoId === parsed.videoId) ?? scope?.find(entry => !entry.videoId))?.captureTimestampsMs ?? []; + if (scope && parsed.timestampsMs.some(time => !allowed.includes(time))) { + const permitted = scope.filter(entry => entry.captureTimestampsMs.length) + .map(entry => `${entry.captureTimestampsMs.join(', ')} ms on ${entry.videoId ?? 'videos found by discovery'}`).join('; '); + throw new Error(`Capture is limited to ${permitted}. Use saved analysis for saved positions. If it fails, report the gap.`); } if (!parsed.analysis && parsed.videoId === context.pinnedVideoId && (!parsed.selections || parsed.selections.every(selection => selection.basis === 'user_requested'))) { diff --git a/platform/src/agents/providers/youtube/tools/get-video-storyboard.ts b/platform/src/agents/providers/youtube/tools/get-video-storyboard.ts index 4389a85c..6c3953f9 100644 --- a/platform/src/agents/providers/youtube/tools/get-video-storyboard.ts +++ b/platform/src/agents/providers/youtube/tools/get-video-storyboard.ts @@ -36,10 +36,19 @@ function validateInput(input: z.infer, ctx: z.RefinementC if (input.sheetIndexes && input.timestampsMs) ctx.addIssue({ code: 'custom', message: 'Choose sheetIndexes or timestampsMs, not both.' }); } export const getVideoStoryboardInputSchema = retrievalInput.superRefine(validateInput); +// Apply the same selection before advertising the tool and before any evidence lookup or billing. +function storyboardInput(context: AgentToolContext) { + const selections = context.storyboardSelections; + return selections ? retrievalInput.extend({ + videoId: z.literal(selections.map(item => item.videoId)), + timestampsMs: z.array(z.literal([...new Set(selections.flatMap(item => item.timestampsMs))])).min(1).max(MAX_STORYBOARD_SHEETS), + sheetIndexes: z.never().optional(), + }) : retrievalInput; +} export function createGetVideoStoryboardTool(context: AgentToolContext) { return tool({ description: 'Retrieve storyboard images and metadata. Supply analysis with a visual question to analyze immediately while saving; omit it for retrieval only. With videoId alone, retrieve up to 3 sheets spread across the video. Use maxSheets, sheetIndexes or timestampsMs to select other coverage. No preliminary metadata call is needed. maxSheets gives a spread overview; use transcript findings to choose timestamps for targeted inspection. Returns saved assetVersions and sampled coverage, without image bytes. Use analyze_video_storyboard for questions about versions already in session inventory. Up to 20 sheets and 8 MiB per selection. Metadata alone is not visual evidence.', - inputSchema: retrievalInput.omit({focus:true}).superRefine(validateInput), + inputSchema: storyboardInput(context).omit({focus:true}).superRefine(validateInput), outputSchema: evidencePacketSchema, // Previews are dashboard images with random IDs. Hidden from the model, which otherwise // mistakes their per-sheet assetId for the analysisAssetVersions analysis needs. @@ -53,7 +62,11 @@ export function createGetVideoStoryboardTool(context: AgentToolContext) { }); } export function executeGetVideoStoryboard(input: z.infer, context: AgentToolContext, toolCallId: string) { - const requested = getVideoStoryboardInputSchema.parse(input); + const requested = storyboardInput(context).superRefine(validateInput).parse(input); + if (context.storyboardSelections) { + const allowed = context.storyboardSelections.find(item => item.videoId === requested.videoId)?.timestampsMs; + if (!allowed || !requested.timestampsMs?.every(time => allowed.includes(time))) throw new Error('Storyboard position is outside the grant for this video.'); + } // Only the old metadata-only shape needs a new key. Preserve existing keys // for explicit selections, and use schema order for the default overview. const parsed = requested.maxSheets === undefined && !requested.sheetIndexes && !requested.timestampsMs @@ -102,6 +115,7 @@ async function retrieveVideoStoryboard(parsed: z.infer withRunDeadline(deadlineAt, context.signal, signal => timeStoryboardStage(parsed.videoId, 'retrieval', () => context.provider.storyboard!(parsed.videoId, parsed.timestampsMs, { maxSheets: parsed.maxSheets ?? MAX_STORYBOARD_SHEETS, sheetIndexes: parsed.sheetIndexes, metadataOnly: false, resolution: requestedResolution(parsed), signal, deadlineAt, + ...(!context.session && (context.refreshEvidence || context.refreshFramesOnly) ? { refresh: true } : {}), }, event => context.onExtractionDiagnostic?.({ ...event, toolCallId }), ...(onReady ? [onReady] : [])), { runId: context.runId, toolCallId }), 'Storyboard retrieval exceeded its budget.'), async (response, signal) => { const storyboard = storyboardSchema.parse(response.value); @@ -131,7 +145,7 @@ async function retrieveVideoStoryboard(parsed: z.infer context.session?.brief().assets.some(asset => asset.version === version && asset.kind === 'storyboard_sheet')), - videoId: parsed.videoId, sessionReused: response.sessionReused === true, selection: storyboard.selection, manifest: storyboard.manifest, + videoId: parsed.videoId, requestedTimestampsMs: parsed.timestampsMs, sessionReused: response.sessionReused === true, selection: storyboard.selection, manifest: storyboard.manifest, resolution: servedResolution(storyboard), sampledRanges: storyboard.sheets.map(sheet => ({ startMs: sheet.firstFrameIndex * sheet.intervalMs, endMs: (sheet.firstFrameIndex + sheet.frameCount - 1) * sheet.intervalMs })), totalFrames: storyboard.frameCount, diff --git a/platform/src/agents/providers/youtube/tools/stored-analysis.ts b/platform/src/agents/providers/youtube/tools/stored-analysis.ts index d5bf8afd..d0cd5250 100644 --- a/platform/src/agents/providers/youtube/tools/stored-analysis.ts +++ b/platform/src/agents/providers/youtube/tools/stored-analysis.ts @@ -73,8 +73,9 @@ export async function readAnalysisAssets( if (scope) { // Read the position from the immutable payload, even when older inventory metadata omits it. const time = storedFramePosition.parse(stored.value).frames[0]!.timestampMs; - const allowed = scope.savedTimestampsMs.includes(time) || (!requiresRetrieval && scope.captureTimestampsMs.includes(time)); - if (stored.asset.videoId !== scope.videoId || !allowed) + const entry = scope.find(item => item.videoId === stored.asset.videoId) ?? scope.find(item => !item.videoId); + const allowed = entry && (entry.savedTimestampsMs.includes(time) || (!requiresRetrieval && entry.captureTimestampsMs.includes(time))); + if (!allowed) throw new Error('This saved version is outside the requested inspection, or its position still requires a fresh capture.'); } return stored; diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index d55dac2b..250c615b 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -8,7 +8,8 @@ import { capabilityRouteDecisionSchema, answerDetailSchema, comparisonVideoIdsSchema, - frameAccessSchema, + frameAccessListSchema, + unscopedImageAccessSchema, requestedEvidenceSchema, numberedItemCountSchema, visualEvidenceSchema, @@ -18,6 +19,9 @@ import { type FinalizeAnswerInput, } from '../contracts'; import { sessionBriefForModel, type SessionBrief } from '../runtime/session-evidence'; +import { extractYouTubeVideoIds, youTubeUrls } from './youtube-references'; +export { extractYouTubeVideoIds } from './youtube-references'; +import { frameRequestIssue, readFrameRequest, requestedSheets, type FrameRequest } from './frame-request'; import { conversationAssistantMessage, type ConversationTurn } from '../runtime/conversation-memory'; import { assertModelCostAvailable, type AgentModelCostBudget } from '../runtime/model-budget'; import { AGENT_CLASSIFICATION_TIMEOUT_MS, withRunDeadline } from '../runtime/deadline'; @@ -30,7 +34,8 @@ const classifierDecisionSchema = z.object({ skill: z.enum(['verify_saved_evidence']).optional().describe('Verify saved evidence: use for follow-ups that read saved original captions/comments, check exact quotations or precise passage links, or correct source claims. It does not by itself prohibit fetching; set savedOnly separately. Omit for broad summaries and visual analysis.'), requestedEvidence: requestedEvidenceSchema.optional().describe('Evidence the user explicitly asks to be read before answering: original_quotes for exact quotations, verbatim wording or precise passage links from captions; comments for what viewers or commenters say. Research must read each or report it unavailable. Omit when not explicitly requested.'), savedOnly: z.boolean().optional().describe('True when the user requires existing saved evidence only or prohibits fresh retrieval. All provider access is disabled for this run; use saved evidence reads and report missing assets.'), - frameAccess: z.union([frameAccessSchema, z.literal('agent_selected')]).optional().describe('For exact captures on one video, list only the permitted NEW capture timestamps in milliseconds. Put saved-only inspection positions separately in savedTimestampsMs, even if their assets are missing. Use agent_selected only for open-ended capture positions.'), + frameAccess: z.union([frameAccessListSchema, unscopedImageAccessSchema]).optional().describe('Permission to retrieve new images. Omit to disable both exact captures and storyboard retrieval. Use storyboard_only for requested storyboard sheets or an overview. A named sheet time limits the retrieval to the sheet containing that time. When the user fixes frame positions, saved or new, list one entry per video the turn touches: captureTimestampsMs holds only the permitted NEW capture positions in milliseconds, empty when nothing may be captured; savedTimestampsMs holds saved positions to re-inspect without recapture, even if their assets are missing. When the videos are still to be found by discovery, give one entry without videoId holding the capture times. Required for exact-timestamp questions and for any limit on what may be captured. Use agent_selected only when new frames may help and the user leaves every position to you.'), + setAsideFrames: z.array(z.object({ videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), timestampMs: z.number().int().nonnegative() })).max(24).optional().describe('Saved frames from the session inventory at a time the user named that this turn must not use: the user excluded them, or the time refers to another video. Every saved frame at a named time belongs either in its video\'s frameAccess entry or here.'), timestampedFrameQuestion: z.string().trim().min(1).max(500).optional().describe('For inspect_video: when the user supplies exact timestamps and asks what is visible, copy the focused visual question here. Include this for a request to describe each specified frame. Omit for image retrieval only or saved-only requests.'), refreshFramesOnly: z.boolean().optional().describe('For inspection or research, including multi-video comparisons: true when the user asks to freshly capture or refresh frames only. Reuse verified video metadata. Leave refreshEvidence unset unless other source evidence must also be refreshed.'), refreshEvidence: z.boolean().optional().describe('Except for a frames-only refresh, true only when the user explicitly asks to fetch again, refresh or get fresh source data. Choose an executable route in that case.'), @@ -48,7 +53,7 @@ const classifierDecisionSchema = z.object({ channelId: capabilityRouteDecisionSchema.options[0].shape.channelId.describe('For research restricted to one supplied channel, copy its channel ID or handle from suppliedChannelIds. Never invent a channel identifier.'), videoId: capabilityRouteDecisionSchema.options[1].shape.videoId.optional(), reason: capabilityRouteDecisionSchema.options[3].shape.reason.optional().describe('Required for finalize: explain why existing context suffices, what scope is missing, or why the request is unsupported. The finalizer writes the response.'), - visualEvidence: visualEvidenceSchema.optional().describe('Required for executable routes. Whether answering needs images: none, helpful or required. helpful and required enable storyboard and frame tools.'), + visualEvidence: visualEvidenceSchema.optional().describe('Required for executable routes. Whether answering needs images: none, helpful or required. helpful and required enable visual analysis. Required visuals need an explicit image grant unless the turn is saved-only. Use agent_selected for an ordinary visual question with delegated positions, or storyboard_only when only sheets are permitted.'), visualRequirements: visualRequirementsSchema.optional().describe('Only when visualEvidence is required: each requested fact that needs images, such as "presenter clothing".'), }); @@ -130,7 +135,10 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio // Keep the provider schema strict. Normalize a copy only at the acceptance // boundary, retaining which fields needed a fallback before the SDK parses them. const input = { ...value } as Record; - if (input.frameAccess && typeof input.frameAccess === 'object' && input.refreshFramesOnly !== true) { + // Fixed captures imply a frames refresh. A saved-only conflict is reported against frameAccess below. + const capturesFrames = Array.isArray(input.frameAccess) && input.frameAccess.some(entry => + Array.isArray((entry as { captureTimestampsMs?: unknown })?.captureTimestampsMs) && (entry as { captureTimestampsMs: unknown[] }).captureTimestampsMs.length > 0); + if (capturesFrames && input.refreshFramesOnly !== true && input.savedOnly !== true) { input.refreshFramesOnly = true; defaultedFields.push('refreshFramesOnly'); } @@ -172,11 +180,24 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio } if (input.refreshFramesOnly === true && input.visualEvidence === 'none') issue('visualEvidence', 'Refreshing frames requires visual tool access.'); const scope = parsed.success && parsed.data.frameAccess; - if (scope && scope !== 'agent_selected') { - if (!suppliedVideoIds.includes(scope.videoId) || (input.route === 'inspect_video' && input.videoId !== scope.videoId)) - issue('frameAccess', 'Scope frame access to the supplied video being inspected.'); - const times = [...scope.savedTimestampsMs, ...scope.captureTimestampsMs]; - if (new Set(times).size !== times.length) issue('frameAccess', 'Give each timestamp exactly one role, saved inspection or fresh capture.'); + if (scope && input.route === 'finalize') issue('route', 'Image retrieval or saved-frame inspection requires an executable route.'); + if (typeof scope === 'string' && input.savedOnly === true) + issue('frameAccess', 'Saved-only requests cannot retrieve new images. Omit frameAccess, or list only savedTimestampsMs.'); + if (Array.isArray(scope)) { + const scoped = scope.flatMap(entry => entry.videoId ? [entry.videoId] : []); + const open = scope.filter(entry => !entry.videoId); + // Every scoped video must be a subject of the route, or its frames cannot be read or captured. + const subjects = input.route === 'inspect_video' ? [input.videoId] + : input.route === 'topic_research' && Array.isArray(input.comparisonVideoIds) ? input.comparisonVideoIds : []; + if (input.savedOnly === true && capturesFrames) + issue('frameAccess', 'Saved-only requests cannot capture frames. List only savedTimestampsMs, or omit savedOnly if the user authorized new captures.'); + else if (scoped.some(id => !suppliedVideoIds.includes(id) || !subjects.includes(id)) || new Set(scoped).size !== scoped.length) + issue('frameAccess', 'List one frameAccess entry per video, only for the route\'s subjects. When saved or new frames span several videos, use topic_research with every one in comparisonVideoIds.'); + else if (open.length && (open.length > 1 || scope.length > 1 || open.some(entry => entry.savedTimestampsMs.length) + || input.route !== 'topic_research' || input.comparisonVideoIds !== undefined)) + issue('frameAccess', 'An entry without videoId fixes capture times for videos that discovery will select. Use it alone, once, without saved positions, and only for topic_research without comparisonVideoIds.'); + else if (scope.some(entry => new Set([...entry.savedTimestampsMs, ...entry.captureTimestampsMs]).size !== entry.savedTimestampsMs.length + entry.captureTimestampsMs.length)) + issue('frameAccess', 'Give each timestamp exactly one role, saved inspection or fresh capture.'); } if (input.timestampedFrameQuestion && input.route !== 'inspect_video') issue('route', 'Exact frame requests require single-video inspection.'); @@ -206,7 +227,6 @@ function classifierToolSchema(options: ClassifierValidationOptions, defaultedFie } }); } -const VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{11}$/; export interface ClassificationDiagnostic { attempt: number; @@ -268,6 +288,7 @@ const RECONSIDERATION_MIN_MS = 1_500; async function classifyWithinDeadline(input: CapabilityClassifierInput, deadlineAt: number): Promise { assertModelCostAvailable(input.modelBudget); + const request = readFrameRequest(input.message); const conversationHistory = input.conversationHistory ?? []; const videoIds = [...new Set([ ...extractYouTubeVideoIds(input.message), @@ -289,7 +310,8 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline 'Select skill verify_saved_evidence for follow-ups that recover saved original captions/comments, verify exact quotations or passage links, or correct source claims. This replaces general discovery and analysis with a verification workflow. The skill does not prohibit fetching: a request that also needs evidence not yet saved, such as comments, still gets it unless savedOnly is set. Omit the skill for broad summaries or visual analysis.', 'Set requestedEvidence for each kind of evidence the user explicitly asks for: original_quotes for exact quotations, verbatim wording or precise passage links; comments for what viewers or commenters say. Include every requested kind in one decision, alongside any explanation. Research cannot complete until each is read or reported unavailable, so choose an executable route.', 'Set savedOnly true whenever the user prohibits fresh retrieval or asks to use existing saved sources only. Keep this restriction on follow-ups when it remains in force. Missing saved evidence is a gap, never permission to fetch. Use the saved reader instead of transcript retrieval or discovery.', - 'When a turn mixes saved and new frames, omit savedOnly and supply frameAccess. For example, re-inspect saved 10:30 and capture only 10:45 means savedTimestampsMs [630000] and captureTimestampsMs [645000]. A prohibition on recapturing 10:30 still allows the requested 10:45. Times mentioned for comparison, narration or history do not authorize capture.', + 'A video length limit filters the source. A capture window limits the seek positions. For a tutorial under five minutes with a capture within the first 30 seconds, choose a captureTimestampsMs value between 0 and 30000 inclusive. Use an exact list even when the user lets you choose a position inside the window.', + 'Set frameAccess from the user request. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. If the images belong to different videos, include both in comparisonVideoIds. A saved image that cannot be read stays a saved inspection. Times mentioned for comparison, narration or history do not authorize capture.', 'Requests for exact quotations from original saved captions/comments, precise passage-start links, or explicit original-asset reads require inspect_video or topic_research with the resolved subjects, even if prior summaries discuss the topic. Research can call search_originals without fetching YouTube. Quoting a source is different from quoting a prior conversation message. Do not route an original-source read directly to finalize.', 'For every context_answer set contextScope: history for listing, quoting, recalling or correcting conversation messages or preferences; video for claims about video content; mixed only when the requested answer needs both. A video URL inside a quoted earlier message does not require video evidence.', 'For finalize give a short routing reason, not a user-facing answer. Choose responseIntent clarification for missing scope, or rejected for unsupported requests. Do not use the legacy clarification or rejected routes for new decisions.', @@ -370,7 +392,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline }] : parsed.success ? [] : parsed.error.issues.map(issue => ({ path: issue.path.map(String).join('.'), code: issue.code, message: issue.message, })); - if (parsed.success && feedback.length === 0) feedback = semanticIssues(parsed.data, input, videoIds); + if (parsed.success && feedback.length === 0) feedback = semanticIssues(parsed.data, input, videoIds, request); // Advisory only: the reconsideration may keep its choice, so a keyword match never fails classification. let advisory = false; if (parsed.success && feedback.length === 0 && context.attempt === 1) { @@ -404,7 +426,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline const lastResort = (evaluations: Evaluation[]) => { // The fallback uses no model, but an exhausted budget still ends the run. assertModelCostAvailable(input.modelBudget); - const built = lastResortDecision(evaluations, input, videoIds, channelIds, explicitVideoIds); + const built = lastResortDecision(evaluations, input, videoIds, channelIds, explicitVideoIds, request); console.warn(JSON.stringify({ event: 'agent_classification_last_resort', modelCallId: callId, method: built.method, candidates: evaluations.length, defaultedFields: built.defaultedFields })); input.onDiagnostic?.({ attempt: 0, stage: 'last_resort', lastResort: built.method, outcome: 'valid', modelId: '', @@ -414,7 +436,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline const decision = await coordinateClassification(call, input.signal, deadlineAt, callId, { fallbackModel: Boolean(input.fallbackModel), managedTimeouts: hasModelTimeouts(input.model), lastResort }); - return finishClassification(decision, videoIds, channelIds, explicitVideoIds, input.message); + return bindStoryboardSelection(finishClassification(decision, videoIds, channelIds, explicitVideoIds, input.message), request); } type ClassificationIssue = { path: string; code: string; message: string }; @@ -584,7 +606,7 @@ const ROUTE_FIELDS = { finalize: ['responseIntent', 'contextScope', 'historySelection', 'reason', 'comparisonVideoIds'], } as const satisfies Record; const SHARED_FIELDS = ['skill', 'savedOnly', 'requestedEvidence', 'answerDetail', 'numberedItemCount', 'refreshEvidence', 'refreshDynamicData', - 'visualEvidence', 'visualRequirements'] as const satisfies readonly DefaultedClassificationField[]; + 'visualEvidence', 'visualRequirements', 'setAsideFrames'] as const satisfies readonly DefaultedClassificationField[]; function candidateObject(value: unknown): Record | undefined { if (typeof value === 'string') { @@ -593,16 +615,22 @@ function candidateObject(value: unknown): Record | undefined { return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : undefined; } -function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[]): ClassificationIssue[] { - // An explicit prohibition rules out an unrestricted fallback. Only the classifier - // resolves which video and positions the user authorized. - const prohibitsRecapture = /\b(?:without|never|do not|don['’]t)\s+re-?captur(?:e|ing)\b/i.test(input.message); - if (decision.frameAccess === 'agent_selected' && prohibitsRecapture) { - return [{ path: 'frameAccess', code: 'prohibited_recapture', message: 'The request prohibits recapture. Supply the fixed fresh positions and keep the saved inspection positions separate.' }]; +function pinnedFrameAccess(entries: T[], videoId: string) { + return entries.map(entry => ({ ...entry, videoId: entry.videoId ?? videoId })); +} + +function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[], request: FrameRequest): ClassificationIssue[] { + // A single linked video pins inspection to itself, which would leave another video's frames out of reach. + const linked = extractYouTubeVideoIds(input.message); + if (decision.frameAccess && decision.route === 'inspect_video' && linked.length === 1 && linked[0] !== decision.videoId) { + return [{ path: 'frameAccess', code: 'frame_scope_outside_link', message: `The message links ${linked[0]}. Inspect that video, or use topic_research with comparisonVideoIds for every video whose frames this turn uses.` }]; } - if (decision.refreshFramesOnly && !decision.frameAccess && !decision.comparisonVideoIds?.length - && (decision.route === 'inspect_video' || videoIds.length === 1)) { - return [{ path: 'frameAccess', code: 'missing_capture_scope', message: 'Specify saved and authorized new timestamps separately, or agent_selected if the user delegated the capture positions.' }]; + if (decision.route !== 'finalize') { + const pin = decision.route === 'topic_research' && !decision.comparisonVideoIds?.length && !decision.playlistListing && linked.length === 1; + const plan = pin ? { ...decision, route: 'inspect_video', videoId: linked[0], + frameAccess: Array.isArray(decision.frameAccess) ? pinnedFrameAccess(decision.frameAccess, linked[0]!) : decision.frameAccess } : decision; + const issue = frameRequestIssue(plan, request, input.sessionBrief?.assets); + if (issue) return [{ path: 'frameAccess', code: 'invalid_frame_access', message: issue }]; } const scope = comparisonScopeIssues(decision, input, videoIds); return scope.length ? scope : searchQueryNumberIssues(decision, input); @@ -615,7 +643,7 @@ function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifie * request defaults alone. Candidates from later attempts are preferred. */ function lastResortDecision( - evaluations: Evaluation[], input: CapabilityClassifierInput, videoIds: string[], channelIds: string[], explicitVideoIds: string[], + evaluations: Evaluation[], input: CapabilityClassifierInput, videoIds: string[], channelIds: string[], explicitVideoIds: string[], request: FrameRequest, ): { decision: ClassifierDecision; method: LastResortMethod; defaultedFields: DefaultedClassificationField[] } { const candidates = [...evaluations].sort((a, b) => b.context.attempt - a.context.attempt) .flatMap(item => { const candidate = candidateObject(item.candidate); return candidate ? [candidate] : []; }); @@ -636,10 +664,16 @@ function lastResortDecision( // Requirements any candidate stated, or the request implies, survive every fallback: // fresh data needs an executable route, required visuals stay required, and a // follow-up comparison keeps both of its subjects. + // A bounded list that no attempt stated validly is never widened, not even to another attempt's agent_selected. + const listing = candidates.filter(candidate => Array.isArray(candidate.frameAccess)); + const frameAccess = firstValid('frameAccess', listing.length ? listing : candidates); + if (listing.length && !frameAccess) { + const clarification = { route: 'finalize', responseIntent: 'clarification', reason: 'Clarify the saved and permitted fresh frame positions for each video.', answerDetail: 'standard' }; + return { decision: classifierDecisionSchema.parse(clarification), method: 'defaults', + defaultedFields: Object.keys(clarification) as DefaultedClassificationField[] }; + } const constraints = { - // Preserve a bounded list even when a later failed attempt proposed open-ended selection. - frameAccess: firstValid('frameAccess', candidates.filter(candidate => candidate.frameAccess !== 'agent_selected')) - ?? firstValid('frameAccess'), + frameAccess, savedOnly: candidates.some(candidate => candidate.savedOnly === true), requestedEvidence: [...new Set(candidates.flatMap(candidate => requestedEvidenceSchema.safeParse(candidate.requestedEvidence).data ?? []))], refreshFramesOnly: candidates.some(candidate => candidate.refreshFramesOnly === true), @@ -654,7 +688,7 @@ function lastResortDecision( return kept.length >= 2 ? [kept] : []; })[0] ?? []])].slice(0, 8), }; - const needsFreshData = constraints.frameAccess || constraints.refreshFramesOnly || constraints.refreshEvidence || constraints.refreshDynamicData; + const needsFreshData = (Array.isArray(constraints.frameAccess) && constraints.frameAccess.some(entry => entry.captureTimestampsMs.length)) || constraints.refreshFramesOnly || constraints.refreshEvidence || constraints.refreshDynamicData; if (needsFreshData && constraints.savedOnly) { const clarification = { route: 'finalize', responseIntent: 'clarification', reason: 'Routing attempts disagree about saved-only access and fresh retrieval. Ask which sources may be refreshed before retrieving or answering from stale data.', answerDetail: 'standard' }; @@ -736,7 +770,7 @@ function lastResortDecision( }; const accept = (decision: Record) => { const parsed = validateClassifierDecision(decision, validation); - return parsed.success ? { parsed: parsed.data, issues: semanticIssues(parsed.data, input, videoIds) } : undefined; + return parsed.success ? { parsed: parsed.data, issues: semanticIssues(parsed.data, input, videoIds, request) } : undefined; }; // Recover a missing route from one candidate as returned. The recovered decision @@ -818,11 +852,13 @@ function expectedComparisonSubjects(input: CapabilityClassifierInput): string[] function finishClassification( decision: z.infer, videoIds: string[], channelIds: string[], explicitVideoIds: string[], message: string, ): CapabilityRouteDecision { - const { explicitSourceCount, frameAccess, ...fields } = withVisualAccess(decision); - const classified = { ...fields, ...(frameAccess && frameAccess !== 'agent_selected' ? { frameAccess } : {}) }; + // Set-aside frames only check the classification; the route carries what research may use. + const { explicitSourceCount, setAsideFrames: _setAside, ...classified } = withVisualAccess(decision); const pinned = !(decision.route === 'topic_research' && decision.playlistListing) && explicitVideoIds.length === 1 && !decision.comparisonVideoIds?.length && (decision.route !== 'finalize' || (decision.responseIntent === 'context_answer' && decision.contextScope !== 'history')); - const route = pinned ? { ...classified, route: 'inspect_video' as const, videoId: explicitVideoIds[0]!, + const pinnedAccess = pinned && Array.isArray(classified.frameAccess) + ? pinnedFrameAccess(classified.frameAccess, explicitVideoIds[0]!) : classified.frameAccess; + const route = pinned ? { ...classified, frameAccess: pinnedAccess, route: 'inspect_video' as const, videoId: explicitVideoIds[0]!, visualEvidence: classified.visualEvidence ?? 'helpful', useStoryboard: classified.visualEvidence !== 'none' } : classified; const researchVideoCount = route.route === 'inspect_video' ? 1 : route.route === 'topic_research' ? Math.min(8, route.comparisonVideoIds?.length ?? explicitSourceCount ?? (route.researchBreadth === 'comparative' ? 4 : 2)) : 0; @@ -917,47 +953,39 @@ export function extractYouTubeChannelIds(message: string): string[] { return [...ids]; } -// Consume the whole URL before checking its host. Embedded links and v= parameters belong to that URL. -const urlCandidates = /(?:https?:\/\/|(?"']+/gi; -function* youTubeUrls(message: string): Generator { - for (const candidate of message.match(urlCandidates) ?? []) { - const normalized = candidate.replace(/[\])},.!?;:'"]+$/g, ''); - let url: URL; - try { - url = new URL(/^https?:\/\//i.test(normalized) ? normalized : `https://${normalized}`); - } catch { - continue; - } - const host = url.hostname.toLowerCase(); - if (['youtube.com', 'www.youtube.com', 'm.youtube.com', 'music.youtube.com', 'youtu.be', 'www.youtu.be', 'm.youtu.be'].includes(host)) yield url; - } -} - -export function extractYouTubeVideoIds(message: string): string[] { - const ids = new Set(); - for (const url of youTubeUrls(message)) { - if (url.hostname.endsWith('youtu.be')) { - addVideoId(ids, url.pathname.split('/').filter(Boolean)[0]); - continue; - } - addVideoId(ids, url.searchParams.get('v') ?? undefined); - const [kind, pathId] = url.pathname.split('/').filter(Boolean); - if (kind === 'shorts' || kind === 'live' || kind === 'embed') addVideoId(ids, pathId); - } - - const labelledIdPattern = /(?:^|[\s(])(?:video(?:\s+id)?|v)\s*[:=]\s*([A-Za-z0-9_-]{11})(?=$|\s|[),.!?;])/gi; - for (const match of message.replace(urlCandidates, ' ').matchAll(labelledIdPattern)) addVideoId(ids, match[1]); - const trimmed = message.trim(); - if (VIDEO_ID_PATTERN.test(trimmed)) ids.add(trimmed); - return [...ids]; +function bindStoryboardSelection(decision: CapabilityRouteDecision, request: FrameRequest): CapabilityRouteDecision { + if ((decision.route !== 'inspect_video' && decision.route !== 'topic_research') || decision.frameAccess !== 'storyboard_only') return decision; + const selections = requestedSheets(decision, request); + const scoped = request.refreshSheets ? { ...decision, refreshFramesOnly: true } : decision; + if (!selections?.length) return scoped; + return { ...scoped, frameAccess: decision.route === 'inspect_video' + ? { storyboardTimestampsMs: selections[0]!.timestampsMs } : { storyboardSelections: selections } }; } export async function resolveCapabilityRoute(options: { + message?: string; + priorAssets?: SessionBrief['assets']; persisted?: unknown; classify: () => Promise; persist: (decision: CapabilityRouteDecision) => void | Promise; }): Promise { - if (options.persisted !== undefined) return capabilityRouteDecisionSchema.parse(options.persisted); + if (options.persisted !== undefined) { + const decision = capabilityRouteDecisionSchema.parse(options.persisted); + // Routes persisted before every scoped video had to be a route subject could pin another + // video after validation. Resuming must not run them without the access the user gave. + const subjects = decision.route === 'inspect_video' ? [decision.videoId] + : decision.route === 'topic_research' ? decision.comparisonVideoIds ?? [] : []; + if ('frameAccess' in decision && Array.isArray(decision.frameAccess) && decision.frameAccess.some(entry => entry.videoId ? !subjects.includes(entry.videoId) : decision.route === 'inspect_video' || subjects.length > 0)) { + return { route: 'finalize', responseIntent: 'clarification', reason: 'Clarify which video the saved and fresh frame positions refer to.' }; + } + if (options.message && (decision.route === 'inspect_video' || decision.route === 'topic_research')) { + const request = readFrameRequest(options.message); + const issue = frameRequestIssue(decision, request, options.priorAssets, true); + if (issue) return { route: 'finalize', responseIntent: 'clarification', reason: issue }; + return bindStoryboardSelection(decision, request); + } + return decision; + } const decision = capabilityRouteDecisionSchema.parse(await options.classify()); await options.persist(decision); return decision; @@ -985,7 +1013,3 @@ function resolveClassification( : 'Which supplied YouTube video would you like me to inspect?', }; } - -function addVideoId(ids: Set, candidate: string | undefined): void { - if (candidate && VIDEO_ID_PATTERN.test(candidate)) ids.add(candidate); -} diff --git a/platform/src/agents/research/frame-request.ts b/platform/src/agents/research/frame-request.ts new file mode 100644 index 00000000..d82b2815 --- /dev/null +++ b/platform/src/agents/research/frame-request.ts @@ -0,0 +1,209 @@ +import { timestampExpressions } from './timestamp-expression'; +import type { RouteFrameAccess } from '../contracts'; +import { clockTime } from '../runtime/clock-time'; +import { extractYouTubeVideoIds, linkedSeek, urlCandidates } from './youtube-references'; +import type { SessionBrief } from '../runtime/session-evidence'; + +// A comparison such as "within" or "at most" can limit either video length or capture position. +// Exclude a time when the surrounding words describe duration ("videos under 5 minutes", "5 minutes long"). +const COMPARISON = /\b(?:under|over|within|up to|at (?:least|most)|(?:no )?(?:longer|shorter|less|more) than)\s*$/i; +const DURATION_SUBJECT = /\b(?:duration|length|runtime)(?:\s+(?:of|is|must be|should be))?\s*$/i; +const SOURCE_SELECTION = /\b(?:find|search|recommend|list|videos?|clips?|tutorials?|lectures?|recordings?|episodes?)\b/i; +const FRAME_ACTION = /\b(?:frames?|stills?|timestamps?|capture|capturing|inspect|sample|extract)\b/i; +const VIDEO_LENGTH_SUFFIX = /^\s*-?\s*(?:long|in (?:length|duration)|videos?|clips?|lectures?|tutorials?|recordings?|episodes?|of (?:content|footage|material)|ads?|intros?|segments?|versions?|cuts?|shorts?|views?)\b/i; +const START_WINDOW = /\b(?:within(?: the first)?|up to|no later than|in the first|during the first)\s*$/i; +const START_ANCHOR = /^\s+(?:of|from)\s+(?:the\s+)?(?:(?:video|clip)(?:'s|’s)\s+)?(?:start|beginning)\b/i; +const RELATIVE_ANCHOR = /^\s+(?:of|from|after|before)\b/i; + +function isVideoLength(before: string, after: string): boolean { + if (VIDEO_LENGTH_SUFFIX.test(after)) return true; + // An article plus a time quantity modifying a noun describes its length ("a 2-hour documentary"). + if (/\b(?:a|an)\s*$/i.test(before) && !FRAME_ACTION.test(before.split(CLAUSE).at(-1) ?? '') + && /^\s+(?!(?:mark|position|frame|image|picture|still|timestamp|screenshot)\b)[a-z]/i.test(after)) return true; + const comparison = COMPARISON.exec(before); + if (!comparison) return false; + const subject = before.slice(0, comparison.index); + if (DURATION_SUBJECT.test(subject)) return true; + if (!START_WINDOW.test(before) && /\b(?:video|clip|tutorial|lecture|recording|episode)s?\s*$/i.test(subject)) return true; + const clause = subject.split(/[.!?;,\n]|\b(?:and|but|then)\b/i).at(-1) ?? ''; + return SOURCE_SELECTION.test(clause) && !FRAME_ACTION.test(clause); +} + +// A list continues the preceding operation ("capture 0:30, 0:45 and 1:00"). +// A new clause can change it ("reuse 0:30, then capture 1:00"). +const CLAUSE = /[:;.!?]\s+|\n|(?:,\s*|\b(?:then|and|but)\s+)(?!https?:\/\/|at\b)(?=[a-z])|\b(?:with|versus|vs\.?)\s+(?=(?:(?:a|an|the)\s+)?(?:new|fresh|saved|previous|prior)\b)/i; +function positionRole(clause: string, after: string) { + if (/\b(?:no|not|never|without|don['’]t|except|instead of|rather than|exclude|ignore|skip)\b/i.test(clause)) return 'excluded'; + if (/\bstoryboards?\b/i.test(clause) || /^\s+storyboard sheet\b/i.test(after)) return 'sheet'; + if (/\b(?:re-?capture|capture|refresh|(?:new|fresh) (?:image|still|frame))\b/i.test(clause) + || (/\b(?:new|fresh)\s*$/i.test(clause) && /^\s+(?:image|still|frame)\b/i.test(after))) return 'capture'; + if (/\b(?:transcript|caption|comment)s?\b/i.test(clause)) return; + if (/\b(?:reuse|re-?inspect|saved)\b/i.test(clause) || /^\s+saved (?:frame|image)\b/i.test(after)) return 'saved'; +} + +// The classifier resolves the subject. The user's named positions bound what the classifier may grant. +export function readFrameRequest(message: string) { + const positions: { time: number; role: ReturnType; videoIds: string[] }[] = []; + let captureEndMs: number | undefined; + const urls = [...message.matchAll(urlCandidates)]; + const inUrl = (index: number) => urls.some(url => index >= url.index && index < url.index + url[0].length); + // Preserve offsets while keeping URL parameters out of prose expressions. + const expressions = timestampExpressions(message.replace(urlCandidates, url => '#'.repeat(url.length))); + let ambiguousTime = false; + function addProseTime(start: number, end: number, time: number, ambiguous = false) { + const before = message.slice(0, start); + const after = message.slice(end); + const fromStart = START_ANCHOR.test(after); + if (!fromStart && isVideoLength(before, after)) return; + if (ambiguous) { ambiguousTime = true; return; } + // A window end limits new captures. It does not request inspection of a saved frame at the endpoint. + if (START_WINDOW.test(before) && (fromStart || !RELATIVE_ANCHOR.test(after) || /\bthe first\s*$/i.test(before))) { + captureEndMs = Math.min(captureEndMs ?? time, time); + } else { + const prefix = before.split(CLAUSE).at(-1) ?? ''; + const suffix = after.split(CLAUSE, 1)[0] ?? ''; + const role = positionRole(prefix, suffix); + if (/\b(?:previous|prior|earlier|last) (?:answer|response|turn|message)\b/i.test(prefix) + && (!role || /\b(?:described|mentioned|cited|showed|reported)\b/i.test(prefix))) return; + const expression = message.slice(start, end); + const clause = prefix + expression + suffix; + const subjects = extractYouTubeVideoIds(clause); + // Multiple timed operands have separate subjects. One time followed by two links applies to both. + const operand = /\s+(?:and|with|versus|vs\.?|to)\s+|,\s*/i; + const localPrefix = prefix.split(operand).at(-1) ?? ''; + const localSuffix = suffix.split(operand)[0] ?? ''; + const separate = subjects.length > 1 && expressions.some(other => other.start >= start - prefix.length && other.end <= end + suffix.length + && (other.end <= start - localPrefix.length || other.start >= end + localSuffix.length)); + const localSubjects = separate ? extractYouTubeVideoIds(localPrefix + expression + localSuffix) : subjects; + positions.push({ time, role, videoIds: localSubjects.length ? localSubjects : subjects }); + } + } + for (const time of expressions) addProseTime(time.start, time.end, time.milliseconds, time.ambiguous); + for (const match of message.matchAll(/\b(?:opening|first) (?:exact )?(?:frame|still|image)\b/gi)) { + if (!inUrl(match.index)) addProseTime(match.index, match.index + match[0].length, 0); + } + // An unspecified interval has no safe numeric end. Ask for it before retrieving images. + const vagueWindow = /\b(?:first|last|opening|final) (?:[a-z]+(?:[- ](?:of|[a-z]+))?) (?:seconds?|minutes?|frames?)\b|\b(?:last|closing|final) (?:frame|still)\b/i.test(message); + // An explicit position or window in the request takes precedence over the link's seek time. + if (!positions.length && captureEndMs === undefined) { + for (const url of urls) { + const seek = linkedSeek(url[0]); + if (!seek) continue; + const prefix = message.slice(0, url.index).split(CLAUSE).at(-1) ?? ''; + const suffix = message.slice(url.index + url[0].length).split(CLAUSE, 1)[0] ?? ''; + positions.push({ ...seek, role: positionRole(prefix, suffix) }); + } + } + // An explicit exact-time instruction still limits retrieval when its time could not be read. + const missingExactTime = !positions.length && captureEndMs === undefined + && /\b(?:(?:only|exactly|just)\s+(?:the (?:frame|still|image)\s+)?at|at (?:exactly|only)|no other (?:timestamps?|positions?))\b/i.test(message); + const noRecapture = /\b(?:without|never|do not|don['’]t)\s+re-?captur(?:e|ing)\b/i.test(message); + const noCapture = /\b(?:never|do not|don['’]t)\s+capture\s+(?:any\s+)?(?:new |exact )?(?:frames?|images?|stills?)\b/i.test(message); + const sheetTimes = positions.every(position => position.role === 'sheet') && captureEndMs === undefined + ? [...new Set(positions.map(position => position.time))] : []; + const refreshSheets = message.split(CLAUSE).some(clause => + /\b(?:refresh|reload|re-?fetch|fresh)\s+(?:(?:only|just|the|these|both|saved|video)\s+)*storyboards?\b|\bstoryboard(?:-only)?\s+refresh\b/i.test(clause) + && !/\b(?:not|never|without|don['’]t)\s+(?:refresh|reload|re-?fetch)\b/i.test(clause)); + const requestsSaved = positions.some(pos => pos.role === 'saved') || /\b(?:reuse|re-?inspect|inspect|analy[sz]e)\s+(?:the |my |our |that )?saved (?:video |exact )?(?:frames?|images?|stills?)\b/i.test(message); + return { positions, ambiguousTime, missingExactTime, requestsSaved, refreshSheets, sheetTimes, captureEndMs, noRecapture, noCapture, vagueWindow }; +} + +export type FrameRequest = ReturnType; +interface FramePlan { + route: string; + videoId?: string; + comparisonVideoIds?: string[]; + frameAccess?: RouteFrameAccess; + savedOnly?: boolean; + timestampedFrameQuestion?: string; + refreshFramesOnly?: boolean; + visualEvidence?: string; + setAsideFrames?: { videoId: string; timestampMs: number }[]; +} + +/** Bind each sheet position to the named subjects before the model can call the provider. */ +export function requestedSheets(plan: FramePlan, request: FrameRequest) { + if (!request.sheetTimes.length || request.sheetTimes.length > 24) return; + const subjects = plan.route === 'inspect_video' && plan.videoId ? [plan.videoId] : plan.comparisonVideoIds ?? []; + if (!subjects.length || request.positions.some(pos => pos.videoIds.some(id => !subjects.includes(id)))) return; + return subjects.flatMap(videoId => { + const times = request.positions.filter(pos => !pos.videoIds.length || pos.videoIds.includes(videoId)).map(pos => pos.time); + return times.length ? [{ videoId, timestampsMs: [...new Set(times)] }] : []; + }); +} + +/** Bound the classifier's grant to operations named in the request, including on resume. */ +export function frameRequestIssue(plan: FramePlan, request: FrameRequest, assets: SessionBrief['assets'] = [], resumed = false): string | undefined { + const access = plan.frameAccess; + const { positions, captureEndMs, noRecapture, noCapture } = request; + const limited = positions.length > 0 || captureEndMs !== undefined || noRecapture || noCapture; + const sameSubject = (ids: string[], videoId?: string) => !ids.length || (videoId !== undefined && ids.includes(videoId)); + const targetedSheets = requestedSheets(plan, request); + if ((request.ambiguousTime || request.missingExactTime) && access) return 'Ask for an unambiguous timestamp or an explicit list of positions.'; + if (request.vagueWindow && access && !(Array.isArray(access) && access.every(entry => !entry.captureTimestampsMs.length))) + return 'Ask for the exact capture interval before retrieving images.'; + if (plan.visualEvidence === 'required' && !plan.savedOnly && (!access || (Array.isArray(access) + && !access.some(entry => entry.savedTimestampsMs.length || entry.captureTimestampsMs.length)))) + return 'Required visual evidence needs capture permission or a saved inspection in frameAccess. Select permitted new images or specify the saved inspection.'; + if (access && typeof access === 'object' && !Array.isArray(access)) { + // A previously persisted targeted sheet cannot widen during recovery. + const actual = 'storyboardSelections' in access ? access.storyboardSelections + : plan.route === 'inspect_video' ? [{ videoId: plan.videoId, timestampsMs: access.storyboardTimestampsMs }] : []; + if (!targetedSheets || actual.length !== targetedSheets.length || targetedSheets.some(expected => { + const times = actual.find(item => item.videoId === expected.videoId)?.timestampsMs; + return !times || times.length !== expected.timestampsMs.length || expected.timestampsMs.some(time => !times.includes(time)); + })) return 'Clarify the requested storyboard sheet positions.'; + return; + } + + const captures = positions.filter(position => position.role === 'capture'); + const matches = (time: number, videoId?: string) => positions.filter(pos => pos.time === time && sameSubject(pos.videoIds, videoId)); + if (request.refreshSheets && !captures.length && !positions.some(pos => pos.role === 'saved') && access !== 'storyboard_only') + return 'Refresh the requested storyboard sheets with storyboard_only access.'; + if (Array.isArray(access)) { + for (const entry of access) { + for (const time of entry.captureTimestampsMs) { + const named = matches(time, entry.videoId); + if (captureEndMs !== undefined && time > captureEndMs) + return `Select capture positions within the requested window ending at ${captureEndMs} ms.`; + const explicit = named.find(pos => pos.role === 'capture'); + // A finite grant needs a position or window in the request. Delegated selection has its own route. + const inRequestedRange = captureEndMs !== undefined || named.length > 0; + const authorized = captures.length ? !!explicit : inRequestedRange; + const stored = assets.some(asset => asset.kind === 'frame' && asset.videoId === entry.videoId && asset.details.timestampMs === time); + // Two videos may share a timestamp. An explicit capture on B can coexist with a saved inspection on A. + const savedHere = named.some(pos => pos.role === 'saved') && (!explicit || (stored && !explicit.videoIds.length)); + const forbidden = noCapture || savedHere || named.some(pos => pos.role === 'excluded' || pos.role === 'sheet'); + if (!authorized || forbidden || (noRecapture && !explicit && (stored || captureEndMs === undefined))) + return `The user did not authorize a new exact capture at ${clockTime(time)} on this video. List only the authorized capture positions.`; + } + } + for (const pos of captures) { + if (!access.some(entry => sameSubject(pos.videoIds, entry.videoId) && entry.captureTimestampsMs.includes(pos.time))) + return `Include the requested new capture at ${clockTime(pos.time)} on its video.`; + } + } else { + if (access === 'storyboard_only' && (!plan.refreshFramesOnly || request.refreshSheets) && (targetedSheets || (!positions.length && captureEndMs === undefined && !noRecapture))) return; + if (!plan.savedOnly && (plan.timestampedFrameQuestion || (access && limited))) + return 'The user fixed the frame positions. List the saved inspection positions and the permitted capture positions in frameAccess, one entry per video.'; + if (plan.refreshFramesOnly && access !== 'agent_selected') return 'A frame refresh needs capture permission. List the positions in frameAccess, or set agent_selected if the user left them to you.'; + } + + // Classification can set aside named inventory from an unrelated video. That choice + // has already been validated and is not part of the persisted execution grant. + if (!access && !plan.refreshFramesOnly && !plan.timestampedFrameQuestion && plan.visualEvidence === 'none') return; + if (!resumed) { + for (const asset of assets) { + const time = asset.details.timestampMs; + if (asset.kind !== 'frame' || typeof time !== 'number' || !positions.some(pos => pos.time === time)) continue; + const entry = Array.isArray(access) && access.find(entry => entry.videoId === asset.videoId); + if (entry && (entry.savedTimestampsMs.includes(time) || entry.captureTimestampsMs.includes(time))) continue; + if (plan.setAsideFrames?.some(frame => frame.videoId === asset.videoId && frame.timestampMs === time)) continue; + return `The session holds a saved ${clockTime(time)} frame of ${asset.videoId}. Include its requested operation in frameAccess, or in setAsideFrames if the user excluded it or meant another video.`; + } + } + for (const pos of positions.filter(pos => pos.role === 'saved')) { + if (!Array.isArray(access) || !access.some(entry => sameSubject(pos.videoIds, entry.videoId) && entry.savedTimestampsMs.includes(pos.time))) + return `List the requested saved ${clockTime(pos.time)} inspection even if its image is unavailable.`; + } +} diff --git a/platform/src/agents/research/research-agent.ts b/platform/src/agents/research/research-agent.ts index 7c2edb38..e5caf534 100644 --- a/platform/src/agents/research/research-agent.ts +++ b/platform/src/agents/research/research-agent.ts @@ -1,5 +1,5 @@ import { inspectionRecordsForModel, INSPECTION_RECORD_GUIDANCE } from '../runtime/inspection-records'; -import { frameActivity } from '../runtime/frame-access'; +import { frameActivity, imagePermissions, isFrameTurn } from '../runtime/frame-access'; import { VISUAL_INSPECTION_SKILL } from './skills/visual-inspection.generated'; import { scheduleEvidenceTools } from '../runtime/evidence-scheduler'; import { withRunAssetCache, type RunAssetCache } from '../runtime/run-asset-cache'; @@ -11,7 +11,7 @@ import { durationLimitNotice, withDurationLimitNotice } from './duration-limit-a import { canAnalyzeStoryboard, storyboardRetrievalBudget, STORYBOARD_RETRIEVAL_MIN_MS } from '../runtime/storyboard-budget'; import { transcriptFailureCode, YOUTUBE_UNAVAILABLE_MESSAGE } from '../providers/youtube/tools/transcript-tool-errors'; import { traceToolCallRepair, traceToolSet, type TraceToolCall } from '../runtime/tool-call-trace'; -import { sessionBriefForModel, type SessionEvidenceStore } from '../runtime/session-evidence'; +import { sessionBriefForModel, type SessionBrief, type SessionEvidenceStore } from '../runtime/session-evidence'; import { sessionProvider } from '../runtime/session-provider'; import { createReadPriorEvidenceTool, isHistoryOnlyRoute, preparePriorEvidence, PRIOR_EVIDENCE_GUIDANCE, READ_PRIOR_EVIDENCE_TOOL_NAME, type DeliverEvidence, type PriorEvidenceAccess } from '../runtime/prior-evidence'; @@ -41,6 +41,7 @@ import { type AgentWarning, type AgentTurnResult, type CapabilityRouteDecision, + type FrameAccess, type EvidenceOperation, type EvidencePacket, type FinalizeAnswerInput, @@ -139,6 +140,7 @@ async function executeResearchRunWithAssets(options: { env: Env; modelFailover?: ModelFailoverState; runId: string; + startedAt?: number; message: string; /** Trusted line naming the run's date, from its admission time and the user's time zone. */ currentDate?: string; @@ -187,6 +189,9 @@ async function executeResearchRunWithAssets(options: { if (!options.persistedRoute) await options.onClassifying?.(classificationDeadlineAt); const decision = await resolveCapabilityRoute({ persisted: options.persistedRoute, + message: options.message, + priorAssets: options.persistedRoute && 'frameAccess' in options.persistedRoute + ? options.session?.brief().assets.filter(asset => asset.collectedAt < (options.startedAt ?? Infinity)) : undefined, classify: () => withRunDeadline(classificationDeadlineAt, options.signal, signal => classifyCapabilityWithModel({ message: options.message, conversationHistory: options.conversationHistory, @@ -276,7 +281,10 @@ async function executeResearchRunWithAssets(options: { const upstream = decision.savedOnly ? savedOnlyProvider() : createYouTubeAgentProvider(options.env, undefined, researchDeadlineAt); const refreshFrames = decision.refreshEvidence || decision.refreshFramesOnly; - const provider = createCapabilityProvider(options.session ? sessionProvider(upstream, options.session, decision.refreshEvidence, options.registerRetrievedAsset, assetCache, refreshFrames) : upstream, decision); + const permissions = imagePermissions(decision); + const refreshSheets = decision.refreshEvidence || (decision.refreshFramesOnly && permissions.canRetrieveSheets && !permissions.canCapture); + const provider = createCapabilityProvider(options.session ? sessionProvider(upstream, options.session, decision.refreshEvidence, + options.registerRetrievedAsset, assetCache, refreshFrames, refreshSheets) : upstream, decision); const transcriptAnalyst = createTranscriptAnalyst( runModel(options.env, options.sessionAffinity, 'low', { ...modelMetadata, @@ -460,9 +468,12 @@ async function runResearchAgentWithModelWithinDeadline(options: { const savedEvidenceSkill = usesSavedEvidenceSkill(options.decision); const playlistListing = options.decision.route === 'topic_research' && options.decision.playlistListing && !options.decision.savedOnly ? options.decision : undefined; - // Missing flags belong to legacy persisted routes, which retain their tool set. + // A storyboard sheet includes neighboring images, even when requested at one timestamp. + const { frameScope, sheetSelections, canCapture, canRetrieveSheets } = imagePermissions(options.decision); const toolNames = (options.toolNames ?? capability.toolNames) - .filter(name => !options.decision.frameAccess || name !== 'get_video_storyboard') + .filter(name => name !== 'get_video_storyboard' || canRetrieveSheets) + // New captures are closed unless the route lists their positions or delegates them. + .filter(name => name !== 'get_video_frames' || canCapture) .filter(name => !playlistListing || (!playlistListing.playlistId && name === 'get_playlist') || name === FINALIZE_ANSWER_TOOL_NAME) .filter(name => !verifySaved || name === FINALIZE_ANSWER_TOOL_NAME || (!options.decision.savedOnly && (SAVED_VERIFICATION_RETRIEVAL_TOOLS as readonly string[]).includes(name))) @@ -626,13 +637,16 @@ async function runResearchAgentWithModelWithinDeadline(options: { } return pending; }; + const failedFrameVersions = new Set(); + const failedCaptures = new Set(); const trackedContext: AgentToolContext = { ...options.context, researchDeadlineAt: options.researchDeadlineAt, researchQuestion: options.message, refreshEvidence: options.decision.refreshEvidence, refreshFramesOnly: options.decision.refreshFramesOnly, - frameAccess: options.decision.frameAccess, + frameAccess: frameScope, + storyboardSelections: sheetSelections, timestampedFrameQuestion: options.decision.route === 'inspect_video' && !options.decision.savedOnly ? options.decision.timestampedFrameQuestion : undefined, pinnedVideoId: options.decision.route === 'inspect_video' ? options.decision.videoId : undefined, @@ -666,17 +680,14 @@ async function runResearchAgentWithModelWithinDeadline(options: { const requiredVideos = options.decision.comparisonVideoIds?.length ?? (options.decision.route === 'topic_research' ? options.decision.requiredVideoCount : undefined); const warnings = mergeWarnings(input.warnings.filter(warning => warning.code !== 'RESEARCH_COVERAGE_SHORTFALL'), youtubeAvailabilityWarnings([...toolFailures.values()])); - if (frameScope?.savedTimestampsMs.length) { - // The unified finalizer renders this in prose. Direct and fallback answers retain it as a note. - const { report } = frameActivity(frameScope, options.context.runId, evidence.values()); - if (!input.answer.includes(report)) warnings.unshift({ code: 'FRAME_ACTIVITY', message: report.slice(0, 1000) }); - } + // Application notes lead the list so the warning cap never drops them. + const notes: AgentWarning[] = []; if (visualRequired && !hasVisualObservations()) { - warnings.push({ code: 'VISUAL_EVIDENCE_INCOMPLETE', + notes.push({ code: 'VISUAL_EVIDENCE_INCOMPLETE', message: `The request required visual evidence${visualRequirements.length ? ` (${visualRequirements.join('; ')})` : ''}, but no analyzed visual observations were collected. Visual portions of the answer remain unverified.` }); } if (requiredVideos !== undefined && reviewedVideos.size < requiredVideos) { - warnings.push({ code: 'PARTIAL_EVIDENCE', + notes.push({ code: 'PARTIAL_EVIDENCE', message: `The user requested ${requiredVideos} source videos; usable ${visualRequired ? '' : 'transcript '}evidence was reviewed from ${reviewedVideos.size}.` }); } const artifacts = [...input.artifacts.filter(artifact => artifact.type !== 'research_coverage'), { @@ -685,10 +696,10 @@ async function runResearchAgentWithModelWithinDeadline(options: { }]; if (options.decision.route === 'topic_research' && options.decision.channelId && !playlistListing && ![...evidence.values()].some(packet => packet.kind === 'youtube_channel_videos')) { - warnings.push({ code: 'CHANNEL_INSPECTION_INCOMPLETE', + notes.push({ code: 'CHANNEL_INSPECTION_INCOMPLETE', message: 'The requested channel catalog could not be inspected. Do not treat this response as complete channel research.' }); } - const result = await options.context.finalize(id, withDurationLimitNotice({ ...input, warnings: warnings.slice(0, 50), artifacts }, currentDurationNotice())); + const result = await options.context.finalize(id, withDurationLimitNotice({ ...input, warnings: mergeWarnings(notes, warnings), artifacts }, currentDurationNotice())); finalized = true; return result; }, @@ -743,6 +754,16 @@ async function runResearchAgentWithModelWithinDeadline(options: { for (const id of evidence.keys()) if (!retained.has(id)) evidence.delete(id); return packet; } catch (error) { + // A failed saved-frame read is a reported gap, not work to repeat until the deadline. + if (frameScope && execution.toolName === 'analyze_video_frames') { + for (const version of (execution.input as { assetVersions?: string[] }).assetVersions ?? []) failedFrameVersions.add(version); + } + // A failed capture closes only its own positions; other authorized captures stay required. + if (frameScope && execution.toolName === 'get_video_frames') { + const { videoId, timestampsMs } = execution.input as { videoId?: string; timestampsMs?: number[] }; + // A time fixed before discovery chose the video is closed by a failure on any video. + for (const time of timestampsMs ?? []) { failedCaptures.add(`${videoId}:${time}`); failedCaptures.add(`*:${time}`); } + } toolFailures.set(execution.toolCallId, { toolCallId: execution.toolCallId, toolName: execution.toolName, @@ -759,39 +780,71 @@ async function runResearchAgentWithModelWithinDeadline(options: { }; const visualRequired = visualEvidenceLevel(options.decision) === 'required'; const visualRequirements = options.decision.visualRequirements ?? []; - const frameScope = options.decision.frameAccess; + let savedFramesAtStart: SessionBrief['assets'] | undefined; const hasVisualObservations = () => [...evidence.values()].some(packet => packet.excerpts.length > 0 && packet.artifacts.some(artifact => ['youtube_frame_analysis', 'youtube_storyboard_analysis'].includes(artifact.type))); // Transcript breadth is not completion when the requested facts need images. // Keep the existing time, total-call, step and cost ceilings authoritative. + const discoveryCaptureUnavailable = () => frameScope?.some(entry => !entry.videoId && entry.captureTimestampsMs.length) + && ![...evidence.values()].some(packet => packet.packetId.startsWith(`packet:${options.context.runId}:`) + && packet.sources.some(source => source.videoId && !trackedContext.transcriptSelection?.tooLong?.has(source.videoId) + && !trackedContext.transcriptSelection?.regionRestricted?.has(source.videoId))); const needsVisualWork = () => { const failed = new Set([...toolFailures.values()].map(failure => failure.toolName)); if (frameScope) { - const { saved, captures } = frameActivity(frameScope, options.context.runId, evidence.values()); - const available = options.context.session?.brief().assets ?? []; - const needsSavedAnalysis = saved.some(item => !item.inspected && available.some(asset => asset.kind === 'frame' - && asset.videoId === frameScope.videoId && asset.details.timestampMs === item.time)); - const needsCaptureAnalysis = options.decision.route === 'inspect_video' && options.decision.timestampedFrameQuestion - && captures.some(item => item.retrieved && !item.inspected); - return (toolNames.includes('analyze_video_frames') && !!options.context.analyzeFrames && !failed.has('analyze_video_frames') - && (needsSavedAnalysis || needsCaptureAnalysis)) - || (toolNames.includes('get_video_frames') && !!options.context.provider.frames && !failed.has('get_video_frames') - && frameExtractionBudget(options.researchDeadlineAt) >= FRAME_EXTRACTION_MIN_MS && captures.some(item => !item.requested)); + const { saved, captures, inspectedVersions, pendingCaptureVersions } = frameActivity(frameScope, options.context.runId, evidence.values()); + // Saved frames predate the run, so read the inventory once rather than on every step. + savedFramesAtStart ??= (options.context.session?.brief().assets ?? []).filter(asset => asset.kind === 'frame' + && frameScope.some(entry => entry.videoId === asset.videoId && entry.savedTimestampsMs.length)); + // Each image is tracked by version: one video's inspection never stands in for another's. + const open = (version: string) => !failedFrameVersions.has(version) && options.context.session?.has?.(version) !== false; + // Older inventory can omit the position the payload holds, so such a frame is tried once. + const needsSavedAnalysis = saved.some(item => !item.inspected && savedFramesAtStart!.some(asset => asset.videoId === item.videoId + && (asset.details.timestampMs === item.time || typeof asset.details.timestampMs !== 'number') + && !inspectedVersions.has(asset.version) && open(asset.version))); + const needsCaptureAnalysis = (visualRequired || (options.decision.route === 'inspect_video' && !!options.decision.timestampedFrameQuestion)) + && pendingCaptureVersions.some(open); + + return (toolNames.includes('analyze_video_frames') && !!options.context.analyzeFrames && (needsSavedAnalysis || needsCaptureAnalysis)) + || (toolNames.includes('get_video_frames') && !!options.context.provider.frames + && frameExtractionBudget(options.researchDeadlineAt) >= FRAME_EXTRACTION_MIN_MS + && !discoveryCaptureUnavailable() && captures.some(item => !item.requested && !failedCaptures.has(`${item.videoId}:${item.time}`))); + } + if (sheetSelections) { + // A completed or failed sheet on one subject cannot stand in for another subject. + const own = [...evidence.values()].filter(packet => packet.kind === 'youtube_storyboard' + && packet.packetId.startsWith(`packet:${options.context.runId}:`)).flatMap(packet => packet.artifacts); + return sheetSelections.some(({ videoId, timestampsMs }) => { + const failedForVideo = (name: string) => [...toolFailures.values()].some(failure => failure.toolName === name + && (!failure.videoId || failure.videoId === videoId)); + return timestampsMs.some(time => { + const retrieved = own.find(artifact => artifact.type === 'youtube_storyboard_retrieval' && artifact.data.videoId === videoId + && Array.isArray(artifact.data.requestedTimestampsMs) && artifact.data.requestedTimestampsMs.includes(time)); + const inspected = own.some(artifact => artifact.type === 'youtube_storyboard_analysis' && artifact.data.videoId === videoId + && Array.isArray(artifact.data.sampledRanges) && artifact.data.sampledRanges.some(range => + range && typeof range.startMs === 'number' && typeof range.endMs === 'number' + && range.startMs <= time && time < range.endMs + Number(artifact.data.intervalMs))); + if (inspected) return false; + if (retrieved) return visualRequired && Array.isArray(retrieved.data.analysisAssetVersions) && retrieved.data.analysisAssetVersions.length > 0 + && toolNames.includes('analyze_video_storyboard') && !!options.context.analyzeStoryboard + && !failedForVideo('analyze_video_storyboard') && canAnalyzeStoryboard(options.researchDeadlineAt); + return toolNames.includes('get_video_storyboard') && !!options.context.provider.storyboard && !failedForVideo('get_video_storyboard') + && storyboardRetrievalBudget(options.researchDeadlineAt) >= STORYBOARD_RETRIEVAL_MIN_MS; + }); + }); } if (!visualRequired || hasVisualObservations()) return false; const savedFrames = [...evidence.values()].some(packet => packet.kind === 'youtube_frames' && packet.assetVersions?.length); - const framesPossible = toolNames.includes('get_video_frames') && toolNames.includes('analyze_video_frames') - && !!options.context.provider.frames && !!options.context.analyzeFrames - && !failed.has('get_video_frames') && !failed.has('analyze_video_frames') - && (savedFrames || frameExtractionBudget(options.researchDeadlineAt) >= FRAME_EXTRACTION_MIN_MS); + const framesPossible = toolNames.includes('analyze_video_frames') && !!options.context.analyzeFrames && !failed.has('analyze_video_frames') + && (savedFrames || (toolNames.includes('get_video_frames') && !!options.context.provider.frames + && !failed.has('get_video_frames') && frameExtractionBudget(options.researchDeadlineAt) >= FRAME_EXTRACTION_MIN_MS)); const savedStoryboards = [...evidence.values()].some(packet => packet.kind === 'youtube_storyboard' && packet.artifacts.some(artifact => artifact.type === 'youtube_storyboard_retrieval' && Number(artifact.data.sampledFrames) > 0)) || options.context.session?.brief().assets.some(asset => asset.kind === 'storyboard_sheet') === true; - const storyboardPossible = toolNames.includes('get_video_storyboard') && toolNames.includes('analyze_video_storyboard') - && !!options.context.provider.storyboard && !!options.context.analyzeStoryboard + const storyboardPossible = toolNames.includes('analyze_video_storyboard') && !!options.context.analyzeStoryboard && !failed.has('analyze_video_storyboard') && canAnalyzeStoryboard(options.researchDeadlineAt) - && (savedStoryboards || (!failed.has('get_video_storyboard') + && (savedStoryboards || (toolNames.includes('get_video_storyboard') && !!options.context.provider.storyboard && !failed.has('get_video_storyboard') && storyboardRetrievalBudget(options.researchDeadlineAt) >= STORYBOARD_RETRIEVAL_MIN_MS)); return framesPossible || storyboardPossible; }; @@ -865,7 +918,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { ...(savedEvidenceSkill && !verifySaved ? ['Reuse supplied visual observations. Saved visual analysis tools remain available only for requested visual facts that are still missing; they do not replace required original caption/comment reads.'] : []), ...(!verifySaved ? ['For exact quotations or source corrections, use search_originals: omit assetVersion to find which saved source says it, then read that source by its version. Analyst summaries are not verbatim evidence. Read original passages before completing research.'] : []), ...(options.decision.savedOnly ? ['This run is saved-only. Retrieval and discovery are disabled, including incidental metadata calls. Missing saved assets are gaps, not permission to fetch.'] : []), - ...(options.decision.frameAccess ? [`Exact frame access for this turn: ${JSON.stringify(options.decision.frameAccess)}. Analyze the saved positions by their session asset versions. If saved inspection fails, stop that inspection and report the gap. Independently retrieve the authorized new positions. Earlier observations cannot stand in for a successful re-inspection.`] : []), + ...(frameScope ? [`Exact frame access for this turn: ${JSON.stringify(frameScope)}. Analyze the saved positions by their session asset versions. If saved inspection fails, stop that inspection and report the gap. Independently retrieve the authorized new positions. Earlier observations cannot stand in for a successful re-inspection.`] : []), ...(requestedKinds.length ? [requestedEvidenceGuidance(requestedKinds)] : []), // The skill accompanies exactly the visual tools this model can call. ...(providerToolNames.some(name => VISUAL_TOOL_NAMES.includes(name)) ? [VISUAL_INSPECTION_SKILL] : []), @@ -889,7 +942,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { ...(options.finalizationModel ? { finalizationReserveMs: 1_000 } : {}) } : {}), unavailableTools: () => [ ...(searchUsed || !!options.decision.comparisonVideoIds?.length || (options.decision.route === 'topic_research' && !!options.decision.channelId) ? ['search_youtube'] : []), - ...(frameExtractionBudget(options.researchDeadlineAt) < FRAME_EXTRACTION_MIN_MS ? ['get_video_frames'] : []), + ...((discoveryCaptureUnavailable() || frameExtractionBudget(options.researchDeadlineAt) < FRAME_EXTRACTION_MIN_MS) ? ['get_video_frames'] : []), ...(storyboardRetrievalBudget(options.researchDeadlineAt) < STORYBOARD_RETRIEVAL_MIN_MS ? ['get_video_storyboard'] : []), ...(!canAnalyzeStoryboard(options.researchDeadlineAt) ? ['analyze_video_storyboard'] : []), ...(needsVisualWork() ? [completionTool] : []), @@ -1097,8 +1150,7 @@ async function runUnifiedFinalizer(options: { .filter(packet => packet.sources.some(source => source.videoId && subjects.has(source.videoId))); const inspectionRecords = inspected.length ? inspectionRecordsForModel(inspected, options.context.evidenceRunRecords?.(inspected.map(packet => packet.packetId))) : undefined; - const frameReport = 'frameAccess' in options.decision && options.decision.frameAccess?.savedTimestampsMs.length - ? frameActivity(options.decision.frameAccess, options.context.runId, options.evidence).report : undefined; + const frameReport = isFrameTurn(options.decision) || options.evidence.some(packet => packet.kind === 'youtube_frames'); const baseOutputSchema = conversational ? conversationalFinalizationOutputSchema : intent === 'context_answer' ? contextFinalizationOutputSchema : finalizationOutputSchema; const numberedItemCount = 'numberedItemCount' in options.decision ? options.decision.numberedItemCount : undefined; @@ -1201,7 +1253,7 @@ async function runUnifiedFinalizer(options: { 'Keep JSON compact. Use short ref_N citations rather than full evidence IDs. For specific-video comparisons cite every subject, or explicitly state the missing side and add ANSWER_SCOPE_SHORTFALL. If contextIncomplete is true, do not claim exhaustive coverage unless the supplied evidence establishes it.', EVIDENCE_DELIVERY_GUIDANCE, INSPECTION_RECORD_GUIDANCE, - ...(frameReport ? ['The application will insert frameActivityReport into the answer. Write the findings and their limitations. Use only the inspections confirmed in that report as current visual evidence. If a saved inspection is missing, disclose what could not be verified. Leave the account of tool actions to the application.'] : []), + ...(frameReport ? ['Distinguish earlier visual observations from inspections made in this turn. Explain evidence gaps. The application appends frame activity and the provenance of earlier frame citations from the recorded operations.'] : []), 'Recovery has a limited token budget. Preserve the requested count where evidence permits by shortening each item before reducing the count. If scope remains incomplete, state the shortfall and add ANSWER_SCOPE_SHORTFALL. Do not pad or invent findings.', 'State important evidence gaps plainly. Do not claim that a failed provider operation succeeded.', 'For list evidence such as playlists, take counts from collection: entriesShown are the entries supplied here, and collection.coverage states what was not retained, not shown, or not stated by the provider. Never report fewer available entries than are shown. Describe unshown or unretained entries as a context limit, not as absent from the source. When the answer cites a playlist entry, or reviewed transcript or visual content for a listed video, the application appends a playlist coverage note stating the provider total, the 12-video listing limit and which videos had content reviewed; do not contradict it. Otherwise no note is appended, so state any material playlist coverage limit yourself. Never present listing details as reviewed video content. evidenceOmittedFromContext counts saved list packets that did not fit at all; notListed of them are counted but not named.', @@ -1226,7 +1278,6 @@ async function runUnifiedFinalizer(options: { providerFailures: groupedToolFailures(options.toolFailures).map(failure => failure.durationLimit ? { ...failure, message: 'VIDEO_TOO_LONG: The video exceeded the configured Agent duration limit.' } : failure), inspectionRecords, - frameActivityReport: frameReport, validationFeedback: feedback, })}, ...(usesSavedEvidenceSkill(options.decision) ? (options.conversationHistory ?? []).flatMap(turn => [ @@ -1324,7 +1375,6 @@ async function runUnifiedFinalizer(options: { for (const block of output.blocks) { block.evidenceIds = block.evidenceIds.map(id => prepared.fullIds.get(id) ?? id); } - if (frameReport && output.blocks.length) output.blocks[0]!.text += ` ${frameReport}`; validationStage = 'answer_notes'; // Judge each block by every reference it will render, declared or inline. const references = output.blocks.map(block => blockReferences(block, prepared.fullIds)); diff --git a/platform/src/agents/research/skills/visual-inspection.generated.ts b/platform/src/agents/research/skills/visual-inspection.generated.ts index cfe466c3..a22bb272 100644 --- a/platform/src/agents/research/skills/visual-inspection.generated.ts +++ b/platform/src/agents/research/skills/visual-inspection.generated.ts @@ -1,2 +1,2 @@ // Generated from visual-inspection/SKILL.md. Run npx tsx scripts/generate-agent-skills.ts to update. -export const VISUAL_INSPECTION_SKILL = "---\nname: visual-inspection\ndescription: Locate and inspect YouTube images for visible details, resolve conflicts with captions or metadata, and explain recorded frame choices on follow-up.\n---\n\n# Visual inspection\n\nAnswer visible questions from analyzed images. Metadata and captions locate moments and identify speakers; they do not establish clothing, scenery or readable on-screen text.\n\n## Locate\n\nReuse adequate saved analysis first. Choose how to locate by what the question needs:\n\n- The look of a whole video, such as camera style, colors, set or diagram style: a small spread of storyboard sheets. Captions add nothing.\n- A visual moment with no spoken cue: scan storyboard sheets with an analysis question. One call can include many sheets.\n- A moment defined by topic or speech, such as the slide for a named concept or a spoken introduction: use title, description or chapter timestamps first, then captions. Retrieve the transcript without an analysis question and find the passage with search_originals. A transcript analyst never answers visual questions.\n\nA chapter is a timestamped section listed by the publisher, not an inferred scene boundary. In topic research, fetch captions only for videos you will inspect. The last provider calls are kept for get_video_frames, so do not spend them on other retrieval. Respect the user's chosen inspection method.\n\nFor a presenter question, an introduction is a lead, not proof the presenter is on screen. A storyboard overview can locate her actual appearance. Prefer a few frames that answer the question over a six-frame quota or unexplained samples across the whole video. Questions about changes across the video need wider coverage.\n\n## Inspect\n\nUse get_video_frames for exact seek positions when storyboard images are too small, text is ambiguous, or captions conflict with visible labels. If storyboard retrieval or analysis fails, switch to get_video_frames with an analysis question while time remains; never repeat the failed storyboard request unchanged. Supply selections with one basis category and the locating evidenceIds for each timestamp. user_requested and exploratory selections cite no excerpts; describe an exploratory sample as exploratory. A basis records the kind of choice, not private reasoning or proof the desired scene exists.\n\nFor new images, supply an analysis question to get_video_frames or get_video_storyboard so the images are analyzed while they are saved. Retrieval without analysis provides no observations; a RETRIEVAL_ANALYSIS_FAILED warning means the images were saved but still need analyze_video_frames or analyze_video_storyboard. Inspect only analysisAssetVersions from storyboard results, never the manifest. The runtime supplies title, full description, channel and question to image analysts. Analyze saved images directly for new questions rather than fetching the same images again.\n\nCheck whether each requested detail is visible and readable. A transition or irrelevant shot does not answer a clothing question. When needed, try a nearby timestamp informed by the overview, within the remaining time and provider budget. When names conflict, inspect a readable on-screen label at an exact frame and distinguish the label from noisy captions. A name in metadata is context, not visual identification.\n\n## Finish and explain\n\nPresent the visible findings with citations and material uncertainty. Explain inspection steps only when the user explicitly asks about the workflow. Finishing transcript research does not finish a visual question. Before finishing, check every requested subject and attribute against analyzed images; stop when they are covered or a real retrieval, analysis or budget limit prevents it, not merely because captions omit visual details. Keep image quality and missing-image warnings. Cite each frame-specific observation at its own timestamp. Describe shared observations only over their supported sampled frames. State missing details and actual retrieval or analysis limits. Sampled stills do not establish continuous appearance or motion.\n\nFor a follow-up about how inspection happened, use frameCoverage, selection bases and delivery records. Separate requested or retrieved positions from analyzed frames and storyboard samples. A storyboard analyst receives every tile in the selected sheets; its reported finding times may cover only a subset. Sheet spread selection does not choose 3:20 over 3:30 within the same supplied sheet. A recorded basis can explain why a timestamp was chosen, such as a spoken introduction; what a cited excerpt says is evidence only when that excerpt is supplied. Without a basis, say the reason was not recorded. Do not reconstruct a past rationale from what now seems sensible. Treat user examples such as “why 3:20 instead of 3:30?” as alternatives, not captured frames. Claim transcript reading or fresh image analysis only when the supplied records establish it. Historical evidence and memory alone do not prove a current operation.\n"; +export const VISUAL_INSPECTION_SKILL = "---\nname: visual-inspection\ndescription: Locate and inspect YouTube images for visible details, resolve conflicts with captions or metadata, and explain recorded frame choices on follow-up.\n---\n\n# Visual inspection\n\nAnswer visible questions from analyzed images. Metadata and captions locate moments and identify speakers; they do not establish clothing, scenery or readable on-screen text.\n\n## Locate\n\nUse only the visual tools offered on this turn and the videos and positions the user permitted. If retrieval is unavailable, inspect permitted saved images or report the missing evidence. Reuse adequate saved analysis first. Choose how to locate by what the question needs:\n\n- The look of a whole video, such as camera style, colors, set or diagram style: use a small spread of storyboard sheets when that retrieval is available.\n- A visual moment with no spoken cue: scan permitted storyboard sheets with an analysis question when available. One call can include many sheets.\n- A moment defined by topic or speech, such as the slide for a named concept or a spoken introduction: use title, description or chapter timestamps first, then captions. Retrieve the transcript without an analysis question and find the passage with search_originals. A transcript analyst never answers visual questions.\n\nA chapter is a timestamped section listed by the publisher, not an inferred scene boundary. In topic research, fetch captions only for videos you will inspect. When exact capture is available, keep the reserved provider calls for it. Respect the user's chosen inspection method.\n\nFor a presenter question, an introduction is a lead, not proof the presenter is on screen. A storyboard overview can locate her actual appearance. Prefer a few frames that answer the question over a six-frame quota or unexplained samples across the whole video. Questions about changes across the video need wider coverage.\n\n## Inspect\n\nWhen get_video_frames is available, use it for permitted exact seek positions if storyboard images are too small, text is ambiguous, or captions conflict with visible labels. If a sheet fails, use an authorized exact capture with an analysis question when available. Otherwise record the gap. Supply selections with one basis category and the locating evidenceIds for each timestamp. user_requested and exploratory selections cite no excerpts; describe an exploratory sample as exploratory. A basis records the kind of choice, not private reasoning or proof the desired scene exists.\n\nWhen a retrieval tool is offered, include the analysis question so the analyst sees the images as they are saved. Retrieval without analysis provides no observations; a RETRIEVAL_ANALYSIS_FAILED warning means the images were saved but still need analyze_video_frames or analyze_video_storyboard. Inspect only analysisAssetVersions from storyboard results, never the manifest. The runtime supplies title, full description, channel and question to image analysts. Analyze saved images directly for new questions rather than fetching the same images again.\n\nCheck whether each requested detail is visible and readable. A transition or irrelevant shot does not answer a clothing question. When needed, try a permitted nearby timestamp informed by the overview, within the remaining time and provider budget. When names conflict, inspect a readable on-screen label at an exact frame and distinguish the label from noisy captions. A name in metadata is context, not visual identification.\n\n## Finish and explain\n\nPresent the visible findings with citations and material uncertainty. Explain inspection steps only when the user explicitly asks about the workflow. Finishing transcript research does not finish a visual question. Before finishing, check every requested subject and attribute against analyzed images; stop when they are covered or a real retrieval, analysis or budget limit prevents it, not merely because captions omit visual details. Keep image quality and missing-image warnings. Cite each frame-specific observation at its own timestamp. Describe shared observations only over their supported sampled frames. State missing details and actual retrieval or analysis limits. Sampled stills do not establish continuous appearance or motion.\n\nFor a follow-up about how inspection happened, use frameCoverage, selection bases and delivery records. Separate requested or retrieved positions from analyzed frames and storyboard samples. A storyboard analyst receives every tile in the selected sheets; its reported finding times may cover only a subset. Sheet spread selection does not choose 3:20 over 3:30 within the same supplied sheet. A recorded basis can explain why a timestamp was chosen, such as a spoken introduction; what a cited excerpt says is evidence only when that excerpt is supplied. Without a basis, say the reason was not recorded. Do not reconstruct a past rationale from what now seems sensible. Treat user examples such as “why 3:20 instead of 3:30?” as alternatives, not captured frames. Claim transcript reading or fresh image analysis only when the supplied records establish it. Historical evidence and memory alone do not prove a current operation.\n"; diff --git a/platform/src/agents/research/skills/visual-inspection/SKILL.md b/platform/src/agents/research/skills/visual-inspection/SKILL.md index aecb181e..ee2e8700 100644 --- a/platform/src/agents/research/skills/visual-inspection/SKILL.md +++ b/platform/src/agents/research/skills/visual-inspection/SKILL.md @@ -9,23 +9,23 @@ Answer visible questions from analyzed images. Metadata and captions locate mome ## Locate -Reuse adequate saved analysis first. Choose how to locate by what the question needs: +Use only the visual tools offered on this turn and the videos and positions the user permitted. If retrieval is unavailable, inspect permitted saved images or report the missing evidence. Reuse adequate saved analysis first. Choose how to locate by what the question needs: -- The look of a whole video, such as camera style, colors, set or diagram style: a small spread of storyboard sheets. Captions add nothing. -- A visual moment with no spoken cue: scan storyboard sheets with an analysis question. One call can include many sheets. +- The look of a whole video, such as camera style, colors, set or diagram style: use a small spread of storyboard sheets when that retrieval is available. +- A visual moment with no spoken cue: scan permitted storyboard sheets with an analysis question when available. One call can include many sheets. - A moment defined by topic or speech, such as the slide for a named concept or a spoken introduction: use title, description or chapter timestamps first, then captions. Retrieve the transcript without an analysis question and find the passage with search_originals. A transcript analyst never answers visual questions. -A chapter is a timestamped section listed by the publisher, not an inferred scene boundary. In topic research, fetch captions only for videos you will inspect. The last provider calls are kept for get_video_frames, so do not spend them on other retrieval. Respect the user's chosen inspection method. +A chapter is a timestamped section listed by the publisher, not an inferred scene boundary. In topic research, fetch captions only for videos you will inspect. When exact capture is available, keep the reserved provider calls for it. Respect the user's chosen inspection method. For a presenter question, an introduction is a lead, not proof the presenter is on screen. A storyboard overview can locate her actual appearance. Prefer a few frames that answer the question over a six-frame quota or unexplained samples across the whole video. Questions about changes across the video need wider coverage. ## Inspect -Use get_video_frames for exact seek positions when storyboard images are too small, text is ambiguous, or captions conflict with visible labels. If storyboard retrieval or analysis fails, switch to get_video_frames with an analysis question while time remains; never repeat the failed storyboard request unchanged. Supply selections with one basis category and the locating evidenceIds for each timestamp. user_requested and exploratory selections cite no excerpts; describe an exploratory sample as exploratory. A basis records the kind of choice, not private reasoning or proof the desired scene exists. +When get_video_frames is available, use it for permitted exact seek positions if storyboard images are too small, text is ambiguous, or captions conflict with visible labels. If a sheet fails, use an authorized exact capture with an analysis question when available. Otherwise record the gap. Supply selections with one basis category and the locating evidenceIds for each timestamp. user_requested and exploratory selections cite no excerpts; describe an exploratory sample as exploratory. A basis records the kind of choice, not private reasoning or proof the desired scene exists. -For new images, supply an analysis question to get_video_frames or get_video_storyboard so the images are analyzed while they are saved. Retrieval without analysis provides no observations; a RETRIEVAL_ANALYSIS_FAILED warning means the images were saved but still need analyze_video_frames or analyze_video_storyboard. Inspect only analysisAssetVersions from storyboard results, never the manifest. The runtime supplies title, full description, channel and question to image analysts. Analyze saved images directly for new questions rather than fetching the same images again. +When a retrieval tool is offered, include the analysis question so the analyst sees the images as they are saved. Retrieval without analysis provides no observations; a RETRIEVAL_ANALYSIS_FAILED warning means the images were saved but still need analyze_video_frames or analyze_video_storyboard. Inspect only analysisAssetVersions from storyboard results, never the manifest. The runtime supplies title, full description, channel and question to image analysts. Analyze saved images directly for new questions rather than fetching the same images again. -Check whether each requested detail is visible and readable. A transition or irrelevant shot does not answer a clothing question. When needed, try a nearby timestamp informed by the overview, within the remaining time and provider budget. When names conflict, inspect a readable on-screen label at an exact frame and distinguish the label from noisy captions. A name in metadata is context, not visual identification. +Check whether each requested detail is visible and readable. A transition or irrelevant shot does not answer a clothing question. When needed, try a permitted nearby timestamp informed by the overview, within the remaining time and provider budget. When names conflict, inspect a readable on-screen label at an exact frame and distinguish the label from noisy captions. A name in metadata is context, not visual identification. ## Finish and explain diff --git a/platform/src/agents/research/timestamp-expression.ts b/platform/src/agents/research/timestamp-expression.ts new file mode 100644 index 00000000..d88f02a4 --- /dev/null +++ b/platform/src/agents/research/timestamp-expression.ts @@ -0,0 +1,65 @@ +// An expression occupies one span in the request. Its component numbers are never separate grants. +export interface TimestampExpression { + start: number; + end: number; + milliseconds: number; + ambiguous: boolean; +} + +const unitScale = new Map(); +for (const [scale, names] of [ + [3600000, ['hours', 'hour', 'hrs', 'hr', 'h']], + [60000, ['minutes', 'minute', 'mins', 'min', 'm']], + [1000, ['seconds', 'second', 'secs', 'sec', 's']], + [1, ['milliseconds', 'millisecond', 'ms']], +] as const) { + for (const name of names) unitScale.set(name, scale); +} +const component = String.raw`(\d+(?:\.\d+)?)\s*-?\s*(${[...unitScale.keys()].join('|')})(?![a-z])`; +const clock = String.raw`(?:\d+:)?\d{1,2}:\d{2}s?\b`; + +export function timestampExpressions(input: string): TimestampExpression[] { + const start = new RegExp(String.raw`(? seconds * 60 + field, 0) * 1000; + } else { + let scale = unitScale.get(first[2]!.toLowerCase())!; + milliseconds = Number(first[1]) * scale; + for (;;) { + continuation.lastIndex = start.lastIndex; + const next = continuation.exec(input); + if (!next) break; + const nextScale = unitScale.get(next[3]!.toLowerCase())!; + // "30 seconds and 45 seconds" lists two positions. Descending units form one + // position. A comma between descending units could also introduce a list. + if (nextScale >= scale && /,|and/i.test(next[1]!)) break; + ambiguous ||= nextScale >= scale || next[1]!.includes(','); + milliseconds += Number(next[2]) * nextScale; + scale = nextScale; + start.lastIndex = continuation.lastIndex; + } + } + milliseconds = Math.round(milliseconds); + result.push({ start: first.index, end: start.lastIndex, milliseconds, + ambiguous: ambiguous || !Number.isSafeInteger(milliseconds) }); + } + return result; +} + +/** URL seek values also allow bare seconds. The entire parameter must be a single timestamp. */ +export function seekMilliseconds(value: string): number | undefined { + if (/^\d+(?:\.\d+)?$/.test(value)) { + const time = Math.round(Number(value) * 1000); + return Number.isSafeInteger(time) ? time : undefined; + } + // A seek parameter uses compact h/m/s or a clock. Prose-only forms have no URL meaning. + if (!/^(?:(?:\d+h)?(?:\d+m)?(?:\d+(?:\.\d+)?s)?|(?:\d+:)?\d{1,2}:\d{2})$/i.test(value)) return; + const [time] = timestampExpressions(value); + if (time?.start === 0 && time.end === value.length && !time.ambiguous) return time.milliseconds; +} diff --git a/platform/src/agents/research/youtube-references.ts b/platform/src/agents/research/youtube-references.ts new file mode 100644 index 00000000..99434d1e --- /dev/null +++ b/platform/src/agents/research/youtube-references.ts @@ -0,0 +1,51 @@ +import { seekMilliseconds } from './timestamp-expression'; + +const VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{11}$/; + +// Consume the whole URL before checking its host. Embedded links and v= parameters belong to that URL. +export const urlCandidates = /(?:https?:\/\/|(?"']+/gi; +function youTubeUrl(candidate: string): URL | undefined { + const normalized = candidate.replace(/[\])},.!?;:'"]+$/g, ''); + try { + const url = new URL(/^https?:\/\//i.test(normalized) ? normalized : `https://${normalized}`); + if (['youtube.com', 'www.youtube.com', 'm.youtube.com', 'music.youtube.com', 'youtu.be', 'www.youtu.be', 'm.youtu.be'].includes(url.hostname)) return url; + } catch { /* Non-URL text can match the candidate pattern. */ } +} + +export function* youTubeUrls(message: string): Generator { + for (const candidate of message.match(urlCandidates) ?? []) { + const url = youTubeUrl(candidate); + if (url) yield url; + } +} + +function videoIdsFromUrl(url: URL): string[] { + const [kind, id] = url.pathname.split('/').filter(Boolean); + const candidates = url.hostname.endsWith('youtu.be') ? [kind] + : [url.searchParams.get('v'), ...(['shorts', 'live', 'embed'].includes(kind ?? '') ? [id] : [])]; + return candidates.filter((value): value is string => !!value && VIDEO_ID_PATTERN.test(value)); +} + +/** Seek parameters belong to their linked video. Numeric values are seconds. */ +export function linkedSeek(candidate: string): { videoIds: string[]; time: number } | undefined { + const url = youTubeUrl(candidate); + if (!url) return; + const videoIds = videoIdsFromUrl(url); + const value = url.searchParams.get('t') ?? url.searchParams.get('start') ?? new URLSearchParams(url.hash.slice(1)).get('t'); + if (!videoIds.length || !value) return; + const time = seekMilliseconds(value); + if (time !== undefined) return { videoIds, time }; +} + +export function extractYouTubeVideoIds(message: string): string[] { + const ids = new Set(); + for (const url of youTubeUrls(message)) { + for (const videoId of videoIdsFromUrl(url)) ids.add(videoId); + } + + const labelledIdPattern = /(?:^|[\s(])(?:video(?:\s+id)?|v)\s*[:=]\s*([A-Za-z0-9_-]{11})(?=$|\s|[),.!?;])/gi; + for (const match of message.replace(urlCandidates, ' ').matchAll(labelledIdPattern)) ids.add(match[1]!); + const trimmed = message.trim(); + if (VIDEO_ID_PATTERN.test(trimmed)) ids.add(trimmed); + return [...ids]; +} diff --git a/platform/src/agents/runtime/frame-access.ts b/platform/src/agents/runtime/frame-access.ts index 2e4a8956..b59497e6 100644 --- a/platform/src/agents/runtime/frame-access.ts +++ b/platform/src/agents/runtime/frame-access.ts @@ -1,37 +1,156 @@ -import type { EvidencePacket, FrameAccess } from '../contracts'; +import type { CapabilityRouteDecision, EvidencePacket, FrameAccess } from '../contracts'; import { clockTime } from './clock-time'; import { frameCoverageSchema } from './model-evidence'; /** A requested position, a returned image and an analyst result are different facts. * Only packets produced by this run can establish that it did the work. */ export function frameActivity(scope: FrameAccess, runId: string, evidence: Iterable) { - const inspected = new Set(); - const retrieved = new Set(); - const requested = new Set(); - for (const packet of evidence) { - if (!packet.packetId.startsWith(`packet:${runId}:`) || packet.kind !== 'youtube_frames' - || !packet.sources.some(source => source.videoId === scope.videoId)) continue; - // A partial inline analysis may cover fewer positions than its paired retrieval. + const packets = [...evidence]; + const own = packets.filter(packet => packet.packetId.startsWith(`packet:${runId}:`) && packet.kind === 'youtube_frames'); + // Versions are tracked separately from positions: inspecting one video's frame + // must never count as inspecting another video's frame at the same time. + const inspectedVersions = new Set(); + const retrievedVersions = new Set(); + const observed = new Map; retrieved: Set; inspected: Set }>(); + for (const packet of own) { for (const artifact of packet.artifacts) { if (artifact.type !== 'youtube_frame_analysis' && artifact.type !== 'youtube_frame_retrieval') continue; const coverage = frameCoverageSchema.safeParse(artifact.data); if (!coverage.success) continue; const analyzing = artifact.type === 'youtube_frame_analysis'; - if (!analyzing) for (const time of coverage.data.requestedTimestampsMs) requested.add(time); - for (const frame of coverage.data.frames) (analyzing ? inspected : retrieved).add(frame.timestampMs); + for (const version of packet.assetVersions ?? []) (analyzing ? inspectedVersions : retrievedVersions).add(version); + // A partial inline analysis may cover fewer positions than its paired retrieval. + for (const videoId of new Set(packet.sources.flatMap(source => source.videoId ? [source.videoId] : []))) { + const sets = observed.get(videoId) ?? { requested: new Set(), retrieved: new Set(), inspected: new Set() }; + observed.set(videoId, sets); + if (!analyzing) for (const time of coverage.data.requestedTimestampsMs) sets.requested.add(time); + for (const frame of coverage.data.frames) (analyzing ? sets.inspected : sets.retrieved).add(frame.timestampMs); + } } } - const saved = scope.savedTimestampsMs.map(time => ({ time, inspected: inspected.has(time) })); - const captures = scope.captureTimestampsMs.map(time => ({ - time, requested: requested.has(time), retrieved: retrieved.has(time), inspected: inspected.has(time), - })); - const report = [ - ...saved.map(({ time, inspected }) => inspected ? `I re-inspected the saved ${clockTime(time)} frame.` - : `I could not re-inspect the saved ${clockTime(time)} frame in this turn.`), - ...captures.map(({ time, requested, retrieved, inspected }) => retrieved - ? `I retrieved the ${clockTime(time)} frame${inspected ? ' and inspected it' : ''}.` - : requested ? `I requested ${clockTime(time)}, but no frame was returned.` - : `I have no confirmed retrieval of the ${clockTime(time)} frame in this turn.`), - ].join(' '); - return { saved, captures, report }; + const has = (videoId: string, kind: 'requested' | 'retrieved' | 'inspected', time: number) => !!observed.get(videoId)?.[kind].has(time); + const titleOf = (videoId: string) => packets.flatMap(packet => packet.sources).find(source => source.videoId === videoId && source.title)?.title ?? videoId; + const saved: { videoId: string; time: number; inspected: boolean }[] = []; + const captures: { videoId: string; time: number; requested: boolean; retrieved: boolean; inspected: boolean }[] = []; + const sentences: string[] = []; + for (const { videoId, savedTimestampsMs, captureTimestampsMs } of scope) { + if (!videoId) { + // Times fixed before discovery chose the videos: report each video that was captured there. + for (const time of captureTimestampsMs) { + const videos = [...observed.keys()].filter(id => has(id, 'retrieved', time)); + const requested = [...observed.keys()].some(id => has(id, 'requested', time)); + captures.push({ videoId: '*', time, requested, retrieved: videos.length > 0, inspected: videos.some(id => has(id, 'inspected', time)) }); + sentences.push(...(videos.length ? videos.map(id => `I retrieved the ${clockTime(time)} frame of "${titleOf(id)}"${has(id, 'inspected', time) ? ' and inspected it' : ''}.`) + : [requested ? `I requested ${clockTime(time)} frames, but none was returned.` : `I have no confirmed retrieval of a ${clockTime(time)} frame in this turn.`])); + } + continue; + } + // With several videos, each position names its video so a gap cannot be read as another video's. + const frame = (time: number) => `${clockTime(time)} frame${scope.length > 1 ? ` of "${titleOf(videoId)}"` : ''}`; + for (const time of savedTimestampsMs) { + saved.push({ videoId, time, inspected: has(videoId, 'inspected', time) }); + sentences.push(has(videoId, 'inspected', time) ? `I re-inspected the saved ${frame(time)}.` + : `I could not re-inspect the saved ${frame(time)} in this turn.`); + } + for (const time of captureTimestampsMs) { + const item = { videoId, time, requested: has(videoId, 'requested', time), retrieved: has(videoId, 'retrieved', time), inspected: has(videoId, 'inspected', time) }; + captures.push(item); + sentences.push(item.retrieved ? `I retrieved the ${frame(time)}${item.inspected ? ' and inspected it' : ''}.` + : item.requested ? `I requested the ${frame(time)}, but no frame was returned.` + : `I have no confirmed retrieval of the ${frame(time)} in this turn.`); + } + } + return { saved, captures, inspectedVersions, + pendingCaptureVersions: [...retrievedVersions].filter(version => !inspectedVersions.has(version)), report: sentences.join(' ') }; +} + +/** The exact frame positions a route grants, if it fixed them. */ +export function frameList(decision: CapabilityRouteDecision): FrameAccess | undefined { + return 'frameAccess' in decision && Array.isArray(decision.frameAccess) ? decision.frameAccess : undefined; +} + +export function imagePermissions(decision: Extract) { + const access = decision.frameAccess; + const frameScope = frameList(decision); + const object = access && typeof access === 'object' && !Array.isArray(access) ? access : undefined; + const sheetSelections = object && ('storyboardSelections' in object ? object.storyboardSelections + : decision.route === 'inspect_video' ? [{ videoId: decision.videoId, timestampsMs: object.storyboardTimestampsMs }] : undefined); + const retrieval = !decision.savedOnly && decision.useStoryboard !== false; + return { frameScope, sheetSelections, + canCapture: retrieval && (access === 'agent_selected' || !!frameScope?.some(entry => entry.captureTimestampsMs.length)), + canRetrieveSheets: retrieval && (access === 'storyboard_only' || access === 'agent_selected' || !!sheetSelections) }; +} + +/** A turn that asked for frame work, whose answer carries the application's account of it. */ +export function isFrameTurn(decision: CapabilityRouteDecision): boolean { + if (decision.route !== 'inspect_video' && decision.route !== 'topic_research') return false; + const { frameScope, canCapture, canRetrieveSheets } = imagePermissions(decision); + return !!frameScope || canCapture || (!!decision.refreshFramesOnly && !canRetrieveSheets) + || (decision.route === 'inspect_video' && !!decision.timestampedFrameQuestion); +} + +/** A saved frame as the session inventory records it. */ +export interface InventoryFrame { version: string; videoId: string; timestampMs?: number; collectedAt: number } + +/** + * The application's account of a frame turn, built from what the route requested and what + * this run recorded, never from the answer's wording or citations: + * - each listed position: re-inspected, retrieved, inspected, or a gap; + * - saved frames re-inspected outside the list, or that none was; + * - cited observations whose exact image this turn did not inspect again. + * An image is saved when the session stored it before this run started, however this run reached it. + */ +export function frameTurnNotes(options: { + runId: string; startedAt: number; access?: FrameAccess; requestedSaved?: boolean; inventory: readonly InventoryFrame[]; packets: readonly EvidencePacket[]; + cited: ReadonlyArray<{ packet: EvidencePacket; startMs: number; videoId?: string; title?: string }>; +}): string[] { + const { runId, access, inventory, packets } = options; + const own = packets.filter(packet => packet.packetId.startsWith(`packet:${runId}:`) && packet.kind === 'youtube_frames'); + const analyzed = new Set(own.filter(packet => packet.artifacts.some(artifact => artifact.type === 'youtube_frame_analysis')) + .flatMap(packet => packet.assetVersions ?? [])); + const savedBefore = inventory.filter(frame => frame.collectedAt < options.startedAt); + // Where each image sits: from the inventory, or from any packet that holds exactly one image. + const positions = new Map(inventory.flatMap(frame => frame.timestampMs === undefined ? [] : [[frame.version, `${frame.videoId}:${frame.timestampMs}`] as const])); + for (const packet of packets) { + const coverage = packet.kind === 'youtube_frames' && packet.assetVersions?.length === 1 + ? packet.artifacts.map(artifact => frameCoverageSchema.safeParse(artifact.data)).find(result => result.success && result.data.frames.length === 1) : undefined; + const videoId = packet.sources.find(source => source.videoId)?.videoId; + if (coverage?.success && videoId && !positions.has(packet.assetVersions![0]!)) + positions.set(packet.assetVersions![0]!, `${videoId}:${coverage.data.frames[0]!.timestampMs}`); + } + const title = (videoId: string) => packets.flatMap(packet => packet.sources).find(source => source.videoId === videoId && source.title)?.title; + const named = (videoId: string, time: number) => `${clockTime(time)} frame${title(videoId) ? ` of "${title(videoId)}"` : ''}`; + const notes: string[] = []; + if (access?.length) notes.push(frameActivity(access, runId, packets).report); + + const listed = new Set(access?.flatMap(entry => entry.videoId + ? [...entry.savedTimestampsMs, ...entry.captureTimestampsMs].map(time => `${entry.videoId}:${time}`) : [])); + const reinspected = savedBefore.filter(frame => analyzed.has(frame.version)); + const additional = reinspected.filter(frame => frame.timestampMs === undefined || !listed.has(`${frame.videoId}:${frame.timestampMs}`)); + if (additional.length === 1 && additional[0]!.timestampMs !== undefined) + notes.push(`I re-inspected the saved ${named(additional[0]!.videoId, additional[0]!.timestampMs)}.`); + else if (additional.length) notes.push(`I re-inspected ${additional.length} saved frame${additional.length === 1 ? '' : 's'} in this turn.`); + const plannedSaved = access?.filter(entry => entry.savedTimestampsMs.length) ?? []; + if (options.requestedSaved && !plannedSaved.length && !reinspected.length) + notes.push('No frame saved before this turn was re-inspected in this turn.'); + const unplaced = new Set(savedBefore.filter(frame => frame.timestampMs === undefined && !analyzed.has(frame.version) + && plannedSaved.some(entry => entry.videoId === frame.videoId)).map(frame => frame.videoId)); + for (const videoId of unplaced) notes.push(`Saved frames of "${title(videoId) ?? videoId}" have no recorded position, so this turn could not match them to requested times or re-inspect them.`); + + // Each citation is judged by the image behind it. When the record cannot tie the citation + // to one image, the answer says so rather than guessing from the rest of the batch. + const earlier = new Map(); + for (const { packet, startMs, videoId, title: cited } of options.cited) { + const key = `${videoId}:${startMs}`; + if (packet.packetId.startsWith(`packet:${runId}:`) || packet.kind !== 'youtube_frames' || earlier.has(key) + || !packet.artifacts.some(artifact => artifact.type === 'youtube_frame_analysis')) continue; + const all = packet.assetVersions ?? []; + const exact = all.filter(version => positions.get(version) === key); + const of = cited ? ` of "${cited}"` : ''; + if (exact.length ? exact.some(version => analyzed.has(version)) : all.length > 0 && all.every(version => analyzed.has(version))) continue; + earlier.set(key, !exact.length && all.some(version => analyzed.has(version)) + ? `It is not recorded whether the cited ${clockTime(startMs)} observation${of} was re-inspected in this turn.` + : `The ${clockTime(startMs)} observation${of} comes from an image inspected in an earlier turn, not in this one.`); + } + return [...notes, ...earlier.values()]; } diff --git a/platform/src/agents/runtime/session-evidence.ts b/platform/src/agents/runtime/session-evidence.ts index 603ca3d1..9a32ea4f 100644 --- a/platform/src/agents/runtime/session-evidence.ts +++ b/platform/src/agents/runtime/session-evidence.ts @@ -42,6 +42,13 @@ export interface SessionAsset { current: boolean; details: Record; } +/** Stored details, with a frame's position recovered from its resource key when older rows omitted it. */ +function assetDetails(row: AssetRow): SessionAsset['details'] { + const details = JSON.parse(row.details_json) as SessionAsset['details']; + const position = row.kind === 'frame' && typeof details.timestampMs !== 'number' + ? /^frame:[\w-]{11}:\d+:(\d+)$/.exec(row.resource_key)?.[1] : undefined; + return position === undefined ? details : { ...details, timestampMs: Number(position) }; +} type AssetRow = { version: string; resource_key: string; @@ -202,7 +209,7 @@ export class SessionEvidenceStore implements SessionAccess { videoId: row.video_id, collectedAt: row.created_at, current: current.has(row.version), - details: JSON.parse(row.details_json), + details: assetDetails(row), })), memories: this.sql .exec<{ memory_json: string }>('SELECT memory_json FROM session_memories ORDER BY updated_at DESC') diff --git a/platform/src/agents/runtime/session-provider.ts b/platform/src/agents/runtime/session-provider.ts index 4eaf0191..32d81f02 100644 --- a/platform/src/agents/runtime/session-provider.ts +++ b/platform/src/agents/runtime/session-provider.ts @@ -19,6 +19,7 @@ export function sessionProvider( onRetrieved?: (claim: string, version: string) => void, assetCache?: RunAssetCache, refreshFrames = refresh, + refreshSheets = refresh, ): YouTubeAgentProvider { const refreshed = new Set(); const remember = (result: CachedResult, generation: number) => { @@ -162,7 +163,7 @@ export function sessionProvider( // Sheet indexes are level-specific, so each served resolution has its own manifest. const requested = options.resolution ?? 'standard'; const manifestKey = (resolution: StoryboardResolution) => `storyboard:${id}:manifest${resolution === 'high' ? ':high' : ''}`; - const fresh = rediscovering || (refresh && !refreshed.has(manifestKey(requested))) || !!options.refresh; + const fresh = rediscovering || (refreshSheets && !refreshed.has(manifestKey(requested))) || !!options.refresh; let prefetched: CachedResult | undefined; let loaded = false; const retrieveManifest = (resolution: StoryboardResolution, reload: boolean, load: () => Promise>) => store.retrieve( @@ -216,7 +217,7 @@ export function sessionProvider( if (generation !== store.generation()) throw new Error('Session assets changed during retrieval.'); const key = `storyboard:${id}:${manifestVersion}:${index}`; const hit = - !prefetched && !(refresh && !refreshed.has(key)) && !options.refresh && (await store.lookup(key)); + !prefetched && !(refreshSheets && !refreshed.has(key)) && !options.refresh && (await store.lookup(key)); return { index, hit }; })); for (const { index, hit } of cached) { @@ -233,7 +234,7 @@ export function sessionProvider( const fetched = prefetched ?? await provider.storyboard!( id, undefined, - { sheetIndexes: missing, maxSheets: missing.length, ...(served === 'high' ? { resolution: served } : {}), ...(options.deadlineAt === undefined ? {} : { deadlineAt: options.deadlineAt }), ...(options.signal ? {signal:options.signal} : {}), ...((refresh || options.refresh) ? {refresh:true} : {}) }, + { sheetIndexes: missing, maxSheets: missing.length, ...(served === 'high' ? { resolution: served } : {}), ...(options.deadlineAt === undefined ? {} : { deadlineAt: options.deadlineAt }), ...(options.signal ? {signal:options.signal} : {}), ...((refreshSheets || options.refresh) ? {refresh:true} : {}) }, diagnostic, onReady ? value => { if (sameManifest(value)) onReady(storyboardSchema.parse({ ...value, sheets: [...results.flatMap(result => result.value.sheets), ...value.sheets].sort((a, b) => a.firstFrameIndex - b.firstFrameIndex), diff --git a/platform/test/classifier-route-recovery.test.ts b/platform/test/classifier-route-recovery.test.ts index 993ba207..a06e04e6 100644 --- a/platform/test/classifier-route-recovery.test.ts +++ b/platform/test/classifier-route-recovery.test.ts @@ -173,19 +173,19 @@ describe('classifier last resort', () => { it.each([ ['5d3f5302', 'dQw4w9WgXcQ', 'inspect_videovideoId'], ['794f5142', 'tXcT3OE7G1g', 'inspect_videovideoId'], - ])('recovers run %s from attempt 1 when the repair has corrupted keys and the fallback errors', async (_run, videoId, key) => { + ])('asks for missing frame permission for run %s when the repair has corrupted keys and the fallback errors', async (_run, videoId, key) => { const result = await classify(framesMessage(videoId), rawClassifier(production.inspectVideo(videoId), production.corruptedRepair(videoId, key)), { fallbackModel: rawClassifier({ error: 'fallback unavailable' }) }); - expect(result.decision).toMatchObject({ route: 'inspect_video', videoId, visualEvidence: 'required' }); - expect(result.lastResort).toMatchObject({ lastResort: 'route_recovery', defaultedFields: ['route'] }); + expect(result.decision).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + expect(result.decision).not.toHaveProperty('frameAccess'); }); it.each([ 'inspect_videovideoId', 'inspect_videovideoId', ])('never reads corrupted keys, and assembles the remaining valid fields: %s', async key => { - const corrupted = production.corruptedRepair('aaaaaaaaaaa', key); + const corrupted = { ...production.corruptedRepair('aaaaaaaaaaa', key), frameAccess: 'agent_selected' }; const result = await classify(framesMessage('dQw4w9WgXcQ'), rawClassifier(corrupted)); // The video comes from the request, not from the corrupted key's value. expect(result.decision).toMatchObject({ route: 'inspect_video', videoId: 'dQw4w9WgXcQ', visualEvidence: 'required' }); @@ -239,7 +239,7 @@ describe('classifier last resort', () => { it('keeps required visuals mandatory, taking the requirements from the request when the model gave none', async () => { const message = 'What did the keynote presenters wear?'; const result = await classify(message, rawClassifier({ answerDetail: 'standard', - researchBreadth: 'focused', searchQuery: 'keynote presenter outfits', visualEvidence: 'required' })); + researchBreadth: 'focused', searchQuery: 'keynote presenter outfits', frameAccess: 'agent_selected', visualEvidence: 'required' })); expect(result.decision).toMatchObject({ route: 'topic_research', visualEvidence: 'required', visualRequirements: [message], useStoryboard: true, searchQuery: 'keynote presenter outfits' }); expect(result.lastResort!.defaultedFields).toContain('visualRequirements'); @@ -248,7 +248,7 @@ describe('classifier last resort', () => { it('keeps required visuals even when the decision falls back to request defaults', async () => { // A corrupted key blocks recovery; the candidate still said the answer needs images. const result = await classify('What did the presenter wear in https://youtu.be/dQw4w9WgXcQ?', - rawClassifier({ answerDetail: 'standard', 'route': 'inspect_video', visualEvidence: 'required', + rawClassifier({ answerDetail: 'standard', 'route': 'inspect_video', frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['presenter clothing'] })); expect(result.decision).toMatchObject({ route: 'inspect_video', videoId: 'dQw4w9WgXcQ', visualEvidence: 'required', visualRequirements: ['presenter clothing'] }); @@ -286,7 +286,7 @@ describe('classifier last resort', () => { // its route, has no refresh flag, and says no visuals are needed. const first = { route: 'finalize', answerDetail: 'standard', responseIntent: 'context_answer', contextScope: 'video', reason: 'Saved data answers this.', refreshDynamicData: true, visualEvidence: 'required', visualRequirements: ['thumbnail text'] }; - const repair = { answerDetail: 'standard', researchBreadth: 'focused', searchQuery: 'MrBeast current views', visualEvidence: 'none' }; + const repair = { answerDetail: 'standard', researchBreadth: 'focused', searchQuery: 'MrBeast current views', frameAccess: 'agent_selected', visualEvidence: 'none' }; const result = await classify('What do the current MrBeast thumbnails say, and how many views do they have now?', rawClassifier(first, repair)); expect(result.lastResort).toMatchObject({ lastResort: 'route_recovery' }); expect(result.decision).toMatchObject({ route: 'topic_research', searchQuery: 'MrBeast current views', refreshDynamicData: true, diff --git a/platform/test/fixtures/analysis-session.ts b/platform/test/fixtures/analysis-session.ts index a5be6545..ddbd69dd 100644 --- a/platform/test/fixtures/analysis-session.ts +++ b/platform/test/fixtures/analysis-session.ts @@ -14,7 +14,8 @@ export function attachTestAssetStore(context: AgentToolContext) { details: SessionAsset['details'] = {}, ) => { const version = (++ordinal).toString(16).padStart(64, '0'); - saved.set(version, { asset: { version, kind, videoId, current: true, collectedAt: 1, details }, value }); + // Collection order stands in for time, so a test can place a run start between saves. + saved.set(version, { asset: { version, kind, videoId, current: true, collectedAt: ordinal, details }, value }); return version; }; context.session = { @@ -87,5 +88,7 @@ export function attachTestAssetStore(context: AgentToolContext) { generation++; if (version) saved.delete(version); else saved.clear(); }; - return { saved, put, remove }; + /** A run start after every save so far. */ + const runStart = () => ordinal + 0.5; + return { saved, put, remove, runStart }; } diff --git a/platform/test/frame-timestamp-expression.test.ts b/platform/test/frame-timestamp-expression.test.ts new file mode 100644 index 00000000..cfb5e77b --- /dev/null +++ b/platform/test/frame-timestamp-expression.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest'; +import { seekMilliseconds, timestampExpressions } from '../src/agents/research/timestamp-expression'; +import { frameRequestIssue, readFrameRequest } from '../src/agents/research/frame-request'; + +describe('frame timestamp expressions', () => { + it('reads equivalent clocks, quantities, and seek values as one complete position', () => { + for (const hours of [1, 2, 12]) for (const minutes of [0, 1, 30, 59]) for (const seconds of [0, 5, 59]) { + const expected = ((hours * 60 + minutes) * 60 + seconds) * 1000; + for (const expression of [ + `${hours}:${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')}`, + `${hours}h${minutes}m${seconds}s`, + `${hours} hours ${minutes} minutes and ${seconds} seconds`, + `${hours * 60 + minutes} minutes ${seconds} seconds`, + ]) { + const prefix = 'Inspect at '; + expect(timestampExpressions(prefix + expression + '.')).toEqual([ + { start: prefix.length, end: prefix.length + expression.length, milliseconds: expected, ambiguous: false }, + ]); + if (!expression.includes(' ')) expect(seekMilliseconds(expression)).toBe(expected); + } + } + }); + + it.each([ + ['at 1 hour and at 30 minutes', [3600000, 1800000]], + ['at 1 hour 30 minutes and at 1 hour 45 minutes', [5400000, 6300000]], + ['at 30 seconds and 45 seconds', [30000, 45000]], + ['at 1.5 hours and at 250 ms', [5400000, 250]], + ] as const)('preserves separately specified positions in "%s"', (message, positions) => { + expect(timestampExpressions(message).map(time => time.milliseconds)).toEqual(positions); + }); + + it('asks for clarification before granting a comma-separated quantity with two readings', () => { + const request = readFrameRequest('Capture at 1 hour, 30 minutes.'); + for (const times of [[5400000], [1800000, 3600000]]) { + expect(frameRequestIssue({ route: 'inspect_video', frameAccess: [{ videoId: 'abcdefghijk', + savedTimestampsMs: [], captureTimestampsMs: times }] }, request)).toContain('unambiguous timestamp'); + } + }); + + it('keeps an unread exact-time instruction closed for both finite and delegated grants', () => { + const request = readFrameRequest('Capture only the frame at half past the first hour.'); + const plan = { route: 'inspect_video' }; + expect(frameRequestIssue({ ...plan, frameAccess: 'agent_selected' }, request)).toContain('unambiguous timestamp'); + expect(frameRequestIssue({ ...plan, frameAccess: [{ savedTimestampsMs: [], captureTimestampsMs: [30000] }] }, request)) + .toContain('unambiguous timestamp'); + }); + + it('keeps a compound source duration out of image permissions', () => { + expect(readFrameRequest('Find a 1 hour 30 minute lecture and capture only at 1 minute 10 seconds.').positions) + .toEqual([{ role: 'capture', time: 70000, videoIds: [] }]); + }); +}); diff --git a/platform/test/session-evidence.integration.test.ts b/platform/test/session-evidence.integration.test.ts index 849e918c..d0bf9736 100644 --- a/platform/test/session-evidence.integration.test.ts +++ b/platform/test/session-evidence.integration.test.ts @@ -22,6 +22,14 @@ import { bicycleHistory, gcnCaptions, officialCaptions, gcnAnalysis, officialAna import { readSavedEvidence, searchSavedEvidence, SAVED_EVIDENCE_READ_LIMIT } from '../src/agents/runtime/saved-evidence-reader'; import { annaId, kelogId, watercolorTranscript, watercolorCorrection, watercolorFollowup } from './fixtures/watercolor-session'; +test('the inventory recovers a saved frame position that older details omitted', async () => + within('legacy-frame-position', async (store, reopen) => { + // Older rows stored no timestampMs; the frame's resource key still records it. + await store.retrieve('frame:abcdefghijk:640:1000', 'frame', id, false, async () => ({ value: oneFrame(id), cacheStatus: 'miss' as const }), () => ({})); + const frame = reopen().brief().assets.find(asset => asset.kind === 'frame')!; + expect(frame.details.timestampMs).toBe(1000); + })); + test('saved original comments include positions 13 and 17 and long tails, with bounded pagination', async () => within('original-comments', async (store, reopen) => { const value = { videoId: id, replyContinuations: [], meta: transcript().meta, @@ -1465,6 +1473,12 @@ test('single-sheet storyboard reuses the processor middle sheet across restored expect(warm.value.value.sheets).toEqual(value.sheets); expect(warm.value.sessionReused).toBe(true); expect(storyboard).toHaveBeenCalledOnce(); + // A sheet-only refresh must reach the provider even when the requested sheet is already saved. + const refresh = sessionProvider(p, reopen(), false, undefined, undefined, false, true); + await refresh.storyboard!(id, undefined, { maxSheets: 1 }); + await refresh.storyboard!(id, undefined, { maxSheets: 1 }); + expect(storyboard).toHaveBeenCalledTimes(2); + expect(storyboard).toHaveBeenLastCalledWith(id, undefined, expect.objectContaining({ refresh: true }), undefined); for (const { diagnostics } of [cold, warm]) { for (const span of diagnostics.spans) { const parent = diagnostics.spans.find(candidate => candidate.id === span.parentId); diff --git a/platform/test/youtube-agent-analysis-context.test.ts b/platform/test/youtube-agent-analysis-context.test.ts index eadd2579..860494e5 100644 --- a/platform/test/youtube-agent-analysis-context.test.ts +++ b/platform/test/youtube-agent-analysis-context.test.ts @@ -166,7 +166,7 @@ it.each(['required', 'helpful'] as const)('keeps provider calls for exact frames usage: { inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 }, outputTokens: { total: 10, text: 10, reasoning: 0 } }, }) }); await runResearchAgentWithModel({ model, message: 'Compare the camera angles', context, decision: { - route: 'topic_research', researchBreadth: 'comparative', researchVideoCount: 4, useStoryboard: true, visualEvidence, + route: 'topic_research', researchBreadth: 'comparative', researchVideoCount: 4, useStoryboard: true, visualEvidence, frameAccess: 'agent_selected', ...(visualEvidence === 'required' ? { visualRequirements: ['camera angle'] } : {}), } }); const reserve = visualEvidence === 'required' ? 2 : undefined; @@ -210,7 +210,7 @@ it.each([ usage: { inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 }, outputTokens: { total: 10, text: 10, reasoning: 0 } }, }) }); await runResearchAgentWithModel({ model, message: 'Compare how the presenters look and what they say', context, decision: { - route: 'topic_research', researchBreadth: 'comparative', researchVideoCount: 2, useStoryboard: true, visualEvidence: 'required', + route: 'topic_research', researchBreadth: 'comparative', researchVideoCount: 2, useStoryboard: true, visualEvidence: 'required', frameAccess: 'agent_selected', visualRequirements: ['presenter appearance'], ...(subjects ? { comparisonVideoIds: subjects } : {}), } }); expect(executions.filter(execution => execution.toolName === 'get_video_transcript').map(execution => execution.frameReserveCalls)).toEqual(reserves); @@ -262,7 +262,7 @@ it.each([ usage: { inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 }, outputTokens: { total: 10, text: 10, reasoning: 0 } }, }) }); await runResearchAgentWithModel({ model, message: 'Compare how the presenters look and what they say', context, recoveredEvidence: recovered, - decision: { route: 'topic_research', comparisonVideoIds: [first, second], useStoryboard: true, visualEvidence: 'required', + decision: { route: 'topic_research', comparisonVideoIds: [first, second], useStoryboard: true, visualEvidence: 'required', frameAccess: 'agent_selected', visualRequirements: ['presenter appearance'] } }); expect(executions.find(execution => execution.toolName === 'get_video_transcript')!.frameReserveCalls).toBe(reserve); }); diff --git a/platform/test/youtube-agent-frame-deadlines.test.ts b/platform/test/youtube-agent-frame-deadlines.test.ts index 7a3bcccd..17060ded 100644 --- a/platform/test/youtube-agent-frame-deadlines.test.ts +++ b/platform/test/youtube-agent-frame-deadlines.test.ts @@ -47,7 +47,7 @@ function setup(delayMs: number) { confidence: 'low', warnings: [], blocks: [{ text: 'The available evidence is limited.', evidenceIds: ['ref_1'] }], }) }], finishReason: { unified: 'stop', raw: undefined }, usage, warnings: [] }) }); const options = { model, finalizationModel: finalizer, context, message: 'Look at frames to confirm player names.', - decision: { route: 'inspect_video' as const, videoId, useStoryboard: true, visualEvidence: 'required' as const, visualRequirements: ['player names on shirts'] }, + decision: { route: 'inspect_video' as const, videoId, useStoryboard: true, visualEvidence: 'required' as const, visualRequirements: ['player names on shirts'], frameAccess: 'agent_selected' as const }, toolNames: ['get_video_frames', 'analyze_video_frames', 'finalize_answer'] as const }; return { options, context, analyzed, finalizer }; } @@ -123,7 +123,7 @@ test('continues after the transcript target and a failed storyboard to retrieve } }); const run = runResearchAgentWithModel({ ...options, model, message: 'What were the names of the presenters and what were they wearing on stage?', - decision: { route: 'topic_research', useStoryboard: true, visualEvidence: 'required', visualRequirements: ['presenter clothing'], researchVideoCount: 1 }, + decision: { route: 'topic_research', useStoryboard: true, visualEvidence: 'required', visualRequirements: ['presenter clothing'], researchVideoCount: 1, frameAccess: 'agent_selected' }, recoveredSearchUsed: true, recoveredEvidence: [transcript], toolNames: ['get_video_storyboard', 'get_video_frames', 'analyze_video_frames', 'finalize_answer'] }); await vi.advanceTimersByTimeAsync(0); @@ -231,7 +231,7 @@ test('does not withhold finalization when visual evidence is only helpful', asyn }) }], finishReason: { unified: 'tool-calls', raw: undefined }, usage, warnings: [] }; } }); await runResearchAgentWithModel({ ...options, model, recoveredEvidence: [metadata], - decision: { route: 'inspect_video', videoId, useStoryboard: true, visualEvidence: 'helpful' } }); + decision: { route: 'inspect_video', videoId, useStoryboard: true, visualEvidence: 'helpful', frameAccess: 'agent_selected' } }); expect(model.doGenerateCalls).toHaveLength(1); expect(context.provider.frames).not.toHaveBeenCalled(); expect(context.finalize).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ warnings: @@ -243,8 +243,11 @@ test('names the required visual facts when finalizing without visual observation const model = new MockLanguageModelV4({ doGenerate: async () => ({ content: [{ type: 'tool-call', toolCallId: 'finish', toolName: 'finalize_answer', input: JSON.stringify({ intent: 'inspect_video', confidence: 'low', artifacts: [], warnings: [], blocks: [{ text: 'Unverified.', evidenceIds: ['ref_1'] }] }) }], finishReason: { unified: 'tool-calls', raw: undefined }, usage, warnings: [] }) }); - await runResearchAgentWithModel({ ...options, model, recoveredEvidence: [metadata], + // A full warning list must not crowd out the application's own visual-evidence note. + const warnings = Array.from({ length: 50 }, (_, index) => ({ code: 'TRANSCRIPT_CONTEXT_TRUNCATED', message: `Omitted passage ${index}.` })); + await runResearchAgentWithModel({ ...options, model, recoveredEvidence: [{ ...metadata, warnings }], recoveredToolFailures: [{ toolCallId: 'frames', toolName: 'get_video_frames', operation: 'frames', message: 'Frame extraction failed' }] }); + expect(vi.mocked(context.finalize).mock.calls[0]![1].warnings).toHaveLength(50); expect(JSON.stringify(model.doGenerateCalls[0]?.prompt)).toContain('Required visual evidence: player names on shirts'); expect(context.finalize).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ warnings: expect.arrayContaining([expect.objectContaining({ code: 'VISUAL_EVIDENCE_INCOMPLETE', message: expect.stringContaining('player names on shirts') })]) })); diff --git a/platform/test/youtube-agent-loop-control.test.ts b/platform/test/youtube-agent-loop-control.test.ts index ea7cb839..f53d2ff6 100644 --- a/platform/test/youtube-agent-loop-control.test.ts +++ b/platform/test/youtube-agent-loop-control.test.ts @@ -497,6 +497,25 @@ describe('YouTube AgentCore loop control', () => { decision: { route: 'topic_research', researchBreadth: 'comparative', searchQuery: 'model practical use cases' }, context }); }); + it('allows completion on the first model step when there is no video for a discovery capture', async () => { + const context = inspectContext(); + context.provider.frames = vi.fn(); + context.provider.search = vi.fn(async () => ({ cacheStatus: 'miss' as const, value: { + query: 'exercise', results: [], videos: [], channels: [], playlists: [], + meta: { source: 'allthingsyoutube' as const, fetchedAt: new Date().toISOString(), partial: false, warnings: [] }, + } })); + const model = new MockLanguageModelV4({ doGenerate: async () => modelResult({ toolCallId: 'finish', toolName: 'complete_research', input: '{}' }) }); + const finalizationModel = new MockLanguageModelV4({ doGenerate: async () => finalizerModelResult({ confidence: 'low', warnings: [], + blocks: [{ text: 'No matching video was found.', evidenceIds: [] }] }) }); + await runResearchAgentWithModel({ model, finalizationModel, context, message: 'Find an exercise video and capture 0:30.', + decision: { route: 'topic_research', searchQuery: 'exercise', frameAccess: [{ captureTimestampsMs: [30000], savedTimestampsMs: [] }] } }); + expect(context.provider.search).toHaveBeenCalledOnce(); + expect(model.doGenerateCalls).toHaveLength(1); + expect(model.doGenerateCalls[0]?.tools?.map(tool => tool.name)).toContain('complete_research'); + expect(model.doGenerateCalls[0]?.tools?.map(tool => tool.name)).not.toContain('get_video_frames'); + expect(context.provider.frames).not.toHaveBeenCalled(); + }); + it('does not retry a failed classified search in the model loop', async () => { const context = inspectContext(); context.provider.search = vi.fn(async () => { throw new Error('Search unavailable'); }); diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index daaed224..36b244b2 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -6,6 +6,7 @@ import { runResearchAgentWithModel, } from '../src/agents/research/research-agent'; import { INSPECT_VIDEO_TOOL_NAMES } from '../src/agents/research/capabilities/inspect-video'; +import { capabilityRouteDecisionSchema } from '../src/agents/contracts'; import { classifyCapabilityWithModel, extractYouTubeVideoIds, @@ -17,6 +18,7 @@ import { createCapabilityProvider, savedOnlyProvider } from '../src/agents/resea import type { ConversationTurn } from '../src/agents/runtime/conversation-memory'; import type { YouTubeAgentProvider } from '../src/agents/providers/youtube/provider'; import type { AgentToolContext } from '../src/agents/providers/youtube/tool-context'; +import { readFrameRequest } from '../src/agents/research/frame-request'; import { currentDateGuidance } from '../src/agents/runtime/current-date'; describe('YouTube agent capability router', () => { @@ -164,7 +166,7 @@ describe('YouTube agent capability router', () => { it.each([false, true])('recovers the captured fixed-video follow-up with missing discovery fields (answerDetail present: %s)', async hasDetail => { const decision = { route: 'topic_research', comparisonVideoIds: ['abcdefghijk', 'lmnopqrstuv'], - visualEvidence: 'required', visualRequirements: ['exercise form'] }; + frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['exercise form'] }; const model = sequenceClassifier([{ ...decision, answerDetail: hasDetail ? 'standard' : undefined }, { ...decision, answerDetail: 'standard' }]); const diagnostics = vi.fn(); @@ -179,7 +181,7 @@ describe('YouTube agent capability router', () => { }); it('gives repair the previous candidate and all missing fields without losing valid choices', async () => { - const candidate = { route: 'topic_research', answerDetail: 'detailed', visualEvidence: 'required', + const candidate = { route: 'topic_research', answerDetail: 'detailed', frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['exercise form'] }; let calls = 0; const model = new MockLanguageModelV4({ doGenerate: async ({ prompt }) => { @@ -608,7 +610,7 @@ describe('YouTube agent capability router', () => { it.each([0, 1])('accepts the captured DevDay classification despite obsolete conflicting count fields (%s)', async requiredVideoCount => { const model = classifierModel({ answerDetail: 'standard', researchBreadth: 'focused', researchVideoCount: 3, requiredVideoCount, route: 'topic_research', searchQuery: 'OpenAI DevDay 2026 presenters on stage keynote', - visualEvidence: 'required', visualRequirements: ['presenter clothing'] }); + frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['presenter clothing'] }); const decision = await classifyCapabilityWithModel({ message: 'What were the names of presenters from OpenAI DevDay 2026 and what were they wearing?', model, signal: new AbortController().signal }); expect(model.doGenerateCalls).toHaveLength(1); expect(decision).toMatchObject({ route: 'topic_research', researchVideoCount: 2, visualEvidence: 'required' }); @@ -621,7 +623,7 @@ describe('YouTube agent capability router', () => { it.each(['https://youtu.be/abcdefghijk', 'https://www.youtube.com/watch?v=abcdefghijk', 'https://youtube.com/shorts/abcdefghijk'])('pins a classifier research decision to the supplied video: %s', async url => { const model = classifierModel({ route: 'topic_research', researchBreadth: 'focused', searchQuery: 'keynote presenters', - visualEvidence: 'required', visualRequirements: ['presenter clothing'] }); + frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['presenter clothing'] }); const decision = await classifyCapabilityWithModel({ message: `Research who presented and what they wore: ${url}`, model, conversationHistory: [{ userMessageId: 'previous-user', agentMessageId: 'previous-agent', user: 'An earlier video', assistant: 'Prior research', resourceIds: ['lmnopqrstuv'] }], signal: new AbortController().signal }); @@ -761,12 +763,12 @@ describe('YouTube agent capability router', () => { ['none', undefined, false], ['helpful', undefined, true], ['required', ['presenter clothing'], true], ] as const)('persists visual evidence %s with derived tool access', async (visualEvidence, visualRequirements, useStoryboard) => { const decision = await classifyCapabilityWithModel({ message: 'Inspect https://youtu.be/abcdefghijk', - model: classifierModel({ route: 'inspect_video', videoId: 'abcdefghijk', visualEvidence, visualRequirements }), + model: classifierModel({ route: 'inspect_video', videoId: 'abcdefghijk', visualEvidence, visualRequirements, ...(visualRequirements ? { frameAccess: 'agent_selected' } : {}) }), signal: new AbortController().signal, }); const recovered = await resolveCapabilityRoute({ persisted: decision, classify: vi.fn(), persist: vi.fn() }); expect(recovered).toEqual({ route: 'inspect_video', videoId: 'abcdefghijk', researchVideoCount: 1, useStoryboard, visualEvidence, - ...(visualRequirements ? { visualRequirements } : {}), answerDetail: 'standard' }); + ...(visualRequirements ? { visualRequirements, frameAccess: 'agent_selected' } : {}), answerDetail: 'standard' }); }); it('requires visual requirements when visual evidence is required, and drops them otherwise', async () => { @@ -782,7 +784,7 @@ describe('YouTube agent capability router', () => { it('asks once to reconsider visual cues, then accepts the reconsidered decision', async () => { const outputs = [ { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'OpenAI DevDay keynote presenters', visualEvidence: 'none' }, - { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'OpenAI DevDay keynote presenters', visualEvidence: 'required', visualRequirements: ['presenter clothing'] }, + { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'OpenAI DevDay keynote presenters', frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['presenter clothing'] }, ]; const model = sequenceClassifier(outputs); const decision = await classifyCapabilityWithModel({ message: 'Who presented at the DevDay keynote and what were they wearing?', @@ -792,7 +794,7 @@ describe('YouTube agent capability router', () => { expect(repair).toContain('possible_visual_requirement'); expect(repair).toContain('Reconsider the previous classification'); expect(repair).toMatch(/The request mentions \W+wearing\W+\. If any requested fact depends on what is visible/); - expect(decision).toMatchObject({ visualEvidence: 'required', visualRequirements: ['presenter clothing'], useStoryboard: true }); + expect(decision).toMatchObject({ frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['presenter clothing'], useStoryboard: true }); }); it('keeps the classifier choice when it declines a visual cue, without a third call', async () => { @@ -808,7 +810,7 @@ describe('YouTube agent capability router', () => { const message = 'Summarize the slide design tips in popular talks'; const first = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'slide design tips', visualEvidence: 'helpful' }; const full = (output: Record): Record => ({ answerDetail: 'standard', ...output }); - const required = full({ ...first, visualEvidence: 'required', visualRequirements: ['slide layouts'] }); + const required = full({ ...first, frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['slide layouts'] }); // Missing route and search query, so route recovery cannot apply. const { route: _omitted, searchQuery: _query, ...withoutRoute } = first; const missing = full(withoutRoute); @@ -937,7 +939,7 @@ describe('YouTube agent capability router', () => { await classifyCapabilityWithModel({ message: 'Help me understand reservoir computing', model: plain, signal: new AbortController().signal }); expect(plain.doGenerateCalls).toHaveLength(1); const required = sequenceClassifier([{ route: 'topic_research', researchBreadth: 'focused', searchQuery: 'keynote outfits', - visualEvidence: 'required', visualRequirements: ['presenter clothing'] }]); + frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['presenter clothing'] }]); await classifyCapabilityWithModel({ message: 'What were the keynote presenters wearing?', model: required, signal: new AbortController().signal }); expect(required.doGenerateCalls).toHaveLength(1); }); @@ -1071,32 +1073,33 @@ describe('YouTube agent capability router', () => { it.each(['direct', 'recovered'])('keeps a frame-only refresh on a %s two-video comparison', async mode => { const ids = ['abcdefghijk', 'lmnopqrstuv']; const output = mode === 'direct' - ? { route: 'topic_research', comparisonVideoIds: ids, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['read the clocks'] } - : { route: 'inspect_video', videoId: ids[0], comparisonVideoIds: ids, refreshFramesOnly: true, visualEvidence: 'invalid' }; + ? { route: 'topic_research', comparisonVideoIds: ids, refreshFramesOnly: true, frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['read the clocks'] } + : { route: 'inspect_video', videoId: ids[0], comparisonVideoIds: ids, refreshFramesOnly: true, frameAccess: 'agent_selected', visualEvidence: 'invalid' }; const pending = classifyCapabilityWithModel({ message: 'Compare fresh frames from https://youtu.be/abcdefghijk and https://youtu.be/lmnopqrstuv', model: sequenceClassifier([output]), signal: new AbortController().signal }); const route = mode === 'direct' ? await pending : await expectLastResort(pending); - expect(route).toMatchObject({ route: 'topic_research', comparisonVideoIds: ids, refreshFramesOnly: true, useStoryboard: true }); + expect(route).toMatchObject({ route: 'topic_research', comparisonVideoIds: ids, refreshFramesOnly: true, frameAccess: 'agent_selected', useStoryboard: true }); expect(route).not.toHaveProperty('refreshEvidence', true); }); it('carries the timestamp question across recovery candidates for the same video', async () => { const videoId = 'abcdefghijk', question = 'What does the scoreboard show at 1:00?'; + const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; const model = sequenceClassifier([ - { route: 'inspect_video', videoId, timestampedFrameQuestion: question, visualEvidence: 'invalid' }, + { route: 'inspect_video', videoId, timestampedFrameQuestion: question, visualEvidence: 'invalid', frameAccess }, { videoId, visualEvidence: 'required', visualRequirements: ['scoreboard'] }, ]); const decision = await expectLastResort(classifyCapabilityWithModel({ message: `At 1:00 in https://youtu.be/${videoId}, read the scoreboard`, model, signal: new AbortController().signal })); - expect(decision).toMatchObject({ route: 'inspect_video', videoId, timestampedFrameQuestion: question }); + expect(decision).toMatchObject({ route: 'inspect_video', videoId, timestampedFrameQuestion: question, frameAccess }); }); - it.each(['overlap', 'unrestricted'])('repairs an unscoped refresh and rejects %s capture permission', async invalid => { + it.each(['overlap', 'omitted'])('repairs an %s capture list before authorizing a bounded refresh', async invalid => { const videoId = 'abcdefghijk'; - const frameAccess = { videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; + const frameAccess = [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]; const base = { route: 'inspect_video', videoId, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['dough shape'] }; const model = sequenceClassifier([base, { ...base, frameAccess: invalid === 'overlap' - ? { ...frameAccess, captureTimestampsMs: [630000, 645000] } : 'agent_selected' }]); + ? [{ ...frameAccess[0]!, captureTimestampsMs: [630000, 645000] }] : undefined }]); const fallbackModel = sequenceClassifier([{ ...base, frameAccess }]); const route = await classifyCapabilityWithModel({ model, fallbackModel, message: 'Reuse saved 10:30 without recapturing it and capture only 10:45.', sessionBrief: { memories: [], assets: [{ videoId, version: 'a'.repeat(64), kind: 'frame', current: true, @@ -1106,6 +1109,478 @@ describe('YouTube agent capability router', () => { expect(fallbackModel.doGenerateCalls).toHaveLength(1); }); + it('routes a saved frame on one video and a capture on another as a comparison of both', async () => { + const [saved, linked] = ['abcdefghijk', 'lmnopqrstuv']; + const frameAccess = [{ videoId: saved, savedTimestampsMs: [30000], captureTimestampsMs: [] }, + { videoId: linked, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + const visual = { refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['outfits'] }; + // Each rejected shape left the saved video's frame unreadable in production. + const model = sequenceClassifier([ + { route: 'inspect_video', videoId: linked, ...visual, frameAccess }, + { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'outfits', ...visual, frameAccess }, + ]); + const fallbackModel = sequenceClassifier([{ route: 'topic_research', comparisonVideoIds: [saved, linked], ...visual, frameAccess }]); + const route = await classifyCapabilityWithModel({ model, fallbackModel, + message: `Compare this with https://www.youtube.com/watch?v=${linked}: re-inspect the saved 0:30 frame without recapturing it, and capture only 1:00 of the new video.`, + sessionBrief: { memories: [], assets: [{ videoId: saved, version: 'a'.repeat(64), kind: 'frame', current: true, + collectedAt: 1, details: { timestampMs: 30000 } }] }, signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'topic_research', comparisonVideoIds: [saved, linked], frameAccess }); + expect(fallbackModel.doGenerateCalls).toHaveLength(1); + }); + + it('keeps frame access on the linked video when one video is linked', async () => { + const [saved, linked] = ['abcdefghijk', 'lmnopqrstuv']; + const visual = { refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['outfit'] }; + const model = sequenceClassifier([ + { route: 'inspect_video', videoId: saved, ...visual, frameAccess: [{ videoId: saved, savedTimestampsMs: [], captureTimestampsMs: [45000] }] }, + { route: 'inspect_video', videoId: linked, ...visual, frameAccess: [{ videoId: linked, savedTimestampsMs: [], captureTimestampsMs: [45000] }] }, + ]); + const route = await classifyCapabilityWithModel({ model, message: `Capture only 0:45 of https://youtu.be/${linked}`, + sessionBrief: { memories: [], assets: [{ videoId: saved, version: 'a'.repeat(64), kind: 'frame', current: true, collectedAt: 1, details: { timestampMs: 30000 } }] }, + signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'inspect_video', videoId: linked, frameAccess: [{ videoId: linked }] }); + expect(model.doGenerateCalls).toHaveLength(2); + }); + + it('reads routes persisted with single-video frame access', () => { + const legacy = { videoId: 'abcdefghijk', savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; + expect(capabilityRouteDecisionSchema.parse({ route: 'inspect_video', videoId: legacy.videoId, researchVideoCount: 1, + useStoryboard: true, answerDetail: 'standard', frameAccess: legacy })).toMatchObject({ frameAccess: [legacy] }); + }); + + it('runs a delegated refresh, and asks an unstated one for capture permission', async () => { + const base = { route: 'inspect_video', videoId: 'abcdefghijk', refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'] }; + const model = sequenceClassifier([base, { ...base, frameAccess: 'agent_selected' }]); + const route = await classifyCapabilityWithModel({ model, message: 'Refresh the frames of https://youtu.be/abcdefghijk', signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'inspect_video', refreshFramesOnly: true, frameAccess: 'agent_selected' }); + expect(model.doGenerateCalls).toHaveLength(2); + expect(JSON.stringify(model.doGenerateCalls[1]!.prompt)).toContain('needs capture permission'); + }); + + it('repairs missing image access for a required visual question', async () => { + const videoId = 'abcdefghijk'; + const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['car color'] }; + const model = sequenceClassifier([base, { ...base, frameAccess: 'storyboard_only' }]); + const message = `What color is the car in https://youtu.be/${videoId}?`; + const decision = await classifyCapabilityWithModel({ model, message, signal: new AbortController().signal }); + expect(decision).toMatchObject({ frameAccess: 'storyboard_only' }); + expect(model.doGenerateCalls).toHaveLength(2); + }); + + it.each([ + ['Capture only the opening frame', 0], ['Capture the frame at 1:30s', 90000], + ] as const)('binds "%s" to its actual timestamp', async (message, requested) => { + const videoId = 'abcdefghijk'; + const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['frame'] }; + const at = (time: number) => [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [time] }]; + const model = sequenceClassifier([{ ...base, frameAccess: 'agent_selected' }, { ...base, frameAccess: at(30000) }]); + const fallbackModel = classifierModel({ ...base, frameAccess: at(requested) }); + const decision = await classifyCapabilityWithModel({ model, fallbackModel, message: `${message} of https://youtu.be/${videoId}.`, signal: new AbortController().signal }); + expect(fallbackModel.doGenerateCalls).toHaveLength(1); + expect(decision).toMatchObject({ frameAccess: at(requested) }); + }); + + it('asks for an exact bound before capturing the first few seconds', async () => { + const videoId = 'abcdefghijk'; + const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'] }; + const model = sequenceClassifier([{ ...base, frameAccess: 'agent_selected' }, + { ...base, frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [120000] }] }]); + expect(await classifyCapabilityWithModel({ model, message: `Sample only the first few seconds of https://youtu.be/${videoId}`, + signal: new AbortController().signal })).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it.each(['Your previous answer described 0:30.', 'Find a 2 hour lecture and describe the cover.', + 'Find 5 hours of content about exercise.'])('does not authorize or require a position from "%s"', message => { + expect(readFrameRequest(message).positions).toEqual([]); + }); + + it('pins a discovery capture to the linked inspection subject', async () => { + const videoId = 'abcdefghijk'; + const model = classifierModel({ route: 'topic_research', researchBreadth: 'focused', searchQuery: 'exercise', + visualEvidence: 'required', visualRequirements: ['movement'], frameAccess: [{ captureTimestampsMs: [30000], savedTimestampsMs: [] }] }); + const message = `Capture only 0:30 of https://youtu.be/${videoId}.`; + const decision = await classifyCapabilityWithModel({ model, message, signal: new AbortController().signal }); + expect(decision).toMatchObject({ route: 'inspect_video', videoId, frameAccess: [{ videoId, captureTimestampsMs: [30000] }] }); + expect(await resolveCapabilityRoute({ message, persisted: { ...decision, frameAccess: [{ captureTimestampsMs: [30000], savedTimestampsMs: [] }] }, + classify: vi.fn(), persist: vi.fn() })).toMatchObject({ responseIntent: 'clarification' }); + }); + + it('assigns comparison sheet times joined by "with" to their own videos', () => { + const a = 'abcdefghijk', b = 'lmnopqrstuv'; + const message = `Compare the storyboard sheet at 0:30 of https://youtu.be/${a} with the one at 1:00 of https://youtu.be/${b}.`; + expect(readFrameRequest(message).positions.map(pos => [pos.time, pos.videoIds])).toEqual([[30000, [a]], [60000, [b]]]); + const common = `Compare the storyboard sheet at 0:30 in both https://youtu.be/${a} and https://youtu.be/${b}.`; + expect(readFrameRequest(common).positions.map(pos => [pos.time, pos.videoIds])).toEqual([[30000, [a, b]]]); + }); + + it('keeps paired storyboard positions on comparison routes, including resume', async () => { + const a = 'abcdefghijk', b = 'lmnopqrstuv'; + const model = classifierModel({ route: 'topic_research', comparisonVideoIds: [a, b], frameAccess: 'storyboard_only', + visualEvidence: 'required', visualRequirements: ['compare the scenes'] }); + const message = `Compare the storyboard sheets at 0:30 of https://youtu.be/${a} and 1:00 of https://youtu.be/${b}.`; + const decision = await classifyCapabilityWithModel({ message, model, signal: new AbortController().signal }); + const storyboardSelections = [{ videoId: a, timestampsMs: [30000] }, { videoId: b, timestampsMs: [60000] }]; + expect(decision).toMatchObject({ route: 'topic_research', frameAccess: { storyboardSelections } }); + const resume = (selections: typeof storyboardSelections) => resolveCapabilityRoute({ message, classify: vi.fn(), persist: vi.fn(), + persisted: { ...decision, frameAccess: { storyboardSelections: selections } } }); + await expect(resume(storyboardSelections)).resolves.toEqual(decision); + await expect(resume(storyboardSelections.map(item => ({ ...item, timestampsMs: [30000, 60000] })))).resolves.toMatchObject({ responseIntent: 'clarification' }); + if (decision.route !== 'topic_research') throw new Error('Expected comparison.'); + const context = inspectContext(); + const storyboard = vi.fn>(async (videoId, times) => ({ + cacheStatus: 'miss', value: { videoId, frameCount: 20, intervalMs: 5000, + sheets: [{ imageBase64: '/9j/2Q==', tileWidth: 100, tileHeight: 100, columns: 2, rows: 1, firstFrameIndex: times![0]! / 5000, + frameCount: 2, intervalMs: 5000 }], meta: { partial: false, warnings: [] } }, + })); + context.provider.storyboard = storyboard; + const research = finalizingModel([ + { name: 'get_video_storyboard', input: { videoId: a, timestampsMs: [60000] } }, + ...storyboardSelections.map(input => ({ name: 'get_video_storyboard', input })), + ]); + await runResearchAgentWithModel({ message, decision, context, model: research }); + expect(storyboard.mock.calls.map(([videoId, timestampsMs]) => ({ videoId, timestampsMs }))).toEqual(storyboardSelections); + expect(research.doGenerateCalls[0]?.tools?.map(tool => tool.name)).not.toContain('get_video_frames'); + }); + + it('repairs an unrestricted storyboard refresh without granting exact capture', async () => { + const videoId = 'abcdefghijk'; + const route = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['overview'] }; + const model = sequenceClassifier([{ ...route, frameAccess: 'agent_selected' }, { ...route, frameAccess: 'storyboard_only' }]); + const message = `Refresh only the storyboard for https://youtu.be/${videoId}.`; + const decision = await classifyCapabilityWithModel({ model, message, signal: new AbortController().signal }); + expect(model.doGenerateCalls).toHaveLength(2); + expect(decision).toMatchObject({ frameAccess: 'storyboard_only', refreshFramesOnly: true }); + }); + + it('leaves capture closed when the classifier grants nothing for an exact capture request', async () => { + const videoId = 'abcdefghijk'; + // Even repeated omissions cannot open either retrieval method. + const route = await classifyCapabilityWithModel({ model: sequenceClassifier([{ route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['the 1:00 still'] }]), + message: `Capture only 1:00 of https://youtu.be/${videoId}. Just the still, no analysis.`, signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it.each([ + ['Capture only 0:45 of https://youtu.be/abcdefghijk and describe it. Do not retrieve any other images.', undefined, 0], + ['Use a storyboard overview of https://youtu.be/abcdefghijk to describe the scenery.', 'storyboard_only', 1], + ] as const)('enforces storyboard access through the provider for "%s"', async (message, frameAccess, expectedCalls) => { + const videoId = 'abcdefghijk'; + const decision = await classifyCapabilityWithModel({ message, signal: new AbortController().signal, + model: classifierModel({ route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'], frameAccess }) }); + if (!frameAccess) { + expect(decision).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + return; + } + if (decision.route !== 'inspect_video') throw new Error('Expected inspection.'); + const context = inspectContext(); + const storyboard = vi.fn(async () => { throw new Error('Storyboard unavailable'); }); + context.provider.storyboard = storyboard; + const model = finalizingModel([{ name: 'get_video_storyboard', input: { videoId, timestampsMs: [90000] } }]); + await runResearchAgentWithModel({ model, message, decision, context }); + expect(storyboard).toHaveBeenCalledTimes(expectedCalls); + expect(model.doGenerateCalls[0]?.tools?.some(tool => tool.name === 'get_video_frames')).toBe(false); + expect(model.doGenerateCalls[0]?.tools?.some(tool => tool.name === 'get_video_storyboard')).toBe(expectedCalls > 0); + }); + + it('repairs a storyboard grant into the requested exact capture', async () => { + const videoId = 'abcdefghijk'; + const route = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene at 0:45'] }; + const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [45000] }]; + const classifier = sequenceClassifier([{ ...route, frameAccess: 'storyboard_only' }, { ...route, frameAccess }]); + const message = `Capture only 0:45 of https://youtu.be/${videoId} and describe it. Do not retrieve any other images.`; + const decision = await classifyCapabilityWithModel({ model: classifier, message, signal: new AbortController().signal }); + expect(classifier.doGenerateCalls).toHaveLength(2); + expect(decision).toMatchObject({ frameAccess }); + if (decision.route !== 'inspect_video') throw new Error('Expected inspection.'); + const model = finalizingModel(); + await runResearchAgentWithModel({ model, message, decision, context: inspectContext() }); + const names = model.doGenerateCalls[0]?.tools?.map(tool => tool.name); + expect(names).toContain('get_video_frames'); + expect(names).not.toContain('get_video_storyboard'); + }); + + it.each([ + ['1 hour 30 minutes', 5400000, [3600000, 1800000]], + ['1h30m15s', 5415000, [3600000, 1800000, 15000]], + ['1 hour 30 minutes and 15 seconds', 5415000, [3600000, 1815000]], + ] as const)('only grants the complete timestamp in "%s"', async (expression, expected, parts) => { + const videoId = 'abcdefghijk'; + const message = `Capture only the frame at ${expression} of https://youtu.be/${videoId}.`; + const scope = (times: readonly number[]) => [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [...times] }]; + const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'] }; + const model = sequenceClassifier([{ ...base, frameAccess: scope(parts) }, { ...base, frameAccess: scope([expected]) }]); + const decision = await classifyCapabilityWithModel({ message, model, signal: new AbortController().signal }); + expect(decision).toMatchObject({ frameAccess: scope([expected]) }); + expect(model.doGenerateCalls).toHaveLength(2); + expect(readFrameRequest(message).positions).toEqual([{ time: expected, role: 'capture', videoIds: [videoId] }]); + await expect(resolveCapabilityRoute({ persisted: { ...decision, frameAccess: scope(parts) }, message, + classify: vi.fn(), persist: vi.fn() })).resolves.toMatchObject({ responseIntent: 'clarification' }); + }); + + it.each([ + ['Capture only the frame at 1 hour 30 minutes of https://youtu.be/abcdefghijk.', [], 5400000, 1800000], + ['Capture only the linked instant at https://youtu.be/abcdefghijk?t=45 and describe it.', [], 45000, 90000], + ['Capture only the linked instant at https://www.youtube.com/watch?v=abcdefghijk&t=45.', [], 45000, 90000], + ['Capture only 0:45 of https://youtu.be/abcdefghijk.', [], 45000, 90000], + ['Capture only 0:45 of https://youtu.be/abcdefghijk?t=90s.', [], 45000, 90000], + ['Re-inspect the saved 0:30 frame of https://youtu.be/abcdefghijk. Do not recapture it if reuse fails. Capture only 1:00.', [30000], 60000, 30000], + ] as const)('repairs an unauthorized capture before the provider can receive it: %s', async (message, saved, fresh, forbidden) => { + const videoId = 'abcdefghijk'; + const route = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'] }; + const valid = [{ videoId, savedTimestampsMs: [...saved], captureTimestampsMs: [fresh] }]; + const invalid = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [fresh, forbidden] }]; + const model = sequenceClassifier([{ ...route, frameAccess: invalid }, { ...route, frameAccess: valid }]); + const assets = saved.map(time => ({ kind: 'frame' as const, version: 'a'.repeat(64), videoId, current: true, collectedAt: 1, details: { timestampMs: time } })); + const decision = await classifyCapabilityWithModel({ model, message, sessionBrief: { assets, memories: [] }, signal: new AbortController().signal }); + expect(model.doGenerateCalls).toHaveLength(2); + expect(decision).toMatchObject({ frameAccess: valid }); + if (decision.route !== 'inspect_video') throw new Error('Expected inspection.'); + const context = inspectContext(); + const frames = vi.fn>().mockResolvedValue({ value: { videoId, + frames: [{ timestampMs: fresh, width: 1, height: 1, mimeType: 'image/jpeg', imageBase64: '/9j/2Q==' }], + failures: [], meta: { partial: false, warnings: [] } }, cacheStatus: 'miss' }); + context.provider.frames = frames; + await runResearchAgentWithModel({ message, decision, context, model: finalizingModel([ + { name: 'get_video_frames', input: { videoId, timestampsMs: [fresh, forbidden] } }, + { name: 'get_video_frames', input: { videoId, timestampsMs: [fresh] } }, + ]) }); + expect(frames.mock.calls.map(([request]) => request.timestampsMs)).toEqual([[fresh]]); + + // A pre-upgrade turn being resumed must also close this path. + await expect(resolveCapabilityRoute({ persisted: { ...decision, frameAccess: invalid }, message, priorAssets: assets, + classify: vi.fn(), persist: vi.fn() })).resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it.each([ + ['Capture the linked instant at https://youtu.be/abcdefghijk?t=0', 0], + ['Capture the linked instant at https://www.youtube.com/embed/abcdefghijk?start=45', 45000], + ['Capture the linked instant at https://youtu.be/abcdefghijk#t=1m30s', 90000], + ['Capture only 0:30 of https://youtu.be/abcdefghijk?t=45', 30000], + ] as const)('resolves the exact capture position for %s', (message, time) => { + expect(readFrameRequest(message).positions).toEqual([{ videoIds: ['abcdefghijk'], role: 'capture', time }]); + }); + + it('ignores unrelated URL parameters and seek times on other hosts', () => { + expect(readFrameRequest('Inspect https://youtu.be/abcdefghijk?tracking=90s and https://example.org/watch?v=lmnopqrstuv&t=45s').positions).toEqual([]); + }); + + it.each([['0:30', 30000], ['1:00', 60000]] as const)('compares a saved frame with a new %s frame on another video', async (clock, time) => { + const a = 'abcdefghijk', b = 'lmnopqrstuv'; + const frameAccess = [{ videoId: a, savedTimestampsMs: [30000], captureTimestampsMs: [] }, + { videoId: b, savedTimestampsMs: [], captureTimestampsMs: [time] }]; + const route = { route: 'topic_research', comparisonVideoIds: [a, b], frameAccess, visualEvidence: 'required', visualRequirements: ['compare frames'] }; + const model = classifierModel(route); + const message = `Compare the saved 0:30 frame of https://youtu.be/${a} with a new ${clock} frame from https://youtu.be/${b}.`; + const assets = [{ videoId: a, kind: 'frame' as const, version: 'a'.repeat(64), current: true, collectedAt: 1, details: { timestampMs: 30000 } }]; + const decision = await classifyCapabilityWithModel({ model, message, sessionBrief: { assets, memories: [] }, signal: new AbortController().signal }); + expect(decision).toMatchObject({ route: 'topic_research', frameAccess }); + expect(model.doGenerateCalls).toHaveLength(1); + const swapped = frameAccess.map(entry => ({ ...entry, videoId: entry.videoId === a ? b : a })); + await expect(resolveCapabilityRoute({ persisted: { ...decision, frameAccess: swapped }, message, priorAssets: assets, + classify: vi.fn(), persist: vi.fn() })).resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it('keeps exact capture permissions paired with each linked video', async () => { + const [a, b] = ['abcdefghijk', 'lmnopqrstuv'] as const; + const at = (videoId: string, time: number) => ({ videoId, savedTimestampsMs: [], captureTimestampsMs: [time] }); + const route = { route: 'topic_research', comparisonVideoIds: [a, b], visualEvidence: 'required', visualRequirements: ['compare scenes'] }; + const valid = [at(a, 30000), at(b, 60000)]; + const model = sequenceClassifier([{ ...route, frameAccess: [at(a, 60000), at(b, 30000)] }, { ...route, frameAccess: valid }]); + const message = `Capture only 0:30 of https://youtu.be/${a} and capture 1:00 of https://youtu.be/${b}. Compare the scenes.`; + await expect(classifyCapabilityWithModel({ model, message, signal: new AbortController().signal })).resolves.toMatchObject({ frameAccess: valid }); + expect(model.doGenerateCalls).toHaveLength(2); + }); + + it('closes a bad capture list when every classification attempt repeats it', async () => { + const videoId = 'abcdefghijk'; + const model = classifierModel({ route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'], + frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [45000, 90000] }] }); + await expect(classifyCapabilityWithModel({ message: `Capture only 0:45 of https://youtu.be/${videoId}.`, model, + signal: new AbortController().signal })).resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it('retrieves only the requested storyboard sheet while exact capture remains closed', async () => { + const videoId = 'abcdefghijk'; + const message = `Retrieve the storyboard sheet containing 1:30 of https://youtu.be/${videoId}. Do not capture any exact frames.`; + const classifier = classifierModel({ route: 'inspect_video', videoId, frameAccess: 'storyboard_only', + visualEvidence: 'required', visualRequirements: ['scene'] }); + const decision = await classifyCapabilityWithModel({ message, model: classifier, signal: new AbortController().signal }); + expect(classifier.doGenerateCalls).toHaveLength(1); + expect(decision).toMatchObject({ frameAccess: { storyboardTimestampsMs: [90000] } }); + if (decision.route !== 'inspect_video') throw new Error('Expected inspection.'); + const resume = (persisted: unknown) => resolveCapabilityRoute({ persisted, message, classify: vi.fn(), persist: vi.fn() }); + await expect(resume(JSON.parse(JSON.stringify(decision)))).resolves.toEqual(decision); + await expect(resume({ ...decision, frameAccess: { storyboardTimestampsMs: [90000, 120000] } })) + .resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + const context = inspectContext(); + const storyboard = vi.fn>().mockResolvedValue({ cacheStatus: 'miss', + value: { videoId, frameCount: 20, intervalMs: 5000, sheets: [{ imageBase64: '/9j/2Q==', tileWidth: 100, tileHeight: 100, + columns: 2, rows: 1, firstFrameIndex: 18, frameCount: 2, intervalMs: 5000 }], meta: { partial: false, warnings: [] } } }); + context.provider.storyboard = storyboard; + const model = finalizingModel([ + { name: 'get_video_storyboard', input: { videoId, sheetIndexes: [0] } }, + { name: 'get_video_storyboard', input: { videoId, timestampsMs: [90000, 120000] } }, + { name: 'get_video_storyboard', input: { videoId, timestampsMs: [90000] } }, + ]); + await runResearchAgentWithModel({ model, message, decision, context }); + expect(storyboard.mock.calls.map(([id, times]) => [id, times])).toEqual([[videoId, [90000]]]); + expect(model.doGenerateCalls[0]?.tools?.map(tool => tool.name)).not.toContain('get_video_frames'); + }); + + it('accepts a capture list that excludes a saved frame the user set aside', async () => { + const videoId = 'abcdefghijk'; + const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + const model = sequenceClassifier([{ route: 'inspect_video', videoId, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'], frameAccess, + setAsideFrames: [{ videoId, timestampMs: 30000 }] }]); + const route = await classifyCapabilityWithModel({ model, message: `Ignore the old 0:30 frame entirely. Capture only 1:00 of https://youtu.be/${videoId}.`, + sessionBrief: { memories: [], assets: [{ videoId, version: 'a'.repeat(64), kind: 'frame', current: true, collectedAt: 1, details: { timestampMs: 30000 } }] }, + signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'inspect_video', frameAccess }); + expect(route).not.toHaveProperty('setAsideFrames'); + expect(model.doGenerateCalls).toHaveLength(1); + }); + + it.each([[false], [true]])('limits a discovery capture to 1m30s or asks (classifier keeps agent_selected: %s)', async persists => { + const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', visualEvidence: 'required', visualRequirements: ['dough'] }; + const frameAccess = [{ savedTimestampsMs: [], captureTimestampsMs: [90000] }]; + const model = sequenceClassifier([{ ...discovery, frameAccess: 'agent_selected' }, ...(persists ? [] : [{ ...discovery, frameAccess }])]); + const route = await classifyCapabilityWithModel({ model, signal: new AbortController().signal, + message: 'Find a sourdough tutorial and capture only 1m30s of that video. No other timestamps.' }); + if (persists) expect(route).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + else expect(route).toMatchObject({ route: 'topic_research', frameAccess }); + expect(route).not.toHaveProperty('frameAccess', 'agent_selected'); + }); + + // One parser serves both checks, so every format limits capture and requires the saved frame alike. + it.each(['1:30', '1m30s', '90 seconds', '1 minute 30 seconds', 'the 90-second mark'])('treats %s as a named time in both checks', async time => { + const videoId = 'abcdefghijk'; + const sessionBrief = { memories: [], assets: [{ videoId, version: 'a'.repeat(64), kind: 'frame' as const, current: true, collectedAt: 1, details: { timestampMs: 90000 } }] }; + const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'] }; + const permission = sequenceClassifier([{ ...base, frameAccess: 'agent_selected' }, + { ...base, frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [90000] }] }]); + // No saved frame here, so only the capture-permission check can reject agent_selected. + await classifyCapabilityWithModel({ model: permission, signal: new AbortController().signal, + message: `Capture only ${time} of https://youtu.be/${videoId}.` }); + expect(permission.doGenerateCalls).toHaveLength(2); + const saved = sequenceClassifier([{ ...base, frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [120000] }] }, + { ...base, frameAccess: [{ videoId, savedTimestampsMs: [90000], captureTimestampsMs: [120000] }] }]); + await classifyCapabilityWithModel({ model: saved, sessionBrief, signal: new AbortController().signal, + message: `Re-inspect the saved frame at ${time} and capture 2:00 of https://youtu.be/${videoId}.` }); + expect(saved.doGenerateCalls).toHaveLength(2); + }); + + it.each([ + 'Find sourdough videos under 5 minutes and show what the dough looks like.', + 'Find tutorials about shaping dough that are shorter than 5:00 and choose a frame.', + 'Choose a frame from a sourdough tutorial under 5 minutes.', + ])('keeps delegated capture when a length bound is the only time in the message: %s', async message => { + const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough under 5 minutes', visualEvidence: 'required', visualRequirements: ['dough'] }; + const model = sequenceClassifier([{ ...discovery, frameAccess: 'agent_selected' }]); + const route = await classifyCapabilityWithModel({ model, signal: new AbortController().signal, + message }); + expect(route).toMatchObject({ frameAccess: 'agent_selected' }); + expect(model.doGenerateCalls).toHaveLength(1); + }); + + it.each(['0:30', '30 seconds', '0m30s'])('enforces a capture window within %s of the start during discovery', async limit => { + const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', + refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['dough'] }; + const captureAt = (time: number) => [{ savedTimestampsMs: [], captureTimestampsMs: [time] }]; + const model = sequenceClassifier([{ ...discovery, frameAccess: 'agent_selected' }, { ...discovery, frameAccess: captureAt(30001) }]); + const fallbackModel = sequenceClassifier([{ ...discovery, frameAccess: captureAt(30000) }]); + const route = await classifyCapabilityWithModel({ model, fallbackModel, signal: new AbortController().signal, + message: `Find a sourdough tutorial under 5 minutes and capture a frame only within ${limit} of the start. Do not capture anything later.` }); + expect(route).toMatchObject({ route: 'topic_research', frameAccess: captureAt(30000) }); + expect(fallbackModel.doGenerateCalls).toHaveLength(1); + }); + + it('asks for clarification if every attempt leaves the capture window unrestricted', async () => { + const model = sequenceClassifier([{ route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', + frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['dough'] }]); + const route = await classifyCapabilityWithModel({ model, signal: new AbortController().signal, + message: 'Find a sourdough tutorial and capture a frame only within 0:30 of the start. Do not capture anything later.' }); + expect(route).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it('keeps a fixed capture time when discovery chooses the video', async () => { + const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', visualEvidence: 'required', visualRequirements: ['dough'] }; + const frameAccess = [{ savedTimestampsMs: [], captureTimestampsMs: [30000] }]; + const model = sequenceClassifier([{ ...discovery, frameAccess: 'agent_selected' }, { ...discovery, frameAccess }]); + const route = await classifyCapabilityWithModel({ model, message: 'Find a sourdough tutorial and capture only 0:30.', signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'topic_research', frameAccess }); + expect(model.doGenerateCalls).toHaveLength(2); + }); + + it.each([ + ['different times', 30000, 60000, 'Reuse the saved 0:30 frame and capture 1:00 of https://youtu.be/lmnopqrstuv. Compare the outfits.'], + ['the same time', 30000, 30000, 'Re-inspect the saved 0:30 frame and capture a new 0:30 frame of https://youtu.be/lmnopqrstuv. Compare the outfits.'], + ['a time in seconds', 30000, 60000, 'Reuse the saved frame at 30 seconds and capture 1:00 of https://youtu.be/lmnopqrstuv. Compare the outfits.'], + ])('recovers both requested operations when a plan drops the saved video (%s)', async (_name, savedTime, captureTime, message) => { + const [saved, linked] = ['abcdefghijk', 'lmnopqrstuv']; + const visual = { visualEvidence: 'required', visualRequirements: ['outfits'] }; + const both = [{ videoId: saved, savedTimestampsMs: [savedTime], captureTimestampsMs: [] }, { videoId: linked, savedTimestampsMs: [], captureTimestampsMs: [captureTime] }]; + const model = sequenceClassifier([ + { route: 'inspect_video', videoId: linked, ...visual, frameAccess: [both[1]] }, + { route: 'topic_research', comparisonVideoIds: [saved, linked], ...visual, frameAccess: both }, + ]); + const route = await classifyCapabilityWithModel({ model, message, signal: new AbortController().signal, + sessionBrief: { memories: [], assets: [{ videoId: saved, version: 'a'.repeat(64), kind: 'frame', current: true, collectedAt: 1, details: { timestampMs: savedTime } }] } }); + expect(route).toMatchObject({ route: 'topic_research', comparisonVideoIds: [saved, linked], frameAccess: both }); + expect(model.doGenerateCalls).toHaveLength(2); + expect(JSON.stringify(model.doGenerateCalls[1]!.prompt)).toContain(`saved 0:30 frame of ${saved}`); + }); + + it.each([ + ['a named position', 'Capture only 1:00 of https://youtu.be/abcdefghijk. No other timestamps.'], + ['a recapture prohibition', 'Look at https://youtu.be/abcdefghijk again without recapturing the saved frame, refresh the rest'], + ])('asks rather than widen %s when no attempt lists the positions', async (_name, message) => { + const model = sequenceClassifier([{ route: 'inspect_video', videoId: 'abcdefghijk', refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'] }]); + const route = await expectLastResort(classifyCapabilityWithModel({ model, message, signal: new AbortController().signal })); + expect(route).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it('asks rather than widen a capture list that no attempt stated validly', async () => { + const videoId = 'abcdefghijk'; + const model = sequenceClassifier([{ route: 'inspect_video', videoId, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'], + frameAccess: [{ videoId, savedTimestampsMs: [60000], captureTimestampsMs: [60000] }] }]); + const route = await expectLastResort(classifyCapabilityWithModel({ model, message: `Refresh the frames of https://youtu.be/${videoId}`, signal: new AbortController().signal })); + expect(route).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it('keeps frame access outside the persisted route closed on resume', async () => { + const frameAccess = { videoId: 'lmnopqrstuv', savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; + const decision = await resolveCapabilityRoute({ persisted: { route: 'inspect_video', videoId: 'abcdefghijk', researchVideoCount: 1, + useStoryboard: true, answerDetail: 'standard', frameAccess }, classify: vi.fn(), persist: vi.fn() }); + expect(decision).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + + it('names frameAccess when a saved-only request also asks for captures', async () => { + const videoId = 'abcdefghijk'; + const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [45000] }]; + const model = sequenceClassifier([ + { route: 'inspect_video', videoId, savedOnly: true, visualEvidence: 'required', visualRequirements: ['scene'], frameAccess }, + { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'], frameAccess }, + ]); + const route = await classifyCapabilityWithModel({ model, message: `Capture only 0:45 of https://youtu.be/${videoId}`, signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'inspect_video', frameAccess, refreshFramesOnly: true }); + expect(JSON.stringify(model.doGenerateCalls[1]!.prompt)).toContain('Saved-only requests cannot capture frames'); + }); + + it.each([ + ['an exact-frame question without a refresh', 'Capture only 1:00 of https://youtu.be/abcdefghijk. No other timestamps.', 60000, + { timestampedFrameQuestion: 'What is on screen at 1:00?' }], + ['a refresh at a position given in seconds', 'Refresh the frame of https://youtu.be/abcdefghijk at only 60 seconds.', 60000, + { refreshFramesOnly: true }], + ])('requires a capture list for %s', async (_name, message, time, fields) => { + const videoId = 'abcdefghijk'; + const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['screen'], ...fields }; + const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [time] }]; + const model = sequenceClassifier([base, { ...base, frameAccess }]); + const route = await classifyCapabilityWithModel({ model, message, signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'inspect_video', frameAccess }); + expect(model.doGenerateCalls).toHaveLength(2); + }); + it('pins inspect_video provider calls to the classified video', async () => { const video = vi.fn(async () => ({ cacheStatus: 'miss' as const, value: {} as never })); const provider = createCapabilityProvider(providerWith({ video }), { @@ -1135,7 +1610,8 @@ describe('YouTube agent capability router', () => { }); const toolNames = model.doGenerateCalls[0]?.tools?.map((candidate) => candidate.name); - expect(toolNames).toEqual([...INSPECT_VIDEO_TOOL_NAMES]); + // Without a retrieval grant, only stored images can be analyzed. + expect(toolNames).toEqual(INSPECT_VIDEO_TOOL_NAMES.filter(name => name !== 'get_video_frames' && name !== 'get_video_storyboard')); expect(JSON.stringify(model.doGenerateCalls[0]?.prompt)).toContain('Pinned video ID: abcdefghijk'); expect(JSON.stringify(model.doGenerateCalls[0]?.prompt)).toContain('Start with this video.'); expect(JSON.stringify(model.doGenerateCalls[0]?.prompt)).toContain('I inspected its main argument.'); @@ -1178,7 +1654,7 @@ describe('YouTube agent capability router', () => { ] as const)('gates the %s storyboard tool using classifier choice %s', async (route, useStoryboard) => { const model = finalizingModel(); await runResearchAgentWithModel({ model, message: 'Inspect the video', - decision: route === 'inspect_video' ? { route, videoId: 'abcdefghijk', useStoryboard } : { route, useStoryboard }, + decision: route === 'inspect_video' ? { route, videoId: 'abcdefghijk', useStoryboard, frameAccess: 'storyboard_only' } : { route, useStoryboard, frameAccess: 'storyboard_only' }, context: inspectContext(), // Even an explicit caller tool list cannot override the classifier's decision. toolNames: ['get_video_storyboard', 'finalize_answer'], @@ -1247,29 +1723,28 @@ function conversationTurn(overrides: Pick ({ - content: [{ - type: 'tool-call', - toolCallId: 'finalize-inspect', - toolName: 'finalize_answer', - input: JSON.stringify({ - blocks: [{ text: 'A supported finding from the video.', evidenceIds: ['ref_1'] }], - intent: 'inspect_video', - confidence: 'low', - citations: [], - artifacts: [], - warnings: [], - }), - }], - finishReason: { unified: 'tool-calls', raw: undefined }, - usage: { - inputTokens: { total: 50, noCache: 50, cacheRead: undefined, cacheWrite: undefined }, - outputTokens: { total: 20, text: 20, reasoning: undefined }, - }, - warnings: [], - }), + doGenerate: async request => { + // Invalid calls stay invalid during argument repair. Research can then try the next call. + const repair = request.responseFormat?.type === 'json'; + const call = repair ? finish : calls[step++] ?? finish; + return { + content: repair ? [{ type: 'text', text: '{}' }] : [{ type: 'tool-call', toolCallId: `inspect-${step}`, + toolName: call.name, input: JSON.stringify(call.input) }], + finishReason: { unified: repair ? 'stop' : 'tool-calls', raw: undefined }, + usage: { + inputTokens: { total: 50, noCache: 50, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 20, text: 20, reasoning: undefined }, + }, + warnings: [], + }; + }, }); } diff --git a/platform/test/youtube-agent-storyboard.test.ts b/platform/test/youtube-agent-storyboard.test.ts index 5eb15776..ec712788 100644 --- a/platform/test/youtube-agent-storyboard.test.ts +++ b/platform/test/youtube-agent-storyboard.test.ts @@ -51,6 +51,27 @@ async function retrieveAndAnalyze(input: Parameters { + it('refuses a timestamp permitted only for the other comparison video before lookup', () => { + const ctx = context(); + ctx.storyboardSelections = [{ videoId: storyboard.videoId, timestampsMs: [10000] }, { videoId: 'abcdefghijk', timestampsMs: [55000] }]; + ctx.executeEvidenceTool = vi.fn(); + expect(() => executeGetVideoStoryboard({ videoId: storyboard.videoId, timestampsMs: [55000] }, ctx, 'swapped')).toThrow('outside the grant'); + expect(ctx.executeEvidenceTool).not.toHaveBeenCalled(); + }); + + it.each([ + { videoId: storyboard.videoId }, { videoId: storyboard.videoId, maxSheets: 3 }, + { videoId: storyboard.videoId, sheetIndexes: [0] }, { videoId: storyboard.videoId, timestampsMs: [55000, 50000] }, + { videoId: 'abcdefghijk', timestampsMs: [55000] }, + ])('rejects out-of-scope sheet requests before evidence lookup or billing: %j', input => { + const ctx = context(); + ctx.storyboardSelections = [{ videoId: storyboard.videoId, timestampsMs: [55000] }]; + ctx.executeEvidenceTool = vi.fn(); + expect(() => executeGetVideoStoryboard(input, ctx, 'denied')).toThrow(); + expect(ctx.executeEvidenceTool).not.toHaveBeenCalled(); + expect(ctx.provider.storyboard).not.toHaveBeenCalled(); + }); + it.each([ { sheetIndexes: [6] }, { sheetIndexes: [5, 6, 7] }, { timestampsMs: [60000] }, diff --git a/platform/test/youtube-stored-analysis.test.ts b/platform/test/youtube-stored-analysis.test.ts index dbc74bec..3ed63a58 100644 --- a/platform/test/youtube-stored-analysis.test.ts +++ b/platform/test/youtube-stored-analysis.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it, vi } from 'vitest'; +import { buildAgentTurnResult } from '../src/agents/finalizer'; import { MockLanguageModelV4 } from 'ai/test'; import { runResearchAgentWithModel } from '../src/agents/research/research-agent'; import { z } from 'zod'; import type { AgentToolContext } from '../src/agents/providers/youtube/tool-context'; -import { evidencePacketSchema, type EvidencePacket } from '../src/agents/contracts'; +import { evidencePacketSchema, type EvidencePacket, type FinalizeAnswerInput, type FrameAccess } from '../src/agents/contracts'; import { createCapabilityToolSet } from '../src/agents/providers/youtube/tool-library'; import { executeGetVideoTranscriptForModel } from '../src/agents/providers/youtube/tools/get-video-transcript'; import { executeGetVideoFrames } from '../src/agents/providers/youtube/tools/get-video-frames'; @@ -114,10 +115,29 @@ function setup() { } const kinds = ['transcript', 'frame', 'storyboard_sheet'] as const; +/** The session's saved frames as the runtime passes them to the answer builder. */ +function inventoryOf(context: AgentToolContext) { + return context.session!.brief().assets.flatMap(asset => asset.kind === 'frame' + ? [{ version: asset.version, videoId: asset.videoId, collectedAt: asset.collectedAt, + ...(typeof asset.details.timestampMs === 'number' ? { timestampMs: asset.details.timestampMs } : {}) }] : []); +} + +/** Finalizes through the application's answer builder, which owns the frame activity notes. */ +function finalizeThroughApplication(context: AgentToolContext, packets: EvidencePacket[], access: FrameAccess, startedAt: number) { + context.runId = crypto.randomUUID(); + const identity = { runId: context.runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }; + const finalize = vi.fn(async (_id: string, input: FinalizeAnswerInput) => buildAgentTurnResult(identity, { userId: 'user', creditsRemaining: 100 }, + input, packets, 0, undefined, { access, inventory: inventoryOf(context), startedAt })); + context.finalize = finalize; + return async () => (await finalize.mock.results.at(-1)!.value).answer as string; +} + it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and only fetches 10:45 (saved image %s)', async savedState => { - const { context, provider, store, frames } = setup(); + const { context, provider, store, frames, packets } = setup(); + const frameAccess = [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]; const at = (timestampMs: number) => ({ ...frames, frames: [{ ...frames.frames[0]!, timestampMs }] }); const saved = store.put('frame', videoId, at(630000), { timestampMs: 630000 }); + const answer = finalizeThroughApplication(context, packets, frameAccess, store.runStart()); if (savedState === 'missing') store.remove(saved); provider.frames.mockResolvedValue({ value: at(645000), cacheStatus: 'miss' }); context.analyzeFrames = vi.fn(async ({ frames }) => { @@ -146,8 +166,7 @@ it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and on await runResearchAgentWithModel({ model, finalizationModel, context, toolNames: ['analyze_video_frames', 'get_video_frames', 'finalize_answer'], message: 'Re-inspect the saved 10:30 image and capture only 10:45. Report the gap if reuse fails.', - decision: { route: 'inspect_video', videoId, useStoryboard: true, refreshFramesOnly: true, visualEvidence: 'required', - frameAccess: { videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] } }, + decision: { route: 'inspect_video', videoId, useStoryboard: true, refreshFramesOnly: true, visualEvidence: 'required', frameAccess }, }); expect(provider.frames).toHaveBeenCalledOnce(); expect(provider.frames).toHaveBeenCalledWith(expect.objectContaining({ timestampsMs: [645000] }), expect.anything(), @@ -155,18 +174,318 @@ it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and on expect(vi.mocked(context.analyzeFrames).mock.calls.map(([input]) => input.frames.frames[0]!.timestampMs)) .toEqual(savedState === 'missing' ? [645000] : [630000, 645000]); const savedReport = savedState === 'available' ? 'I re-inspected the saved 10:30 frame' : 'I could not re-inspect the saved 10:30 frame'; - expect(JSON.stringify(finalizationModel.doGenerateCalls[0]!.prompt)).toContain(savedReport); - expect(context.finalize).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ - answer: expect.stringContaining(savedReport), - })); - expect(context.finalize).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ - answer: expect.stringContaining('I retrieved the 10:45 frame and inspected it.'), - })); + // Findings keep their citations; the account of tool actions is a separate, uncited paragraph. + const [findings, activity] = (await answer()).split('\n\n'); + expect(findings).toContain('[cite:'); + expect(activity).toContain(savedReport); + expect(activity).toContain('I retrieved the 10:45 frame and inspected it.'); + expect(activity).not.toContain('[cite:'); +}); + +it('re-inspects a saved frame of one video while capturing only the other video', async () => { + const { context, provider, store, frames, packets } = setup(); + const other = 'lmnopqrstuv'; + const frameAccess = [{ videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, { videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + const at = (id: string, timestampMs: number) => ({ ...frames, videoId: id, frames: [{ ...frames.frames[0]!, timestampMs }] }); + const saved = store.put('frame', videoId, at(videoId, 30000), { timestampMs: 30000 }); + const answer = finalizeThroughApplication(context, packets, frameAccess, store.runStart()); + provider.frames.mockResolvedValue({ value: at(other, 60000), cacheStatus: 'miss' }); + context.analyzeFrames = vi.fn(async ({ frames }) => ({ + findings: [{ observation: 'A blue shirt.', timestampsMs: [frames.frames[0]!.timestampMs] }], warnings: [] })); + const calls = [ + { name: 'analyze_video_frames', input: { assetVersions: [saved], focus: 'What is he wearing?' } }, + { name: 'get_video_frames', input: { videoId: other, timestampsMs: [60000], analysis: 'What is he wearing?' } }, + { name: 'complete_research', input: {} }, + ]; + const usage = { inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 1, text: 1, reasoning: 0 } }; + let step = 0; + const model = new MockLanguageModelV4({ doGenerate: async () => { + const call = calls[step++]!; + return { content: [{ type: 'tool-call', toolCallId: `step-${step}`, toolName: call.name, input: JSON.stringify(call.input) }], + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, usage, warnings: [] }; + } }); + const finalizationModel = new MockLanguageModelV4({ doGenerate: async () => ({ + content: [{ type: 'text', text: JSON.stringify({ confidence: 'medium', warnings: [], + blocks: [{ text: 'Both wear shirts.', evidenceIds: ['ref_1'] }] }) }], + finishReason: { unified: 'stop', raw: 'stop' }, usage, warnings: [], + }) }); + await runResearchAgentWithModel({ model, finalizationModel, context, + toolNames: ['analyze_video_frames', 'get_video_frames', 'finalize_answer'], + message: 'Re-inspect the saved 0:30 frame without recapturing it, and capture only 1:00 of the new video.', + decision: { route: 'topic_research', comparisonVideoIds: [videoId, other], researchVideoCount: 2, useStoryboard: true, + refreshFramesOnly: true, visualEvidence: 'required', frameAccess }, + }); + expect(provider.frames).toHaveBeenCalledOnce(); + expect(provider.frames).toHaveBeenCalledWith(expect.objectContaining({ videoId: other, timestampsMs: [60000] }), expect.anything(), + expect.anything(), expect.anything(), expect.anything()); + expect(vi.mocked(context.analyzeFrames).mock.calls.map(([input]) => [input.frames.videoId, input.frames.frames[0]!.timestampMs])) + .toEqual([[videoId, 30000], [other, 60000]]); + expect(await answer()).toMatch(/\n\nI re-inspected the saved 0:30 frame of "[^"]+"\. I retrieved the 1:00 frame of "[^"]+" and inspected it\./); +}); + +it('keeps research open until a required capture is analyzed, without an exact-frame question', async () => { + const { context, provider, packets, frames } = setup(); + provider.frames.mockResolvedValue({ value: { ...frames, frames: [{ ...frames.frames[0]!, timestampMs: 45000 }] }, cacheStatus: 'miss' }); + context.analyzeFrames = vi.fn(async ({ frames }) => ({ + findings: [{ observation: 'A blue chart.', timestampsMs: [frames.frames[0]!.timestampMs] }], warnings: [] })); + const usage = { inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 1, text: 1, reasoning: 0 } }; + const calls = [ + () => ({ name: 'get_video_frames', input: { videoId, timestampsMs: [45000] } }), + () => ({ name: 'complete_research', input: {} }), + () => ({ name: 'analyze_video_frames', input: { assetVersions: packets.at(-1)!.assetVersions, focus: 'Describe the scene.' } }), + () => ({ name: 'complete_research', input: {} }), + ]; + let step = 0; + const model = new MockLanguageModelV4({ doGenerate: async () => { + const call = calls[step++]!(); + return { content: [{ type: 'tool-call', toolCallId: `step-${step}`, toolName: call.name, input: JSON.stringify(call.input) }], + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, usage, warnings: [] }; + } }); + const finalizationModel = new MockLanguageModelV4({ doGenerate: async () => ({ + content: [{ type: 'text', text: JSON.stringify({ confidence: 'medium', warnings: [], blocks: [{ text: 'A blue chart.', evidenceIds: ['ref_1'] }] }) }], + finishReason: { unified: 'stop', raw: 'stop' }, usage, warnings: [], + }) }); + const warn = vi.spyOn(console, 'warn'); + try { + await runResearchAgentWithModel({ model, finalizationModel, context, + toolNames: ['analyze_video_frames', 'get_video_frames', 'finalize_answer'], message: 'Capture only 0:45 and describe the scene.', + decision: { route: 'inspect_video', videoId, useStoryboard: true, refreshFramesOnly: true, visualEvidence: 'required', + frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [45000] }] }, + }); + expect(warn.mock.calls.filter(([line]) => String(line).includes('complete_research did not execute'))).toHaveLength(1); + } finally { warn.mockRestore(); } + expect(vi.mocked(context.analyzeFrames).mock.calls.map(([input]) => input.frames.frames[0]!.timestampMs)).toEqual([45000]); +}); + +/** Runs scripted research steps, then keeps asking to complete. Returns how often completion was refused. */ +async function refusedCompletions(context: AgentToolContext, steps: Array<() => { name: string; input: unknown }>, + decision: Parameters[0]['decision']) { + const usage = { inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, outputTokens: { total: 1, text: 1, reasoning: 0 } }; + let step = 0; + const model = new MockLanguageModelV4({ doGenerate: async () => { + const call = steps[step++]?.() ?? { name: 'complete_research', input: {} }; + return { content: [{ type: 'tool-call', toolCallId: `step-${step}`, toolName: call.name, input: JSON.stringify(call.input) }], + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, usage, warnings: [] }; + } }); + const finalizationModel = new MockLanguageModelV4({ doGenerate: async () => ({ + content: [{ type: 'text', text: JSON.stringify({ confidence: 'medium', warnings: [], blocks: [{ text: 'Both wear shirts.', evidenceIds: ['ref_1'] }] }) }], + finishReason: { unified: 'stop', raw: 'stop' }, usage, warnings: [] }) }); + const warn = vi.spyOn(console, 'warn'); + try { + await runResearchAgentWithModel({ model, finalizationModel, context, decision, + toolNames: ['analyze_video_frames', 'get_video_frames', 'finalize_answer'], message: 'Compare the saved 0:30 frame with a new 1:00 capture.' }); + return warn.mock.calls.filter(([line]) => String(line).includes('complete_research did not execute')).length; + } finally { warn.mockRestore(); } +} + +describe('cross-video frame completion', () => { + const other = 'lmnopqrstuv'; + const frameAccess = [{ videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, { videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + const decision = { route: 'topic_research' as const, comparisonVideoIds: [videoId, other], researchVideoCount: 2, useStoryboard: true, + refreshFramesOnly: true, visualEvidence: 'required' as const, frameAccess }; + function scene(legacy = false) { + const fixture = setup(); + const at = (id: string, timestampMs: number) => ({ ...fixture.frames, videoId: id, frames: [{ ...fixture.frames.frames[0]!, timestampMs }] }); + // Older inventory rows can omit the position that the stored payload still holds. + const saved = fixture.store.put('frame', videoId, at(videoId, 30000), legacy ? undefined : { timestampMs: 30000 }); + fixture.provider.frames.mockResolvedValue({ value: at(other, 60000), cacheStatus: 'miss' }); + fixture.context.analyzeFrames = vi.fn(async ({ frames }) => ({ + findings: [{ observation: 'A shirt.', timestampsMs: [frames.frames[0]!.timestampMs] }], warnings: [] })); + const analyzed = () => vi.mocked(fixture.context.analyzeFrames!).mock.calls.map(([input]) => input.frames.videoId); + return { ...fixture, saved, analyzed }; + } + + it('keeps research open for the other video after one video is inspected', async () => { + const { context, packets, saved, analyzed } = scene(); + const refused = await refusedCompletions(context, [ + () => ({ name: 'analyze_video_frames', input: { assetVersions: [saved], focus: 'What is he wearing?' } }), + () => ({ name: 'get_video_frames', input: { videoId: other, timestampsMs: [60000] } }), + () => ({ name: 'complete_research', input: {} }), + () => ({ name: 'analyze_video_frames', input: { assetVersions: packets.at(-1)!.assetVersions, focus: 'What is he wearing?' } }), + ], decision); + expect(refused).toBe(1); + expect(analyzed()).toEqual([videoId, other]); + }); + + it('keeps other authorized captures required after one capture fails', async () => { + const { context, provider, frames } = scene(); + const at = (id: string, timestampMs: number) => ({ ...frames, videoId: id, frames: [{ ...frames.frames[0]!, timestampMs }] }); + provider.frames.mockImplementation((async (request: { videoId: string }) => { + if (request.videoId === videoId) throw new Error('Extraction failed.'); + return { value: at(other, 60000), cacheStatus: 'miss' }; + }) as never); + const refused = await refusedCompletions(context, [ + () => ({ name: 'get_video_frames', input: { videoId, timestampsMs: [35000] } }), + () => ({ name: 'complete_research', input: {} }), + () => ({ name: 'get_video_frames', input: { videoId: other, timestampsMs: [60000], analysis: 'What is he wearing?' } }), + ], { ...decision, frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [35000] }, + { videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }] }); + expect(refused).toBe(1); + expect((provider.frames.mock.calls as unknown as [{ videoId: string }][]).map(([request]) => request.videoId)).toEqual([videoId, other]); + }); + + it('tries a saved frame whose inventory omits its position before completing', async () => { + const { context, saved, analyzed } = scene(true); + const refused = await refusedCompletions(context, [ + () => ({ name: 'get_video_frames', input: { videoId: other, timestampsMs: [60000], analysis: 'What is he wearing?' } }), + () => ({ name: 'complete_research', input: {} }), + () => ({ name: 'analyze_video_frames', input: { assetVersions: [saved], focus: 'What is he wearing?' } }), + ], decision); + expect(refused).toBe(1); + expect(analyzed()).toEqual([other, videoId]); + }); +}); + +describe('application notes on frame turns', () => { + const other = 'lmnopqrstuv'; + /** An earlier turn inspected the first video at 0:30; this turn's route listed only the second video. */ + async function turns(beforeTurn?: (put: (id: string, timestampMs: number) => void) => void) { + const { context, store, frames, provider } = setup(); + const at = (id: string, timestampMs: number, width = 640) => ({ ...frames, videoId: id, frames: [{ ...frames.frames[0]!, timestampMs, width }] }); + context.analyzeFrames = vi.fn(async ({ frames }) => ({ + findings: [{ observation: 'A shirt.', timestampsMs: [frames.frames[0]!.timestampMs] }], warnings: [] })); + context.runId = crypto.randomUUID(); + const savedVersion = store.put('frame', videoId, at(videoId, 30000), { timestampMs: 30000 }); + const earlier = await analyze('frame', savedVersion, context, 'What is he wearing?'); + // Older frames saved without a recorded position, before this turn starts. + beforeTurn?.((id, timestampMs) => { store.put('frame', id, at(id, timestampMs, 480)); }); + context.runId = crypto.randomUUID(); + let startedAt = store.runStart(); + context.frameAccess = [{ videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + provider.frames.mockResolvedValue({ value: at(other, 60000), cacheStatus: 'miss' }); + const current = await executeGetVideoFrames({ videoId: other, timestampsMs: [60000], analysis: 'What is he wearing?' }, context, 'capture'); + const answer = (text: string, packets: EvidencePacket[], requestedSaved = false) => buildAgentTurnResult({ runId: context.runId, conversationId: crypto.randomUUID(), + userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }, { userId: 'user', creditsRemaining: 100 }, + { answer: text, intent: 'topic_research', confidence: 'medium', citations: [], artifacts: [], warnings: [] }, + [earlier, ...packets], 0, undefined, { requestedSaved, access: context.frameAccess, inventory: inventoryOf(context), startedAt }).answer; + /** Moves this turn's start after saves made by a test's own earlier turn. */ + const restart = () => { startedAt = store.runStart(); }; + return { context, store, at, savedVersion, earlier, current, answer, restart }; + } + const earlierNote = /The 0:30 observation( of "[^"]+")? comes from an image inspected in an earlier turn, not in this one\./; + const claimNote = 'No frame saved before this turn was re-inspected in this turn'; + + it('labels an earlier frame citation on a history answer with no frame route or inventory', async () => { + const { context, earlier } = await turns(); + const result = buildAgentTurnResult({ runId: context.runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }, + { userId: 'user', creditsRemaining: 100 }, { answer: `I re-inspected it [cite:${earlier.excerpts[0]!.id}].`, + intent: 'context_answer', confidence: 'medium', citations: [], artifacts: [], warnings: [] }, [earlier], 0); + expect(result.answer).toMatch(earlierNote); + }); + + it('summarizes several saved versions inspected in an open capture', async () => { + const { context, store, at, current, answer, restart } = await turns(); + context.frameAccess = undefined; + const versions = [30000, 40000, 50000].map(time => store.put('frame', videoId, at(videoId, time, 480), { timestampMs: time })); + restart(); + const analysis = await executeAnalyzeVideoFrames({ assetVersions: versions, focus: 'Describe the shirt' }, context, 'batch'); + const text = answer(`A shirt [cite:${current.excerpts[0]!.id}].`, [analysis, current]); + expect(text).toMatch(/I re-inspected \d+ saved frames in this turn/); + expect(text.match(/I re-inspected/g)).toHaveLength(1); + }); + + it('states that a cited observation came from an earlier turn', async () => { + const { earlier, current, answer } = await turns(); + const text = answer(`The saved frame shows a shirt [cite:${earlier.excerpts[0]!.id}]. The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current]); + expect(text).toMatch(earlierNote); + expect(text).not.toContain('The 1:00 observation'); + }); + + it.each([ + 'I inspected the saved 0:30 frame again.', + 'Both frames show shirts.', + ])('states that no saved frame was re-inspected, whatever the answer says: %s', async claim => { + const { current, answer } = await turns(); + const text = answer(`${claim} The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current], true); + expect(text).toContain(claimNote); + expect(text).toContain('I retrieved the 1:00 frame and inspected it.'); + }); + + it('counts an image served from session storage as a re-inspected saved frame', async () => { + const { savedVersion, current, answer } = await turns(); + // A capture request answered from the session returns the image saved by the earlier turn. + const reused = { ...current, assetVersions: [savedVersion] }; + const text = answer(`The frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [reused]); + expect(text).not.toContain(claimNote); + }); + + it.each([ + ['re-inspected alone', 'single', false], + ['re-inspected with another image', 'batch', true], + ])('handles a cited batch image without inventory positions when it was %s', async (_name, mode, uncertain) => { + const { context, store, at, current, answer, restart } = await turns(); + context.runId = crypto.randomUUID(); + context.frameAccess = undefined; + // Older inventory rows omit positions, so only the recorded analyses can place each image. + const [thirty, fortyFive, other] = [30000, 45000, 50000].map(time => store.put('frame', videoId, at(videoId, time, 480))); + const batch = await executeAnalyzeVideoFrames({ assetVersions: [thirty!, fortyFive!], focus: 'What is he wearing?' }, context, 'batch'); + restart(); + context.runId = current.packetId.split(':')[1]!; + const again = await executeAnalyzeVideoFrames({ assetVersions: mode === 'single' ? [thirty!] : [fortyFive!, other!], focus: 'What is he wearing?' }, context, 'again'); + const cited = batch.excerpts.find(excerpt => excerpt.startMs === 30000)!; + const text = answer(`The saved frame shows a shirt [cite:${cited.id}] [cite:${current.excerpts[0]!.id}].`, [batch, current, again]); + expect(text).not.toMatch(earlierNote); + expect(text.includes('It is not recorded whether the cited 0:30 observation')).toBe(uncertain); + }); + + it('omits unrelated saved inventory from a capture-only answer', async () => { + const { current, answer } = await turns(put => put(other, 30000)); + const text = answer(`The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current]); + expect(text).not.toContain('no recorded position'); + expect(text).not.toContain(claimNote); + }); + + it('judges each citation by its own image when an earlier batch held several', async () => { + const { context, store, at, current, answer, restart } = await turns(); + // An earlier turn analyzed 0:30 and 0:45 together; this turn re-inspects only the exact 0:30 image. + context.runId = crypto.randomUUID(); + context.frameAccess = undefined; + const [thirty, fortyFive] = [30000, 45000].map(time => store.put('frame', videoId, at(videoId, time, 480), { timestampMs: time })); + const batch = await executeAnalyzeVideoFrames({ assetVersions: [thirty!, fortyFive!], focus: 'What is he wearing?' }, context, 'batch'); + restart(); + context.runId = current.packetId.split(':')[1]!; + context.frameAccess = [{ videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, { videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; + const again = await analyze('frame', thirty!, context, 'What is he wearing?'); + const cited = batch.excerpts.find(excerpt => excerpt.startMs === 30000)!; + const text = answer(`The saved frame shows a shirt [cite:${cited.id}] [cite:${current.excerpts[0]!.id}].`, [batch, current, again]); + expect(text).toContain('I re-inspected the saved 0:30 frame'); + expect(text).not.toMatch(earlierNote); + expect(text).not.toContain(claimNote); + }); + + it('keeps the earlier-turn note when this turn inspected a different image at the same moment', async () => { + const { context, store, at, earlier, current, answer } = await turns(); + context.frameAccess = [{ videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, ...context.frameAccess!]; + const replacement = await analyze('frame', store.put('frame', videoId, at(videoId, 30000, 320), { timestampMs: 30000 }), context, 'What is he wearing?'); + const text = answer(`The saved frame shows a shirt [cite:${earlier.excerpts[0]!.id}] [cite:${current.excerpts[0]!.id}].`, [current, replacement]); + expect(text).toMatch(earlierNote); + }); + + it('adds neither note when this turn re-inspected the cited image', async () => { + const { context, savedVersion, earlier, current, answer } = await turns(); + context.frameAccess = [{ videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }, ...context.frameAccess!]; + const again = await analyze('frame', savedVersion, context, 'What is he wearing?'); + const text = answer(`I re-inspected the saved frame: a shirt [cite:${earlier.excerpts[0]!.id}] [cite:${current.excerpts[0]!.id}].`, [current, again]); + expect(text).not.toMatch(earlierNote); + expect(text).not.toContain(claimNote); + }); +}); + +it('enforces a capture time fixed before discovery on whichever video is chosen', async () => { + const { context, provider, frames } = setup(); + context.frameAccess = [{ savedTimestampsMs: [], captureTimestampsMs: [30000] }]; + provider.frames.mockResolvedValue({ value: { ...frames, videoId: 'lmnopqrstuv', frames: [{ ...frames.frames[0]!, timestampMs: 30000 }] }, cacheStatus: 'miss' }); + expect(() => executeGetVideoFrames({ videoId: 'lmnopqrstuv', timestampsMs: [60000] }, context, 'other-time')).toThrow('Capture is limited'); + await executeGetVideoFrames({ videoId: 'lmnopqrstuv', timestampsMs: [30000] }, context, 'fixed-time'); + expect((provider.frames.mock.calls as unknown as [{ videoId: string; timestampsMs: number[] }][])[0]![0]) + .toMatchObject({ videoId: 'lmnopqrstuv', timestampsMs: [30000] }); }); it('rejects forbidden positions and other videos before metadata, billing or extraction', () => { const { context, provider } = setup(); - context.frameAccess = { videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; + context.frameAccess = [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]; context.maxVideoSeconds = 7200; context.executeEvidenceTool = vi.fn(); for (const request of [ @@ -174,6 +493,14 @@ it('rejects forbidden positions and other videos before metadata, billing or ext { videoId, timestampsMs: [646000] }, { videoId: 'lmnopqrstuv', timestampsMs: [645000] }, ]) expect(() => executeGetVideoFrames(request, context, 'forbidden')).toThrow('Capture is limited'); + // With several videos, each position is authorized only on its own video. + context.frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [645000] }, + { videoId: 'lmnopqrstuv', savedTimestampsMs: [30000], captureTimestampsMs: [60000] }]; + for (const request of [ + { videoId, timestampsMs: [60000] }, + { videoId: 'lmnopqrstuv', timestampsMs: [645000] }, + { videoId: 'lmnopqrstuv', timestampsMs: [30000] }, + ]) expect(() => executeGetVideoFrames(request, context, 'forbidden')).toThrow('Capture is limited'); expect(context.executeEvidenceTool).not.toHaveBeenCalled(); expect(provider.frames).not.toHaveBeenCalled(); }); @@ -181,7 +508,7 @@ it('rejects forbidden positions and other videos before metadata, billing or ext it('keeps the freshness requirement for a previously saved version at the new capture position', async () => { const { context, store, frames } = setup(); context.refreshFramesOnly = true; - context.frameAccess = { videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; + context.frameAccess = [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]; const stale = store.put('frame', videoId, { ...frames, frames: [{ ...frames.frames[0]!, timestampMs: 645000 }] }); await expect(analyze('frame', stale, context)).rejects.toThrow('requires a fresh capture'); expect(context.analyzeFrames).not.toHaveBeenCalled(); From a04f652faca5a41737a100df61ec70ce29d0d025 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Mon, 9 Nov 2026 15:59:01 +0530 Subject: [PATCH 2/2] let the agent interpret frame requests and enforce its scope in the tools --- platform/package.json | 3 +- .../replays/visual-inspection.config.ts | 2 +- .../scripts/replays/weather-map.live.test.ts | 79 ++++++ platform/src/agents/agent-runtime-do.ts | 6 +- platform/src/agents/contracts.ts | 8 +- platform/src/agents/finalizer.ts | 4 +- .../src/agents/research/capability-router.ts | 115 ++++---- platform/src/agents/research/frame-request.ts | 209 --------------- .../src/agents/research/research-agent.ts | 8 +- .../skills/visual-inspection.generated.ts | 2 +- .../skills/visual-inspection/SKILL.md | 2 +- .../agents/research/timestamp-expression.ts | 65 ----- .../src/agents/research/youtube-references.ts | 15 +- platform/src/agents/runtime/frame-access.ts | 4 +- .../test/frame-timestamp-expression.test.ts | 53 ---- platform/test/youtube-agent-router.test.ts | 250 ++---------------- platform/test/youtube-stored-analysis.test.ts | 80 +++--- platform/tsconfig.agent-live.json | 2 +- 18 files changed, 227 insertions(+), 680 deletions(-) create mode 100644 platform/scripts/replays/weather-map.live.test.ts delete mode 100644 platform/src/agents/research/frame-request.ts delete mode 100644 platform/src/agents/research/timestamp-expression.ts delete mode 100644 platform/test/frame-timestamp-expression.test.ts diff --git a/platform/package.json b/platform/package.json index 0ed20e45..599c3ee9 100644 --- a/platform/package.json +++ b/platform/package.json @@ -18,7 +18,8 @@ "test:auth": "vitest run test/auth-config.test.ts test/authentication.test.ts test/cli-auth-boundary.test.ts test/account-route.test.ts", "test:container": "npm --prefix youtube-processor test", "test:agent:dashboard": "npm --prefix ../web run build -- --webpack && playwright test --config playwright.agent-dashboard.config.ts", - "test:agent:visual-replay": "vitest run --config scripts/replays/visual-inspection.config.ts", + "test:agent:visual-replay": "vitest run --config scripts/replays/visual-inspection.config.ts visual-inspection.live.test.ts", + "test:agent:weather-replay": "vitest run --config scripts/replays/visual-inspection.config.ts weather-map.live.test.ts", "test:agent:live": "tsc --noEmit -p tsconfig.agent-live.json && tsx scripts/test-youtube-agent-live.ts", "test:user-account": "vitest run --config vitest.user-account.config.ts", "types": "wrangler types", diff --git a/platform/scripts/replays/visual-inspection.config.ts b/platform/scripts/replays/visual-inspection.config.ts index c26c5613..e32faa2a 100644 --- a/platform/scripts/replays/visual-inspection.config.ts +++ b/platform/scripts/replays/visual-inspection.config.ts @@ -4,5 +4,5 @@ import { parseEnv } from 'node:util'; import { resolve } from 'node:path'; const vars = process.env.AGENT_REPLAY_ENV_FILE ? parseEnv(readFileSync(process.env.AGENT_REPLAY_ENV_FILE, 'utf8')) : process.env; export default defineConfig({ test: { root: import.meta.dirname, environment: 'node', globals: true, - include: ['visual-inspection.live.test.ts'], setupFiles: [resolve(import.meta.dirname, '../../test/setup.ts')], + include: ['*.live.test.ts'], setupFiles: [resolve(import.meta.dirname, '../../test/setup.ts')], testTimeout: 900_000, env: { FIREWORKS_API_KEY: vars.FIREWORKS_API_KEY ?? vars.FIREWORKS_API_KEY_1 ?? '' } } }); diff --git a/platform/scripts/replays/weather-map.live.test.ts b/platform/scripts/replays/weather-map.live.test.ts new file mode 100644 index 00000000..7386b387 --- /dev/null +++ b/platform/scripts/replays/weather-map.live.test.ts @@ -0,0 +1,79 @@ +// Live language decisions, local image tools. The saved analysis fails deterministically. +// This fixture contains no private session records or original video images. +import { expect, it, vi } from 'vitest'; +import { createAgentModel, createClassifierFallbackModel } from '../../src/agents/model'; +import { classifyCapabilityWithModel } from '../../src/agents/research/capability-router'; +import { runResearchAgentWithModel } from '../../src/agents/research/research-agent'; +import { buildAgentTurnResult } from '../../src/agents/finalizer'; +import type { EvidencePacket } from '../../src/agents/contracts'; +import type { AgentToolContext } from '../../src/agents/providers/youtube/tool-context'; +import { attachTestAssetStore } from '../../test/fixtures/analysis-session'; + +it('replays the 3:30 saved-analysis failure and the separately permitted 3:48 capture', async () => { + const videoId = 'abcdefghijk'; + const message = 'Re-inspect the saved 3:30 weather map. If that fails, report the gap and do not recapture it. ' + + 'Capture only one new still at 3:48. Read its displayed day/time and any readable temperature or wind numbers with their locations. ' + + 'Compare the overlays and say whether Ireland becomes more involved, only if the images support that.'; + const runId = crypto.randomUUID(); + const signal = AbortSignal.timeout(200_000); + const env = { FIREWORKS_API_KEY: process.env.FIREWORKS_API_KEY, AGENT_GLM_PROVIDER: 'fireworks', + AGENT_FINALIZER_PROVIDER: 'fireworks', AGENT_FINALIZER_MODEL: 'glm-5p3-flash', AGENT_FINALIZER_REASONING_EFFORT: 'medium', + AI_GATEWAY_ID: '' } as unknown as Env; + const model = (model_role: string) => createAgentModel(env, runId, 'low', { model_role, agent_run_id: runId }); + const frame = (timestampMs: number) => ({ videoId, frames: [{ timestampMs, width: 640, height: 360, + mimeType: 'image/jpeg', imageBase64: '/9j/2Q==' }], failures: [], meta: { partial: false, warnings: [] } }); + const frames = vi.fn(async (request: { timestampsMs: number[] }) => { + expect(request.timestampsMs).toEqual([228000]); + return { value: frame(228000), cacheStatus: 'miss' }; + }); + const storyboard = vi.fn(async () => { throw new Error('Unexpected storyboard retrieval'); }); + const packets: EvidencePacket[] = []; + const inspections: number[] = []; + const imageRequests: { name: string; input: unknown }[] = []; + const context: AgentToolContext = { runId, signal, transcriptPolicy: { mode: 'complete_transcript' }, provider: { frames, storyboard } as unknown as AgentToolContext['provider'], + traceToolCall: async call => { + if (call.source === 'model' && ['get_video_frames', 'get_video_storyboard'].includes(call.name)) + imageRequests.push({ name: call.name, input: call.input }); + return call.execute(); + }, + analyzeFrames: async ({ frames }) => { + const time = frames.frames[0]!.timestampMs; + inspections.push(time); + if (time === 210000) throw new Error('Saved image analysis failed.'); + return { findings: [{ observation: 'The day label reads Tuesday; a 12 appears next to Dublin.', timestampsMs: [time] }], warnings: [] }; + }, + executeEvidenceTool: async call => { const result = await call.execute(); packets.push(result); return result; }, + getEvidence: () => packets, finalize: vi.fn(), + }; + const store = attachTestAssetStore(context); + const saved = store.put('frame', videoId, frame(210000), { timestampMs: 210000 }); + const startedAt = store.runStart(); + const decision = await classifyCapabilityWithModel({ message, sessionBrief: context.session!.brief(), model: model('classifier'), + fallbackModel: createClassifierFallbackModel(env, runId), signal }); + if (decision.route !== 'inspect_video') throw new Error('Expected bounded inspection: ' + JSON.stringify(decision)); + expect(decision.frameAccess).toEqual([{ videoId, savedTimestampsMs: [210000], captureTimestampsMs: [228000] }]); + if (!Array.isArray(decision.frameAccess)) throw new Error('Missing exact positions'); + const access = decision.frameAccess; + let answer = ''; + let modelAnswer = ''; + context.finalize = async (_id, input) => { + modelAnswer = input.answer; + const result = buildAgentTurnResult({ runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }, + { userId: 'replay', creditsRemaining: 100 }, input, packets, 0, undefined, + { access, startedAt, inventory: [{ videoId, version: saved, collectedAt: 1, timestampMs: 210000 }] }); + answer = result.answer; + return result; + }; + await runResearchAgentWithModel({ model: model('agent_core'), finalizationModel: model('finalizer'), context, message, decision }); + process.stdout.write(JSON.stringify({ replay: 'weather-map', inspections, imageRequests, captureRequests: frames.mock.calls.map(([request]) => request.timestampsMs), modelAnswer, answer }) + '\n'); + expect(imageRequests).toEqual([{ name: 'get_video_frames', input: expect.objectContaining({ videoId, timestampsMs: [228000] }) }]); + expect(inspections).toContain(210000); + expect(inspections).toContain(228000); + expect(frames.mock.calls.map(([request]) => request.timestampsMs)).toEqual([[228000]]); + expect(storyboard).not.toHaveBeenCalled(); + expect(modelAnswer).toMatch(/Tuesday/i); + expect(modelAnswer).toContain('3:30'); + expect(modelAnswer).toMatch(/could(?:n't| not)|unable|cannot|failed|unavailable|couldn’t|wasn.t/i); + expect(answer).toContain('I could not re-inspect the saved 3:30 frame'); + expect(answer).toContain('I retrieved the 3:48 frame and inspected it.'); +}); diff --git a/platform/src/agents/agent-runtime-do.ts b/platform/src/agents/agent-runtime-do.ts index bc69f9a6..8b675fe7 100644 --- a/platform/src/agents/agent-runtime-do.ts +++ b/platform/src/agents/agent-runtime-do.ts @@ -14,7 +14,6 @@ import { storedExtractionDiagnosticSchema, type StoredExtractionDiagnostic } fro import { transcriptDiagnosticSchema, type TranscriptDiagnostic } from './runtime/transcript-diagnostics'; import { agentDraftSchema, agentRunProgressSchema, toolTrace, type AgentDraft } from './runtime/run-progress'; import { framePreviewKey, recordedFramePreviews, saveFramePreviews, type SavedFrame } from './runtime/frame-previews'; -import { readFrameRequest } from './research/frame-request'; import { frameList, isFrameTurn } from './runtime/frame-access'; import { saveStoryboardPreviews } from './runtime/storyboard-previews'; import { compactAgentRun, compactAgentResult } from './response'; @@ -1041,8 +1040,7 @@ export class AgentRuntimeDO extends Agent { const history = this.readConversationHistory(run); const citedIds=answerCitationIds(parsedInput.answer); const citedSessionEvidence=this.sessionStore.evidenceForCitations(citedIds); - const requestedSaved = readFrameRequest(run.execution_message ?? run.message).requestsSaved; - const needsFrameInventory = requestedSaved || isFrameTurn(decision) || citedSessionEvidence.some(packet => packet.kind === 'youtube_frames'); + const needsFrameInventory = isFrameTurn(decision) || citedSessionEvidence.some(packet => packet.kind === 'youtube_frames'); const result = buildAgentTurnResult({ runId, conversationId: run.conversation_id, @@ -1055,7 +1053,7 @@ export class AgentRuntimeDO extends Agent { requestedVideoIds: decision.route === 'inspect_video' ? [decision.videoId] : decision.comparisonVideoIds ?? [], // An application fallback is checked against what this run had, not earlier turns' cited content. runPackets: this.admittedRunEvidence(runId, history), - } : undefined, needsFrameInventory ? { requestedSaved, access: frameList(decision), startedAt: run.created_at, + } : undefined, needsFrameInventory ? { access: frameList(decision), startedAt: run.created_at, inventory: this.sessionStore.brief().assets.flatMap(asset => asset.kind === 'frame' ? [{ version: asset.version, videoId: asset.videoId, collectedAt: asset.collectedAt, ...(typeof asset.details.timestampMs === 'number' ? { timestampMs: asset.details.timestampMs } : {}) }] : []) } : undefined); // Hash the accepted text: unmatched citations may have been removed from the input. diff --git a/platform/src/agents/contracts.ts b/platform/src/agents/contracts.ts index 91d0312c..24aa66ef 100644 --- a/platform/src/agents/contracts.ts +++ b/platform/src/agents/contracts.ts @@ -31,9 +31,13 @@ export const frameAccessSchema = z.preprocess( export type FrameAccess = z.infer; /** Model choices for visual retrieval without a per-video exact-frame list. */ export const unscopedImageAccessSchema = z.enum(['storyboard_only', 'agent_selected']); +const storyboardSelectionsSchema = z.object({ + storyboardSelections: z.array(z.object({ videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), timestampsMs: frameTimes.min(1) })).min(1).max(8), +}); +export const classifierFrameAccessSchema = z.union([frameAccessListSchema, unscopedImageAccessSchema, storyboardSelectionsSchema]); export const routeFrameAccessSchema = z.union([frameAccessSchema, unscopedImageAccessSchema, - z.object({ storyboardTimestampsMs: frameTimes.min(1) }), - z.object({ storyboardSelections: z.array(z.object({ videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), timestampsMs: frameTimes.min(1) })).min(1).max(8) })]); + // Older single-video routes stored sheet positions without the video ID. + z.object({ storyboardTimestampsMs: frameTimes.min(1) }), storyboardSelectionsSchema]); export type RouteFrameAccess = z.infer; /** Evidence the user explicitly asked to be read before answering. Research cannot complete until each is read or unavailable. */ diff --git a/platform/src/agents/finalizer.ts b/platform/src/agents/finalizer.ts index cfdf4e8d..117b3d02 100644 --- a/platform/src/agents/finalizer.ts +++ b/platform/src/agents/finalizer.ts @@ -31,7 +31,7 @@ export function buildAgentTurnResult( creditsCharged: number, durationLimitContext?: DurationLimitAnswerContext, /** Present when the route asked for frame work: its exact access, and the session's saved frames. */ - frameTurn?: { requestedSaved?: boolean; access?: FrameAccess; inventory: readonly InventoryFrame[]; startedAt: number }, + frameTurn?: { access?: FrameAccess; inventory: readonly InventoryFrame[]; startedAt: number }, ): AgentTurnResult { const markers = answerCitationIds(input.answer); const citations: AgentCitation[] = []; @@ -84,7 +84,7 @@ export function buildAgentTurnResult( const notes = [...playlistCoverage.notes]; // The application, not the model, states which frames this turn actually inspected. if (citedObservations.length || (frameTurn && input.intent !== 'clarification' && input.intent !== 'rejected')) { - notes.push(...frameTurnNotes({ runId: identity.runId, startedAt: frameTurn?.startedAt ?? 0, requestedSaved: frameTurn?.requestedSaved, access: frameTurn?.access, inventory: frameTurn?.inventory ?? [], packets, cited: citedObservations })); + notes.push(...frameTurnNotes({ runId: identity.runId, startedAt: frameTurn?.startedAt ?? 0, access: frameTurn?.access, inventory: frameTurn?.inventory ?? [], packets, cited: citedObservations })); } const disclosedAnswer = appendNotices(answer, notes); diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index 250c615b..76f0ffba 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -8,8 +8,7 @@ import { capabilityRouteDecisionSchema, answerDetailSchema, comparisonVideoIdsSchema, - frameAccessListSchema, - unscopedImageAccessSchema, + classifierFrameAccessSchema, requestedEvidenceSchema, numberedItemCountSchema, visualEvidenceSchema, @@ -21,7 +20,6 @@ import { import { sessionBriefForModel, type SessionBrief } from '../runtime/session-evidence'; import { extractYouTubeVideoIds, youTubeUrls } from './youtube-references'; export { extractYouTubeVideoIds } from './youtube-references'; -import { frameRequestIssue, readFrameRequest, requestedSheets, type FrameRequest } from './frame-request'; import { conversationAssistantMessage, type ConversationTurn } from '../runtime/conversation-memory'; import { assertModelCostAvailable, type AgentModelCostBudget } from '../runtime/model-budget'; import { AGENT_CLASSIFICATION_TIMEOUT_MS, withRunDeadline } from '../runtime/deadline'; @@ -34,8 +32,7 @@ const classifierDecisionSchema = z.object({ skill: z.enum(['verify_saved_evidence']).optional().describe('Verify saved evidence: use for follow-ups that read saved original captions/comments, check exact quotations or precise passage links, or correct source claims. It does not by itself prohibit fetching; set savedOnly separately. Omit for broad summaries and visual analysis.'), requestedEvidence: requestedEvidenceSchema.optional().describe('Evidence the user explicitly asks to be read before answering: original_quotes for exact quotations, verbatim wording or precise passage links from captions; comments for what viewers or commenters say. Research must read each or report it unavailable. Omit when not explicitly requested.'), savedOnly: z.boolean().optional().describe('True when the user requires existing saved evidence only or prohibits fresh retrieval. All provider access is disabled for this run; use saved evidence reads and report missing assets.'), - frameAccess: z.union([frameAccessListSchema, unscopedImageAccessSchema]).optional().describe('Permission to retrieve new images. Omit to disable both exact captures and storyboard retrieval. Use storyboard_only for requested storyboard sheets or an overview. A named sheet time limits the retrieval to the sheet containing that time. When the user fixes frame positions, saved or new, list one entry per video the turn touches: captureTimestampsMs holds only the permitted NEW capture positions in milliseconds, empty when nothing may be captured; savedTimestampsMs holds saved positions to re-inspect without recapture, even if their assets are missing. When the videos are still to be found by discovery, give one entry without videoId holding the capture times. Required for exact-timestamp questions and for any limit on what may be captured. Use agent_selected only when new frames may help and the user leaves every position to you.'), - setAsideFrames: z.array(z.object({ videoId: z.string().regex(/^[A-Za-z0-9_-]{11}$/), timestampMs: z.number().int().nonnegative() })).max(24).optional().describe('Saved frames from the session inventory at a time the user named that this turn must not use: the user excluded them, or the time refers to another video. Every saved frame at a named time belongs either in its video\'s frameAccess entry or here.'), + frameAccess: classifierFrameAccessSchema.optional().describe('Image access for this turn. Omit for no retrieval. For exact positions, list savedTimestampsMs to inspect from storage and captureTimestampsMs to request, in milliseconds per video. Omit videoId only for positions fixed before discovery. For named storyboard sheets use {storyboardSelections: [{videoId, timestampsMs}]}; storyboard_only allows any sheet but no exact captures. Use agent_selected only when the user delegates the choice of images and positions.'), timestampedFrameQuestion: z.string().trim().min(1).max(500).optional().describe('For inspect_video: when the user supplies exact timestamps and asks what is visible, copy the focused visual question here. Include this for a request to describe each specified frame. Omit for image retrieval only or saved-only requests.'), refreshFramesOnly: z.boolean().optional().describe('For inspection or research, including multi-video comparisons: true when the user asks to freshly capture or refresh frames only. Reuse verified video metadata. Leave refreshEvidence unset unless other source evidence must also be refreshed.'), refreshEvidence: z.boolean().optional().describe('Except for a frames-only refresh, true only when the user explicitly asks to fetch again, refresh or get fresh source data. Choose an executable route in that case.'), @@ -181,23 +178,9 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio if (input.refreshFramesOnly === true && input.visualEvidence === 'none') issue('visualEvidence', 'Refreshing frames requires visual tool access.'); const scope = parsed.success && parsed.data.frameAccess; if (scope && input.route === 'finalize') issue('route', 'Image retrieval or saved-frame inspection requires an executable route.'); - if (typeof scope === 'string' && input.savedOnly === true) - issue('frameAccess', 'Saved-only requests cannot retrieve new images. Omit frameAccess, or list only savedTimestampsMs.'); - if (Array.isArray(scope)) { - const scoped = scope.flatMap(entry => entry.videoId ? [entry.videoId] : []); - const open = scope.filter(entry => !entry.videoId); - // Every scoped video must be a subject of the route, or its frames cannot be read or captured. - const subjects = input.route === 'inspect_video' ? [input.videoId] - : input.route === 'topic_research' && Array.isArray(input.comparisonVideoIds) ? input.comparisonVideoIds : []; - if (input.savedOnly === true && capturesFrames) - issue('frameAccess', 'Saved-only requests cannot capture frames. List only savedTimestampsMs, or omit savedOnly if the user authorized new captures.'); - else if (scoped.some(id => !suppliedVideoIds.includes(id) || !subjects.includes(id)) || new Set(scoped).size !== scoped.length) - issue('frameAccess', 'List one frameAccess entry per video, only for the route\'s subjects. When saved or new frames span several videos, use topic_research with every one in comparisonVideoIds.'); - else if (open.length && (open.length > 1 || scope.length > 1 || open.some(entry => entry.savedTimestampsMs.length) - || input.route !== 'topic_research' || input.comparisonVideoIds !== undefined)) - issue('frameAccess', 'An entry without videoId fixes capture times for videos that discovery will select. Use it alone, once, without saved positions, and only for topic_research without comparisonVideoIds.'); - else if (scope.some(entry => new Set([...entry.savedTimestampsMs, ...entry.captureTimestampsMs]).size !== entry.savedTimestampsMs.length + entry.captureTimestampsMs.length)) - issue('frameAccess', 'Give each timestamp exactly one role, saved inspection or fresh capture.'); + if (parsed.success && input.route !== 'finalize') { + const issueText = imageScopeIssue(parsed.data, suppliedVideoIds); + if (issueText) issue('frameAccess', issueText); } if (input.timestampedFrameQuestion && input.route !== 'inspect_video') issue('route', 'Exact frame requests require single-video inspection.'); @@ -288,7 +271,6 @@ const RECONSIDERATION_MIN_MS = 1_500; async function classifyWithinDeadline(input: CapabilityClassifierInput, deadlineAt: number): Promise { assertModelCostAvailable(input.modelBudget); - const request = readFrameRequest(input.message); const conversationHistory = input.conversationHistory ?? []; const videoIds = [...new Set([ ...extractYouTubeVideoIds(input.message), @@ -304,14 +286,14 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline 'Requests for current view counts, likes, or comments require an executable route with refreshDynamicData true, even when past values are in history. Use saved data for historical questions. This does not require refreshing transcripts or images.', 'Classify the current request for an agent that researches and synthesizes information from YouTube videos. Decide scope before selecting tools.', 'Playlist discovery and listing are supported. For a fresh YouTube playlist listing by URL or name choose topic_research, playlistListing true, focused breadth, a playlist search query and visualEvidence none. Listing titles, links, durations, totals and arithmetic from those durations are supported, including when the user gives their answer a study-note name. Earlier assistant refusals are not capability limits. For a saved-only listing reuse retained evidence instead.', - 'The session inventory describes available raw assets, their collection times and coverage. Session memories are derived hints, not proof. A saved transcript can support new questions through research or inspection, which must prepare its content before finalization. Counts of frames or sheets do not prove that a requested scene was observed. Select inspection if new visual interpretation is needed. For fresh frames, including a request comparing several videos, set refreshFramesOnly true and leave refreshEvidence unset. If timestamps and a visual question are supplied, set timestampedFrameQuestion even when the user also names get_video_frames. Keep raw-image-only requests free of an analysis question. Refreshing transcripts or all source evidence requires refreshEvidence true. Requests limited to current statistics or comments use refreshDynamicData true instead.', + 'The session inventory describes available raw assets, their collection times and coverage. Session memories are derived hints, not proof. A saved transcript can support new questions through research or inspection, which must prepare its content before finalization. Counts of frames or sheets do not prove that a requested scene was observed. Select inspection if new visual interpretation is needed. For fresh frames, including a request comparing several videos, set refreshFramesOnly true and leave refreshEvidence unset. If exact frames and a visual question are requested, set timestampedFrameQuestion even when the user also names get_video_frames. Keep raw-image-only requests free of an analysis question. Refreshing transcripts or all source evidence requires refreshEvidence true. Requests limited to current statistics or comments use refreshDynamicData true instead.', 'Use prior completed turns and availableEvidence to choose the next action. Choose finalize with responseIntent context_answer when the request can be answered from the conversation or supplied evidence without further retrieval or analysis. Prior assistant claims are not verified source evidence. Questions about what was previously said may use history alone; new video facts require supplied evidence. An inventory reference alone does not mean its content is ready for the answer. Finalization cannot search or read saved assets. Choose research or inspection if answering requires reading a stored transcript, analyzing an asset, or loading additional evidence. When evidence is insufficient or the user asks for new inspection or fresh data, choose inspect_video or topic_research.', 'Requests to list, quote, summarize, or correct messages in this conversation are supported. Route them to finalize with responseIntent context_answer. The runtime supplies a bounded chronological history page before finalization, alongside the recent conversation window. Finalization cannot search history, paginate, read assets, or retrieve provider evidence. It must state any history coverage limit. Choose inspect_video or topic_research when source evidence needs reading or analysis; use context_answer only for prepared context. Do not list the messages yourself.', 'Select skill verify_saved_evidence for follow-ups that recover saved original captions/comments, verify exact quotations or passage links, or correct source claims. This replaces general discovery and analysis with a verification workflow. The skill does not prohibit fetching: a request that also needs evidence not yet saved, such as comments, still gets it unless savedOnly is set. Omit the skill for broad summaries or visual analysis.', 'Set requestedEvidence for each kind of evidence the user explicitly asks for: original_quotes for exact quotations, verbatim wording or precise passage links; comments for what viewers or commenters say. Include every requested kind in one decision, alongside any explanation. Research cannot complete until each is read or reported unavailable, so choose an executable route.', - 'Set savedOnly true whenever the user prohibits fresh retrieval or asks to use existing saved sources only. Keep this restriction on follow-ups when it remains in force. Missing saved evidence is a gap, never permission to fetch. Use the saved reader instead of transcript retrieval or discovery.', + 'Set savedOnly true whenever the user prohibits fresh retrieval or asks to use existing saved sources only. Keep this restriction on follow-ups when it remains in force. For mixed requests, leave savedOnly unset and list saved and new positions separately. Missing saved evidence is a gap, never permission to fetch. Use the saved reader instead of transcript retrieval or discovery.', 'A video length limit filters the source. A capture window limits the seek positions. For a tutorial under five minutes with a capture within the first 30 seconds, choose a captureTimestampsMs value between 0 and 30000 inclusive. Use an exact list even when the user lets you choose a position inside the window.', - 'Set frameAccess from the user request. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. If the images belong to different videos, include both in comparisonVideoIds. A saved image that cannot be read stays a saved inspection. Times mentioned for comparison, narration or history do not authorize capture.', + 'Set frameAccess from the user request. For example, re-inspect saved 10:30 and capture only 10:45 means [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]. If the images belong to different videos, include both in comparisonVideoIds. A saved image that cannot be read stays a saved inspection. Preserve restrictions in linked seek times and compound times (1 hour 30 minutes is one position at 5400000 ms). Excluded images and times mentioned only as history do not authorize retrieval. If a limit is unclear, ask for clarification instead of granting unrestricted retrieval.', 'Requests for exact quotations from original saved captions/comments, precise passage-start links, or explicit original-asset reads require inspect_video or topic_research with the resolved subjects, even if prior summaries discuss the topic. Research can call search_originals without fetching YouTube. Quoting a source is different from quoting a prior conversation message. Do not route an original-source read directly to finalize.', 'For every context_answer set contextScope: history for listing, quoting, recalling or correcting conversation messages or preferences; video for claims about video content; mixed only when the requested answer needs both. A video URL inside a quoted earlier message does not require video evidence.', 'For finalize give a short routing reason, not a user-facing answer. Choose responseIntent clarification for missing scope, or rejected for unsupported requests. Do not use the legacy clarification or rejected routes for new decisions.', @@ -392,7 +374,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline }] : parsed.success ? [] : parsed.error.issues.map(issue => ({ path: issue.path.map(String).join('.'), code: issue.code, message: issue.message, })); - if (parsed.success && feedback.length === 0) feedback = semanticIssues(parsed.data, input, videoIds, request); + if (parsed.success && feedback.length === 0) feedback = semanticIssues(parsed.data, input, videoIds); // Advisory only: the reconsideration may keep its choice, so a keyword match never fails classification. let advisory = false; if (parsed.success && feedback.length === 0 && context.attempt === 1) { @@ -426,7 +408,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline const lastResort = (evaluations: Evaluation[]) => { // The fallback uses no model, but an exhausted budget still ends the run. assertModelCostAvailable(input.modelBudget); - const built = lastResortDecision(evaluations, input, videoIds, channelIds, explicitVideoIds, request); + const built = lastResortDecision(evaluations, input, videoIds, channelIds, explicitVideoIds); console.warn(JSON.stringify({ event: 'agent_classification_last_resort', modelCallId: callId, method: built.method, candidates: evaluations.length, defaultedFields: built.defaultedFields })); input.onDiagnostic?.({ attempt: 0, stage: 'last_resort', lastResort: built.method, outcome: 'valid', modelId: '', @@ -436,7 +418,7 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline const decision = await coordinateClassification(call, input.signal, deadlineAt, callId, { fallbackModel: Boolean(input.fallbackModel), managedTimeouts: hasModelTimeouts(input.model), lastResort }); - return bindStoryboardSelection(finishClassification(decision, videoIds, channelIds, explicitVideoIds, input.message), request); + return finishClassification(decision, videoIds, channelIds, explicitVideoIds, input.message); } type ClassificationIssue = { path: string; code: string; message: string }; @@ -606,7 +588,7 @@ const ROUTE_FIELDS = { finalize: ['responseIntent', 'contextScope', 'historySelection', 'reason', 'comparisonVideoIds'], } as const satisfies Record; const SHARED_FIELDS = ['skill', 'savedOnly', 'requestedEvidence', 'answerDetail', 'numberedItemCount', 'refreshEvidence', 'refreshDynamicData', - 'visualEvidence', 'visualRequirements', 'setAsideFrames'] as const satisfies readonly DefaultedClassificationField[]; + 'visualEvidence', 'visualRequirements'] as const satisfies readonly DefaultedClassificationField[]; function candidateObject(value: unknown): Record | undefined { if (typeof value === 'string') { @@ -619,19 +601,12 @@ function pinnedFrameAccess(entries: T[], videoId return entries.map(entry => ({ ...entry, videoId: entry.videoId ?? videoId })); } -function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[], request: FrameRequest): ClassificationIssue[] { +function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[]): ClassificationIssue[] { // A single linked video pins inspection to itself, which would leave another video's frames out of reach. const linked = extractYouTubeVideoIds(input.message); if (decision.frameAccess && decision.route === 'inspect_video' && linked.length === 1 && linked[0] !== decision.videoId) { return [{ path: 'frameAccess', code: 'frame_scope_outside_link', message: `The message links ${linked[0]}. Inspect that video, or use topic_research with comparisonVideoIds for every video whose frames this turn uses.` }]; } - if (decision.route !== 'finalize') { - const pin = decision.route === 'topic_research' && !decision.comparisonVideoIds?.length && !decision.playlistListing && linked.length === 1; - const plan = pin ? { ...decision, route: 'inspect_video', videoId: linked[0], - frameAccess: Array.isArray(decision.frameAccess) ? pinnedFrameAccess(decision.frameAccess, linked[0]!) : decision.frameAccess } : decision; - const issue = frameRequestIssue(plan, request, input.sessionBrief?.assets); - if (issue) return [{ path: 'frameAccess', code: 'invalid_frame_access', message: issue }]; - } const scope = comparisonScopeIssues(decision, input, videoIds); return scope.length ? scope : searchQueryNumberIssues(decision, input); } @@ -643,7 +618,7 @@ function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifie * request defaults alone. Candidates from later attempts are preferred. */ function lastResortDecision( - evaluations: Evaluation[], input: CapabilityClassifierInput, videoIds: string[], channelIds: string[], explicitVideoIds: string[], request: FrameRequest, + evaluations: Evaluation[], input: CapabilityClassifierInput, videoIds: string[], channelIds: string[], explicitVideoIds: string[], ): { decision: ClassifierDecision; method: LastResortMethod; defaultedFields: DefaultedClassificationField[] } { const candidates = [...evaluations].sort((a, b) => b.context.attempt - a.context.attempt) .flatMap(item => { const candidate = candidateObject(item.candidate); return candidate ? [candidate] : []; }); @@ -664,10 +639,11 @@ function lastResortDecision( // Requirements any candidate stated, or the request implies, survive every fallback: // fresh data needs an executable route, required visuals stay required, and a // follow-up comparison keeps both of its subjects. - // A bounded list that no attempt stated validly is never widened, not even to another attempt's agent_selected. - const listing = candidates.filter(candidate => Array.isArray(candidate.frameAccess)); - const frameAccess = firstValid('frameAccess', listing.length ? listing : candidates); - if (listing.length && !frameAccess) { + // Last-resort assembly must not mint image permissions by combining failed decisions. + // Preserve an agreed grant; ask for clarity if their explicit grants differ. + const grants = candidates.flatMap(candidate => candidate.frameAccess == null ? [] : [candidate.frameAccess]); + const frameAccess = firstValid('frameAccess'); + if (grants.length && (!frameAccess || new Set(grants.map(grant => JSON.stringify(shape.frameAccess.safeParse(grant).data))).size !== 1)) { const clarification = { route: 'finalize', responseIntent: 'clarification', reason: 'Clarify the saved and permitted fresh frame positions for each video.', answerDetail: 'standard' }; return { decision: classifierDecisionSchema.parse(clarification), method: 'defaults', defaultedFields: Object.keys(clarification) as DefaultedClassificationField[] }; @@ -770,7 +746,7 @@ function lastResortDecision( }; const accept = (decision: Record) => { const parsed = validateClassifierDecision(decision, validation); - return parsed.success ? { parsed: parsed.data, issues: semanticIssues(parsed.data, input, videoIds, request) } : undefined; + return parsed.success ? { parsed: parsed.data, issues: semanticIssues(parsed.data, input, videoIds) } : undefined; }; // Recover a missing route from one candidate as returned. The recovered decision @@ -852,8 +828,7 @@ function expectedComparisonSubjects(input: CapabilityClassifierInput): string[] function finishClassification( decision: z.infer, videoIds: string[], channelIds: string[], explicitVideoIds: string[], message: string, ): CapabilityRouteDecision { - // Set-aside frames only check the classification; the route carries what research may use. - const { explicitSourceCount, setAsideFrames: _setAside, ...classified } = withVisualAccess(decision); + const { explicitSourceCount, ...classified } = withVisualAccess(decision); const pinned = !(decision.route === 'topic_research' && decision.playlistListing) && explicitVideoIds.length === 1 && !decision.comparisonVideoIds?.length && (decision.route !== 'finalize' || (decision.responseIntent === 'context_answer' && decision.contextScope !== 'history')); const pinnedAccess = pinned && Array.isArray(classified.frameAccess) @@ -953,36 +928,44 @@ export function extractYouTubeChannelIds(message: string): string[] { return [...ids]; } -function bindStoryboardSelection(decision: CapabilityRouteDecision, request: FrameRequest): CapabilityRouteDecision { - if ((decision.route !== 'inspect_video' && decision.route !== 'topic_research') || decision.frameAccess !== 'storyboard_only') return decision; - const selections = requestedSheets(decision, request); - const scoped = request.refreshSheets ? { ...decision, refreshFramesOnly: true } : decision; - if (!selections?.length) return scoped; - return { ...scoped, frameAccess: decision.route === 'inspect_video' - ? { storyboardTimestampsMs: selections[0]!.timestampsMs } : { storyboardSelections: selections } }; +// Validate relationships between structured fields. Only the agent interprets the user's words. +function imageScopeIssue(plan: Extract | ClassifierDecision, suppliedVideoIds?: readonly string[]) { + const scope = plan.frameAccess; + if (plan.savedOnly && scope && !Array.isArray(scope)) return 'Saved-only requests cannot retrieve new images.'; + if (!plan.savedOnly && (plan.visualEvidence === 'required' || plan.refreshFramesOnly) && (!scope || + (Array.isArray(scope) && !scope.some(entry => entry.savedTimestampsMs.length || entry.captureTimestampsMs.length)))) + return 'Required visual work needs capture permission or a saved inspection in frameAccess.'; + if ('timestampedFrameQuestion' in plan && plan.timestampedFrameQuestion && !plan.savedOnly && !Array.isArray(scope)) + return 'An exact-frame question needs a list of permitted positions.'; + const subjects = plan.route === 'inspect_video' ? [plan.videoId] : plan.comparisonVideoIds ?? []; + const validSubject = (id: string) => subjects.includes(id) && (!suppliedVideoIds || suppliedVideoIds.includes(id)); + if (Array.isArray(scope)) { + const ids = scope.flatMap(entry => entry.videoId ? [entry.videoId] : []); + if (plan.savedOnly && scope.some(entry => entry.captureTimestampsMs.length)) return 'Saved-only requests cannot capture frames.'; + if (ids.some(id => !validSubject(id)) || new Set(ids).size !== ids.length) + return 'List one frameAccess entry per subject video. Use topic_research with comparisonVideoIds for multiple subjects.'; + if (scope.some(entry => !entry.videoId && (scope.length !== 1 || entry.savedTimestampsMs.length || plan.route !== 'topic_research' || subjects.length))) + return 'Only discovery can leave the video open for a fixed capture list.'; + if (scope.some(entry => new Set([...entry.savedTimestampsMs, ...entry.captureTimestampsMs]).size !== entry.savedTimestampsMs.length + entry.captureTimestampsMs.length)) + return 'Give each timestamp exactly one role per video: saved inspection or capture.'; + } else if (scope && typeof scope === 'object') { + if ('storyboardTimestampsMs' in scope) return plan.route === 'inspect_video' ? undefined : 'Specify the video for each storyboard selection.'; + const ids = scope.storyboardSelections.map(entry => entry.videoId); + if (new Set(ids).size !== ids.length || ids.some(id => !validSubject(id))) + return 'Choose one storyboard selection per video, from the route subjects.'; + } } export async function resolveCapabilityRoute(options: { - message?: string; - priorAssets?: SessionBrief['assets']; persisted?: unknown; classify: () => Promise; persist: (decision: CapabilityRouteDecision) => void | Promise; }): Promise { if (options.persisted !== undefined) { const decision = capabilityRouteDecisionSchema.parse(options.persisted); - // Routes persisted before every scoped video had to be a route subject could pin another - // video after validation. Resuming must not run them without the access the user gave. - const subjects = decision.route === 'inspect_video' ? [decision.videoId] - : decision.route === 'topic_research' ? decision.comparisonVideoIds ?? [] : []; - if ('frameAccess' in decision && Array.isArray(decision.frameAccess) && decision.frameAccess.some(entry => entry.videoId ? !subjects.includes(entry.videoId) : decision.route === 'inspect_video' || subjects.length > 0)) { - return { route: 'finalize', responseIntent: 'clarification', reason: 'Clarify which video the saved and fresh frame positions refer to.' }; - } - if (options.message && (decision.route === 'inspect_video' || decision.route === 'topic_research')) { - const request = readFrameRequest(options.message); - const issue = frameRequestIssue(decision, request, options.priorAssets, true); + if (decision.route === 'inspect_video' || decision.route === 'topic_research') { + const issue = imageScopeIssue(decision); if (issue) return { route: 'finalize', responseIntent: 'clarification', reason: issue }; - return bindStoryboardSelection(decision, request); } return decision; } diff --git a/platform/src/agents/research/frame-request.ts b/platform/src/agents/research/frame-request.ts deleted file mode 100644 index d82b2815..00000000 --- a/platform/src/agents/research/frame-request.ts +++ /dev/null @@ -1,209 +0,0 @@ -import { timestampExpressions } from './timestamp-expression'; -import type { RouteFrameAccess } from '../contracts'; -import { clockTime } from '../runtime/clock-time'; -import { extractYouTubeVideoIds, linkedSeek, urlCandidates } from './youtube-references'; -import type { SessionBrief } from '../runtime/session-evidence'; - -// A comparison such as "within" or "at most" can limit either video length or capture position. -// Exclude a time when the surrounding words describe duration ("videos under 5 minutes", "5 minutes long"). -const COMPARISON = /\b(?:under|over|within|up to|at (?:least|most)|(?:no )?(?:longer|shorter|less|more) than)\s*$/i; -const DURATION_SUBJECT = /\b(?:duration|length|runtime)(?:\s+(?:of|is|must be|should be))?\s*$/i; -const SOURCE_SELECTION = /\b(?:find|search|recommend|list|videos?|clips?|tutorials?|lectures?|recordings?|episodes?)\b/i; -const FRAME_ACTION = /\b(?:frames?|stills?|timestamps?|capture|capturing|inspect|sample|extract)\b/i; -const VIDEO_LENGTH_SUFFIX = /^\s*-?\s*(?:long|in (?:length|duration)|videos?|clips?|lectures?|tutorials?|recordings?|episodes?|of (?:content|footage|material)|ads?|intros?|segments?|versions?|cuts?|shorts?|views?)\b/i; -const START_WINDOW = /\b(?:within(?: the first)?|up to|no later than|in the first|during the first)\s*$/i; -const START_ANCHOR = /^\s+(?:of|from)\s+(?:the\s+)?(?:(?:video|clip)(?:'s|’s)\s+)?(?:start|beginning)\b/i; -const RELATIVE_ANCHOR = /^\s+(?:of|from|after|before)\b/i; - -function isVideoLength(before: string, after: string): boolean { - if (VIDEO_LENGTH_SUFFIX.test(after)) return true; - // An article plus a time quantity modifying a noun describes its length ("a 2-hour documentary"). - if (/\b(?:a|an)\s*$/i.test(before) && !FRAME_ACTION.test(before.split(CLAUSE).at(-1) ?? '') - && /^\s+(?!(?:mark|position|frame|image|picture|still|timestamp|screenshot)\b)[a-z]/i.test(after)) return true; - const comparison = COMPARISON.exec(before); - if (!comparison) return false; - const subject = before.slice(0, comparison.index); - if (DURATION_SUBJECT.test(subject)) return true; - if (!START_WINDOW.test(before) && /\b(?:video|clip|tutorial|lecture|recording|episode)s?\s*$/i.test(subject)) return true; - const clause = subject.split(/[.!?;,\n]|\b(?:and|but|then)\b/i).at(-1) ?? ''; - return SOURCE_SELECTION.test(clause) && !FRAME_ACTION.test(clause); -} - -// A list continues the preceding operation ("capture 0:30, 0:45 and 1:00"). -// A new clause can change it ("reuse 0:30, then capture 1:00"). -const CLAUSE = /[:;.!?]\s+|\n|(?:,\s*|\b(?:then|and|but)\s+)(?!https?:\/\/|at\b)(?=[a-z])|\b(?:with|versus|vs\.?)\s+(?=(?:(?:a|an|the)\s+)?(?:new|fresh|saved|previous|prior)\b)/i; -function positionRole(clause: string, after: string) { - if (/\b(?:no|not|never|without|don['’]t|except|instead of|rather than|exclude|ignore|skip)\b/i.test(clause)) return 'excluded'; - if (/\bstoryboards?\b/i.test(clause) || /^\s+storyboard sheet\b/i.test(after)) return 'sheet'; - if (/\b(?:re-?capture|capture|refresh|(?:new|fresh) (?:image|still|frame))\b/i.test(clause) - || (/\b(?:new|fresh)\s*$/i.test(clause) && /^\s+(?:image|still|frame)\b/i.test(after))) return 'capture'; - if (/\b(?:transcript|caption|comment)s?\b/i.test(clause)) return; - if (/\b(?:reuse|re-?inspect|saved)\b/i.test(clause) || /^\s+saved (?:frame|image)\b/i.test(after)) return 'saved'; -} - -// The classifier resolves the subject. The user's named positions bound what the classifier may grant. -export function readFrameRequest(message: string) { - const positions: { time: number; role: ReturnType; videoIds: string[] }[] = []; - let captureEndMs: number | undefined; - const urls = [...message.matchAll(urlCandidates)]; - const inUrl = (index: number) => urls.some(url => index >= url.index && index < url.index + url[0].length); - // Preserve offsets while keeping URL parameters out of prose expressions. - const expressions = timestampExpressions(message.replace(urlCandidates, url => '#'.repeat(url.length))); - let ambiguousTime = false; - function addProseTime(start: number, end: number, time: number, ambiguous = false) { - const before = message.slice(0, start); - const after = message.slice(end); - const fromStart = START_ANCHOR.test(after); - if (!fromStart && isVideoLength(before, after)) return; - if (ambiguous) { ambiguousTime = true; return; } - // A window end limits new captures. It does not request inspection of a saved frame at the endpoint. - if (START_WINDOW.test(before) && (fromStart || !RELATIVE_ANCHOR.test(after) || /\bthe first\s*$/i.test(before))) { - captureEndMs = Math.min(captureEndMs ?? time, time); - } else { - const prefix = before.split(CLAUSE).at(-1) ?? ''; - const suffix = after.split(CLAUSE, 1)[0] ?? ''; - const role = positionRole(prefix, suffix); - if (/\b(?:previous|prior|earlier|last) (?:answer|response|turn|message)\b/i.test(prefix) - && (!role || /\b(?:described|mentioned|cited|showed|reported)\b/i.test(prefix))) return; - const expression = message.slice(start, end); - const clause = prefix + expression + suffix; - const subjects = extractYouTubeVideoIds(clause); - // Multiple timed operands have separate subjects. One time followed by two links applies to both. - const operand = /\s+(?:and|with|versus|vs\.?|to)\s+|,\s*/i; - const localPrefix = prefix.split(operand).at(-1) ?? ''; - const localSuffix = suffix.split(operand)[0] ?? ''; - const separate = subjects.length > 1 && expressions.some(other => other.start >= start - prefix.length && other.end <= end + suffix.length - && (other.end <= start - localPrefix.length || other.start >= end + localSuffix.length)); - const localSubjects = separate ? extractYouTubeVideoIds(localPrefix + expression + localSuffix) : subjects; - positions.push({ time, role, videoIds: localSubjects.length ? localSubjects : subjects }); - } - } - for (const time of expressions) addProseTime(time.start, time.end, time.milliseconds, time.ambiguous); - for (const match of message.matchAll(/\b(?:opening|first) (?:exact )?(?:frame|still|image)\b/gi)) { - if (!inUrl(match.index)) addProseTime(match.index, match.index + match[0].length, 0); - } - // An unspecified interval has no safe numeric end. Ask for it before retrieving images. - const vagueWindow = /\b(?:first|last|opening|final) (?:[a-z]+(?:[- ](?:of|[a-z]+))?) (?:seconds?|minutes?|frames?)\b|\b(?:last|closing|final) (?:frame|still)\b/i.test(message); - // An explicit position or window in the request takes precedence over the link's seek time. - if (!positions.length && captureEndMs === undefined) { - for (const url of urls) { - const seek = linkedSeek(url[0]); - if (!seek) continue; - const prefix = message.slice(0, url.index).split(CLAUSE).at(-1) ?? ''; - const suffix = message.slice(url.index + url[0].length).split(CLAUSE, 1)[0] ?? ''; - positions.push({ ...seek, role: positionRole(prefix, suffix) }); - } - } - // An explicit exact-time instruction still limits retrieval when its time could not be read. - const missingExactTime = !positions.length && captureEndMs === undefined - && /\b(?:(?:only|exactly|just)\s+(?:the (?:frame|still|image)\s+)?at|at (?:exactly|only)|no other (?:timestamps?|positions?))\b/i.test(message); - const noRecapture = /\b(?:without|never|do not|don['’]t)\s+re-?captur(?:e|ing)\b/i.test(message); - const noCapture = /\b(?:never|do not|don['’]t)\s+capture\s+(?:any\s+)?(?:new |exact )?(?:frames?|images?|stills?)\b/i.test(message); - const sheetTimes = positions.every(position => position.role === 'sheet') && captureEndMs === undefined - ? [...new Set(positions.map(position => position.time))] : []; - const refreshSheets = message.split(CLAUSE).some(clause => - /\b(?:refresh|reload|re-?fetch|fresh)\s+(?:(?:only|just|the|these|both|saved|video)\s+)*storyboards?\b|\bstoryboard(?:-only)?\s+refresh\b/i.test(clause) - && !/\b(?:not|never|without|don['’]t)\s+(?:refresh|reload|re-?fetch)\b/i.test(clause)); - const requestsSaved = positions.some(pos => pos.role === 'saved') || /\b(?:reuse|re-?inspect|inspect|analy[sz]e)\s+(?:the |my |our |that )?saved (?:video |exact )?(?:frames?|images?|stills?)\b/i.test(message); - return { positions, ambiguousTime, missingExactTime, requestsSaved, refreshSheets, sheetTimes, captureEndMs, noRecapture, noCapture, vagueWindow }; -} - -export type FrameRequest = ReturnType; -interface FramePlan { - route: string; - videoId?: string; - comparisonVideoIds?: string[]; - frameAccess?: RouteFrameAccess; - savedOnly?: boolean; - timestampedFrameQuestion?: string; - refreshFramesOnly?: boolean; - visualEvidence?: string; - setAsideFrames?: { videoId: string; timestampMs: number }[]; -} - -/** Bind each sheet position to the named subjects before the model can call the provider. */ -export function requestedSheets(plan: FramePlan, request: FrameRequest) { - if (!request.sheetTimes.length || request.sheetTimes.length > 24) return; - const subjects = plan.route === 'inspect_video' && plan.videoId ? [plan.videoId] : plan.comparisonVideoIds ?? []; - if (!subjects.length || request.positions.some(pos => pos.videoIds.some(id => !subjects.includes(id)))) return; - return subjects.flatMap(videoId => { - const times = request.positions.filter(pos => !pos.videoIds.length || pos.videoIds.includes(videoId)).map(pos => pos.time); - return times.length ? [{ videoId, timestampsMs: [...new Set(times)] }] : []; - }); -} - -/** Bound the classifier's grant to operations named in the request, including on resume. */ -export function frameRequestIssue(plan: FramePlan, request: FrameRequest, assets: SessionBrief['assets'] = [], resumed = false): string | undefined { - const access = plan.frameAccess; - const { positions, captureEndMs, noRecapture, noCapture } = request; - const limited = positions.length > 0 || captureEndMs !== undefined || noRecapture || noCapture; - const sameSubject = (ids: string[], videoId?: string) => !ids.length || (videoId !== undefined && ids.includes(videoId)); - const targetedSheets = requestedSheets(plan, request); - if ((request.ambiguousTime || request.missingExactTime) && access) return 'Ask for an unambiguous timestamp or an explicit list of positions.'; - if (request.vagueWindow && access && !(Array.isArray(access) && access.every(entry => !entry.captureTimestampsMs.length))) - return 'Ask for the exact capture interval before retrieving images.'; - if (plan.visualEvidence === 'required' && !plan.savedOnly && (!access || (Array.isArray(access) - && !access.some(entry => entry.savedTimestampsMs.length || entry.captureTimestampsMs.length)))) - return 'Required visual evidence needs capture permission or a saved inspection in frameAccess. Select permitted new images or specify the saved inspection.'; - if (access && typeof access === 'object' && !Array.isArray(access)) { - // A previously persisted targeted sheet cannot widen during recovery. - const actual = 'storyboardSelections' in access ? access.storyboardSelections - : plan.route === 'inspect_video' ? [{ videoId: plan.videoId, timestampsMs: access.storyboardTimestampsMs }] : []; - if (!targetedSheets || actual.length !== targetedSheets.length || targetedSheets.some(expected => { - const times = actual.find(item => item.videoId === expected.videoId)?.timestampsMs; - return !times || times.length !== expected.timestampsMs.length || expected.timestampsMs.some(time => !times.includes(time)); - })) return 'Clarify the requested storyboard sheet positions.'; - return; - } - - const captures = positions.filter(position => position.role === 'capture'); - const matches = (time: number, videoId?: string) => positions.filter(pos => pos.time === time && sameSubject(pos.videoIds, videoId)); - if (request.refreshSheets && !captures.length && !positions.some(pos => pos.role === 'saved') && access !== 'storyboard_only') - return 'Refresh the requested storyboard sheets with storyboard_only access.'; - if (Array.isArray(access)) { - for (const entry of access) { - for (const time of entry.captureTimestampsMs) { - const named = matches(time, entry.videoId); - if (captureEndMs !== undefined && time > captureEndMs) - return `Select capture positions within the requested window ending at ${captureEndMs} ms.`; - const explicit = named.find(pos => pos.role === 'capture'); - // A finite grant needs a position or window in the request. Delegated selection has its own route. - const inRequestedRange = captureEndMs !== undefined || named.length > 0; - const authorized = captures.length ? !!explicit : inRequestedRange; - const stored = assets.some(asset => asset.kind === 'frame' && asset.videoId === entry.videoId && asset.details.timestampMs === time); - // Two videos may share a timestamp. An explicit capture on B can coexist with a saved inspection on A. - const savedHere = named.some(pos => pos.role === 'saved') && (!explicit || (stored && !explicit.videoIds.length)); - const forbidden = noCapture || savedHere || named.some(pos => pos.role === 'excluded' || pos.role === 'sheet'); - if (!authorized || forbidden || (noRecapture && !explicit && (stored || captureEndMs === undefined))) - return `The user did not authorize a new exact capture at ${clockTime(time)} on this video. List only the authorized capture positions.`; - } - } - for (const pos of captures) { - if (!access.some(entry => sameSubject(pos.videoIds, entry.videoId) && entry.captureTimestampsMs.includes(pos.time))) - return `Include the requested new capture at ${clockTime(pos.time)} on its video.`; - } - } else { - if (access === 'storyboard_only' && (!plan.refreshFramesOnly || request.refreshSheets) && (targetedSheets || (!positions.length && captureEndMs === undefined && !noRecapture))) return; - if (!plan.savedOnly && (plan.timestampedFrameQuestion || (access && limited))) - return 'The user fixed the frame positions. List the saved inspection positions and the permitted capture positions in frameAccess, one entry per video.'; - if (plan.refreshFramesOnly && access !== 'agent_selected') return 'A frame refresh needs capture permission. List the positions in frameAccess, or set agent_selected if the user left them to you.'; - } - - // Classification can set aside named inventory from an unrelated video. That choice - // has already been validated and is not part of the persisted execution grant. - if (!access && !plan.refreshFramesOnly && !plan.timestampedFrameQuestion && plan.visualEvidence === 'none') return; - if (!resumed) { - for (const asset of assets) { - const time = asset.details.timestampMs; - if (asset.kind !== 'frame' || typeof time !== 'number' || !positions.some(pos => pos.time === time)) continue; - const entry = Array.isArray(access) && access.find(entry => entry.videoId === asset.videoId); - if (entry && (entry.savedTimestampsMs.includes(time) || entry.captureTimestampsMs.includes(time))) continue; - if (plan.setAsideFrames?.some(frame => frame.videoId === asset.videoId && frame.timestampMs === time)) continue; - return `The session holds a saved ${clockTime(time)} frame of ${asset.videoId}. Include its requested operation in frameAccess, or in setAsideFrames if the user excluded it or meant another video.`; - } - } - for (const pos of positions.filter(pos => pos.role === 'saved')) { - if (!Array.isArray(access) || !access.some(entry => sameSubject(pos.videoIds, entry.videoId) && entry.savedTimestampsMs.includes(pos.time))) - return `List the requested saved ${clockTime(pos.time)} inspection even if its image is unavailable.`; - } -} diff --git a/platform/src/agents/research/research-agent.ts b/platform/src/agents/research/research-agent.ts index e5caf534..37039380 100644 --- a/platform/src/agents/research/research-agent.ts +++ b/platform/src/agents/research/research-agent.ts @@ -189,9 +189,6 @@ async function executeResearchRunWithAssets(options: { if (!options.persistedRoute) await options.onClassifying?.(classificationDeadlineAt); const decision = await resolveCapabilityRoute({ persisted: options.persistedRoute, - message: options.message, - priorAssets: options.persistedRoute && 'frameAccess' in options.persistedRoute - ? options.session?.brief().assets.filter(asset => asset.collectedAt < (options.startedAt ?? Infinity)) : undefined, classify: () => withRunDeadline(classificationDeadlineAt, options.signal, signal => classifyCapabilityWithModel({ message: options.message, conversationHistory: options.conversationHistory, @@ -791,6 +788,9 @@ async function runResearchAgentWithModelWithinDeadline(options: { && packet.sources.some(source => source.videoId && !trackedContext.transcriptSelection?.tooLong?.has(source.videoId) && !trackedContext.transcriptSelection?.regionRestricted?.has(source.videoId))); const needsVisualWork = () => { + // Permission alone does not make optional images prerequisites for an answer. + if (!visualRequired && !options.decision.refreshFramesOnly + && !(options.decision.route === 'inspect_video' && options.decision.timestampedFrameQuestion)) return false; const failed = new Set([...toolFailures.values()].map(failure => failure.toolName)); if (frameScope) { const { saved, captures, inspectedVersions, pendingCaptureVersions } = frameActivity(frameScope, options.context.runId, evidence.values()); @@ -918,7 +918,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { ...(savedEvidenceSkill && !verifySaved ? ['Reuse supplied visual observations. Saved visual analysis tools remain available only for requested visual facts that are still missing; they do not replace required original caption/comment reads.'] : []), ...(!verifySaved ? ['For exact quotations or source corrections, use search_originals: omit assetVersion to find which saved source says it, then read that source by its version. Analyst summaries are not verbatim evidence. Read original passages before completing research.'] : []), ...(options.decision.savedOnly ? ['This run is saved-only. Retrieval and discovery are disabled, including incidental metadata calls. Missing saved assets are gaps, not permission to fetch.'] : []), - ...(frameScope ? [`Exact frame access for this turn: ${JSON.stringify(frameScope)}. Analyze the saved positions by their session asset versions. If saved inspection fails, stop that inspection and report the gap. Independently retrieve the authorized new positions. Earlier observations cannot stand in for a successful re-inspection.`] : []), + ...(frameScope ? [`Exact frame access for this turn: ${JSON.stringify(frameScope)}.`] : []), ...(requestedKinds.length ? [requestedEvidenceGuidance(requestedKinds)] : []), // The skill accompanies exactly the visual tools this model can call. ...(providerToolNames.some(name => VISUAL_TOOL_NAMES.includes(name)) ? [VISUAL_INSPECTION_SKILL] : []), diff --git a/platform/src/agents/research/skills/visual-inspection.generated.ts b/platform/src/agents/research/skills/visual-inspection.generated.ts index a22bb272..9b64fbd5 100644 --- a/platform/src/agents/research/skills/visual-inspection.generated.ts +++ b/platform/src/agents/research/skills/visual-inspection.generated.ts @@ -1,2 +1,2 @@ // Generated from visual-inspection/SKILL.md. Run npx tsx scripts/generate-agent-skills.ts to update. -export const VISUAL_INSPECTION_SKILL = "---\nname: visual-inspection\ndescription: Locate and inspect YouTube images for visible details, resolve conflicts with captions or metadata, and explain recorded frame choices on follow-up.\n---\n\n# Visual inspection\n\nAnswer visible questions from analyzed images. Metadata and captions locate moments and identify speakers; they do not establish clothing, scenery or readable on-screen text.\n\n## Locate\n\nUse only the visual tools offered on this turn and the videos and positions the user permitted. If retrieval is unavailable, inspect permitted saved images or report the missing evidence. Reuse adequate saved analysis first. Choose how to locate by what the question needs:\n\n- The look of a whole video, such as camera style, colors, set or diagram style: use a small spread of storyboard sheets when that retrieval is available.\n- A visual moment with no spoken cue: scan permitted storyboard sheets with an analysis question when available. One call can include many sheets.\n- A moment defined by topic or speech, such as the slide for a named concept or a spoken introduction: use title, description or chapter timestamps first, then captions. Retrieve the transcript without an analysis question and find the passage with search_originals. A transcript analyst never answers visual questions.\n\nA chapter is a timestamped section listed by the publisher, not an inferred scene boundary. In topic research, fetch captions only for videos you will inspect. When exact capture is available, keep the reserved provider calls for it. Respect the user's chosen inspection method.\n\nFor a presenter question, an introduction is a lead, not proof the presenter is on screen. A storyboard overview can locate her actual appearance. Prefer a few frames that answer the question over a six-frame quota or unexplained samples across the whole video. Questions about changes across the video need wider coverage.\n\n## Inspect\n\nWhen get_video_frames is available, use it for permitted exact seek positions if storyboard images are too small, text is ambiguous, or captions conflict with visible labels. If a sheet fails, use an authorized exact capture with an analysis question when available. Otherwise record the gap. Supply selections with one basis category and the locating evidenceIds for each timestamp. user_requested and exploratory selections cite no excerpts; describe an exploratory sample as exploratory. A basis records the kind of choice, not private reasoning or proof the desired scene exists.\n\nWhen a retrieval tool is offered, include the analysis question so the analyst sees the images as they are saved. Retrieval without analysis provides no observations; a RETRIEVAL_ANALYSIS_FAILED warning means the images were saved but still need analyze_video_frames or analyze_video_storyboard. Inspect only analysisAssetVersions from storyboard results, never the manifest. The runtime supplies title, full description, channel and question to image analysts. Analyze saved images directly for new questions rather than fetching the same images again.\n\nCheck whether each requested detail is visible and readable. A transition or irrelevant shot does not answer a clothing question. When needed, try a permitted nearby timestamp informed by the overview, within the remaining time and provider budget. When names conflict, inspect a readable on-screen label at an exact frame and distinguish the label from noisy captions. A name in metadata is context, not visual identification.\n\n## Finish and explain\n\nPresent the visible findings with citations and material uncertainty. Explain inspection steps only when the user explicitly asks about the workflow. Finishing transcript research does not finish a visual question. Before finishing, check every requested subject and attribute against analyzed images; stop when they are covered or a real retrieval, analysis or budget limit prevents it, not merely because captions omit visual details. Keep image quality and missing-image warnings. Cite each frame-specific observation at its own timestamp. Describe shared observations only over their supported sampled frames. State missing details and actual retrieval or analysis limits. Sampled stills do not establish continuous appearance or motion.\n\nFor a follow-up about how inspection happened, use frameCoverage, selection bases and delivery records. Separate requested or retrieved positions from analyzed frames and storyboard samples. A storyboard analyst receives every tile in the selected sheets; its reported finding times may cover only a subset. Sheet spread selection does not choose 3:20 over 3:30 within the same supplied sheet. A recorded basis can explain why a timestamp was chosen, such as a spoken introduction; what a cited excerpt says is evidence only when that excerpt is supplied. Without a basis, say the reason was not recorded. Do not reconstruct a past rationale from what now seems sensible. Treat user examples such as “why 3:20 instead of 3:30?” as alternatives, not captured frames. Claim transcript reading or fresh image analysis only when the supplied records establish it. Historical evidence and memory alone do not prove a current operation.\n"; +export const VISUAL_INSPECTION_SKILL = "---\nname: visual-inspection\ndescription: Locate and inspect YouTube images for visible details, resolve conflicts with captions or metadata, and explain recorded frame choices on follow-up.\n---\n\n# Visual inspection\n\nAnswer visible questions from analyzed images. Metadata and captions locate moments and identify speakers; they do not establish clothing, scenery or readable on-screen text.\n\n## Locate\n\nUse only the visual tools offered on this turn and the videos and positions the user permitted. If retrieval is unavailable, inspect permitted saved images or report the missing evidence. Reuse adequate saved analysis first. Choose how to locate by what the question needs:\n\n- The look of a whole video, such as camera style, colors, set or diagram style: use a small spread of storyboard sheets when that retrieval is available.\n- A visual moment with no spoken cue: scan permitted storyboard sheets with an analysis question when available. One call can include many sheets.\n- A moment defined by topic or speech, such as the slide for a named concept or a spoken introduction: use title, description or chapter timestamps first, then captions. Retrieve the transcript without an analysis question and find the passage with search_originals. A transcript analyst never answers visual questions.\n\nA chapter is a timestamped section listed by the publisher, not an inferred scene boundary. In topic research, fetch captions only for videos you will inspect. When exact capture is available, keep the reserved provider calls for it. Respect the user's chosen inspection method.\n\nFor a presenter question, an introduction is a lead, not proof the presenter is on screen. A storyboard overview can locate her actual appearance. Prefer a few frames that answer the question over a six-frame quota or unexplained samples across the whole video. Questions about changes across the video need wider coverage.\n\n## Inspect\n\nWhen get_video_frames is available, use it for permitted exact seek positions if storyboard images are too small, text is ambiguous, or captions conflict with visible labels. If a sheet fails, use an authorized exact capture with an analysis question when available. Otherwise record the gap. Supply selections with one basis category and the locating evidenceIds for each timestamp. user_requested and exploratory selections cite no excerpts; describe an exploratory sample as exploratory. A basis records the kind of choice, not private reasoning or proof the desired scene exists.\n\nWhen a retrieval tool is offered, include the analysis question so the analyst sees the images as they are saved. Retrieval without analysis provides no observations; a RETRIEVAL_ANALYSIS_FAILED warning means the images were saved but still need analyze_video_frames or analyze_video_storyboard. Inspect only analysisAssetVersions from storyboard results, never the manifest. The runtime supplies title, full description, channel and question to image analysts. When the user asks to re-inspect a saved image, analyze its session asset version. An earlier observation is not a new inspection. If that read or analysis fails, report which image you could not inspect. Continue with any separately permitted capture; do not replace the failed read with a capture or a storyboard. Compare only what the inspected images support.\n\nCheck whether each requested detail is visible and readable. A transition or irrelevant shot does not answer a clothing question. When needed, try a permitted nearby timestamp informed by the overview, within the remaining time and provider budget. When names conflict, inspect a readable on-screen label at an exact frame and distinguish the label from noisy captions. A name in metadata is context, not visual identification.\n\n## Finish and explain\n\nPresent the visible findings with citations and material uncertainty. Explain inspection steps only when the user explicitly asks about the workflow. Finishing transcript research does not finish a visual question. Before finishing, check every requested subject and attribute against analyzed images; stop when they are covered or a real retrieval, analysis or budget limit prevents it, not merely because captions omit visual details. Keep image quality and missing-image warnings. Cite each frame-specific observation at its own timestamp. Describe shared observations only over their supported sampled frames. State missing details and actual retrieval or analysis limits. Sampled stills do not establish continuous appearance or motion.\n\nFor a follow-up about how inspection happened, use frameCoverage, selection bases and delivery records. Separate requested or retrieved positions from analyzed frames and storyboard samples. A storyboard analyst receives every tile in the selected sheets; its reported finding times may cover only a subset. Sheet spread selection does not choose 3:20 over 3:30 within the same supplied sheet. A recorded basis can explain why a timestamp was chosen, such as a spoken introduction; what a cited excerpt says is evidence only when that excerpt is supplied. Without a basis, say the reason was not recorded. Do not reconstruct a past rationale from what now seems sensible. Treat user examples such as “why 3:20 instead of 3:30?” as alternatives, not captured frames. Claim transcript reading or fresh image analysis only when the supplied records establish it. Historical evidence and memory alone do not prove a current operation.\n"; diff --git a/platform/src/agents/research/skills/visual-inspection/SKILL.md b/platform/src/agents/research/skills/visual-inspection/SKILL.md index ee2e8700..3c027795 100644 --- a/platform/src/agents/research/skills/visual-inspection/SKILL.md +++ b/platform/src/agents/research/skills/visual-inspection/SKILL.md @@ -23,7 +23,7 @@ For a presenter question, an introduction is a lead, not proof the presenter is When get_video_frames is available, use it for permitted exact seek positions if storyboard images are too small, text is ambiguous, or captions conflict with visible labels. If a sheet fails, use an authorized exact capture with an analysis question when available. Otherwise record the gap. Supply selections with one basis category and the locating evidenceIds for each timestamp. user_requested and exploratory selections cite no excerpts; describe an exploratory sample as exploratory. A basis records the kind of choice, not private reasoning or proof the desired scene exists. -When a retrieval tool is offered, include the analysis question so the analyst sees the images as they are saved. Retrieval without analysis provides no observations; a RETRIEVAL_ANALYSIS_FAILED warning means the images were saved but still need analyze_video_frames or analyze_video_storyboard. Inspect only analysisAssetVersions from storyboard results, never the manifest. The runtime supplies title, full description, channel and question to image analysts. Analyze saved images directly for new questions rather than fetching the same images again. +When a retrieval tool is offered, include the analysis question so the analyst sees the images as they are saved. Retrieval without analysis provides no observations; a RETRIEVAL_ANALYSIS_FAILED warning means the images were saved but still need analyze_video_frames or analyze_video_storyboard. Inspect only analysisAssetVersions from storyboard results, never the manifest. The runtime supplies title, full description, channel and question to image analysts. When the user asks to re-inspect a saved image, analyze its session asset version. An earlier observation is not a new inspection. If that read or analysis fails, report which image you could not inspect. Continue with any separately permitted capture; do not replace the failed read with a capture or a storyboard. Compare only what the inspected images support. Check whether each requested detail is visible and readable. A transition or irrelevant shot does not answer a clothing question. When needed, try a permitted nearby timestamp informed by the overview, within the remaining time and provider budget. When names conflict, inspect a readable on-screen label at an exact frame and distinguish the label from noisy captions. A name in metadata is context, not visual identification. diff --git a/platform/src/agents/research/timestamp-expression.ts b/platform/src/agents/research/timestamp-expression.ts deleted file mode 100644 index d88f02a4..00000000 --- a/platform/src/agents/research/timestamp-expression.ts +++ /dev/null @@ -1,65 +0,0 @@ -// An expression occupies one span in the request. Its component numbers are never separate grants. -export interface TimestampExpression { - start: number; - end: number; - milliseconds: number; - ambiguous: boolean; -} - -const unitScale = new Map(); -for (const [scale, names] of [ - [3600000, ['hours', 'hour', 'hrs', 'hr', 'h']], - [60000, ['minutes', 'minute', 'mins', 'min', 'm']], - [1000, ['seconds', 'second', 'secs', 'sec', 's']], - [1, ['milliseconds', 'millisecond', 'ms']], -] as const) { - for (const name of names) unitScale.set(name, scale); -} -const component = String.raw`(\d+(?:\.\d+)?)\s*-?\s*(${[...unitScale.keys()].join('|')})(?![a-z])`; -const clock = String.raw`(?:\d+:)?\d{1,2}:\d{2}s?\b`; - -export function timestampExpressions(input: string): TimestampExpression[] { - const start = new RegExp(String.raw`(? seconds * 60 + field, 0) * 1000; - } else { - let scale = unitScale.get(first[2]!.toLowerCase())!; - milliseconds = Number(first[1]) * scale; - for (;;) { - continuation.lastIndex = start.lastIndex; - const next = continuation.exec(input); - if (!next) break; - const nextScale = unitScale.get(next[3]!.toLowerCase())!; - // "30 seconds and 45 seconds" lists two positions. Descending units form one - // position. A comma between descending units could also introduce a list. - if (nextScale >= scale && /,|and/i.test(next[1]!)) break; - ambiguous ||= nextScale >= scale || next[1]!.includes(','); - milliseconds += Number(next[2]) * nextScale; - scale = nextScale; - start.lastIndex = continuation.lastIndex; - } - } - milliseconds = Math.round(milliseconds); - result.push({ start: first.index, end: start.lastIndex, milliseconds, - ambiguous: ambiguous || !Number.isSafeInteger(milliseconds) }); - } - return result; -} - -/** URL seek values also allow bare seconds. The entire parameter must be a single timestamp. */ -export function seekMilliseconds(value: string): number | undefined { - if (/^\d+(?:\.\d+)?$/.test(value)) { - const time = Math.round(Number(value) * 1000); - return Number.isSafeInteger(time) ? time : undefined; - } - // A seek parameter uses compact h/m/s or a clock. Prose-only forms have no URL meaning. - if (!/^(?:(?:\d+h)?(?:\d+m)?(?:\d+(?:\.\d+)?s)?|(?:\d+:)?\d{1,2}:\d{2})$/i.test(value)) return; - const [time] = timestampExpressions(value); - if (time?.start === 0 && time.end === value.length && !time.ambiguous) return time.milliseconds; -} diff --git a/platform/src/agents/research/youtube-references.ts b/platform/src/agents/research/youtube-references.ts index 99434d1e..e800723c 100644 --- a/platform/src/agents/research/youtube-references.ts +++ b/platform/src/agents/research/youtube-references.ts @@ -1,9 +1,7 @@ -import { seekMilliseconds } from './timestamp-expression'; - const VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{11}$/; // Consume the whole URL before checking its host. Embedded links and v= parameters belong to that URL. -export const urlCandidates = /(?:https?:\/\/|(?"']+/gi; +const urlCandidates = /(?:https?:\/\/|(?"']+/gi; function youTubeUrl(candidate: string): URL | undefined { const normalized = candidate.replace(/[\])},.!?;:'"]+$/g, ''); try { @@ -26,17 +24,6 @@ function videoIdsFromUrl(url: URL): string[] { return candidates.filter((value): value is string => !!value && VIDEO_ID_PATTERN.test(value)); } -/** Seek parameters belong to their linked video. Numeric values are seconds. */ -export function linkedSeek(candidate: string): { videoIds: string[]; time: number } | undefined { - const url = youTubeUrl(candidate); - if (!url) return; - const videoIds = videoIdsFromUrl(url); - const value = url.searchParams.get('t') ?? url.searchParams.get('start') ?? new URLSearchParams(url.hash.slice(1)).get('t'); - if (!videoIds.length || !value) return; - const time = seekMilliseconds(value); - if (time !== undefined) return { videoIds, time }; -} - export function extractYouTubeVideoIds(message: string): string[] { const ids = new Set(); for (const url of youTubeUrls(message)) { diff --git a/platform/src/agents/runtime/frame-access.ts b/platform/src/agents/runtime/frame-access.ts index b59497e6..1a07b2ff 100644 --- a/platform/src/agents/runtime/frame-access.ts +++ b/platform/src/agents/runtime/frame-access.ts @@ -101,7 +101,7 @@ export interface InventoryFrame { version: string; videoId: string; timestampMs? * An image is saved when the session stored it before this run started, however this run reached it. */ export function frameTurnNotes(options: { - runId: string; startedAt: number; access?: FrameAccess; requestedSaved?: boolean; inventory: readonly InventoryFrame[]; packets: readonly EvidencePacket[]; + runId: string; startedAt: number; access?: FrameAccess; inventory: readonly InventoryFrame[]; packets: readonly EvidencePacket[]; cited: ReadonlyArray<{ packet: EvidencePacket; startMs: number; videoId?: string; title?: string }>; }): string[] { const { runId, access, inventory, packets } = options; @@ -131,8 +131,6 @@ export function frameTurnNotes(options: { notes.push(`I re-inspected the saved ${named(additional[0]!.videoId, additional[0]!.timestampMs)}.`); else if (additional.length) notes.push(`I re-inspected ${additional.length} saved frame${additional.length === 1 ? '' : 's'} in this turn.`); const plannedSaved = access?.filter(entry => entry.savedTimestampsMs.length) ?? []; - if (options.requestedSaved && !plannedSaved.length && !reinspected.length) - notes.push('No frame saved before this turn was re-inspected in this turn.'); const unplaced = new Set(savedBefore.filter(frame => frame.timestampMs === undefined && !analyzed.has(frame.version) && plannedSaved.some(entry => entry.videoId === frame.videoId)).map(frame => frame.videoId)); for (const videoId of unplaced) notes.push(`Saved frames of "${title(videoId) ?? videoId}" have no recorded position, so this turn could not match them to requested times or re-inspect them.`); diff --git a/platform/test/frame-timestamp-expression.test.ts b/platform/test/frame-timestamp-expression.test.ts deleted file mode 100644 index cfb5e77b..00000000 --- a/platform/test/frame-timestamp-expression.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { seekMilliseconds, timestampExpressions } from '../src/agents/research/timestamp-expression'; -import { frameRequestIssue, readFrameRequest } from '../src/agents/research/frame-request'; - -describe('frame timestamp expressions', () => { - it('reads equivalent clocks, quantities, and seek values as one complete position', () => { - for (const hours of [1, 2, 12]) for (const minutes of [0, 1, 30, 59]) for (const seconds of [0, 5, 59]) { - const expected = ((hours * 60 + minutes) * 60 + seconds) * 1000; - for (const expression of [ - `${hours}:${String(minutes).padStart(2, '0')}:${String(seconds).padStart(2, '0')}`, - `${hours}h${minutes}m${seconds}s`, - `${hours} hours ${minutes} minutes and ${seconds} seconds`, - `${hours * 60 + minutes} minutes ${seconds} seconds`, - ]) { - const prefix = 'Inspect at '; - expect(timestampExpressions(prefix + expression + '.')).toEqual([ - { start: prefix.length, end: prefix.length + expression.length, milliseconds: expected, ambiguous: false }, - ]); - if (!expression.includes(' ')) expect(seekMilliseconds(expression)).toBe(expected); - } - } - }); - - it.each([ - ['at 1 hour and at 30 minutes', [3600000, 1800000]], - ['at 1 hour 30 minutes and at 1 hour 45 minutes', [5400000, 6300000]], - ['at 30 seconds and 45 seconds', [30000, 45000]], - ['at 1.5 hours and at 250 ms', [5400000, 250]], - ] as const)('preserves separately specified positions in "%s"', (message, positions) => { - expect(timestampExpressions(message).map(time => time.milliseconds)).toEqual(positions); - }); - - it('asks for clarification before granting a comma-separated quantity with two readings', () => { - const request = readFrameRequest('Capture at 1 hour, 30 minutes.'); - for (const times of [[5400000], [1800000, 3600000]]) { - expect(frameRequestIssue({ route: 'inspect_video', frameAccess: [{ videoId: 'abcdefghijk', - savedTimestampsMs: [], captureTimestampsMs: times }] }, request)).toContain('unambiguous timestamp'); - } - }); - - it('keeps an unread exact-time instruction closed for both finite and delegated grants', () => { - const request = readFrameRequest('Capture only the frame at half past the first hour.'); - const plan = { route: 'inspect_video' }; - expect(frameRequestIssue({ ...plan, frameAccess: 'agent_selected' }, request)).toContain('unambiguous timestamp'); - expect(frameRequestIssue({ ...plan, frameAccess: [{ savedTimestampsMs: [], captureTimestampsMs: [30000] }] }, request)) - .toContain('unambiguous timestamp'); - }); - - it('keeps a compound source duration out of image permissions', () => { - expect(readFrameRequest('Find a 1 hour 30 minute lecture and capture only at 1 minute 10 seconds.').positions) - .toEqual([{ role: 'capture', time: 70000, videoIds: [] }]); - }); -}); diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index 36b244b2..5ff85e67 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -18,7 +18,6 @@ import { createCapabilityProvider, savedOnlyProvider } from '../src/agents/resea import type { ConversationTurn } from '../src/agents/runtime/conversation-memory'; import type { YouTubeAgentProvider } from '../src/agents/providers/youtube/provider'; import type { AgentToolContext } from '../src/agents/providers/youtube/tool-context'; -import { readFrameRequest } from '../src/agents/research/frame-request'; import { currentDateGuidance } from '../src/agents/runtime/current-date'; describe('YouTube agent capability router', () => { @@ -1167,33 +1166,6 @@ describe('YouTube agent capability router', () => { expect(model.doGenerateCalls).toHaveLength(2); }); - it.each([ - ['Capture only the opening frame', 0], ['Capture the frame at 1:30s', 90000], - ] as const)('binds "%s" to its actual timestamp', async (message, requested) => { - const videoId = 'abcdefghijk'; - const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['frame'] }; - const at = (time: number) => [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [time] }]; - const model = sequenceClassifier([{ ...base, frameAccess: 'agent_selected' }, { ...base, frameAccess: at(30000) }]); - const fallbackModel = classifierModel({ ...base, frameAccess: at(requested) }); - const decision = await classifyCapabilityWithModel({ model, fallbackModel, message: `${message} of https://youtu.be/${videoId}.`, signal: new AbortController().signal }); - expect(fallbackModel.doGenerateCalls).toHaveLength(1); - expect(decision).toMatchObject({ frameAccess: at(requested) }); - }); - - it('asks for an exact bound before capturing the first few seconds', async () => { - const videoId = 'abcdefghijk'; - const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'] }; - const model = sequenceClassifier([{ ...base, frameAccess: 'agent_selected' }, - { ...base, frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [120000] }] }]); - expect(await classifyCapabilityWithModel({ model, message: `Sample only the first few seconds of https://youtu.be/${videoId}`, - signal: new AbortController().signal })).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); - }); - - it.each(['Your previous answer described 0:30.', 'Find a 2 hour lecture and describe the cover.', - 'Find 5 hours of content about exercise.'])('does not authorize or require a position from "%s"', message => { - expect(readFrameRequest(message).positions).toEqual([]); - }); - it('pins a discovery capture to the linked inspection subject', async () => { const videoId = 'abcdefghijk'; const model = classifierModel({ route: 'topic_research', researchBreadth: 'focused', searchQuery: 'exercise', @@ -1201,30 +1173,22 @@ describe('YouTube agent capability router', () => { const message = `Capture only 0:30 of https://youtu.be/${videoId}.`; const decision = await classifyCapabilityWithModel({ model, message, signal: new AbortController().signal }); expect(decision).toMatchObject({ route: 'inspect_video', videoId, frameAccess: [{ videoId, captureTimestampsMs: [30000] }] }); - expect(await resolveCapabilityRoute({ message, persisted: { ...decision, frameAccess: [{ captureTimestampsMs: [30000], savedTimestampsMs: [] }] }, + expect(await resolveCapabilityRoute({ persisted: { ...decision, frameAccess: [{ captureTimestampsMs: [30000], savedTimestampsMs: [] }] }, classify: vi.fn(), persist: vi.fn() })).toMatchObject({ responseIntent: 'clarification' }); }); - it('assigns comparison sheet times joined by "with" to their own videos', () => { - const a = 'abcdefghijk', b = 'lmnopqrstuv'; - const message = `Compare the storyboard sheet at 0:30 of https://youtu.be/${a} with the one at 1:00 of https://youtu.be/${b}.`; - expect(readFrameRequest(message).positions.map(pos => [pos.time, pos.videoIds])).toEqual([[30000, [a]], [60000, [b]]]); - const common = `Compare the storyboard sheet at 0:30 in both https://youtu.be/${a} and https://youtu.be/${b}.`; - expect(readFrameRequest(common).positions.map(pos => [pos.time, pos.videoIds])).toEqual([[30000, [a, b]]]); - }); - it('keeps paired storyboard positions on comparison routes, including resume', async () => { const a = 'abcdefghijk', b = 'lmnopqrstuv'; - const model = classifierModel({ route: 'topic_research', comparisonVideoIds: [a, b], frameAccess: 'storyboard_only', + const storyboardSelections = [{ videoId: a, timestampsMs: [30000] }, { videoId: b, timestampsMs: [60000] }]; + const model = classifierModel({ route: 'topic_research', comparisonVideoIds: [a, b], frameAccess: { storyboardSelections }, visualEvidence: 'required', visualRequirements: ['compare the scenes'] }); const message = `Compare the storyboard sheets at 0:30 of https://youtu.be/${a} and 1:00 of https://youtu.be/${b}.`; const decision = await classifyCapabilityWithModel({ message, model, signal: new AbortController().signal }); - const storyboardSelections = [{ videoId: a, timestampsMs: [30000] }, { videoId: b, timestampsMs: [60000] }]; expect(decision).toMatchObject({ route: 'topic_research', frameAccess: { storyboardSelections } }); - const resume = (selections: typeof storyboardSelections) => resolveCapabilityRoute({ message, classify: vi.fn(), persist: vi.fn(), + const resume = (selections: typeof storyboardSelections) => resolveCapabilityRoute({ classify: vi.fn(), persist: vi.fn(), persisted: { ...decision, frameAccess: { storyboardSelections: selections } } }); await expect(resume(storyboardSelections)).resolves.toEqual(decision); - await expect(resume(storyboardSelections.map(item => ({ ...item, timestampsMs: [30000, 60000] })))).resolves.toMatchObject({ responseIntent: 'clarification' }); + await expect(resume([{ videoId: 'zzzzzzzzzzz', timestampsMs: [30000] }])).resolves.toMatchObject({ responseIntent: 'clarification' }); if (decision.route !== 'topic_research') throw new Error('Expected comparison.'); const context = inspectContext(); const storyboard = vi.fn>(async (videoId, times) => ({ @@ -1242,13 +1206,13 @@ describe('YouTube agent capability router', () => { expect(research.doGenerateCalls[0]?.tools?.map(tool => tool.name)).not.toContain('get_video_frames'); }); - it('repairs an unrestricted storyboard refresh without granting exact capture', async () => { + it('accepts a storyboard-only refresh without granting exact capture', async () => { const videoId = 'abcdefghijk'; const route = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['overview'] }; - const model = sequenceClassifier([{ ...route, frameAccess: 'agent_selected' }, { ...route, frameAccess: 'storyboard_only' }]); + const model = classifierModel({ ...route, refreshFramesOnly: true, frameAccess: 'storyboard_only' }); const message = `Refresh only the storyboard for https://youtu.be/${videoId}.`; const decision = await classifyCapabilityWithModel({ model, message, signal: new AbortController().signal }); - expect(model.doGenerateCalls).toHaveLength(2); + expect(model.doGenerateCalls).toHaveLength(1); expect(decision).toMatchObject({ frameAccess: 'storyboard_only', refreshFramesOnly: true }); }); @@ -1284,7 +1248,7 @@ describe('YouTube agent capability router', () => { it('repairs a storyboard grant into the requested exact capture', async () => { const videoId = 'abcdefghijk'; - const route = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene at 0:45'] }; + const route = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene at 0:45'], timestampedFrameQuestion: 'What is visible?' }; const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [45000] }]; const classifier = sequenceClassifier([{ ...route, frameAccess: 'storyboard_only' }, { ...route, frameAccess }]); const message = `Capture only 0:45 of https://youtu.be/${videoId} and describe it. Do not retrieve any other images.`; @@ -1299,119 +1263,20 @@ describe('YouTube agent capability router', () => { expect(names).not.toContain('get_video_storyboard'); }); - it.each([ - ['1 hour 30 minutes', 5400000, [3600000, 1800000]], - ['1h30m15s', 5415000, [3600000, 1800000, 15000]], - ['1 hour 30 minutes and 15 seconds', 5415000, [3600000, 1815000]], - ] as const)('only grants the complete timestamp in "%s"', async (expression, expected, parts) => { - const videoId = 'abcdefghijk'; - const message = `Capture only the frame at ${expression} of https://youtu.be/${videoId}.`; - const scope = (times: readonly number[]) => [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [...times] }]; - const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'] }; - const model = sequenceClassifier([{ ...base, frameAccess: scope(parts) }, { ...base, frameAccess: scope([expected]) }]); - const decision = await classifyCapabilityWithModel({ message, model, signal: new AbortController().signal }); - expect(decision).toMatchObject({ frameAccess: scope([expected]) }); - expect(model.doGenerateCalls).toHaveLength(2); - expect(readFrameRequest(message).positions).toEqual([{ time: expected, role: 'capture', videoIds: [videoId] }]); - await expect(resolveCapabilityRoute({ persisted: { ...decision, frameAccess: scope(parts) }, message, - classify: vi.fn(), persist: vi.fn() })).resolves.toMatchObject({ responseIntent: 'clarification' }); - }); - - it.each([ - ['Capture only the frame at 1 hour 30 minutes of https://youtu.be/abcdefghijk.', [], 5400000, 1800000], - ['Capture only the linked instant at https://youtu.be/abcdefghijk?t=45 and describe it.', [], 45000, 90000], - ['Capture only the linked instant at https://www.youtube.com/watch?v=abcdefghijk&t=45.', [], 45000, 90000], - ['Capture only 0:45 of https://youtu.be/abcdefghijk.', [], 45000, 90000], - ['Capture only 0:45 of https://youtu.be/abcdefghijk?t=90s.', [], 45000, 90000], - ['Re-inspect the saved 0:30 frame of https://youtu.be/abcdefghijk. Do not recapture it if reuse fails. Capture only 1:00.', [30000], 60000, 30000], - ] as const)('repairs an unauthorized capture before the provider can receive it: %s', async (message, saved, fresh, forbidden) => { - const videoId = 'abcdefghijk'; - const route = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'] }; - const valid = [{ videoId, savedTimestampsMs: [...saved], captureTimestampsMs: [fresh] }]; - const invalid = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [fresh, forbidden] }]; - const model = sequenceClassifier([{ ...route, frameAccess: invalid }, { ...route, frameAccess: valid }]); - const assets = saved.map(time => ({ kind: 'frame' as const, version: 'a'.repeat(64), videoId, current: true, collectedAt: 1, details: { timestampMs: time } })); - const decision = await classifyCapabilityWithModel({ model, message, sessionBrief: { assets, memories: [] }, signal: new AbortController().signal }); - expect(model.doGenerateCalls).toHaveLength(2); - expect(decision).toMatchObject({ frameAccess: valid }); - if (decision.route !== 'inspect_video') throw new Error('Expected inspection.'); - const context = inspectContext(); - const frames = vi.fn>().mockResolvedValue({ value: { videoId, - frames: [{ timestampMs: fresh, width: 1, height: 1, mimeType: 'image/jpeg', imageBase64: '/9j/2Q==' }], - failures: [], meta: { partial: false, warnings: [] } }, cacheStatus: 'miss' }); - context.provider.frames = frames; - await runResearchAgentWithModel({ message, decision, context, model: finalizingModel([ - { name: 'get_video_frames', input: { videoId, timestampsMs: [fresh, forbidden] } }, - { name: 'get_video_frames', input: { videoId, timestampsMs: [fresh] } }, - ]) }); - expect(frames.mock.calls.map(([request]) => request.timestampsMs)).toEqual([[fresh]]); - - // A pre-upgrade turn being resumed must also close this path. - await expect(resolveCapabilityRoute({ persisted: { ...decision, frameAccess: invalid }, message, priorAssets: assets, - classify: vi.fn(), persist: vi.fn() })).resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); - }); - - it.each([ - ['Capture the linked instant at https://youtu.be/abcdefghijk?t=0', 0], - ['Capture the linked instant at https://www.youtube.com/embed/abcdefghijk?start=45', 45000], - ['Capture the linked instant at https://youtu.be/abcdefghijk#t=1m30s', 90000], - ['Capture only 0:30 of https://youtu.be/abcdefghijk?t=45', 30000], - ] as const)('resolves the exact capture position for %s', (message, time) => { - expect(readFrameRequest(message).positions).toEqual([{ videoIds: ['abcdefghijk'], role: 'capture', time }]); - }); - - it('ignores unrelated URL parameters and seek times on other hosts', () => { - expect(readFrameRequest('Inspect https://youtu.be/abcdefghijk?tracking=90s and https://example.org/watch?v=lmnopqrstuv&t=45s').positions).toEqual([]); - }); - - it.each([['0:30', 30000], ['1:00', 60000]] as const)('compares a saved frame with a new %s frame on another video', async (clock, time) => { - const a = 'abcdefghijk', b = 'lmnopqrstuv'; - const frameAccess = [{ videoId: a, savedTimestampsMs: [30000], captureTimestampsMs: [] }, - { videoId: b, savedTimestampsMs: [], captureTimestampsMs: [time] }]; - const route = { route: 'topic_research', comparisonVideoIds: [a, b], frameAccess, visualEvidence: 'required', visualRequirements: ['compare frames'] }; - const model = classifierModel(route); - const message = `Compare the saved 0:30 frame of https://youtu.be/${a} with a new ${clock} frame from https://youtu.be/${b}.`; - const assets = [{ videoId: a, kind: 'frame' as const, version: 'a'.repeat(64), current: true, collectedAt: 1, details: { timestampMs: 30000 } }]; - const decision = await classifyCapabilityWithModel({ model, message, sessionBrief: { assets, memories: [] }, signal: new AbortController().signal }); - expect(decision).toMatchObject({ route: 'topic_research', frameAccess }); - expect(model.doGenerateCalls).toHaveLength(1); - const swapped = frameAccess.map(entry => ({ ...entry, videoId: entry.videoId === a ? b : a })); - await expect(resolveCapabilityRoute({ persisted: { ...decision, frameAccess: swapped }, message, priorAssets: assets, - classify: vi.fn(), persist: vi.fn() })).resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); - }); - - it('keeps exact capture permissions paired with each linked video', async () => { - const [a, b] = ['abcdefghijk', 'lmnopqrstuv'] as const; - const at = (videoId: string, time: number) => ({ videoId, savedTimestampsMs: [], captureTimestampsMs: [time] }); - const route = { route: 'topic_research', comparisonVideoIds: [a, b], visualEvidence: 'required', visualRequirements: ['compare scenes'] }; - const valid = [at(a, 30000), at(b, 60000)]; - const model = sequenceClassifier([{ ...route, frameAccess: [at(a, 60000), at(b, 30000)] }, { ...route, frameAccess: valid }]); - const message = `Capture only 0:30 of https://youtu.be/${a} and capture 1:00 of https://youtu.be/${b}. Compare the scenes.`; - await expect(classifyCapabilityWithModel({ model, message, signal: new AbortController().signal })).resolves.toMatchObject({ frameAccess: valid }); - expect(model.doGenerateCalls).toHaveLength(2); - }); - - it('closes a bad capture list when every classification attempt repeats it', async () => { - const videoId = 'abcdefghijk'; - const model = classifierModel({ route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'], - frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [45000, 90000] }] }); - await expect(classifyCapabilityWithModel({ message: `Capture only 0:45 of https://youtu.be/${videoId}.`, model, - signal: new AbortController().signal })).resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); - }); - it('retrieves only the requested storyboard sheet while exact capture remains closed', async () => { const videoId = 'abcdefghijk'; const message = `Retrieve the storyboard sheet containing 1:30 of https://youtu.be/${videoId}. Do not capture any exact frames.`; - const classifier = classifierModel({ route: 'inspect_video', videoId, frameAccess: 'storyboard_only', + const classifier = classifierModel({ route: 'inspect_video', videoId, frameAccess: { storyboardSelections: [{ videoId, timestampsMs: [90000] }] }, visualEvidence: 'required', visualRequirements: ['scene'] }); const decision = await classifyCapabilityWithModel({ message, model: classifier, signal: new AbortController().signal }); expect(classifier.doGenerateCalls).toHaveLength(1); - expect(decision).toMatchObject({ frameAccess: { storyboardTimestampsMs: [90000] } }); + expect(decision).toMatchObject({ frameAccess: { storyboardSelections: [{ videoId, timestampsMs: [90000] }] } }); if (decision.route !== 'inspect_video') throw new Error('Expected inspection.'); - const resume = (persisted: unknown) => resolveCapabilityRoute({ persisted, message, classify: vi.fn(), persist: vi.fn() }); + const resume = (persisted: unknown) => resolveCapabilityRoute({ persisted, classify: vi.fn(), persist: vi.fn() }); await expect(resume(JSON.parse(JSON.stringify(decision)))).resolves.toEqual(decision); - await expect(resume({ ...decision, frameAccess: { storyboardTimestampsMs: [90000, 120000] } })) - .resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + // An older single-video route has the same effective tool limit. + const legacy = { ...decision, frameAccess: { storyboardTimestampsMs: [90000] } }; + await expect(resume(legacy)).resolves.toEqual(legacy); const context = inspectContext(); const storyboard = vi.fn>().mockResolvedValue({ cacheStatus: 'miss', value: { videoId, frameCount: 20, intervalMs: 5000, sheets: [{ imageBase64: '/9j/2Q==', tileWidth: 100, tileHeight: 100, @@ -1430,45 +1295,14 @@ describe('YouTube agent capability router', () => { it('accepts a capture list that excludes a saved frame the user set aside', async () => { const videoId = 'abcdefghijk'; const frameAccess = [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; - const model = sequenceClassifier([{ route: 'inspect_video', videoId, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'], frameAccess, - setAsideFrames: [{ videoId, timestampMs: 30000 }] }]); + const model = sequenceClassifier([{ route: 'inspect_video', videoId, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['scene'], frameAccess }]); const route = await classifyCapabilityWithModel({ model, message: `Ignore the old 0:30 frame entirely. Capture only 1:00 of https://youtu.be/${videoId}.`, sessionBrief: { memories: [], assets: [{ videoId, version: 'a'.repeat(64), kind: 'frame', current: true, collectedAt: 1, details: { timestampMs: 30000 } }] }, signal: new AbortController().signal }); expect(route).toMatchObject({ route: 'inspect_video', frameAccess }); - expect(route).not.toHaveProperty('setAsideFrames'); expect(model.doGenerateCalls).toHaveLength(1); }); - it.each([[false], [true]])('limits a discovery capture to 1m30s or asks (classifier keeps agent_selected: %s)', async persists => { - const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', visualEvidence: 'required', visualRequirements: ['dough'] }; - const frameAccess = [{ savedTimestampsMs: [], captureTimestampsMs: [90000] }]; - const model = sequenceClassifier([{ ...discovery, frameAccess: 'agent_selected' }, ...(persists ? [] : [{ ...discovery, frameAccess }])]); - const route = await classifyCapabilityWithModel({ model, signal: new AbortController().signal, - message: 'Find a sourdough tutorial and capture only 1m30s of that video. No other timestamps.' }); - if (persists) expect(route).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); - else expect(route).toMatchObject({ route: 'topic_research', frameAccess }); - expect(route).not.toHaveProperty('frameAccess', 'agent_selected'); - }); - - // One parser serves both checks, so every format limits capture and requires the saved frame alike. - it.each(['1:30', '1m30s', '90 seconds', '1 minute 30 seconds', 'the 90-second mark'])('treats %s as a named time in both checks', async time => { - const videoId = 'abcdefghijk'; - const sessionBrief = { memories: [], assets: [{ videoId, version: 'a'.repeat(64), kind: 'frame' as const, current: true, collectedAt: 1, details: { timestampMs: 90000 } }] }; - const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['scene'] }; - const permission = sequenceClassifier([{ ...base, frameAccess: 'agent_selected' }, - { ...base, frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [90000] }] }]); - // No saved frame here, so only the capture-permission check can reject agent_selected. - await classifyCapabilityWithModel({ model: permission, signal: new AbortController().signal, - message: `Capture only ${time} of https://youtu.be/${videoId}.` }); - expect(permission.doGenerateCalls).toHaveLength(2); - const saved = sequenceClassifier([{ ...base, frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [120000] }] }, - { ...base, frameAccess: [{ videoId, savedTimestampsMs: [90000], captureTimestampsMs: [120000] }] }]); - await classifyCapabilityWithModel({ model: saved, sessionBrief, signal: new AbortController().signal, - message: `Re-inspect the saved frame at ${time} and capture 2:00 of https://youtu.be/${videoId}.` }); - expect(saved.doGenerateCalls).toHaveLength(2); - }); - it.each([ 'Find sourdough videos under 5 minutes and show what the dough looks like.', 'Find tutorials about shaping dough that are shorter than 5:00 and choose a frame.', @@ -1482,52 +1316,13 @@ describe('YouTube agent capability router', () => { expect(model.doGenerateCalls).toHaveLength(1); }); - it.each(['0:30', '30 seconds', '0m30s'])('enforces a capture window within %s of the start during discovery', async limit => { - const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', - refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['dough'] }; - const captureAt = (time: number) => [{ savedTimestampsMs: [], captureTimestampsMs: [time] }]; - const model = sequenceClassifier([{ ...discovery, frameAccess: 'agent_selected' }, { ...discovery, frameAccess: captureAt(30001) }]); - const fallbackModel = sequenceClassifier([{ ...discovery, frameAccess: captureAt(30000) }]); - const route = await classifyCapabilityWithModel({ model, fallbackModel, signal: new AbortController().signal, - message: `Find a sourdough tutorial under 5 minutes and capture a frame only within ${limit} of the start. Do not capture anything later.` }); - expect(route).toMatchObject({ route: 'topic_research', frameAccess: captureAt(30000) }); - expect(fallbackModel.doGenerateCalls).toHaveLength(1); - }); - - it('asks for clarification if every attempt leaves the capture window unrestricted', async () => { - const model = sequenceClassifier([{ route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', - frameAccess: 'agent_selected', visualEvidence: 'required', visualRequirements: ['dough'] }]); - const route = await classifyCapabilityWithModel({ model, signal: new AbortController().signal, - message: 'Find a sourdough tutorial and capture a frame only within 0:30 of the start. Do not capture anything later.' }); - expect(route).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); - }); - it('keeps a fixed capture time when discovery chooses the video', async () => { const discovery = { route: 'topic_research', researchBreadth: 'focused', searchQuery: 'sourdough tutorial', visualEvidence: 'required', visualRequirements: ['dough'] }; const frameAccess = [{ savedTimestampsMs: [], captureTimestampsMs: [30000] }]; - const model = sequenceClassifier([{ ...discovery, frameAccess: 'agent_selected' }, { ...discovery, frameAccess }]); + const model = classifierModel({ ...discovery, frameAccess }); const route = await classifyCapabilityWithModel({ model, message: 'Find a sourdough tutorial and capture only 0:30.', signal: new AbortController().signal }); expect(route).toMatchObject({ route: 'topic_research', frameAccess }); - expect(model.doGenerateCalls).toHaveLength(2); - }); - - it.each([ - ['different times', 30000, 60000, 'Reuse the saved 0:30 frame and capture 1:00 of https://youtu.be/lmnopqrstuv. Compare the outfits.'], - ['the same time', 30000, 30000, 'Re-inspect the saved 0:30 frame and capture a new 0:30 frame of https://youtu.be/lmnopqrstuv. Compare the outfits.'], - ['a time in seconds', 30000, 60000, 'Reuse the saved frame at 30 seconds and capture 1:00 of https://youtu.be/lmnopqrstuv. Compare the outfits.'], - ])('recovers both requested operations when a plan drops the saved video (%s)', async (_name, savedTime, captureTime, message) => { - const [saved, linked] = ['abcdefghijk', 'lmnopqrstuv']; - const visual = { visualEvidence: 'required', visualRequirements: ['outfits'] }; - const both = [{ videoId: saved, savedTimestampsMs: [savedTime], captureTimestampsMs: [] }, { videoId: linked, savedTimestampsMs: [], captureTimestampsMs: [captureTime] }]; - const model = sequenceClassifier([ - { route: 'inspect_video', videoId: linked, ...visual, frameAccess: [both[1]] }, - { route: 'topic_research', comparisonVideoIds: [saved, linked], ...visual, frameAccess: both }, - ]); - const route = await classifyCapabilityWithModel({ model, message, signal: new AbortController().signal, - sessionBrief: { memories: [], assets: [{ videoId: saved, version: 'a'.repeat(64), kind: 'frame', current: true, collectedAt: 1, details: { timestampMs: savedTime } }] } }); - expect(route).toMatchObject({ route: 'topic_research', comparisonVideoIds: [saved, linked], frameAccess: both }); - expect(model.doGenerateCalls).toHaveLength(2); - expect(JSON.stringify(model.doGenerateCalls[1]!.prompt)).toContain(`saved 0:30 frame of ${saved}`); + expect(model.doGenerateCalls).toHaveLength(1); }); it.each([ @@ -1547,6 +1342,15 @@ describe('YouTube agent capability router', () => { expect(route).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); }); + it('asks when failed classifier attempts disagree about capture scope', async () => { + const videoId = 'abcdefghijk'; + const invalidRoute = { videoId, route: 'inspect_video', visualEvidence: 'invalid' }; + const model = sequenceClassifier([{ ...invalidRoute, frameAccess: [{ videoId, savedTimestampsMs: [], captureTimestampsMs: [30000] }] }, + { ...invalidRoute, frameAccess: 'agent_selected' }]); + await expect(classifyCapabilityWithModel({ model, message: `Refresh https://youtu.be/${videoId}`, signal: new AbortController().signal })) + .resolves.toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); + }); + it('keeps frame access outside the persisted route closed on resume', async () => { const frameAccess = { videoId: 'lmnopqrstuv', savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; const decision = await resolveCapabilityRoute({ persisted: { route: 'inspect_video', videoId: 'abcdefghijk', researchVideoCount: 1, diff --git a/platform/test/youtube-stored-analysis.test.ts b/platform/test/youtube-stored-analysis.test.ts index 3ed63a58..433eb1ff 100644 --- a/platform/test/youtube-stored-analysis.test.ts +++ b/platform/test/youtube-stored-analysis.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; import { buildAgentTurnResult } from '../src/agents/finalizer'; import { MockLanguageModelV4 } from 'ai/test'; +import { classifyCapabilityWithModel } from '../src/agents/research/capability-router'; import { runResearchAgentWithModel } from '../src/agents/research/research-agent'; import { z } from 'zod'; import type { AgentToolContext } from '../src/agents/providers/youtube/tool-context'; @@ -132,22 +133,38 @@ function finalizeThroughApplication(context: AgentToolContext, packets: Evidence return async () => (await finalize.mock.results.at(-1)!.value).answer as string; } -it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and only fetches 10:45 (saved image %s)', async savedState => { +// Replay from the classifier's structured output through the real tool gates and answer builder. +// The model and image provider are scripted: this tests enforcement, not language understanding. +it.each([ + ...['available', 'missing', 'analysis fails'].map(savedState => ({ + name: 'sourdough', savedState, before: 630000, after: 645000, savedLabel: '10:30', newLabel: '10:45', + question: 'Describe the dough shape.', observation: 'The dough is round.', + message: 'Re-inspect the saved 10:30 image and capture only 10:45. Report the gap if reuse fails.', + })), + { name: 'weather-map 3:30/3:48', savedState: 'analysis fails', before: 210000, after: 228000, savedLabel: '3:30', newLabel: '3:48', + question: 'Read the weather map overlays.', observation: 'The map displays Tuesday.', + message: 'Re-inspect the saved 3:30 weather map. If reuse fails, report the gap and do not recapture it. Capture only 3:48. Read its labels and compare the overlays if possible.' }, +])('mixed-frame replay: $name (saved image $savedState)', async ({ name, savedState, before, after, savedLabel, newLabel, question, observation, message }) => { const { context, provider, store, frames, packets } = setup(); - const frameAccess = [{ videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }]; + const frameAccess = [{ videoId, savedTimestampsMs: [before], captureTimestampsMs: [after] }]; const at = (timestampMs: number) => ({ ...frames, frames: [{ ...frames.frames[0]!, timestampMs }] }); - const saved = store.put('frame', videoId, at(630000), { timestampMs: 630000 }); - const answer = finalizeThroughApplication(context, packets, frameAccess, store.runStart()); + const saved = store.put('frame', videoId, at(before), { timestampMs: before }); + if (savedState === 'missing') store.remove(saved); - provider.frames.mockResolvedValue({ value: at(645000), cacheStatus: 'miss' }); + provider.frames.mockResolvedValue({ value: at(after), cacheStatus: 'miss' }); context.analyzeFrames = vi.fn(async ({ frames }) => { const time = frames.frames[0]!.timestampMs; - if (savedState === 'analysis fails' && time === 630000) throw new Error('Saved image analysis failed.'); - return { findings: [{ observation: 'The dough is round.', timestampsMs: [time] }], warnings: [] }; + if (savedState === 'analysis fails' && time === before) throw new Error('Saved image analysis failed.'); + return { findings: [{ observation, timestampsMs: [time] }], warnings: [] }; }); const calls = [ - { name: 'analyze_video_frames', input: { assetVersions: [saved], focus: 'Inspect the saved dough shape.' } }, - { name: 'get_video_frames', input: { videoId, timestampsMs: [645000], analysis: 'Describe the dough.' } }, + { name: 'analyze_video_frames', input: { assetVersions: [saved], focus: question } }, + // Reproduce the forbidden recovery attempt, including a mixed batch and the storyboard alternative. + ...(name.startsWith('weather') ? [ + { name: 'get_video_frames', input: { videoId, timestampsMs: [before, after] } }, + { name: 'get_video_storyboard', input: { videoId, timestampsMs: [before] } }, + ] : []), + { name: 'get_video_frames', input: { videoId, timestampsMs: [after], analysis: question } }, { name: 'complete_research', input: {} }, ]; const usage = { inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, @@ -160,25 +177,31 @@ it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and on } }); const finalizationModel = new MockLanguageModelV4({ doGenerate: async () => ({ content: [{ type: 'text', text: JSON.stringify({ confidence: 'medium', warnings: [], - blocks: [{ text: 'The dough is round.', evidenceIds: ['ref_1'] }] }) }], + blocks: [{ text: observation, evidenceIds: ['ref_1'] }] }) }], finishReason: { unified: 'stop', raw: 'stop' }, usage, warnings: [], }) }); - await runResearchAgentWithModel({ model, finalizationModel, context, - toolNames: ['analyze_video_frames', 'get_video_frames', 'finalize_answer'], - message: 'Re-inspect the saved 10:30 image and capture only 10:45. Report the gap if reuse fails.', - decision: { route: 'inspect_video', videoId, useStoryboard: true, refreshFramesOnly: true, visualEvidence: 'required', frameAccess }, - }); + const classifier = new MockLanguageModelV4({ doGenerate: async () => ({ + content: [{ type: 'tool-call', toolCallId: 'route', toolName: 'classify_request', input: JSON.stringify({ + route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: [question], frameAccess, + }) }], finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, usage, warnings: [], + }) }); + const decision = await classifyCapabilityWithModel({ model: classifier, message, sessionBrief: context.session!.brief(), signal: context.signal }); + if (decision.route !== 'inspect_video' || !Array.isArray(decision.frameAccess)) throw new Error('Expected scoped inspection'); + const answer = finalizeThroughApplication(context, packets, decision.frameAccess, store.runStart()); + await runResearchAgentWithModel({ model, finalizationModel, context, message, decision }); + expect(provider.storyboard).not.toHaveBeenCalled(); expect(provider.frames).toHaveBeenCalledOnce(); - expect(provider.frames).toHaveBeenCalledWith(expect.objectContaining({ timestampsMs: [645000] }), expect.anything(), + expect(provider.frames).toHaveBeenCalledWith(expect.objectContaining({ timestampsMs: [after] }), expect.anything(), expect.anything(), expect.anything(), expect.anything()); expect(vi.mocked(context.analyzeFrames).mock.calls.map(([input]) => input.frames.frames[0]!.timestampMs)) - .toEqual(savedState === 'missing' ? [645000] : [630000, 645000]); - const savedReport = savedState === 'available' ? 'I re-inspected the saved 10:30 frame' : 'I could not re-inspect the saved 10:30 frame'; + .toEqual(savedState === 'missing' ? [after] : [before, after]); + const savedReport = savedState === 'available' ? `I re-inspected the saved ${savedLabel} frame` : `I could not re-inspect the saved ${savedLabel} frame`; // Findings keep their citations; the account of tool actions is a separate, uncited paragraph. const [findings, activity] = (await answer()).split('\n\n'); + expect(findings).toContain(observation); expect(findings).toContain('[cite:'); expect(activity).toContain(savedReport); - expect(activity).toContain('I retrieved the 10:45 frame and inspected it.'); + expect(activity).toContain(`I retrieved the ${newLabel} frame and inspected it.`); expect(activity).not.toContain('[cite:'); }); @@ -356,16 +379,15 @@ describe('application notes on frame turns', () => { context.frameAccess = [{ videoId: other, savedTimestampsMs: [], captureTimestampsMs: [60000] }]; provider.frames.mockResolvedValue({ value: at(other, 60000), cacheStatus: 'miss' }); const current = await executeGetVideoFrames({ videoId: other, timestampsMs: [60000], analysis: 'What is he wearing?' }, context, 'capture'); - const answer = (text: string, packets: EvidencePacket[], requestedSaved = false) => buildAgentTurnResult({ runId: context.runId, conversationId: crypto.randomUUID(), + const answer = (text: string, packets: EvidencePacket[]) => buildAgentTurnResult({ runId: context.runId, conversationId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }, { userId: 'user', creditsRemaining: 100 }, { answer: text, intent: 'topic_research', confidence: 'medium', citations: [], artifacts: [], warnings: [] }, - [earlier, ...packets], 0, undefined, { requestedSaved, access: context.frameAccess, inventory: inventoryOf(context), startedAt }).answer; + [earlier, ...packets], 0, undefined, { access: context.frameAccess, inventory: inventoryOf(context), startedAt }).answer; /** Moves this turn's start after saves made by a test's own earlier turn. */ const restart = () => { startedAt = store.runStart(); }; return { context, store, at, savedVersion, earlier, current, answer, restart }; } const earlierNote = /The 0:30 observation( of "[^"]+")? comes from an image inspected in an earlier turn, not in this one\./; - const claimNote = 'No frame saved before this turn was re-inspected in this turn'; it('labels an earlier frame citation on a history answer with no frame route or inventory', async () => { const { context, earlier } = await turns(); @@ -397,10 +419,11 @@ describe('application notes on frame turns', () => { 'I inspected the saved 0:30 frame again.', 'Both frames show shirts.', ])('states that no saved frame was re-inspected, whatever the answer says: %s', async claim => { - const { current, answer } = await turns(); - const text = answer(`${claim} The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current], true); - expect(text).toContain(claimNote); - expect(text).toContain('I retrieved the 1:00 frame and inspected it.'); + const { context, current, answer } = await turns(); + context.frameAccess!.push({ videoId, savedTimestampsMs: [30000], captureTimestampsMs: [] }); + const text = answer(`${claim} The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current]); + expect(text).toContain('I could not re-inspect the saved 0:30 frame'); + expect(text).toContain('I retrieved the 1:00 frame of "lmnopqrstuv" and inspected it.'); }); it('counts an image served from session storage as a re-inspected saved frame', async () => { @@ -408,7 +431,7 @@ describe('application notes on frame turns', () => { // A capture request answered from the session returns the image saved by the earlier turn. const reused = { ...current, assetVersions: [savedVersion] }; const text = answer(`The frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [reused]); - expect(text).not.toContain(claimNote); + expect(text).toContain('I re-inspected the saved 0:30 frame'); }); it.each([ @@ -434,7 +457,6 @@ describe('application notes on frame turns', () => { const { current, answer } = await turns(put => put(other, 30000)); const text = answer(`The new frame shows a shirt [cite:${current.excerpts[0]!.id}].`, [current]); expect(text).not.toContain('no recorded position'); - expect(text).not.toContain(claimNote); }); it('judges each citation by its own image when an earlier batch held several', async () => { @@ -452,7 +474,6 @@ describe('application notes on frame turns', () => { const text = answer(`The saved frame shows a shirt [cite:${cited.id}] [cite:${current.excerpts[0]!.id}].`, [batch, current, again]); expect(text).toContain('I re-inspected the saved 0:30 frame'); expect(text).not.toMatch(earlierNote); - expect(text).not.toContain(claimNote); }); it('keeps the earlier-turn note when this turn inspected a different image at the same moment', async () => { @@ -469,7 +490,6 @@ describe('application notes on frame turns', () => { const again = await analyze('frame', savedVersion, context, 'What is he wearing?'); const text = answer(`I re-inspected the saved frame: a shirt [cite:${earlier.excerpts[0]!.id}] [cite:${current.excerpts[0]!.id}].`, [current, again]); expect(text).not.toMatch(earlierNote); - expect(text).not.toContain(claimNote); }); }); diff --git a/platform/tsconfig.agent-live.json b/platform/tsconfig.agent-live.json index 955a936c..8957aed5 100644 --- a/platform/tsconfig.agent-live.json +++ b/platform/tsconfig.agent-live.json @@ -1,4 +1,4 @@ { "extends": "./tsconfig.json", - "include": ["scripts/test-youtube-agent-live.ts", "src/secrets.d.ts", "worker-configuration.d.ts"] + "include": ["scripts/test-youtube-agent-live.ts", "scripts/replays/weather-map.live.test.ts", "src/secrets.d.ts", "worker-configuration.d.ts"] }