diff --git a/platform/src/agents/agent-runtime-do.ts b/platform/src/agents/agent-runtime-do.ts index 9e7bac7b..ebb04cbd 100644 --- a/platform/src/agents/agent-runtime-do.ts +++ b/platform/src/agents/agent-runtime-do.ts @@ -1050,7 +1050,7 @@ export class AgentRuntimeDO extends Agent { requestedVideoIds: decision.route === 'inspect_video' ? [decision.videoId] : decision.comparisonVideoIds ?? [], // An application fallback is checked against what this run had, not earlier turns' cited content. runPackets: this.admittedRunEvidence(runId, history), - } : undefined); + } : undefined, 'frameAccess' in decision ? decision.frameAccess : undefined); // Hash the accepted text: unmatched citations may have been removed from the input. const answerHash = createHash('sha256').update(result.answer).digest('hex'); const serialized = JSON.stringify(result); diff --git a/platform/src/agents/finalizer.ts b/platform/src/agents/finalizer.ts index c778af4b..92f1ddb1 100644 --- a/platform/src/agents/finalizer.ts +++ b/platform/src/agents/finalizer.ts @@ -8,7 +8,9 @@ import { type AgentTurnResult, type EvidencePacket, type FinalizeAnswerInput, + type FrameAccess, } from './contracts'; +import { frameActivityReport } from './runtime/frame-access'; import { appendNotices, playlistCoverageDisclosures } from './playlist-coverage'; import { CITED_FRAME_PREVIEWS, citedPacketFramePreviews } from './runtime/frame-previews'; @@ -28,6 +30,7 @@ export function buildAgentTurnResult( packets: EvidencePacket[], creditsCharged: number, durationLimitContext?: DurationLimitAnswerContext, + frameAccess?: FrameAccess, ): AgentTurnResult { const markers = answerCitationIds(input.answer); const citations: AgentCitation[] = []; @@ -75,7 +78,11 @@ export function buildAgentTurnResult( // The application states playlist totals, the listing limit and reviewed content itself. const playlistCoverage = playlistCoverageDisclosures(packets, citations.map(citation => citation.id)); - const disclosedAnswer = appendNotices(answer, playlistCoverage.notes); + const notes = [...playlistCoverage.notes]; + if (frameAccess?.savedTimestampsMs.length && input.intent !== 'clarification' && input.intent !== 'rejected') { + notes.push(frameActivityReport(frameAccess, identity.runId, packets)); + } + const disclosedAnswer = appendNotices(answer, notes); const warnings = answerWarnings(packets, [ ...input.warnings, diff --git a/platform/src/agents/providers/youtube/tools/get-video-frames.ts b/platform/src/agents/providers/youtube/tools/get-video-frames.ts index a427ce9f..bf71eff2 100644 --- a/platform/src/agents/providers/youtube/tools/get-video-frames.ts +++ b/platform/src/agents/providers/youtube/tools/get-video-frames.ts @@ -106,7 +106,7 @@ export function executeGetVideoFrames(input: z.input analyzeFrameEvidence(validateFrameResponse(request, value), parsed.analysis!, analysisContext, toolCallId), } : undefined, onReady => visualSpan('retrieval', () => context.provider.frames!(request, context.signal, { extractionTimeoutMs, - ...(!context.session && (context.refreshFramesOnly || context.refreshEvidence) ? { refresh: true } : {}) }, + ...(!context.session && (context.frameAccess || context.refreshFramesOnly || context.refreshEvidence) ? { refresh: true } : {}) }, event => { extractionDiagnostics.push(event); context.onExtractionDiagnostic?.({ ...event, toolCallId }); }, ...(onReady ? [onReady] : []))), async (response, signal) => { const frames = validateFrameResponse(request, response.value); if (!context.saveFramePreviews) return; diff --git a/platform/src/agents/providers/youtube/tools/stored-analysis.ts b/platform/src/agents/providers/youtube/tools/stored-analysis.ts index d5bf8afd..8c78c82a 100644 --- a/platform/src/agents/providers/youtube/tools/stored-analysis.ts +++ b/platform/src/agents/providers/youtube/tools/stored-analysis.ts @@ -46,7 +46,7 @@ export async function readAnalysisAssets( () => mapInBatches(versions, async (version) => { context.signal.throwIfAborted(); const requiresRetrieval = - (context.refreshEvidence || (context.refreshFramesOnly && kind === 'frame')) && + (context.refreshEvidence || ((scope || context.refreshFramesOnly) && kind === 'frame')) && !(context.getEvidence?.() ?? []).some( (packet) => packet.packetId.startsWith(`packet:${context.runId}:`) && diff --git a/platform/src/agents/research/capability-provider.ts b/platform/src/agents/research/capability-provider.ts index 931337d2..65867130 100644 --- a/platform/src/agents/research/capability-provider.ts +++ b/platform/src/agents/research/capability-provider.ts @@ -27,7 +27,7 @@ export function createCapabilityProvider( ? { ...options, refresh: true, ...(decision.refreshDynamicData ? { includeSignals: true } : {}) } : options; let result = await savedProvider.video(id, readOptions); const needsDuration = options?.requireFrameDuration || (decision.route === 'inspect_video' && - (decision.refreshFramesOnly || decision.timestampedFrameQuestion)); + (decision.refreshFramesOnly || decision.frameAccess || decision.timestampedFrameQuestion)); if (!decision.savedOnly && needsDuration) { const missingDuration = result.value.isLive !== true && eligibleDuration(result.value.durationSeconds) === undefined; if (!readOptions?.refresh && (result.cacheStatus === 'hit' || result.cacheStatus === 'stale') && (result.value.id !== id || missingDuration)) diff --git a/platform/src/agents/research/capability-router.ts b/platform/src/agents/research/capability-router.ts index d55dac2b..35bf19cf 100644 --- a/platform/src/agents/research/capability-router.ts +++ b/platform/src/agents/research/capability-router.ts @@ -30,9 +30,8 @@ const classifierDecisionSchema = z.object({ skill: z.enum(['verify_saved_evidence']).optional().describe('Verify saved evidence: use for follow-ups that read saved original captions/comments, check exact quotations or precise passage links, or correct source claims. It does not by itself prohibit fetching; set savedOnly separately. Omit for broad summaries and visual analysis.'), requestedEvidence: requestedEvidenceSchema.optional().describe('Evidence the user explicitly asks to be read before answering: original_quotes for exact quotations, verbatim wording or precise passage links from captions; comments for what viewers or commenters say. Research must read each or report it unavailable. Omit when not explicitly requested.'), savedOnly: z.boolean().optional().describe('True when the user requires existing saved evidence only or prohibits fresh retrieval. All provider access is disabled for this run; use saved evidence reads and report missing assets.'), - frameAccess: z.union([frameAccessSchema, z.literal('agent_selected')]).optional().describe('For exact captures on one video, list only the permitted NEW capture timestamps in milliseconds. Put saved-only inspection positions separately in savedTimestampsMs, even if their assets are missing. Use agent_selected only for open-ended capture positions.'), timestampedFrameQuestion: z.string().trim().min(1).max(500).optional().describe('For inspect_video: when the user supplies exact timestamps and asks what is visible, copy the focused visual question here. Include this for a request to describe each specified frame. Omit for image retrieval only or saved-only requests.'), - refreshFramesOnly: z.boolean().optional().describe('For inspection or research, including multi-video comparisons: true when the user asks to freshly capture or refresh frames only. Reuse verified video metadata. Leave refreshEvidence unset unless other source evidence must also be refreshed.'), + refreshFramesOnly: z.union([z.boolean(), frameAccessSchema]).optional().describe('Use true for an open-ended frame refresh or multi-video refresh. For one video with fixed capture positions, supply the videoId, savedTimestampsMs to inspect from storage, and captureTimestampsMs permitted for fresh retrieval. Include missing saved positions. Omit for saved-only inspection.'), refreshEvidence: z.boolean().optional().describe('Except for a frames-only refresh, true only when the user explicitly asks to fetch again, refresh or get fresh source data. Choose an executable route in that case.'), refreshDynamicData: z.boolean().optional().describe('True when the user asks for current views, likes, or comments. Refresh metadata, statistics and comments, keeping saved transcripts and images. Choose an executable route. False for historical questions and dashboard context.'), responseIntent: z.enum(['context_answer', 'clarification', 'rejected']).optional().describe('Required for finalize: answer using existing context, ask for missing scope, or decline an unsupported request.'), @@ -101,7 +100,7 @@ function inferMissingRoute(input: Record, suppliedVideoIds: rea const discovery = DISCOVERY_KEYS.some(present); const inspection = present('videoId'); const finalization = FINALIZATION_KEYS.some(present); - const refresh = input.refreshEvidence === true || input.refreshFramesOnly === true || input.refreshDynamicData === true; + const refresh = input.refreshEvidence === true || Boolean(input.refreshFramesOnly) || input.refreshDynamicData === true; const signals = [discovery, inspection, finalization].filter(Boolean).length; if (signals !== 1) return undefined; // comparisonVideoIds and researchBreadth alone are valid in several routes, so @@ -130,10 +129,6 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio // Keep the provider schema strict. Normalize a copy only at the acceptance // boundary, retaining which fields needed a fallback before the SDK parses them. const input = { ...value } as Record; - if (input.frameAccess && typeof input.frameAccess === 'object' && input.refreshFramesOnly !== true) { - input.refreshFramesOnly = true; - defaultedFields.push('refreshFramesOnly'); - } if (Array.isArray(input.requestedEvidence) && !input.requestedEvidence.length) delete input.requestedEvidence; if (!answerDetailSchema.safeParse(input.answerDetail).success) { input.answerDetail = 'standard'; @@ -170,20 +165,20 @@ function validateClassifierDecision(value: unknown, options: ClassifierValidatio || input.comparisonVideoIds !== undefined || input.requestedEvidence !== undefined || input.visualEvidence !== 'none')) { issue('playlistListing', 'Use playlistListing only for fresh playlist metadata research. For video content or saved verification omit it and keep the required evidence tools.'); } - if (input.refreshFramesOnly === true && input.visualEvidence === 'none') issue('visualEvidence', 'Refreshing frames requires visual tool access.'); - const scope = parsed.success && parsed.data.frameAccess; - if (scope && scope !== 'agent_selected') { - if (!suppliedVideoIds.includes(scope.videoId) || (input.route === 'inspect_video' && input.videoId !== scope.videoId)) - issue('frameAccess', 'Scope frame access to the supplied video being inspected.'); + if (Boolean(input.refreshFramesOnly) && input.visualEvidence === 'none') issue('visualEvidence', 'Refreshing frames requires visual tool access.'); + const scope = parsed.success && parsed.data.refreshFramesOnly; + if (typeof scope === 'object') { + if (input.route !== 'inspect_video' || input.comparisonVideoIds !== undefined || !suppliedVideoIds.includes(scope.videoId) || input.videoId !== scope.videoId) + issue('refreshFramesOnly', 'A fixed capture list requires one inspected video. For multiple subjects, clarify if capture permissions differ by video.'); const times = [...scope.savedTimestampsMs, ...scope.captureTimestampsMs]; - if (new Set(times).size !== times.length) issue('frameAccess', 'Give each timestamp exactly one role, saved inspection or fresh capture.'); + if (new Set(times).size !== times.length) issue('refreshFramesOnly', 'Give each timestamp exactly one role, saved inspection or fresh capture.'); } if (input.timestampedFrameQuestion && input.route !== 'inspect_video') issue('route', 'Exact frame requests require single-video inspection.'); - if (input.savedOnly === true && (input.refreshEvidence === true || input.refreshFramesOnly === true || input.refreshDynamicData === true)) issue('savedOnly', 'Saved-only requests cannot refresh source evidence.'); + if (input.savedOnly === true && (input.refreshEvidence === true || Boolean(input.refreshFramesOnly) || input.refreshDynamicData === true)) issue('savedOnly', scope ? 'For mixed frames, omit savedOnly and give the permitted capture positions in refreshFramesOnly. For saved-only inspection, omit refreshFramesOnly.' : 'Saved-only requests cannot refresh source evidence.'); if (input.route === 'finalize' && Array.isArray(input.requestedEvidence) && input.requestedEvidence.length) issue('route', 'Requested evidence must be read by research or inspection before answering.'); if (input.historySelection && input.contextScope === 'video') issue('contextScope', 'History selection requires history or mixed context.'); - if (input.route === 'finalize' && (input.refreshEvidence === true || input.refreshFramesOnly === true || input.refreshDynamicData === true)) issue('route', 'Fresh retrieval requires an executable route.'); + if (input.route === 'finalize' && (input.refreshEvidence === true || Boolean(input.refreshFramesOnly) || input.refreshDynamicData === true)) issue('route', 'Fresh retrieval requires an executable route.'); // Run conditional presence checks even when essential base fields failed validation, // so one bounded repair sees every missing field instead of only the first layer. return issues.length ? { success: false, error: new z.ZodError(issues), defaultedFields } : { ...parsed, defaultedFields }; @@ -283,13 +278,13 @@ async function classifyWithinDeadline(input: CapabilityClassifierInput, deadline 'Requests for current view counts, likes, or comments require an executable route with refreshDynamicData true, even when past values are in history. Use saved data for historical questions. This does not require refreshing transcripts or images.', 'Classify the current request for an agent that researches and synthesizes information from YouTube videos. Decide scope before selecting tools.', 'Playlist discovery and listing are supported. For a fresh YouTube playlist listing by URL or name choose topic_research, playlistListing true, focused breadth, a playlist search query and visualEvidence none. Listing titles, links, durations, totals and arithmetic from those durations are supported, including when the user gives their answer a study-note name. Earlier assistant refusals are not capability limits. For a saved-only listing reuse retained evidence instead.', - 'The session inventory describes available raw assets, their collection times and coverage. Session memories are derived hints, not proof. A saved transcript can support new questions through research or inspection, which must prepare its content before finalization. Counts of frames or sheets do not prove that a requested scene was observed. Select inspection if new visual interpretation is needed. For fresh frames, including a request comparing several videos, set refreshFramesOnly true and leave refreshEvidence unset. If timestamps and a visual question are supplied, set timestampedFrameQuestion even when the user also names get_video_frames. Keep raw-image-only requests free of an analysis question. Refreshing transcripts or all source evidence requires refreshEvidence true. Requests limited to current statistics or comments use refreshDynamicData true instead.', + 'The session inventory describes available raw assets, their collection times and coverage. Session memories are derived hints, not proof. A saved transcript can support new questions through research or inspection, which must prepare its content before finalization. Counts of frames or sheets do not prove that a requested scene was observed. Select inspection if new visual interpretation is needed. For fresh frames, including a request comparing several videos, use refreshFramesOnly and leave refreshEvidence unset. If timestamps and a visual question are supplied, set timestampedFrameQuestion even when the user also names get_video_frames. Keep raw-image-only requests free of an analysis question. Refreshing transcripts or all source evidence requires refreshEvidence true. Requests limited to current statistics or comments use refreshDynamicData true instead.', 'Use prior completed turns and availableEvidence to choose the next action. Choose finalize with responseIntent context_answer when the request can be answered from the conversation or supplied evidence without further retrieval or analysis. Prior assistant claims are not verified source evidence. Questions about what was previously said may use history alone; new video facts require supplied evidence. An inventory reference alone does not mean its content is ready for the answer. Finalization cannot search or read saved assets. Choose research or inspection if answering requires reading a stored transcript, analyzing an asset, or loading additional evidence. When evidence is insufficient or the user asks for new inspection or fresh data, choose inspect_video or topic_research.', 'Requests to list, quote, summarize, or correct messages in this conversation are supported. Route them to finalize with responseIntent context_answer. The runtime supplies a bounded chronological history page before finalization, alongside the recent conversation window. Finalization cannot search history, paginate, read assets, or retrieve provider evidence. It must state any history coverage limit. Choose inspect_video or topic_research when source evidence needs reading or analysis; use context_answer only for prepared context. Do not list the messages yourself.', 'Select skill verify_saved_evidence for follow-ups that recover saved original captions/comments, verify exact quotations or passage links, or correct source claims. This replaces general discovery and analysis with a verification workflow. The skill does not prohibit fetching: a request that also needs evidence not yet saved, such as comments, still gets it unless savedOnly is set. Omit the skill for broad summaries or visual analysis.', 'Set requestedEvidence for each kind of evidence the user explicitly asks for: original_quotes for exact quotations, verbatim wording or precise passage links; comments for what viewers or commenters say. Include every requested kind in one decision, alongside any explanation. Research cannot complete until each is read or reported unavailable, so choose an executable route.', 'Set savedOnly true whenever the user prohibits fresh retrieval or asks to use existing saved sources only. Keep this restriction on follow-ups when it remains in force. Missing saved evidence is a gap, never permission to fetch. Use the saved reader instead of transcript retrieval or discovery.', - 'When a turn mixes saved and new frames, omit savedOnly and supply frameAccess. For example, re-inspect saved 10:30 and capture only 10:45 means savedTimestampsMs [630000] and captureTimestampsMs [645000]. A prohibition on recapturing 10:30 still allows the requested 10:45. Times mentioned for comparison, narration or history do not authorize capture.', + 'When a turn mixes saved and new frames, omit savedOnly and supply the fixed timestamp lists in refreshFramesOnly. For example, re-inspect saved 10:30 and capture only 10:45 means savedTimestampsMs [630000] and captureTimestampsMs [645000]. A prohibition on recapturing 10:30 still allows the requested 10:45. Times mentioned for comparison, narration or history do not authorize capture.', 'Requests for exact quotations from original saved captions/comments, precise passage-start links, or explicit original-asset reads require inspect_video or topic_research with the resolved subjects, even if prior summaries discuss the topic. Research can call search_originals without fetching YouTube. Quoting a source is different from quoting a prior conversation message. Do not route an original-source read directly to finalize.', 'For every context_answer set contextScope: history for listing, quoting, recalling or correcting conversation messages or preferences; video for claims about video content; mixed only when the requested answer needs both. A video URL inside a quoted earlier message does not require video evidence.', 'For finalize give a short routing reason, not a user-facing answer. Choose responseIntent clarification for missing scope, or rejected for unsupported requests. Do not use the legacy clarification or rejected routes for new decisions.', @@ -594,15 +589,19 @@ function candidateObject(value: unknown): Record | undefined { } function semanticIssues(decision: ClassifierDecision, input: CapabilityClassifierInput, videoIds: string[]): ClassificationIssue[] { - // An explicit prohibition rules out an unrestricted fallback. Only the classifier - // resolves which video and positions the user authorized. - const prohibitsRecapture = /\b(?:without|never|do not|don['’]t)\s+re-?captur(?:e|ing)\b/i.test(input.message); - if (decision.frameAccess === 'agent_selected' && prohibitsRecapture) { - return [{ path: 'frameAccess', code: 'prohibited_recapture', message: 'The request prohibits recapture. Supply the fixed fresh positions and keep the saved inspection positions separate.' }]; + const refresh = decision.refreshFramesOnly; + const links = extractYouTubeVideoIds(input.message); + if (typeof refresh === 'object' && links.some(id => id !== refresh.videoId)) { + return [{ path: 'refreshFramesOnly', code: 'wrong_frame_video', message: 'The capture scope must match the video linked in the current request.' }]; } - if (decision.refreshFramesOnly && !decision.frameAccess && !decision.comparisonVideoIds?.length - && (decision.route === 'inspect_video' || videoIds.length === 1)) { - return [{ path: 'frameAccess', code: 'missing_capture_scope', message: 'Specify saved and authorized new timestamps separately, or agent_selected if the user delegated the capture positions.' }]; + // Explicit positions require a bounded list. An ordinary request to refresh frames can + // keep delegating the positions, including when all classifier attempts used the old boolean. + const namesPositions = /\b(?:\d+:\d{2}|\d+\s*ms\b)/i.test(input.message); + const forbidsRecapture = /\b(?:without|never|do not|don['’]t)\s+re-?captur(?:e|ing)\b/i.test(input.message); + const unrestricted = decision.route !== 'finalize' && !decision.savedOnly && typeof refresh !== 'object'; + if (unrestricted && (forbidsRecapture || (refresh === true && namesPositions && !decision.comparisonVideoIds?.length + && (decision.route === 'inspect_video' || videoIds.length === 1)))) { + return [{ path: 'refreshFramesOnly', code: 'missing_capture_scope', message: 'Separate the saved inspection positions from the permitted capture positions in the frame refresh.' }]; } const scope = comparisonScopeIssues(decision, input, videoIds); return scope.length ? scope : searchQueryNumberIssues(decision, input); @@ -636,13 +635,17 @@ function lastResortDecision( // Requirements any candidate stated, or the request implies, survive every fallback: // fresh data needs an executable route, required visuals stay required, and a // follow-up comparison keeps both of its subjects. + const boundedCandidates = candidates.filter(candidate => candidate.refreshFramesOnly && typeof candidate.refreshFramesOnly === 'object'); + const boundedRefresh = firstValid('refreshFramesOnly', boundedCandidates); + if (boundedCandidates.length && !boundedRefresh) { + const clarification = { route: 'finalize', responseIntent: 'clarification', reason: 'Clarify the saved and permitted fresh frame positions.', answerDetail: 'standard' }; + return { decision: classifierDecisionSchema.parse(clarification), method: 'defaults', + defaultedFields: Object.keys(clarification) as DefaultedClassificationField[] }; + } const constraints = { - // Preserve a bounded list even when a later failed attempt proposed open-ended selection. - frameAccess: firstValid('frameAccess', candidates.filter(candidate => candidate.frameAccess !== 'agent_selected')) - ?? firstValid('frameAccess'), savedOnly: candidates.some(candidate => candidate.savedOnly === true), requestedEvidence: [...new Set(candidates.flatMap(candidate => requestedEvidenceSchema.safeParse(candidate.requestedEvidence).data ?? []))], - refreshFramesOnly: candidates.some(candidate => candidate.refreshFramesOnly === true), + refreshFramesOnly: boundedRefresh ?? candidates.some(candidate => candidate.refreshFramesOnly === true), refreshEvidence: candidates.some(candidate => candidate.refreshEvidence === true), refreshDynamicData: candidates.some(candidate => candidate.refreshDynamicData === true), visualsRequired: candidates.some(candidate => candidate.visualEvidence === 'required'), @@ -654,7 +657,7 @@ function lastResortDecision( return kept.length >= 2 ? [kept] : []; })[0] ?? []])].slice(0, 8), }; - const needsFreshData = constraints.frameAccess || constraints.refreshFramesOnly || constraints.refreshEvidence || constraints.refreshDynamicData; + const needsFreshData = constraints.refreshFramesOnly || constraints.refreshEvidence || constraints.refreshDynamicData; if (needsFreshData && constraints.savedOnly) { const clarification = { route: 'finalize', responseIntent: 'clarification', reason: 'Routing attempts disagree about saved-only access and fresh retrieval. Ask which sources may be refreshed before retrieving or answering from stale data.', answerDetail: 'standard' }; @@ -697,14 +700,13 @@ function lastResortDecision( } if (decision.answerDetail === undefined) fill('answerDetail', 'standard'); if (decision.route !== 'finalize') { - if (constraints.frameAccess) fill('frameAccess', constraints.frameAccess); if (decision.route === 'inspect_video' && !constraints.savedOnly) { const matching = candidates.filter(candidate => candidate.videoId === decision.videoId || (candidate.videoId === undefined && decision.videoId === soleVideoId)); fill('timestampedFrameQuestion', firstValid('timestampedFrameQuestion', matching)); } if (constraints.refreshFramesOnly && !constraints.savedOnly) { - fill('refreshFramesOnly', true); + fill('refreshFramesOnly', constraints.refreshFramesOnly); if (!decision.visualEvidence || decision.visualEvidence === 'none') fill('visualEvidence', 'helpful'); } if (constraints.refreshEvidence && !constraints.savedOnly) fill('refreshEvidence', true); @@ -818,8 +820,8 @@ function expectedComparisonSubjects(input: CapabilityClassifierInput): string[] function finishClassification( decision: z.infer, videoIds: string[], channelIds: string[], explicitVideoIds: string[], message: string, ): CapabilityRouteDecision { - const { explicitSourceCount, frameAccess, ...fields } = withVisualAccess(decision); - const classified = { ...fields, ...(frameAccess && frameAccess !== 'agent_selected' ? { frameAccess } : {}) }; + const { explicitSourceCount, refreshFramesOnly, ...fields } = withVisualAccess(decision); + const classified = { ...fields, ...(typeof refreshFramesOnly === 'object' ? { frameAccess: refreshFramesOnly } : { refreshFramesOnly }) }; const pinned = !(decision.route === 'topic_research' && decision.playlistListing) && explicitVideoIds.length === 1 && !decision.comparisonVideoIds?.length && (decision.route !== 'finalize' || (decision.responseIntent === 'context_answer' && decision.contextScope !== 'history')); const route = pinned ? { ...classified, route: 'inspect_video' as const, videoId: explicitVideoIds[0]!, @@ -957,7 +959,16 @@ export async function resolveCapabilityRoute(options: { classify: () => Promise; persist: (decision: CapabilityRouteDecision) => void | Promise; }): Promise { - if (options.persisted !== undefined) return capabilityRouteDecisionSchema.parse(options.persisted); + if (options.persisted !== undefined) { + const decision = capabilityRouteDecisionSchema.parse(options.persisted); + // Older runs could pin a different video after validating the scope. Resuming + // must never discard that scope to make the mismatched route executable. + if ('frameAccess' in decision && decision.frameAccess && (decision.comparisonVideoIds?.length + || (decision.route === 'inspect_video' && decision.videoId !== decision.frameAccess.videoId))) { + return { route: 'finalize', responseIntent: 'clarification', reason: 'Clarify which video the saved and fresh frame positions refer to.' }; + } + return decision; + } const decision = capabilityRouteDecisionSchema.parse(await options.classify()); await options.persist(decision); return decision; diff --git a/platform/src/agents/research/research-agent.ts b/platform/src/agents/research/research-agent.ts index 7c2edb38..314ec2f8 100644 --- a/platform/src/agents/research/research-agent.ts +++ b/platform/src/agents/research/research-agent.ts @@ -11,7 +11,7 @@ import { durationLimitNotice, withDurationLimitNotice } from './duration-limit-a import { canAnalyzeStoryboard, storyboardRetrievalBudget, STORYBOARD_RETRIEVAL_MIN_MS } from '../runtime/storyboard-budget'; import { transcriptFailureCode, YOUTUBE_UNAVAILABLE_MESSAGE } from '../providers/youtube/tools/transcript-tool-errors'; import { traceToolCallRepair, traceToolSet, type TraceToolCall } from '../runtime/tool-call-trace'; -import { sessionBriefForModel, type SessionEvidenceStore } from '../runtime/session-evidence'; +import { sessionBriefForModel, type SessionBrief, type SessionEvidenceStore } from '../runtime/session-evidence'; import { sessionProvider } from '../runtime/session-provider'; import { createReadPriorEvidenceTool, isHistoryOnlyRoute, preparePriorEvidence, PRIOR_EVIDENCE_GUIDANCE, READ_PRIOR_EVIDENCE_TOOL_NAME, type DeliverEvidence, type PriorEvidenceAccess } from '../runtime/prior-evidence'; @@ -275,7 +275,7 @@ async function executeResearchRunWithAssets(options: { await options.onCapabilityLoaded(decision.route, researchDeadlineAt); const upstream = decision.savedOnly ? savedOnlyProvider() : createYouTubeAgentProvider(options.env, undefined, researchDeadlineAt); - const refreshFrames = decision.refreshEvidence || decision.refreshFramesOnly; + const refreshFrames = decision.refreshEvidence || decision.refreshFramesOnly || !!decision.frameAccess; const provider = createCapabilityProvider(options.session ? sessionProvider(upstream, options.session, decision.refreshEvidence, options.registerRetrievedAsset, assetCache, refreshFrames) : upstream, decision); const transcriptAnalyst = createTranscriptAnalyst( runModel(options.env, options.sessionAffinity, 'low', { @@ -565,7 +565,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { ...outstandingRequested().map(({ kind, videoId }) => ({ reason: 'requested_evidence_missing', kind, ...(videoId ? { videoId } : {}) })), ]; }; - const pendingOriginals = () => pendingOriginalSources(options.decision, options.context.session?.brief(), evidence.values(), + const pendingOriginals = () => !savedEvidenceSkill ? [] : pendingOriginalSources(options.decision, options.context.session?.brief(), evidence.values(), options.context.session?.savedEvidenceReadState?.(options.context.runId)); const toolFailures = new Map( (options.recoveredToolFailures ?? []).map((failure) => [failure.toolCallId, failure]), @@ -626,6 +626,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { } return pending; }; + const failedFrameVersions = new Set(); const trackedContext: AgentToolContext = { ...options.context, researchDeadlineAt: options.researchDeadlineAt, @@ -664,13 +665,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { } const target = researchVideoTarget(options.decision); const requiredVideos = options.decision.comparisonVideoIds?.length ?? (options.decision.route === 'topic_research' ? options.decision.requiredVideoCount : undefined); - const warnings = mergeWarnings(input.warnings.filter(warning => warning.code !== 'RESEARCH_COVERAGE_SHORTFALL'), - youtubeAvailabilityWarnings([...toolFailures.values()])); - if (frameScope?.savedTimestampsMs.length) { - // The unified finalizer renders this in prose. Direct and fallback answers retain it as a note. - const { report } = frameActivity(frameScope, options.context.runId, evidence.values()); - if (!input.answer.includes(report)) warnings.unshift({ code: 'FRAME_ACTIVITY', message: report.slice(0, 1000) }); - } + const warnings: AgentWarning[] = []; if (visualRequired && !hasVisualObservations()) { warnings.push({ code: 'VISUAL_EVIDENCE_INCOMPLETE', message: `The request required visual evidence${visualRequirements.length ? ` (${visualRequirements.join('; ')})` : ''}, but no analyzed visual observations were collected. Visual portions of the answer remain unverified.` }); @@ -688,7 +683,11 @@ async function runResearchAgentWithModelWithinDeadline(options: { warnings.push({ code: 'CHANNEL_INSPECTION_INCOMPLETE', message: 'The requested channel catalog could not be inspected. Do not treat this response as complete channel research.' }); } - const result = await options.context.finalize(id, withDurationLimitNotice({ ...input, warnings: warnings.slice(0, 50), artifacts }, currentDurationNotice())); + const answerWarnings = mergeWarnings( + [...warnings, ...youtubeAvailabilityWarnings([...toolFailures.values()])], + input.warnings.filter(warning => warning.code !== 'RESEARCH_COVERAGE_SHORTFALL'), + ); + const result = await options.context.finalize(id, withDurationLimitNotice({ ...input, warnings: answerWarnings, artifacts }, currentDurationNotice())); finalized = true; return result; }, @@ -743,6 +742,10 @@ async function runResearchAgentWithModelWithinDeadline(options: { for (const id of evidence.keys()) if (!retained.has(id)) evidence.delete(id); return packet; } catch (error) { + if (options.decision.frameAccess && execution.toolName === 'analyze_video_frames') { + const versions = (execution.input as { assetVersions: string[] }).assetVersions; + for (const version of versions) failedFrameVersions.add(version); + } toolFailures.set(execution.toolCallId, { toolCallId: execution.toolCallId, toolName: execution.toolName, @@ -760,6 +763,9 @@ async function runResearchAgentWithModelWithinDeadline(options: { const visualRequired = visualEvidenceLevel(options.decision) === 'required'; const visualRequirements = options.decision.visualRequirements ?? []; const frameScope = options.decision.frameAccess; + let initialBrief: SessionBrief | undefined; + let savedFramesAtStart: SessionBrief['assets'] = []; + const hasVisualObservations = () => [...evidence.values()].some(packet => packet.excerpts.length > 0 && packet.artifacts.some(artifact => ['youtube_frame_analysis', 'youtube_storyboard_analysis'].includes(artifact.type))); @@ -768,16 +774,16 @@ async function runResearchAgentWithModelWithinDeadline(options: { const needsVisualWork = () => { const failed = new Set([...toolFailures.values()].map(failure => failure.toolName)); if (frameScope) { - const { saved, captures } = frameActivity(frameScope, options.context.runId, evidence.values()); - const available = options.context.session?.brief().assets ?? []; - const needsSavedAnalysis = saved.some(item => !item.inspected && available.some(asset => asset.kind === 'frame' - && asset.videoId === frameScope.videoId && asset.details.timestampMs === item.time)); - const needsCaptureAnalysis = options.decision.route === 'inspect_video' && options.decision.timestampedFrameQuestion - && captures.some(item => item.retrieved && !item.inspected); - return (toolNames.includes('analyze_video_frames') && !!options.context.analyzeFrames && !failed.has('analyze_video_frames') + const { saved, captures, inspectedVersions, pendingCaptureVersions } = frameActivity(frameScope, options.context.runId, evidence.values()); + const needsSavedAnalysis = saved.some(item => !item.inspected && savedFramesAtStart.some(asset => + (asset.details.timestampMs === item.time || (typeof asset.details.timestampMs !== 'number' && !failed.has('analyze_video_frames'))) + && !inspectedVersions.has(asset.version) && !failedFrameVersions.has(asset.version) && options.context.session?.has?.(asset.version) !== false)); + const needsCaptureAnalysis = (visualRequired || (options.decision.route === 'inspect_video' && options.decision.timestampedFrameQuestion)) + && pendingCaptureVersions.some(version => !failedFrameVersions.has(version) && options.context.session?.has?.(version) !== false); + if ((toolNames.includes('analyze_video_frames') && !!options.context.analyzeFrames && (needsSavedAnalysis || needsCaptureAnalysis)) || (toolNames.includes('get_video_frames') && !!options.context.provider.frames && !failed.has('get_video_frames') - && frameExtractionBudget(options.researchDeadlineAt) >= FRAME_EXTRACTION_MIN_MS && captures.some(item => !item.requested)); + && frameExtractionBudget(options.researchDeadlineAt) >= FRAME_EXTRACTION_MIN_MS && captures.some(item => !item.requested))) return true; } if (!visualRequired || hasVisualObservations()) return false; const savedFrames = [...evidence.values()].some(packet => packet.kind === 'youtube_frames' && packet.assetVersions?.length); @@ -787,7 +793,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { && (savedFrames || frameExtractionBudget(options.researchDeadlineAt) >= FRAME_EXTRACTION_MIN_MS); const savedStoryboards = [...evidence.values()].some(packet => packet.kind === 'youtube_storyboard' && packet.artifacts.some(artifact => artifact.type === 'youtube_storyboard_retrieval' && Number(artifact.data.sampledFrames) > 0)) - || options.context.session?.brief().assets.some(asset => asset.kind === 'storyboard_sheet') === true; + || initialBrief?.assets.some(asset => asset.kind === 'storyboard_sheet') === true; const storyboardPossible = toolNames.includes('get_video_storyboard') && toolNames.includes('analyze_video_storyboard') && !!options.context.provider.storyboard && !!options.context.analyzeStoryboard && !failed.has('analyze_video_storyboard') && canAnalyzeStoryboard(options.researchDeadlineAt) @@ -821,7 +827,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { }, }; if (options.decision.route === 'inspect_video' && toolNames.includes('get_video') && (options.decision.refreshDynamicData || options.decision.refreshEvidence || - ((options.decision.timestampedFrameQuestion || options.decision.refreshFramesOnly) && verifiedFrameMetadata(options.decision.videoId, [...evidence.values()]) === undefined) + ((options.decision.timestampedFrameQuestion || options.decision.refreshFramesOnly || options.decision.frameAccess) && verifiedFrameMetadata(options.decision.videoId, [...evidence.values()]) === undefined) || !transcriptSourceContext(options.decision.videoId, [...evidence.values()]).title)) { try { await executeGetVideo({ videoId: options.decision.videoId }, phaseContext, `initial-video:${options.decision.videoId}`); @@ -835,6 +841,8 @@ async function runResearchAgentWithModelWithinDeadline(options: { signal.throwIfAborted(); } } + initialBrief = options.context.session?.brief(); + if (frameScope) savedFramesAtStart = initialBrief?.assets.filter(asset => asset.kind === 'frame' && asset.videoId === frameScope.videoId) ?? []; const sessionTools: ToolSet = { ...contextHandoff.tools(await phaseContext.session?.searchTools?.((packets, source = 'search_context') => { const admitted = options.context.deliverEvidence ? options.context.deliverEvidence(packets, source).admitted : packets; @@ -909,7 +917,7 @@ async function runResearchAgentWithModelWithinDeadline(options: { options.conversationHistory ?? [], options.message, [...evidence.values()].map(evidencePacketForModel), - options.context.session ? sessionBriefForModel(options.context.session.brief()) : undefined, + initialBrief ? sessionBriefForModel(initialBrief) : undefined, options.inheritedResolved && options.context.session ? options.prior?.pointers ?? [] : undefined, ), context: phaseContext, @@ -1097,8 +1105,6 @@ async function runUnifiedFinalizer(options: { .filter(packet => packet.sources.some(source => source.videoId && subjects.has(source.videoId))); const inspectionRecords = inspected.length ? inspectionRecordsForModel(inspected, options.context.evidenceRunRecords?.(inspected.map(packet => packet.packetId))) : undefined; - const frameReport = 'frameAccess' in options.decision && options.decision.frameAccess?.savedTimestampsMs.length - ? frameActivity(options.decision.frameAccess, options.context.runId, options.evidence).report : undefined; const baseOutputSchema = conversational ? conversationalFinalizationOutputSchema : intent === 'context_answer' ? contextFinalizationOutputSchema : finalizationOutputSchema; const numberedItemCount = 'numberedItemCount' in options.decision ? options.decision.numberedItemCount : undefined; @@ -1201,7 +1207,8 @@ async function runUnifiedFinalizer(options: { 'Keep JSON compact. Use short ref_N citations rather than full evidence IDs. For specific-video comparisons cite every subject, or explicitly state the missing side and add ANSWER_SCOPE_SHORTFALL. If contextIncomplete is true, do not claim exhaustive coverage unless the supplied evidence establishes it.', EVIDENCE_DELIVERY_GUIDANCE, INSPECTION_RECORD_GUIDANCE, - ...(frameReport ? ['The application will insert frameActivityReport into the answer. Write the findings and their limitations. Use only the inspections confirmed in that report as current visual evidence. If a saved inspection is missing, disclose what could not be verified. Leave the account of tool actions to the application.'] : []), + ...('frameAccess' in options.decision && options.decision.frameAccess?.savedTimestampsMs.length + ? ['Base visual claims on successful inspections. Explain any gap. The application adds the frame activity to the answer from the recorded operations.'] : []), 'Recovery has a limited token budget. Preserve the requested count where evidence permits by shortening each item before reducing the count. If scope remains incomplete, state the shortfall and add ANSWER_SCOPE_SHORTFALL. Do not pad or invent findings.', 'State important evidence gaps plainly. Do not claim that a failed provider operation succeeded.', 'For list evidence such as playlists, take counts from collection: entriesShown are the entries supplied here, and collection.coverage states what was not retained, not shown, or not stated by the provider. Never report fewer available entries than are shown. Describe unshown or unretained entries as a context limit, not as absent from the source. When the answer cites a playlist entry, or reviewed transcript or visual content for a listed video, the application appends a playlist coverage note stating the provider total, the 12-video listing limit and which videos had content reviewed; do not contradict it. Otherwise no note is appended, so state any material playlist coverage limit yourself. Never present listing details as reviewed video content. evidenceOmittedFromContext counts saved list packets that did not fit at all; notListed of them are counted but not named.', @@ -1226,7 +1233,6 @@ async function runUnifiedFinalizer(options: { providerFailures: groupedToolFailures(options.toolFailures).map(failure => failure.durationLimit ? { ...failure, message: 'VIDEO_TOO_LONG: The video exceeded the configured Agent duration limit.' } : failure), inspectionRecords, - frameActivityReport: frameReport, validationFeedback: feedback, })}, ...(usesSavedEvidenceSkill(options.decision) ? (options.conversationHistory ?? []).flatMap(turn => [ @@ -1324,7 +1330,6 @@ async function runUnifiedFinalizer(options: { for (const block of output.blocks) { block.evidenceIds = block.evidenceIds.map(id => prepared.fullIds.get(id) ?? id); } - if (frameReport && output.blocks.length) output.blocks[0]!.text += ` ${frameReport}`; validationStage = 'answer_notes'; // Judge each block by every reference it will render, declared or inline. const references = output.blocks.map(block => blockReferences(block, prepared.fullIds)); diff --git a/platform/src/agents/runtime/frame-access.ts b/platform/src/agents/runtime/frame-access.ts index 2e4a8956..1e5290bd 100644 --- a/platform/src/agents/runtime/frame-access.ts +++ b/platform/src/agents/runtime/frame-access.ts @@ -5,6 +5,8 @@ import { frameCoverageSchema } from './model-evidence'; /** A requested position, a returned image and an analyst result are different facts. * Only packets produced by this run can establish that it did the work. */ export function frameActivity(scope: FrameAccess, runId: string, evidence: Iterable) { + const inspectedVersions = new Set(); + const retrievedVersions = new Set(); const inspected = new Set(); const retrieved = new Set(); const requested = new Set(); @@ -17,6 +19,7 @@ export function frameActivity(scope: FrameAccess, runId: string, evidence: Itera const coverage = frameCoverageSchema.safeParse(artifact.data); if (!coverage.success) continue; const analyzing = artifact.type === 'youtube_frame_analysis'; + for (const version of packet.assetVersions ?? []) (analyzing ? inspectedVersions : retrievedVersions).add(version); if (!analyzing) for (const time of coverage.data.requestedTimestampsMs) requested.add(time); for (const frame of coverage.data.frames) (analyzing ? inspected : retrieved).add(frame.timestampMs); } @@ -25,7 +28,12 @@ export function frameActivity(scope: FrameAccess, runId: string, evidence: Itera const captures = scope.captureTimestampsMs.map(time => ({ time, requested: requested.has(time), retrieved: retrieved.has(time), inspected: inspected.has(time), })); - const report = [ + return { saved, captures, inspectedVersions, pendingCaptureVersions: [...retrievedVersions].filter(version => !inspectedVersions.has(version)) }; +} + +export function frameActivityReport(scope: FrameAccess, runId: string, packets: Iterable): string { + const { saved, captures } = frameActivity(scope, runId, packets); + return [ ...saved.map(({ time, inspected }) => inspected ? `I re-inspected the saved ${clockTime(time)} frame.` : `I could not re-inspect the saved ${clockTime(time)} frame in this turn.`), ...captures.map(({ time, requested, retrieved, inspected }) => retrieved @@ -33,5 +41,4 @@ export function frameActivity(scope: FrameAccess, runId: string, evidence: Itera : requested ? `I requested ${clockTime(time)}, but no frame was returned.` : `I have no confirmed retrieval of the ${clockTime(time)} frame in this turn.`), ].join(' '); - return { saved, captures, report }; } diff --git a/platform/test/youtube-agent-frame-deadlines.test.ts b/platform/test/youtube-agent-frame-deadlines.test.ts index 7a3bcccd..9b75884f 100644 --- a/platform/test/youtube-agent-frame-deadlines.test.ts +++ b/platform/test/youtube-agent-frame-deadlines.test.ts @@ -243,8 +243,10 @@ test('names the required visual facts when finalizing without visual observation const model = new MockLanguageModelV4({ doGenerate: async () => ({ content: [{ type: 'tool-call', toolCallId: 'finish', toolName: 'finalize_answer', input: JSON.stringify({ intent: 'inspect_video', confidence: 'low', artifacts: [], warnings: [], blocks: [{ text: 'Unverified.', evidenceIds: ['ref_1'] }] }) }], finishReason: { unified: 'tool-calls', raw: undefined }, usage, warnings: [] }) }); - await runResearchAgentWithModel({ ...options, model, recoveredEvidence: [metadata], + const warnings = Array.from({ length: 50 }, (_, index) => ({ code: 'TRANSCRIPT_CONTEXT_TRUNCATED', message: `Omitted passage ${index}.` })); + await runResearchAgentWithModel({ ...options, model, recoveredEvidence: [{ ...metadata, warnings }], recoveredToolFailures: [{ toolCallId: 'frames', toolName: 'get_video_frames', operation: 'frames', message: 'Frame extraction failed' }] }); + expect(vi.mocked(context.finalize).mock.calls[0]![1].warnings).toHaveLength(50); expect(JSON.stringify(model.doGenerateCalls[0]?.prompt)).toContain('Required visual evidence: player names on shirts'); expect(context.finalize).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ warnings: expect.arrayContaining([expect.objectContaining({ code: 'VISUAL_EVIDENCE_INCOMPLETE', message: expect.stringContaining('player names on shirts') })]) })); diff --git a/platform/test/youtube-agent-router.test.ts b/platform/test/youtube-agent-router.test.ts index daaed224..76454f45 100644 --- a/platform/test/youtube-agent-router.test.ts +++ b/platform/test/youtube-agent-router.test.ts @@ -1091,19 +1091,40 @@ describe('YouTube agent capability router', () => { expect(decision).toMatchObject({ route: 'inspect_video', videoId, timestampedFrameQuestion: question }); }); - it.each(['overlap', 'unrestricted'])('repairs an unscoped refresh and rejects %s capture permission', async invalid => { + it.each(['direct', 'recovery'])('runs an open-ended frame refresh after %s routing', async mode => { const videoId = 'abcdefghijk'; + const model = sequenceClassifier([{ route: 'inspect_video', videoId, refreshFramesOnly: true, + visualEvidence: mode === 'direct' ? 'helpful' : 'invalid' }]); + const decision = await classifyCapabilityWithModel({ model, message: 'Refresh the frames in https://youtu.be/abcdefghijk', + signal: new AbortController().signal }); + expect(decision).toMatchObject({ route: 'inspect_video', videoId, refreshFramesOnly: true }); + expect(decision).not.toHaveProperty('frameAccess'); + }); + + it.each(['overlap', 'unrestricted', 'omitted', 'wrong video', 'comparison'] as const)('repairs %s before authorizing a bounded frame refresh', async invalid => { + const videoId = 'abcdefghijk', previous = 'lmnopqrstuv'; const frameAccess = { videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; - const base = { route: 'inspect_video', videoId, refreshFramesOnly: true, visualEvidence: 'required', visualRequirements: ['dough shape'] }; - const model = sequenceClassifier([base, { ...base, frameAccess: invalid === 'overlap' - ? { ...frameAccess, captureTimestampsMs: [630000, 645000] } : 'agent_selected' }]); - const fallbackModel = sequenceClassifier([{ ...base, frameAccess }]); - const route = await classifyCapabilityWithModel({ model, fallbackModel, message: 'Reuse saved 10:30 without recapturing it and capture only 10:45.', - sessionBrief: { memories: [], assets: [{ videoId, version: 'a'.repeat(64), kind: 'frame', current: true, - collectedAt: 1, details: { timestampMs: 630000 } }] }, signal: new AbortController().signal }); - expect(route).toMatchObject({ route: 'inspect_video', videoId, frameAccess, refreshFramesOnly: true }); + const base = { route: 'inspect_video', videoId, visualEvidence: 'required', visualRequirements: ['dough shape'] }; + const candidates = { + omitted: base, + overlap: { ...base, refreshFramesOnly: { ...frameAccess, captureTimestampsMs: [645000, 630000] } }, + unrestricted: { ...base, refreshFramesOnly: true }, + 'wrong video': { ...base, videoId: previous, refreshFramesOnly: { ...frameAccess, videoId: previous } }, + comparison: { ...base, route: 'topic_research', comparisonVideoIds: [previous, videoId], refreshFramesOnly: frameAccess }, + }; + const model = sequenceClassifier([candidates[invalid], { ...base, refreshFramesOnly: frameAccess }]); + const route = await classifyCapabilityWithModel({ model, message: 'Reuse saved 10:30 without recapturing it and capture only 10:45 in https://youtu.be/abcdefghijk.', + sessionBrief: { memories: [], assets: [{ videoId: previous, version: 'a'.repeat(64), kind: 'frame', current: true, + collectedAt: 1, details: {} }] }, signal: new AbortController().signal }); + expect(route).toMatchObject({ route: 'inspect_video', videoId, frameAccess }); expect(model.doGenerateCalls).toHaveLength(2); - expect(fallbackModel.doGenerateCalls).toHaveLength(1); + }); + + it('keeps conflicting persisted frame permissions closed on resume', async () => { + const frameAccess = { videoId: 'lmnopqrstuv', savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; + const decision = await resolveCapabilityRoute({ persisted: { route: 'inspect_video', videoId: 'abcdefghijk', frameAccess }, + classify: vi.fn(), persist: vi.fn() }); + expect(decision).toMatchObject({ route: 'finalize', responseIntent: 'clarification' }); }); it('pins inspect_video provider calls to the classified video', async () => { diff --git a/platform/test/youtube-stored-analysis.test.ts b/platform/test/youtube-stored-analysis.test.ts index dbc74bec..72f8b35d 100644 --- a/platform/test/youtube-stored-analysis.test.ts +++ b/platform/test/youtube-stored-analysis.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it, vi } from 'vitest'; +import { buildAgentTurnResult } from '../src/agents/finalizer'; import { MockLanguageModelV4 } from 'ai/test'; import { runResearchAgentWithModel } from '../src/agents/research/research-agent'; import { z } from 'zod'; import type { AgentToolContext } from '../src/agents/providers/youtube/tool-context'; -import { evidencePacketSchema, type EvidencePacket } from '../src/agents/contracts'; +import { evidencePacketSchema, type EvidencePacket, type FinalizeAnswerInput } from '../src/agents/contracts'; import { createCapabilityToolSet } from '../src/agents/providers/youtube/tool-library'; import { executeGetVideoTranscriptForModel } from '../src/agents/providers/youtube/tools/get-video-transcript'; import { executeGetVideoFrames } from '../src/agents/providers/youtube/tools/get-video-frames'; @@ -115,9 +116,16 @@ function setup() { const kinds = ['transcript', 'frame', 'storyboard_sheet'] as const; it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and only fetches 10:45 (saved image %s)', async savedState => { - const { context, provider, store, frames } = setup(); + const { context, provider, store, frames, packets } = setup(); + const identity = { runId: crypto.randomUUID(), conversationId: crypto.randomUUID(), + userMessageId: crypto.randomUUID(), agentMessageId: crypto.randomUUID() }; + context.runId = identity.runId; + const frameAccess = { videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; + const finalize = vi.fn(async (_id: string, input: FinalizeAnswerInput) => buildAgentTurnResult(identity, { userId: 'user', creditsRemaining: 100 }, + input, packets, 0, undefined, frameAccess)); + context.finalize = finalize; const at = (timestampMs: number) => ({ ...frames, frames: [{ ...frames.frames[0]!, timestampMs }] }); - const saved = store.put('frame', videoId, at(630000), { timestampMs: 630000 }); + const saved = store.put('frame', videoId, at(630000)); if (savedState === 'missing') store.remove(saved); provider.frames.mockResolvedValue({ value: at(645000), cacheStatus: 'miss' }); context.analyzeFrames = vi.fn(async ({ frames }) => { @@ -125,16 +133,21 @@ it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and on if (savedState === 'analysis fails' && time === 630000) throw new Error('Saved image analysis failed.'); return { findings: [{ observation: 'The dough is round.', timestampsMs: [time] }], warnings: [] }; }); - const calls = [ - { name: 'analyze_video_frames', input: { assetVersions: [saved], focus: 'Inspect the saved dough shape.' } }, - { name: 'get_video_frames', input: { videoId, timestampsMs: [645000], analysis: 'Describe the dough.' } }, - { name: 'complete_research', input: {} }, - ]; + const inspectSaved = { name: 'analyze_video_frames', input: { assetVersions: [saved], focus: 'Inspect the saved dough shape.' } }; + const capture = { name: 'get_video_frames', input: { videoId, timestampsMs: [645000] } }; + const inspectCapture = { name: 'analyze_video_frames', input: { assetVersions: [] as string[], focus: 'Describe the dough.' } }; + const calls = savedState === 'analysis fails' ? [inspectSaved, capture, inspectCapture] : [capture, inspectCapture, inspectSaved]; const usage = { inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 }, outputTokens: { total: 1, text: 1, reasoning: 0 } }; let step = 0; - const model = new MockLanguageModelV4({ doGenerate: async () => { - const call = calls[step++]!; + const model = new MockLanguageModelV4({ doGenerate: async ({ tools }) => { + const call = calls[step++] ?? { name: 'complete_research', input: {} }; + if (call.name !== 'complete_research' && savedState !== 'missing') { + expect(tools?.map(tool => tool.name)).not.toContain('complete_research'); + } + if (call === inspectCapture) { + inspectCapture.input.assetVersions = packets.find(packet => packet.artifacts.some(artifact => artifact.type === 'youtube_frame_retrieval'))!.assetVersions!; + } return { content: [{ type: 'tool-call', toolCallId: `step-${step}`, toolName: call.name, input: JSON.stringify(call.input) }], finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, usage, warnings: [] }; } }); @@ -146,22 +159,21 @@ it.each(['available', 'missing', 'analysis fails'])('inspects saved 10:30 and on await runResearchAgentWithModel({ model, finalizationModel, context, toolNames: ['analyze_video_frames', 'get_video_frames', 'finalize_answer'], message: 'Re-inspect the saved 10:30 image and capture only 10:45. Report the gap if reuse fails.', - decision: { route: 'inspect_video', videoId, useStoryboard: true, refreshFramesOnly: true, visualEvidence: 'required', - frameAccess: { videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] } }, + decision: { route: 'inspect_video', videoId, useStoryboard: true, visualEvidence: 'required', frameAccess }, }); expect(provider.frames).toHaveBeenCalledOnce(); expect(provider.frames).toHaveBeenCalledWith(expect.objectContaining({ timestampsMs: [645000] }), expect.anything(), - expect.anything(), expect.anything(), expect.anything()); + expect.anything(), expect.anything()); expect(vi.mocked(context.analyzeFrames).mock.calls.map(([input]) => input.frames.frames[0]!.timestampMs)) - .toEqual(savedState === 'missing' ? [645000] : [630000, 645000]); + .toEqual(savedState === 'missing' ? [645000] : savedState === 'analysis fails' ? [630000, 645000] : [645000, 630000]); const savedReport = savedState === 'available' ? 'I re-inspected the saved 10:30 frame' : 'I could not re-inspect the saved 10:30 frame'; - expect(JSON.stringify(finalizationModel.doGenerateCalls[0]!.prompt)).toContain(savedReport); - expect(context.finalize).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ - answer: expect.stringContaining(savedReport), - })); - expect(context.finalize).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ - answer: expect.stringContaining('I retrieved the 10:45 frame and inspected it.'), - })); + const result = await finalize.mock.results[0]!.value; + const [findings, activity] = result.answer.split('\n\n'); + expect(findings).toContain('[cite:'); + expect(activity).toContain(savedReport); + expect(activity).toContain('I retrieved the 10:45 frame and inspected it.'); + expect(activity).not.toContain('[cite:'); + expect(result.warnings).not.toContainEqual(expect.objectContaining({ code: 'FRAME_ACTIVITY' })); }); it('rejects forbidden positions and other videos before metadata, billing or extraction', () => { @@ -180,7 +192,6 @@ it('rejects forbidden positions and other videos before metadata, billing or ext it('keeps the freshness requirement for a previously saved version at the new capture position', async () => { const { context, store, frames } = setup(); - context.refreshFramesOnly = true; context.frameAccess = { videoId, savedTimestampsMs: [630000], captureTimestampsMs: [645000] }; const stale = store.put('frame', videoId, { ...frames, frames: [{ ...frames.frames[0]!, timestampMs: 645000 }] }); await expect(analyze('frame', stale, context)).rejects.toThrow('requires a fresh capture');