diff --git a/sdk_v2/INFERENCING.md b/sdk_v2/INFERENCING.md index 2284eb761..3bdda0c16 100644 --- a/sdk_v2/INFERENCING.md +++ b/sdk_v2/INFERENCING.md @@ -228,8 +228,10 @@ For complete setup and inference examples, see the native vision samples: #### Current limitations Remote HTTP(S) image fetching and image `file_id` references are not supported. -Image input is restricted to the first conversation turn and cannot be combined -with tool definitions. Start a new conversation to submit another image turn. +Images may appear in any user turn. Earlier images are retained and replayed until +the conversation history is discarded, increasing memory usage. Multi-image support +and input limits depend on the model and its chat template. Media cannot be combined +with tool definitions or a conversation containing tool calls or results. ## Run a request, then stream a follow-up diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index a795f2ef5..16cc3adfd 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -1015,8 +1015,8 @@ bool IsGuidedToolRetryEligible( int attempted_tokens, std::optional output_budget) { return tool_ctx.tool_plan && natural_tool_output_end && - (!semantic_output_seen || safe_prefix_available || !tool_ctx.text_output) && - !tool_call_published && + (!semantic_output_seen || safe_prefix_available || !tool_ctx.text_output) && + !tool_call_published && (!output_budget || attempted_tokens < *output_budget) && options.stop_sequences.empty() && tool_ctx.tool_output && @@ -1372,13 +1372,10 @@ std::unique_ptr PrepareChatRequest( } const auto effective_kvp = MergeKeyValuePairs(base_session_options, internal_request.options); - auto media = CollectMediaInput(internal_request); - const bool media_turn = !media.Empty(); - if (media_turn) { + if (RequestHasMedia(internal_request)) { ToolCallContext declarations; chat_session_internal::PopulateToolDefinitions(request_definitions, declarations); - ValidateMediaTurn(media, BuildTranscriptMessages(internal_request.items), - {.tools_declared = declarations.HasTools()}); + ValidateMediaToolDefinitions(declarations.HasTools()); } prepared->tool_context = BuildToolCallContextForRequest(internal_request, effective_kvp, request_definitions, @@ -1390,7 +1387,10 @@ std::unique_ptr PrepareChatRequest( const ChatTranscript payload_transcript; payload_transcript.ValidateInputs(messages); prepared->messages.emplace(message_preparer(std::move(messages), model.HasPositionalToolResults())); + auto media = CollectMediaInput(prepared->messages->Messages()); + const bool media_turn = !media.Empty(); if (media_turn) { + ValidateMediaTurn(media, prepared->messages->Messages(), {}); prepared->prompt = PrepareMediaChatPrompt(media.messages, model, media.images, media.audios, prepared->tool_context); } else { @@ -1404,13 +1404,11 @@ std::unique_ptr PrepareChatRequest( auto effective_kvp = MergeKeyValuePairs(base_session_options, request.options); chat_completions::ApplyCatalogDefaults(effective_kvp, model_info.model_settings); - auto media = CollectMediaInput(request); - const bool media_turn = !media.Empty(); - if (media_turn) { + const bool request_has_media = RequestHasMedia(request); + if (request_has_media || CarriesMedia(transcript.Messages())) { ToolCallContext declarations; chat_session_internal::PopulateToolDefinitions(tool_definitions, declarations); - ValidateMediaTurn(media, BuildTranscriptMessages(request.items), - {.session_has_history = !transcript.Empty(), .tools_declared = declarations.HasTools()}); + ValidateMediaToolDefinitions(declarations.HasTools()); } prepared->tool_context = BuildToolCallContextForRequest(request, effective_kvp, tool_definitions, @@ -1423,7 +1421,7 @@ std::unique_ptr PrepareChatRequest( prepared->system_prompt = GetOptionOrEmpty(effective_kvp, kSystemPromptOption); if (!TurnCanGenerate(prepared->ingest.messages, - {.media = media_turn, + {.media = request_has_media, .history = !transcript.Empty(), .system_prefix = !prepared->system_prompt.empty()})) { FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, @@ -1440,15 +1438,12 @@ std::unique_ptr PrepareChatRequest( all_messages = WithSystemPrompt(prepared->system_prompt, std::move(all_messages)); prepared->messages.emplace(message_preparer(std::move(all_messages), model.HasPositionalToolResults())); + auto media = CollectMediaInput(prepared->messages->Messages()); + const bool media_turn = !media.Empty(); if (media_turn) { - auto media_messages = std::move(media.messages); - if (!prepared->system_prompt.empty()) { - media_messages.insert(media_messages.begin(), - MessageItem(FOUNDRY_LOCAL_ROLE_SYSTEM, prepared->system_prompt)); - } - + ValidateMediaTurn(media, prepared->messages->Messages(), {}); prepared->prompt = - PrepareMediaChatPrompt(media_messages, model, media.images, media.audios, prepared->tool_context); + PrepareMediaChatPrompt(media.messages, model, media.images, media.audios, prepared->tool_context); } else { prepared->prompt = PrepareTextChatPrompt(*prepared->messages, model, prepared->tool_context); } @@ -1651,7 +1646,7 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) !cached_tool_ctx_.HasSameTools(turn_tool_ctx); if (cached_generator_ && - (turn_system_prompt != system_prompt_ || retained_tool_context_changed || + (media_turn || turn_system_prompt != system_prompt_ || retained_tool_context_changed || (backend_kind == ChatBackendKind::kGenerator && (inputs.empty() || CarriesToolActivity(inputs))))) { InvalidateCachedGenerator(); } @@ -2081,8 +2076,7 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response) // from plain text turns, so a rebuild restores correct behavior. This matches C#, which always applies the full // template per turn. // - // Media is single-shot too: AppendMessages cannot extend a media-decoded sequence, so any text follow-up must - // rebuild rather than silently feed text into a state that includes media-derived tokens. + // AppendMessages cannot extend a media-decoded sequence. Rebuild later turns from the retained media transcript. // // A turn stopped at a tool call because the model kept talking afterwards is truncated mid-sequence with no turn // terminator, so its KV cache is no basis for the next turn either. diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h index 55d327279..729f6e3ca 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h @@ -191,10 +191,7 @@ class ChatSession : public Session { /// each turn's input messages and assistant reply from the transcript. /// If all turns are undone, the cached generator is destroyed. /// - /// Vision turns: image input is only allowed while the conversation has no - /// history. UndoTurns rolls back messages, so undoing every turn does make - /// the session accept media again — but the media bytes of an undone turn - /// are gone either way, because they never entered the transcript. + /// Media retained by an undone turn is removed with that turn; earlier media remains available to future prompts. /// /// Blocks until any in-flight request on this session completes. /// diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc index 15d9bb9eb..a29d274f1 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc @@ -4,6 +4,8 @@ #include "exception.h" #include "inferencing/session/tool_registry.h" +#include "items/audio_item.h" +#include "items/image_item.h" #include "items/message_item.h" #include "items/text_item.h" #include "items/tool_call_item.h" @@ -147,6 +149,12 @@ bool TranscriptMessage::HasToolCalls() const { return false; } +bool TranscriptMessage::HasMedia() const { + return std::any_of(entries.begin(), entries.end(), [](const TranscriptEntry& entry) { + return entry.kind == TranscriptEntry::Kind::kImage || entry.kind == TranscriptEntry::Kind::kAudio; + }); +} + bool TranscriptMessage::HasVisibleTextAfterToolCall() const { bool seen_call = false; for (const auto& entry : entries) { @@ -225,6 +233,10 @@ void MergeAssistantTurn(TranscriptMessage& open, TranscriptMessage&& next) { case TranscriptEntry::Kind::kToolCall: open.AppendToolCall(std::move(entry.tool_call)); break; + case TranscriptEntry::Kind::kImage: + case TranscriptEntry::Kind::kAudio: + open.entries.push_back(std::move(entry)); + break; } } } @@ -286,15 +298,33 @@ TranscriptIngest IngestRequestItems(const std::vector& items, const std:: message.name = message_item.name; for (const auto& part : message_item.content) { - if (!part.view || part.view->type != FOUNDRY_LOCAL_ITEM_TEXT) { + if (!part.view) { continue; } - const auto& text_item = static_cast(*part.view); - if (text_item.text_type == FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING) { - message.AppendReasoning(text_item.text); - } else { - message.AppendText(text_item.text); + if (part.view->type == FOUNDRY_LOCAL_ITEM_TEXT) { + const auto& text_item = static_cast(*part.view); + if (text_item.text_type == FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING) { + message.AppendReasoning(text_item.text); + } else { + message.AppendText(text_item.text); + } + } else if (part.view->type == FOUNDRY_LOCAL_ITEM_IMAGE) { + const auto& image = static_cast(*part.view); + auto snapshot = std::make_shared(image.ReadBytes(), image.format); + message.entries.push_back(TranscriptEntry::Media(TranscriptEntry::Kind::kImage, std::move(snapshot))); + } else if (part.view->type == FOUNDRY_LOCAL_ITEM_AUDIO) { + const auto& audio = static_cast(*part.view); + if (audio.data == nullptr || audio.data_size == 0) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "audio entry must contain bytes"); + } + + const auto* begin = static_cast(audio.data); + auto snapshot = + std::make_shared(std::vector(begin, begin + audio.data_size), audio.format); + snapshot->sample_rate = audio.sample_rate; + snapshot->channels = audio.channels; + message.entries.push_back(TranscriptEntry::Media(TranscriptEntry::Kind::kAudio, std::move(snapshot))); } } @@ -366,6 +396,11 @@ bool CarriesToolActivity(const std::vector& messages) { }); } +bool CarriesMedia(const std::vector& messages) { + return std::any_of(messages.begin(), messages.end(), + [](const TranscriptMessage& message) { return message.HasMedia(); }); +} + bool CarriesPriorTurnHistory(const std::vector& messages) { return std::any_of(messages.begin(), messages.end(), [](const TranscriptMessage& message) { return message.role == FOUNDRY_LOCAL_ROLE_ASSISTANT || message.role == FOUNDRY_LOCAL_ROLE_TOOL; @@ -374,7 +409,8 @@ bool CarriesPriorTurnHistory(const std::vector& messages) { bool CarriesRespondableContent(const std::vector& messages) { return std::any_of(messages.begin(), messages.end(), [](const TranscriptMessage& message) { - return !message.VisibleText().empty() || message.HasToolCalls() || message.role == FOUNDRY_LOCAL_ROLE_TOOL; + return !message.VisibleText().empty() || message.HasMedia() || message.HasToolCalls() || + message.role == FOUNDRY_LOCAL_ROLE_TOOL; }); } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h index dbce3dac8..de22596a8 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -45,30 +46,36 @@ struct TranscriptEntry { kText, // visible assistant/user text kReasoning, // chain-of-thought text kToolCall, // a tool invocation + kImage, + kAudio, }; Kind kind = Kind::kText; std::string text; TranscriptToolCall tool_call; + std::shared_ptr media; static TranscriptEntry Text(std::string value) { - return {Kind::kText, std::move(value), {}}; + return {Kind::kText, std::move(value), {}, {}}; } static TranscriptEntry Reasoning(std::string value) { - return {Kind::kReasoning, std::move(value), {}}; + return {Kind::kReasoning, std::move(value), {}, {}}; } static TranscriptEntry ToolCall(TranscriptToolCall call) { - return {Kind::kToolCall, {}, std::move(call)}; + return {Kind::kToolCall, {}, std::move(call), {}}; + } + + static TranscriptEntry Media(Kind kind, std::shared_ptr value) { + return {kind, {}, {}, std::move(value)}; } }; /// A message in the authoritative transcript. /// -/// Unlike MessageItem (the API-facing type) a transcript message keeps visible text, reasoning, and tool calls as -/// separate ordered events rather than folding everything into modality content parts. Media parts are intentionally -/// absent: media input is single-shot and is fed to the generator directly from the request items. +/// Visible text, reasoning, media, and tool calls remain separate ordered events. Media owns an immutable snapshot +/// shared by transcript copies, so later turns and preflight operations see the original bytes. struct TranscriptMessage { flMessageRole role = FOUNDRY_LOCAL_ROLE_NONE; std::vector entries; @@ -91,6 +98,7 @@ struct TranscriptMessage { void AppendToolCall(TranscriptToolCall call); bool HasToolCalls() const; + bool HasMedia() const; /// True when the message carries visible text that follows a tool call and is more than whitespace. /// @@ -155,10 +163,9 @@ struct TranscriptIngest { /// Convert request items into transcript messages, preserving the order in which they were supplied. /// -/// - MESSAGE items become role-tagged messages; typed TextItem parts keep their visible / reasoning distinction and -/// non-text parts are skipped (media is handled separately by the caller). A message with neither content nor a -/// participant name carries nothing and is dropped; a content-free named message survives to attribute the tool -/// calls that follow it. +/// - MESSAGE items become role-tagged messages; text keeps its visible / reasoning distinction and image and audio +/// parts retain owned byte snapshots in their original positions. A message with neither content nor a participant +/// name carries nothing and is dropped; a content-free named message survives to attribute following tool calls. /// - TOOL_CALL items become assistant messages, merged into an open assistant turn when there is one, so replayed /// assistant content and its calls stay together exactly as the model produced them. /// - TOOL_RESULT items become role="tool" messages carrying the call ID. An empty result string is preserved. @@ -209,6 +216,9 @@ std::vector BuildTranscriptMessages( /// relative to the surrounding conversation — and the media prompt path cannot render it at all. bool CarriesToolActivity(const std::vector& messages); +/// True when any message contains image or audio input. +bool CarriesMedia(const std::vector& messages); + /// True when `text` is empty or contains nothing but whitespace. /// /// Shared by the transcript's ordering invariant and by generation. Whitespace after a tool call does not end the @@ -219,14 +229,13 @@ bool IsWhitespaceOnly(std::string_view text); /// whatever the caller is sending now, whether it came from a live transcript or from a replayed chain. bool CarriesPriorTurnHistory(const std::vector& messages); -/// True when `messages` carry something the model can actually respond to: visible text, a tool call, or a tool -/// result. Assistant turn boundaries alone are not — they only say a turn happened. +/// True when `messages` carry something the model can actually respond to: visible text, media, a tool call, or a +/// tool result. Assistant turn boundaries alone are not — they only say a turn happened. bool CarriesRespondableContent(const std::vector& messages); /// Everything outside a turn's messages that can still give the model something to generate from. struct TurnContent { - /// The request carries image or audio bytes. They reach the generator directly rather than as message text, so a - /// media-only turn has no entries at all — the bytes are the question. + /// The request carries image or audio bytes. bool media = false; /// The session transcript already holds a conversation, so an empty request asks for the next turn of it. bool history = false; diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/media_input.cc b/sdk_v2/cpp/src/inferencing/generative/chat/media_input.cc index f04503fd4..ab1e45a68 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/media_input.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/media_input.cc @@ -3,9 +3,28 @@ #include "inferencing/generative/chat/media_input.h" #include "exception.h" +#include "items/text_item.h" namespace fl { +bool RequestHasMedia(const Request& request) { + for (const auto* item : request.items) { + if (item == nullptr || item->type != FOUNDRY_LOCAL_ITEM_MESSAGE) { + continue; + } + + const auto& message = static_cast(*item); + for (const auto& part : message.content) { + if (part.view && (part.view->type == FOUNDRY_LOCAL_ITEM_IMAGE || + part.view->type == FOUNDRY_LOCAL_ITEM_AUDIO)) { + return true; + } + } + } + + return false; +} + MediaInput CollectMediaInput(const Request& request) { MediaInput media; std::vector message_items; @@ -43,6 +62,53 @@ MediaInput CollectMediaInput(const Request& request) { return media; } +MediaInput CollectMediaInput(const std::vector& messages) { + MediaInput media; + if (!CarriesMedia(messages)) { + return media; + } + + media.messages.reserve(messages.size()); + for (const auto& message : messages) { + MessageItem projected; + projected.role = message.role; + projected.name = message.name; + for (const auto& entry : message.entries) { + switch (entry.kind) { + case TranscriptEntry::Kind::kText: + projected.content.push_back(MessagePart::Own(std::make_unique(entry.text))); + break; + case TranscriptEntry::Kind::kReasoning: + projected.content.push_back( + MessagePart::Own(std::make_unique(entry.text, FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING))); + break; + case TranscriptEntry::Kind::kImage: + media.images.push_back(static_cast(entry.media.get())); + projected.content.push_back(MessagePart::Borrow(*entry.media)); + break; + case TranscriptEntry::Kind::kAudio: + media.audios.push_back(static_cast(entry.media.get())); + projected.content.push_back(MessagePart::Borrow(*entry.media)); + break; + case TranscriptEntry::Kind::kToolCall: + break; + } + } + + media.messages.push_back(std::move(projected)); + } + + return media; +} + +void ValidateMediaToolDefinitions(bool tools_declared) { + if (tools_declared) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, + "image or audio input cannot be combined with tool definitions; the multimodal prompt does not " + "support tool calls or results"); + } +} + void ValidateMediaTurn(const MediaInput& media, const std::vector& inputs, const MediaTurnContext& context) { @@ -50,18 +116,10 @@ void ValidateMediaTurn(const MediaInput& media, return; } - if (context.session_has_history || CarriesPriorTurnHistory(inputs)) { - FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, - "image or audio input is only allowed on the first turn of a conversation; the bytes are not part of " - "the conversation record, so no later prompt can show them to the model again. Start a new " - "conversation to send media."); - } - - if (context.tools_declared) { + ValidateMediaToolDefinitions(context.tools_declared); + if (CarriesToolActivity(inputs)) { FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, - "image or audio input cannot be combined with tool definitions; the model could answer with a tool " - "call, and the turn carrying that call's result could no longer show it the media. Send the media in a " - "request that declares no tools."); + "image or audio input cannot be combined with tool calls or results in the conversation"); } } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/media_input.h b/sdk_v2/cpp/src/inferencing/generative/chat/media_input.h index 417d13daa..bea958048 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/media_input.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/media_input.h @@ -12,14 +12,12 @@ namespace fl { -/// Media parts referenced by a request, plus copies of the messages that carry them. +/// Media parts and the messages that carry them. /// -/// The media prompt path (OnnxChatGenerator::CreateWithMedia) renders these messages and nothing else, so it cannot -/// use the transcript projection and has no way to render a tool call or a tool result. +/// The media prompt path renders these messages and has no way to render a tool call or a tool result. /// -/// Lifetime: `images` and `audios` are **borrowed** from the Request that CollectMediaInput was given — they point -/// into the items that request owns. A MediaInput must not outlive that request, and the bytes must not be captured -/// beyond the turn that reads them. `messages` are copies, so they carry no such constraint. +/// The pointers borrow from the request or transcript supplied to CollectMediaInput. Transcript-derived messages +/// also borrow their media parts, so the MediaInput must not outlive that transcript. struct MediaInput { std::vector messages; std::vector images; @@ -34,26 +32,22 @@ struct MediaInput { /// The returned image and audio pointers borrow from `request` — see MediaInput's lifetime note. MediaInput CollectMediaInput(const Request& request); +/// Project the complete transcript into media messages and buffers in conversation order. +MediaInput CollectMediaInput(const std::vector& messages); + +/// Check request media without copying its contents. +bool RequestHasMedia(const Request& request); + /// Everything outside the request's own items that decides whether a media turn is allowed. struct MediaTurnContext { - /// The session transcript already holds a conversation. - bool session_has_history = false; /// This turn declares tools, so the model may answer it with a call. bool tools_declared = false; }; -/// Reject a media turn this runtime cannot serve. -/// -/// Media is single-shot: image and audio bytes go straight to the generator and never enter the transcript, so no -/// later prompt can reproduce them. That gives one conversation-scoped rule which does not depend on whether the -/// session happened to still be cached: -/// -/// - Media is only allowed while the conversation has no history. Prior history reaches a turn two ways — the live -/// transcript, or a chain replayed into this request's inputs after the session cache dropped it — and both are -/// rejected identically, so a continuation behaves the same warm and cold. -/// - A media turn must not declare tools. The model would be free to answer with a call, and the result for that -/// call would arrive on a later turn that can no longer show the model the image it was looking at. Rejecting up -/// front is the only way that follow-up is never stranded. +/// Reject tool declarations before their schemas are admitted on a media turn. +void ValidateMediaToolDefinitions(bool tools_declared); + +/// Reject media combined with declared tools or historical tool activity, which the media template cannot render. /// /// @throws fl::Exception FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT — the caller sent a request this runtime cannot /// serve, so it maps to HTTP 400. No-op when the request carries no media. diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc index 22a51a777..b6978953e 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc @@ -312,11 +312,8 @@ void OnnxChatGenerator::ResetTurnState() { // Media helpers // --------------------------------------------------------------------------- -std::string OnnxChatGenerator::TransformMessagesForMedia(const std::vector& messages) { - // Find the index of the last user message so we can rewrite only that one - // into structured content, preserving the positions of text and media. - // Other messages are emitted in plain `{"role","content"}` form so the chat - // template renders them the same way it does for text-only requests. +std::string OnnxChatGenerator::TransformMessagesForMedia(const std::vector& messages, + bool preserve_reasoning_history) { size_t last_user_idx = messages.size(); for (size_t i = messages.size(); i-- > 0;) { if (messages[i].role == FOUNDRY_LOCAL_ROLE_USER) { @@ -330,8 +327,15 @@ std::string OnnxChatGenerator::TransformMessagesForMedia(const std::vectortype == FOUNDRY_LOCAL_ITEM_IMAGE || + part.view->type == FOUNDRY_LOCAL_ITEM_AUDIO); + }); + if (has_media && msg.role != FOUNDRY_LOCAL_ROLE_USER) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "image or audio input must belong to a user message"); + } - if (i == last_user_idx) { + if (i == last_user_idx || has_media) { auto content = nlohmann::json::array(); for (const auto& part : msg.content) { if (!part.view) { @@ -351,15 +355,24 @@ std::string OnnxChatGenerator::TransformMessagesForMedia(const std::vectortype == FOUNDRY_LOCAL_ITEM_IMAGE || part.view->type == FOUNDRY_LOCAL_ITEM_AUDIO)) { - FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, - "media input must belong to the final user message"); + if (part.view && part.view->type == FOUNDRY_LOCAL_ITEM_TEXT) { + const auto& text = static_cast(*part.view); + if (text.text_type == FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING) { + reasoning += text.text; + } } } - // Non-final message: render visible text parts only via the canonical helper. - entry["content"] = RenderMessageForPrompt(msg); + + entry["reasoning_content"] = std::move(reasoning); + } + if (!msg.name.empty()) { + entry["name"] = msg.name; } arr.push_back(std::move(entry)); diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h index 933afd195..893cb3be0 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.h @@ -140,14 +140,9 @@ class OnnxChatGenerator : public ChatGenerator { // ---- Static helpers exposed for unit testing ---- - /// Build the JSON messages array fed to OgaTokenizer::ApplyChatTemplate when - /// the request includes one or more images. The last user message's content - /// is rewritten to the structured form - /// `[{"type":"image"},{"type":"text","text":"..."}]` so that the model's - /// chat template inserts the appropriate vision sentinel tokens. Other - /// messages are emitted in their plain `{"role","content"}` form. - /// - static std::string TransformMessagesForMedia(const std::vector& messages); + /// Project all media-bearing user messages and the final user message into structured template content. + static std::string TransformMessagesForMedia(const std::vector& messages, + bool preserve_reasoning_history = false); private: OnnxChatGenerator(std::unique_ptr gen_params, diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.cc b/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.cc index a4d6aa41a..055aea911 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/prepared_chat_prompt.cc @@ -115,7 +115,8 @@ PreparedChatPrompt PrepareMediaChatPrompt(const std::vector& messag } PreparedChatPrompt prepared; - const auto messages_json = OnnxChatGenerator::TransformMessagesForMedia(messages); + const auto messages_json = + OnnxChatGenerator::TransformMessagesForMedia(messages, tool_ctx.supports_reasoning_history); const char* tools = tool_ctx.tools_json.empty() ? nullptr : tool_ctx.tools_json.c_str(); const char* kwargs = tool_ctx.template_kwargs_json.empty() ? nullptr : tool_ctx.template_kwargs_json.c_str(); prepared.prompt = diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_chain.h b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_chain.h index 2859c7cb4..a2b9ba39e 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_chain.h +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_chain.h @@ -4,6 +4,7 @@ #include +#include #include #include #include @@ -30,6 +31,8 @@ struct ResponseChainHop { raw_envelope_call_ids(std::move(raw_call_ids)) {} nlohmann::json input_items = nlohmann::json::array(); + /// Private input override when a local image was snapshotted for replay. + std::shared_ptr replay_input_items; nlohmann::json output_items = nlohmann::json::array(); std::unordered_set raw_envelope_call_ids; }; diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc index 33d9660b7..b3f85c1a8 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.cc @@ -170,8 +170,7 @@ namespace { /// What a stored message contributes to replay. struct StoredMessageContent { std::string text; - /// The message carried image or audio parts. Their bytes are not stored and cannot be replayed, but the turn - /// itself still happened and must not vanish from the rebuilt conversation. + /// Media messages use the typed input converter so their parts and byte buffers are replayed in order. bool has_media = false; }; @@ -324,6 +323,8 @@ flMessageRole ReplayRole(const std::string& role) { } // namespace +static void AddTypedInputMessage(Request& request, const InputMessage& msg); + /// Replay a hop's own input items — the request the caller sent for that hop, in the shape they sent it. static void AddJsonItemsToRequest(Request& request, const nlohmann::json& items) { if (!items.is_array()) { @@ -378,17 +379,9 @@ static void AddJsonItemsToRequest(Request& request, const nlohmann::json& items) auto r = ReplayRole(role); auto content = ReadStoredContent(entry); - if (content.text.empty() && content.has_media) { - // A media-only message. The placeholder is not an invention standing in for the image: it is exactly what the - // live path recorded for this turn. A media-only message has no text of its own, so AddTypedInputItems gives - // it this single space to satisfy the chat template, and that space — not the bytes — is what reached the - // transcript. Replaying it therefore reproduces the live record verbatim. - // - // The bytes are absent on both sides: a media turn drops its generator, so the very next turn of the live - // session also rebuilds its prompt from a transcript that never held them. Dropping the message instead would - // silently remove a user turn the model saw; failing here would make the cold path reject a continuation the - // warm path serves. - content.text = kMediaOnlyPlaceholder; + if (content.has_media) { + AddTypedInputMessage(request, entry.get()); + continue; } if (content.text.empty()) { @@ -464,7 +457,7 @@ static void AddHopOutputToRequest(Request& request, const nlohmann::json& output static void AddChainContextToRequest(Request& request, const ResponseChainContext& context) { for (const auto& hop : context) { request.BeginItemSegment(); - AddJsonItemsToRequest(request, hop.input_items); + AddJsonItemsToRequest(request, hop.replay_input_items ? *hop.replay_input_items : hop.input_items); request.BeginItemSegment(); AddHopOutputToRequest(request, hop.output_items, hop.raw_envelope_call_ids); } @@ -605,6 +598,40 @@ std::unique_ptr MakeAudioItemFromInputAudio(const InputAudioContent& // Helper: add typed InputItems to a session request // --------------------------------------------------------------------------- +static void AddTypedInputMessage(Request& request, const InputMessage& msg) { + std::vector> parts; + bool has_text = false; + for (const auto& c : msg.content) { + if (auto* tc = std::get_if(&c)) { + if (!tc->text.empty()) { + parts.push_back(std::make_unique(tc->text)); + has_text = true; + } + } else if (auto* ic = std::get_if(&c)) { + parts.push_back(MakeImageItemFromInputImage(*ic)); + } else if (auto* ac = std::get_if(&c)) { + parts.push_back(MakeAudioItemFromInputAudio(*ac)); + } else { + FL_THROW(FOUNDRY_LOCAL_ERROR_NOT_IMPLEMENTED, + "input message content type not supported (only input_text, input_image, and input_audio)"); + } + } + + if (parts.empty()) { + if (ReplayRole(msg.role) == FOUNDRY_LOCAL_ROLE_ASSISTANT) { + request.AddOwnedItem(MakeAssistantTurnBoundary()); + } + + return; + } + + if (!has_text) { + parts.push_back(std::make_unique(kMediaOnlyPlaceholder)); + } + + request.AddOwnedItem(std::make_unique(ReplayRole(msg.role), std::move(parts))); +} + static void AddTypedInputItems(Request& request, const std::vector& input_items) { for (const auto& input_item : input_items) { @@ -631,47 +658,7 @@ static void AddTypedInputItems(Request& request, ToolKind::kCustom); request.AddOwnedItem(std::move(i)); } else if (auto* msg = std::get_if(&input_item)) { - // Build typed parts from the message's content array. - std::vector> parts; - bool has_text = false; - for (const auto& c : msg->content) { - if (auto* tc = std::get_if(&c)) { - if (!tc->text.empty()) { - parts.push_back(std::make_unique(tc->text)); - has_text = true; - } - } else if (auto* ic = std::get_if(&c)) { - parts.push_back(MakeImageItemFromInputImage(*ic)); - } else if (auto* ac = std::get_if(&c)) { - parts.push_back(MakeAudioItemFromInputAudio(*ac)); - } else { - FL_THROW(FOUNDRY_LOCAL_ERROR_NOT_IMPLEMENTED, - "input message content type not supported (only input_text, input_image, and input_audio)"); - } - } - - // A message with no replayable content is normally nothing to say. An assistant message is the exception: it - // is the boundary of a turn whose output did not survive replay (a reasoning-only turn, or one truncated - // before any visible text), and the live session commits that boundary. - if (parts.empty()) { - if (ReplayRole(msg->role) == FOUNDRY_LOCAL_ROLE_ASSISTANT) { - request.AddOwnedItem(MakeAssistantTurnBoundary()); - } - - continue; - } - - // The chat template requires every message to carry at least one - // text part. A pure-image message (e.g. "input_image" with no - // accompanying "input_text") would render as an empty content - // string. Inject the placeholder so the template still renders the - // message and the model receives the image sentinel. - if (!has_text) { - parts.push_back(std::make_unique(kMediaOnlyPlaceholder)); - } - - auto i = std::make_unique(ReplayRole(msg->role), std::move(parts)); - request.AddOwnedItem(std::move(i)); + AddTypedInputMessage(request, *msg); } } } @@ -1210,5 +1197,91 @@ nlohmann::json ToInputItems(const nlohmann::json& req_json) { return items; } +std::shared_ptr SnapshotReplayInputItems(const nlohmann::json& input_items, + const Request& request) { + const auto visit_images = [](auto& items, auto&& visit) { + if (!items.is_array()) { + return; + } + + for (auto& item : items) { + if (!item.is_object()) { + continue; + } + const auto type = item.value("type", ""); + if (type == "function_call" || type == "function_call_output" || type == "custom_tool_call" || + type == "custom_tool_call_output" || type == "reasoning" || item.value("role", "").empty()) { + continue; + } + const auto content = item.find("content"); + if (content == item.end() || !content->is_array()) { + continue; + } + for (auto& part : *content) { + if (part.is_object() && part.value("type", "") == "input_image") { + visit(part); + } + } + } + }; + const auto is_local_image = [](const nlohmann::json& part) { + const auto url = part.find("image_url"); + if (url == part.end() || !url->is_string()) { + return false; + } + const auto& value = url->get_ref(); + return !value.empty() && !value.starts_with(kDataUrlPrefix); + }; + + bool needs_snapshot = false; + visit_images(input_items, [&](const auto& part) { needs_snapshot |= is_local_image(part); }); + if (!needs_snapshot) { + return {}; + } + + const size_t current_start = request.item_segment_starts.empty() ? 0 : request.item_segment_starts.back(); + if (current_start > request.items.size()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "cannot match stored image input to the current request segment"); + } + + std::vector images; + for (size_t i = current_start; i < request.items.size(); ++i) { + const auto* item = request.items[i]; + if (!item || item->type != FOUNDRY_LOCAL_ITEM_MESSAGE) { + continue; + } + for (const auto& part : static_cast(*item).content) { + if (part.view && part.view->type == FOUNDRY_LOCAL_ITEM_IMAGE) { + images.push_back(static_cast(part.view)); + } + } + } + + auto replay = std::make_shared(input_items); + size_t image_index = 0; + visit_images(*replay, [&](nlohmann::json& part) { + if (image_index >= images.size()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "cannot match stored image input to resolved request images"); + } + const auto& image = *images[image_index++]; + if (!is_local_image(part)) { + return; + } + if (!image.data || image.data_size == 0) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "local image input was not resolved to owned bytes"); + } + + auto mime = image.format; + if (mime.empty() || mime.find_first_of(";,") != std::string::npos) { + mime = "application/octet-stream"; + } + part["image_url"] = "data:" + mime + ";base64," + Azure::Core::Convert::Base64Encode(image.ReadBytes()); + }); + if (image_index != images.size()) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "cannot match stored image input to resolved request images"); + } + return replay; +} + } // namespace ResponseConverter } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.h b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.h index 56ff0429c..d48a7bd77 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.h +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_converter.h @@ -9,6 +9,7 @@ #include +#include #include #include #include @@ -116,5 +117,10 @@ ToolCallStreamOutput BuildToolCallStreamOutput(const ToolCallItem& call, /// reports what the caller sent, and a replayed chain takes its instructions from the current request. nlohmann::json ToInputItems(const nlohmann::json& req_json); +/// Snapshot local image references using the already-resolved bytes in this request's own input segment. The public +/// input items remain unchanged. Returns null when no local image needs a private replay representation. +std::shared_ptr SnapshotReplayInputItems(const nlohmann::json& input_items, + const Request& request); + } // namespace ResponseConverter } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.cc b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.cc index cabd988de..299135f6d 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.cc +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.cc @@ -117,6 +117,7 @@ void ResponseStore::StoreLocked(StoredResponse response, MetadataState& state) { .response = std::move(response.response), .input_items = std::move(response.input_items), .raw_envelope_call_ids = std::move(response.raw_envelope_call_ids), + .replay_input_items = std::move(response.replay_input_items), .replay_prefix = std::move(replay_prefix), .insertion_sequence = state.next_insertion_sequence++}); state.index[response.id] = state.entries.begin(); @@ -592,6 +593,7 @@ ResponseChainHop ResponseStore::ToReplayHop(const Entry& entry) { if (entry.input_items.is_array()) { replay.input_items = entry.input_items; } + replay.replay_input_items = entry.replay_input_items; const auto output = entry.response.find("output"); if (output != entry.response.end() && output->is_array()) { diff --git a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.h b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.h index 6d90ca1ca..73b68d0fb 100644 --- a/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.h +++ b/sdk_v2/cpp/src/inferencing/generative/openresponses/response_store.h @@ -120,12 +120,14 @@ class ResponseStore { struct StoredResponse { StoredResponse(std::string id_in, std::string model_id_in, nlohmann::json response_in, nlohmann::json input_items_in, - std::unordered_set raw_call_ids_in = {}) + std::unordered_set raw_call_ids_in = {}, + std::shared_ptr replay_input_items_in = {}) : id(std::move(id_in)), model_id(std::move(model_id_in)), response(std::move(response_in)), input_items(std::move(input_items_in)), - raw_envelope_call_ids(std::move(raw_call_ids_in)) {} + raw_envelope_call_ids(std::move(raw_call_ids_in)), + replay_input_items(std::move(replay_input_items_in)) {} std::string id; /// Resolved model that produced it. A continuation must resolve to the same model; empty means unbound, which @@ -136,6 +138,7 @@ class ResponseStore { /// This hop's own request items — exactly what the /input_items endpoint returns. nlohmann::json input_items; std::unordered_set raw_envelope_call_ids; + std::shared_ptr replay_input_items; }; /// Outcome of ResponseStore::BeginResponse. @@ -269,6 +272,7 @@ class ResponseStore { nlohmann::json response; nlohmann::json input_items; std::unordered_set raw_envelope_call_ids; + std::shared_ptr replay_input_items; std::shared_ptr replay_prefix; uint64_t insertion_sequence = 0; }; diff --git a/sdk_v2/cpp/src/inferencing/session/request.cc b/sdk_v2/cpp/src/inferencing/session/request.cc index 4686ce63a..16d2d03e9 100644 --- a/sdk_v2/cpp/src/inferencing/session/request.cc +++ b/sdk_v2/cpp/src/inferencing/session/request.cc @@ -22,8 +22,20 @@ std::unique_ptr CloneChatItem(const Item& item) { const auto& text = static_cast(item); return std::make_unique(text.text, text.text_type); } - case FOUNDRY_LOCAL_ITEM_MESSAGE: - return std::make_unique(static_cast(item)); + case FOUNDRY_LOCAL_ITEM_MESSAGE: { + const auto& message = static_cast(item); + auto clone = std::make_unique(); + clone->role = message.role; + clone->name = message.name; + clone->content.reserve(message.content.size()); + for (const auto& part : message.content) { + if (part.view) { + clone->content.push_back(MessagePart::Own(CloneChatItem(*part.view))); + } + } + + return clone; + } case FOUNDRY_LOCAL_ITEM_TOOL_CALL: { const auto& call = static_cast(item); auto clone = std::make_unique(call.call_id, call.name, call.arguments, call.replayed_from_store, @@ -38,14 +50,9 @@ std::unique_ptr CloneChatItem(const Item& item) { } case FOUNDRY_LOCAL_ITEM_IMAGE: { const auto& image = static_cast(item); - if (image.data != nullptr && image.data_size > 0) { - const auto* begin = static_cast(image.data); - auto clone = std::make_unique(std::vector(begin, begin + image.data_size), image.format); - clone->uri = image.uri; - return clone; - } - - return std::make_unique(image.uri, image.format); + auto clone = std::make_unique(image.ReadBytes(), image.format); + clone->uri = image.uri; + return clone; } case FOUNDRY_LOCAL_ITEM_AUDIO: { const auto& audio = static_cast(item); diff --git a/sdk_v2/cpp/src/inferencing/session/request.h b/sdk_v2/cpp/src/inferencing/session/request.h index 4bf0d944a..881539af2 100644 --- a/sdk_v2/cpp/src/inferencing/session/request.h +++ b/sdk_v2/cpp/src/inferencing/session/request.h @@ -102,7 +102,7 @@ struct Request { Request(const Request&) = delete; Request& operator=(const Request&) = delete; - /// Deep-copy every field read during chat preparation, including inline media bytes. + /// Deep-copy every field read during chat preparation, including media bytes resolved from local files. Request CaptureChatSnapshot() const; /// Add a pre-allocated owned item. diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index fcc652bea..3491b08a5 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -62,6 +62,7 @@ struct PublishRequest { nlohmann::json response; nlohmann::json input_items; std::unordered_set raw_envelope_call_ids; + std::shared_ptr replay_input_items; }; /// Publish a completed response: store its metadata and cache its session as one indivisible step. @@ -79,7 +80,8 @@ void PublishResponse(PublishRequest request) { const bool published = request.store.Commit( request.lease, ResponseStore::StoredResponse{request.response_id, request.model_id, std::move(request.response), - std::move(request.input_items), std::move(request.raw_envelope_call_ids)}, + std::move(request.input_items), std::move(request.raw_envelope_call_ids), + std::move(request.replay_input_items)}, &admission); if (!published) { @@ -440,6 +442,8 @@ std::shared_ptr ResponsesHandler::HandleNo nlohmann::json response_json = response; if (params.store) { + auto input_items = ResponseConverter::ToInputItems(req_json); + auto replay_input_items = ResponseConverter::SnapshotReplayInputItems(input_items, session_request); PublishResponse(PublishRequest{.store = ctx_.response_store, .session_manager = ctx_.session_manager, .logger = ctx_.logger, @@ -449,8 +453,9 @@ std::shared_ptr ResponsesHandler::HandleNo .response_id = turn.response_id, .model_id = turn.model_id, .response = response_json, - .input_items = ResponseConverter::ToInputItems(req_json), - .raw_envelope_call_ids = std::move(raw_envelope_call_ids)}); + .input_items = std::move(input_items), + .raw_envelope_call_ids = std::move(raw_envelope_call_ids), + .replay_input_items = std::move(replay_input_items)}); } return JsonResponse(Status::CODE_200, response_json); @@ -769,6 +774,8 @@ std::shared_ptr ResponsesHandler::HandleSt // PublishResponse throws and the stream ends with response.failed instead of claiming an unstored completion. if (should_store && !stream->RunIfConnected([&] { nlohmann::json response_json = completed_response; + auto input_items = ResponseConverter::ToInputItems(req_copy); + auto replay_input_items = ResponseConverter::SnapshotReplayInputItems(input_items, *req); PublishResponse(PublishRequest{.store = store, .session_manager = session_manager, .logger = logger, @@ -778,8 +785,9 @@ std::shared_ptr ResponsesHandler::HandleSt .response_id = turn.response_id, .model_id = turn.model_id, .response = std::move(response_json), - .input_items = ResponseConverter::ToInputItems(req_copy), - .raw_envelope_call_ids = std::move(raw_envelope_call_ids)}); + .input_items = std::move(input_items), + .raw_envelope_call_ids = std::move(raw_envelope_call_ids), + .replay_input_items = std::move(replay_input_items)}); })) { stream->Finish(); return; diff --git a/sdk_v2/cpp/test/internal_api/chat/chat_transcript_test.cc b/sdk_v2/cpp/test/internal_api/chat/chat_transcript_test.cc index ecc6f3f89..96a36b33a 100644 --- a/sdk_v2/cpp/test/internal_api/chat/chat_transcript_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/chat_transcript_test.cc @@ -15,10 +15,13 @@ #include "items/text_item.h" #include "items/tool_call_item.h" #include "items/tool_result_item.h" +#include "utils/temp_path.h" #include #include +#include +#include #include #include #include @@ -1006,11 +1009,11 @@ TEST(TranscriptIngestTest, ATopLevelTextItemIsRejectedRatherThanDropped) { } } -TEST(TranscriptIngestTest, MediaPartsInsideAMessageAreStillSkipped) { - // The message itself is ingested for its text; the media bytes travel to the generator separately. +TEST(TranscriptIngestTest, ImagePartsRemainInTheTranscriptBetweenTextParts) { std::vector> parts; - parts.push_back(std::make_unique("What is this?")); + parts.push_back(std::make_unique("What is ")); parts.push_back(std::make_unique(std::vector{1, 2, 3}, "image/png")); + parts.push_back(std::make_unique("this?")); Request request; request.AddOwnedItem(std::make_unique(FOUNDRY_LOCAL_ROLE_USER, std::move(parts))); @@ -1019,6 +1022,46 @@ TEST(TranscriptIngestTest, MediaPartsInsideAMessageAreStillSkipped) { ASSERT_EQ(messages.size(), 1u); EXPECT_EQ(messages[0].VisibleText(), "What is this?"); + ASSERT_EQ(messages[0].entries.size(), 3u); + EXPECT_EQ(messages[0].entries[0].text, "What is "); + EXPECT_EQ(messages[0].entries[2].text, "this?"); +} + +TEST(TranscriptIngestTest, FileSnapshotRemainsOwnedThroughCommitCopyAndUndo) { + const auto dir = fl::test::TempPath::CreateTempDir("fl_chat_snapshot_"); + const auto image_path = dir.path() / "image.png"; + { + std::ofstream file(image_path, std::ios::binary); + file.write("\x01\x02\x03", 3); + ASSERT_TRUE(file.good()); + } + + Request request; + std::vector> parts; + parts.push_back(std::make_unique(image_path.string(), "image/png")); + request.AddOwnedItem(std::make_unique(FOUNDRY_LOCAL_ROLE_USER, std::move(parts))); + auto snapshot = request.CaptureChatSnapshot(); + ASSERT_TRUE(std::filesystem::remove(image_path)); + + auto inputs = BuildTranscriptMessages(snapshot.items); + ASSERT_EQ(inputs.size(), 1u); + ASSERT_EQ(inputs[0].entries.size(), 1u); + const auto* image = static_cast(inputs[0].entries[0].media.get()); + ASSERT_NE(image, nullptr); + EXPECT_EQ(image->ReadBytes(), (std::vector{1, 2, 3})); + + ChatTranscript transcript; + transcript.CommitTurn(std::move(inputs), MakeAssistant("first", {}), {}); + auto copy = transcript; + CommitTextTurn(transcript, "follow-up", "second"); + transcript.UndoTurns(1); + ASSERT_EQ(transcript.Messages().size(), 2u); + EXPECT_EQ(transcript.Messages()[0].entries[0].media.get(), copy.Messages()[0].entries[0].media.get()); + EXPECT_EQ(static_cast(*copy.Messages()[0].entries[0].media).ReadBytes(), + (std::vector{1, 2, 3})); + transcript.UndoTurns(1); + EXPECT_TRUE(transcript.Empty()); + EXPECT_TRUE(copy.Messages()[0].HasMedia()); } TEST(TranscriptIngestTest, CarriesRespondableContentDistinguishesABoundaryFromRealInput) { diff --git a/sdk_v2/cpp/test/internal_api/chat/media_input_test.cc b/sdk_v2/cpp/test/internal_api/chat/media_input_test.cc index 3574981f1..247e34e4e 100644 --- a/sdk_v2/cpp/test/internal_api/chat/media_input_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/media_input_test.cc @@ -1,11 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -// Tests for media collection and the conversation-scoped media policy. -// -// The media prompt path (OnnxChatGenerator::CreateWithMedia) renders MediaInput::messages and nothing else, and the -// image / audio bytes never enter the transcript. These tests pin the one rule that follows: media is allowed only -// while a conversation has no history and declares no tools, and the rule does not depend on whether the session -// happened to still be cached. +// Tests for media collection, conversation history, and the media policy. #include "inferencing/generative/chat/media_input.h" @@ -19,11 +14,14 @@ #include "items/text_item.h" #include "items/tool_call_item.h" #include "items/tool_result_item.h" +#include "utils/temp_path.h" #include #include #include +#include +#include #include #include #include @@ -102,34 +100,40 @@ TEST(MediaInputTest, ToolActivityWithoutMediaIsAccepted) { request.AddOwnedItem(std::make_unique("call_1", "get_weather", R"({"city":"Seattle"})")); request.AddOwnedItem(std::make_unique("call_1", "sunny")); - EXPECT_FALSE(RejectionFor(request, {.session_has_history = true, .tools_declared = true}).has_value()); + EXPECT_FALSE(RejectionFor(request, {.tools_declared = true}).has_value()); } // --------------------------------------------------------------------------- -// Prior history — rejected identically whether it is live or replayed. +// Image history is accepted identically whether it is live or replayed. // --------------------------------------------------------------------------- -TEST(MediaInputTest, MediaOnAWarmContinuationIsRejected) { - // The session still holds the conversation, so the request carries only the new media turn. +TEST(MediaInputTest, ImageOnAWarmContinuationIsAccepted) { Request request; request.AddOwnedItem(ImageMessage("Does this photo match?")); - EXPECT_EQ(RejectionFor(request, {.session_has_history = true, .tools_declared = false}), - FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + std::vector conversation{ + {FOUNDRY_LOCAL_ROLE_USER, "Weather in Seattle?"}, + {FOUNDRY_LOCAL_ROLE_ASSISTANT, "It is sunny."}, + }; + auto current = BuildTranscriptMessages(request.items); + conversation.insert(conversation.end(), current.begin(), current.end()); + auto media = CollectMediaInput(conversation); + + ASSERT_EQ(media.messages.size(), 3u); + EXPECT_NO_THROW(ValidateMediaTurn(media, conversation, {})); } -TEST(MediaInputTest, MediaOnAColdContinuationIsRejectedTheSameWay) { +TEST(MediaInputTest, ImageOnAColdContinuationIsAccepted) { // Same conversation, but the session cache dropped it, so the history arrives replayed in the request items. Request request; request.AddOwnedItem(std::make_unique(FOUNDRY_LOCAL_ROLE_USER, "Weather in Seattle?")); request.AddOwnedItem(std::make_unique(FOUNDRY_LOCAL_ROLE_ASSISTANT, "It is sunny.")); request.AddOwnedItem(ImageMessage("Does this photo match?")); - EXPECT_EQ(RejectionFor(request, {.session_has_history = false, .tools_declared = false}), - FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + EXPECT_FALSE(RejectionFor(request, {}).has_value()); } -TEST(MediaInputTest, WarmAndColdMediaContinuationsRejectIdentically) { +TEST(MediaInputTest, WarmAndColdImageContinuationsAgree) { Request warm; warm.AddOwnedItem(ImageMessage("Does this photo match?")); @@ -138,8 +142,7 @@ TEST(MediaInputTest, WarmAndColdMediaContinuationsRejectIdentically) { cold.AddOwnedItem(std::make_unique(FOUNDRY_LOCAL_ROLE_ASSISTANT, "It is sunny.")); cold.AddOwnedItem(ImageMessage("Does this photo match?")); - EXPECT_EQ(RejectionFor(warm, {.session_has_history = true, .tools_declared = false}), - RejectionFor(cold, {.session_has_history = false, .tools_declared = false})); + EXPECT_EQ(RejectionFor(warm, {}), RejectionFor(cold, {})); } TEST(MediaInputTest, ReplayedToolExchangeWithANewImageIsRejectedNotDropped) { @@ -165,7 +168,7 @@ TEST(MediaInputTest, ReplayedToolExchangeWithANewImageIsRejectedNotDropped) { FAIL() << "expected the media + replayed tool exchange to be rejected"; } catch (const fl::Exception& ex) { EXPECT_EQ(ex.code(), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); - EXPECT_NE(std::string(ex.what()).find("only allowed on the first turn"), std::string::npos) << ex.what(); + EXPECT_NE(std::string(ex.what()).find("tool"), std::string::npos) << ex.what(); } } @@ -179,9 +182,7 @@ TEST(MediaInputTest, ToolResultOnlyWithANewAudioClipIsRejected) { EXPECT_EQ(RejectionFor(request, {}), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); } -TEST(MediaInputTest, ReplayedAssistantBoundaryAloneStillCountsAsHistory) { - // A hop whose output was reasoning-only replays as an assistant message with no entries. It is still a prior turn, - // so a media continuation behind it must be rejected exactly as any other continuation is. +TEST(MediaInputTest, ImageCanFollowAReplayedAssistantBoundary) { std::vector> boundary_parts; boundary_parts.push_back(std::make_unique(std::string{})); @@ -190,11 +191,11 @@ TEST(MediaInputTest, ReplayedAssistantBoundaryAloneStillCountsAsHistory) { request.AddOwnedItem(std::make_unique(FOUNDRY_LOCAL_ROLE_ASSISTANT, std::move(boundary_parts))); request.AddOwnedItem(ImageMessage("Now look at this.")); - EXPECT_EQ(RejectionFor(request, {}), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + EXPECT_FALSE(RejectionFor(request, {}).has_value()); } // --------------------------------------------------------------------------- -// Declared tools — a generated call on a media turn could never be answered. +// The multimodal template cannot represent tool calls or results. // --------------------------------------------------------------------------- TEST(MediaInputTest, FirstTurnMediaWithDeclaredToolsIsRejected) { @@ -204,7 +205,7 @@ TEST(MediaInputTest, FirstTurnMediaWithDeclaredToolsIsRejected) { try { auto media = CollectMediaInput(request); auto inputs = BuildTranscriptMessages(request.items); - ValidateMediaTurn(media, inputs, {.session_has_history = false, .tools_declared = true}); + ValidateMediaTurn(media, inputs, {.tools_declared = true}); FAIL() << "expected first-turn media with declared tools to be rejected"; } catch (const fl::Exception& ex) { EXPECT_EQ(ex.code(), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); @@ -216,15 +217,14 @@ TEST(MediaInputTest, AudioWithDeclaredToolsIsRejected) { Request request; request.AddOwnedItem(AudioMessage("Transcribe this.")); - EXPECT_EQ(RejectionFor(request, {.session_has_history = false, .tools_declared = true}), - FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + EXPECT_EQ(RejectionFor(request, {.tools_declared = true}), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); } TEST(MediaInputTest, TextTurnWithDeclaredToolsIsUnaffected) { Request request; request.AddOwnedItem(std::make_unique(FOUNDRY_LOCAL_ROLE_USER, "Weather in Seattle?")); - EXPECT_FALSE(RejectionFor(request, {.session_has_history = false, .tools_declared = true}).has_value()); + EXPECT_FALSE(RejectionFor(request, {.tools_declared = true}).has_value()); } // --------------------------------------------------------------------------- @@ -253,7 +253,7 @@ TEST(MediaInputTest, ResponsesRequestMixingAnImageWithAToolExchangeIsRejected) { EXPECT_EQ(RejectionFor(request, {}), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); } -TEST(MediaInputTest, ResponsesChainContinuationWithAnImageIsRejected) { +TEST(MediaInputTest, ResponsesChainContinuationWithAnImageIsAccepted) { // The full cold path: a stored conversation is reconstructed and the new turn carries an image. ResponseStore store; @@ -280,12 +280,10 @@ TEST(MediaInputTest, ResponsesChainContinuationWithAnImageIsRejected) { auto params = body.get(); auto request = ResponseConverter::ToSessionRequest(params, &(*context)); - EXPECT_EQ(RejectionFor(request, {}), FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT); + EXPECT_FALSE(RejectionFor(request, {}).has_value()); } -TEST(MediaInputTest, StoredMediaOnlyUserTurnIsReplayedNotSilentlyDropped) { - // A first turn that was image-only is stored with no text. Replay cannot reproduce the bytes, but the turn must - // still appear in the rebuilt conversation — dropping it would remove a user turn the model saw. +TEST(MediaInputTest, StoredImageOnlyUserTurnReplaysTheOriginalImage) { ResponseStore store; json response; @@ -306,23 +304,23 @@ TEST(MediaInputTest, StoredMediaOnlyUserTurnIsReplayedNotSilentlyDropped) { params.input = std::string("What colour was it?"); auto request = ResponseConverter::ToSessionRequest(params, &(*context)); + auto media = CollectMediaInput(request); auto messages = BuildTranscriptMessages(request.items); + ASSERT_EQ(media.images.size(), 1u); + EXPECT_EQ(media.images[0]->ReadBytes(), (std::vector{0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a})); ASSERT_EQ(messages.size(), 3u); EXPECT_EQ(messages[0].role, FOUNDRY_LOCAL_ROLE_USER); EXPECT_EQ(messages[0].VisibleText(), " ") << "the media-only user turn must survive replay"; EXPECT_EQ(messages[1].VisibleText(), "A cat."); EXPECT_EQ(messages[2].VisibleText(), "What colour was it?"); - // The continuation itself carries no media, so it is a normal text turn and is accepted. EXPECT_FALSE(RejectionFor(request, {}).has_value()); - // Warm parity, so the placeholder is not a silent degradation unique to replay: a live session ingests the same - // image-only message into exactly the same transcript message. The bytes go straight to the generator and never - // enter the transcript, so the very next turn of that live session rebuilds its prompt without them too. + // An image-only live message and the replayed message have the same visible text and media-bearing shape. std::vector> live_parts; - live_parts.push_back(std::make_unique(" ")); live_parts.push_back(std::make_unique(std::vector{1, 2, 3}, "image/png")); + live_parts.push_back(std::make_unique(" ")); Request live; live.AddOwnedItem(std::make_unique(FOUNDRY_LOCAL_ROLE_USER, std::move(live_parts))); @@ -331,4 +329,142 @@ TEST(MediaInputTest, StoredMediaOnlyUserTurnIsReplayedNotSilentlyDropped) { ASSERT_EQ(live_messages.size(), 1u); EXPECT_EQ(live_messages[0].role, messages[0].role); EXPECT_EQ(live_messages[0].VisibleText(), messages[0].VisibleText()); + ASSERT_EQ(live_messages[0].entries.size(), messages[0].entries.size()); + for (size_t i = 0; i < live_messages[0].entries.size(); ++i) { + EXPECT_EQ(live_messages[0].entries[i].kind, messages[0].entries[i].kind); + } +} + +TEST(MediaInputTest, StoredMixedMediaReplaysInTheSameOrderAsTypedInput) { + const json body = { + {"model", "test-model"}, + {"input", json::array({{{"role", "user"}, + {"content", json::array({{{"type", "input_text"}, {"text", "before"}}, + {{"type", "input_image"}, + {"image_url", "data:image/png;base64,AQI="}}, + {{"type", "input_audio"}, {"format", "wav"}, {"data", "AwQ="}}, + {{"type", "input_text"}, {"text", "after"}}})}}})}}; + auto warm = ResponseConverter::ToSessionRequest(body.get()); + ResponseChainContext context; + context.emplace_back(ResponseConverter::ToInputItems(body), json::array()); + + ResponseCreateParams next; + next.model = "test-model"; + next.input = std::string("continue"); + auto cold = ResponseConverter::ToSessionRequest(next, &context); + const auto warm_messages = BuildTranscriptMessages(warm.items); + const auto cold_messages = BuildTranscriptMessages(cold.items); + + ASSERT_FALSE(warm_messages.empty()); + ASSERT_FALSE(cold_messages.empty()); + ASSERT_EQ(warm_messages[0].entries.size(), 4u); + ASSERT_EQ(cold_messages[0].entries.size(), 4u); + for (size_t i = 0; i < warm_messages[0].entries.size(); ++i) { + EXPECT_EQ(warm_messages[0].entries[i].kind, cold_messages[0].entries[i].kind); + EXPECT_EQ(warm_messages[0].entries[i].text, cold_messages[0].entries[i].text); + } + + auto media = CollectMediaInput(cold); + ASSERT_EQ(media.images.size(), 1u); + ASSERT_EQ(media.audios.size(), 1u); + EXPECT_EQ(media.images[0]->ReadBytes(), (std::vector{1, 2})); + const auto* audio = static_cast(media.audios[0]->data); + EXPECT_EQ(std::vector(audio, audio + media.audios[0]->data_size), + (std::vector{3, 4})); +} + +TEST(MediaInputTest, LocalImageReplaySnapshotsCurrentSegmentWithoutChangingPublicInput) { + const auto dir = fl::test::TempPath::CreateTempDir("fl_media_replay_"); + const auto image_path = dir.path() / "image.png"; + { + std::ofstream file(image_path, std::ios::binary); + file.write("\x01\x02\x03", 3); + ASSERT_TRUE(file.good()); + } + + ResponseChainContext prior; + prior.emplace_back( + json::array({{{"role", "user"}, + {"content", json::array({{{"type", "input_image"}, + {"image_url", "data:image/png;base64,CA=="}}})}}}), + json::array()); + const json body = { + {"model", "test-model"}, + {"input", json::array({{{"role", "user"}, + {"content", json::array({{{"type", "input_image"}, + {"image_url", "data:image/png;base64,CQ=="}}, + {{"type", "input_text"}, {"text", "compare"}}, + {{"type", "input_image"}, + {"image_url", image_path.string()}}})}}})}}; + const auto public_items = ResponseConverter::ToInputItems(body); + auto request = ResponseConverter::ToSessionRequest(body.get(), &prior); + ASSERT_TRUE(std::filesystem::remove(image_path)); + + const auto snapshot = ResponseConverter::SnapshotReplayInputItems(public_items, request); + ASSERT_TRUE(snapshot); + EXPECT_EQ(public_items[0]["content"][2]["image_url"], image_path.string()); + EXPECT_EQ((*snapshot)[0]["content"][0]["image_url"], "data:image/png;base64,CQ=="); + EXPECT_EQ((*snapshot)[0]["content"][2]["image_url"], "data:image/png;base64,AQID"); + + ResponseChainContext chain; + chain.emplace_back(public_items, json::array()); + chain[0].replay_input_items = snapshot; + ResponseCreateParams next; + next.model = "test-model"; + next.input = std::string("continue"); + auto replay = ResponseConverter::ToSessionRequest(next, &chain); + auto media = CollectMediaInput(replay); + ASSERT_EQ(media.images.size(), 2u); + EXPECT_EQ(media.images[0]->ReadBytes(), (std::vector{9})); + EXPECT_EQ(media.images[1]->ReadBytes(), (std::vector{1, 2, 3})); +} + +TEST(MediaInputTest, InlineImagesNeedNoPrivateReplayCopy) { + const json body = {{"model", "test-model"}, + {"input", json::array({{{"role", "user"}, + {"content", json::array({{{"type", "input_image"}, + {"image_url", "data:image/png;base64,AQI="}}})}}})}}; + auto request = ResponseConverter::ToSessionRequest(body.get()); + + EXPECT_FALSE(ResponseConverter::SnapshotReplayInputItems(ResponseConverter::ToInputItems(body), request)); +} + +TEST(MediaInputTest, LocalImageReplayRejectsMissingResolvedImage) { + const json input = json::array({{{"role", "user"}, + {"content", json::array({{{"type", "input_image"}, + {"image_url", "/missing-image.png"}}})}}}); + Request request; + + EXPECT_THROW(ResponseConverter::SnapshotReplayInputItems(input, request), fl::Exception); +} + +TEST(MediaInputTest, LocalImageReplayHandlesUnsafeFormatHints) { + const auto dir = fl::test::TempPath::CreateTempDir("fl_media_mime_"); + const auto image_path = dir.path() / "image.png"; + { + std::ofstream file(image_path, std::ios::binary); + file.write("\x01\x02\x03", 3); + ASSERT_TRUE(file.good()); + } + const json body = { + {"model", "test-model"}, + {"input", json::array({{{"role", "user"}, + {"content", json::array({{{"type", "input_image"}, + {"image_url", image_path.string()}, + {"media_type", "image/png;extra,invalid"}}})}}})}}; + auto request = ResponseConverter::ToSessionRequest(body.get()); + ASSERT_TRUE(std::filesystem::remove(image_path)); + + const auto snapshot = ResponseConverter::SnapshotReplayInputItems(ResponseConverter::ToInputItems(body), request); + ASSERT_TRUE(snapshot); + ResponseChainContext chain; + chain.emplace_back(json::array(), json::array()); + chain[0].replay_input_items = snapshot; + ResponseCreateParams next; + next.model = "test-model"; + next.input = std::string("continue"); + auto replay = ResponseConverter::ToSessionRequest(next, &chain); + auto media = CollectMediaInput(replay); + ASSERT_EQ(media.images.size(), 1u); + EXPECT_EQ(media.images[0]->ReadBytes(), (std::vector{1, 2, 3})); } diff --git a/sdk_v2/cpp/test/internal_api/chat/onnx_chat_generator_vision_test.cc b/sdk_v2/cpp/test/internal_api/chat/onnx_chat_generator_vision_test.cc index 7e1dc91e3..61fba7e51 100644 --- a/sdk_v2/cpp/test/internal_api/chat/onnx_chat_generator_vision_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/onnx_chat_generator_vision_test.cc @@ -10,6 +10,7 @@ #include "inferencing/generative/chat/onnx_chat_generator.h" +#include "exception.h" #include "items/audio_item.h" #include "items/image_item.h" #include "items/message_item.h" @@ -152,8 +153,7 @@ TEST(OnnxChatGeneratorVision, TransformPreservesPriorTextOnlyMessages) { EXPECT_EQ(json[2]["content"][1]["text"], "what is this"); } -TEST(OnnxChatGeneratorVision, TransformOnlyRewritesFinalUserMessage) { - // Two user messages; only the latter should be rewritten. +TEST(OnnxChatGeneratorVision, TransformKeepsEarlierTextOnlyUserUnstructured) { std::vector messages; messages.push_back(MakeTextMessage(FOUNDRY_LOCAL_ROLE_USER, "first")); messages.push_back(MakeTextMessage(FOUNDRY_LOCAL_ROLE_ASSISTANT, "ok")); @@ -190,10 +190,50 @@ TEST(OnnxChatGeneratorMedia, TransformIncludesImageAndAudioMarkers) { EXPECT_EQ(json[0]["content"][2]["text"], "describe both"); } -TEST(OnnxChatGeneratorMedia, TransformRejectsMediaOutsideFinalUserMessage) { +TEST(OnnxChatGeneratorVision, TransformPreservesImagesAcrossUserTurns) { std::vector messages; - messages.push_back(MakeAudioMessage(FOUNDRY_LOCAL_ROLE_USER, "transcribe this")); - messages.push_back(MakeTextMessage(FOUNDRY_LOCAL_ROLE_USER, "then summarize it")); + messages.push_back(MakeImageMessage(FOUNDRY_LOCAL_ROLE_USER, "first image")); + messages.push_back(MakeTextMessage(FOUNDRY_LOCAL_ROLE_ASSISTANT, "first answer")); + messages.push_back(MakeImageMessage(FOUNDRY_LOCAL_ROLE_USER, "second image")); + messages.push_back(MakeTextMessage(FOUNDRY_LOCAL_ROLE_ASSISTANT, "second answer")); + messages.push_back(MakeTextMessage(FOUNDRY_LOCAL_ROLE_USER, "compare them")); + + const auto json = nlohmann::json::parse(OnnxChatGenerator::TransformMessagesForMedia(messages)); + + ASSERT_EQ(json.size(), 5u); + EXPECT_EQ(json[0]["content"][0]["type"], "image"); + EXPECT_EQ(json[0]["content"][1]["text"], "first image"); + EXPECT_EQ(json[1]["content"], "first answer"); + EXPECT_EQ(json[2]["content"][0]["type"], "image"); + EXPECT_EQ(json[2]["content"][1]["text"], "second image"); + EXPECT_EQ(json[3]["content"], "second answer"); + EXPECT_EQ(json[4]["content"][0]["text"], "compare them"); +} + +TEST(OnnxChatGeneratorVision, TransformPreservesNamesAndConditionalAssistantReasoning) { + std::vector> assistant_parts; + assistant_parts.push_back(std::make_unique("private", FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING)); + assistant_parts.push_back(std::make_unique("answer")); + std::vector messages; + messages.push_back(MakeImageMessage(FOUNDRY_LOCAL_ROLE_USER, "first")); + messages[0].name = "observer"; + messages.emplace_back(FOUNDRY_LOCAL_ROLE_ASSISTANT, std::move(assistant_parts), "helper"); + messages.push_back(MakeTextMessage(FOUNDRY_LOCAL_ROLE_USER, "continue")); + + const auto ordinary = nlohmann::json::parse(OnnxChatGenerator::TransformMessagesForMedia(messages)); + const auto reasoning = nlohmann::json::parse(OnnxChatGenerator::TransformMessagesForMedia(messages, true)); + + EXPECT_EQ(ordinary[0]["name"], "observer"); + EXPECT_EQ(ordinary[1]["name"], "helper"); + EXPECT_EQ(ordinary[1]["content"], "answer"); + EXPECT_FALSE(ordinary[1].contains("reasoning_content")); + EXPECT_EQ(reasoning[1]["reasoning_content"], "private"); +} - EXPECT_THROW(OnnxChatGenerator::TransformMessagesForMedia(messages), fl::Exception); +TEST(OnnxChatGeneratorVision, TransformRejectsImagesOutsideUserMessages) { + for (const auto role : {FOUNDRY_LOCAL_ROLE_SYSTEM, FOUNDRY_LOCAL_ROLE_ASSISTANT, FOUNDRY_LOCAL_ROLE_TOOL}) { + std::vector messages; + messages.push_back(MakeImageMessage(role, "image")); + EXPECT_THROW(OnnxChatGenerator::TransformMessagesForMedia(messages), fl::Exception); + } } diff --git a/sdk_v2/cpp/test/internal_api/response_lifecycle_test.cc b/sdk_v2/cpp/test/internal_api/response_lifecycle_test.cc index f9c97cac1..c935c7359 100644 --- a/sdk_v2/cpp/test/internal_api/response_lifecycle_test.cc +++ b/sdk_v2/cpp/test/internal_api/response_lifecycle_test.cc @@ -235,6 +235,42 @@ TEST(ResponseContinuationTest, NormalCompactionAndLeaseSnapshotDoNotDuplicateRep EXPECT_EQ((*context)[2].input_items[0]["content"], "child"); } +TEST(ResponseContinuationTest, PrivateMediaReplaySurvivesLeaseSnapshotAndDeletion) { + ResponseStore store(1); + const auto public_items = json::array({{{"role", "user"}, + {"content", json::array({{{"type", "input_image"}, + {"image_url", "/local/image.png"}}})}}}); + auto private_items = public_items; + private_items[0]["content"][0]["image_url"] = "data:image/png;base64,AQID"; + const auto snapshot = std::make_shared(std::move(private_items)); + + auto root = store.BeginResponse("", "model-a"); + ASSERT_TRUE(store.Commit(root.lease, + ResponseStore::StoredResponse{"parent", "model-a", ResponseJson("parent"), public_items, {}, snapshot}, + nullptr)); + const auto visible = store.GetInputItems("parent"); + ASSERT_TRUE(visible); + EXPECT_EQ(*visible, public_items); + + auto child = store.BeginResponse("parent", "model-a"); + ASSERT_EQ(child.status, ContinuationStatus::kOk); + store.Store("other", ResponseJson("other"), json::array(), "model-a"); + ASSERT_FALSE(store.Get("parent").has_value()); + ASSERT_TRUE(store.Commit(child.lease, + ResponseStore::StoredResponse{"child", "model-a", ResponseJson("child", "parent"), + json::array()}, + nullptr)); + + const auto context = store.BuildChainContext("child"); + ASSERT_TRUE(context); + ASSERT_EQ(context->size(), 2u); + EXPECT_EQ((*context)[0].input_items, public_items); + ASSERT_TRUE((*context)[0].replay_input_items); + EXPECT_EQ((*context)[0].replay_input_items.get(), snapshot.get()); + EXPECT_TRUE(store.Delete("parent")); + EXPECT_FALSE(store.BuildChainContext("child").has_value()); +} + // ======================================================================== // Lease lifetime // ======================================================================== diff --git a/sdk_v2/cpp/test/internal_api/web_service_test.cc b/sdk_v2/cpp/test/internal_api/web_service_test.cc index f9eeb0348..d6cd87048 100644 --- a/sdk_v2/cpp/test/internal_api/web_service_test.cc +++ b/sdk_v2/cpp/test/internal_api/web_service_test.cc @@ -1951,7 +1951,7 @@ TEST_F(WebServiceTest, ChatCompletionsAppliesMediaPolicyBeforeInference) { const auto result = PostJson(base_url_ + "/v1/chat/completions", body); EXPECT_EQ(result.status, 400) << result.body.dump(2); - EXPECT_NE(ErrorMessageOf(result.body).find(with_tools ? "tool definitions" : "first turn"), std::string::npos) + EXPECT_NE(ErrorMessageOf(result.body).find(with_tools ? "tool definitions" : "multimodal model"), std::string::npos) << result.body.dump(2); } diff --git a/sdk_v2/cpp/test/sdk_api/responses_vision_test.cc b/sdk_v2/cpp/test/sdk_api/responses_vision_test.cc index 48cc5d31f..6f9a97478 100644 --- a/sdk_v2/cpp/test/sdk_api/responses_vision_test.cc +++ b/sdk_v2/cpp/test/sdk_api/responses_vision_test.cc @@ -16,8 +16,12 @@ // that CI on machines without vision-model storage stays green. #include "model_fixture.h" +#include "utils/temp_path.h" #include "web_service_fixture.h" +#include +#include +#include #include #include #include @@ -34,6 +38,16 @@ constexpr const char* kRedPngBase64 = "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAb0lEQVR4nO3PAQkAAAyEwO9feoshgnABdLep8QUNyPEFDcjxBQ3I" "8QUNyPEFDcjxBQ3I8QUNyPEFDcjxBQ3I8QUNyPEFDcjxBQ3I8QUNyPEFDcjxBQ3I8QUNyPEFDcjxBQ3I8QUNyPEFDcjxBQ3IPanc" "8OLDQitxAAAAAElFTkSuQmCC"; +constexpr char kRedPngBytes[] = + "\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52\x00\x00\x00\x40" + "\x00\x00\x00\x40\x08\x02\x00\x00\x00\x25\x0b\xe6\x89\x00\x00\x00\x6f\x49\x44\x41" + "\x54\x78\x9c\xed\xcf\x01\x09\x00\x00\x0c\x84\xc0\xef\x5f\x7a\x8b\x21\x82\x70\x01" + "\x74\xb7\xa9\xf1\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1" + "\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1" + "\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1" + "\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1\x05\x0d\xc8\xf1" + "\x05\x0d\xc8\xf1\x05\x0d\xc8\x3d\xa9\xdc\xf0\xe2\xc3\x42\x2b\x71\x00\x00\x00\x00" + "\x49\x45\x4e\x44\xae\x42\x60\x82"; constexpr const char* kBluePngBase64 = "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAb0lEQVR4nO3PAQkAAAyEwO9feoshgnABdNvJ8QUNyPEFDcjxBQ3I" "8QUNyPEFDcjxBQ3I8QUNyPEFDcjxBQ3I8QUNyPEFDcjxBQ3I8QUNyPEFDcjxBQ3I8QUNyPEFDcjxBQ3I8QUNyPEFDcjxBQ2oPcf8" @@ -78,6 +92,26 @@ std::string ReadChatStream(const std::string& body) { return text; } +json ReadCompletedResponse(const std::string& body) { + json response; + std::istringstream stream(body); + std::string line; + while (std::getline(stream, line)) { + if (line.rfind("data: ", 0) != 0 || line.substr(6) == "[DONE]") { + continue; + } + + const auto event = json::parse(line.substr(6)); + const auto type = event.value("type", ""); + EXPECT_NE(type, "response.failed") << body; + if (type == "response.completed" || type == "response.incomplete") { + response = event.at("response"); + } + } + + return response; +} + } // namespace // ---------------------------------------------------------------------- @@ -169,6 +203,7 @@ TEST_F(ResponsesVisionIntegrationTest, ChatCompletionsUsesImageColorIncludingStr {"messages", json::array({{{"role", "user"}, {"content", content}}})}, {"max_completion_tokens", 32}, {"temperature", 0}, + {"reasoning_effort", "none"}, {"stream", streaming}, {"stream_options", {{"include_usage", true}}}, }; @@ -185,6 +220,91 @@ TEST_F(ResponsesVisionIntegrationTest, ChatCompletionsUsesImageColorIncludingStr } } +TEST_F(ResponsesVisionIntegrationTest, ChatCompletionsRetainsImagesAcrossUserTurns) { + auto client = MakeClient(600); + for (const bool streaming : {false, true}) { + for (const bool reverse : {false, true}) { + SCOPED_TRACE(std::string(streaming ? "streaming" : "nonstreaming") + (reverse ? " blue first" : " red first")); + auto first = ChatImageContent(reverse ? kBluePngBase64 : kRedPngBase64); + first.insert(first.begin(), json{{"type", "text"}, {"text", "Picture 1. Remember this image. Reply only OK."}}); + auto second = ChatImageContent(reverse ? kRedPngBase64 : kBluePngBase64); + second.insert(second.begin(), + json{{"type", "text"}, + {"text", "Picture 2. What color was the first image? Answer with one word."}}); + const json body = { + {"model", vision_model_id()}, + {"messages", json::array({{{"role", "user"}, {"content", first}}, + {{"role", "assistant"}, {"content", "OK"}}, + {{"role", "user"}, {"content", second}}})}, + {"max_completion_tokens", 32}, + {"temperature", 0}, + {"reasoning_effort", "none"}, + {"stream", streaming}, + {"stream_options", {{"include_usage", true}}}, + }; + const auto result = client.Post(VisionEndpoint("/v1/chat/completions"), body.dump(), "application/json"); + ASSERT_TRUE(result) << httplib::to_string(result.error()); + ASSERT_EQ(result->status, 200) << result->body; + const auto text = streaming ? ReadChatStream(result->body) + : json::parse(result->body)["choices"][0]["message"]["content"].get(); + const auto normalized = fl::test::ToLower(text); + EXPECT_TRUE(std::regex_search(normalized, std::regex(reverse ? "\\bblue\\b" : "\\bred\\b"))) << text; + EXPECT_FALSE(std::regex_search(normalized, std::regex(reverse ? "\\bred\\b" : "\\bblue\\b"))) << text; + } + } +} + +TEST_F(ResponsesVisionIntegrationTest, CancelledAndUndoneImageTurnsLeaveRetainedHistoryIntact) { + using namespace foundry_local; + + ChatSession session(*SharedTestEnv::Get().vision_model()); + RequestOptions options; + options.search.temperature = 0.0f; + options.search.max_output_tokens = 32; + options.additional_options.Set("chat_template_kwargs", R"({"enable_thinking":false})"); + session.SetOptions(options); + + const auto image_request = [](const char* text) { + std::vector content; + content.push_back(Item::Text(text)); + content.push_back(Item::ImageFromData("png", kRedPngBytes, sizeof(kRedPngBytes) - 1)); + return Request{MessageItem(FOUNDRY_LOCAL_ROLE_USER, std::move(content))}; + }; + const auto first = image_request("Remember this image. Reply only OK."); + const auto first_text = fl::test::ToLower(CollectResponseText(session.ProcessRequest(first))); + ASSERT_EQ(session.TurnCount(), 1u); + ASSERT_EQ(first_text.find("red"), std::string::npos) << first_text; + + const Request probe{UserMessage("What color was the image? Answer with one word.")}; + const auto baseline = session.PreflightRequest(probe); + ASSERT_TRUE(baseline.fits); + + const auto second = image_request("Describe the second image in one sentence."); + std::atomic callbacks{0}; + session.SetStreamingCallback([&](Item) { + ++callbacks; + return 1; + }); + try { + (void)session.ProcessRequest(second); + FAIL() << "Expected streaming callback cancellation"; + } catch (const Error& error) { + EXPECT_EQ(error.Code(), FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED) << error.what(); + } + session.SetStreamingCallback(nullptr); + EXPECT_GT(callbacks.load(), 0); + EXPECT_EQ(session.TurnCount(), 1u); + EXPECT_EQ(session.PreflightRequest(probe).prompt_tokens, baseline.prompt_tokens); + + (void)session.ProcessRequest(second); + ASSERT_EQ(session.TurnCount(), 2u); + session.UndoTurns(1); + EXPECT_EQ(session.TurnCount(), 1u); + EXPECT_EQ(session.PreflightRequest(probe).prompt_tokens, baseline.prompt_tokens); + const auto answer = CollectResponseText(session.ProcessRequest(probe)); + EXPECT_TRUE(std::regex_search(fl::test::ToLower(answer), std::regex("\\bred\\b"))) << answer; +} + TEST_F(ResponsesVisionIntegrationTest, ChatCompletionsImageOnlyProducesOutput) { auto client = MakeClient(600); const json body = { @@ -403,8 +523,7 @@ TEST_F(ResponsesVisionIntegrationTest, TwoImagesProduceOutput) { TEST_F(ResponsesVisionIntegrationTest, ImageWithDeclaredToolsIsRejected) { auto client = MakeClient(); - // The model could answer an image turn with a tool call, and the turn carrying that call's result could no longer - // show it the image. The combination is rejected before any generation happens. + // The multimodal prompt does not represent tool calls or results, so reject this combination before generation. json request_body = { {"model", vision_model_id()}, {"input", json::array({ @@ -430,7 +549,7 @@ TEST_F(ResponsesVisionIntegrationTest, ImageWithDeclaredToolsIsRejected) { EXPECT_NE(result->body.find("tool definitions"), std::string::npos) << result->body; } -TEST_F(ResponsesVisionIntegrationTest, ImageOnAContinuationIsRejected) { +TEST_F(ResponsesVisionIntegrationTest, ImagesAndTextContinueWarmAndColdConversations) { auto client = MakeClient(); client.set_read_timeout(600, 0); @@ -440,6 +559,7 @@ TEST_F(ResponsesVisionIntegrationTest, ImageOnAContinuationIsRejected) { {"store", true}, {"max_output_tokens", 64}, {"temperature", 0}, + {"reasoning", {{"effort", "none"}}}, }; auto first_result = client.Post(VisionEndpoint("/v1/responses"), first.dump(), "application/json"); @@ -447,36 +567,122 @@ TEST_F(ResponsesVisionIntegrationTest, ImageOnAContinuationIsRejected) { ASSERT_EQ(first_result->status, 200) << first_result->body; const std::string first_id = json::parse(first_result->body)["id"].get(); - auto send_image_continuation = [&]() { - json body = { - {"model", vision_model_id()}, - {"previous_response_id", first_id}, - {"input", json::array({ - {{"role", "user"}, - {"content", json::array({ - {{"type", "input_text"}, {"text", "And this image?"}}, - {{"type", "input_image"}, - {"detail", "low"}, - {"image_url", std::string("data:image/png;base64,") + kTinyPngBase64}}, - })}}, - })}, - {"max_output_tokens", 64}, - {"store", false}, - }; - return client.Post(VisionEndpoint("/v1/responses"), body.dump(), "application/json"); + const json red_turn = { + {"model", vision_model_id()}, + {"previous_response_id", first_id}, + {"input", json::array({{{"role", "user"}, + {"content", json::array({{{"type", "input_text"}, + {"text", "Picture 1. Remember this image. Reply only OK."}}, + {{"type", "input_image"}, + {"image_url", std::string("data:image/png;base64,") + + kRedPngBase64}}})}}})}, + {"max_output_tokens", 32}, + {"temperature", 0}, + {"reasoning", {{"effort", "none"}}}, + {"store", true}, }; + const auto red_result = client.Post(VisionEndpoint("/v1/responses"), red_turn.dump(), "application/json"); + ASSERT_TRUE(red_result); + ASSERT_EQ(red_result->status, 200) << red_result->body; + const auto red_response = json::parse(red_result->body); + const auto remembered_reply = fl::test::ToLower(red_response.value("output_text", "")); + ASSERT_EQ(remembered_reply.find("red"), std::string::npos) << remembered_reply; + ASSERT_EQ(remembered_reply.find("blue"), std::string::npos) << remembered_reply; + const auto red_id = red_response["id"].get(); + + const json blue_turn = { + {"model", vision_model_id()}, + {"previous_response_id", red_id}, + {"input", json::array({{{"role", "user"}, + {"content", json::array({{{"type", "input_text"}, + {"text", "Picture 2. What color was the first image? One word."}}, + {{"type", "input_image"}, + {"image_url", std::string("data:image/png;base64,") + + kBluePngBase64}}})}}})}, + {"max_output_tokens", 32}, + {"temperature", 0}, + {"reasoning", {{"effort", "none"}}}, + {"store", false}, + }; + for (const char* path : {"warm", "cold"}) { + SCOPED_TRACE(path); + const auto result = client.Post(VisionEndpoint("/v1/responses"), blue_turn.dump(), "application/json"); + ASSERT_TRUE(result); + ASSERT_EQ(result->status, 200) << result->body; + const auto text = json::parse(result->body).value("output_text", ""); + const auto normalized = fl::test::ToLower(text); + EXPECT_TRUE(std::regex_search(normalized, std::regex("\\bred\\b"))) << text; + EXPECT_FALSE(std::regex_search(normalized, std::regex("\\bblue\\b"))) << text; + } + + const json text_turn = {{"model", vision_model_id()}, + {"previous_response_id", red_id}, + {"input", "What color was the image? Answer with one word."}, + {"max_output_tokens", 32}, + {"temperature", 0}, + {"reasoning", {{"effort", "none"}}}, + {"store", false}}; + const auto text_result = client.Post(VisionEndpoint("/v1/responses"), text_turn.dump(), "application/json"); + ASSERT_TRUE(text_result); + ASSERT_EQ(text_result->status, 200) << text_result->body; + const auto text = json::parse(text_result->body).value("output_text", ""); + EXPECT_TRUE(std::regex_search(fl::test::ToLower(text), std::regex("\\bred\\b"))) << text; +} + +TEST_F(ResponsesVisionIntegrationTest, LocalImageReplaySurvivesDeletionForStreamingAndNonStreaming) { + auto client = MakeClient(600); + for (const bool streaming : {false, true}) { + SCOPED_TRACE(streaming ? "streaming" : "nonstreaming"); + const auto dir = fl::test::TempPath::CreateTempDir("fl_vision_replay_"); + const auto image_path = dir.path() / "red.png"; + { + std::ofstream file(image_path, std::ios::binary); + file.write(kRedPngBytes, static_cast(sizeof(kRedPngBytes) - 1)); + ASSERT_TRUE(file.good()); + } - // Warm: the session cached under first_id still holds the conversation. - auto warm = send_image_continuation(); - ASSERT_TRUE(warm) << "HTTP request failed"; - EXPECT_EQ(warm->status, 400) << "media on a continuation must be a client error: " << warm->body; - EXPECT_NE(warm->body.find("first turn of a conversation"), std::string::npos) << warm->body; - - // Cold: that request checked the session out and the failure did not put it back, so this one rebuilds the chain - // from the store. The rejection must be identical. - auto cold = send_image_continuation(); - ASSERT_TRUE(cold) << "HTTP request failed"; - EXPECT_EQ(cold->status, warm->status) << "warm and cold must reject a media continuation identically: " - << cold->body; - EXPECT_NE(cold->body.find("first turn of a conversation"), std::string::npos) << cold->body; + const json first = { + {"model", vision_model_id()}, + {"input", json::array({{{"role", "user"}, + {"content", json::array({{{"type", "input_text"}, + {"text", "Remember this image. Reply only OK."}}, + {{"type", "input_image"}, + {"image_url", image_path.string()}}})}}})}, + {"stream", streaming}, + {"store", true}, + {"max_output_tokens", 32}, + {"temperature", 0}, + {"reasoning", {{"effort", "none"}}}, + }; + const auto first_result = client.Post(VisionEndpoint("/v1/responses"), first.dump(), "application/json"); + ASSERT_TRUE(first_result); + ASSERT_EQ(first_result->status, 200) << first_result->body; + const auto first_response = streaming ? ReadCompletedResponse(first_result->body) : json::parse(first_result->body); + ASSERT_TRUE(first_response.is_object()) << first_result->body; + const auto id = first_response.at("id").get(); + const auto first_text = fl::test::ToLower(first_response.value("output_text", "")); + ASSERT_EQ(first_text.find("red"), std::string::npos) << first_text; + ASSERT_TRUE(std::filesystem::remove(image_path)); + + const auto public_items = client.Get(VisionEndpoint("/v1/responses") + "/" + id + "/input_items"); + ASSERT_TRUE(public_items); + ASSERT_EQ(public_items->status, 200) << public_items->body; + EXPECT_EQ(json::parse(public_items->body)["data"][0]["content"][1]["image_url"], image_path.string()); + + const json followup = {{"model", vision_model_id()}, + {"previous_response_id", id}, + {"input", "What color was the image? Answer with one word."}, + {"store", false}, + {"max_output_tokens", 32}, + {"temperature", 0}, + {"reasoning", {{"effort", "none"}}}}; + for (const char* path : {"warm", "cold"}) { + SCOPED_TRACE(path); + const auto result = client.Post(VisionEndpoint("/v1/responses"), followup.dump(), "application/json"); + ASSERT_TRUE(result); + ASSERT_EQ(result->status, 200) << result->body; + const auto text = json::parse(result->body).value("output_text", ""); + EXPECT_TRUE(std::regex_search(fl::test::ToLower(text), std::regex("\\bred\\b"))) << text; + } + } }