Preserve images across conversation turns

sdk_v2/INFERENCING.md

@@ -228,8 +228,10 @@ For complete setup and inference examples, see the native vision samples:
 #### Current limitations
 
 Remote HTTP(S) image fetching and image `file_id` references are not supported.
-Image input is restricted to the first conversation turn and cannot be combined
-with tool definitions. Start a new conversation to submit another image turn.
+Images may appear in any user turn. Earlier images are retained and replayed until
+the conversation history is discarded, increasing memory usage. Multi-image support
+and input limits depend on the model and its chat template. Media cannot be combined
+with tool definitions or a conversation containing tool calls or results.
 
 ## Run a request, then stream a follow-up
 

sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc

@@ -1015,8 +1015,8 @@ bool IsGuidedToolRetryEligible(
     int attempted_tokens,
     std::optional<int> output_budget) {
   return tool_ctx.tool_plan && natural_tool_output_end &&
-           (!semantic_output_seen || safe_prefix_available || !tool_ctx.text_output) &&
-           !tool_call_published &&
+         (!semantic_output_seen || safe_prefix_available || !tool_ctx.text_output) &&
+         !tool_call_published &&
          (!output_budget || attempted_tokens < *output_budget) &&
          options.stop_sequences.empty() &&
          tool_ctx.tool_output &&
@@ -1372,13 +1372,10 @@ std::unique_ptr<PreparedChatRequest> PrepareChatRequest(
     }
 
     const auto effective_kvp = MergeKeyValuePairs(base_session_options, internal_request.options);
-    auto media = CollectMediaInput(internal_request);
-    const bool media_turn = !media.Empty();
-    if (media_turn) {
+    if (RequestHasMedia(internal_request)) {
       ToolCallContext declarations;
       chat_session_internal::PopulateToolDefinitions(request_definitions, declarations);
-      ValidateMediaTurn(media, BuildTranscriptMessages(internal_request.items),
-                        {.tools_declared = declarations.HasTools()});
+      ValidateMediaToolDefinitions(declarations.HasTools());
     }
 
     prepared->tool_context = BuildToolCallContextForRequest(internal_request, effective_kvp, request_definitions,
@@ -1390,7 +1387,10 @@ std::unique_ptr<PreparedChatRequest> PrepareChatRequest(
     const ChatTranscript payload_transcript;
     payload_transcript.ValidateInputs(messages);
     prepared->messages.emplace(message_preparer(std::move(messages), model.HasPositionalToolResults()));
+    auto media = CollectMediaInput(prepared->messages->Messages());
+    const bool media_turn = !media.Empty();
     if (media_turn) {
+      ValidateMediaTurn(media, prepared->messages->Messages(), {});
       prepared->prompt =
           PrepareMediaChatPrompt(media.messages, model, media.images, media.audios, prepared->tool_context);
     } else {
@@ -1404,13 +1404,11 @@ std::unique_ptr<PreparedChatRequest> PrepareChatRequest(
 
   auto effective_kvp = MergeKeyValuePairs(base_session_options, request.options);
   chat_completions::ApplyCatalogDefaults(effective_kvp, model_info.model_settings);
-  auto media = CollectMediaInput(request);
-  const bool media_turn = !media.Empty();
-  if (media_turn) {
+  const bool request_has_media = RequestHasMedia(request);
+  if (request_has_media || CarriesMedia(transcript.Messages())) {
     ToolCallContext declarations;
     chat_session_internal::PopulateToolDefinitions(tool_definitions, declarations);
-    ValidateMediaTurn(media, BuildTranscriptMessages(request.items),
-                      {.session_has_history = !transcript.Empty(), .tools_declared = declarations.HasTools()});
+    ValidateMediaToolDefinitions(declarations.HasTools());
   }
 
   prepared->tool_context = BuildToolCallContextForRequest(request, effective_kvp, tool_definitions,
@@ -1423,7 +1421,7 @@ std::unique_ptr<PreparedChatRequest> PrepareChatRequest(
   prepared->system_prompt = GetOptionOrEmpty(effective_kvp, kSystemPromptOption);
 
   if (!TurnCanGenerate(prepared->ingest.messages,
-                       {.media = media_turn,
+                       {.media = request_has_media,
                         .history = !transcript.Empty(),
                         .system_prefix = !prepared->system_prompt.empty()})) {
     FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT,
@@ -1440,15 +1438,12 @@ std::unique_ptr<PreparedChatRequest> PrepareChatRequest(
   all_messages = WithSystemPrompt(prepared->system_prompt, std::move(all_messages));
   prepared->messages.emplace(message_preparer(std::move(all_messages), model.HasPositionalToolResults()));
 
+  auto media = CollectMediaInput(prepared->messages->Messages());
+  const bool media_turn = !media.Empty();
   if (media_turn) {
-    auto media_messages = std::move(media.messages);
-    if (!prepared->system_prompt.empty()) {
-      media_messages.insert(media_messages.begin(),
-                            MessageItem(FOUNDRY_LOCAL_ROLE_SYSTEM, prepared->system_prompt));
-    }
-
+    ValidateMediaTurn(media, prepared->messages->Messages(), {});
     prepared->prompt =
-        PrepareMediaChatPrompt(media_messages, model, media.images, media.audios, prepared->tool_context);
+        PrepareMediaChatPrompt(media.messages, model, media.images, media.audios, prepared->tool_context);
   } else {
     prepared->prompt = PrepareTextChatPrompt(*prepared->messages, model, prepared->tool_context);
   }
@@ -1651,7 +1646,7 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response)
       !cached_tool_ctx_.HasSameTools(turn_tool_ctx);
 
   if (cached_generator_ &&
-      (turn_system_prompt != system_prompt_ || retained_tool_context_changed ||
+      (media_turn || turn_system_prompt != system_prompt_ || retained_tool_context_changed ||
        (backend_kind == ChatBackendKind::kGenerator && (inputs.empty() || CarriesToolActivity(inputs))))) {
     InvalidateCachedGenerator();
   }
@@ -2081,8 +2076,7 @@ void ChatSession::ProcessRequestImpl(const Request& request, Response& response)
   // from plain text turns, so a rebuild restores correct behavior. This matches C#, which always applies the full
   // template per turn.
   //
-  // Media is single-shot too: AppendMessages cannot extend a media-decoded sequence, so any text follow-up must
-  // rebuild rather than silently feed text into a state that includes media-derived tokens.
+  // AppendMessages cannot extend a media-decoded sequence. Rebuild later turns from the retained media transcript.
   //
   // A turn stopped at a tool call because the model kept talking afterwards is truncated mid-sequence with no turn
   // terminator, so its KV cache is no basis for the next turn either.

sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h

@@ -191,10 +191,7 @@ class ChatSession : public Session {
   /// each turn's input messages and assistant reply from the transcript.
   /// If all turns are undone, the cached generator is destroyed.
   ///
-  /// Vision turns: image input is only allowed while the conversation has no
-  /// history. UndoTurns rolls back messages, so undoing every turn does make
-  /// the session accept media again — but the media bytes of an undone turn
-  /// are gone either way, because they never entered the transcript.
+  /// Media retained by an undone turn is removed with that turn; earlier media remains available to future prompts.
   ///
   /// Blocks until any in-flight request on this session completes.
   ///

sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.cc

@@ -4,6 +4,8 @@
 
 #include "exception.h"
 #include "inferencing/session/tool_registry.h"
+#include "items/audio_item.h"
+#include "items/image_item.h"
 #include "items/message_item.h"
 #include "items/text_item.h"
 #include "items/tool_call_item.h"
@@ -147,6 +149,12 @@ bool TranscriptMessage::HasToolCalls() const {
   return false;
 }
 
+bool TranscriptMessage::HasMedia() const {
+  return std::any_of(entries.begin(), entries.end(), [](const TranscriptEntry& entry) {
+    return entry.kind == TranscriptEntry::Kind::kImage || entry.kind == TranscriptEntry::Kind::kAudio;
+  });
+}
+
 bool TranscriptMessage::HasVisibleTextAfterToolCall() const {
   bool seen_call = false;
   for (const auto& entry : entries) {
@@ -225,6 +233,10 @@ void MergeAssistantTurn(TranscriptMessage& open, TranscriptMessage&& next) {
       case TranscriptEntry::Kind::kToolCall:
         open.AppendToolCall(std::move(entry.tool_call));
         break;
+      case TranscriptEntry::Kind::kImage:
+      case TranscriptEntry::Kind::kAudio:
+        open.entries.push_back(std::move(entry));
+        break;
     }
   }
 }
@@ -286,15 +298,33 @@ TranscriptIngest IngestRequestItems(const std::vector<Item*>& items, const std::
       message.name = message_item.name;
 
       for (const auto& part : message_item.content) {
-        if (!part.view || part.view->type != FOUNDRY_LOCAL_ITEM_TEXT) {
+        if (!part.view) {
           continue;
         }
 
-        const auto& text_item = static_cast<const TextItem&>(*part.view);
-        if (text_item.text_type == FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING) {
-          message.AppendReasoning(text_item.text);
-        } else {
-          message.AppendText(text_item.text);
+        if (part.view->type == FOUNDRY_LOCAL_ITEM_TEXT) {
+          const auto& text_item = static_cast<const TextItem&>(*part.view);
+          if (text_item.text_type == FOUNDRY_LOCAL_TEXT_ITEM_TYPE_REASONING) {
+            message.AppendReasoning(text_item.text);
+          } else {
+            message.AppendText(text_item.text);
+          }
+        } else if (part.view->type == FOUNDRY_LOCAL_ITEM_IMAGE) {
+          const auto& image = static_cast<const ImageItem&>(*part.view);
+          auto snapshot = std::make_shared<ImageItem>(image.ReadBytes(), image.format);
+          message.entries.push_back(TranscriptEntry::Media(TranscriptEntry::Kind::kImage, std::move(snapshot)));
+        } else if (part.view->type == FOUNDRY_LOCAL_ITEM_AUDIO) {
+          const auto& audio = static_cast<const AudioItem&>(*part.view);
+          if (audio.data == nullptr || audio.data_size == 0) {
+            FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "audio entry must contain bytes");
+          }
+
+          const auto* begin = static_cast<const std::uint8_t*>(audio.data);
+          auto snapshot =
+              std::make_shared<AudioItem>(std::vector<std::uint8_t>(begin, begin + audio.data_size), audio.format);
+          snapshot->sample_rate = audio.sample_rate;
+          snapshot->channels = audio.channels;
+          message.entries.push_back(TranscriptEntry::Media(TranscriptEntry::Kind::kAudio, std::move(snapshot)));
         }
       }
 
@@ -366,6 +396,11 @@ bool CarriesToolActivity(const std::vector<TranscriptMessage>& messages) {
   });
 }
 
+bool CarriesMedia(const std::vector<TranscriptMessage>& messages) {
+  return std::any_of(messages.begin(), messages.end(),
+                     [](const TranscriptMessage& message) { return message.HasMedia(); });
+}
+
 bool CarriesPriorTurnHistory(const std::vector<TranscriptMessage>& messages) {
   return std::any_of(messages.begin(), messages.end(), [](const TranscriptMessage& message) {
     return message.role == FOUNDRY_LOCAL_ROLE_ASSISTANT || message.role == FOUNDRY_LOCAL_ROLE_TOOL;
@@ -374,7 +409,8 @@ bool CarriesPriorTurnHistory(const std::vector<TranscriptMessage>& messages) {
 
 bool CarriesRespondableContent(const std::vector<TranscriptMessage>& messages) {
   return std::any_of(messages.begin(), messages.end(), [](const TranscriptMessage& message) {
-    return !message.VisibleText().empty() || message.HasToolCalls() || message.role == FOUNDRY_LOCAL_ROLE_TOOL;
+    return !message.VisibleText().empty() || message.HasMedia() || message.HasToolCalls() ||
+           message.role == FOUNDRY_LOCAL_ROLE_TOOL;
   });
 }
 

sdk_v2/cpp/src/inferencing/generative/chat/chat_transcript.h

@@ -9,6 +9,7 @@
 
 #include <cstddef>
 #include <functional>
+#include <memory>
 #include <optional>
 #include <string>
 #include <string_view>
@@ -45,30 +46,36 @@ struct TranscriptEntry {
     kText,       // visible assistant/user text
     kReasoning,  // chain-of-thought text
     kToolCall,   // a tool invocation
+    kImage,
+    kAudio,
   };
 
   Kind kind = Kind::kText;
   std::string text;
   TranscriptToolCall tool_call;
+  std::shared_ptr<const Item> media;
 
   static TranscriptEntry Text(std::string value) {
-    return {Kind::kText, std::move(value), {}};
+    return {Kind::kText, std::move(value), {}, {}};
   }
 
   static TranscriptEntry Reasoning(std::string value) {
-    return {Kind::kReasoning, std::move(value), {}};
+    return {Kind::kReasoning, std::move(value), {}, {}};
   }
 
   static TranscriptEntry ToolCall(TranscriptToolCall call) {
-    return {Kind::kToolCall, {}, std::move(call)};
+    return {Kind::kToolCall, {}, std::move(call), {}};
+  }
+
+  static TranscriptEntry Media(Kind kind, std::shared_ptr<const Item> value) {
+    return {kind, {}, {}, std::move(value)};
   }
 };
 
 /// A message in the authoritative transcript.
 ///
-/// Unlike MessageItem (the API-facing type) a transcript message keeps visible text, reasoning, and tool calls as
-/// separate ordered events rather than folding everything into modality content parts. Media parts are intentionally
-/// absent: media input is single-shot and is fed to the generator directly from the request items.
+/// Visible text, reasoning, media, and tool calls remain separate ordered events. Media owns an immutable snapshot
+/// shared by transcript copies, so later turns and preflight operations see the original bytes.
 struct TranscriptMessage {
   flMessageRole role = FOUNDRY_LOCAL_ROLE_NONE;
   std::vector<TranscriptEntry> entries;
@@ -91,6 +98,7 @@ struct TranscriptMessage {
   void AppendToolCall(TranscriptToolCall call);
 
   bool HasToolCalls() const;
+  bool HasMedia() const;
 
   /// True when the message carries visible text that follows a tool call and is more than whitespace.
   ///
@@ -155,10 +163,9 @@ struct TranscriptIngest {
 
 /// Convert request items into transcript messages, preserving the order in which they were supplied.
 ///
-/// - MESSAGE items become role-tagged messages; typed TextItem parts keep their visible / reasoning distinction and
-///   non-text parts are skipped (media is handled separately by the caller). A message with neither content nor a
-///   participant name carries nothing and is dropped; a content-free named message survives to attribute the tool
-///   calls that follow it.
+/// - MESSAGE items become role-tagged messages; text keeps its visible / reasoning distinction and image and audio
+///   parts retain owned byte snapshots in their original positions. A message with neither content nor a participant
+///   name carries nothing and is dropped; a content-free named message survives to attribute following tool calls.
 /// - TOOL_CALL items become assistant messages, merged into an open assistant turn when there is one, so replayed
 ///   assistant content and its calls stay together exactly as the model produced them.
 /// - TOOL_RESULT items become role="tool" messages carrying the call ID. An empty result string is preserved.
@@ -209,6 +216,9 @@ std::vector<TranscriptMessage> BuildTranscriptMessages(
 /// relative to the surrounding conversation — and the media prompt path cannot render it at all.
 bool CarriesToolActivity(const std::vector<TranscriptMessage>& messages);
 
+/// True when any message contains image or audio input.
+bool CarriesMedia(const std::vector<TranscriptMessage>& messages);
+
 /// True when `text` is empty or contains nothing but whitespace.
 ///
 /// Shared by the transcript's ordering invariant and by generation. Whitespace after a tool call does not end the
@@ -219,14 +229,13 @@ bool IsWhitespaceOnly(std::string_view text);
 /// whatever the caller is sending now, whether it came from a live transcript or from a replayed chain.
 bool CarriesPriorTurnHistory(const std::vector<TranscriptMessage>& messages);
 
-/// True when `messages` carry something the model can actually respond to: visible text, a tool call, or a tool
-/// result. Assistant turn boundaries alone are not — they only say a turn happened.
+/// True when `messages` carry something the model can actually respond to: visible text, media, a tool call, or a
+/// tool result. Assistant turn boundaries alone are not — they only say a turn happened.
 bool CarriesRespondableContent(const std::vector<TranscriptMessage>& messages);
 
 /// Everything outside a turn's messages that can still give the model something to generate from.
 struct TurnContent {
-  /// The request carries image or audio bytes. They reach the generator directly rather than as message text, so a
-  /// media-only turn has no entries at all — the bytes are the question.
+  /// The request carries image or audio bytes.
   bool media = false;
   /// The session transcript already holds a conversation, so an empty request asks for the next turn of it.
   bool history = false;