From 365a429b6b6620c266d4d4c87cb9e2d1dbfbcad2 Mon Sep 17 00:00:00 2001 From: Tan Jia Huei Date: Tue, 27 Oct 2026 16:34:34 +0000 Subject: [PATCH 1/5] fix: address hosted web search review findings - Project a reason-less Responses incomplete as unknown instead of inferring pause, and keep pause for sources that report it. - Replace the client-side citation prediction with an explicit citation_policy on the Messages adapter, defaulting to drop, and never emit empty or whitespace-only text blocks when splitting at citation boundaries. - Never drop reasoning that Chat cannot represent; fail at the signature boundary. - Make the hosted search request count a normalized Usage field. - Treat extra_body and extra_headers as opaque transport extensions. - Drop unrelated churn and document the new contracts. --- docs/README.md | 3 +- docs/protocols/anthropic-messages.md | 2 +- docs/protocols/comparison.md | 2 +- docs/protocols/openai-responses.md | 1 + docs/web-search.md | 14 +- python/ARCHITECTURE.md | 9 +- python/CONVENTIONS.md | 2 +- python/molto/adapters/anthropic.py | 60 +++-- python/molto/adapters/openai.py | 44 +--- python/molto/core.py | 10 +- python/molto/types/normalized.py | 22 +- python/molto/upstream/anthropic.py | 14 +- python/molto/upstream/openai.py | 60 +---- python/molto/utils/test.py | 2 +- python/molto/utils/web_search.py | 101 ++++++-- scripts/match_anthropic_cassette_events.py | 9 +- scripts/match_anthropic_cassette_responses.py | 6 +- scripts/match_openai_cassette_events.py | 6 +- scripts/match_openai_cassette_responses.py | 16 +- tests/adapters/test_anthropic_search.py | 61 +++-- tests/adapters/test_common.py | 111 +++++++++ tests/adapters/test_openai_search.py | 4 +- tests/core/test_web_search.py | 198 ++++++++++----- tests/test_match_anthropic_cassette_events.py | 9 +- ...test_match_anthropic_cassette_responses.py | 12 +- tests/test_match_openai_cassette_events.py | 6 +- tests/test_match_openai_cassette_responses.py | 17 +- tests/upstream/test_anthropic.py | 5 +- tests/upstream/test_anthropic_search.py | 2 +- tests/upstream/test_openai.py | 45 +++- tests/upstream/test_openai_search.py | 235 ++++++++---------- 31 files changed, 652 insertions(+), 436 deletions(-) create mode 100644 tests/adapters/test_common.py diff --git a/docs/README.md b/docs/README.md index 870952d..4a423e0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -209,7 +209,7 @@ The translated path currently supports: - system prompts and supported assistant history; - raw reasoning across all three protocols, and reasoning summaries on supported translations between Responses and Messages; - client-defined function tools, selection, parallel-call control, calls, successful results, history, and streamed argument deltas; -- basic [hosted web search](web-search.md), URL attribution between Responses and Messages, and mandatory search with text output for downstream Chat; +- basic [hosted web search](web-search.md), URL attribution between Responses and Messages when a Messages caller selects `citation_policy="buffer"`, and mandatory search with text output for downstream Chat; - model, output-token limit, reasoning effort, temperature, and `top_p`; - prompt-cache settings and supported text/image breakpoints under the [shared mapping policy](#prompt-caching), including silent omission of caller-defined keys on Messages; - response identity, model, lifecycle metadata, termination, and semantic token usage; and @@ -297,6 +297,7 @@ Molto first preserves a provider-neutral termination reason, then projects it in | `tool_call` | `tool_use` | `tool_calls` | completed | | `model_context_window_exceeded` | same value | `stop` | completed | | `pause` | `pause_turn` | `length` | incomplete with null details | +| `unknown` | `end_turn` | `stop` | incomplete with null details | | `refusal` | `refusal` | `stop` | completed | | `content_filter` | `end_turn` | `content_filter` | incomplete: `content_filter` | diff --git a/docs/protocols/anthropic-messages.md b/docs/protocols/anthropic-messages.md index d8d3c0a..b68b84a 100644 --- a/docs/protocols/anthropic-messages.md +++ b/docs/protocols/anthropic-messages.md @@ -360,7 +360,7 @@ Recognized errors use the Anthropic SDK namespace and the [shared error contract Declare `{"type":"web_search_20250305","name":"web_search"}` in `tools`. Translation to Responses supports allowed domains, approximate location, and search budgets; explicit blocked domains are rejected. -With Responses upstream, Messages receives server-tool blocks and URL citations with reduced source metadata. Citation-bearing text waits for complete attribution. See [Hosted web search](../web-search.md) for mappings, placeholders, and continuation limits. +With Responses upstream, Messages receives server-tool blocks and URL citations with reduced source metadata. Citation-bearing text waits for complete attribution only when the caller selects `citation_policy="buffer"` on `Molto.messages()`; the default `drop` policy streams text immediately and omits citations. A Responses response that ends incomplete without a stated cause arrives as `end_turn`, because only a Messages `pause_turn` is a reported pause. See [Hosted web search](../web-search.md) for mappings, placeholders, and continuation limits. ## Current implementation limitations diff --git a/docs/protocols/comparison.md b/docs/protocols/comparison.md index e9495b1..c464d9c 100644 --- a/docs/protocols/comparison.md +++ b/docs/protocols/comparison.md @@ -51,7 +51,7 @@ Responses is the richer lifecycle protocol. Translating into Responses requires ## Hosted-search fidelity -Responses and Messages translate basic search calls, results, and URL attribution. Messages excerpts and opaque references cannot be reconstructed from Responses; translated Chat retains only answer text and inline links. See [Hosted web search](../web-search.md) for directional mappings and limitations. +Responses and Messages translate basic search calls, results, and URL attribution; a Messages target preserves citations only when the caller selects `citation_policy="buffer"`, and gaps that contain only whitespace are folded into a neighbouring block so no emitted block is empty. Messages excerpts and opaque references cannot be reconstructed from Responses; translated Chat retains only answer text and inline links, and a search turn carrying a reasoning signature is rejected rather than flattened. See [Hosted web search](../web-search.md) for directional mappings and limitations. ## Role and placement diff --git a/docs/protocols/openai-responses.md b/docs/protocols/openai-responses.md index 469a10f..2f0a8d3 100644 --- a/docs/protocols/openai-responses.md +++ b/docs/protocols/openai-responses.md @@ -332,6 +332,7 @@ Termination projection is: | `tool_call` | `response.completed`, coerced | | `model_context_window_exceeded` | `response.completed`, coerced | | `pause` | `response.incomplete` with null details | +| `unknown` | `response.incomplete` with null details | | `refusal` | `response.completed`, coerced | Molto retains the more specific source reason in the normalized stream. Reasons without a corresponding Responses termination value use the coercions above. diff --git a/docs/web-search.md b/docs/web-search.md index 02dce8a..89e60f8 100644 --- a/docs/web-search.md +++ b/docs/web-search.md @@ -25,9 +25,9 @@ Responses declares `{"type":"web_search"}` in `tools`. Messages declares `{"type | Blocked domains | Native Messages preserves `blocked_domains`; translation to Responses rejects an explicit blocklist | | Source inclusion | Translated searches to Responses request `web_search_call.action.sources`, preserving existing include values without adding duplicates | -Omitted options remain omitted; native options pass through unchanged. Native Chat forwards `web_search_options` supplied in either the request or `extra_body`. Translated routes targeting Chat reject hosted search before transport, including search enabled through `extra_body.web_search_options`. +Omitted options remain omitted; native options pass through unchanged. Native Chat forwards `web_search_options` supplied in either the request or `extra_body`. Translated routes targeting Chat reject hosted search declared in the request before transport. -On translated routes, hosted search supplied through `extra_body.tools` uses the same output policy as search declared in the downstream request. When `extra_body.tools` replaces the projected tools, the effective upstream declarations determine whether Chat omits signature-only reasoning. +`extra_body` and `extra_headers` are opaque extensions: Molto merges them into the upstream request without inspecting them, so they never enable, disable, or reshape translation. Search supplied only through `extra_body.tools` or `extra_body.web_search_options` is therefore forwarded verbatim, not translated, and the resulting annotations are projection input only where the declared request already enables the policy that consumes them. Responses and Messages preserve automatic, disabled, required, and supported named selection. Translated Chat requests require hosted search specifically, using Responses required `allowed_tools` restricted to search or Messages named `web_search` selection. Conflicting explicit Chat tool choices are rejected on translated routes. A client function named `web_search` remains distinct from hosted search, but Messages cannot select between them by name; rename the function when using named selection. @@ -82,9 +82,9 @@ It becomes these illustrative Messages content blocks: Offsets use Python string characters without normalization. Recordings verify ASCII ranges; synthetic Unicode tests do not establish the provider's Unicode indexing convention. -Messages buffers text parts until their citations are complete, preserving part order. This can increase text latency and change chunk boundaries. +Citations reach a Messages caller only when the caller opts in with `citation_policy="buffer"`, which buffers text parts until their citations are complete and preserves part order. That option increases text latency and changes chunk boundaries for every text part, including uncited ones. The default `citation_policy="drop"` streams text immediately and omits citations. Neither mode drops answer text, and citations are never inferred from a request's tools, history, or `extra_body`. -Translated Chat omits structured annotations in both modes. It preserves actual answer text and existing inline links, without inserting replacement citations or narration. Signature-only reasoning with no representable text is omitted on the hosted-search Chat route; other reasoning follows the existing protocol restrictions. +Translated Chat omits structured annotations in both modes. It preserves actual answer text and existing inline links, without inserting replacement citations or narration. Chat cannot carry a reasoning signature, so a hosted-search turn whose reasoning is encrypted or signed, as OpenAI and Anthropic both return, is rejected at the signature boundary instead of losing that output. Chat remains usable for providers whose reasoning is unsigned. ## Search activity and results @@ -138,11 +138,13 @@ A Messages pause has these terminal values: A paused search can remain `in_progress` without a result. Chat's `length` approximates unfinished generation; increasing the token limit may not resolve the pause. Continuation is caller-managed. -Native usage is preserved. Token translation is unchanged. Messages search counts have no public equivalent on translated Responses or Chat and are omitted. When Responses explicitly reports `tool_usage.web_search.num_requests`, translated Messages copies that number to `usage.server_tool_use.web_search_requests` and supplies `web_fetch_requests: 0` for this search-only scope. Explicit zero remains zero; absent/null counts remain unknown. Counts are not derived from operations, queries, sources, citations, or replayed history, and repeated usage snapshots are updates rather than additions. This is a numeric compatibility approximation, not equivalent billing or accounting. +A Responses response that ends `incomplete` without `incomplete_details` does not establish a pause. Molto normalizes it as `unknown`, which Messages reports as `end_turn` and Chat as `stop`; only a Messages `stop_reason: "pause_turn"` produces the pause row above. + +Native usage is preserved. Token translation is unchanged. Messages search counts have no public equivalent on translated Responses or Chat and are omitted. When Responses explicitly reports `tool_usage.web_search.num_requests`, translated Messages copies that number to `usage.server_tool_use.web_search_requests` and supplies `web_fetch_requests: 0` for this search-only scope. Explicit zero remains zero; absent/null counts remain unknown, and a provider that reports no token usage reports no search count either, because the count lives on the normalized usage object. Counts are not derived from operations, queries, sources, citations, or replayed history, and repeated usage snapshots are updates rather than additions. This is a numeric compatibility approximation, not equivalent billing or accounting. ## Follow-ups and recovery -Keep the upstream and provider fixed across a conversation. Native routes preserve search history. Translated Responses/Messages routes map supported calls, results, and citations with the reductions above; translated Chat history retains only visible text and supported function calls. History can carry citations without redeclaring search. When targeting Responses, citation buffering follows the effective upstream history, including replacement history supplied through `extra_body.input`. +Keep the upstream and provider fixed across a conversation. Native routes preserve search history. Translated Responses/Messages routes map supported calls, results, and citations with the reductions above; translated Chat history retains only visible text and supported function calls. History can carry citations without redeclaring search. Citation projection follows `citation_policy` only, so a caller that supplies search context solely through `extra_body.input` must select `buffer` explicitly. Projection to a Messages upstream rejects search-history calls containing more than one query with `UnsupportedInput`. Responses pools results across those queries; expanding that history into separate Messages calls would leave calls without matching results. Single-query history preserves the call/result link. diff --git a/python/ARCHITECTURE.md b/python/ARCHITECTURE.md index 5a13bdd..57a383e 100644 --- a/python/ARCHITECTURE.md +++ b/python/ARCHITECTURE.md @@ -148,7 +148,7 @@ Each implemented coordinator performs the following work: 5. Select the upstream path from `endpoint`. 6. Return either an async iterator of downstream SDK event models or the downstream SDK response model. -Every coordinator also accepts request-scoped `extra_body` and `extra_headers` transport options. They bypass downstream request normalization and are applied by the selected provider SDK after upstream request projection. Before forwarding extra headers, the facade removes client credentials, account selectors, connection-specific fields, request framing, and proxy topology headers; the provider SDK then constructs its own authentication and transport headers. This preserves Molto's credential boundary while allowing application-specific headers and provider body extensions. +Every coordinator also accepts request-scoped `extra_body` and `extra_headers` transport options. They are opaque: the selected provider SDK merges them into the upstream request after Molto projects it, and Molto never reads them to choose a route, normalize input, construct a request, or project output. Translation guarantees are therefore defined over the normalized request only, and a caller that supplies request semantics through `extra_body` opts out of them. Before forwarding extra headers, the facade removes client credentials, account selectors, connection-specific fields, request framing, and proxy topology headers; the provider SDK then constructs its own authentication and transport headers. This preserves Molto's credential boundary while allowing application-specific headers and provider body extensions. The returned `MoltoStream` exposes a copy of the successful upstream response headers as `response_headers`. Coordinator prefetching opens the SDK stream and captures those headers before returning, including on translated routes. Molto retains all response headers; an HTTP server using Molto remains responsible for filtering connection-specific and framing headers before constructing its downstream response. @@ -344,9 +344,9 @@ Service-tier labels are retained as upstream metadata, but target protocols do n `SearchCall` and `SearchResult` use the existing part lifecycle and link by call identity. They are assistant history/output parts. An unfinished call may have no result, and a search error does not determine overall termination. Responses folds a call and result into one item; Messages emits separate server-tool blocks. -Responses and Messages clients set `ResponseStartEvent.web_search_enabled` from the effective upstream tool declarations, including `extra_body` overrides. Chat projection uses this state for its hosted-search output policy, falling back to normalized request tools when an event source omits it. This keeps transport interpretation in clients and output projection in adapters. +A block-based target cannot split text it has already emitted, so `AnthropicMessagesAdapter` takes a `citation_policy` selected by the caller through `Molto.messages()`. `buffer` holds every text part until its citation snapshot is final so blocks can be split at citation boundaries, trading text latency and chunk boundaries for exact attribution. `drop`, the default, streams text immediately and omits citations. The policy governs output projection only: request construction keeps input attribution, and search call/result parts are buffered in both modes because that buffering learns the final action rather than splitting text. -`TextOutput.citations` holds `WebCitation` ranges local to each text part. `CitationDelta` adds attribution, and the final snapshot is authoritative. `defer_citations` tells block-based projectors to wait for that snapshot. The Responses normalizer and Messages projector buffer unresolved search or citation parts in start order, allowing ordinary parts to stream when they reach the front. Projection assigns output indices because parts can expand or disappear. +`TextOutput.citations` holds `WebCitation` ranges local to each text part. `CitationDelta` adds attribution, and the final snapshot is authoritative. The Messages projector buffers unresolved search parts in start order, and buffers text parts when `citation_policy` is `buffer`, allowing other parts to stream when they reach the front. Projection assigns output indices because parts can expand or disappear. No client predicts whether a response will carry citations; only the caller selects that trade. The [hosted-search contract](../docs/web-search.md) defines supported routes, reductions, failures, usage, and continuation. @@ -409,6 +409,9 @@ The Chat client normalizes token usage and the Responses adapter projects it as: | `output_tokens` | `completion_tokens` | `output_tokens` | | `output_tokens_details.reasoning_tokens` | `completion_tokens_details.reasoning_tokens` | same semantic field | | `total_tokens` | `total_tokens` | `total_tokens` | +| `web_search_requests` | `tool_usage.web_search.num_requests` | no Responses field | + +A hosted-search count is projected into Messages `usage.server_tool_use.web_search_requests` when the normalized request declares a search tool or the observed count is nonzero, because Messages reports server-tool usage only where server tools apply. `web_fetch_requests` is `0` in that search-only scope. A provider that reports no token usage reports no search count either. Projection builds the vendor usage object explicitly. Dumping an `ExtraModel` directly would expose its internal `extra_fields` container at the public boundary. diff --git a/python/CONVENTIONS.md b/python/CONVENTIONS.md index 4769c0e..bf03b9c 100644 --- a/python/CONVENTIONS.md +++ b/python/CONVENTIONS.md @@ -16,7 +16,7 @@ - Upstream clients construct provider requests, invoke provider SDK transport, apply compatibility validation where required, and normalize provider output. They must not format downstream responses. - Type modules define schemas and validation. Normalized types must not contain protocol conversions, select routes, or instantiate clients. -Each protocol adapter and upstream client translates individual tool declarations through its `_translate_tool` method. Keep request assembly, selection, call/result linking, buffering, and stream lifecycle in those layers; normalized models contain only schemas and provider-neutral validation. +Each protocol adapter and upstream client translates individual tool declarations through its `_translate_tool` method. Keep request assembly, selection, call/result linking, buffering, and stream lifecycle in those layers; normalized models contain only schemas and provider-neutral validation. Hosted-search wire conversions that both the adapter and the client of one protocol need live in `molto/utils/web_search.py`; do not copy them into either layer, and do not add conversions for other protocols there. A future `chat()` method is a dispatcher over those coordinators. It should select and delegate to a protocol method rather than duplicate that method's validation, normalization, composition, or projection logic. diff --git a/python/molto/adapters/anthropic.py b/python/molto/adapters/anthropic.py index e07ca44..657319b 100644 --- a/python/molto/adapters/anthropic.py +++ b/python/molto/adapters/anthropic.py @@ -9,7 +9,7 @@ from collections import deque from collections.abc import AsyncIterator from contextlib import aclosing -from typing import Any, NoReturn +from typing import Any, Literal, NoReturn, TypeAlias from anthropic import APIError as AnthropicAPIError from anthropic import APIStatusError as AnthropicAPIStatusError @@ -19,6 +19,7 @@ from pydantic_core import from_json from molto.adapters.base import BaseAdapter +from molto.providers import ProviderCapabilities from molto.types.anthropic_messages import ( AnthropicMessagesCreateRequest, AnthropicMessagesResponse, @@ -86,8 +87,20 @@ "stop": "end_turn", "stop_sequence": "stop_sequence", "tool_call": "tool_use", + # Messages cannot report an unfinished generation without a cause, so an + # unknown source reason is coerced to normal completion rather than pretending + # the turn paused. + "unknown": "end_turn", } +CitationPolicy: TypeAlias = Literal["buffer", "drop"] +"""How a block-based target projects text that may carry URL citations. + +`buffer` holds every text part until its citation snapshot is final so blocks can +be split at citation boundaries, trading text latency and chunk boundaries for +exact attribution. `drop` streams text immediately and omits citations. +""" + _EXTENDED_THINKING_EFFORTS: tuple[tuple[int, ReasoningEffort], ...] = ( (1024, "minimal"), (2048, "low"), @@ -157,16 +170,20 @@ def _citations(citations: list[dict[str, Any]] | None, text: str) -> list[WebCit async def _ordered_parts( events: AsyncIterator[ModelResponseStreamEvent], + *, + buffer_text: bool, ) -> AsyncIterator[ModelResponseStreamEvent]: """Stream parts in start order, yielding only the end for snapshot-based parts.""" - # Responses may stream "Alpha. Beta." before citing only "Beta.". Messages - # needs separate uncited/cited blocks; an emitted block cannot be split later. + # A block-based target cannot split text it has already emitted, and Responses + # may stream "Alpha. Beta." before citing only "Beta.". Buffering every text + # part therefore trades text latency for exact attribution; the caller selects + # that trade with `citation_policy`. # Search actions and queries can also arrive late: the final snapshot tells # us to expand a query batch into searches or omit browsing. # If part A expands into two blocks, part B must follow both, so later parts # wait to preserve order and receive the correct consecutive block indices. - # Ordinary parts stream their chunks at the FIFO head; citation/search parts - # wait for their final snapshots, without buffering the whole response. + # Ordinary parts stream their chunks at the FIFO head; snapshot parts wait for + # their final snapshots, without buffering the whole response. pending: dict[int, deque[ModelResponseStreamEvent]] = {} snapshots: set[int] = set() async for event in events: @@ -179,7 +196,7 @@ async def _ordered_parts( pending[event.index] = deque() part = event.part if isinstance(part, (SearchCall, SearchResult)) or ( - isinstance(part, TextOutput) and (part.defer_citations or part.citations) + buffer_text and isinstance(part, TextOutput) ): snapshots.add(event.index) if isinstance(event, PartEndEvent): @@ -212,6 +229,15 @@ class AnthropicMessagesAdapter( ): """Translate the downstream Anthropic Messages protocol to and from normalized IR.""" + def __init__( + self, + capabilities: ProviderCapabilities, + *, + citation_policy: CitationPolicy = "drop", + ) -> None: + super().__init__(capabilities) + self._citation_policy = citation_policy + @staticmethod def _translate_tool(tool: dict[str, Any]) -> FunctionTool | WebSearchTool: """Translate a Messages tool declaration into its normalized schema.""" @@ -627,7 +653,9 @@ def event(data: dict[str, Any]) -> AnthropicMessagesStreamEvent: thinking_parts: set[int] = set() thinking_signatures: dict[int, str] = {} - async with aclosing(_ordered_parts(events)) as ordered_events: + async with aclosing( + _ordered_parts(events, buffer_text=self._citation_policy == "buffer") + ) as ordered_events: async for normalized_event in ordered_events: if isinstance(normalized_event, ResponseStartEvent): yield event( @@ -712,9 +740,10 @@ def event(data: dict[str, Any]) -> AnthropicMessagesStreamEvent: elif isinstance(normalized_event, PartDeltaEvent): index = indices[normalized_event.index] if isinstance(normalized_event.delta, CitationDelta): - raise ValueError( - "Citation-bearing text must declare deferred citations before streaming." - ) + # Buffered parts consume their citation deltas in + # `_ordered_parts`; in the `drop` policy no text part is + # buffered, so citations are omitted deliberately. + continue if isinstance(normalized_event.delta, ThinkingDelta): delta = normalized_event.delta if index not in thinking_parts: @@ -866,10 +895,13 @@ def event(data: dict[str, Any]) -> AnthropicMessagesStreamEvent: "thinking_tokens": usage.output_tokens_details.reasoning_tokens }, } - search_requests = normalized_event.extra_fields.get("web_search_requests") - if search_requests is None and usage is not None: - search_requests = usage.extra_fields.get("web_search_requests") - if search_requests is not None: + search_requests = None if usage is None else usage.web_search_requests + declares_search = any( + isinstance(tool, WebSearchTool) for tool in request.tools or [] + ) + # Messages reports server-tool usage only where server tools apply, + # so an observed zero is projected only for a search request. + if search_requests is not None and (declares_search or search_requests > 0): raw_usage["server_tool_use"] = { "web_search_requests": search_requests, "web_fetch_requests": 0, diff --git a/python/molto/adapters/openai.py b/python/molto/adapters/openai.py index d4cd820..ca1058a 100644 --- a/python/molto/adapters/openai.py +++ b/python/molto/adapters/openai.py @@ -91,6 +91,9 @@ "stop": "stop", "stop_sequence": "stop", "tool_call": "tool_calls", + # Chat has no incomplete state, so an unknown source reason is coerced to normal + # completion rather than claiming the output limit was reached. + "unknown": "stop", } @@ -1101,7 +1104,12 @@ def response_data( } ) search_calls.clear() - if termination_reason in {"content_filter", "max_output_tokens", "pause"}: + if termination_reason in { + "content_filter", + "max_output_tokens", + "pause", + "unknown", + }: event_type = "response.incomplete" status = "incomplete" else: @@ -1114,7 +1122,9 @@ def response_data( ) if status == "incomplete": final_response["incomplete_details"] = ( - None if termination_reason == "pause" else {"reason": termination_reason} + None + if termination_reason in {"pause", "unknown"} + else {"reason": termination_reason} ) yield event( { @@ -1371,7 +1381,6 @@ async def _project_events( response_start: ResponseStartEvent | None = None initial_chunk_sent = False tool_indices: dict[int, int] = {} - search_enabled = any(isinstance(tool, WebSearchTool) for tool in request.tools or []) def chunk( metadata: ResponseStartEvent | ResponseEndEvent, @@ -1410,19 +1419,10 @@ def delta_chunk( async for normalized_event in events: if isinstance(normalized_event, ResponseStartEvent): response_start = normalized_event - if normalized_event.web_search_enabled is not None: - search_enabled = normalized_event.web_search_enabled elif isinstance(normalized_event, PartStartEvent): if isinstance(normalized_event.part, (SearchCall, SearchResult)): continue - if ( - isinstance(normalized_event.part, Thinking) - and not normalized_event.part.content - and not normalized_event.part.summary - and search_enabled - ): - continue if not isinstance(normalized_event.part, (Thinking, TextOutput, ToolCall)): raise ValueError(f"Unsupported output part: {type(normalized_event.part)}") if response_start is None: @@ -1493,14 +1493,6 @@ def delta_chunk( elif isinstance(normalized_event, PartDeltaEvent): if isinstance(normalized_event.delta, CitationDelta): continue - if ( - isinstance(normalized_event.delta, ThinkingDelta) - and normalized_event.delta.signature_delta is not None - and not normalized_event.delta.content_delta - and not normalized_event.delta.summary_delta - and search_enabled - ): - continue if response_start is None: raise ValueError("Received a delta before its response started.") if isinstance(normalized_event.delta, ThinkingDelta): @@ -1559,13 +1551,6 @@ def delta_chunk( elif isinstance(normalized_event, PartEndEvent): if isinstance(normalized_event.part, (SearchCall, SearchResult)): continue - if ( - isinstance(normalized_event.part, Thinking) - and not normalized_event.part.content - and not normalized_event.part.summary - and search_enabled - ): - continue if not isinstance(normalized_event.part, (Thinking, TextOutput, ToolCall)): raise ValueError(f"Unsupported output part: {type(normalized_event.part)}") if isinstance(normalized_event.part, Thinking): @@ -1604,11 +1589,6 @@ def delta_chunk( ), }, } - if not initial_chunk_sent and search_enabled: - yield delta_chunk( - normalized_event, {"role": "assistant", "content": "", "refusal": None} - ) - initial_chunk_sent = True yield chunk( normalized_event, choices=[ diff --git a/python/molto/core.py b/python/molto/core.py index c47e5cc..a1b86fc 100644 --- a/python/molto/core.py +++ b/python/molto/core.py @@ -4,7 +4,7 @@ from httpx import AsyncClient, Headers from pydantic import BaseModel -from molto.adapters.anthropic import AnthropicMessagesAdapter +from molto.adapters.anthropic import AnthropicMessagesAdapter, CitationPolicy from molto.adapters.openai import OpenAIChatCompletionsAdapter, OpenAIResponsesAdapter from molto.configs import ENV_CONFIG from molto.providers import ( @@ -337,12 +337,16 @@ async def messages( base_url: str | None = None, extra_body: Mapping[str, Any] | None = None, extra_headers: Mapping[str, str] | None = None, + citation_policy: CitationPolicy = "drop", ) -> MoltoStream[AnthropicMessagesStreamEvent] | AnthropicMessagesResponse: """Consume an Anthropic Messages request using any supported upstream protocol. `endpoint` selects the wire protocol, `provider` selects the serving implementation and its capabilities, and `base_url` selects only the - HTTP destination. + HTTP destination. `citation_policy` selects how translated responses + project text that may carry URL citations: `buffer` withholds each text + part until its citations are final so blocks can be split at citation + boundaries, and `drop` streams text immediately without citations. """ request = AnthropicMessagesCreateRequest.model_validate(body) resolved_provider = self._resolve_provider(endpoint, provider) @@ -366,7 +370,7 @@ async def messages( ) return await adapter.project_native_response(upstream.native_response(request)) - adapter = AnthropicMessagesAdapter(capabilities) + adapter = AnthropicMessagesAdapter(capabilities, citation_policy=citation_policy) normalized_request = adapter.normalize_request(request) if endpoint == "responses": upstream = OpenAIResponsesClient( diff --git a/python/molto/types/normalized.py b/python/molto/types/normalized.py index 9c90643..f9b2c3e 100644 --- a/python/molto/types/normalized.py +++ b/python/molto/types/normalized.py @@ -328,10 +328,6 @@ class TextOutput(ExtraModel): citations: Annotated[ list[WebCitation], Field(description="URL attributions local to this text part.") ] = [] - defer_citations: Annotated[ - bool, - Field(description="Delay block-based projection until the citation snapshot is complete."), - ] = False content_type: Annotated[ Literal["text-output"], Field(description="Content type identifier as a discriminator."), @@ -413,9 +409,8 @@ def _validate_representation(self) -> Self: def _reject_input_extras(model_type: type[ExtraModel]) -> Callable[[Any], Any]: def validate(value: Any) -> Any: if isinstance(value, dict): - extra_fields = { - key for key in value if key not in model_type.model_fields or key == "extra_fields" - } + allowed_fields = model_type.model_fields.keys() - {"extra_fields"} + extra_fields = value.keys() - allowed_fields elif isinstance(value, model_type): extra_fields = value.extra_fields.keys() else: @@ -830,6 +825,10 @@ class Usage(ExtraModel): ZeroIfNoneInt, Field(description="The total number of tokens used."), ] + web_search_requests: Annotated[ + int | None, + Field(description="Hosted search requests reported by the provider, when known."), + ] = None class ResponseStartEvent(ExtraModel): @@ -851,12 +850,6 @@ class ResponseStartEvent(ExtraModel): str | None, Field(description="The service tier used for the response."), ] = None - web_search_enabled: Annotated[ - bool | None, - Field( - description="Whether the effective upstream request enables hosted web search, when known." - ), - ] = None event_kind: Annotated[ Literal["response_start"], Field(description="Event type identifier, used as a discriminator."), @@ -872,6 +865,9 @@ class ResponseStartEvent(ExtraModel): "stop", "stop_sequence", "tool_call", + # The source reported an incomplete generation without naming a cause. This is + # deliberately distinct from `pause`, which is a cause reported by the source. + "unknown", ] diff --git a/python/molto/upstream/anthropic.py b/python/molto/upstream/anthropic.py index 8fc9ce3..fddaf73 100644 --- a/python/molto/upstream/anthropic.py +++ b/python/molto/upstream/anthropic.py @@ -437,7 +437,7 @@ def _assistant_blocks( for citation in item.citations ): raise UnsupportedInput( - "Citation ranges must within the associated answer text." + "Citation ranges must be within the associated answer text." ) text_blocks = messages_text_blocks(item) if item.cache_breakpoint is not None: @@ -687,8 +687,6 @@ def start_part( native_request = AnthropicMessagesCreateRequest.model_validate( self._request_body(request, capabilities=self._capabilities) ) - tools = (self._extra_body or {}).get("tools", native_request.tools) or [] - search_enabled = any(tool.get("type") == "web_search_20250305" for tool in tools) async with aclosing(self.native_events(native_request)) as native_events: async for event in native_events: if isinstance(event, RawMessageStartEvent): @@ -697,7 +695,6 @@ def start_part( id=message.id, model=message.model, service_tier=message.usage.service_tier, - web_search_enabled=search_enabled, ) input_tokens = message.usage.input_tokens cache_creation = message.usage.cache_creation_input_tokens or 0 @@ -846,8 +843,12 @@ def start_part( if state.content else initial_input ) + if not isinstance(search_input, dict): + # An interrupted or malformed input is provider output, so + # fall back to the observed snapshot instead of rejecting it. + search_input = initial_input if isinstance(initial_input, dict) else {} completed_part = search_call_from_messages( - {"id": part.id, "input": search_input} + {"id": part.id, "input": search_input}, strict=False ) yield start_part(event.index, completed_part) elif isinstance(part, SearchResult): @@ -908,9 +909,8 @@ def start_part( + cache_creation + cache_read + raw_usage.output_tokens, + web_search_requests=search_usage, ) - if search_usage is not None: - usage.extra_fields["anthropic_web_search_requests"] = search_usage elif isinstance(event, RawMessageStopEvent): if metadata is None: raise ValueError("Anthropic stream ended before its message started.") diff --git a/python/molto/upstream/openai.py b/python/molto/upstream/openai.py index 3170dc6..a24764a 100644 --- a/python/molto/upstream/openai.py +++ b/python/molto/upstream/openai.py @@ -669,18 +669,7 @@ def _request_body(cls, request: NormalizedRequest) -> dict[str, Any]: raise UnsupportedInput(str(error)) from error async def _events(self, request: NormalizedRequest) -> AsyncIterator[ModelResponseStreamEvent]: - request_body = self._request_body(request) - native_request = OpenAIResponsesCreateRequest.model_validate(request_body) - # SDK extra_body can introduce or replace the projected tool declarations. - tools = (self._extra_body or {}).get("tools", native_request.tools) or [] - search_enabled = any(tool["type"] == "web_search" for tool in tools) - if search_enabled and self._extra_body is not None: - # SDK extra_body wins over typed request parameters. Retain caller - # include entries while keeping search sources enabled on translation. - includes = ["web_search_call.action.sources", *(native_request.include or [])] - includes.extend(self._extra_body.get("include") or []) - unique_includes = list(dict.fromkeys(includes)) - self._extra_body = {**self._extra_body, "include": unique_includes} + native_request = OpenAIResponsesCreateRequest.model_validate(self._request_body(request)) # Only a reasoning item whose `added` snapshot is completely empty (no # content, summary, or encrypted state) is deferred: if nothing ever # arrives, its part lifecycle is omitted from the IR entirely. An item @@ -694,22 +683,6 @@ async def _events(self, request: NormalizedRequest) -> AsyncIterator[ModelRespon started_text_items: set[int] = set() ended_text_items: set[int] = set() next_index = 0 - # SDK overrides replace history as well as tools; buffering must reflect - # the input actually sent upstream, including cited follow-ups without tools. - input_items = (self._extra_body or {}).get("input", request_body.get("input")) - defer_citations = search_enabled or ( - isinstance(input_items, list) - and any( - item.get("type") == "web_search_call" - or any( - isinstance(part, dict) - and part.get("type") == "output_text" - and part.get("annotations") - for part in item.get("content") or [] - ) - for item in input_items - ) - ) def start_part( source_index: int, part: Thinking | TextOutput | ToolCall | SearchCall | SearchResult @@ -749,12 +722,9 @@ def part_index(source_index: int) -> int: created_at=int(metadata.created_at), model=metadata.model, service_tier=metadata.service_tier, - web_search_enabled=search_enabled, ) elif event_type == "response.output_item.added": part = _started_output_item(event.item) - if isinstance(part, TextOutput): - part.defer_citations = defer_citations if isinstance(part, Thinking): _validate_responses_reasoning_capability(part, self._capabilities.thinking) if ( @@ -817,9 +787,7 @@ def part_index(source_index: int) -> int: text_indices[key] = index yield PartStartEvent( index=index, - part=TextOutput( - content="", id=event.item_id, defer_citations=defer_citations - ), + part=TextOutput(content="", id=event.item_id), ) if added_part.text: yield PartDeltaEvent( @@ -881,7 +849,6 @@ def part_index(source_index: int) -> int: part=TextOutput( content=event.part.text, id=event.item_id, - defer_citations=defer_citations, citations=[ citation_from_responses(annotation.model_dump()) for annotation in event.part.annotations @@ -928,23 +895,21 @@ def part_index(source_index: int) -> int: ) else: incomplete_details = metadata.incomplete_details + # A reason-less incomplete response does not establish a pause; + # the normalized stream keeps the cause unknown. if incomplete_details is None or incomplete_details.reason is None: - termination_reason = "pause" + termination_reason = "unknown" else: termination_reason = incomplete_details.reason usage = _responses_usage(metadata.usage) - tool_usage = getattr(metadata, "tool_usage", None) + tool_usage = metadata.model_dump().get("tool_usage") search_usage = ( (tool_usage.get("web_search") or {}) if isinstance(tool_usage, dict) else {} ) - if ( - defer_citations - and usage is not None - and search_usage.get("num_requests") is not None - ): - usage.extra_fields["web_search_requests"] = search_usage["num_requests"] + if usage is not None and search_usage.get("num_requests") is not None: + usage.web_search_requests = search_usage["num_requests"] yield ResponseEndEvent( id=metadata.id, created_at=int(metadata.created_at), @@ -952,11 +917,6 @@ def part_index(source_index: int) -> int: service_tier=metadata.service_tier, termination_reason=termination_reason, usage=usage, - **( - {"web_search_requests": search_usage["num_requests"]} - if defer_citations and search_usage.get("num_requests") is not None - else {} - ), ) elif event_type in _IGNORED_RESPONSES_EVENTS: continue @@ -1328,10 +1288,6 @@ def _validate_request(request: NormalizedRequest) -> None: raise UnsupportedInput("Chat Completions cannot represent tool return names.") async def _events(self, request: NormalizedRequest) -> AsyncIterator[ModelResponseStreamEvent]: - if (self._extra_body or {}).get("web_search_options") is not None: - raise UnsupportedInput( - "Hosted web search cannot use Chat Completions upstream; select Responses or Messages." - ) response_metadata: ResponseStartEvent | None = None part_text = "" usage: Usage | None = None diff --git a/python/molto/utils/test.py b/python/molto/utils/test.py index 2ce04a6..6ba8c80 100644 --- a/python/molto/utils/test.py +++ b/python/molto/utils/test.py @@ -52,7 +52,7 @@ def assert_contains_expected(actual: Any, expected: Any, path: str = "$") -> Non """Assert that actual recursively contains the expected value.""" if isinstance(expected, dict): assert isinstance(actual, dict), f"Expected an object at {path}, got {type(actual)}" - missing_keys = {key for key in expected if key not in actual} + missing_keys = expected.keys() - actual.keys() assert not missing_keys, f"Missing keys at {path}: {sorted(missing_keys)}" for key, expected_value in expected.items(): assert_contains_expected(actual[key], expected_value, f"{path}.{key}") diff --git a/python/molto/utils/web_search.py b/python/molto/utils/web_search.py index bae57ce..b09a0f7 100644 --- a/python/molto/utils/web_search.py +++ b/python/molto/utils/web_search.py @@ -16,10 +16,24 @@ def messages_text_blocks( part: TextOutput, ) -> list[dict[str, Any]]: - """Split text at citation boundaries while preserving attribution.""" + """Split text at citation boundaries into blocks Anthropic accepts. + + Citation boundaries fall on character offsets, so a boundary can isolate + whitespace or name a zero-length range. Messages rejects an empty text block + and a block without non-whitespace text, so whitespace-only ranges are merged + into a neighbouring block and a point annotation is attached to the block that + contains its offset. Concatenating the returned blocks reproduces the part. + """ text = part.content + if not part.citations: + return [{"type": "text", "text": text}] if text else [] + boundaries = {0, len(text)} for citation in part.citations: + if citation.start_index == citation.end_index: + # A point annotation names a position, not a range, so it must not + # split the text into separate blocks. + continue boundaries.add(citation.start_index) boundaries.add(citation.end_index) @@ -29,28 +43,51 @@ def messages_text_blocks( raise ValueError("Citation end precedes its start.") ordered_boundaries = sorted(boundaries) - intervals = set(zip(ordered_boundaries, ordered_boundaries[1:], strict=False)) - # Point annotations have no answer characters but still carry attribution. - for citation in part.citations: - if citation.start_index == citation.end_index: - intervals.add((citation.start_index, citation.end_index)) + # Leading spacing has no earlier block to merge into, so it is carried into + # the first block that has content. + ranges: list[list[int]] = [] + pending_start: int | None = None + for start, end in zip(ordered_boundaries, ordered_boundaries[1:], strict=False): + if start == end: + continue + if not text[start:end].strip(): + if ranges: + ranges[-1][1] = end + else: + pending_start = start + continue + ranges.append([start if pending_start is None else pending_start, end]) + pending_start = None + + if not ranges: + # The part has no splittable content; keep it as one block rather than + # inventing attribution for a block that carries no text. + return [{"type": "text", "text": text}] if text else [] blocks: list[dict[str, Any]] = [] - for start, end in sorted(intervals): + for index, (start, end) in enumerate(ranges): block: dict[str, Any] = {"type": "text", "text": text[start:end]} - block_citations = [] - for citation in part.citations: - if start == end: - matches = citation.start_index == start and citation.end_index == end - else: - matches = citation.start_index <= start and end <= citation.end_index - if matches: - block_citations.append(citation_to_messages(citation, text=text)) - + block_citations = [ + citation_to_messages(citation, text=text, block_start=start, block_end=end) + for citation in part.citations + if _cites_block(citation, start=start, end=end, is_last=index == len(ranges) - 1) + ] if block_citations: block["citations"] = block_citations blocks.append(block) - return blocks or [{"type": "text", "text": text}] + return blocks + + +def _cites_block(citation: WebCitation, *, start: int, end: int, is_last: bool) -> bool: + """Report whether an attribution applies to the block spanning `start`..`end`.""" + if citation.start_index != citation.end_index: + # A block may extend past the cited range when surrounding whitespace was + # folded into it, so any overlap associates the attribution. + return start < citation.end_index and citation.start_index < end + if start <= citation.start_index < end: + return True + # A point annotation at the end of the text belongs to the final block. + return is_last and citation.start_index == end def citation_from_responses(annotation: dict[str, Any]) -> WebCitation: @@ -86,15 +123,20 @@ def citation_to_responses(citation: WebCitation) -> dict[str, Any]: } -def citation_to_messages(citation: WebCitation, *, text: str) -> dict[str, Any]: +def citation_to_messages( + citation: WebCitation, *, text: str, block_start: int, block_end: int +) -> dict[str, Any]: """Build a Messages citation with the documented answer-range/state fallbacks.""" + cited_text = citation.cited_text + if cited_text is None: + # A point annotation has no answer characters of its own, so its excerpt + # falls back to the block that carries it rather than to an empty string. + cited_text = text[citation.start_index : citation.end_index] or text[block_start:block_end] return { "type": "web_search_result_location", "url": citation.url, "title": citation.title if citation.title is not None else "", - "cited_text": citation.cited_text - if citation.cited_text is not None - else text[citation.start_index : citation.end_index], + "cited_text": cited_text, "encrypted_index": citation.encrypted_index if citation.encrypted_index is not None else "N/A", @@ -150,10 +192,19 @@ def search_call_from_responses(item: dict[str, Any]) -> SearchCall: ) -def search_call_from_messages(block: dict[str, Any]) -> SearchCall: - """Convert Messages server-search input after its arguments have been collected.""" - search_input = block["input"] - if not isinstance(search_input, dict) or set(search_input) - {"query"}: +def search_call_from_messages(block: dict[str, Any], *, strict: bool = True) -> SearchCall: + """Convert Messages server-search input after its arguments have been collected. + + `strict` validates caller-supplied history. Provider output is normalized with + `strict=False` so an unknown or partially parsed input shape cannot turn a + provider defect into a request-validation error. + """ + search_input = block.get("input") + if not isinstance(search_input, dict): + if strict: + raise UnsupportedInput("Unsupported Anthropic web search input fields.") + return SearchCall(id=block["id"]) + if strict and set(search_input) - {"query"}: raise UnsupportedInput("Unsupported Anthropic web search input fields.") return SearchCall(id=block["id"], query=search_input.get("query")) diff --git a/scripts/match_anthropic_cassette_events.py b/scripts/match_anthropic_cassette_events.py index f8abce8..d97ce9e 100644 --- a/scripts/match_anthropic_cassette_events.py +++ b/scripts/match_anthropic_cassette_events.py @@ -591,11 +591,10 @@ def _response_terminal( return response, "end_turn" incomplete_details = response.get("incomplete_details") - if incomplete_details is None and ( - has_search - or any(item.get("type") == "web_search_call" for item in response.get("output", [])) - ): - return response, "pause_turn" + # A reason-less incomplete response does not establish a pause, so the + # translated stream reports the normalized unknown reason as normal completion. + if incomplete_details is None or incomplete_details.get("reason") is None: + return response, "end_turn" if not isinstance(incomplete_details, dict): raise ValueError("response.incomplete must contain incomplete_details.") reason = incomplete_details.get("reason") diff --git a/scripts/match_anthropic_cassette_responses.py b/scripts/match_anthropic_cassette_responses.py index dad0d72..905c66b 100644 --- a/scripts/match_anthropic_cassette_responses.py +++ b/scripts/match_anthropic_cassette_responses.py @@ -680,8 +680,10 @@ def _stop_reason( raise ValueError("Responses response status must be completed or incomplete.") incomplete_details = response.get("incomplete_details") - if incomplete_details is None and has_search: - return "pause_turn" + # A reason-less incomplete response is normalized as unknown and coerced to + # normal completion; it is not evidence that the turn paused. + if incomplete_details is None or incomplete_details.get("reason") is None: + return "end_turn" if not isinstance(incomplete_details, dict): raise ValueError("Incomplete Responses responses must contain incomplete_details.") if incomplete_details.get("reason") == "max_output_tokens": diff --git a/scripts/match_openai_cassette_events.py b/scripts/match_openai_cassette_events.py index 84753ab..f207d92 100644 --- a/scripts/match_openai_cassette_events.py +++ b/scripts/match_openai_cassette_events.py @@ -377,8 +377,10 @@ def _response_terminal( return response, "stop" incomplete_details = response.get("incomplete_details") - if has_search and incomplete_details is None: - return response, "length" + # A reason-less incomplete response is normalized as unknown, which Chat + # coerces to normal completion rather than to the output-token limit. + if incomplete_details is None or incomplete_details.get("reason") is None: + return response, "stop" if not isinstance(incomplete_details, dict): raise ValueError("response.incomplete must contain incomplete_details.") reason = incomplete_details.get("reason") diff --git a/scripts/match_openai_cassette_responses.py b/scripts/match_openai_cassette_responses.py index d4a1618..1991dbf 100644 --- a/scripts/match_openai_cassette_responses.py +++ b/scripts/match_openai_cassette_responses.py @@ -9,7 +9,8 @@ Hosted-search fixtures cover Chat downstream only. Hosted calls and structured URL citations are omitted while actual answer text and inline links are retained. -Signature-only search reasoning is omitted because Chat has no corresponding state. +Encrypted or signed reasoning is not representable in Chat Completions, so a +hosted-search turn that carries it is rejected rather than omitted. """ from __future__ import annotations @@ -250,13 +251,6 @@ def _response_output( if item_type == "web_search_call": continue if item_type == "reasoning": - if ( - has_search - and item.get("encrypted_content") is not None - and item.get("content") in (None, []) - and item.get("summary") in (None, []) - ): - continue if reasoning_content is not None: raise ValueError( "Multiple Responses reasoning items cannot map to Chat Completions." @@ -315,8 +309,10 @@ def _finish_reason( raise ValueError("Responses response status must be completed or incomplete.") incomplete_details = response.get("incomplete_details") - if has_search and incomplete_details is None: - return "length" + # A reason-less incomplete response is normalized as unknown, which Chat + # coerces to normal completion rather than to the output-token limit. + if incomplete_details is None or incomplete_details.get("reason") is None: + return "stop" if not isinstance(incomplete_details, dict): raise ValueError("Incomplete Responses responses must contain incomplete_details.") return { diff --git a/tests/adapters/test_anthropic_search.py b/tests/adapters/test_anthropic_search.py index 5d505ff..8d1a39f 100644 --- a/tests/adapters/test_anthropic_search.py +++ b/tests/adapters/test_anthropic_search.py @@ -31,16 +31,19 @@ ) -def setup_adapter(): - adapter = AnthropicMessagesAdapter(PROVIDER_CAPABILITIES["anthropic"]) - request = adapter.normalize_request( - AnthropicMessagesCreateRequest( - model="m", - max_tokens=100, - messages=[{"role": "user", "content": "Search"}], - stream=True, - ) +def setup_adapter(*, search: bool = False, citation_policy: str = "buffer"): + adapter = AnthropicMessagesAdapter( + PROVIDER_CAPABILITIES["anthropic"], citation_policy=citation_policy ) + body = { + "model": "m", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Search"}], + "stream": True, + } + if search: + body["tools"] = [{"type": "web_search_20250305", "name": "web_search"}] + request = adapter.normalize_request(AnthropicMessagesCreateRequest(**body)) return adapter, request @@ -235,11 +238,10 @@ async def test_buffered_citations_search_omission_and_collection(): events += [PartStartEvent(index=index, part=part), PartEndEvent(index=index, part=part)] text = TextOutput( content="Alpha. Beta.", - defer_citations=True, citations=[WebCitation(url="https://a", start_index=7, end_index=12)], ) events += [ - PartStartEvent(index=4, part=TextOutput(content="", defer_citations=True)), + PartStartEvent(index=4, part=TextOutput(content="")), PartDeltaEvent(index=4, delta=TextDelta(content_delta=text.content)), PartEndEvent(index=4, part=text), ResponseEndEvent(id="r", model="m", created_at=0, termination_reason="pause"), @@ -302,16 +304,15 @@ async def source(): @pytest.mark.parametrize("has_token_usage", [False, True]) async def test_search_usage_observation(count, has_token_usage): """Ensures unknown usage remains unknown while an explicitly reported zero survives.""" - adapter, request = setup_adapter() + adapter, request = setup_adapter(search=True) usage = Usage( input_tokens=1, input_tokens_details=InputTokenDetails(), output_tokens=1, output_tokens_details=OutputTokenDetails(), total_tokens=2, + web_search_requests=count, ) - if count is not None: - usage.extra_fields["web_search_requests"] = count end = ResponseEndEvent( id="r", model="m", @@ -319,13 +320,12 @@ async def test_search_usage_observation(count, has_token_usage): termination_reason="stop", usage=usage if has_token_usage else None, ) - if not has_token_usage and count is not None: - end.extra_fields["web_search_requests"] = count events = [ResponseStartEvent(id="r", model="m"), end] response = await adapter.project_response( adapter.project_events(iterate(events), request), request ) - if count is None: + if count is None or not has_token_usage: + # A provider that reports no token usage reports no search count either. assert response.usage.server_tool_use is None else: assert response.usage.server_tool_use.web_search_requests == count @@ -333,15 +333,28 @@ async def test_search_usage_observation(count, has_token_usage): def test_point_citation_preserves_attribution(): - """Ensures an empty range keeps its source without associating unrelated text.""" + """Ensures an empty range keeps its source without emitting an empty block.""" blocks = messages_text_blocks( TextOutput( content="ab", citations=[WebCitation(url="https://x", start_index=1, end_index=1)] ) ) - assert [block["text"] for block in blocks] == ["a", "", "b"] - assert blocks[1]["citations"][0]["cited_text"] == "" - assert "citations" not in blocks[0] and "citations" not in blocks[2] + assert [block["text"] for block in blocks] == ["ab"] + assert blocks[0]["citations"][0]["cited_text"] == "ab" + + +def test_whitespace_gap_does_not_emit_a_spacing_block(): + """Ensures a citation boundary on spacing cannot produce a block Messages rejects.""" + blocks = messages_text_blocks( + TextOutput( + content="ab cd", + citations=[WebCitation(url="https://x", start_index=2, end_index=3)], + ) + ) + assert [block["text"] for block in blocks] == ["ab ", "cd"] + assert [block["citations"][0]["cited_text"] for block in blocks if "citations" in block] == [ + " " + ] @pytest.mark.parametrize("kind", ["search", "deferred_text"]) @@ -349,11 +362,7 @@ async def test_early_close_releases_source_after_buffered_projection(kind): """Ensures closing a buffered block start deterministically releases upstream input.""" adapter, request = setup_adapter() closed = False - part = ( - SearchCall(id="s", query="q") - if kind == "search" - else TextOutput(content="answer", defer_citations=True) - ) + part = SearchCall(id="s", query="q") if kind == "search" else TextOutput(content="answer") async def source(): nonlocal closed diff --git a/tests/adapters/test_common.py b/tests/adapters/test_common.py new file mode 100644 index 0000000..43c5f2a --- /dev/null +++ b/tests/adapters/test_common.py @@ -0,0 +1,111 @@ +"""Cross-adapter projection of normalized terminal and usage state. + +One normalized stream must project into three downstream protocols, so the +behaviour for a reason the target cannot express belongs to a single table here +rather than to each adapter's own tests. +""" + +import pytest + +from molto.adapters.anthropic import AnthropicMessagesAdapter +from molto.adapters.openai import OpenAIChatCompletionsAdapter, OpenAIResponsesAdapter +from molto.providers import PROVIDER_CAPABILITIES +from molto.types.anthropic_messages import AnthropicMessagesCreateRequest +from molto.types.normalized import ( + PartEndEvent, + PartStartEvent, + ResponseEndEvent, + ResponseStartEvent, + TerminationReason, + TextOutput, +) +from molto.types.openai_chat_completions import OpenAIChatCompletionsCreateRequest +from molto.types.openai_responses import OpenAIResponsesCreateRequest + +CAPABILITIES = PROVIDER_CAPABILITIES["openai"] + + +async def _events(reason: TerminationReason): + yield ResponseStartEvent(id="r", created_at=1, model="m") + part = TextOutput(content="answer") + yield PartStartEvent(index=0, part=part) + yield PartEndEvent(index=0, part=part) + yield ResponseEndEvent(id="r", created_at=1, model="m", termination_reason=reason) + + +async def _chat(reason: TerminationReason): + adapter = OpenAIChatCompletionsAdapter(CAPABILITIES) + request = adapter.normalize_request( + OpenAIChatCompletionsCreateRequest( + model="m", messages=[{"role": "user", "content": "q"}], stream=True + ) + ) + events = [event async for event in adapter.project_events(_events(reason), request)] + return next( + choice.finish_reason + for event in events + for choice in event.choices + if choice.finish_reason + ) + + +async def _messages(reason: TerminationReason): + adapter = AnthropicMessagesAdapter(CAPABILITIES) + request = adapter.normalize_request( + AnthropicMessagesCreateRequest( + model="m", + max_tokens=10, + messages=[{"role": "user", "content": "q"}], + stream=True, + ) + ) + events = [event async for event in adapter.project_events(_events(reason), request)] + return next(event.delta.stop_reason for event in events if event.type == "message_delta") + + +async def _responses(reason: TerminationReason): + adapter = OpenAIResponsesAdapter(CAPABILITIES) + request = adapter.normalize_request(OpenAIResponsesCreateRequest(model="m", input="q")) + events = [event async for event in adapter.project_events(_events(reason), request)] + terminal = events[-1] + assert terminal.type in {"response.completed", "response.incomplete"} + return terminal.response.status, terminal.response.incomplete_details + + +@pytest.mark.parametrize( + ("reason", "chat", "messages"), + [ + ("stop", "stop", "end_turn"), + ("max_output_tokens", "length", "max_tokens"), + ("content_filter", "content_filter", "end_turn"), + ("pause", "length", "pause_turn"), + # Unknown is not a pause: a target without an incomplete state reports + # normal completion rather than the output limit or a continuation hint. + ("unknown", "stop", "end_turn"), + ("refusal", "stop", "refusal"), + ], +) +async def test_terminal_reason_projects_into_chat_and_messages( + reason: TerminationReason, chat: str, messages: str +): + """Ensures each normalized terminal reason maps to the documented target value.""" + assert await _chat(reason) == chat + assert await _messages(reason) == messages + + +@pytest.mark.parametrize( + ("reason", "status", "details"), + [ + ("stop", "completed", None), + ("max_output_tokens", "incomplete", {"reason": "max_output_tokens"}), + ("content_filter", "incomplete", {"reason": "content_filter"}), + ("pause", "incomplete", None), + ("unknown", "incomplete", None), + ], +) +async def test_terminal_reason_projects_into_responses(reason, status, details): + """Ensures Responses keeps incomplete states and leaves a reason-less cause null.""" + projected_status, incomplete_details = await _responses(reason) + assert projected_status == status + actual_details = None if incomplete_details is None else incomplete_details.model_dump() + assert actual_details == details diff --git a/tests/adapters/test_openai_search.py b/tests/adapters/test_openai_search.py index 48c490e..4e168d8 100644 --- a/tests/adapters/test_openai_search.py +++ b/tests/adapters/test_openai_search.py @@ -190,7 +190,7 @@ async def test_paused_search_remains_incomplete_without_fake_result(): @pytest.mark.parametrize("answer", ["answer", None]) async def test_chat_search_failure_omits_hosted_blocks_and_maps_pause(answer): - """Ensures Chat preserves real answer text or empty content after a failed paused search.""" + """Ensures Chat preserves real answer text and invents no content after a failed paused search.""" adapter = OpenAIChatCompletionsAdapter(CAPABILITIES) request = adapter.normalize_request( OpenAIChatCompletionsCreateRequest(model="m", messages=[], web_search_options={}) @@ -204,7 +204,7 @@ async def test_chat_search_failure_omits_hosted_blocks_and_maps_pause(answer): response = await adapter.project_response( adapter.project_events(_events(parts, "pause"), request), request ) - assert response.choices[0].message.content == (answer or "") + assert response.choices[0].message.content == answer assert response.choices[0].message.tool_calls is None assert response.choices[0].message.annotations is None assert response.choices[0].finish_reason == "length" diff --git a/tests/core/test_web_search.py b/tests/core/test_web_search.py index 67d91b7..ffb2b10 100644 --- a/tests/core/test_web_search.py +++ b/tests/core/test_web_search.py @@ -29,7 +29,6 @@ RESPONSES_MESSAGES_PROTOCOLS, Protocol, _assert_responses_stream_invariants, - _coordinator, _get_http_client, _matched_cassette_path, _read_cassette, @@ -37,6 +36,22 @@ ) SEARCH_ROUTES = tuple(product(PROTOCOLS, RESPONSES_MESSAGES_PROTOCOLS)) +# Chat cannot carry provider reasoning signatures, so a search turn that returns +# encrypted or signed reasoning is a documented rejection rather than a projection. +SIGNED_REASONING_SAFE_ROUTES = tuple( + (downstream, upstream) + for downstream, upstream in SEARCH_ROUTES + if downstream != "chat/completions" +) + + +def _coordinator(molto: Molto, protocol: Protocol): + """Select a coordinator, opting Messages into citations for search assertions.""" + if protocol == "messages": + return lambda **kwargs: molto.messages(**kwargs, citation_policy="buffer") + if protocol == "responses": + return molto.responses + return molto.chat_completions def _path(protocol: Protocol, stream: bool = True): @@ -313,7 +328,8 @@ async def test_native_search_matches_sdk( @pytest.mark.parametrize( - ("downstream", "upstream"), [route for route in SEARCH_ROUTES if route[0] != route[1]] + ("downstream", "upstream"), + [route for route in SIGNED_REASONING_SAFE_ROUTES if route[0] != route[1]], ) @pytest.mark.parametrize("stream", [True, False]) async def test_translated_search_matches_sdk( @@ -354,6 +370,22 @@ async def test_translated_search_matches_sdk( assert _chat_semantics(actual, upstream) == _chat_semantics(expected, upstream) +@pytest.mark.parametrize("upstream", RESPONSES_MESSAGES_PROTOCOLS) +@pytest.mark.parametrize("stream", [True, False]) +async def test_chat_downstream_search_rejects_signed_reasoning(upstream: Protocol, stream: bool): + """Ensures a search turn that carries provider reasoning fails instead of dropping it.""" + body, expected_request = _request_pair("chat/completions", upstream, stream) + async with _get_http_client( + [_path(upstream)], upstream, expected_body=expected_request + ) as http_client: + with pytest.raises(ValueError, match="reasoning signatures"): + result = await Molto(http_client=http_client).chat_completions( + body=body, endpoint=upstream, api_key="test-api-key" + ) + if stream: + _ = [event async for event in result] + + @pytest.mark.parametrize("stream", [True, False]) async def test_messages_ambiguous_search_selection_rejected_before_transport(stream: bool): """Ensures named search collisions fail before HTTP in both public output modes.""" @@ -370,7 +402,7 @@ def handler(request: Request) -> Response: async with AsyncClient(transport=MockTransport(handler)) as http_client: with pytest.raises(UnsupportedInput, match="ambiguous"): await Molto(http_client=http_client).messages( - body=body, endpoint="responses", api_key="test-api-key" + body=body, endpoint="responses", api_key="test-api-key", citation_policy="buffer" ) assert requests == [] @@ -405,7 +437,7 @@ async def test_messages_search_selection_matches_sdk(stream: bool, choice_type: [_path("responses")], "responses", expected_body=expected_request ) as http_client: result = await Molto(http_client=http_client).messages( - body=body, endpoint="responses", api_key="test-api-key" + body=body, endpoint="responses", api_key="test-api-key", citation_policy="buffer" ) if stream: events = [event async for event in result] @@ -448,7 +480,7 @@ async def annotation_events(client, request): [_path("responses")], "responses", expected_body=expected_request ) as http_client: result = await Molto(http_client=http_client).messages( - body=body, endpoint="responses", api_key="test-api-key" + body=body, endpoint="responses", api_key="test-api-key", citation_policy="buffer" ) if stream: events = [event async for event in result] @@ -475,16 +507,18 @@ async def test_messages_search_extra_body_matches_sdk(stream: bool, include: lis } if include is not None: extra_body["include"] = include + expected_request.pop("include", None) expected_request.update(extra_body) - expected_request["include"] = list( - dict.fromkeys(["web_search_call.action.sources", *(include or [])]) - ) expected = _collect("messages", await _sdk_result("messages", True)) async with _get_http_client( [_path("responses")], "responses", expected_body=expected_request ) as http_client: result = await Molto(http_client=http_client).messages( - body=body, endpoint="responses", api_key="test-api-key", extra_body=extra_body + body=body, + endpoint="responses", + api_key="test-api-key", + citation_policy="buffer", + extra_body=extra_body, ) if stream: events = [event async for event in result] @@ -497,31 +531,21 @@ async def test_messages_search_extra_body_matches_sdk(stream: bool, include: lis @pytest.mark.parametrize("stream", [True, False]) @pytest.mark.parametrize("upstream", RESPONSES_MESSAGES_PROTOCOLS) -async def test_chat_search_extra_body_matches_sdk(upstream: Protocol, stream: bool): - """Ensures transport-only hosted search preserves SDK Chat output in both modes.""" +async def test_translated_search_ignores_extra_body_tools(upstream: Protocol, stream: bool): + """Ensures transport-only tool overrides never change Molto's search projection.""" body, expected_request = _request_pair("chat/completions", upstream, stream) - body.pop("web_search_options") - extra_body = { - "tools": expected_request["tools"], - "tool_choice": "required" - if upstream == "responses" - else {"type": "tool", "name": "web_search"}, - } - expected_request.update(extra_body) - expected = _collect("chat/completions", await _sdk_result("chat/completions", True)) + # extra_body replaces the projected tools but not the search-derived include + # or selection, and it cannot restore signature support for Chat output. + expected_request["tools"] = None async with _get_http_client( [_path(upstream)], upstream, expected_body=expected_request ) as http_client: - result = await Molto(http_client=http_client).chat_completions( - body=body, endpoint=upstream, api_key="test-api-key", extra_body=extra_body - ) - if stream: - events = [event async for event in result] - _assert_lifecycle("chat/completions", events) - actual = _collect("chat/completions", events) - else: - actual = result - assert _chat_semantics(actual, upstream) == _chat_semantics(expected, upstream) + with pytest.raises(ValueError, match="reasoning signatures"): + result = await Molto(http_client=http_client).chat_completions( + body=body, endpoint=upstream, api_key="test-api-key", extra_body={"tools": None} + ) + if stream: + _ = [event async for event in result] @pytest.mark.parametrize("stream", [True, False]) @@ -603,39 +627,53 @@ async def test_chat_search_raw_reasoning_starts_with_assistant_role(): @pytest.mark.parametrize("downstream", RESPONSES_MESSAGES_PROTOCOLS) @pytest.mark.parametrize("stream", [True, False]) -@pytest.mark.parametrize("search_in_extra_body", [False, True]) -async def test_chat_upstream_search_rejected_before_transport( - downstream: Protocol, stream: bool, search_in_extra_body: bool +async def test_chat_upstream_declared_search_rejected_before_transport( + downstream: Protocol, stream: bool ): - """Ensures translated search and Chat body extensions fail before opening HTTP.""" + """Ensures translated hosted search fails before HTTP when the request declares it.""" requests = [] - if search_in_extra_body: - body = deepcopy( - _read_cassette(_matched_cassette_path("simple_text", downstream, "stream-00.yaml"))[ - "request" - ]["body"] - ) - body["stream"] = stream - else: - body = _body(downstream, stream) def handler(request: Request) -> Response: requests.append(request) - raise AssertionError("Hosted search must fail before transport") + raise AssertionError("Declared hosted search must fail before transport") async with AsyncClient(transport=MockTransport(handler)) as http_client: with pytest.raises(UnsupportedInput, match="[Ss]earch"): result = await _coordinator(Molto(http_client=http_client), downstream)( - body=body, + body=_body(downstream, stream), endpoint="chat/completions", api_key="test-api-key", - extra_body={"web_search_options": {}} if search_in_extra_body else None, ) if stream: _ = [event async for event in result] assert requests == [] +@pytest.mark.parametrize("downstream", RESPONSES_MESSAGES_PROTOCOLS) +@pytest.mark.parametrize("stream", [True, False]) +async def test_chat_upstream_extra_body_search_is_forwarded(downstream: Protocol, stream: bool): + """Ensures transport extensions bypass Molto logic and reach the upstream verbatim.""" + body = deepcopy( + _read_cassette(_matched_cassette_path("simple_text", downstream, "stream-00.yaml"))[ + "request" + ]["body"] + ) + body["stream"] = stream + path = _matched_cassette_path("simple_text", "chat/completions", "stream-00.yaml") + expected = deepcopy(_read_cassette(path)["request"]["body"]) + # extra_body is merged by the SDK after Molto builds its request. + expected["web_search_options"] = {} + async with _get_http_client([path], "chat/completions", expected_body=expected) as http_client: + result = await _coordinator(Molto(http_client=http_client), downstream)( + body=body, + endpoint="chat/completions", + api_key="test-api-key", + extra_body={"web_search_options": {}}, + ) + if stream: + _ = [event async for event in result] + + @pytest.mark.parametrize("stream", [True, False]) @pytest.mark.parametrize( ("downstream", "upstream"), [("responses", "messages"), ("messages", "responses")] @@ -733,14 +771,12 @@ async def test_chat_search_required_with_client_function( body=body, endpoint=upstream, api_key="test-api-key" ) return - result = await Molto(http_client=http_client).chat_completions( - body=body, endpoint=upstream, api_key="test-api-key" - ) - if stream: - events = [event async for event in result] - _assert_lifecycle("chat/completions", events) - else: - assert not result.choices[0].message.tool_calls + with pytest.raises(ValueError, match="reasoning signatures"): + result = await Molto(http_client=http_client).chat_completions( + body=body, endpoint=upstream, api_key="test-api-key" + ) + if stream: + _ = [event async for event in result] @pytest.mark.parametrize("stream", [True, False]) @@ -872,9 +908,8 @@ def _history(protocol: Protocol, search: bool) -> list[dict[str, Any]]: ("downstream", "upstream", "search"), [ (downstream, upstream, search) - for downstream, upstream in SEARCH_ROUTES + for downstream, upstream in SIGNED_REASONING_SAFE_ROUTES for search in (False, True) - if not (search and downstream == "chat/completions") ], ) async def test_fixed_route_followup_history( @@ -955,7 +990,10 @@ async def test_split_assistant_search_history_matches_combined_sdk(stream: bool, [_path("responses")], "responses", expected_body=expected_request ) as http_client: result = await Molto(http_client=http_client).messages( - body=request, endpoint="responses", api_key="test-api-key" + body=request, + endpoint="responses", + api_key="test-api-key", + citation_policy="buffer", ) if stream: events = [event async for event in result] @@ -1016,7 +1054,7 @@ async def aclose(self): self.closed = True -@pytest.mark.parametrize(("downstream", "upstream"), SEARCH_ROUTES) +@pytest.mark.parametrize(("downstream", "upstream"), SIGNED_REASONING_SAFE_ROUTES) @pytest.mark.parametrize("cancel", [False, True]) async def test_search_stream_close_releases_provider( downstream: Protocol, upstream: Protocol, cancel: bool @@ -1261,6 +1299,12 @@ async def test_synthetic_responses_failed_search(stream: bool, downstream: Proto if item["status"] == "completed": item["status"] = "failed" item["action"].pop("sources", None) + if downstream == "chat/completions": + # Chat cannot carry provider reasoning signatures, so the same recorded + # stream fails instead of silently dropping the reasoning part. + with pytest.raises(ValueError, match="reasoning signatures"): + await _synthetic_replay(downstream, "responses", stream, events) + return result, reference = await _synthetic_replay(downstream, "responses", stream, events) if downstream == "messages": call = next(block for block in result.content if block.type == "server_tool_use") @@ -1292,6 +1336,18 @@ async def test_synthetic_messages_search_error( block["content"] = {"type": "web_search_tool_result_error", "error_code": error_code} if not answer: events = [event for event in events if event.get("index", -1) < 3] + if downstream == "chat/completions": + # Chat cannot carry provider reasoning signatures, so the same recorded + # stream fails instead of silently dropping the reasoning part. + with pytest.raises(ValueError, match="reasoning signatures"): + await _synthetic_replay(downstream, "messages", stream, events) + return + if downstream == "chat/completions": + # Chat cannot carry provider reasoning signatures, so the same recorded + # stream fails instead of silently dropping the reasoning part. + with pytest.raises(ValueError, match="reasoning signatures"): + await _synthetic_replay(downstream, "messages", stream, events) + return result, reference = await _synthetic_replay(downstream, "messages", stream, events) expected_text = "".join(block.text for block in reference.content if block.type == "text") if downstream == "responses": @@ -1316,6 +1372,12 @@ async def test_synthetic_pause_has_no_invented_result(stream: bool, downstream: for event in events: if event["type"] == "message_delta": event["delta"]["stop_reason"] = "pause_turn" + if downstream == "chat/completions": + # Chat cannot carry provider reasoning signatures, so the paused search + # turn fails instead of dropping the reasoning part. + with pytest.raises(ValueError, match="reasoning signatures"): + await _synthetic_replay(downstream, "messages", stream, events) + return result, reference = await _synthetic_replay(downstream, "messages", stream, events) assert reference.stop_reason == "pause_turn" assert not any(block.type == "web_search_tool_result" for block in reference.content) @@ -1419,8 +1481,12 @@ async def test_synthetic_observed_search_usage(stream: bool, count, token_usage: if not token_usage: response["usage"] = None result, reference = await _synthetic_replay("messages", "responses", stream, events) - if count in ("missing", None): + if count in ("missing", None) or not token_usage: + # A provider that reports no token usage reports no search count either; + # the observed count is not synthesized into a token-usage object. assert result.usage.server_tool_use is None + if not token_usage and count not in ("missing", None): + assert reference.tool_usage["web_search"]["num_requests"] == count else: assert reference.tool_usage["web_search"]["num_requests"] == count assert result.usage.server_tool_use.web_search_requests == count @@ -1732,13 +1798,13 @@ async def test_synthetic_multiple_responses_text_parts_have_local_offsets( result, reference = await _synthetic_replay("messages", "responses", stream, events) blocks = [block for block in result.content if block.type == "text"] assert "".join(block.text for block in blocks) == reference.output_text - assert [block.text for block in blocks[-3:]] == [" ", "Ω", " "] - assert not blocks[-3].citations - assert not blocks[-1].citations - assert len(blocks[-2].citations) == 1 - assert blocks[-2].citations[0].url == citation["url"] - assert blocks[-2].citations[0].cited_text == "Ω" - assert blocks[-2].citations[0].encrypted_index == "N/A" + # Surrounding whitespace folds into the cited block so Messages never sees a + # block that is empty or contains only spacing. + assert [block.text for block in blocks[-1:]] == [" Ω "] + assert len(blocks[-1].citations) == 1 + assert blocks[-1].citations[0].url == citation["url"] + assert blocks[-1].citations[0].cited_text == "Ω" + assert blocks[-1].citations[0].encrypted_index == "N/A" @pytest.mark.parametrize("stream", [True, False]) diff --git a/tests/test_match_anthropic_cassette_events.py b/tests/test_match_anthropic_cassette_events.py index 562a85d..2b0dc91 100644 --- a/tests/test_match_anthropic_cassette_events.py +++ b/tests/test_match_anthropic_cassette_events.py @@ -1988,13 +1988,14 @@ def test_search_stream_without_search_clears_recorded_activity(keep_tool: bool) ) -def test_ordinary_stream_null_incomplete_details_still_rejected() -> None: - """Ensures search pause translation does not change ordinary stream termination validation.""" +def test_stream_null_incomplete_details_maps_to_end_turn() -> None: + """Ensures a reason-less incomplete stream is unknown rather than a pause.""" messages = _messages_cassette(thinking_deltas=None, text_deltas=["answer"]) responses = _responses_cassette(reasoning_deltas=None, text_deltas=["answer"]) events = _data_events(responses) events[-1]["type"] = "response.incomplete" events[-1]["response"]["incomplete_details"] = None responses["response"]["sse"] = [f"data: {json_dumps(event)}\n\n" for event in events] - with pytest.raises(ValueError, match="incomplete_details"): - match_messages_events(messages, responses) + match_messages_events(messages, responses) + delta = next(event for event in messages["response"]["sse"] if '"message_delta"' in event) + assert '"stop_reason":"end_turn"' in delta.replace(" ", "") diff --git a/tests/test_match_anthropic_cassette_responses.py b/tests/test_match_anthropic_cassette_responses.py index 61021da..37083e2 100644 --- a/tests/test_match_anthropic_cassette_responses.py +++ b/tests/test_match_anthropic_cassette_responses.py @@ -510,7 +510,7 @@ def test_search_counter_is_observed_not_inferred(count: int | None) -> None: def test_search_failed_and_paused_states() -> None: - """Ensures failed searches use unavailable and unfinished searches do not gain results.""" + """Ensures failed searches use unavailable and unfinished searches gain no results or pause.""" messages, responses = _search_cassettes() response = responses["response"]["body"] response["output"][0]["status"] = "failed" @@ -522,7 +522,7 @@ def test_search_failed_and_paused_states() -> None: response.update(status="incomplete", incomplete_details=None) response["output"][0]["status"] = "in_progress" match_messages_response(messages, responses) - assert messages["response"]["body"]["stop_reason"] == "pause_turn" + assert messages["response"]["body"]["stop_reason"] == "end_turn" assert not any( block["type"] == "web_search_tool_result" for block in messages["response"]["body"]["content"] @@ -538,9 +538,9 @@ def test_search_context_size_rejected(size: str) -> None: match_messages_response(messages, responses) -def test_ordinary_response_null_incomplete_details_still_rejected() -> None: - """Ensures search pause translation does not weaken ordinary response validation.""" +def test_null_incomplete_details_maps_to_end_turn() -> None: + """Ensures a reason-less incomplete response is unknown rather than a pause.""" messages, responses = _cassettes() responses["response"]["body"].update(status="incomplete", incomplete_details=None) - with pytest.raises(ValueError, match="incomplete_details"): - match_messages_response(messages, responses) + match_messages_response(messages, responses) + assert messages["response"]["body"]["stop_reason"] == "end_turn" diff --git a/tests/test_match_openai_cassette_events.py b/tests/test_match_openai_cassette_events.py index 617858f..2029da7 100644 --- a/tests/test_match_openai_cassette_events.py +++ b/tests/test_match_openai_cassette_events.py @@ -962,8 +962,8 @@ def test_search_stream_keeps_text_from_multiple_output_parts() -> None: assert "".join(_chat_content(event) or "" for event in _data_events(chat)) == expected -def test_search_paused_stream_maps_to_length() -> None: - """Ensures a paused search's incomplete terminal state cannot become Chat stop.""" +def test_search_unknown_incomplete_stream_maps_to_stop() -> None: + """Ensures a reason-less search incomplete is unknown rather than a pause.""" chat = read_yaml(str(CASSETTES / "web_search" / "openai-chat_completions-stream-00.yaml")) responses = read_yaml(str(CASSETTES / "web_search" / "openai-responses-stream-00.yaml")) events = _data_events(responses) @@ -979,4 +979,4 @@ def test_search_paused_stream_maps_to_length() -> None: for event in _data_events(chat) for choice in event["choices"] if choice.get("finish_reason") - ] == ["length"] + ] == ["stop"] diff --git a/tests/test_match_openai_cassette_responses.py b/tests/test_match_openai_cassette_responses.py index 8d399f3..cbb56f7 100644 --- a/tests/test_match_openai_cassette_responses.py +++ b/tests/test_match_openai_cassette_responses.py @@ -279,11 +279,9 @@ def _search_cassettes() -> tuple[dict, dict]: ) -def test_search_response_omits_signature_only_reasoning() -> None: - """Ensures signature-only search reasoning leaves the matched answer unchanged.""" +def test_search_response_rejects_signature_only_reasoning() -> None: + """Ensures signature-only search reasoning cannot be silently omitted.""" chat, responses = _search_cassettes() - expected = deepcopy(chat) - match_chat_completions_response(expected, responses) responses["response"]["body"]["output"].insert( 0, { @@ -294,9 +292,8 @@ def test_search_response_omits_signature_only_reasoning() -> None: }, ) - match_chat_completions_response(chat, responses) - - assert chat == expected + with pytest.raises(ValueError, match="Encrypted Responses reasoning"): + match_chat_completions_response(chat, responses) @pytest.mark.parametrize("field", ["content", "summary"]) @@ -448,14 +445,14 @@ def test_search_failure_and_browsing_without_text_stay_empty(action: str) -> Non assert choice["finish_reason"] == "stop" -def test_search_paused_response_maps_to_length() -> None: - """Ensures the reduced paused-search state does not claim successful Chat completion.""" +def test_search_unknown_incomplete_maps_to_stop() -> None: + """Ensures a reason-less search incomplete is unknown, not a token-limit stop.""" chat, responses = _search_cassettes() responses["response"]["body"].update(status="incomplete", incomplete_details=None) match_chat_completions_response(chat, responses) - assert chat["response"]["body"]["choices"][0]["finish_reason"] == "length" + assert chat["response"]["body"]["choices"][0]["finish_reason"] == "stop" @pytest.mark.parametrize("field", ["filters", "max_tool_calls"]) diff --git a/tests/upstream/test_anthropic.py b/tests/upstream/test_anthropic.py index 043a5d8..0604437 100644 --- a/tests/upstream/test_anthropic.py +++ b/tests/upstream/test_anthropic.py @@ -837,7 +837,7 @@ def body(prompt_cache: PromptCache) -> dict[str, object]: def test_client_preserves_assistant_message_boundaries_and_ordered_blocks(): - """Ensures Messages retains adjacent containers, empty text, markers, and block ordering.""" + """Ensures Messages retains adjacent containers, markers, and block ordering.""" request = NormalizedRequest( model="compatible-model", max_output_tokens=2048, @@ -861,7 +861,8 @@ def test_client_preserves_assistant_message_boundaries_and_ordered_blocks(): { "role": "assistant", "content": [ - {"type": "text", "text": ""}, + # An empty text part carries no content and Messages rejects an + # empty block, so it is omitted. {"type": "thinking", "thinking": "Thought"}, {"type": "text", "text": "Cached", "cache_control": {"type": "ephemeral"}}, {"type": "tool_use", "id": "call_1", "name": "lookup", "input": {}}, diff --git a/tests/upstream/test_anthropic_search.py b/tests/upstream/test_anthropic_search.py index b8483cc..5af1211 100644 --- a/tests/upstream/test_anthropic_search.py +++ b/tests/upstream/test_anthropic_search.py @@ -339,7 +339,7 @@ async def native_events(request): assert len(citations) == 1 and citations[0].end_index == 2 terminal = next(event for event in events if isinstance(event, ResponseEndEvent)) assert terminal.termination_reason == "pause" - assert terminal.usage.extra_fields["anthropic_web_search_requests"] == 2 + assert terminal.usage.web_search_requests == 2 @pytest.mark.parametrize("nested", [False, True]) diff --git a/tests/upstream/test_openai.py b/tests/upstream/test_openai.py index f8d00dc..2c566bb 100644 --- a/tests/upstream/test_openai.py +++ b/tests/upstream/test_openai.py @@ -1894,7 +1894,6 @@ async def stub_native_events( created_at=1, model="reasoning-model", service_tier="default", - web_search_enabled=False, ), PartStartEvent( index=0, @@ -2996,7 +2995,6 @@ async def stub_native_events( created_at=1, model="reasoning-model", service_tier="default", - web_search_enabled=False, ), PartStartEvent( index=0, @@ -3833,3 +3831,46 @@ def handler(request: httpx.Request) -> httpx.Response: error = exc_info.value assert error.response.status_code == 404 assert isinstance(error.__cause__, APIStatusError) + + +@pytest.mark.parametrize( + ("incomplete_details", "expected"), + [ + ({"reason": "max_output_tokens"}, "max_output_tokens"), + ({"reason": "content_filter"}, "content_filter"), + (None, "unknown"), + ("absent", "unknown"), + ], +) +async def test_responses_incomplete_without_a_reason_is_unknown( + monkeypatch: pytest.MonkeyPatch, incomplete_details, expected +): + """Ensures a reason-less incomplete response is reported as unknown, never as a pause.""" + incomplete_response = _response() + incomplete_response["status"] = "incomplete" + if incomplete_details != "absent": + incomplete_response["incomplete_details"] = incomplete_details + native_events = [ + _response_event( + {"type": "response.created", "response": _response(), "sequence_number": 0} + ), + _response_event( + { + "type": "response.incomplete", + "response": incomplete_response, + "sequence_number": 1, + } + ), + ] + + async def stub_native_events(native_request: OpenAIResponsesCreateRequest): + for event in native_events: + yield event + + async with httpx.AsyncClient(transport=httpx.MockTransport(_unexpected_http)) as http_client: + client = OpenAIResponsesClient(http_client, capabilities=OPENAI_CAPABILITIES) + monkeypatch.setattr(client, "native_events", stub_native_events) + events = [event async for event in client.events(NormalizedRequest(model="m"))] + + response_end = next(event for event in events if isinstance(event, ResponseEndEvent)) + assert response_end.termination_reason == expected diff --git a/tests/upstream/test_openai_search.py b/tests/upstream/test_openai_search.py index 79e4fa8..53399b3 100644 --- a/tests/upstream/test_openai_search.py +++ b/tests/upstream/test_openai_search.py @@ -19,6 +19,7 @@ WebSearchTool, ) from molto.upstream.openai import OpenAIChatCompletionsClient, OpenAIResponsesClient +from molto.utils.web_search import messages_text_blocks CAPABILITIES = PROVIDER_CAPABILITIES["openai"] @@ -221,7 +222,6 @@ async def test_text_parts_keep_local_citation_offsets(refusal_first: bool): assert [event.part.content for event in texts] == ["first", "second"] assert [event.part.citations[0].start_index for event in texts] == [0, 0] assert [event.part.citations[0].end_index for event in texts] == [5, 6] - assert all(event.part.defer_citations for event in texts) @pytest.mark.parametrize("count", [None, 0, 3]) @@ -244,8 +244,7 @@ async def test_search_usage_retains_reported_count_only(count): _event("response.completed", response=response), ] ) - assert events[-1].usage.extra_fields.get("web_search_requests") == count - assert ("web_search_requests" in events[-1].usage.extra_fields) == (count is not None) + assert events[-1].usage.web_search_requests == count @pytest.mark.parametrize( @@ -381,7 +380,7 @@ async def test_unclassified_partial_search_does_not_invent_invocation(): ] ) assert not any(isinstance(event, PartStartEvent) for event in events) - assert events[-1].termination_reason == "pause" + assert events[-1].termination_reason == "unknown" async def test_later_text_cannot_overtake_unclassified_search(): @@ -441,140 +440,37 @@ async def test_later_text_cannot_overtake_unclassified_search(): assert [event.index for event in starts] == [0, 1, 2] -@pytest.mark.parametrize( - "include", - [ - ["reasoning.encrypted_content"], - ["web_search_call.action.sources", "reasoning.encrypted_content"], - ], -) -async def test_translated_search_merges_transport_include_without_duplicates(include): - """Ensures SDK extra_body cannot remove required search sources or caller include values.""" - await _normalize( - [ - _event("response.created", response=_response()), - _event("response.completed", response=_response()), - ], - extra_body={"include": include}, - expected_include=["web_search_call.action.sources", "reasoning.encrypted_content"], - ) - +async def test_translated_search_keeps_projected_include_only(): + """Ensures the projected include comes from the normalized request, not extra_body.""" + seen = {} -@pytest.mark.parametrize( - "history", - [ - [SearchCall(id="ws_1")], - [ - AssistantInput( - content=[ - TextOutput( - content="old", - citations=[ - WebCitation(url="https://example.com", start_index=0, end_index=3) - ], - ) + def replay(request): + seen["body"] = loads(request.content) + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text="".join( + f"data: {dumps(event)}\n\n" + for event in [ + _event("response.created", response=_response()), + _event("response.completed", response=_response()), ] - ) - ], - ], -) -@pytest.mark.parametrize("override", [False, True]) -async def test_search_history_without_tool_declaration_still_defers_citations(history, override): - """Ensures typed and overridden search or citation history buffers follow-ups without tools.""" - request = NormalizedRequest(model="m", input=history, stream=True) - extra_body = None - if override: - extra_body = {"input": OpenAIResponsesClient._request_body(request)["input"]} - request.input = [] - part = {"type": "output_text", "text": "answer", "annotations": []} - item = { - "id": "msg_1", - "type": "message", - "role": "assistant", - "status": "completed", - "content": [part], - } - events = await _normalize( - [ - _event("response.created", response=_response()), - _event("response.output_item.added", output_index=0, item={**item, "content": []}), - _event( - "response.content_part.added", - output_index=0, - content_index=0, - item_id="msg_1", - part={**part, "text": ""}, ), - _event( - "response.content_part.done", - output_index=0, - content_index=0, - item_id="msg_1", - part=part, - ), - _event("response.output_item.done", output_index=0, item=item), - _event("response.completed", response=_response([item])), - ], - normalized_request=request, - extra_body=extra_body, - ) - assert all( - event.part.defer_citations - for event in events - if isinstance(event, (PartStartEvent, PartEndEvent)) and isinstance(event.part, TextOutput) - ) - + ) -@pytest.mark.parametrize( - "input_override", [None, [], "New question", [{"role": "user", "content": "New question"}]] -) -async def test_input_override_removes_search_history_buffering(input_override): - """Ensures replacing search history with plain input does not retain stale citation buffering.""" - part = {"type": "output_text", "text": "answer", "annotations": []} - item = { - "id": "msg_1", - "type": "message", - "role": "assistant", - "status": "completed", - "content": [part], - } - events = await _normalize( - [ - _event("response.created", response=_response()), - _event("response.output_item.added", output_index=0, item={**item, "content": []}), - _event( - "response.content_part.added", - output_index=0, - content_index=0, - item_id="msg_1", - part={**part, "text": ""}, - ), - _event( - "response.content_part.done", - output_index=0, - content_index=0, - item_id="msg_1", - part=part, - ), - _event("response.output_item.done", output_index=0, item=item), - _event("response.completed", response=_response([item])), - ], - normalized_request=NormalizedRequest( - model="m", input=[SearchCall(id="ws_1")], stream=True - ), - extra_body={"input": input_override}, - ) - texts = [ - event.part - for event in events - if isinstance(event, PartStartEvent) and isinstance(event.part, TextOutput) - ] - assert len(texts) == 1 - assert not texts[0].defer_citations + async with httpx.AsyncClient(transport=httpx.MockTransport(replay)) as http_client: + client = OpenAIResponsesClient(http_client, capabilities=CAPABILITIES, api_key="test") + _ = [ + event + async for event in client.events( + NormalizedRequest(model="m", tools=[WebSearchTool()], stream=True) + ) + ] + assert seen["body"]["include"] == ["web_search_call.action.sources"] -async def test_reported_search_usage_without_token_usage_is_retained(): - """Ensures an independent search counter survives without inventing token usage.""" +async def test_reported_search_usage_requires_token_usage(): + """Ensures a count without token usage stays unknown instead of inventing usage.""" events = await _normalize( [ _event("response.created", response=_response()), @@ -585,7 +481,7 @@ async def test_reported_search_usage_without_token_usage_is_retained(): ] ) assert events[-1].usage is None - assert events[-1].extra_fields["web_search_requests"] == 2 + assert "web_search_requests" not in events[-1].extra_fields async def test_null_search_usage_remains_unknown(): @@ -593,10 +489,22 @@ async def test_null_search_usage_remains_unknown(): events = await _normalize( [ _event("response.created", response=_response()), - _event("response.completed", response=_response(tool_usage={"web_search": None})), + _event( + "response.completed", + response=_response( + usage={ + "input_tokens": 1, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 3, + }, + tool_usage={"web_search": None}, + ), + ), ] ) - assert "web_search_requests" not in events[-1].extra_fields + assert events[-1].usage.web_search_requests is None async def test_item_done_text_snapshot_rebases_local_citations(): @@ -642,3 +550,60 @@ async def test_item_done_text_snapshot_rebases_local_citations(): (0, 5), (5, 11), ] + + +@pytest.mark.parametrize( + ("text", "ranges"), + [ + ("ab cd", [(2, 3)]), + ("a b c", [(0, 1), (2, 5)]), + ("Ω x", [(1, 2)]), + ], +) +async def test_citation_ir_never_projects_blank_blocks(text, ranges): + """Ensures client citation ranges never become empty or whitespace-only blocks.""" + part = { + "type": "output_text", + "text": text, + "annotations": [ + { + "type": "url_citation", + "url": f"https://example.com/{index}", + "title": "", + "start_index": start, + "end_index": end, + } + for index, (start, end) in enumerate(ranges) + ], + } + item = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [part], + } + events = await _normalize( + [ + _event("response.created", response=_response()), + _event("response.output_item.added", output_index=0, item={**item, "content": []}), + _event( + "response.content_part.done", + output_index=0, + content_index=0, + item_id="msg_1", + part=part, + ), + _event("response.output_item.done", output_index=0, item=item), + _event("response.completed", response=_response([item])), + ] + ) + texts = [ + event.part + for event in events + if isinstance(event, PartEndEvent) and isinstance(event.part, TextOutput) + ] + assert texts and all(event.part.content for event in events if isinstance(event, PartEndEvent)) + assert "".join(part.content for part in texts) == text + projected = [block for part in texts for block in messages_text_blocks(part)] + assert [block["text"] for block in projected if not block["text"].strip()] == [] From 243ba26dbaa9c14be9da94936009aef1239ed5e4 Mon Sep 17 00:00:00 2001 From: Tan Jia Huei Date: Thu, 29 Oct 2026 03:17:30 +0000 Subject: [PATCH 2/5] fix: buffer citations only when the request announces search - citation_policy=buffer now withholds text parts only when the normalized request declares a hosted-search tool or carries search or citation history, falling back to the drop outcome otherwise. - Document that extra_body is opaque while extra_headers is filtered once at the credential boundary, and that neither is validated or used for translation. --- docs/protocols/anthropic-messages.md | 2 +- docs/web-search.md | 2 +- python/ARCHITECTURE.md | 6 +- python/CONVENTIONS.md | 2 +- python/molto/adapters/anthropic.py | 31 ++++++--- python/molto/core.py | 7 +- tests/adapters/test_anthropic_search.py | 91 ++++++++++++++++++++++++- tests/core/test_web_search.py | 12 ++-- 8 files changed, 131 insertions(+), 22 deletions(-) diff --git a/docs/protocols/anthropic-messages.md b/docs/protocols/anthropic-messages.md index b68b84a..d58a768 100644 --- a/docs/protocols/anthropic-messages.md +++ b/docs/protocols/anthropic-messages.md @@ -360,7 +360,7 @@ Recognized errors use the Anthropic SDK namespace and the [shared error contract Declare `{"type":"web_search_20250305","name":"web_search"}` in `tools`. Translation to Responses supports allowed domains, approximate location, and search budgets; explicit blocked domains are rejected. -With Responses upstream, Messages receives server-tool blocks and URL citations with reduced source metadata. Citation-bearing text waits for complete attribution only when the caller selects `citation_policy="buffer"` on `Molto.messages()`; the default `drop` policy streams text immediately and omits citations. A Responses response that ends incomplete without a stated cause arrives as `end_turn`, because only a Messages `pause_turn` is a reported pause. See [Hosted web search](../web-search.md) for mappings, placeholders, and continuation limits. +With Responses upstream, Messages receives server-tool blocks and URL citations with reduced source metadata. Citation-bearing text waits for complete attribution only when the caller selects `citation_policy="buffer"` on `Molto.messages()` and the normalized request declares hosted search or carries search or citation history; the default `drop` policy always streams text immediately and omits citations. A Responses response that ends incomplete without a stated cause arrives as `end_turn`, because only a Messages `pause_turn` is a reported pause. See [Hosted web search](../web-search.md) for mappings, placeholders, and continuation limits. ## Current implementation limitations diff --git a/docs/web-search.md b/docs/web-search.md index 89e60f8..077e123 100644 --- a/docs/web-search.md +++ b/docs/web-search.md @@ -82,7 +82,7 @@ It becomes these illustrative Messages content blocks: Offsets use Python string characters without normalization. Recordings verify ASCII ranges; synthetic Unicode tests do not establish the provider's Unicode indexing convention. -Citations reach a Messages caller only when the caller opts in with `citation_policy="buffer"`, which buffers text parts until their citations are complete and preserves part order. That option increases text latency and changes chunk boundaries for every text part, including uncited ones. The default `citation_policy="drop"` streams text immediately and omits citations. Neither mode drops answer text, and citations are never inferred from a request's tools, history, or `extra_body`. +Citations reach a Messages caller only when the caller opts in with `citation_policy="buffer"`. That policy buffers text parts until their citations are complete, preserving part order, but only for requests that announce a citation signal: a declared hosted-search tool, or search or citation history in the normalized request. Those requests pay the latency and chunk-boundary cost for every text part, including uncited ones; other requests stream unchanged. The default `citation_policy="drop"` always streams text immediately and omits citations. Neither mode drops answer text, and no trigger consults `extra_body`, so history supplied only that way must select `buffer` with a declared signal or expect omitted citations. A `buffer` request that receives citations without announcing a signal falls back to the `drop` outcome for that response, because its text has already streamed. Translated Chat omits structured annotations in both modes. It preserves actual answer text and existing inline links, without inserting replacement citations or narration. Chat cannot carry a reasoning signature, so a hosted-search turn whose reasoning is encrypted or signed, as OpenAI and Anthropic both return, is rejected at the signature boundary instead of losing that output. Chat remains usable for providers whose reasoning is unsigned. diff --git a/python/ARCHITECTURE.md b/python/ARCHITECTURE.md index 57a383e..51df139 100644 --- a/python/ARCHITECTURE.md +++ b/python/ARCHITECTURE.md @@ -148,7 +148,9 @@ Each implemented coordinator performs the following work: 5. Select the upstream path from `endpoint`. 6. Return either an async iterator of downstream SDK event models or the downstream SDK response model. -Every coordinator also accepts request-scoped `extra_body` and `extra_headers` transport options. They are opaque: the selected provider SDK merges them into the upstream request after Molto projects it, and Molto never reads them to choose a route, normalize input, construct a request, or project output. Translation guarantees are therefore defined over the normalized request only, and a caller that supplies request semantics through `extra_body` opts out of them. Before forwarding extra headers, the facade removes client credentials, account selectors, connection-specific fields, request framing, and proxy topology headers; the provider SDK then constructs its own authentication and transport headers. This preserves Molto's credential boundary while allowing application-specific headers and provider body extensions. +Every coordinator also accepts request-scoped `extra_body` and `extra_headers` transport options. Neither is validated against a schema, and neither is consulted to choose a route, normalize input, construct a request, or project output, so translation guarantees are defined over the normalized request only and a caller that supplies request semantics through `extra_body` opts out of them. + +`extra_body` is opaque: the selected provider SDK merges it into the upstream request after Molto projects that request. `extra_headers` is filtered once at the credential boundary before the SDK merges it, because forwarding a caller's credential or framing headers would replace the provider's own authentication and transport framing. The facade drops client credentials and account selectors, hop-by-hop headers, ingress transport headers, and every name listed in `Connection`. The provider SDK then constructs its own authentication and transport headers. This preserves Molto's credential boundary while allowing application-specific headers and provider body extensions. The returned `MoltoStream` exposes a copy of the successful upstream response headers as `response_headers`. Coordinator prefetching opens the SDK stream and captures those headers before returning, including on translated routes. Molto retains all response headers; an HTTP server using Molto remains responsible for filtering connection-specific and framing headers before constructing its downstream response. @@ -344,7 +346,7 @@ Service-tier labels are retained as upstream metadata, but target protocols do n `SearchCall` and `SearchResult` use the existing part lifecycle and link by call identity. They are assistant history/output parts. An unfinished call may have no result, and a search error does not determine overall termination. Responses folds a call and result into one item; Messages emits separate server-tool blocks. -A block-based target cannot split text it has already emitted, so `AnthropicMessagesAdapter` takes a `citation_policy` selected by the caller through `Molto.messages()`. `buffer` holds every text part until its citation snapshot is final so blocks can be split at citation boundaries, trading text latency and chunk boundaries for exact attribution. `drop`, the default, streams text immediately and omits citations. The policy governs output projection only: request construction keeps input attribution, and search call/result parts are buffered in both modes because that buffering learns the final action rather than splitting text. +A block-based target cannot split text it has already emitted, so `AnthropicMessagesAdapter` takes a `citation_policy` selected by the caller through `Molto.messages()`. `buffer` holds text parts until their citation snapshot is final so blocks can be split at citation boundaries, and applies that only when the normalized request declares a hosted-search tool or carries search or citation history. It therefore trades text latency and chunk boundaries for exact attribution on those requests alone. `drop`, the default, always streams text immediately and omits citations. The policy governs output projection only: request construction keeps input attribution, and search call/result parts are buffered in both modes because that buffering learns the final action rather than splitting text. `TextOutput.citations` holds `WebCitation` ranges local to each text part. `CitationDelta` adds attribution, and the final snapshot is authoritative. The Messages projector buffers unresolved search parts in start order, and buffers text parts when `citation_policy` is `buffer`, allowing other parts to stream when they reach the front. Projection assigns output indices because parts can expand or disappear. No client predicts whether a response will carry citations; only the caller selects that trade. diff --git a/python/CONVENTIONS.md b/python/CONVENTIONS.md index bf03b9c..4b2f1e3 100644 --- a/python/CONVENTIONS.md +++ b/python/CONVENTIONS.md @@ -156,7 +156,7 @@ The bypass belongs in the relevant coordinator; transport remains in the matchin Construct each short-lived `AsyncOpenAI` wrapper with Molto's injected `httpx.AsyncClient`, the resolved API key, and an explicit supplied or default base URL. Resolve an absent vLLM API key to the fixed non-secret `EMPTY` credential, and suppress OpenAI organization and project environment headers for compatible providers. -Treat coordinator `extra_body` and `extra_headers` as request-scoped upstream transport options, not native protocol body fields or normalized semantics. Strip client credentials, account selectors, connection-specific fields, request framing, and proxy topology headers before passing `extra_headers` to a provider SDK. Capture successful SDK response headers before consuming events and expose a copy on the returned `MoltoStream`; do not filter response headers in the library, because an HTTP adapter must apply filtering for its own downstream transport. +Treat coordinator `extra_body` and `extra_headers` as request-scoped upstream transport options, not native protocol body fields or normalized semantics. Never validate them against a schema or consult them when routing, normalizing, constructing a request, or projecting output; request semantics belong to the validated request body. Strip client credentials, account selectors, connection-specific fields, request framing, and proxy topology headers before passing `extra_headers` to a provider SDK. Capture successful SDK response headers before consuming events and expose a copy on the returned `MoltoStream`; do not filter response headers in the library, because an HTTP adapter must apply filtering for its own downstream transport. Context-manage every returned SDK stream so completion and early termination release its response. Do not close the short-lived SDK wrapper because it borrows Molto's shared HTTP client; only `Molto.aclose()` closes that client. diff --git a/python/molto/adapters/anthropic.py b/python/molto/adapters/anthropic.py index 657319b..8f69cb9 100644 --- a/python/molto/adapters/anthropic.py +++ b/python/molto/adapters/anthropic.py @@ -96,9 +96,10 @@ CitationPolicy: TypeAlias = Literal["buffer", "drop"] """How a block-based target projects text that may carry URL citations. -`buffer` holds every text part until its citation snapshot is final so blocks can -be split at citation boundaries, trading text latency and chunk boundaries for -exact attribution. `drop` streams text immediately and omits citations. +`buffer` withholds text parts until their citation snapshot is final whenever the +request shows a citation signal, so blocks can be split at citation boundaries. +That trades text latency and chunk boundaries for exact attribution on those +requests only. `drop` always streams text immediately and omits citations. """ _EXTENDED_THINKING_EFFORTS: tuple[tuple[int, ReasoningEffort], ...] = ( @@ -168,6 +169,20 @@ def _citations(citations: list[dict[str, Any]] | None, text: str) -> list[WebCit return result +def _citations_are_possible(request: NormalizedRequest) -> bool: + """Report whether the request itself shows a hosted-search or citation signal.""" + if any(isinstance(tool, WebSearchTool) for tool in request.tools or []): + return True + for item in request.input or []: + parts = item.content if isinstance(item, AssistantInput) else [item] + for part in parts: + if isinstance(part, (SearchCall, SearchResult)): + return True + if isinstance(part, TextOutput) and part.citations: + return True + return False + + async def _ordered_parts( events: AsyncIterator[ModelResponseStreamEvent], *, @@ -653,9 +668,8 @@ def event(data: dict[str, Any]) -> AnthropicMessagesStreamEvent: thinking_parts: set[int] = set() thinking_signatures: dict[int, str] = {} - async with aclosing( - _ordered_parts(events, buffer_text=self._citation_policy == "buffer") - ) as ordered_events: + buffer_text = self._citation_policy == "buffer" and _citations_are_possible(request) + async with aclosing(_ordered_parts(events, buffer_text=buffer_text)) as ordered_events: async for normalized_event in ordered_events: if isinstance(normalized_event, ResponseStartEvent): yield event( @@ -741,8 +755,9 @@ def event(data: dict[str, Any]) -> AnthropicMessagesStreamEvent: index = indices[normalized_event.index] if isinstance(normalized_event.delta, CitationDelta): # Buffered parts consume their citation deltas in - # `_ordered_parts`; in the `drop` policy no text part is - # buffered, so citations are omitted deliberately. + # `_ordered_parts`. Reaching here means the request announced + # no citation signal, so `buffer` falls back to the `drop` + # outcome: the text has already streamed and cannot be split. continue if isinstance(normalized_event.delta, ThinkingDelta): delta = normalized_event.delta diff --git a/python/molto/core.py b/python/molto/core.py index a1b86fc..6e3623b 100644 --- a/python/molto/core.py +++ b/python/molto/core.py @@ -344,9 +344,10 @@ async def messages( `endpoint` selects the wire protocol, `provider` selects the serving implementation and its capabilities, and `base_url` selects only the HTTP destination. `citation_policy` selects how translated responses - project text that may carry URL citations: `buffer` withholds each text - part until its citations are final so blocks can be split at citation - boundaries, and `drop` streams text immediately without citations. + project text that may carry URL citations: `buffer` withholds text parts + until their citations are final whenever the request declares hosted search + or carries search or citation history, and `drop` always streams text + immediately without citations. """ request = AnthropicMessagesCreateRequest.model_validate(body) resolved_provider = self._resolve_provider(endpoint, provider) diff --git a/tests/adapters/test_anthropic_search.py b/tests/adapters/test_anthropic_search.py index 8d1a39f..d2f970f 100644 --- a/tests/adapters/test_anthropic_search.py +++ b/tests/adapters/test_anthropic_search.py @@ -7,6 +7,8 @@ from molto.types.anthropic_messages import AnthropicMessagesCreateRequest from molto.types.exceptions import UnsupportedInput from molto.types.normalized import ( + AssistantInput, + CitationDelta, FunctionToolChoice, InputTokenDetails, OutputTokenDetails, @@ -226,7 +228,7 @@ def test_overlapping_unicode_citations(): async def test_buffered_citations_search_omission_and_collection(): """Ensures delayed citation splits follow linked search results with dense lifecycles.""" - adapter, request = setup_adapter() + adapter, request = setup_adapter(search=True) parts = [ SearchCall(id="browse", action="open_page"), SearchResult(tool_call_id="browse", sources=[]), @@ -360,7 +362,7 @@ def test_whitespace_gap_does_not_emit_a_spacing_block(): @pytest.mark.parametrize("kind", ["search", "deferred_text"]) async def test_early_close_releases_source_after_buffered_projection(kind): """Ensures closing a buffered block start deterministically releases upstream input.""" - adapter, request = setup_adapter() + adapter, request = setup_adapter(search=True) closed = False part = SearchCall(id="s", query="q") if kind == "search" else TextOutput(content="answer") @@ -406,3 +408,88 @@ def test_dynamic_search_caller_is_outside_basic_scope(): ) with pytest.raises(UnsupportedInput, match="directly invoked"): adapter.normalize_request(request) + + +@pytest.mark.parametrize( + "declare", + ["tool", "search_history", "citation_history"], +) +async def test_buffer_policy_buffers_announced_search(declare: str): + """Ensures buffer withholds text when the request declares search or carries search history.""" + body = { + "model": "m", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Search"}], + "stream": True, + } + if declare == "tool": + body["tools"] = [{"type": "web_search_20250305", "name": "web_search"}] + adapter = AnthropicMessagesAdapter( + PROVIDER_CAPABILITIES["anthropic"], citation_policy="buffer" + ) + native = AnthropicMessagesCreateRequest(**body) + request = adapter.normalize_request(native) + if declare != "tool": + part = ( + SearchCall(id="ws_1", query="q") + if declare == "search_history" + else TextOutput( + content="old", + citations=[WebCitation(url="https://example.com", start_index=0, end_index=3)], + ) + ) + request = request.model_copy( + update={"input": [*request.input, AssistantInput(content=[part])]} + ) + events = [ + ResponseStartEvent(id="r", model="m"), + PartStartEvent(index=0, part=TextOutput(content="")), + PartDeltaEvent(index=0, delta=TextDelta(content_delta="answer")), + PartEndEvent(index=0, part=TextOutput(content="answer")), + ResponseEndEvent(id="r", model="m", created_at=0, termination_reason="stop"), + ] + projected = [event async for event in adapter.project_events(iterate(events), request)] + texts = [event for event in projected if event.type == "content_block_start"] + # Buffered parts emit their completed block instead of streaming deltas. + assert not [event for event in projected if event.type == "content_block_delta"] + assert [block.content_block.text for block in texts] == ["answer"] + + +async def test_buffer_policy_falls_back_to_drop_without_a_signal(): + """Ensures buffer without an announced signal streams text and omits citations.""" + adapter, request = setup_adapter(citation_policy="buffer") + citation = WebCitation(url="https://example.com", start_index=0, end_index=3) + events = [ + ResponseStartEvent(id="r", model="m"), + PartStartEvent(index=0, part=TextOutput(content="")), + PartDeltaEvent(index=0, delta=TextDelta(content_delta="answer")), + PartDeltaEvent(index=0, delta=CitationDelta(citation=citation)), + PartEndEvent(index=0, part=TextOutput(content="answer", citations=[citation])), + ResponseEndEvent(id="r", model="m", created_at=0, termination_reason="stop"), + ] + projected = [event async for event in adapter.project_events(iterate(events), request)] + assert [event.delta.text for event in projected if event.type == "content_block_delta"] == [ + "answer" + ] + start = next(event for event in projected if event.type == "content_block_start") + assert start.content_block.citations is None + + +async def test_drop_policy_omits_citations_for_declared_search(): + """Ensures drop streams declared search text immediately and omits citations.""" + adapter, request = setup_adapter(search=True, citation_policy="drop") + citation = WebCitation(url="https://example.com", start_index=0, end_index=3) + events = [ + ResponseStartEvent(id="r", model="m"), + PartStartEvent(index=0, part=TextOutput(content="")), + PartDeltaEvent(index=0, delta=TextDelta(content_delta="answer")), + PartDeltaEvent(index=0, delta=CitationDelta(citation=citation)), + PartEndEvent(index=0, part=TextOutput(content="answer", citations=[citation])), + ResponseEndEvent(id="r", model="m", created_at=0, termination_reason="stop"), + ] + projected = [event async for event in adapter.project_events(iterate(events), request)] + assert [event.delta.text for event in projected if event.type == "content_block_delta"] == [ + "answer" + ] + start = next(event for event in projected if event.type == "content_block_start") + assert start.content_block.citations is None diff --git a/tests/core/test_web_search.py b/tests/core/test_web_search.py index ffb2b10..f9a8c0b 100644 --- a/tests/core/test_web_search.py +++ b/tests/core/test_web_search.py @@ -499,15 +499,14 @@ async def annotation_events(client, request): async def test_messages_search_extra_body_matches_sdk(stream: bool, include: list[str] | None): """Ensures transport-only search preserves SDK search semantics and citation lifecycle.""" body, expected_request = _request_pair("messages", "responses", stream) - body.pop("tools") - body.pop("tool_choice") extra_body = { "tools": [{"type": "web_search"}], "tool_choice": "required", } if include is not None: extra_body["include"] = include - expected_request.pop("include", None) + # The projected include still comes from the declared search; extra_body only + # replaces it when the caller supplies one. expected_request.update(extra_body) expected = _collect("messages", await _sdk_result("messages", True)) async with _get_http_client( @@ -1673,7 +1672,7 @@ async def test_synthetic_repeated_searches_preserve_identity(stream: bool): async def test_search_history_without_tools_still_buffers_citations( stream: bool, override: bool, new_search: bool ): - """Ensures typed and overridden history preserve SDK citations without redeclaring search.""" + """Ensures typed search history preserves citations while transport-only history cannot.""" body, expected = _request_pair("messages", "responses", stream) for request in (body, expected): request.pop("tools") @@ -1710,6 +1709,11 @@ async def test_search_history_without_tools_still_buffers_citations( text_blocks = [block for block in result.content if block.type == "text"] assert "".join(block.text for block in text_blocks) == reference.output_text citations = [citation for block in text_blocks for citation in block.citations or []] + if override: + # History supplied only through extra_body is invisible to the citation + # trigger, so the buffered policy falls back to omitting citations. + assert citations == [] + return annotation = ( next(item for item in reference.output if item.type == "message").content[0].annotations[0] ) From e2e7b038e4eb6f6440e89c305035120b50eb3557 Mon Sep 17 00:00:00 2001 From: Tan Jia Huei Date: Thu, 29 Oct 2026 04:35:49 +0000 Subject: [PATCH 3/5] fix: reject search history that lost provider ciphertext Anthropic decrypts web_search_result.encrypted_content and web_search_result_location.encrypted_index on later turns and rejects a request where either is missing or modified. Translated history cannot supply those values, so request construction now rejects it with UnsupportedInput instead of sending an N/A placeholder, and the docs stop recommending replay of translated Messages responses. --- docs/protocols/anthropic-messages.md | 2 +- docs/web-search.md | 8 +++-- python/molto/upstream/anthropic.py | 16 +++++++++ tests/cassettes/README.md | 2 +- tests/core/test_web_search.py | 27 ++++++++++++++- tests/upstream/test_anthropic_search.py | 44 ++++++++++++++++++++++--- 6 files changed, 88 insertions(+), 11 deletions(-) diff --git a/docs/protocols/anthropic-messages.md b/docs/protocols/anthropic-messages.md index d58a768..4135c9e 100644 --- a/docs/protocols/anthropic-messages.md +++ b/docs/protocols/anthropic-messages.md @@ -360,7 +360,7 @@ Recognized errors use the Anthropic SDK namespace and the [shared error contract Declare `{"type":"web_search_20250305","name":"web_search"}` in `tools`. Translation to Responses supports allowed domains, approximate location, and search budgets; explicit blocked domains are rejected. -With Responses upstream, Messages receives server-tool blocks and URL citations with reduced source metadata. Citation-bearing text waits for complete attribution only when the caller selects `citation_policy="buffer"` on `Molto.messages()` and the normalized request declares hosted search or carries search or citation history; the default `drop` policy always streams text immediately and omits citations. A Responses response that ends incomplete without a stated cause arrives as `end_turn`, because only a Messages `pause_turn` is a reported pause. See [Hosted web search](../web-search.md) for mappings, placeholders, and continuation limits. +With Responses upstream, Messages receives server-tool blocks and URL citations with reduced source metadata. Citation-bearing text waits for complete attribution only when the caller selects `citation_policy="buffer"` on `Molto.messages()` and the normalized request declares hosted search or carries search or citation history; the default `drop` policy always streams text immediately and omits citations. A Responses response that ends incomplete without a stated cause arrives as `end_turn`, because only a Messages `pause_turn` is a reported pause. Replaying search history to a Messages upstream requires Anthropic's original `encrypted_content` and `encrypted_index`; translated history cannot supply them, so Molto rejects it with `UnsupportedInput` and the caller sends text-only history instead. See [Hosted web search](../web-search.md) for mappings, placeholders, and continuation limits. ## Current implementation limitations diff --git a/docs/web-search.md b/docs/web-search.md index 077e123..844ca95 100644 --- a/docs/web-search.md +++ b/docs/web-search.md @@ -112,7 +112,7 @@ Observed Responses sources become result entries. A URL-only source produces thi } ``` -Available titles and `page_age` are preserved; unavailable age is omitted or null. The result placeholder supplies neither retrieved content nor valid continuation state. Native Messages retains original encrypted content. An explicitly empty source list becomes an empty result list. Missing sources remain unknown and are never reconstructed from answer citations. An unfinished invocation gets no invented successful result. +Available titles and `page_age` are preserved; unavailable age is omitted or null. The `"N/A"` placeholder only satisfies the response schema: it supplies neither retrieved content nor decryptable continuation state, and Anthropic rejects a request that replays it. Native Messages retains the original encrypted content, which is the only form that can be sent back. An explicitly empty source list becomes an empty result list. Missing sources remain unknown and are never reconstructed from answer citations. An unfinished invocation gets no invented successful result. Translated Messages omits Responses `open_page` and `find_in_page` operations, retaining the answer and citations. @@ -150,13 +150,15 @@ Projection to a Messages upstream rejects search-history calls containing more t When projecting Messages history to Responses, a search result updates the preceding call with the same ID even when they occupy separate assistant messages. The call preserves its query metadata while applying the result's status and supplied sources. Assistant text messages and text runs around search blocks retain their boundaries. -For a Messages response named `response`, preserve supported assistant blocks with: +Anthropic decrypts `web_search_result.encrypted_content` and `web_search_result_location.encrypted_index` on later turns and rejects a request where either is missing or modified, so a translated Messages response cannot be replayed: those values were issued by a different provider and Molto never has them. Molto therefore rejects such history locally with `UnsupportedInput` instead of sending a placeholder that the API would reject. + +Replay only native Messages content, whose encrypted values are intact: ```python assistant_content = [block.model_dump(exclude_none=True) for block in response.content] ``` -The SDK/API decides whether reduced history, including `"N/A"` placeholders, is usable. If it rejects replay, the caller can retry with text-only history: +For a translated response, send text-only history instead: ```python assistant_content = "".join(block.text for block in response.content if block.type == "text") diff --git a/python/molto/upstream/anthropic.py b/python/molto/upstream/anthropic.py index fddaf73..7093ba4 100644 --- a/python/molto/upstream/anthropic.py +++ b/python/molto/upstream/anthropic.py @@ -439,6 +439,13 @@ def _assistant_blocks( raise UnsupportedInput( "Citation ranges must be within the associated answer text." ) + if any(citation.encrypted_index is None for citation in item.citations): + # Anthropic decrypts the original index on later turns and rejects + # a placeholder, so replaying reduced history cannot succeed. + raise UnsupportedInput( + "Anthropic Messages cannot replay citations without their original " + "encrypted_index; send text-only history instead." + ) text_blocks = messages_text_blocks(item) if item.cache_breakpoint is not None: text_blocks[-1]["cache_control"] = _content_block(item, request)[ @@ -455,6 +462,15 @@ def _assistant_blocks( ) blocks.extend(search_calls_to_messages(item)) elif isinstance(item, SearchResult): + if item.sources is not None and any( + source.encrypted_content is None for source in item.sources + ): + # Anthropic decrypts the original content on later turns and rejects + # a placeholder, so replaying reduced history cannot succeed. + raise UnsupportedInput( + "Anthropic Messages cannot replay search results without their " + "original encrypted_content; send text-only history instead." + ) if item.sources is not None or item.error is not None: blocks.append(search_result_to_messages(item)) elif isinstance(item, ToolCall): diff --git a/tests/cassettes/README.md b/tests/cassettes/README.md index 0ca5123..523555d 100644 --- a/tests/cassettes/README.md +++ b/tests/cassettes/README.md @@ -109,7 +109,7 @@ The `reasoning_raw` scenario is named with `vllm-*` filenames and its test deriv ### Hosted search -The Responses recordings are unchanged canonical source data. The matched Messages fixtures project source URLs and citation ranges with the documented empty-title and `"N/A"` placeholders; these fixtures do not establish valid Anthropic continuation state. Chat preserves the answer and inline links while omitting structured annotations and hosted activity. Native parity still compares each fixture's complete parsed SDK representation. +The Responses recordings are unchanged canonical source data. The matched Messages fixtures project source URLs and citation ranges with the documented empty-title and `"N/A"` placeholders; these fixtures do not establish valid Anthropic continuation state and must not be replayed, because Anthropic rejects a result whose `encrypted_content` or `encrypted_index` was not issued by Anthropic. Chat preserves the answer and inline links while omitting structured annotations and hosted activity. Native parity still compares each fixture's complete parsed SDK representation. The Responses stream exposes 13 sources at item completion and a fourteenth URL in the terminal snapshot. Messages stream matching uses the item-completion snapshot, as does incremental translation. Its reasoning signature likewise follows the item-completion snapshot rather than a later terminal ciphertext. Translated comparisons reduce only those unavailable differences and separately assert text/citation association, sources, placeholders, linkage, usage, and lifecycle. Do not erase these fields globally to force equality. diff --git a/tests/core/test_web_search.py b/tests/core/test_web_search.py index f9a8c0b..55d4eb9 100644 --- a/tests/core/test_web_search.py +++ b/tests/core/test_web_search.py @@ -928,11 +928,23 @@ async def test_fixed_route_followup_history( ] body[body_key].extend(_history(downstream, search)) expected[expected_key].extend(_history(upstream, search)) + requests = [] async with _get_http_client( [_path(upstream, stream if downstream == upstream else True)], upstream, expected_body=expected, + requests=requests, ) as http_client: + if search and downstream != upstream and upstream == "messages": + # Anthropic cannot decrypt a replayed result that lost its provider state. + with pytest.raises(UnsupportedInput, match="encrypted_content"): + result = await _coordinator(Molto(http_client=http_client), downstream)( + body=body, endpoint=upstream, api_key="test-api-key" + ) + if stream: + _ = [event async for event in result] + assert requests == [] + return result = await _coordinator(Molto(http_client=http_client), downstream)( body=body, endpoint=upstream, api_key="test-api-key" ) @@ -1818,7 +1830,7 @@ async def test_synthetic_multiple_responses_text_parts_have_local_offsets( async def test_search_history_reaches_sdk_error_boundary( downstream: Protocol, upstream: Protocol, stream: bool ): - """Ensures reduced history reaches transport and a recorded SDK rejection propagates without local placeholder validation.""" + """Ensures native history reaches transport while translated history fails before it.""" body, expected = _request_pair(downstream, upstream, stream) for request, protocol in ((body, downstream), (expected, upstream)): key = "input" if protocol == "responses" else "messages" @@ -1829,6 +1841,19 @@ async def test_search_history_reaches_sdk_error_boundary( request[key].extend(_history(protocol, True)) error_path = _matched_cassette_path("error-invalid_request_error", upstream, "stream-00.yaml") error_type = AnthropicBadRequestError if upstream == "messages" else OpenAIBadRequestError + if downstream != upstream and upstream == "messages": + requests = [] + async with _get_http_client( + [error_path], upstream, expected_body=expected, requests=requests + ) as http_client: + with pytest.raises(UnsupportedInput, match="encrypted_content"): + result = await _coordinator(Molto(http_client=http_client), downstream)( + body=body, endpoint=upstream, api_key="test-api-key" + ) + if stream: + _ = [event async for event in result] + assert requests == [] + return async with _get_http_client([error_path], upstream, expected_body=expected) as http_client: sdk = ( AsyncAnthropic(api_key="test-api-key", http_client=http_client) diff --git a/tests/upstream/test_anthropic_search.py b/tests/upstream/test_anthropic_search.py index 5af1211..f2a61fc 100644 --- a/tests/upstream/test_anthropic_search.py +++ b/tests/upstream/test_anthropic_search.py @@ -119,7 +119,7 @@ def fail(*args, **kwargs): def test_request_search_options_and_history(): - """Ensures target-native tools, budget, selection and reduced history are preserved.""" + """Ensures target-native tools, budget, selection and provider replay state are preserved.""" request = NormalizedRequest( model="m", max_output_tokens=100, @@ -136,7 +136,12 @@ def test_request_search_options_and_history(): content=[ SearchCall(id="s", query="q"), SearchResult( - tool_call_id="s", sources=[WebSearchSource(url="https://x.test")] + tool_call_id="s", + sources=[ + WebSearchSource( + url="https://x.test", encrypted_content="opaque-search" + ) + ], ), TextOutput( content="Answer", @@ -145,7 +150,7 @@ def test_request_search_options_and_history(): url="https://x.test", start_index=0, end_index=6, - encrypted_index="N/A", + encrypted_index="opaque-index", ) ], ), @@ -171,8 +176,8 @@ def test_request_search_options_and_history(): assert body["tool_choice"] == {"type": "tool", "name": "web_search"} call, result, text = body["messages"][0]["content"] assert call["input"] == {"query": "q"} - assert result["content"][0]["encrypted_content"] == "N/A" - assert text["citations"][0]["encrypted_index"] == "N/A" + assert result["content"][0]["encrypted_content"] == "opaque-search" + assert text["citations"][0]["encrypted_index"] == "opaque-index" @pytest.mark.parametrize("context", ["low", "medium", "high"]) @@ -462,3 +467,32 @@ def handler(request): assert events[-1].termination_reason == ( "pause" if reference.stop_reason == "pause_turn" else "max_output_tokens" ) + + +@pytest.mark.parametrize( + ("kind", "message"), + [ + ("source", "encrypted_content"), + ("citation", "encrypted_index"), + ], +) +def test_rejects_search_history_without_provider_ciphertext(kind: str, message: str): + """Ensures reduced search history is rejected instead of sending a placeholder.""" + if kind == "source": + parts = [ + SearchCall(id="s", query="q"), + SearchResult(tool_call_id="s", sources=[WebSearchSource(url="https://x.test")]), + ] + else: + parts = [ + TextOutput( + content="Answer", + citations=[WebCitation(url="https://x.test", start_index=0, end_index=6)], + ) + ] + with pytest.raises(UnsupportedInput, match=message): + AnthropicMessagesClient._request_body( + NormalizedRequest( + model="m", max_output_tokens=100, input=[AssistantInput(content=parts)] + ) + ) From 233cc3d5671f772b213c6ad14f12254db014c865 Mon Sep 17 00:00:00 2001 From: Tan Jia Huei Date: Thu, 29 Oct 2026 11:32:33 +0000 Subject: [PATCH 4/5] fix: Accept unsigned reasoning output --- docs/README.md | 6 +- docs/protocols/anthropic-messages.md | 8 +- docs/protocols/comparison.md | 2 +- docs/protocols/openai-chat-completions.md | 6 +- docs/protocols/openai-responses.md | 10 +- docs/web-search.md | 4 +- python/ARCHITECTURE.md | 8 +- python/CONVENTIONS.md | 4 +- python/molto/adapters/anthropic.py | 6 +- python/molto/adapters/openai.py | 2 - python/molto/upstream/anthropic.py | 4 - python/molto/upstream/openai.py | 8 +- tests/adapters/test_anthropic.py | 39 ++++-- tests/adapters/test_openai.py | 43 ++++-- tests/core/test_unsigned_reasoning.py | 161 ++++++++++++++++++++++ tests/upstream/test_anthropic.py | 16 ++- tests/upstream/test_openai.py | 78 +++++++++++ 17 files changed, 332 insertions(+), 73 deletions(-) create mode 100644 tests/core/test_unsigned_reasoning.py diff --git a/docs/README.md b/docs/README.md index df8c7a9..41333c8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -207,7 +207,7 @@ The translated path currently supports: - string input and ordered text messages; - ordered mixed text and image user input using remote URLs or base64 data URLs; - system prompts and supported assistant history; -- raw reasoning across all three protocols, and reasoning summaries on supported translations between Responses and Messages; +- raw reasoning across all three protocols, reasoning summaries between Responses and Messages, and unsigned summary output through Chat; - client-defined function tools, selection, parallel-call control, calls, successful results, history, and streamed argument deltas; - basic [hosted web search](web-search.md), URL attribution between Responses and Messages, and mandatory search with text output for downstream Chat when reasoning is absent or unsigned; signed or encrypted reasoning is rejected; - model, output-token limit, reasoning effort, temperature, and `top_p`; @@ -230,9 +230,9 @@ The normalized representation distinguishes raw reasoning from reasoning summari Responses events label those representations explicitly and must agree with the configured provider capability. Chat Completions `reasoning_content` and Messages `thinking_delta` are ambiguous, so the capability profile classifies them before normalization. -A Responses reasoning lifecycle must contain raw reasoning without a summary or signature, a summary accompanied by a signature without raw reasoning, or a signature without reasoning text. A Messages thinking lifecycle follows the same pairing rule. A completely empty start may be deferred until its representation is known, and a lifecycle must not switch representations. +A Responses reasoning output lifecycle contains raw reasoning without a summary or signature, a summary with an optional signature and no raw reasoning, or a signature without reasoning text. A Messages thinking output lifecycle follows the same rule. A completely empty start may be deferred until its representation is known, and a lifecycle must not switch representations. -Although a provider profile can classify Chat `reasoning_content` as a summary, Chat has no signature field. Current translated routes through Chat therefore support only unsigned raw reasoning. Summary translation through Chat fails at the Responses or Messages signature boundary rather than silently discarding required continuation state. +Output processing preserves unsigned summaries rather than rejecting a response because its signature is missing. Chat can carry unsigned raw reasoning or summaries according to the provider profile, but rejects present signatures because it has no signature field. Request translation remains strict: replaying summary history through Responses or Messages requires its original signature. Receiving unsigned reasoning output does not establish usable continuation state. ### Opaque continuation state diff --git a/docs/protocols/anthropic-messages.md b/docs/protocols/anthropic-messages.md index 09514fb..9a0de8c 100644 --- a/docs/protocols/anthropic-messages.md +++ b/docs/protocols/anthropic-messages.md @@ -278,13 +278,13 @@ When the application speaks Messages, an inbound thinking block is normalized as Adaptive requests always use summarized display and an explicit effort. A manual `budget_tokens` value must be less than `max_tokens`; Molto rejects the request before transport when the budget is invalid. Reasoning history enables thinking when the current request omits reasoning configuration, using high effort for adaptive models or the default 10,000-token manual budget. -Inbound `thinking_delta` classification is a separate concern from the outbound adaptive-versus-manual model policy. A valid Messages thinking lifecycle contains one of: +Inbound `thinking_delta` classification is a separate concern from the outbound adaptive-versus-manual model policy. A Messages thinking output lifecycle contains one of: - raw reasoning without a signature; -- a reasoning summary with a signature; or +- a reasoning summary with an optional signature; or - a signature without reasoning text. -It must not mix or switch representations within one block. Official Anthropic continuation requires the original Anthropic-issued signature. Molto's local request model also permits an absent signature for compatible providers such as vLLM; this does not make unsigned thinking history valid against Anthropic's official API. +It must not mix or switch representations within one block. Output processing accepts a missing summary signature and projects it as an empty Messages signature without inventing continuation state. Request translation still rejects unsigned summary history before transport. Official Anthropic continuation requires the original Anthropic-issued signature. Molto's local request model also permits an absent signature for compatible providers such as vLLM; this does not make unsigned thinking history valid against Anthropic's official API. ### Opaque thinking state @@ -368,7 +368,7 @@ When Messages is the translated upstream and the provider is Anthropic, missing, Cross-protocol Messages support includes streaming and non-streaming requests for the explicitly mapped text, portable image input, thinking, custom function tools, direct tool calls, successful tool results, limited failed-result paths, conversation history, lifecycle metadata, termination, and usage semantics described above. -A route targeting Messages must supply a downstream output-token limit because Messages requires `max_tokens`. Reasoning summaries are not currently portable through Chat Completions in either direction because Chat has no signature slot. Molto also does not enforce the same-provider requirement for opaque continuation state; callers must keep signatures with their issuing provider. +A route targeting Messages must supply a downstream output-token limit because Messages requires `max_tokens`. Unsigned reasoning summary output can pass through Chat Completions, but present signatures cannot because Chat has no signature slot. Translated summary-history requests still require signatures. Molto also does not enforce the same-provider requirement for opaque continuation state; callers must keep signatures with their issuing provider. The normalized path does not currently implement image output, non-web citations, refusals as content blocks, redacted thinking, hosted tools other than basic web search, structured output, or multiple candidates. Request fields accepted by the local Messages wrapper are not cross-protocol features unless the adapter and client map them explicitly. diff --git a/docs/protocols/comparison.md b/docs/protocols/comparison.md index 20cb101..d018ffe 100644 --- a/docs/protocols/comparison.md +++ b/docs/protocols/comparison.md @@ -51,7 +51,7 @@ Responses is the richer lifecycle protocol. Translating into Responses requires ## Hosted-search fidelity -Responses and Messages translate basic search calls, results, and URL attribution. The Anthropic Messages-to-Responses route preserves excerpts and opaque references in `provider_metadata.anthropic` for callers that echo the extension in history; standard Responses fields alone cannot reconstruct them. Translated Chat rejects signed or encrypted reasoning, including signature-only parts. This prevents translating the OpenAI encrypted-reasoning search stream and Anthropic search turns with signed thinking to Chat; use Responses or Messages downstream instead. Search turns without reasoning or with unsigned raw reasoning remain representable, retaining answer text and inline links while omitting hosted activity and structured citations. The returned reasoning data determines rejection, not the search declaration. Native Chat passthrough preserves its provider output. See [Hosted web search](../web-search.md) for directional mappings and limitations. +Responses and Messages translate basic search calls, results, and URL attribution. The Anthropic Messages-to-Responses route preserves excerpts and opaque references in `provider_metadata.anthropic` for callers that echo the extension in history; standard Responses fields alone cannot reconstruct them. Translated Chat rejects signed or encrypted reasoning, including signature-only parts. This prevents translating the OpenAI encrypted-reasoning search stream and Anthropic search turns with signed thinking to Chat; use Responses or Messages downstream instead. Search turns without reasoning or with unsigned raw reasoning or summaries remain representable, retaining answer text and inline links while omitting hosted activity and structured citations. The returned reasoning data determines rejection, not the search declaration. Native Chat passthrough preserves its provider output. See [Hosted web search](../web-search.md) for directional mappings and limitations. ## Role and placement diff --git a/docs/protocols/openai-chat-completions.md b/docs/protocols/openai-chat-completions.md index 0cc79f2..e0ba8b6 100644 --- a/docs/protocols/openai-chat-completions.md +++ b/docs/protocols/openai-chat-completions.md @@ -226,7 +226,7 @@ The provider or model renderer can still merge, move, or reinterpret accepted wi ### Reasoning and continuation state -Chat's single `reasoning_content` extension can be classified as either the normalized raw reasoning field or summary field according to the provider profile. Chat has no reasoning signature or encrypted continuation slot. In the current implementation, only unsigned raw reasoning is portable through Chat on a translated route. A Chat summary has no signature and therefore cannot satisfy the completed-summary contract required by Responses or Messages; in the reverse direction, a Responses or Messages summary carries a signature that Chat projection rejects rather than silently discarding. Opaque continuation state and summarized reasoning are therefore not currently portable through Chat. +Chat's single `reasoning_content` extension can be classified as either the normalized raw reasoning field or summary field according to the provider profile. Chat has no reasoning signature or encrypted continuation slot. Translated output supports unsigned raw reasoning and summaries. Responses and Messages output projection accepts a Chat summary without inventing a signature; in the reverse direction, Chat projection rejects any present signature rather than silently discarding it. Request translation still rejects unsigned summary history when normalizing or constructing a Responses or Messages request, so receiving summary text does not establish replayable continuation state. ### Tools and tool results @@ -298,7 +298,7 @@ Native Chat routes forward `web_search_options` from the request or `extra_body` Use `web_search_options: {}` with a Responses or Messages upstream to require hosted search. Translated routes targeting a Chat upstream reject hosted search before transport. -Translated Chat raises `ValueError("Chat Completions cannot preserve reasoning signatures.")` when upstream output contains signed thinking or encrypted reasoning, including signature-only parts. The OpenAI hosted-search streaming recording contains encrypted reasoning; Anthropic hosted-search turns with signed thinking have the same limitation. Use Responses or Messages downstream for these turns. Search without reasoning or with unsigned raw reasoning remains representable; enabling search alone does not cause rejection. Native Chat passthrough is unaffected. +Translated Chat raises `ValueError("Chat Completions cannot preserve reasoning signatures.")` when upstream output contains signed thinking or encrypted reasoning, including signature-only parts. The OpenAI hosted-search streaming recording contains encrypted reasoning; Anthropic hosted-search turns with signed thinking have the same limitation. Use Responses or Messages downstream for these turns. Search without reasoning or with unsigned raw reasoning or summaries remains representable; enabling search alone does not cause rejection. Native Chat passthrough is unaffected. For representable output, translated Chat retains answer text and inline links while omitting hosted activity and structured citations. See [Hosted web search](../web-search.md) for tool selection, option mappings, and continuation limits. @@ -307,7 +307,7 @@ For representable output, translated Chat retains answer text and inline links w These are implementation gaps rather than inherent claims about the Chat protocol: - An upstream Chat stream silently uses `choices[0]`; additional choices are not rejected and are lost. Requests with `n > 1` should not use translation. -- Translated reasoning through Chat currently supports only unsigned raw reasoning, such as the `vllm` provider profile. Summary profiles fail at the target protocol's signature boundary in either direction. +- Chat cannot carry signed reasoning output. Unsigned summary output is supported, but translated Responses and Messages requests still require signatures when replaying summary history. - Refusals, hosted tools other than basic web search, moderation, non-web annotations, audio, prediction output, and log probabilities are not carried through the normalized path. Hosted-search structured annotations are deliberately omitted from translated Chat output. - Chat text chunks therefore use `logprobs=None`, even when another protocol could theoretically represent compatible token log probabilities. - Chat has no normalized per-part status. Target lifecycle and status fields are synthesized where required, and pauses map to `finish_reason="length"`; other terminal reasons without an exact Chat mapping become `finish_reason="stop"`. diff --git a/docs/protocols/openai-responses.md b/docs/protocols/openai-responses.md index 34ef18e..18178fd 100644 --- a/docs/protocols/openai-responses.md +++ b/docs/protocols/openai-responses.md @@ -257,15 +257,15 @@ Responses is explicit where Chat Completions and Messages are ambiguous: Every explicit Responses event must agree with the selected provider capability profile. OpenAI is classified as summarized reasoning and vLLM as raw reasoning. A base URL never chooses this behavior. -A Responses reasoning lifecycle must use exactly one valid representation: +A Responses reasoning output lifecycle uses one of these representations: - raw reasoning content with no summary or encrypted state; -- a reasoning summary accompanied by encrypted state and no raw content; or +- a reasoning summary with optional encrypted state and no raw content; or - encrypted state with neither summary nor raw content. -The item must not combine or switch representations between its added, delta, and done states. A completely empty start may remain deferred until its representation becomes known. When Responses is the downstream target, summary providers require a signature for a completed reasoning summary. +The item must not combine or switch representations between its added, delta, and done states. A completely empty start may remain deferred until its representation becomes known. Output processing accepts completed summaries without encrypted state. Request translation still rejects unsigned summary history before transport. -Chat Completions has one `reasoning_content` extension and no signature. Messages has one `thinking_delta` field plus signature lifecycle. Their configured provider capability decides whether their text is classified as Responses raw reasoning or summary. Only unsigned raw Chat reasoning is currently portable into Responses: a Chat summary cannot supply the signature required at Responses part completion. Responses-to-Chat summary translation also fails because Chat rejects the source signature instead of dropping it. Supported delta boundaries remain observable and are preserved by matched parity coverage. +Chat Completions has one `reasoning_content` extension and no signature. Messages has one `thinking_delta` field plus signature lifecycle. Their configured provider capability decides whether their text is classified as Responses raw reasoning or summary. Unsigned Chat reasoning output can project into Responses as raw reasoning or a summary according to that capability. Responses-to-Chat output rejects any present signature instead of dropping it; unsigned summary output is representable. Supported delta boundaries remain observable and are preserved by matched parity coverage. ### Opaque continuation state @@ -390,7 +390,7 @@ A Responses stream can fail after HTTP `200`: the SDK raises `APIError` on the c The following are implementation gaps, not general claims that the source data cannot exist: - multiple Chat Completions choices; the current translated client reads only `choices[0]`; -- summarized reasoning through Chat Completions; Chat has no signature slot, so current translation fails in either direction rather than discarding required continuation state; +- signed reasoning output through Chat Completions and unsigned summary history on translated requests; Chat has no signature slot, while Responses and Messages request translation requires summary signatures; - refusals as content, hosted tools other than basic search, failed client-function result semantics, redacted thinking, non-web annotations, moderation, and other multimodal output; - log probability transport across the normalized representation; - Chat audio and accepted or rejected prediction-token details; diff --git a/docs/web-search.md b/docs/web-search.md index e1b0ff7..d3f63c9 100644 --- a/docs/web-search.md +++ b/docs/web-search.md @@ -8,7 +8,7 @@ Molto preserves provider-hosted web search on native routes and translates basic | Messages | Search, answer, and reduced URL attribution | Native passthrough | Rejected before transport | | Chat Completions | Mandatory search; signed reasoning rejected | Mandatory search; signed reasoning rejected | Native passthrough | -Translated Chat cannot preserve reasoning signatures or encrypted reasoning. The OpenAI Responses streaming search recording contains encrypted reasoning, and Anthropic Messages search turns can contain signed thinking; these turns raise `ValueError("Chat Completions cannot preserve reasoning signatures.")`. Use Responses or Messages downstream to retain that output. Search without reasoning or with unsigned raw reasoning remains representable. The `vllm` profile supports unsigned raw reasoning; provider support for hosted search is separate. Rejection depends on the returned reasoning data, not whether search was requested. Native Chat passthrough is unaffected. +Translated Chat cannot preserve reasoning signatures or encrypted reasoning. The OpenAI Responses streaming search recording contains encrypted reasoning, and Anthropic Messages search turns can contain signed thinking; these turns raise `ValueError("Chat Completions cannot preserve reasoning signatures.")`. Use Responses or Messages downstream to retain that output. Search without reasoning or with unsigned raw reasoning or summaries remains representable. The `vllm` profile supports unsigned raw reasoning; provider support for hosted search is separate. Rejection depends on the returned reasoning data, not whether search was requested. Native Chat passthrough is unaffected. Messages callers receive translated URL citations with `Molto.messages(..., citation_policy="buffer")`, or with `citation_policy="auto"` when the normalized request declares hosted search or carries search or citation history. The default `"drop"` streams text without citations; native Messages passthrough preserves citations under every policy. @@ -100,7 +100,7 @@ Buffering increases text latency and changes chunk boundaries. All policies reta Each buffered Messages text block starts with empty text and its citations, then emits the full block text in one `text_delta` before stopping. This preserves compatibility with the Anthropic SDK's `text_stream` iterator. -Translated Chat omits structured annotations in both modes. It preserves actual answer text and existing inline links, without inserting replacement citations or narration. Signed or encrypted reasoning raises the error described above, including signature-only parts without visible reasoning text. Unsigned reasoning follows the existing protocol restrictions. +Translated Chat omits structured annotations in both modes. It preserves actual answer text and existing inline links, without inserting replacement citations or narration. Signed or encrypted reasoning raises the error described above, including signature-only parts without visible reasoning text. Unsigned raw reasoning and summary output are representable; translated summary-history requests still require signatures. ## Search activity and results diff --git a/python/ARCHITECTURE.md b/python/ARCHITECTURE.md index c9e1283..2972fd0 100644 --- a/python/ARCHITECTURE.md +++ b/python/ARCHITECTURE.md @@ -10,7 +10,7 @@ Three upstream endpoints are implemented: - `chat/completions`: native Chat Completions passthrough or cross-protocol translation through Chat Completions; - `messages`: native Anthropic Messages passthrough or translated through the normalized representation. -Native same-protocol and cross-protocol routes support streaming and non-streaming requests. Cross-protocol routes cover text, portable image input, raw reasoning, reasoning summaries between Responses and Messages, function tools, tool calls and results, conversation history, prompt-cache intent, lifecycle metadata, and token usage. Because Chat Completions has no signature field, translated routes through Chat currently support only unsigned raw reasoning. +Native same-protocol and cross-protocol routes support streaming and non-streaming requests. Cross-protocol routes cover text, portable image input, raw reasoning, reasoning summaries between Responses and Messages, function tools, tool calls and results, conversation history, prompt-cache intent, lifecycle metadata, and token usage. Translated output through Chat supports unsigned raw reasoning and summaries; present signatures remain unsupported because Chat has no signature field. Translated summary-history requests still require signatures. ## Route configuration @@ -256,7 +256,7 @@ OpenAIChatCompletionsAdapter.project_events() ChatCompletionChunk SDK stream ``` -`reasoning_content` is a provider extension to the OpenAI Chat schema, but the field itself does not distinguish raw reasoning from a reasoning summary. Molto models the extension only on assistant messages and uses `ProviderCapabilities.thinking` to classify it. OpenAI classifies it as a summary and vLLM as raw reasoning. Responses events already distinguish `response.reasoning_text.*` from `response.reasoning_summary_*`; their explicit event types remain authoritative. Classification does not make summaries portable through Chat: Responses and Messages require a signature for completed summaries, while Chat has no signature field. +`reasoning_content` is a provider extension to the OpenAI Chat schema, but the field itself does not distinguish raw reasoning from a reasoning summary. Molto models the extension only on assistant messages and uses `ProviderCapabilities.thinking` to classify it. OpenAI classifies it as a summary and vLLM as raw reasoning. Responses events already distinguish `response.reasoning_text.*` from `response.reasoning_summary_*`; their explicit event types remain authoritative. Output normalization and projection tolerate unsigned summaries, including those received through Chat. Present signatures remain unrepresentable in Chat, and translated Responses and Messages request construction still requires signatures for summary history. ## Normalized request model @@ -348,7 +348,7 @@ The normalized termination reason maps shared semantics explicitly: normal compl `SearchCall` and `SearchResult` use the existing part lifecycle and link by call identity. They are assistant history/output parts. An unfinished call may have no result, and a search error does not determine overall termination. Responses folds a call and result into one item; Messages emits separate server-tool blocks. -Chat projection rejects reasoning signatures and encrypted reasoning based on output data, including signature-only parts. Search declarations and transport overrides do not change that rule; turns without reasoning or with unsigned raw reasoning remain representable. +Chat projection rejects reasoning signatures and encrypted reasoning based on output data, including signature-only parts. Search declarations and transport overrides do not change that rule; turns without reasoning or with unsigned raw reasoning or summaries remain representable. `TextOutput.citations` holds `WebCitation` ranges local to each text part. `CitationDelta` adds attribution, and the final snapshot is authoritative. Citation buffering belongs to the Messages adapter, exposed by the `Molto.messages()` argument `citation_policy`, which accepts `"buffer"`, `"auto"`, or `"drop"`. With Responses upstream, the default `"drop"` streams text and discards attribution; `"buffer"` waits for every text part's completed snapshot, including uncited text, before splitting it at citation boundaries. `"auto"` buffers when the normalized request declares hosted search or carries search calls, search results, or text with citations in its history; otherwise text streams and unexpected citations are dropped. The coordinator selects `"drop"` for Chat upstream regardless of the requested policy, preserving immediate text streaming and chunk boundaries because Chat normalization discards annotations. The policy never changes request construction or native passthrough. The Responses normalizer and Messages projector buffer unresolved search parts in start order, allowing ordinary parts to stream when they reach the front. No upstream client predicts citation buffering, and opaque extensions cannot influence it. The `"auto"` policy inspects only the normalized request; callers supplying search context only through `extra_body` must explicitly select `"buffer"` for attribution. Projection assigns output indices because parts can expand or disappear. @@ -500,7 +500,7 @@ Parity differs by path: ## Current limitations and extension path -The current public paths support native and translated streaming and non-streaming responses for text, portable image input, raw reasoning, reasoning summaries between Responses and Messages, and client-defined function tools. Routes through Chat Completions currently support only unsigned raw reasoning. Function-tool support includes definitions, selection, parallel-call control, successful results, conversation history, and streamed JSON arguments. Native routes preserve hosted web search. Basic hosted search translation is supported through Responses and Messages upstreams, with text-only projection for downstream Chat when reasoning is absent or unsigned; signed or encrypted reasoning raises an error; translated Chat upstreams reject normalized search declarations and history, while opaque `extra_body.web_search_options` is forwarded and returned annotations are dropped. Other hosted tools, failed client-function result semantics outside Messages, other multimodal content, and additional provider capabilities are not complete yet. The matched Responses/Messages fixtures also map the reconstructible opaque reasoning snapshots under the assumption that OpenAI serves both endpoints; opaque state is not portable across providers, the implementation does not enforce its issuer, and the separately recorded terminal Responses ciphertext is not byte-reconstructible through Messages. +The current public paths support native and translated streaming and non-streaming responses for text, portable image input, raw reasoning, reasoning summaries between Responses and Messages, and client-defined function tools. Output through Chat Completions supports unsigned raw reasoning and summaries, but rejects present signatures. Translated summary-history requests still require signatures. Function-tool support includes definitions, selection, parallel-call control, successful results, conversation history, and streamed JSON arguments. Native routes preserve hosted web search. Basic hosted search translation is supported through Responses and Messages upstreams, with text-only projection for downstream Chat when reasoning is absent or unsigned; signed or encrypted reasoning raises an error; translated Chat upstreams reject normalized search declarations and history, while opaque `extra_body.web_search_options` is forwarded and returned annotations are dropped. Other hosted tools, failed client-function result semantics outside Messages, other multimodal content, and additional provider capabilities are not complete yet. The matched Responses/Messages fixtures also map the reconstructible opaque reasoning snapshots under the assumption that OpenAI serves both endpoints; opaque state is not portable across providers, the implementation does not enforce its issuer, and the separately recorded terminal Responses ciphertext is not byte-reconstructible through Messages. Function and hosted-search definitions and tool selection use provider-neutral normalized types. Protocol-specific function schemas and tool-choice shapes remain at their adapters and clients. diff --git a/python/CONVENTIONS.md b/python/CONVENTIONS.md index 4bb984b..9cb5d7b 100644 --- a/python/CONVENTIONS.md +++ b/python/CONVENTIONS.md @@ -133,9 +133,9 @@ Preserve chunk boundaries when they are observable and covered by parity tests. Store raw reasoning in `Thinking.content` and `ThinkingDelta.content_delta`. Store reasoning summaries in `Thinking.summary` and `ThinkingDelta.summary_delta`. Responses wire events label these representations explicitly; preserve that label and require it to match `ProviderCapabilities.thinking`. Chat Completions `reasoning_content` and Messages `thinking_delta` are ambiguous, so classify them with the same capability. Project only the matching normalized representation into each protocol's reasoning field. -A Responses reasoning lifecycle must use exactly one valid representation: raw reasoning content with no summary or signature, a reasoning summary accompanied by a signature with no raw content, or a signature with no reasoning summary or raw content. Preserve that representation across its start, delta, and end state; do not combine or switch representations during the part lifecycle. A completely empty start may remain deferred until its representation is known. +A Responses reasoning output lifecycle must use exactly one representation: raw reasoning content with no summary or signature, a reasoning summary with optional signature and no raw content, or a signature with no reasoning summary or raw content. Preserve that representation across its start, delta, and end state; do not combine or switch representations during the part lifecycle. A completely empty start may remain deferred until its representation is known. -Chat Completions reasoning has no signature and may represent either raw reasoning or a summary according to the provider capability profile. Messages follows the same pairing rule as Responses: raw reasoning has no signature, a reasoning summary must have one, and a signature may appear without reasoning text. Enforce protocol-specific pairing at the relevant wire boundary rather than in the provider-neutral normalized models. Current cross-protocol paths through Chat therefore support only unsigned raw reasoning; reject summaries rather than discarding required opaque state. +Chat Completions reasoning has no signature and may represent either raw reasoning or a summary according to the provider capability profile. Responses and Messages output processing tolerates missing summary signatures and preserves observed opaque state. Chat output may carry unsigned raw reasoning or summaries but rejects present signatures rather than discarding them. Request normalization and construction remain strict: summary history requires its signature on translated Responses and Messages routes. Receiving an unsigned summary does not make it replayable as history. Keep these checks at the relevant wire boundary rather than in the provider-neutral normalized models. Treat `ThinkingDelta.signature_delta` as a complete replacement snapshot of provider-specific opaque state. Never concatenate it with `Thinking.signature`. diff --git a/python/molto/adapters/anthropic.py b/python/molto/adapters/anthropic.py index 2da49b5..bd963e5 100644 --- a/python/molto/adapters/anthropic.py +++ b/python/molto/adapters/anthropic.py @@ -881,11 +881,7 @@ def event(data: dict[str, Any]) -> AnthropicMessagesStreamEvent: raise ValueError( "An Anthropic summary block cannot contain raw reasoning." ) - if not part.signature: - raise ValueError( - "An Anthropic reasoning summary requires a signature." - ) - if thinking_signatures.get(index) != part.signature: + if thinking_signatures.get(index) != (part.signature or None): raise ValueError( "An Anthropic reasoning signature must be streamed before " "the block ends." diff --git a/python/molto/adapters/openai.py b/python/molto/adapters/openai.py index 41af16d..f72e3c2 100644 --- a/python/molto/adapters/openai.py +++ b/python/molto/adapters/openai.py @@ -982,8 +982,6 @@ def response_data( ) elif part.content: raise ValueError("Summarized Responses reasoning cannot contain raw text.") - if part.summary and not part.signature: - raise ValueError("A Responses reasoning summary requires a signature.") representation = self._capabilities.thinking completed_item = { "id": item_id, diff --git a/python/molto/upstream/anthropic.py b/python/molto/upstream/anthropic.py index db98479..a38f185 100644 --- a/python/molto/upstream/anthropic.py +++ b/python/molto/upstream/anthropic.py @@ -852,10 +852,6 @@ def start_part( continue part = state.part if isinstance(part, Thinking): - if self._capabilities.thinking == "summary" and not state.signature: - raise ValueError( - "An Anthropic reasoning summary requires a signature." - ) completed_part: ( Thinking | TextOutput | ToolCall | SearchCall | SearchResult ) = self._classify_thinking(state.content, signature=state.signature) diff --git a/python/molto/upstream/openai.py b/python/molto/upstream/openai.py index e9f2e7d..1d138e2 100644 --- a/python/molto/upstream/openai.py +++ b/python/molto/upstream/openai.py @@ -178,12 +178,6 @@ def _reasoning_part(item: Any) -> Thinking: ) -def _validate_completed_responses_reasoning(part: Thinking) -> Thinking: - if part.summary and not part.signature: - raise ValueError("A Responses reasoning summary requires a signature.") - return part - - def _validate_responses_reasoning_capability( part: Thinking, representation: Literal["raw", "summary"] ) -> None: @@ -208,7 +202,7 @@ def _started_output_item(item: Any) -> Thinking | TextOutput | ToolCall: def _completed_output_item(item: Any) -> Thinking | TextOutput | ToolCall: """Project a completed Responses output item into its normalized part.""" if item.type == "reasoning": - return _validate_completed_responses_reasoning(_reasoning_part(item)) + return _reasoning_part(item) if item.type == "message": text = "" citations = [] diff --git a/tests/adapters/test_anthropic.py b/tests/adapters/test_anthropic.py index 6182f0c..a81f546 100644 --- a/tests/adapters/test_anthropic.py +++ b/tests/adapters/test_anthropic.py @@ -567,22 +567,35 @@ async def test_projection_rejects_reasoning_outside_capability() -> None: ] -async def test_projection_rejects_unsigned_summary_completion() -> None: +@pytest.mark.parametrize("signature", [None, ""]) +async def test_projection_preserves_unsigned_summary_completion(signature: str | None) -> None: + """Ensures missing output signatures do not discard streamed reasoning summaries.""" adapter, request = _adapter() - with pytest.raises(ValueError, match="summary requires a signature"): - [ - event - async for event in adapter.project_events( - _stream( - _MESSAGE_START, - PartStartEvent(index=0, part=Thinking(content="", summary="")), - PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), - PartEndEvent(index=0, part=Thinking(content="", summary="Summary")), + events = [ + event + async for event in adapter.project_events( + _stream( + _MESSAGE_START, + PartStartEvent( + index=0, part=Thinking(content="", summary="", signature=signature) ), - request, - ) - ] + PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), + PartEndEvent( + index=0, part=Thinking(content="", summary="Summary", signature=signature) + ), + _MESSAGE_END, + ), + request, + ) + ] + + deltas = [event.delta for event in events if event.type == "content_block_delta"] + assert [delta.type for delta in deltas] == ["thinking_delta"] + assert deltas[0].thinking == "Summary" + response = await adapter.project_response(_native_stream(*events), request) + assert response.content[0].thinking == "Summary" + assert response.content[0].signature == "" async def test_projection_preserves_non_empty_part_starts() -> None: diff --git a/tests/adapters/test_openai.py b/tests/adapters/test_openai.py index 29e6547..ef892a9 100644 --- a/tests/adapters/test_openai.py +++ b/tests/adapters/test_openai.py @@ -982,22 +982,39 @@ def test_responses_normalization_rejects_unsigned_summary_history() -> None: OpenAIResponsesAdapter(_CAPABILITIES).normalize_request(request) -async def test_responses_projection_rejects_unsigned_summary_completion() -> None: +@pytest.mark.parametrize("signature", [None, ""]) +async def test_responses_projection_preserves_unsigned_summary_completion( + signature: str | None, +) -> None: + """Ensures Responses completes unsigned reasoning summaries without inventing opaque state.""" adapter, request = _responses_adapter(_CAPABILITIES) - with pytest.raises(ValueError, match="summary requires a signature"): - [ - event - async for event in adapter.project_events( - _stream( - _RESPONSE_START, - PartStartEvent(index=0, part=Thinking(content="", summary="")), - PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), - PartEndEvent(index=0, part=Thinking(content="", summary="Summary")), + events = [ + event + async for event in adapter.project_events( + _stream( + _RESPONSE_START, + PartStartEvent( + index=0, part=Thinking(content="", summary="", signature=signature) ), - request, - ) - ] + PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), + PartEndEvent( + index=0, part=Thinking(content="", summary="Summary", signature=signature) + ), + _RESPONSE_END, + ), + request, + ) + ] + + assert [ + event.delta for event in events if event.type == "response.reasoning_summary_text.delta" + ] == ["Summary"] + completed = next(event.item for event in events if event.type == "response.output_item.done") + assert completed.summary[0].text == "Summary" + assert completed.encrypted_content is None + assert events[-1].type == "response.completed" + assert events[-1].response.output[0] == completed async def test_responses_projection_rejects_thinking_part_with_content_and_summary() -> None: diff --git a/tests/core/test_unsigned_reasoning.py b/tests/core/test_unsigned_reasoning.py new file mode 100644 index 0000000..3cf5d84 --- /dev/null +++ b/tests/core/test_unsigned_reasoning.py @@ -0,0 +1,161 @@ +"""SDK parity for synthetic unsigned reasoning derived from recorded streams.""" + +from itertools import permutations + +import pytest +from anthropic import AsyncAnthropic +from httpx import AsyncClient, MockTransport, Request, Response +from openai import AsyncOpenAI + +from molto import Molto +from molto.types.exceptions import UnsupportedInput +from molto.utils.io import dumps_sse, json_loads, load_sse +from tests.core.test_interface import ( + PROTOCOLS, + Protocol, + _coordinator, + _matched_cassette_path, + _read_cassette, + _sse_records, +) +from tests.core.test_web_search import _assert_lifecycle, _collect + + +def _unsigned_events(protocol: Protocol) -> list[dict]: + scenario = "reasoning_raw" if protocol == "chat/completions" else "reasoning_encrypted" + cassette = _read_cassette(_matched_cassette_path(scenario, protocol, "stream-00.yaml")) + events = [load_sse(record) for record in _sse_records(cassette, protocol)] + + def remove_signatures(value): + if isinstance(value, dict): + value.pop("encrypted_content", None) + if "signature" in value: + value["signature"] = "" + for child in value.values(): + remove_signatures(child) + elif isinstance(value, list): + for child in value: + remove_signatures(child) + + # These mutations model synthetic unsigned output, not observed provider behavior. + # Chat has no signature field; its unchanged wire reasoning is classified by + # the OpenAI summary capability selected by the public call below. + events = [ + event + for event in events + if event.get("type") != "content_block_delta" + or event["delta"]["type"] != "signature_delta" + ] + remove_signatures(events) + return events + + +def _body(protocol: Protocol, stream: bool) -> dict: + body = {"model": "test-model", "stream": stream} + if protocol == "responses": + body.update(input="Question", max_output_tokens=2048) + else: + body["messages"] = [{"role": "user", "content": "Question"}] + body["max_tokens" if protocol == "messages" else "max_completion_tokens"] = 2048 + return body + + +def _expected_request(downstream: Protocol, upstream: Protocol) -> dict: + body = _body(upstream, True) + if upstream == "responses": + body["input"] = [{"role": "user", "content": [{"type": "input_text", "text": "Question"}]}] + else: + body["messages"][0]["content"] = [{"type": "text", "text": "Question"}] + if upstream == "chat/completions": + body["stream_options"] = {"include_usage": True} + if downstream == "messages": + body.setdefault("stream_options", {})["include_obfuscation"] = False + return body + + +def _reasoning_and_text(protocol: Protocol, response) -> tuple[str, str]: + if protocol == "responses": + reasoning = [item for item in response.output if item.type == "reasoning"] + assert all(not item.encrypted_content for item in reasoning) + return "".join( + part.text for item in reasoning for part in item.summary + ), response.output_text + if protocol == "messages": + reasoning = [block for block in response.content if block.type == "thinking"] + assert all(not block.signature for block in reasoning) + return ( + "".join(block.thinking for block in reasoning), + "".join(block.text for block in response.content if block.type == "text"), + ) + message = response.choices[0].message + return message.reasoning_content, message.content + + +@pytest.mark.parametrize(("downstream", "upstream"), list(permutations(PROTOCOLS, 2))) +@pytest.mark.parametrize("stream", [True, False]) +async def test_unsigned_reasoning_output_matches_sdk( + downstream: Protocol, upstream: Protocol, stream: bool +): + """Ensures unsigned summaries survive all translated routes but remain invalid replay history.""" + events = _unsigned_events(upstream) + payload = "".join(dumps_sse(event, event.get("type", "")) for event in events) + expected_request = _expected_request(downstream, upstream) + requests = [] + + def handler(request: Request) -> Response: + requests.append(request) + assert json_loads(request.content) == expected_request + return Response(200, text=payload, headers={"content-type": "text/event-stream"}) + + async with AsyncClient(transport=MockTransport(handler)) as http_client: + if upstream == "messages": + sdk = AsyncAnthropic(api_key="test-api-key", http_client=http_client) + create = sdk.messages.create + else: + sdk = AsyncOpenAI(api_key="test-api-key", http_client=http_client) + create = ( + sdk.responses.create if upstream == "responses" else sdk.chat.completions.create + ) + async with await create(**expected_request) as sdk_stream: + reference = _collect(upstream, [event async for event in sdk_stream]) + + client = Molto(http_client=http_client) + result = await _coordinator(client, downstream)( + body=_body(downstream, stream), + endpoint=upstream, + provider="openai", + api_key="test-api-key", + ) + if stream: + projected = [event async for event in result] + _assert_lifecycle(downstream, projected) + result = _collect(downstream, projected) + + expected = _reasoning_and_text(upstream, reference) + assert all(expected) + assert _reasoning_and_text(downstream, result) == expected + assert len(requests) == 2 + + if downstream != "chat/completions": + history_body = _body(downstream, stream) + if downstream == "responses": + history_body["input"] = [ + item.model_dump(exclude_none=True) for item in result.output + ] + else: + history_body["messages"] = [ + { + "role": "assistant", + "content": [ + block.model_dump(exclude_none=True) for block in result.content + ], + } + ] + with pytest.raises(UnsupportedInput, match="summary requires a signature"): + await _coordinator(client, downstream)( + body=history_body, + endpoint=upstream, + provider="openai", + api_key="test-api-key", + ) + assert len(requests) == 2 diff --git a/tests/upstream/test_anthropic.py b/tests/upstream/test_anthropic.py index d5d0640..0ae2429 100644 --- a/tests/upstream/test_anthropic.py +++ b/tests/upstream/test_anthropic.py @@ -594,9 +594,10 @@ async def test_client_preserves_signature_only_thinking_block( ] -async def test_client_rejects_unsigned_summary_completion( +async def test_client_preserves_unsigned_summary_completion( monkeypatch: pytest.MonkeyPatch, ) -> None: + """Ensures an unsigned provider thinking block retains its summary and balanced lifecycle.""" native_events = [ _MESSAGE_START, { @@ -612,10 +613,15 @@ async def test_client_rejects_unsigned_summary_completion( {"type": "content_block_stop", "index": 0}, ] - with pytest.raises(ValueError, match="summary requires a signature"): - await _collect_client_events( - monkeypatch, native_events, PROVIDER_CAPABILITIES["anthropic"] - ) + events = await _collect_client_events( + monkeypatch, native_events, PROVIDER_CAPABILITIES["anthropic"] + ) + + assert events[1:] == [ + PartStartEvent(index=0, part=Thinking(content="", summary="")), + PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), + PartEndEvent(index=0, part=Thinking(content="", summary="Summary")), + ] async def test_client_completes_thinking_block_without_deltas( diff --git a/tests/upstream/test_openai.py b/tests/upstream/test_openai.py index df84ec4..5d29762 100644 --- a/tests/upstream/test_openai.py +++ b/tests/upstream/test_openai.py @@ -2288,6 +2288,84 @@ def test_responses_client_rejects_unsigned_summary_history() -> None: OpenAIResponsesClient._request_body(request) +@pytest.mark.parametrize( + "signature_fields", [{}, {"encrypted_content": None}, {"encrypted_content": ""}] +) +async def test_responses_events_preserve_unsigned_summary_completion( + monkeypatch: pytest.MonkeyPatch, signature_fields: dict[str, str | None] +) -> None: + """Ensures absent, null, or empty output ciphertext does not discard a reasoning summary.""" + native_events = [ + _response_event( + { + "type": "response.output_item.added", + "item": { + "id": "rs_123", + "type": "reasoning", + "summary": [], + **signature_fields, + }, + "output_index": 0, + "sequence_number": 0, + } + ), + _response_event( + { + "type": "response.reasoning_summary_text.delta", + "summary_index": 0, + "delta": "Summary", + "item_id": "rs_123", + "output_index": 0, + "sequence_number": 1, + } + ), + _response_event( + { + "type": "response.output_item.done", + "item": { + "id": "rs_123", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Summary"}], + **signature_fields, + }, + "output_index": 0, + "sequence_number": 2, + } + ), + ] + + async def stub_native_events( + native_request: OpenAIResponsesCreateRequest, + ) -> AsyncIterator[OpenAIResponsesStreamEvent]: + for event in native_events: + yield event + + async with httpx.AsyncClient(transport=httpx.MockTransport(_unexpected_http)) as http_client: + client = OpenAIResponsesClient(http_client, capabilities=OPENAI_CAPABILITIES) + monkeypatch.setattr(client, "native_events", stub_native_events) + events = [event async for event in client.events(NormalizedRequest(model="m"))] + + assert events[0] == PartStartEvent( + index=0, + part=Thinking( + content="", + summary="", + id="rs_123", + signature=signature_fields.get("encrypted_content"), + ), + ) + assert events[1] == PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")) + assert events[-1] == PartEndEvent( + index=0, + part=Thinking( + content="", + summary="Summary", + id="rs_123", + signature=signature_fields.get("encrypted_content"), + ), + ) + + async def test_responses_events_preserve_reasoning_added_snapshot_and_deltas( monkeypatch: pytest.MonkeyPatch, ) -> None: From 7838929641162dc6e9c29d4f12298776d78c326d Mon Sep 17 00:00:00 2001 From: Tan Jia Huei Date: Thu, 29 Oct 2026 12:02:03 +0000 Subject: [PATCH 5/5] revert: Keep PR 89 focused on hosted web search --- docs/README.md | 6 +- docs/protocols/anthropic-messages.md | 8 +- docs/protocols/comparison.md | 2 +- docs/protocols/openai-chat-completions.md | 6 +- docs/protocols/openai-responses.md | 10 +- docs/web-search.md | 4 +- python/ARCHITECTURE.md | 8 +- python/CONVENTIONS.md | 4 +- python/molto/adapters/anthropic.py | 6 +- python/molto/adapters/openai.py | 2 + python/molto/upstream/anthropic.py | 4 + python/molto/upstream/openai.py | 8 +- tests/adapters/test_anthropic.py | 39 ++---- tests/adapters/test_openai.py | 43 ++---- tests/core/test_unsigned_reasoning.py | 161 ---------------------- tests/upstream/test_anthropic.py | 16 +-- tests/upstream/test_openai.py | 78 ----------- 17 files changed, 73 insertions(+), 332 deletions(-) delete mode 100644 tests/core/test_unsigned_reasoning.py diff --git a/docs/README.md b/docs/README.md index 41333c8..df8c7a9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -207,7 +207,7 @@ The translated path currently supports: - string input and ordered text messages; - ordered mixed text and image user input using remote URLs or base64 data URLs; - system prompts and supported assistant history; -- raw reasoning across all three protocols, reasoning summaries between Responses and Messages, and unsigned summary output through Chat; +- raw reasoning across all three protocols, and reasoning summaries on supported translations between Responses and Messages; - client-defined function tools, selection, parallel-call control, calls, successful results, history, and streamed argument deltas; - basic [hosted web search](web-search.md), URL attribution between Responses and Messages, and mandatory search with text output for downstream Chat when reasoning is absent or unsigned; signed or encrypted reasoning is rejected; - model, output-token limit, reasoning effort, temperature, and `top_p`; @@ -230,9 +230,9 @@ The normalized representation distinguishes raw reasoning from reasoning summari Responses events label those representations explicitly and must agree with the configured provider capability. Chat Completions `reasoning_content` and Messages `thinking_delta` are ambiguous, so the capability profile classifies them before normalization. -A Responses reasoning output lifecycle contains raw reasoning without a summary or signature, a summary with an optional signature and no raw reasoning, or a signature without reasoning text. A Messages thinking output lifecycle follows the same rule. A completely empty start may be deferred until its representation is known, and a lifecycle must not switch representations. +A Responses reasoning lifecycle must contain raw reasoning without a summary or signature, a summary accompanied by a signature without raw reasoning, or a signature without reasoning text. A Messages thinking lifecycle follows the same pairing rule. A completely empty start may be deferred until its representation is known, and a lifecycle must not switch representations. -Output processing preserves unsigned summaries rather than rejecting a response because its signature is missing. Chat can carry unsigned raw reasoning or summaries according to the provider profile, but rejects present signatures because it has no signature field. Request translation remains strict: replaying summary history through Responses or Messages requires its original signature. Receiving unsigned reasoning output does not establish usable continuation state. +Although a provider profile can classify Chat `reasoning_content` as a summary, Chat has no signature field. Current translated routes through Chat therefore support only unsigned raw reasoning. Summary translation through Chat fails at the Responses or Messages signature boundary rather than silently discarding required continuation state. ### Opaque continuation state diff --git a/docs/protocols/anthropic-messages.md b/docs/protocols/anthropic-messages.md index 9a0de8c..09514fb 100644 --- a/docs/protocols/anthropic-messages.md +++ b/docs/protocols/anthropic-messages.md @@ -278,13 +278,13 @@ When the application speaks Messages, an inbound thinking block is normalized as Adaptive requests always use summarized display and an explicit effort. A manual `budget_tokens` value must be less than `max_tokens`; Molto rejects the request before transport when the budget is invalid. Reasoning history enables thinking when the current request omits reasoning configuration, using high effort for adaptive models or the default 10,000-token manual budget. -Inbound `thinking_delta` classification is a separate concern from the outbound adaptive-versus-manual model policy. A Messages thinking output lifecycle contains one of: +Inbound `thinking_delta` classification is a separate concern from the outbound adaptive-versus-manual model policy. A valid Messages thinking lifecycle contains one of: - raw reasoning without a signature; -- a reasoning summary with an optional signature; or +- a reasoning summary with a signature; or - a signature without reasoning text. -It must not mix or switch representations within one block. Output processing accepts a missing summary signature and projects it as an empty Messages signature without inventing continuation state. Request translation still rejects unsigned summary history before transport. Official Anthropic continuation requires the original Anthropic-issued signature. Molto's local request model also permits an absent signature for compatible providers such as vLLM; this does not make unsigned thinking history valid against Anthropic's official API. +It must not mix or switch representations within one block. Official Anthropic continuation requires the original Anthropic-issued signature. Molto's local request model also permits an absent signature for compatible providers such as vLLM; this does not make unsigned thinking history valid against Anthropic's official API. ### Opaque thinking state @@ -368,7 +368,7 @@ When Messages is the translated upstream and the provider is Anthropic, missing, Cross-protocol Messages support includes streaming and non-streaming requests for the explicitly mapped text, portable image input, thinking, custom function tools, direct tool calls, successful tool results, limited failed-result paths, conversation history, lifecycle metadata, termination, and usage semantics described above. -A route targeting Messages must supply a downstream output-token limit because Messages requires `max_tokens`. Unsigned reasoning summary output can pass through Chat Completions, but present signatures cannot because Chat has no signature slot. Translated summary-history requests still require signatures. Molto also does not enforce the same-provider requirement for opaque continuation state; callers must keep signatures with their issuing provider. +A route targeting Messages must supply a downstream output-token limit because Messages requires `max_tokens`. Reasoning summaries are not currently portable through Chat Completions in either direction because Chat has no signature slot. Molto also does not enforce the same-provider requirement for opaque continuation state; callers must keep signatures with their issuing provider. The normalized path does not currently implement image output, non-web citations, refusals as content blocks, redacted thinking, hosted tools other than basic web search, structured output, or multiple candidates. Request fields accepted by the local Messages wrapper are not cross-protocol features unless the adapter and client map them explicitly. diff --git a/docs/protocols/comparison.md b/docs/protocols/comparison.md index d018ffe..20cb101 100644 --- a/docs/protocols/comparison.md +++ b/docs/protocols/comparison.md @@ -51,7 +51,7 @@ Responses is the richer lifecycle protocol. Translating into Responses requires ## Hosted-search fidelity -Responses and Messages translate basic search calls, results, and URL attribution. The Anthropic Messages-to-Responses route preserves excerpts and opaque references in `provider_metadata.anthropic` for callers that echo the extension in history; standard Responses fields alone cannot reconstruct them. Translated Chat rejects signed or encrypted reasoning, including signature-only parts. This prevents translating the OpenAI encrypted-reasoning search stream and Anthropic search turns with signed thinking to Chat; use Responses or Messages downstream instead. Search turns without reasoning or with unsigned raw reasoning or summaries remain representable, retaining answer text and inline links while omitting hosted activity and structured citations. The returned reasoning data determines rejection, not the search declaration. Native Chat passthrough preserves its provider output. See [Hosted web search](../web-search.md) for directional mappings and limitations. +Responses and Messages translate basic search calls, results, and URL attribution. The Anthropic Messages-to-Responses route preserves excerpts and opaque references in `provider_metadata.anthropic` for callers that echo the extension in history; standard Responses fields alone cannot reconstruct them. Translated Chat rejects signed or encrypted reasoning, including signature-only parts. This prevents translating the OpenAI encrypted-reasoning search stream and Anthropic search turns with signed thinking to Chat; use Responses or Messages downstream instead. Search turns without reasoning or with unsigned raw reasoning remain representable, retaining answer text and inline links while omitting hosted activity and structured citations. The returned reasoning data determines rejection, not the search declaration. Native Chat passthrough preserves its provider output. See [Hosted web search](../web-search.md) for directional mappings and limitations. ## Role and placement diff --git a/docs/protocols/openai-chat-completions.md b/docs/protocols/openai-chat-completions.md index e0ba8b6..0cc79f2 100644 --- a/docs/protocols/openai-chat-completions.md +++ b/docs/protocols/openai-chat-completions.md @@ -226,7 +226,7 @@ The provider or model renderer can still merge, move, or reinterpret accepted wi ### Reasoning and continuation state -Chat's single `reasoning_content` extension can be classified as either the normalized raw reasoning field or summary field according to the provider profile. Chat has no reasoning signature or encrypted continuation slot. Translated output supports unsigned raw reasoning and summaries. Responses and Messages output projection accepts a Chat summary without inventing a signature; in the reverse direction, Chat projection rejects any present signature rather than silently discarding it. Request translation still rejects unsigned summary history when normalizing or constructing a Responses or Messages request, so receiving summary text does not establish replayable continuation state. +Chat's single `reasoning_content` extension can be classified as either the normalized raw reasoning field or summary field according to the provider profile. Chat has no reasoning signature or encrypted continuation slot. In the current implementation, only unsigned raw reasoning is portable through Chat on a translated route. A Chat summary has no signature and therefore cannot satisfy the completed-summary contract required by Responses or Messages; in the reverse direction, a Responses or Messages summary carries a signature that Chat projection rejects rather than silently discarding. Opaque continuation state and summarized reasoning are therefore not currently portable through Chat. ### Tools and tool results @@ -298,7 +298,7 @@ Native Chat routes forward `web_search_options` from the request or `extra_body` Use `web_search_options: {}` with a Responses or Messages upstream to require hosted search. Translated routes targeting a Chat upstream reject hosted search before transport. -Translated Chat raises `ValueError("Chat Completions cannot preserve reasoning signatures.")` when upstream output contains signed thinking or encrypted reasoning, including signature-only parts. The OpenAI hosted-search streaming recording contains encrypted reasoning; Anthropic hosted-search turns with signed thinking have the same limitation. Use Responses or Messages downstream for these turns. Search without reasoning or with unsigned raw reasoning or summaries remains representable; enabling search alone does not cause rejection. Native Chat passthrough is unaffected. +Translated Chat raises `ValueError("Chat Completions cannot preserve reasoning signatures.")` when upstream output contains signed thinking or encrypted reasoning, including signature-only parts. The OpenAI hosted-search streaming recording contains encrypted reasoning; Anthropic hosted-search turns with signed thinking have the same limitation. Use Responses or Messages downstream for these turns. Search without reasoning or with unsigned raw reasoning remains representable; enabling search alone does not cause rejection. Native Chat passthrough is unaffected. For representable output, translated Chat retains answer text and inline links while omitting hosted activity and structured citations. See [Hosted web search](../web-search.md) for tool selection, option mappings, and continuation limits. @@ -307,7 +307,7 @@ For representable output, translated Chat retains answer text and inline links w These are implementation gaps rather than inherent claims about the Chat protocol: - An upstream Chat stream silently uses `choices[0]`; additional choices are not rejected and are lost. Requests with `n > 1` should not use translation. -- Chat cannot carry signed reasoning output. Unsigned summary output is supported, but translated Responses and Messages requests still require signatures when replaying summary history. +- Translated reasoning through Chat currently supports only unsigned raw reasoning, such as the `vllm` provider profile. Summary profiles fail at the target protocol's signature boundary in either direction. - Refusals, hosted tools other than basic web search, moderation, non-web annotations, audio, prediction output, and log probabilities are not carried through the normalized path. Hosted-search structured annotations are deliberately omitted from translated Chat output. - Chat text chunks therefore use `logprobs=None`, even when another protocol could theoretically represent compatible token log probabilities. - Chat has no normalized per-part status. Target lifecycle and status fields are synthesized where required, and pauses map to `finish_reason="length"`; other terminal reasons without an exact Chat mapping become `finish_reason="stop"`. diff --git a/docs/protocols/openai-responses.md b/docs/protocols/openai-responses.md index 18178fd..34ef18e 100644 --- a/docs/protocols/openai-responses.md +++ b/docs/protocols/openai-responses.md @@ -257,15 +257,15 @@ Responses is explicit where Chat Completions and Messages are ambiguous: Every explicit Responses event must agree with the selected provider capability profile. OpenAI is classified as summarized reasoning and vLLM as raw reasoning. A base URL never chooses this behavior. -A Responses reasoning output lifecycle uses one of these representations: +A Responses reasoning lifecycle must use exactly one valid representation: - raw reasoning content with no summary or encrypted state; -- a reasoning summary with optional encrypted state and no raw content; or +- a reasoning summary accompanied by encrypted state and no raw content; or - encrypted state with neither summary nor raw content. -The item must not combine or switch representations between its added, delta, and done states. A completely empty start may remain deferred until its representation becomes known. Output processing accepts completed summaries without encrypted state. Request translation still rejects unsigned summary history before transport. +The item must not combine or switch representations between its added, delta, and done states. A completely empty start may remain deferred until its representation becomes known. When Responses is the downstream target, summary providers require a signature for a completed reasoning summary. -Chat Completions has one `reasoning_content` extension and no signature. Messages has one `thinking_delta` field plus signature lifecycle. Their configured provider capability decides whether their text is classified as Responses raw reasoning or summary. Unsigned Chat reasoning output can project into Responses as raw reasoning or a summary according to that capability. Responses-to-Chat output rejects any present signature instead of dropping it; unsigned summary output is representable. Supported delta boundaries remain observable and are preserved by matched parity coverage. +Chat Completions has one `reasoning_content` extension and no signature. Messages has one `thinking_delta` field plus signature lifecycle. Their configured provider capability decides whether their text is classified as Responses raw reasoning or summary. Only unsigned raw Chat reasoning is currently portable into Responses: a Chat summary cannot supply the signature required at Responses part completion. Responses-to-Chat summary translation also fails because Chat rejects the source signature instead of dropping it. Supported delta boundaries remain observable and are preserved by matched parity coverage. ### Opaque continuation state @@ -390,7 +390,7 @@ A Responses stream can fail after HTTP `200`: the SDK raises `APIError` on the c The following are implementation gaps, not general claims that the source data cannot exist: - multiple Chat Completions choices; the current translated client reads only `choices[0]`; -- signed reasoning output through Chat Completions and unsigned summary history on translated requests; Chat has no signature slot, while Responses and Messages request translation requires summary signatures; +- summarized reasoning through Chat Completions; Chat has no signature slot, so current translation fails in either direction rather than discarding required continuation state; - refusals as content, hosted tools other than basic search, failed client-function result semantics, redacted thinking, non-web annotations, moderation, and other multimodal output; - log probability transport across the normalized representation; - Chat audio and accepted or rejected prediction-token details; diff --git a/docs/web-search.md b/docs/web-search.md index d3f63c9..e1b0ff7 100644 --- a/docs/web-search.md +++ b/docs/web-search.md @@ -8,7 +8,7 @@ Molto preserves provider-hosted web search on native routes and translates basic | Messages | Search, answer, and reduced URL attribution | Native passthrough | Rejected before transport | | Chat Completions | Mandatory search; signed reasoning rejected | Mandatory search; signed reasoning rejected | Native passthrough | -Translated Chat cannot preserve reasoning signatures or encrypted reasoning. The OpenAI Responses streaming search recording contains encrypted reasoning, and Anthropic Messages search turns can contain signed thinking; these turns raise `ValueError("Chat Completions cannot preserve reasoning signatures.")`. Use Responses or Messages downstream to retain that output. Search without reasoning or with unsigned raw reasoning or summaries remains representable. The `vllm` profile supports unsigned raw reasoning; provider support for hosted search is separate. Rejection depends on the returned reasoning data, not whether search was requested. Native Chat passthrough is unaffected. +Translated Chat cannot preserve reasoning signatures or encrypted reasoning. The OpenAI Responses streaming search recording contains encrypted reasoning, and Anthropic Messages search turns can contain signed thinking; these turns raise `ValueError("Chat Completions cannot preserve reasoning signatures.")`. Use Responses or Messages downstream to retain that output. Search without reasoning or with unsigned raw reasoning remains representable. The `vllm` profile supports unsigned raw reasoning; provider support for hosted search is separate. Rejection depends on the returned reasoning data, not whether search was requested. Native Chat passthrough is unaffected. Messages callers receive translated URL citations with `Molto.messages(..., citation_policy="buffer")`, or with `citation_policy="auto"` when the normalized request declares hosted search or carries search or citation history. The default `"drop"` streams text without citations; native Messages passthrough preserves citations under every policy. @@ -100,7 +100,7 @@ Buffering increases text latency and changes chunk boundaries. All policies reta Each buffered Messages text block starts with empty text and its citations, then emits the full block text in one `text_delta` before stopping. This preserves compatibility with the Anthropic SDK's `text_stream` iterator. -Translated Chat omits structured annotations in both modes. It preserves actual answer text and existing inline links, without inserting replacement citations or narration. Signed or encrypted reasoning raises the error described above, including signature-only parts without visible reasoning text. Unsigned raw reasoning and summary output are representable; translated summary-history requests still require signatures. +Translated Chat omits structured annotations in both modes. It preserves actual answer text and existing inline links, without inserting replacement citations or narration. Signed or encrypted reasoning raises the error described above, including signature-only parts without visible reasoning text. Unsigned reasoning follows the existing protocol restrictions. ## Search activity and results diff --git a/python/ARCHITECTURE.md b/python/ARCHITECTURE.md index 2972fd0..c9e1283 100644 --- a/python/ARCHITECTURE.md +++ b/python/ARCHITECTURE.md @@ -10,7 +10,7 @@ Three upstream endpoints are implemented: - `chat/completions`: native Chat Completions passthrough or cross-protocol translation through Chat Completions; - `messages`: native Anthropic Messages passthrough or translated through the normalized representation. -Native same-protocol and cross-protocol routes support streaming and non-streaming requests. Cross-protocol routes cover text, portable image input, raw reasoning, reasoning summaries between Responses and Messages, function tools, tool calls and results, conversation history, prompt-cache intent, lifecycle metadata, and token usage. Translated output through Chat supports unsigned raw reasoning and summaries; present signatures remain unsupported because Chat has no signature field. Translated summary-history requests still require signatures. +Native same-protocol and cross-protocol routes support streaming and non-streaming requests. Cross-protocol routes cover text, portable image input, raw reasoning, reasoning summaries between Responses and Messages, function tools, tool calls and results, conversation history, prompt-cache intent, lifecycle metadata, and token usage. Because Chat Completions has no signature field, translated routes through Chat currently support only unsigned raw reasoning. ## Route configuration @@ -256,7 +256,7 @@ OpenAIChatCompletionsAdapter.project_events() ChatCompletionChunk SDK stream ``` -`reasoning_content` is a provider extension to the OpenAI Chat schema, but the field itself does not distinguish raw reasoning from a reasoning summary. Molto models the extension only on assistant messages and uses `ProviderCapabilities.thinking` to classify it. OpenAI classifies it as a summary and vLLM as raw reasoning. Responses events already distinguish `response.reasoning_text.*` from `response.reasoning_summary_*`; their explicit event types remain authoritative. Output normalization and projection tolerate unsigned summaries, including those received through Chat. Present signatures remain unrepresentable in Chat, and translated Responses and Messages request construction still requires signatures for summary history. +`reasoning_content` is a provider extension to the OpenAI Chat schema, but the field itself does not distinguish raw reasoning from a reasoning summary. Molto models the extension only on assistant messages and uses `ProviderCapabilities.thinking` to classify it. OpenAI classifies it as a summary and vLLM as raw reasoning. Responses events already distinguish `response.reasoning_text.*` from `response.reasoning_summary_*`; their explicit event types remain authoritative. Classification does not make summaries portable through Chat: Responses and Messages require a signature for completed summaries, while Chat has no signature field. ## Normalized request model @@ -348,7 +348,7 @@ The normalized termination reason maps shared semantics explicitly: normal compl `SearchCall` and `SearchResult` use the existing part lifecycle and link by call identity. They are assistant history/output parts. An unfinished call may have no result, and a search error does not determine overall termination. Responses folds a call and result into one item; Messages emits separate server-tool blocks. -Chat projection rejects reasoning signatures and encrypted reasoning based on output data, including signature-only parts. Search declarations and transport overrides do not change that rule; turns without reasoning or with unsigned raw reasoning or summaries remain representable. +Chat projection rejects reasoning signatures and encrypted reasoning based on output data, including signature-only parts. Search declarations and transport overrides do not change that rule; turns without reasoning or with unsigned raw reasoning remain representable. `TextOutput.citations` holds `WebCitation` ranges local to each text part. `CitationDelta` adds attribution, and the final snapshot is authoritative. Citation buffering belongs to the Messages adapter, exposed by the `Molto.messages()` argument `citation_policy`, which accepts `"buffer"`, `"auto"`, or `"drop"`. With Responses upstream, the default `"drop"` streams text and discards attribution; `"buffer"` waits for every text part's completed snapshot, including uncited text, before splitting it at citation boundaries. `"auto"` buffers when the normalized request declares hosted search or carries search calls, search results, or text with citations in its history; otherwise text streams and unexpected citations are dropped. The coordinator selects `"drop"` for Chat upstream regardless of the requested policy, preserving immediate text streaming and chunk boundaries because Chat normalization discards annotations. The policy never changes request construction or native passthrough. The Responses normalizer and Messages projector buffer unresolved search parts in start order, allowing ordinary parts to stream when they reach the front. No upstream client predicts citation buffering, and opaque extensions cannot influence it. The `"auto"` policy inspects only the normalized request; callers supplying search context only through `extra_body` must explicitly select `"buffer"` for attribution. Projection assigns output indices because parts can expand or disappear. @@ -500,7 +500,7 @@ Parity differs by path: ## Current limitations and extension path -The current public paths support native and translated streaming and non-streaming responses for text, portable image input, raw reasoning, reasoning summaries between Responses and Messages, and client-defined function tools. Output through Chat Completions supports unsigned raw reasoning and summaries, but rejects present signatures. Translated summary-history requests still require signatures. Function-tool support includes definitions, selection, parallel-call control, successful results, conversation history, and streamed JSON arguments. Native routes preserve hosted web search. Basic hosted search translation is supported through Responses and Messages upstreams, with text-only projection for downstream Chat when reasoning is absent or unsigned; signed or encrypted reasoning raises an error; translated Chat upstreams reject normalized search declarations and history, while opaque `extra_body.web_search_options` is forwarded and returned annotations are dropped. Other hosted tools, failed client-function result semantics outside Messages, other multimodal content, and additional provider capabilities are not complete yet. The matched Responses/Messages fixtures also map the reconstructible opaque reasoning snapshots under the assumption that OpenAI serves both endpoints; opaque state is not portable across providers, the implementation does not enforce its issuer, and the separately recorded terminal Responses ciphertext is not byte-reconstructible through Messages. +The current public paths support native and translated streaming and non-streaming responses for text, portable image input, raw reasoning, reasoning summaries between Responses and Messages, and client-defined function tools. Routes through Chat Completions currently support only unsigned raw reasoning. Function-tool support includes definitions, selection, parallel-call control, successful results, conversation history, and streamed JSON arguments. Native routes preserve hosted web search. Basic hosted search translation is supported through Responses and Messages upstreams, with text-only projection for downstream Chat when reasoning is absent or unsigned; signed or encrypted reasoning raises an error; translated Chat upstreams reject normalized search declarations and history, while opaque `extra_body.web_search_options` is forwarded and returned annotations are dropped. Other hosted tools, failed client-function result semantics outside Messages, other multimodal content, and additional provider capabilities are not complete yet. The matched Responses/Messages fixtures also map the reconstructible opaque reasoning snapshots under the assumption that OpenAI serves both endpoints; opaque state is not portable across providers, the implementation does not enforce its issuer, and the separately recorded terminal Responses ciphertext is not byte-reconstructible through Messages. Function and hosted-search definitions and tool selection use provider-neutral normalized types. Protocol-specific function schemas and tool-choice shapes remain at their adapters and clients. diff --git a/python/CONVENTIONS.md b/python/CONVENTIONS.md index 9cb5d7b..4bb984b 100644 --- a/python/CONVENTIONS.md +++ b/python/CONVENTIONS.md @@ -133,9 +133,9 @@ Preserve chunk boundaries when they are observable and covered by parity tests. Store raw reasoning in `Thinking.content` and `ThinkingDelta.content_delta`. Store reasoning summaries in `Thinking.summary` and `ThinkingDelta.summary_delta`. Responses wire events label these representations explicitly; preserve that label and require it to match `ProviderCapabilities.thinking`. Chat Completions `reasoning_content` and Messages `thinking_delta` are ambiguous, so classify them with the same capability. Project only the matching normalized representation into each protocol's reasoning field. -A Responses reasoning output lifecycle must use exactly one representation: raw reasoning content with no summary or signature, a reasoning summary with optional signature and no raw content, or a signature with no reasoning summary or raw content. Preserve that representation across its start, delta, and end state; do not combine or switch representations during the part lifecycle. A completely empty start may remain deferred until its representation is known. +A Responses reasoning lifecycle must use exactly one valid representation: raw reasoning content with no summary or signature, a reasoning summary accompanied by a signature with no raw content, or a signature with no reasoning summary or raw content. Preserve that representation across its start, delta, and end state; do not combine or switch representations during the part lifecycle. A completely empty start may remain deferred until its representation is known. -Chat Completions reasoning has no signature and may represent either raw reasoning or a summary according to the provider capability profile. Responses and Messages output processing tolerates missing summary signatures and preserves observed opaque state. Chat output may carry unsigned raw reasoning or summaries but rejects present signatures rather than discarding them. Request normalization and construction remain strict: summary history requires its signature on translated Responses and Messages routes. Receiving an unsigned summary does not make it replayable as history. Keep these checks at the relevant wire boundary rather than in the provider-neutral normalized models. +Chat Completions reasoning has no signature and may represent either raw reasoning or a summary according to the provider capability profile. Messages follows the same pairing rule as Responses: raw reasoning has no signature, a reasoning summary must have one, and a signature may appear without reasoning text. Enforce protocol-specific pairing at the relevant wire boundary rather than in the provider-neutral normalized models. Current cross-protocol paths through Chat therefore support only unsigned raw reasoning; reject summaries rather than discarding required opaque state. Treat `ThinkingDelta.signature_delta` as a complete replacement snapshot of provider-specific opaque state. Never concatenate it with `Thinking.signature`. diff --git a/python/molto/adapters/anthropic.py b/python/molto/adapters/anthropic.py index bd963e5..2da49b5 100644 --- a/python/molto/adapters/anthropic.py +++ b/python/molto/adapters/anthropic.py @@ -881,7 +881,11 @@ def event(data: dict[str, Any]) -> AnthropicMessagesStreamEvent: raise ValueError( "An Anthropic summary block cannot contain raw reasoning." ) - if thinking_signatures.get(index) != (part.signature or None): + if not part.signature: + raise ValueError( + "An Anthropic reasoning summary requires a signature." + ) + if thinking_signatures.get(index) != part.signature: raise ValueError( "An Anthropic reasoning signature must be streamed before " "the block ends." diff --git a/python/molto/adapters/openai.py b/python/molto/adapters/openai.py index f72e3c2..41af16d 100644 --- a/python/molto/adapters/openai.py +++ b/python/molto/adapters/openai.py @@ -982,6 +982,8 @@ def response_data( ) elif part.content: raise ValueError("Summarized Responses reasoning cannot contain raw text.") + if part.summary and not part.signature: + raise ValueError("A Responses reasoning summary requires a signature.") representation = self._capabilities.thinking completed_item = { "id": item_id, diff --git a/python/molto/upstream/anthropic.py b/python/molto/upstream/anthropic.py index a38f185..db98479 100644 --- a/python/molto/upstream/anthropic.py +++ b/python/molto/upstream/anthropic.py @@ -852,6 +852,10 @@ def start_part( continue part = state.part if isinstance(part, Thinking): + if self._capabilities.thinking == "summary" and not state.signature: + raise ValueError( + "An Anthropic reasoning summary requires a signature." + ) completed_part: ( Thinking | TextOutput | ToolCall | SearchCall | SearchResult ) = self._classify_thinking(state.content, signature=state.signature) diff --git a/python/molto/upstream/openai.py b/python/molto/upstream/openai.py index 1d138e2..e9f2e7d 100644 --- a/python/molto/upstream/openai.py +++ b/python/molto/upstream/openai.py @@ -178,6 +178,12 @@ def _reasoning_part(item: Any) -> Thinking: ) +def _validate_completed_responses_reasoning(part: Thinking) -> Thinking: + if part.summary and not part.signature: + raise ValueError("A Responses reasoning summary requires a signature.") + return part + + def _validate_responses_reasoning_capability( part: Thinking, representation: Literal["raw", "summary"] ) -> None: @@ -202,7 +208,7 @@ def _started_output_item(item: Any) -> Thinking | TextOutput | ToolCall: def _completed_output_item(item: Any) -> Thinking | TextOutput | ToolCall: """Project a completed Responses output item into its normalized part.""" if item.type == "reasoning": - return _reasoning_part(item) + return _validate_completed_responses_reasoning(_reasoning_part(item)) if item.type == "message": text = "" citations = [] diff --git a/tests/adapters/test_anthropic.py b/tests/adapters/test_anthropic.py index a81f546..6182f0c 100644 --- a/tests/adapters/test_anthropic.py +++ b/tests/adapters/test_anthropic.py @@ -567,35 +567,22 @@ async def test_projection_rejects_reasoning_outside_capability() -> None: ] -@pytest.mark.parametrize("signature", [None, ""]) -async def test_projection_preserves_unsigned_summary_completion(signature: str | None) -> None: - """Ensures missing output signatures do not discard streamed reasoning summaries.""" +async def test_projection_rejects_unsigned_summary_completion() -> None: adapter, request = _adapter() - events = [ - event - async for event in adapter.project_events( - _stream( - _MESSAGE_START, - PartStartEvent( - index=0, part=Thinking(content="", summary="", signature=signature) - ), - PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), - PartEndEvent( - index=0, part=Thinking(content="", summary="Summary", signature=signature) + with pytest.raises(ValueError, match="summary requires a signature"): + [ + event + async for event in adapter.project_events( + _stream( + _MESSAGE_START, + PartStartEvent(index=0, part=Thinking(content="", summary="")), + PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), + PartEndEvent(index=0, part=Thinking(content="", summary="Summary")), ), - _MESSAGE_END, - ), - request, - ) - ] - - deltas = [event.delta for event in events if event.type == "content_block_delta"] - assert [delta.type for delta in deltas] == ["thinking_delta"] - assert deltas[0].thinking == "Summary" - response = await adapter.project_response(_native_stream(*events), request) - assert response.content[0].thinking == "Summary" - assert response.content[0].signature == "" + request, + ) + ] async def test_projection_preserves_non_empty_part_starts() -> None: diff --git a/tests/adapters/test_openai.py b/tests/adapters/test_openai.py index ef892a9..29e6547 100644 --- a/tests/adapters/test_openai.py +++ b/tests/adapters/test_openai.py @@ -982,39 +982,22 @@ def test_responses_normalization_rejects_unsigned_summary_history() -> None: OpenAIResponsesAdapter(_CAPABILITIES).normalize_request(request) -@pytest.mark.parametrize("signature", [None, ""]) -async def test_responses_projection_preserves_unsigned_summary_completion( - signature: str | None, -) -> None: - """Ensures Responses completes unsigned reasoning summaries without inventing opaque state.""" +async def test_responses_projection_rejects_unsigned_summary_completion() -> None: adapter, request = _responses_adapter(_CAPABILITIES) - events = [ - event - async for event in adapter.project_events( - _stream( - _RESPONSE_START, - PartStartEvent( - index=0, part=Thinking(content="", summary="", signature=signature) - ), - PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), - PartEndEvent( - index=0, part=Thinking(content="", summary="Summary", signature=signature) + with pytest.raises(ValueError, match="summary requires a signature"): + [ + event + async for event in adapter.project_events( + _stream( + _RESPONSE_START, + PartStartEvent(index=0, part=Thinking(content="", summary="")), + PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), + PartEndEvent(index=0, part=Thinking(content="", summary="Summary")), ), - _RESPONSE_END, - ), - request, - ) - ] - - assert [ - event.delta for event in events if event.type == "response.reasoning_summary_text.delta" - ] == ["Summary"] - completed = next(event.item for event in events if event.type == "response.output_item.done") - assert completed.summary[0].text == "Summary" - assert completed.encrypted_content is None - assert events[-1].type == "response.completed" - assert events[-1].response.output[0] == completed + request, + ) + ] async def test_responses_projection_rejects_thinking_part_with_content_and_summary() -> None: diff --git a/tests/core/test_unsigned_reasoning.py b/tests/core/test_unsigned_reasoning.py deleted file mode 100644 index 3cf5d84..0000000 --- a/tests/core/test_unsigned_reasoning.py +++ /dev/null @@ -1,161 +0,0 @@ -"""SDK parity for synthetic unsigned reasoning derived from recorded streams.""" - -from itertools import permutations - -import pytest -from anthropic import AsyncAnthropic -from httpx import AsyncClient, MockTransport, Request, Response -from openai import AsyncOpenAI - -from molto import Molto -from molto.types.exceptions import UnsupportedInput -from molto.utils.io import dumps_sse, json_loads, load_sse -from tests.core.test_interface import ( - PROTOCOLS, - Protocol, - _coordinator, - _matched_cassette_path, - _read_cassette, - _sse_records, -) -from tests.core.test_web_search import _assert_lifecycle, _collect - - -def _unsigned_events(protocol: Protocol) -> list[dict]: - scenario = "reasoning_raw" if protocol == "chat/completions" else "reasoning_encrypted" - cassette = _read_cassette(_matched_cassette_path(scenario, protocol, "stream-00.yaml")) - events = [load_sse(record) for record in _sse_records(cassette, protocol)] - - def remove_signatures(value): - if isinstance(value, dict): - value.pop("encrypted_content", None) - if "signature" in value: - value["signature"] = "" - for child in value.values(): - remove_signatures(child) - elif isinstance(value, list): - for child in value: - remove_signatures(child) - - # These mutations model synthetic unsigned output, not observed provider behavior. - # Chat has no signature field; its unchanged wire reasoning is classified by - # the OpenAI summary capability selected by the public call below. - events = [ - event - for event in events - if event.get("type") != "content_block_delta" - or event["delta"]["type"] != "signature_delta" - ] - remove_signatures(events) - return events - - -def _body(protocol: Protocol, stream: bool) -> dict: - body = {"model": "test-model", "stream": stream} - if protocol == "responses": - body.update(input="Question", max_output_tokens=2048) - else: - body["messages"] = [{"role": "user", "content": "Question"}] - body["max_tokens" if protocol == "messages" else "max_completion_tokens"] = 2048 - return body - - -def _expected_request(downstream: Protocol, upstream: Protocol) -> dict: - body = _body(upstream, True) - if upstream == "responses": - body["input"] = [{"role": "user", "content": [{"type": "input_text", "text": "Question"}]}] - else: - body["messages"][0]["content"] = [{"type": "text", "text": "Question"}] - if upstream == "chat/completions": - body["stream_options"] = {"include_usage": True} - if downstream == "messages": - body.setdefault("stream_options", {})["include_obfuscation"] = False - return body - - -def _reasoning_and_text(protocol: Protocol, response) -> tuple[str, str]: - if protocol == "responses": - reasoning = [item for item in response.output if item.type == "reasoning"] - assert all(not item.encrypted_content for item in reasoning) - return "".join( - part.text for item in reasoning for part in item.summary - ), response.output_text - if protocol == "messages": - reasoning = [block for block in response.content if block.type == "thinking"] - assert all(not block.signature for block in reasoning) - return ( - "".join(block.thinking for block in reasoning), - "".join(block.text for block in response.content if block.type == "text"), - ) - message = response.choices[0].message - return message.reasoning_content, message.content - - -@pytest.mark.parametrize(("downstream", "upstream"), list(permutations(PROTOCOLS, 2))) -@pytest.mark.parametrize("stream", [True, False]) -async def test_unsigned_reasoning_output_matches_sdk( - downstream: Protocol, upstream: Protocol, stream: bool -): - """Ensures unsigned summaries survive all translated routes but remain invalid replay history.""" - events = _unsigned_events(upstream) - payload = "".join(dumps_sse(event, event.get("type", "")) for event in events) - expected_request = _expected_request(downstream, upstream) - requests = [] - - def handler(request: Request) -> Response: - requests.append(request) - assert json_loads(request.content) == expected_request - return Response(200, text=payload, headers={"content-type": "text/event-stream"}) - - async with AsyncClient(transport=MockTransport(handler)) as http_client: - if upstream == "messages": - sdk = AsyncAnthropic(api_key="test-api-key", http_client=http_client) - create = sdk.messages.create - else: - sdk = AsyncOpenAI(api_key="test-api-key", http_client=http_client) - create = ( - sdk.responses.create if upstream == "responses" else sdk.chat.completions.create - ) - async with await create(**expected_request) as sdk_stream: - reference = _collect(upstream, [event async for event in sdk_stream]) - - client = Molto(http_client=http_client) - result = await _coordinator(client, downstream)( - body=_body(downstream, stream), - endpoint=upstream, - provider="openai", - api_key="test-api-key", - ) - if stream: - projected = [event async for event in result] - _assert_lifecycle(downstream, projected) - result = _collect(downstream, projected) - - expected = _reasoning_and_text(upstream, reference) - assert all(expected) - assert _reasoning_and_text(downstream, result) == expected - assert len(requests) == 2 - - if downstream != "chat/completions": - history_body = _body(downstream, stream) - if downstream == "responses": - history_body["input"] = [ - item.model_dump(exclude_none=True) for item in result.output - ] - else: - history_body["messages"] = [ - { - "role": "assistant", - "content": [ - block.model_dump(exclude_none=True) for block in result.content - ], - } - ] - with pytest.raises(UnsupportedInput, match="summary requires a signature"): - await _coordinator(client, downstream)( - body=history_body, - endpoint=upstream, - provider="openai", - api_key="test-api-key", - ) - assert len(requests) == 2 diff --git a/tests/upstream/test_anthropic.py b/tests/upstream/test_anthropic.py index 0ae2429..d5d0640 100644 --- a/tests/upstream/test_anthropic.py +++ b/tests/upstream/test_anthropic.py @@ -594,10 +594,9 @@ async def test_client_preserves_signature_only_thinking_block( ] -async def test_client_preserves_unsigned_summary_completion( +async def test_client_rejects_unsigned_summary_completion( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Ensures an unsigned provider thinking block retains its summary and balanced lifecycle.""" native_events = [ _MESSAGE_START, { @@ -613,15 +612,10 @@ async def test_client_preserves_unsigned_summary_completion( {"type": "content_block_stop", "index": 0}, ] - events = await _collect_client_events( - monkeypatch, native_events, PROVIDER_CAPABILITIES["anthropic"] - ) - - assert events[1:] == [ - PartStartEvent(index=0, part=Thinking(content="", summary="")), - PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), - PartEndEvent(index=0, part=Thinking(content="", summary="Summary")), - ] + with pytest.raises(ValueError, match="summary requires a signature"): + await _collect_client_events( + monkeypatch, native_events, PROVIDER_CAPABILITIES["anthropic"] + ) async def test_client_completes_thinking_block_without_deltas( diff --git a/tests/upstream/test_openai.py b/tests/upstream/test_openai.py index 5d29762..df84ec4 100644 --- a/tests/upstream/test_openai.py +++ b/tests/upstream/test_openai.py @@ -2288,84 +2288,6 @@ def test_responses_client_rejects_unsigned_summary_history() -> None: OpenAIResponsesClient._request_body(request) -@pytest.mark.parametrize( - "signature_fields", [{}, {"encrypted_content": None}, {"encrypted_content": ""}] -) -async def test_responses_events_preserve_unsigned_summary_completion( - monkeypatch: pytest.MonkeyPatch, signature_fields: dict[str, str | None] -) -> None: - """Ensures absent, null, or empty output ciphertext does not discard a reasoning summary.""" - native_events = [ - _response_event( - { - "type": "response.output_item.added", - "item": { - "id": "rs_123", - "type": "reasoning", - "summary": [], - **signature_fields, - }, - "output_index": 0, - "sequence_number": 0, - } - ), - _response_event( - { - "type": "response.reasoning_summary_text.delta", - "summary_index": 0, - "delta": "Summary", - "item_id": "rs_123", - "output_index": 0, - "sequence_number": 1, - } - ), - _response_event( - { - "type": "response.output_item.done", - "item": { - "id": "rs_123", - "type": "reasoning", - "summary": [{"type": "summary_text", "text": "Summary"}], - **signature_fields, - }, - "output_index": 0, - "sequence_number": 2, - } - ), - ] - - async def stub_native_events( - native_request: OpenAIResponsesCreateRequest, - ) -> AsyncIterator[OpenAIResponsesStreamEvent]: - for event in native_events: - yield event - - async with httpx.AsyncClient(transport=httpx.MockTransport(_unexpected_http)) as http_client: - client = OpenAIResponsesClient(http_client, capabilities=OPENAI_CAPABILITIES) - monkeypatch.setattr(client, "native_events", stub_native_events) - events = [event async for event in client.events(NormalizedRequest(model="m"))] - - assert events[0] == PartStartEvent( - index=0, - part=Thinking( - content="", - summary="", - id="rs_123", - signature=signature_fields.get("encrypted_content"), - ), - ) - assert events[1] == PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")) - assert events[-1] == PartEndEvent( - index=0, - part=Thinking( - content="", - summary="Summary", - id="rs_123", - signature=signature_fields.get("encrypted_content"), - ), - ) - - async def test_responses_events_preserve_reasoning_added_snapshot_and_deltas( monkeypatch: pytest.MonkeyPatch, ) -> None: