From 62e83ce0dab47093265fde9321c5b4cbaefc3378 Mon Sep 17 00:00:00 2001 From: Tan Jia Huei Date: Thu, 29 Oct 2026 12:01:41 +0000 Subject: [PATCH] fix: Accept unsigned reasoning output --- docs/README.md | 6 +- docs/protocols/anthropic-messages.md | 8 +- docs/protocols/openai-chat-completions.md | 4 +- docs/protocols/openai-responses.md | 10 +- python/ARCHITECTURE.md | 6 +- python/CONVENTIONS.md | 4 +- python/molto/adapters/anthropic.py | 8 +- python/molto/adapters/openai.py | 2 - python/molto/upstream/anthropic.py | 4 - python/molto/upstream/openai.py | 8 +- tests/adapters/test_anthropic.py | 39 ++-- tests/adapters/test_openai.py | 43 +++-- tests/core/test_unsigned_reasoning.py | 216 ++++++++++++++++++++++ tests/upstream/test_anthropic.py | 16 +- tests/upstream/test_openai.py | 78 ++++++++ 15 files changed, 384 insertions(+), 68 deletions(-) create mode 100644 tests/core/test_unsigned_reasoning.py diff --git a/docs/README.md b/docs/README.md index d43c5b4..dbcc7ba 100644 --- a/docs/README.md +++ b/docs/README.md @@ -207,7 +207,7 @@ The translated path currently supports: - string input and ordered text messages; - ordered mixed text and image user input using remote URLs or base64 data URLs; - system prompts and supported assistant history; -- raw reasoning across all three protocols, and reasoning summaries on supported translations between Responses and Messages; +- raw reasoning across all three protocols, reasoning summaries between Responses and Messages, and unsigned summary output through Chat; - client-defined function tools, selection, parallel-call control, calls, successful results, history, and streamed argument deltas; - model, output-token limit, reasoning effort, temperature, and `top_p`; - prompt-cache settings and supported text/image breakpoints under the [shared mapping policy](#prompt-caching), including silent omission of caller-defined keys on Messages; @@ -229,9 +229,9 @@ The normalized representation distinguishes raw reasoning from reasoning summari Responses events label those representations explicitly and must agree with the configured provider capability. Chat Completions `reasoning_content` and Messages `thinking_delta` are ambiguous, so the capability profile classifies them before normalization. -A Responses reasoning lifecycle must contain raw reasoning without a summary or signature, a summary accompanied by a signature without raw reasoning, or a signature without reasoning text. A Messages thinking lifecycle follows the same pairing rule. A completely empty start may be deferred until its representation is known, and a lifecycle must not switch representations. +A Responses reasoning output lifecycle contains raw reasoning without a summary or signature, a summary with an optional signature and no raw reasoning, or a signature without reasoning text. A Messages thinking output lifecycle follows the same rule. A completely empty start may be deferred until its representation is known, and a lifecycle must not switch representations. -Although a provider profile can classify Chat `reasoning_content` as a summary, Chat has no signature field. Current translated routes through Chat therefore support only unsigned raw reasoning. Summary translation through Chat fails at the Responses or Messages signature boundary rather than silently discarding required continuation state. +Output processing preserves unsigned summaries rather than rejecting a response because its signature is missing. Chat can carry unsigned raw reasoning or summaries according to the provider profile, but rejects present signatures because it has no signature field. Request translation remains strict: replaying summary history through Responses or Messages requires its original signature. Receiving unsigned reasoning output does not establish usable continuation state. ### Opaque continuation state diff --git a/docs/protocols/anthropic-messages.md b/docs/protocols/anthropic-messages.md index a8d2b83..a0c5e9d 100644 --- a/docs/protocols/anthropic-messages.md +++ b/docs/protocols/anthropic-messages.md @@ -278,13 +278,13 @@ When the application speaks Messages, an inbound thinking block is normalized as Adaptive requests always use summarized display and an explicit effort. A manual `budget_tokens` value must be less than `max_tokens`; Molto rejects the request before transport when the budget is invalid. Reasoning history enables thinking when the current request omits reasoning configuration, using high effort for adaptive models or the default 10,000-token manual budget. -Inbound `thinking_delta` classification is a separate concern from the outbound adaptive-versus-manual model policy. A valid Messages thinking lifecycle contains one of: +Inbound `thinking_delta` classification is a separate concern from the outbound adaptive-versus-manual model policy. A Messages thinking output lifecycle contains one of: - raw reasoning without a signature; -- a reasoning summary with a signature; or +- a reasoning summary with an optional signature; or - a signature without reasoning text. -It must not mix or switch representations within one block. Official Anthropic continuation requires the original Anthropic-issued signature. Molto's local request model also permits an absent signature for compatible providers such as vLLM; this does not make unsigned thinking history valid against Anthropic's official API. +It must not mix or switch representations within one block. Output processing accepts a missing summary signature and projects it as an empty Messages signature without inventing continuation state. Request translation still rejects unsigned summary history before transport. Official Anthropic continuation requires the original Anthropic-issued signature. Molto's local request model also permits an absent signature for compatible providers such as vLLM; this does not make unsigned thinking history valid against Anthropic's official API. ### Opaque thinking state @@ -360,7 +360,7 @@ Recognized errors use the Anthropic SDK namespace and the [shared error contract Cross-protocol Messages support includes streaming and non-streaming requests for the explicitly mapped text, portable image input, thinking, custom function tools, direct tool calls, successful tool results, limited failed-result paths, conversation history, lifecycle metadata, termination, and usage semantics described above. -A route targeting Messages must supply a downstream output-token limit because Messages requires `max_tokens`. Reasoning summaries are not currently portable through Chat Completions in either direction because Chat has no signature slot. Molto also does not enforce the same-provider requirement for opaque continuation state; callers must keep signatures with their issuing provider. +A route targeting Messages must supply a downstream output-token limit because Messages requires `max_tokens`. Unsigned reasoning summary output can pass through Chat Completions, but present signatures cannot because Chat has no signature slot. Translated summary-history requests still require signatures. Molto also does not enforce the same-provider requirement for opaque continuation state; callers must keep signatures with their issuing provider. The normalized path does not currently implement image output, citations, refusals as content blocks, redacted thinking, server or hosted tools, structured output, or multiple candidates. Request fields accepted by the local Messages wrapper are not cross-protocol features unless the adapter and client map them explicitly. diff --git a/docs/protocols/openai-chat-completions.md b/docs/protocols/openai-chat-completions.md index cb2dea7..f32f141 100644 --- a/docs/protocols/openai-chat-completions.md +++ b/docs/protocols/openai-chat-completions.md @@ -226,7 +226,7 @@ The provider or model renderer can still merge, move, or reinterpret accepted wi ### Reasoning and continuation state -Chat's single `reasoning_content` extension can be classified as either the normalized raw reasoning field or summary field according to the provider profile. Chat has no reasoning signature or encrypted continuation slot. In the current implementation, only unsigned raw reasoning is portable through Chat on a translated route. A Chat summary has no signature and therefore cannot satisfy the completed-summary contract required by Responses or Messages; in the reverse direction, a Responses or Messages summary carries a signature that Chat projection rejects rather than silently discarding. Opaque continuation state and summarized reasoning are therefore not currently portable through Chat. +Chat's single `reasoning_content` extension can be classified as either the normalized raw reasoning field or summary field according to the provider profile. Chat has no reasoning signature or encrypted continuation slot. Translated output supports unsigned raw reasoning and summaries. Responses and Messages output projection accepts a Chat summary without inventing a signature; in the reverse direction, Chat projection rejects any present signature rather than silently discarding it. Request translation still rejects unsigned summary history when normalizing or constructing a Responses or Messages request, so receiving summary text does not establish replayable continuation state. ### Tools and tool results @@ -297,7 +297,7 @@ Recognized errors use the OpenAI SDK namespace and the [shared error contract](. These are implementation gaps rather than inherent claims about the Chat protocol: - An upstream Chat stream silently uses `choices[0]`; additional choices are not rejected and are lost. Requests with `n > 1` should not use translation. -- Translated reasoning through Chat currently supports only unsigned raw reasoning, such as the `vllm` provider profile. Summary profiles fail at the target protocol's signature boundary in either direction. +- Chat cannot carry signed reasoning output. Unsigned summary output is supported, but translated Responses and Messages requests still require signatures when replaying summary history. - Refusals, hosted tools, moderation, annotations, audio, prediction output, and log probabilities are not carried through the normalized path. - Chat text chunks therefore use `logprobs=None`, even when another protocol could theoretically represent compatible token log probabilities. - Chat has no normalized per-part status. Target lifecycle and status fields are synthesized where required, and normalized terminal reasons without an exact Chat mapping become `finish_reason="stop"`. diff --git a/docs/protocols/openai-responses.md b/docs/protocols/openai-responses.md index 635e34f..210f1a9 100644 --- a/docs/protocols/openai-responses.md +++ b/docs/protocols/openai-responses.md @@ -257,15 +257,15 @@ Responses is explicit where Chat Completions and Messages are ambiguous: Every explicit Responses event must agree with the selected provider capability profile. OpenAI is classified as summarized reasoning and vLLM as raw reasoning. A base URL never chooses this behavior. -A Responses reasoning lifecycle must use exactly one valid representation: +A Responses reasoning output lifecycle must use exactly one representation: - raw reasoning content with no summary or encrypted state; -- a reasoning summary accompanied by encrypted state and no raw content; or +- a reasoning summary with optional encrypted state and no raw content; or - encrypted state with neither summary nor raw content. -The item must not combine or switch representations between its added, delta, and done states. A completely empty start may remain deferred until its representation becomes known. When Responses is the downstream target, summary providers require a signature for a completed reasoning summary. +The item must not combine or switch representations between its added, delta, and done states. A completely empty start may remain deferred until its representation becomes known. Output processing accepts completed summaries without encrypted state. Request translation still rejects unsigned summary history before transport. -Chat Completions has one `reasoning_content` extension and no signature. Messages has one `thinking_delta` field plus signature lifecycle. Their configured provider capability decides whether their text is classified as Responses raw reasoning or summary. Only unsigned raw Chat reasoning is currently portable into Responses: a Chat summary cannot supply the signature required at Responses part completion. Responses-to-Chat summary translation also fails because Chat rejects the source signature instead of dropping it. Supported delta boundaries remain observable and are preserved by matched parity coverage. +Chat Completions has one `reasoning_content` extension and no signature. Messages has one `thinking_delta` field plus signature lifecycle. Their configured provider capability decides whether their text is classified as Responses raw reasoning or summary. Unsigned Chat reasoning output can project into Responses as raw reasoning or a summary according to that capability. Responses-to-Chat output rejects any present signature instead of dropping it; unsigned summary output is representable. Supported delta boundaries remain observable and are preserved by matched parity coverage. ### Opaque continuation state @@ -379,7 +379,7 @@ A Responses stream can fail after HTTP `200`: the SDK raises `APIError` on the c The following are implementation gaps, not general claims that the source data cannot exist: - multiple Chat Completions choices; the current translated client reads only `choices[0]`; -- summarized reasoning through Chat Completions; Chat has no signature slot, so current translation fails in either direction rather than discarding required continuation state; +- signed reasoning output through Chat Completions and unsigned summary history on translated requests; Chat has no signature slot, while Responses and Messages request translation requires summary signatures; - refusals as content, hosted and server tools, failed tool-result semantics, redacted thinking, citations, annotations, moderation, and other multimodal output; - log probability transport across the normalized representation; - Chat audio and accepted or rejected prediction-token details; diff --git a/python/ARCHITECTURE.md b/python/ARCHITECTURE.md index 9664a86..c9f8b0d 100644 --- a/python/ARCHITECTURE.md +++ b/python/ARCHITECTURE.md @@ -10,7 +10,7 @@ Three upstream endpoints are implemented: - `chat/completions`: native Chat Completions passthrough or cross-protocol translation through Chat Completions; - `messages`: native Anthropic Messages passthrough or translated through the normalized representation. -Native same-protocol and cross-protocol routes support streaming and non-streaming requests. Cross-protocol routes cover text, portable image input, raw reasoning, reasoning summaries between Responses and Messages, function tools, tool calls and results, conversation history, prompt-cache intent, lifecycle metadata, and token usage. Because Chat Completions has no signature field, translated routes through Chat currently support only unsigned raw reasoning. +Native same-protocol and cross-protocol routes support streaming and non-streaming requests. Cross-protocol routes cover text, portable image input, raw reasoning, reasoning summaries between Responses and Messages, function tools, tool calls and results, conversation history, prompt-cache intent, lifecycle metadata, and token usage. Translated output through Chat supports unsigned raw reasoning and summaries; present signatures remain unsupported because Chat has no signature field. Translated summary-history requests still require signatures. ## Route configuration @@ -254,7 +254,7 @@ OpenAIChatCompletionsAdapter.project_events() ChatCompletionChunk SDK stream ``` -`reasoning_content` is a provider extension to the OpenAI Chat schema, but the field itself does not distinguish raw reasoning from a reasoning summary. Molto models the extension only on assistant messages and uses `ProviderCapabilities.thinking` to classify it. OpenAI classifies it as a summary and vLLM as raw reasoning. Responses events already distinguish `response.reasoning_text.*` from `response.reasoning_summary_*`; their explicit event types remain authoritative. Classification does not make summaries portable through Chat: Responses and Messages require a signature for completed summaries, while Chat has no signature field. +`reasoning_content` is a provider extension to the OpenAI Chat schema, but the field itself does not distinguish raw reasoning from a reasoning summary. Molto models the extension only on assistant messages and uses `ProviderCapabilities.thinking` to classify it. OpenAI classifies it as a summary and vLLM as raw reasoning. Responses events already distinguish `response.reasoning_text.*` from `response.reasoning_summary_*`; their explicit event types remain authoritative. Output normalization and projection tolerate unsigned summaries, including those received through Chat. Present signatures remain unrepresentable in Chat, and translated Responses and Messages request construction still requires signatures for summary history. ## Normalized request model @@ -477,7 +477,7 @@ Parity differs by path: ## Current limitations and extension path -The current public paths support native and translated streaming and non-streaming responses for text, portable image input, raw reasoning, reasoning summaries between Responses and Messages, and client-defined function tools. Routes through Chat Completions currently support only unsigned raw reasoning. Function-tool support includes definitions, selection, parallel-call control, successful results, conversation history, and streamed JSON arguments. Hosted and server tools, failed-result semantics outside Messages, other multimodal content, and additional provider capabilities are not complete yet. The matched Responses/Messages fixtures also map the reconstructible opaque reasoning snapshots under the assumption that OpenAI serves both endpoints; opaque state is not portable across providers, the implementation does not enforce its issuer, and the separately recorded terminal Responses ciphertext is not byte-reconstructible through Messages. +The current public paths support native and translated streaming and non-streaming responses for text, portable image input, raw reasoning, reasoning summaries between Responses and Messages, and client-defined function tools. Output through Chat Completions supports unsigned raw reasoning and summaries, but rejects present signatures. Translated summary-history requests still require signatures. Function-tool support includes definitions, selection, parallel-call control, successful results, conversation history, and streamed JSON arguments. Hosted and server tools, failed-result semantics outside Messages, other multimodal content, and additional provider capabilities are not complete yet. The matched Responses/Messages fixtures also map the reconstructible opaque reasoning snapshots under the assumption that OpenAI serves both endpoints; opaque state is not portable across providers, the implementation does not enforce its issuer, and the separately recorded terminal Responses ciphertext is not byte-reconstructible through Messages. Function definitions and tool selection use provider-neutral normalized types. Protocol-specific function schemas and tool-choice shapes remain at their adapters and clients. diff --git a/python/CONVENTIONS.md b/python/CONVENTIONS.md index 0a8394f..b82b929 100644 --- a/python/CONVENTIONS.md +++ b/python/CONVENTIONS.md @@ -127,9 +127,9 @@ Preserve chunk boundaries when they are observable and covered by parity tests. Store raw reasoning in `Thinking.content` and `ThinkingDelta.content_delta`. Store reasoning summaries in `Thinking.summary` and `ThinkingDelta.summary_delta`. Responses wire events label these representations explicitly; preserve that label and require it to match `ProviderCapabilities.thinking`. Chat Completions `reasoning_content` and Messages `thinking_delta` are ambiguous, so classify them with the same capability. Project only the matching normalized representation into each protocol's reasoning field. -A Responses reasoning lifecycle must use exactly one valid representation: raw reasoning content with no summary or signature, a reasoning summary accompanied by a signature with no raw content, or a signature with no reasoning summary or raw content. Preserve that representation across its start, delta, and end state; do not combine or switch representations during the part lifecycle. A completely empty start may remain deferred until its representation is known. +A Responses reasoning output lifecycle must use exactly one representation: raw reasoning content with no summary or signature, a reasoning summary with optional signature and no raw content, or a signature with no reasoning summary or raw content. Preserve that representation across its start, delta, and end state; do not combine or switch representations during the part lifecycle. A completely empty start may remain deferred until its representation is known. -Chat Completions reasoning has no signature and may represent either raw reasoning or a summary according to the provider capability profile. Messages follows the same pairing rule as Responses: raw reasoning has no signature, a reasoning summary must have one, and a signature may appear without reasoning text. Enforce protocol-specific pairing at the relevant wire boundary rather than in the provider-neutral normalized models. Current cross-protocol paths through Chat therefore support only unsigned raw reasoning; reject summaries rather than discarding required opaque state. +Chat Completions reasoning has no signature and may represent either raw reasoning or a summary according to the provider capability profile. Responses and Messages output processing tolerates missing summary signatures and preserves observed opaque state. Chat output may carry unsigned raw reasoning or summaries but rejects present signatures rather than discarding them. Request normalization and construction remain strict: summary history requires its signature on translated Responses and Messages routes. Receiving an unsigned summary does not make it replayable as history. Keep these checks at the relevant wire boundary rather than in the provider-neutral normalized models. Treat `ThinkingDelta.signature_delta` as a complete replacement snapshot of provider-specific opaque state. Never concatenate it with `Thinking.signature`. diff --git a/python/molto/adapters/anthropic.py b/python/molto/adapters/anthropic.py index 0299adf..5c0fd9f 100644 --- a/python/molto/adapters/anthropic.py +++ b/python/molto/adapters/anthropic.py @@ -645,11 +645,9 @@ def event(data: dict[str, Any]) -> AnthropicMessagesStreamEvent: raise ValueError( "An Anthropic summary block cannot contain raw reasoning." ) - if not part.signature: - raise ValueError( - "An Anthropic reasoning summary requires a signature." - ) - if thinking_signatures.get(normalized_event.index) != part.signature: + if thinking_signatures.get(normalized_event.index) != ( + part.signature or None + ): raise ValueError( "An Anthropic reasoning signature must be streamed before " "the block ends." diff --git a/python/molto/adapters/openai.py b/python/molto/adapters/openai.py index e5e6048..73f6d3d 100644 --- a/python/molto/adapters/openai.py +++ b/python/molto/adapters/openai.py @@ -821,8 +821,6 @@ def response_data( ) elif part.content: raise ValueError("Summarized Responses reasoning cannot contain raw text.") - if part.summary and not part.signature: - raise ValueError("A Responses reasoning summary requires a signature.") representation = self._capabilities.thinking completed_item = { "id": item_id, diff --git a/python/molto/upstream/anthropic.py b/python/molto/upstream/anthropic.py index e2fab59..d265c8d 100644 --- a/python/molto/upstream/anthropic.py +++ b/python/molto/upstream/anthropic.py @@ -693,10 +693,6 @@ def start_part( continue part = state.part if isinstance(part, Thinking): - if self._capabilities.thinking == "summary" and not state.signature: - raise ValueError( - "An Anthropic reasoning summary requires a signature." - ) completed_part: Thinking | TextOutput | ToolCall = self._classify_thinking( state.content, signature=state.signature ) diff --git a/python/molto/upstream/openai.py b/python/molto/upstream/openai.py index 37c3edc..71b0ed8 100644 --- a/python/molto/upstream/openai.py +++ b/python/molto/upstream/openai.py @@ -168,12 +168,6 @@ def _reasoning_part(item: Any) -> Thinking: ) -def _validate_completed_responses_reasoning(part: Thinking) -> Thinking: - if part.summary and not part.signature: - raise ValueError("A Responses reasoning summary requires a signature.") - return part - - def _validate_responses_reasoning_capability( part: Thinking, representation: Literal["raw", "summary"] ) -> None: @@ -198,7 +192,7 @@ def _started_output_item(item: Any) -> Thinking | TextOutput | ToolCall: def _completed_output_item(item: Any) -> Thinking | TextOutput | ToolCall: """Project a completed Responses output item into its normalized part.""" if item.type == "reasoning": - return _validate_completed_responses_reasoning(_reasoning_part(item)) + return _reasoning_part(item) if item.type == "message": return TextOutput( content="".join(part.text for part in item.content if part.type == "output_text"), diff --git a/tests/adapters/test_anthropic.py b/tests/adapters/test_anthropic.py index 6182f0c..a81f546 100644 --- a/tests/adapters/test_anthropic.py +++ b/tests/adapters/test_anthropic.py @@ -567,22 +567,35 @@ async def test_projection_rejects_reasoning_outside_capability() -> None: ] -async def test_projection_rejects_unsigned_summary_completion() -> None: +@pytest.mark.parametrize("signature", [None, ""]) +async def test_projection_preserves_unsigned_summary_completion(signature: str | None) -> None: + """Ensures missing output signatures do not discard streamed reasoning summaries.""" adapter, request = _adapter() - with pytest.raises(ValueError, match="summary requires a signature"): - [ - event - async for event in adapter.project_events( - _stream( - _MESSAGE_START, - PartStartEvent(index=0, part=Thinking(content="", summary="")), - PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), - PartEndEvent(index=0, part=Thinking(content="", summary="Summary")), + events = [ + event + async for event in adapter.project_events( + _stream( + _MESSAGE_START, + PartStartEvent( + index=0, part=Thinking(content="", summary="", signature=signature) ), - request, - ) - ] + PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), + PartEndEvent( + index=0, part=Thinking(content="", summary="Summary", signature=signature) + ), + _MESSAGE_END, + ), + request, + ) + ] + + deltas = [event.delta for event in events if event.type == "content_block_delta"] + assert [delta.type for delta in deltas] == ["thinking_delta"] + assert deltas[0].thinking == "Summary" + response = await adapter.project_response(_native_stream(*events), request) + assert response.content[0].thinking == "Summary" + assert response.content[0].signature == "" async def test_projection_preserves_non_empty_part_starts() -> None: diff --git a/tests/adapters/test_openai.py b/tests/adapters/test_openai.py index e0a2b55..7a780b6 100644 --- a/tests/adapters/test_openai.py +++ b/tests/adapters/test_openai.py @@ -982,22 +982,39 @@ def test_responses_normalization_rejects_unsigned_summary_history() -> None: OpenAIResponsesAdapter(_CAPABILITIES).normalize_request(request) -async def test_responses_projection_rejects_unsigned_summary_completion() -> None: +@pytest.mark.parametrize("signature", [None, ""]) +async def test_responses_projection_preserves_unsigned_summary_completion( + signature: str | None, +) -> None: + """Ensures Responses completes unsigned reasoning summaries without inventing opaque state.""" adapter, request = _responses_adapter(_CAPABILITIES) - with pytest.raises(ValueError, match="summary requires a signature"): - [ - event - async for event in adapter.project_events( - _stream( - _RESPONSE_START, - PartStartEvent(index=0, part=Thinking(content="", summary="")), - PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), - PartEndEvent(index=0, part=Thinking(content="", summary="Summary")), + events = [ + event + async for event in adapter.project_events( + _stream( + _RESPONSE_START, + PartStartEvent( + index=0, part=Thinking(content="", summary="", signature=signature) ), - request, - ) - ] + PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), + PartEndEvent( + index=0, part=Thinking(content="", summary="Summary", signature=signature) + ), + _RESPONSE_END, + ), + request, + ) + ] + + assert [ + event.delta for event in events if event.type == "response.reasoning_summary_text.delta" + ] == ["Summary"] + completed = next(event.item for event in events if event.type == "response.output_item.done") + assert completed.summary[0].text == "Summary" + assert completed.encrypted_content is None + assert events[-1].type == "response.completed" + assert events[-1].response.output[0] == completed async def test_responses_projection_rejects_thinking_part_with_content_and_summary() -> None: diff --git a/tests/core/test_unsigned_reasoning.py b/tests/core/test_unsigned_reasoning.py new file mode 100644 index 0000000..26c1b3c --- /dev/null +++ b/tests/core/test_unsigned_reasoning.py @@ -0,0 +1,216 @@ +"""SDK parity for synthetic unsigned reasoning derived from recorded streams.""" + +from itertools import permutations + +import pytest +from anthropic import AsyncAnthropic +from anthropic.lib.streaming._messages import accumulate_event +from httpx import AsyncClient, MockTransport, Request, Response +from openai import AsyncOpenAI +from openai.lib.streaming.chat import ChatCompletionStreamState +from pydantic import BaseModel + +from molto import Molto +from molto.types.exceptions import UnsupportedInput +from molto.utils.io import dumps_sse, json_loads, load_sse +from tests.core.test_interface import ( + PROTOCOLS, + Protocol, + _assert_responses_stream_invariants, + _coordinator, + _matched_cassette_path, + _read_cassette, + _sse_records, +) + + +def _collect(protocol: Protocol, events: list[BaseModel]) -> BaseModel: + if protocol == "responses": + assert events[-1].type == "response.completed" + return events[-1].response + if protocol == "chat/completions": + state = ChatCompletionStreamState() + for event in events: + state.handle_chunk(event.model_copy(deep=True)) + return state.current_completion_snapshot + snapshot = None + for event in events: + snapshot = accumulate_event(event=event.model_copy(deep=True), current_snapshot=snapshot) + assert snapshot is not None + return snapshot + + +def _assert_lifecycle(protocol: Protocol, events: list[BaseModel]) -> None: + if protocol == "responses": + _assert_responses_stream_invariants([dumps_sse(event) for event in events]) + assert events[0].type == "response.created" + assert events[-1].type == "response.completed" + added = [ + event.output_index for event in events if event.type == "response.output_item.added" + ] + done = [ + event.output_index for event in events if event.type == "response.output_item.done" + ] + assert added == done == list(range(len(added))) + elif protocol == "messages": + assert events[0].type == "message_start" + assert events[-1].type == "message_stop" + opened = set() + starts = [] + for event in events: + if event.type == "content_block_start": + assert event.index == len(starts) + starts.append(event.index) + opened.add(event.index) + elif event.type == "content_block_delta": + assert event.index in opened + elif event.type == "content_block_stop": + assert event.index in opened + opened.remove(event.index) + assert not opened + assert sum(event.type == "message_delta" for event in events) == 1 + else: + assert ( + sum(choice.finish_reason is not None for event in events for choice in event.choices) + == 1 + ) + + +def _unsigned_events(protocol: Protocol) -> list[dict]: + scenario = "reasoning_raw" if protocol == "chat/completions" else "reasoning_encrypted" + cassette = _read_cassette(_matched_cassette_path(scenario, protocol, "stream-00.yaml")) + events = [load_sse(record) for record in _sse_records(cassette, protocol)] + + def remove_signatures(value): + if isinstance(value, dict): + value.pop("encrypted_content", None) + if "signature" in value: + value["signature"] = "" + for child in value.values(): + remove_signatures(child) + elif isinstance(value, list): + for child in value: + remove_signatures(child) + + # These mutations model synthetic unsigned output, not observed provider behavior. + # Chat has no signature field; its unchanged wire reasoning is classified by + # the OpenAI summary capability selected by the public call below. + events = [ + event + for event in events + if event.get("type") != "content_block_delta" + or event["delta"]["type"] != "signature_delta" + ] + remove_signatures(events) + return events + + +def _body(protocol: Protocol, stream: bool) -> dict: + body = {"model": "test-model", "stream": stream} + if protocol == "responses": + body.update(input="Question", max_output_tokens=2048) + else: + body["messages"] = [{"role": "user", "content": "Question"}] + body["max_tokens" if protocol == "messages" else "max_completion_tokens"] = 2048 + return body + + +def _expected_request(downstream: Protocol, upstream: Protocol) -> dict: + body = _body(upstream, True) + if upstream == "responses": + body["input"] = [{"role": "user", "content": [{"type": "input_text", "text": "Question"}]}] + else: + body["messages"][0]["content"] = [{"type": "text", "text": "Question"}] + if upstream == "chat/completions": + body["stream_options"] = {"include_usage": True} + if downstream == "messages": + body.setdefault("stream_options", {})["include_obfuscation"] = False + return body + + +def _reasoning_and_text(protocol: Protocol, response) -> tuple[str, str]: + if protocol == "responses": + reasoning = [item for item in response.output if item.type == "reasoning"] + assert all(not item.encrypted_content for item in reasoning) + return "".join( + part.text for item in reasoning for part in item.summary + ), response.output_text + if protocol == "messages": + reasoning = [block for block in response.content if block.type == "thinking"] + assert all(not block.signature for block in reasoning) + return ( + "".join(block.thinking for block in reasoning), + "".join(block.text for block in response.content if block.type == "text"), + ) + message = response.choices[0].message + return message.reasoning_content, message.content + + +@pytest.mark.parametrize(("downstream", "upstream"), list(permutations(PROTOCOLS, 2))) +@pytest.mark.parametrize("stream", [True, False]) +async def test_unsigned_reasoning_output_matches_sdk( + downstream: Protocol, upstream: Protocol, stream: bool +): + """Ensures unsigned summaries survive all translated routes but remain invalid replay history.""" + events = _unsigned_events(upstream) + payload = "".join(dumps_sse(event, event.get("type", "")) for event in events) + expected_request = _expected_request(downstream, upstream) + requests = [] + + def handler(request: Request) -> Response: + requests.append(request) + assert json_loads(request.content) == expected_request + return Response(200, text=payload, headers={"content-type": "text/event-stream"}) + + async with AsyncClient(transport=MockTransport(handler)) as http_client: + if upstream == "messages": + sdk = AsyncAnthropic(api_key="test-api-key", http_client=http_client) + create = sdk.messages.create + else: + sdk = AsyncOpenAI(api_key="test-api-key", http_client=http_client) + create = ( + sdk.responses.create if upstream == "responses" else sdk.chat.completions.create + ) + async with await create(**expected_request) as sdk_stream: + reference = _collect(upstream, [event async for event in sdk_stream]) + + client = Molto(http_client=http_client) + result = await _coordinator(client, downstream)( + body=_body(downstream, stream), + endpoint=upstream, + provider="openai", + api_key="test-api-key", + ) + if stream: + projected = [event async for event in result] + _assert_lifecycle(downstream, projected) + result = _collect(downstream, projected) + + expected = _reasoning_and_text(upstream, reference) + assert all(expected) + assert _reasoning_and_text(downstream, result) == expected + assert len(requests) == 2 + + if downstream != "chat/completions": + history_body = _body(downstream, stream) + if downstream == "responses": + history_body["input"] = [ + item.model_dump(exclude_none=True) for item in result.output + ] + else: + history_body["messages"] = [ + { + "role": "assistant", + "content": [ + block.model_dump(exclude_none=True) for block in result.content + ], + } + ] + with pytest.raises(UnsupportedInput, match="summary requires a signature"): + await _coordinator(client, downstream)( + body=history_body, + endpoint=upstream, + provider="openai", + api_key="test-api-key", + ) + assert len(requests) == 2 diff --git a/tests/upstream/test_anthropic.py b/tests/upstream/test_anthropic.py index 043a5d8..572bd52 100644 --- a/tests/upstream/test_anthropic.py +++ b/tests/upstream/test_anthropic.py @@ -594,9 +594,10 @@ async def test_client_preserves_signature_only_thinking_block( ] -async def test_client_rejects_unsigned_summary_completion( +async def test_client_preserves_unsigned_summary_completion( monkeypatch: pytest.MonkeyPatch, ) -> None: + """Ensures an unsigned provider thinking block retains its summary and balanced lifecycle.""" native_events = [ _MESSAGE_START, { @@ -612,10 +613,15 @@ async def test_client_rejects_unsigned_summary_completion( {"type": "content_block_stop", "index": 0}, ] - with pytest.raises(ValueError, match="summary requires a signature"): - await _collect_client_events( - monkeypatch, native_events, PROVIDER_CAPABILITIES["anthropic"] - ) + events = await _collect_client_events( + monkeypatch, native_events, PROVIDER_CAPABILITIES["anthropic"] + ) + + assert events[1:] == [ + PartStartEvent(index=0, part=Thinking(content="", summary="")), + PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")), + PartEndEvent(index=0, part=Thinking(content="", summary="Summary")), + ] async def test_client_completes_thinking_block_without_deltas( diff --git a/tests/upstream/test_openai.py b/tests/upstream/test_openai.py index e08b332..4fd275e 100644 --- a/tests/upstream/test_openai.py +++ b/tests/upstream/test_openai.py @@ -2285,6 +2285,84 @@ def test_responses_client_rejects_unsigned_summary_history() -> None: OpenAIResponsesClient._request_body(request) +@pytest.mark.parametrize( + "signature_fields", [{}, {"encrypted_content": None}, {"encrypted_content": ""}] +) +async def test_responses_events_preserve_unsigned_summary_completion( + monkeypatch: pytest.MonkeyPatch, signature_fields: dict[str, str | None] +) -> None: + """Ensures absent, null, or empty output ciphertext does not discard a reasoning summary.""" + native_events = [ + _response_event( + { + "type": "response.output_item.added", + "item": { + "id": "rs_123", + "type": "reasoning", + "summary": [], + **signature_fields, + }, + "output_index": 0, + "sequence_number": 0, + } + ), + _response_event( + { + "type": "response.reasoning_summary_text.delta", + "summary_index": 0, + "delta": "Summary", + "item_id": "rs_123", + "output_index": 0, + "sequence_number": 1, + } + ), + _response_event( + { + "type": "response.output_item.done", + "item": { + "id": "rs_123", + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Summary"}], + **signature_fields, + }, + "output_index": 0, + "sequence_number": 2, + } + ), + ] + + async def stub_native_events( + native_request: OpenAIResponsesCreateRequest, + ) -> AsyncIterator[OpenAIResponsesStreamEvent]: + for event in native_events: + yield event + + async with httpx.AsyncClient(transport=httpx.MockTransport(_unexpected_http)) as http_client: + client = OpenAIResponsesClient(http_client, capabilities=OPENAI_CAPABILITIES) + monkeypatch.setattr(client, "native_events", stub_native_events) + events = [event async for event in client.events(NormalizedRequest(model="m"))] + + assert events[0] == PartStartEvent( + index=0, + part=Thinking( + content="", + summary="", + id="rs_123", + signature=signature_fields.get("encrypted_content"), + ), + ) + assert events[1] == PartDeltaEvent(index=0, delta=ThinkingDelta(summary_delta="Summary")) + assert events[-1] == PartEndEvent( + index=0, + part=Thinking( + content="", + summary="Summary", + id="rs_123", + signature=signature_fields.get("encrypted_content"), + ), + ) + + async def test_responses_events_preserve_reasoning_added_snapshot_and_deltas( monkeypatch: pytest.MonkeyPatch, ) -> None: