From 9b38261a9d8060bb61cd0fde773cf4aa8bda4c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=90=BE=20smolpaws?= Date: Tue, 18 Aug 2026 20:28:46 +0200 Subject: [PATCH 01/14] feat(sdk): add cleanup LLM profile for outward agent text (#4344) Co-authored-by: Engel Nyst --- openhands-sdk/openhands/sdk/llm/__init__.py | 8 + .../openhands/sdk/llm/cleanup_profile.py | 166 +++++++++++++ tests/sdk/llm/test_cleanup_profile.py | 227 ++++++++++++++++++ 3 files changed, 401 insertions(+) create mode 100644 openhands-sdk/openhands/sdk/llm/cleanup_profile.py create mode 100644 tests/sdk/llm/test_cleanup_profile.py diff --git a/openhands-sdk/openhands/sdk/llm/__init__.py b/openhands-sdk/openhands/sdk/llm/__init__.py index 0fc8617a76..01048f885a 100644 --- a/openhands-sdk/openhands/sdk/llm/__init__.py +++ b/openhands-sdk/openhands/sdk/llm/__init__.py @@ -4,6 +4,11 @@ OAuthCredentials, OpenAISubscriptionAuth, ) +from openhands.sdk.llm.cleanup_profile import ( + CLEANUP_PROFILE_NAME, + aclean_outward_text, + clean_outward_text, +) from openhands.sdk.llm.fallback_strategy import FallbackStrategy from openhands.sdk.llm.llm import LLM, LLM_PROFILE_SCHEMA_VERSION from openhands.sdk.llm.llm_profile_store import ( @@ -45,6 +50,9 @@ "OpenAISubscriptionAuth", "OPENAI_CODEX_MODELS", # Core + "CLEANUP_PROFILE_NAME", + "aclean_outward_text", + "clean_outward_text", "FallbackStrategy", "LLMResponse", "LLM", diff --git a/openhands-sdk/openhands/sdk/llm/cleanup_profile.py b/openhands-sdk/openhands/sdk/llm/cleanup_profile.py new file mode 100644 index 0000000000..5323040f1a --- /dev/null +++ b/openhands-sdk/openhands/sdk/llm/cleanup_profile.py @@ -0,0 +1,166 @@ +"""Cleanup profile: repair an agent's outward text before a human reads it. + +An agent's reasoning can be sound while the *surface* of a message is broken — +mojibake from mis-encoded emoji, stray control characters, or an inconsistent +format for the target channel. The big reasoning model should not have to police +its own output every turn. + +This module runs the agent's outward text through a small, saved LLM profile — +resolved by convention under the name ``cleanup`` (:data:`CLEANUP_PROFILE_NAME`) — +and returns the repaired text. It mirrors the ``ask_oracle`` pattern: no agent +setting and no wiring; a caller saves a profile named ``cleanup`` and calls +:func:`clean_outward_text` (or the async :func:`aclean_outward_text`). + +The pass is deliberately narrow: + +- It runs on outward, human-facing text only. Callers must not use it on internal + agent or tool messages. +- It is stateless: only the cleanup system prompt plus the draft are sent, with + no conversation history and no tools. The active conversation LLM is never + switched. +- It repairs the surface only. The prompt forbids adding facts, links, or + promises; it must not change meaning. +- It fails open. If no ``cleanup`` profile exists, or the call errors, the + original text is returned unchanged. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +from openhands.sdk.llm.llm import LLM +from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.llm.llm_response import LLMResponse +from openhands.sdk.llm.message import Message, TextContent +from openhands.sdk.logger import get_logger + + +if TYPE_CHECKING: + from openhands.sdk.utils.cipher import Cipher + + +logger = get_logger(__name__) + +# The cleanup model is a saved LLM profile resolved by convention under this +# name. Save a profile named "cleanup" (e.g. via LLMProfileStore.save("cleanup", +# llm)) and callers on the outward path will consult it. No agent setting or +# wiring is required. +CLEANUP_PROFILE_NAME: Final[str] = "cleanup" + +_CLEANUP_SYSTEM_PROMPT = """\ +You repair the surface of a message that an AI agent is about to send to a \ +human. Fix only the presentation. + +Rules: +- Fix mojibake, broken encoding, and stray control characters. +- Keep the meaning exactly. Do not add facts, links, numbers, or promises. +- Do not remove real content. Do not answer or continue the message. +- Keep the author's tone and any intentional formatting. +- Return only the repaired message text, with nothing added around it.""" + +_CLEANUP_USER_PROMPT_TEMPLATE = """\ +Repair this message and return only the repaired text: + +{text}""" + + +def _load_cleanup_llm(cipher: Cipher | None) -> LLM | None: + """Load the ``cleanup`` profile, or ``None`` when cleanup is unavailable. + + Returns ``None`` (feature off / fail-open) when no ``cleanup`` profile is + saved or the profile cannot be loaded, so callers can pass the original text + through unchanged. + """ + try: + return LLMProfileStore().load(CLEANUP_PROFILE_NAME, cipher=cipher) + except FileNotFoundError: + # No cleanup profile configured: feature is simply off. + return None + except Exception as exc: + logger.warning("Cleanup profile could not be loaded: %s", exc) + return None + + +def _cleanup_messages(text: str) -> list[Message]: + return [ + Message(role="system", content=[TextContent(text=_CLEANUP_SYSTEM_PROMPT)]), + Message( + role="user", + content=[TextContent(text=_CLEANUP_USER_PROMPT_TEMPLATE.format(text=text))], + ), + ] + + +def _repaired_or_original(response: LLMResponse, text: str) -> str: + cleaned = "".join( + content.text + for content in response.message.content + if isinstance(content, TextContent) + ).strip() + # An empty reply means the cleanup model gave us nothing usable; keep the + # original rather than sending a blank message. + return cleaned or text + + +def clean_outward_text(text: str, *, cipher: Cipher | None = None) -> str: + """Return ``text`` repaired by the ``cleanup`` LLM profile, or unchanged. + + Resolves the saved profile named :data:`CLEANUP_PROFILE_NAME` from the + default :class:`LLMProfileStore` and runs a single stateless completion to + repair the message surface. This is fail-open: if the profile is missing or + the call fails for any reason, the original ``text`` is returned unchanged so + a cleanup problem can never block or corrupt an outward message. + + Args: + text: The agent's outward, human-facing draft. Do not pass internal + agent or tool messages. + cipher: Optional cipher for decrypting the profile's secrets at rest. + + Returns: + The repaired text, or the original ``text`` when cleanup is unavailable + or fails, or when ``text`` is empty. + """ + if not text.strip(): + return text + + cleanup_llm = _load_cleanup_llm(cipher) + if cleanup_llm is None: + return text + + # Imported lazily: ``agent.utils`` imports from ``openhands.sdk.llm``, so a + # module-level import here would create a circular import at package init. + from openhands.sdk.agent.utils import make_llm_completion + + try: + response = make_llm_completion(cleanup_llm, _cleanup_messages(text)) + except Exception as exc: + logger.warning("Cleanup profile call failed; sending original text: %s", exc) + return text + + return _repaired_or_original(response, text) + + +async def aclean_outward_text(text: str, *, cipher: Cipher | None = None) -> str: + """Async variant of :func:`clean_outward_text`. + + Same fail-open contract, for async outward paths (e.g. the agent-server + outbound surface). See :func:`clean_outward_text` for details. + """ + if not text.strip(): + return text + + cleanup_llm = _load_cleanup_llm(cipher) + if cleanup_llm is None: + return text + + # Imported lazily: ``agent.utils`` imports from ``openhands.sdk.llm``, so a + # module-level import here would create a circular import at package init. + from openhands.sdk.agent.utils import amake_llm_completion + + try: + response = await amake_llm_completion(cleanup_llm, _cleanup_messages(text)) + except Exception as exc: + logger.warning("Cleanup profile call failed; sending original text: %s", exc) + return text + + return _repaired_or_original(response, text) diff --git a/tests/sdk/llm/test_cleanup_profile.py b/tests/sdk/llm/test_cleanup_profile.py new file mode 100644 index 0000000000..1480c13566 --- /dev/null +++ b/tests/sdk/llm/test_cleanup_profile.py @@ -0,0 +1,227 @@ +"""Tests for the cleanup LLM profile (``clean_outward_text``). + +The cleanup profile repairs an agent's outward text before a human reads it. It +resolves a saved LLM profile named ``cleanup`` and runs a single stateless +completion, failing open (returning the original text) whenever the profile is +missing or the call errors. +""" + +from collections.abc import Sequence +from pathlib import Path +from typing import Any, cast + +import pytest +from pydantic import PrivateAttr + +from openhands.sdk.llm import ( + CLEANUP_PROFILE_NAME, + LLM, + LLMResponse, + Message, + TextContent, + aclean_outward_text, + clean_outward_text, + llm_profile_store, +) +from openhands.sdk.llm.llm import LLMCallContext +from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.llm.streaming import TokenCallbackType +from openhands.sdk.testing import TestLLM +from openhands.sdk.tool import ToolDefinition + + +class CapturingTestLLM(TestLLM): + """TestLLM that records the messages and tools of the last completion.""" + + _last_messages: list[Message] = PrivateAttr(default_factory=list) + _last_tools: Sequence[ToolDefinition] | None = PrivateAttr(default=None) + + @property + def last_messages(self) -> list[Message]: + return self._last_messages + + @property + def last_tools(self) -> Sequence[ToolDefinition] | None: + return self._last_tools + + def completion( + self, + messages: list[Message], + tools: Sequence[ToolDefinition] | None = None, + add_security_risk_prediction: bool = False, + on_token: TokenCallbackType | None = None, + call_context: LLMCallContext | None = None, + **kwargs: Any, + ) -> LLMResponse: + self._last_messages = list(messages) + self._last_tools = tools + return super().completion( + messages=messages, + tools=tools, + add_security_risk_prediction=add_security_risk_prediction, + on_token=on_token, + call_context=call_context, + **kwargs, + ) + + +def _assistant_message(text: str) -> Message: + return Message(role="assistant", content=[TextContent(text=text)]) + + +def _message_text(message: Message) -> str: + return "".join( + content.text for content in message.content if isinstance(content, TextContent) + ) + + +def _capturing_llm(*replies: str) -> CapturingTestLLM: + return cast( + CapturingTestLLM, + CapturingTestLLM.from_messages( + [_assistant_message(reply) for reply in replies], + model="cleanup-model", + usage_id="cleanup", + ), + ) + + +def test_returns_cleaned_text_from_cleanup_profile( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cleanup_llm = _capturing_llm("Done! I appreciate the nudge.") + + def load_profile(self: LLMProfileStore, name: str, *, cipher: Any = None) -> LLM: + assert name == CLEANUP_PROFILE_NAME + return cleanup_llm + + monkeypatch.setattr(LLMProfileStore, "load", load_profile) + + original = "Done! I appreciate the nudge! \u00f0" + result = clean_outward_text(original) + + assert result == "Done! I appreciate the nudge." + # Stateless call: only a system + user message, no tools, no history. + assert [message.role for message in cleanup_llm.last_messages] == ["system", "user"] + assert cleanup_llm.last_tools == [] + assert "repair" in _message_text(cleanup_llm.last_messages[0]).lower() + assert original in _message_text(cleanup_llm.last_messages[1]) + + +def test_cipher_is_forwarded_to_profile_load( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The cipher argument must reach LLMProfileStore.load so the encrypted-secret + # path works when a caller passes one. + cleanup_llm = _capturing_llm("clean") + sentinel = object() + seen: dict[str, Any] = {} + + def load_profile(self: LLMProfileStore, name: str, *, cipher: Any = None) -> LLM: + seen["cipher"] = cipher + return cleanup_llm + + monkeypatch.setattr(LLMProfileStore, "load", load_profile) + + clean_outward_text("draft \u00f0", cipher=cast(Any, sentinel)) + + assert seen["cipher"] is sentinel + + +def test_missing_profile_returns_original_text( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # A real, empty profile directory: no "cleanup" profile exists, so the + # feature is off and the original text passes through unchanged. + profile_dir = tmp_path / "profiles" + profile_dir.mkdir() + monkeypatch.setattr(llm_profile_store, "_DEFAULT_PROFILE_DIR", profile_dir) + + original = "Ship it \u00f0" + assert clean_outward_text(original) == original + + +def test_call_failure_returns_original_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Scripted with no replies: the next completion raises (exhausted), so the + # cleanup must fail open and return the untouched original. + failing_llm = _capturing_llm() + + monkeypatch.setattr( + LLMProfileStore, + "load", + lambda self, name, *, cipher=None: failing_llm, + ) + + original = "Keep me exactly \u00e2 as is" + assert clean_outward_text(original) == original + + +def test_empty_cleanup_reply_returns_original_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A blank reply must not blank out the message; keep the original. + cleanup_llm = _capturing_llm(" ") + monkeypatch.setattr( + LLMProfileStore, + "load", + lambda self, name, *, cipher=None: cleanup_llm, + ) + + original = "Real content here" + assert clean_outward_text(original) == original + + +@pytest.mark.parametrize("blank", ["", " ", "\n\t"]) +def test_blank_input_short_circuits_without_loading_profile( + blank: str, monkeypatch: pytest.MonkeyPatch +) -> None: + def fail_load(self: LLMProfileStore, name: str, *, cipher: Any = None) -> LLM: + raise AssertionError("profile should not be loaded for blank input") + + monkeypatch.setattr(LLMProfileStore, "load", fail_load) + + assert clean_outward_text(blank) == blank + + +async def test_async_returns_cleaned_text_from_cleanup_profile( + monkeypatch: pytest.MonkeyPatch, +) -> None: + cleanup_llm = _capturing_llm("Done! I appreciate the nudge.") + + def load_profile(self: LLMProfileStore, name: str, *, cipher: Any = None) -> LLM: + assert name == CLEANUP_PROFILE_NAME + return cleanup_llm + + monkeypatch.setattr(LLMProfileStore, "load", load_profile) + + result = await aclean_outward_text("Done! I appreciate the nudge! \u00f0") + + assert result == "Done! I appreciate the nudge." + assert [message.role for message in cleanup_llm.last_messages] == ["system", "user"] + + +async def test_async_missing_profile_returns_original_text( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + profile_dir = tmp_path / "profiles" + profile_dir.mkdir() + monkeypatch.setattr(llm_profile_store, "_DEFAULT_PROFILE_DIR", profile_dir) + + original = "Ship it \u00f0" + assert await aclean_outward_text(original) == original + + +async def test_async_call_failure_returns_original_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + failing_llm = _capturing_llm() + monkeypatch.setattr( + LLMProfileStore, + "load", + lambda self, name, *, cipher=None: failing_llm, + ) + + original = "Keep me exactly \u00e2 as is" + assert await aclean_outward_text(original) == original From d460d1a0b6bd35e054ad146c6078205df4686387 Mon Sep 17 00:00:00 2001 From: Rohit Malhotra Date: Tue, 18 Aug 2026 14:36:42 -0400 Subject: [PATCH 02/14] feat: Add deployment kind to agent-server telemetry (#4522) Co-authored-by: openhands --- .../openhands/agent_server/config.py | 17 ++++++++++----- .../agent_server/telemetry/__init__.py | 2 ++ .../agent_server/telemetry/factory.py | 8 ++++++- .../agent_server/telemetry/models.py | 4 ++++ .../agent_server/telemetry/service.py | 5 ++++- .../openhands/agent_server/telemetry_types.py | 4 ++++ .../test_telemetry_disabled_by_default.py | 21 +++++++++++++++++-- .../test_telemetry_no_duplication.py | 15 +++++++++++++ .../telemetry/test_telemetry_schema.py | 1 + tests/agent_server/test_config.py | 8 +++++++ 10 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 openhands-agent-server/openhands/agent_server/telemetry_types.py diff --git a/openhands-agent-server/openhands/agent_server/config.py b/openhands-agent-server/openhands/agent_server/config.py index 2dec4d23e0..65d8632fb9 100644 --- a/openhands-agent-server/openhands/agent_server/config.py +++ b/openhands-agent-server/openhands/agent_server/config.py @@ -14,6 +14,7 @@ get_env_parser, merge, ) +from openhands.agent_server.telemetry_types import DeploymentKind from openhands.sdk.marketplace.registration import MarketplaceRegistration from openhands.sdk.utils.cipher import Cipher @@ -135,13 +136,19 @@ class WebhookSpec(BaseModel): class TelemetrySpec(BaseModel): """Deployment-supplied product-analytics transport settings. - This carries *transport* only. Whether telemetry may be delivered is - resolved from consent (``misc_settings.telemetry.consent``, optionally - seeded or overridden by ``OH_TELEMETRY_CONSENT``) — there is no deployment - "mode" here, and nothing in the agent-server special-cases a hosted - deployment. + This carries transport plus the non-identifying deployment tag. Whether + telemetry may be delivered is resolved from consent + (``misc_settings.telemetry.consent``, optionally seeded or overridden by + ``OH_TELEMETRY_CONSENT``). """ + deployment_kind: DeploymentKind = Field( + default="local", + description=( + "Deployment kind attached to diagnostic events. Use 'remote' for " + "hosted OpenHands and 'local' for self-hosted or developer runs." + ), + ) exporter: TelemetryExporterKind = Field( default="none", description=( diff --git a/openhands-agent-server/openhands/agent_server/telemetry/__init__.py b/openhands-agent-server/openhands/agent_server/telemetry/__init__.py index 9ef78a3b47..3e771c0abb 100644 --- a/openhands-agent-server/openhands/agent_server/telemetry/__init__.py +++ b/openhands-agent-server/openhands/agent_server/telemetry/__init__.py @@ -26,6 +26,7 @@ ) from openhands.agent_server.telemetry.models import ( TELEMETRY_SCHEMA_VERSION, + DeploymentKind, DiagnosticEvent, RuntimeProperties, ) @@ -62,6 +63,7 @@ "TELEMETRY_SCHEMA_VERSION", "BufferedTelemetrySink", "ConversationTelemetryContext", + "DeploymentKind", "DiagnosticEvent", "DISTINCT_ID_HEADER", "DiagnosticEventFactory", diff --git a/openhands-agent-server/openhands/agent_server/telemetry/factory.py b/openhands-agent-server/openhands/agent_server/telemetry/factory.py index c648258c7c..926c23f9ae 100644 --- a/openhands-agent-server/openhands/agent_server/telemetry/factory.py +++ b/openhands-agent-server/openhands/agent_server/telemetry/factory.py @@ -15,6 +15,7 @@ from openhands.agent_server.server_details_router import ServerInfo from openhands.agent_server.telemetry.models import ( TELEMETRY_SCHEMA_VERSION, + DeploymentKind, DiagnosticEvent, DiagnosticProperties, EventName, @@ -64,7 +65,11 @@ def _python_version() -> str: return f"{sys.version_info.major}.{sys.version_info.minor}" -def build_runtime_properties(*, deferred_init: bool) -> RuntimeProperties: +def build_runtime_properties( + *, + deferred_init: bool, + deployment_kind: DeploymentKind = "local", +) -> RuntimeProperties: """Snapshot the coarse runtime facts shared by every event.""" # Versions and build metadata come from ServerInfo's own field defaults, so # /server_info and telemetry can never disagree about what is running. @@ -78,6 +83,7 @@ def build_runtime_properties(*, deferred_init: bool) -> RuntimeProperties: python_version=_python_version(), platform=_platform_token(), deferred_init=deferred_init, + deployment_kind=deployment_kind, ) diff --git a/openhands-agent-server/openhands/agent_server/telemetry/models.py b/openhands-agent-server/openhands/agent_server/telemetry/models.py index f62e2504d5..7f498f66ff 100644 --- a/openhands-agent-server/openhands/agent_server/telemetry/models.py +++ b/openhands-agent-server/openhands/agent_server/telemetry/models.py @@ -22,6 +22,8 @@ from pydantic import BaseModel, ConfigDict, Field, StringConstraints +from openhands.agent_server.telemetry_types import DeploymentKind + TELEMETRY_SCHEMA_VERSION: Final[int] = 1 @@ -144,6 +146,7 @@ class RuntimeProperties(BaseModel): python_version: SafeToken platform: SafeToken deferred_init: bool + deployment_kind: DeploymentKind = "local" source: Literal["openhands-agent-server"] = "openhands-agent-server" @@ -290,6 +293,7 @@ def to_payload(self) -> dict[str, object]: "python_version", "platform", "deferred_init", + "deployment_kind", "source", "$insert_id", "conversation_ref", diff --git a/openhands-agent-server/openhands/agent_server/telemetry/service.py b/openhands-agent-server/openhands/agent_server/telemetry/service.py index 97f5dd7c65..a085a9c12c 100644 --- a/openhands-agent-server/openhands/agent_server/telemetry/service.py +++ b/openhands-agent-server/openhands/agent_server/telemetry/service.py @@ -104,7 +104,10 @@ async def build_telemetry_sink(config: Config) -> TelemetrySink: spec = config.telemetry _event_factory = DiagnosticEventFactory( - runtime=build_runtime_properties(deferred_init=config.deferred_init), + runtime=build_runtime_properties( + deferred_init=config.deferred_init, + deployment_kind=spec.deployment_kind, + ), salt=( spec.salt.get_secret_value() if spec.salt is not None diff --git a/openhands-agent-server/openhands/agent_server/telemetry_types.py b/openhands-agent-server/openhands/agent_server/telemetry_types.py new file mode 100644 index 0000000000..13b71afbc4 --- /dev/null +++ b/openhands-agent-server/openhands/agent_server/telemetry_types.py @@ -0,0 +1,4 @@ +from typing import Literal + + +DeploymentKind = Literal["local", "remote"] diff --git a/tests/agent_server/telemetry/test_telemetry_disabled_by_default.py b/tests/agent_server/telemetry/test_telemetry_disabled_by_default.py index c8d869d389..f528d74412 100644 --- a/tests/agent_server/telemetry/test_telemetry_disabled_by_default.py +++ b/tests/agent_server/telemetry/test_telemetry_disabled_by_default.py @@ -6,10 +6,11 @@ from fastapi.testclient import TestClient from openhands.agent_server.api import create_app -from openhands.agent_server.config import Config +from openhands.agent_server.config import Config, TelemetrySpec from openhands.agent_server.telemetry import ( NoOpTelemetrySink, build_telemetry_sink, + get_event_factory, get_telemetry_sink, ) @@ -25,6 +26,22 @@ async def test_building_from_a_default_config_stays_a_noop(temp_persistence_dir) assert sink.enabled is False +async def test_building_sink_carries_configured_deployment_kind( + temp_persistence_dir, +): + sink = await build_telemetry_sink( + Config( + static_files_path=None, telemetry=TelemetrySpec(deployment_kind="remote") + ) + ) + try: + factory = get_event_factory() + assert factory is not None + assert factory.runtime.deployment_kind == "remote" + finally: + await sink.aclose() + + async def test_opt_in_without_an_api_key_stays_inactive(config_factory): """No key means no exporter.""" sink = await build_telemetry_sink(config_factory("posthog")) @@ -57,7 +74,7 @@ def test_importing_the_telemetry_package_does_not_import_posthog(): import sys import openhands.agent_server.telemetry as t -from openhands.agent_server.config import Config +from openhands.agent_server.config import Config, TelemetrySpec from openhands.agent_server.api import create_app assert "posthog" not in sys.modules, "importing telemetry pulled in posthog" diff --git a/tests/agent_server/telemetry/test_telemetry_no_duplication.py b/tests/agent_server/telemetry/test_telemetry_no_duplication.py index 15e330e092..d32327e65d 100644 --- a/tests/agent_server/telemetry/test_telemetry_no_duplication.py +++ b/tests/agent_server/telemetry/test_telemetry_no_duplication.py @@ -74,6 +74,21 @@ def test_no_deployment_mode_concept_remains(): assert "deployment_mode" not in m.RuntimeProperties.model_fields +def test_deployment_kind_is_a_runtime_property_not_a_mode(): + import openhands.agent_server.config as config_mod + + assert ( + config_mod.TelemetrySpec(deployment_kind="remote").deployment_kind == "remote" + ) + assert m.RuntimeProperties.model_fields["deployment_kind"].default == "local" + assert ( + build_runtime_properties( + deferred_init=False, deployment_kind="remote" + ).deployment_kind + == "remote" + ) + + def test_versions_match_server_info(): """Telemetry and /server_info must never disagree about what is running.""" from openhands.agent_server.server_details_router import ServerInfo diff --git a/tests/agent_server/telemetry/test_telemetry_schema.py b/tests/agent_server/telemetry/test_telemetry_schema.py index 349c4956b2..d6619c115b 100644 --- a/tests/agent_server/telemetry/test_telemetry_schema.py +++ b/tests/agent_server/telemetry/test_telemetry_schema.py @@ -161,4 +161,5 @@ def test_payload_carries_schema_version_and_excludes_distinct_id(): # distinct_id is the transport's addressing field, not a property. assert "distinct_id" not in payload assert "kind" not in payload + assert payload["deployment_kind"] == "local" assert set(payload) <= set(m.EXPECTED_PROPERTY_NAMES) diff --git a/tests/agent_server/test_config.py b/tests/agent_server/test_config.py index 1b88e21ba7..0adc8b57d2 100644 --- a/tests/agent_server/test_config.py +++ b/tests/agent_server/test_config.py @@ -40,6 +40,14 @@ def test_load_config_reads_registered_marketplaces_from_env(monkeypatch, tmp_pat assert registration.auto_load is True +def test_load_config_reads_telemetry_deployment_kind_from_env(monkeypatch, tmp_path): + config_path = tmp_path / "missing.json" + monkeypatch.setenv(CONFIG_PATH_ENV, str(config_path)) + monkeypatch.setenv("OH_TELEMETRY_DEPLOYMENT_KIND", "remote") + + assert load_config().telemetry.deployment_kind == "remote" + + def test_conversation_idle_ttl_defaults_to_twenty_minutes(): assert DEFAULT_CONVERSATION_IDLE_TTL_SECONDS == 1200.0 assert Config().conversation_idle_ttl_seconds == 1200.0 From ae18d7cd0fbee080160d42fd0472cff4bfe86866 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:52:21 +0000 Subject: [PATCH 03/14] chore(deps-dev): bump pillow from 12.2.0 to 12.3.0 (#4518) Co-authored-by: Vasco Schiavo <115561717+VascoSch92@users.noreply.github.com> --- openhands-sdk/pyproject.toml | 2 +- pyproject.toml | 4 +- uv.lock | 138 ++++++++++++++++++----------------- 3 files changed, 73 insertions(+), 71 deletions(-) diff --git a/openhands-sdk/pyproject.toml b/openhands-sdk/pyproject.toml index 4afa5cc175..505afe45d9 100644 --- a/openhands-sdk/pyproject.toml +++ b/openhands-sdk/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ "jsonschema>=4.23.0", "joserfc>=1.6.8", "litellm>=1.93.0", - "pillow>=12.1.1", + "pillow>=12.3.0", "pydantic>=2.12.5", "python-frontmatter>=1.1.0", "python-json-logger>=3.3.0", diff --git a/pyproject.toml b/pyproject.toml index 84aacb1302..1dce849060 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ constraint-dependencies = [ "aiohttp>=3.13.3", # CVE-2025-69223 + 7 others "urllib3>=2.6.3", # CVE-2026-21441, CVE-2025-66471, CVE-2025-66418 "protobuf>=6.33.5", # CVE-2026-0994 - "pillow>=12.1.1", # CVE-2026-25990 + "pillow>=12.3.0", # CVE-2026-25990 "orjson>=3.11.7", # CVE-2025-67221 "rich>=14.3.3", # Version 14.3.2 essentially has a denial-of-service vulnerability which is outlined in https://github.com/Textualize/rich/issues/3958 "lupa>=2.8", # CVE-2026-34444 @@ -29,7 +29,7 @@ openhands-agent-server = { workspace = true } dev = [ "pre-commit>=4.3.0", "packaging>=24.2", - "pillow>=12.1.1", + "pillow>=12.3.0", "psutil>=7.0.0", "pyright[nodejs]>=1.1.405", "pytest>=9.0.3", diff --git a/uv.lock b/uv.lock index 749d8a8187..10772d6889 100644 --- a/uv.lock +++ b/uv.lock @@ -26,7 +26,7 @@ constraints = [ { name = "litellm", specifier = "==1.93.0" }, { name = "lupa", specifier = ">=2.8" }, { name = "orjson", specifier = ">=3.11.7" }, - { name = "pillow", specifier = ">=12.1.1" }, + { name = "pillow", specifier = ">=12.3.0" }, { name = "protobuf", specifier = ">=6.33.5" }, { name = "rich", specifier = ">=14.3.3" }, { name = "starlette", specifier = ">=0.49.1" }, @@ -37,7 +37,7 @@ constraints = [ dev = [ { name = "griffe", extras = ["pypi"], specifier = ">=2.0.0" }, { name = "packaging", specifier = ">=24.2" }, - { name = "pillow", specifier = ">=12.1.1" }, + { name = "pillow", specifier = ">=12.3.0" }, { name = "pre-commit", specifier = ">=4.3.0" }, { name = "psutil", specifier = ">=7.0.0" }, { name = "pycodestyle", specifier = ">=2.12.0" }, @@ -2807,7 +2807,7 @@ requires-dist = [ { name = "jsonschema", specifier = ">=4.23.0" }, { name = "litellm", specifier = ">=1.93.0" }, { name = "lmnr", specifier = ">=0.7.56,<0.8.0" }, - { name = "pillow", specifier = ">=12.1.1" }, + { name = "pillow", specifier = ">=12.3.0" }, { name = "pydantic", specifier = ">=2.12.5" }, { name = "python-frontmatter", specifier = ">=1.1.0" }, { name = "python-json-logger", specifier = ">=3.3.0" }, @@ -3130,71 +3130,73 @@ wheels = [ [[package]] name = "pillow" -version = "12.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/58/be/7482c8a5ebebbc6470b3eb791812fff7d5e0216c2be3827b30b8bb6603ed/pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5", size = 5308279, upload-time = "2026-04-01T14:43:13.246Z" }, - { url = "https://files.pythonhosted.org/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421", size = 4695490, upload-time = "2026-04-01T14:43:15.584Z" }, - { url = "https://files.pythonhosted.org/packages/de/af/4e8e6869cbed569d43c416fad3dc4ecb944cb5d9492defaed89ddd6fe871/pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987", size = 6284462, upload-time = "2026-04-01T14:43:18.268Z" }, - { url = "https://files.pythonhosted.org/packages/e9/9e/c05e19657fd57841e476be1ab46c4d501bffbadbafdc31a6d665f8b737b6/pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76", size = 8094744, upload-time = "2026-04-01T14:43:20.716Z" }, - { url = "https://files.pythonhosted.org/packages/2b/54/1789c455ed10176066b6e7e6da1b01e50e36f94ba584dc68d9eebfe9156d/pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005", size = 6398371, upload-time = "2026-04-01T14:43:23.443Z" }, - { url = "https://files.pythonhosted.org/packages/43/e3/fdc657359e919462369869f1c9f0e973f353f9a9ee295a39b1fea8ee1a77/pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780", size = 7087215, upload-time = "2026-04-01T14:43:26.758Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f8/2f6825e441d5b1959d2ca5adec984210f1ec086435b0ed5f52c19b3b8a6e/pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5", size = 6509783, upload-time = "2026-04-01T14:43:29.56Z" }, - { url = "https://files.pythonhosted.org/packages/67/f9/029a27095ad20f854f9dba026b3ea6428548316e057e6fc3545409e86651/pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5", size = 7212112, upload-time = "2026-04-01T14:43:32.091Z" }, - { url = "https://files.pythonhosted.org/packages/be/42/025cfe05d1be22dbfdb4f264fe9de1ccda83f66e4fc3aac94748e784af04/pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940", size = 6378489, upload-time = "2026-04-01T14:43:34.601Z" }, - { url = "https://files.pythonhosted.org/packages/5d/7b/25a221d2c761c6a8ae21bfa3874988ff2583e19cf8a27bf2fee358df7942/pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5", size = 7084129, upload-time = "2026-04-01T14:43:37.213Z" }, - { url = "https://files.pythonhosted.org/packages/10/e1/542a474affab20fd4a0f1836cb234e8493519da6b76899e30bcc5d990b8b/pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414", size = 2463612, upload-time = "2026-04-01T14:43:39.421Z" }, - { url = "https://files.pythonhosted.org/packages/4a/01/53d10cf0dbad820a8db274d259a37ba50b88b24768ddccec07355382d5ad/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c", size = 4100837, upload-time = "2026-04-01T14:43:41.506Z" }, - { url = "https://files.pythonhosted.org/packages/0f/98/f3a6657ecb698c937f6c76ee564882945f29b79bad496abcba0e84659ec5/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2", size = 4176528, upload-time = "2026-04-01T14:43:43.773Z" }, - { url = "https://files.pythonhosted.org/packages/69/bc/8986948f05e3ea490b8442ea1c1d4d990b24a7e43d8a51b2c7d8b1dced36/pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c", size = 3640401, upload-time = "2026-04-01T14:43:45.87Z" }, - { url = "https://files.pythonhosted.org/packages/34/46/6c717baadcd62bc8ed51d238d521ab651eaa74838291bda1f86fe1f864c9/pillow-12.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795", size = 5308094, upload-time = "2026-04-01T14:43:48.438Z" }, - { url = "https://files.pythonhosted.org/packages/71/43/905a14a8b17fdb1ccb58d282454490662d2cb89a6bfec26af6d3520da5ec/pillow-12.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f", size = 4695402, upload-time = "2026-04-01T14:43:51.292Z" }, - { url = "https://files.pythonhosted.org/packages/73/dd/42107efcb777b16fa0393317eac58f5b5cf30e8392e266e76e51cff28c3d/pillow-12.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed", size = 6280005, upload-time = "2026-04-01T14:43:54.242Z" }, - { url = "https://files.pythonhosted.org/packages/a8/68/b93e09e5e8549019e61acf49f65b1a8530765a7f812c77a7461bca7e4494/pillow-12.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9", size = 8090669, upload-time = "2026-04-01T14:43:57.335Z" }, - { url = "https://files.pythonhosted.org/packages/4b/6e/3ccb54ce8ec4ddd1accd2d89004308b7b0b21c4ac3d20fa70af4760a4330/pillow-12.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed", size = 6395194, upload-time = "2026-04-01T14:43:59.864Z" }, - { url = "https://files.pythonhosted.org/packages/67/ee/21d4e8536afd1a328f01b359b4d3997b291ffd35a237c877b331c1c3b71c/pillow-12.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3", size = 7082423, upload-time = "2026-04-01T14:44:02.74Z" }, - { url = "https://files.pythonhosted.org/packages/78/5f/e9f86ab0146464e8c133fe85df987ed9e77e08b29d8d35f9f9f4d6f917ba/pillow-12.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9", size = 6505667, upload-time = "2026-04-01T14:44:05.381Z" }, - { url = "https://files.pythonhosted.org/packages/ed/1e/409007f56a2fdce61584fd3acbc2bbc259857d555196cedcadc68c015c82/pillow-12.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795", size = 7208580, upload-time = "2026-04-01T14:44:08.39Z" }, - { url = "https://files.pythonhosted.org/packages/23/c4/7349421080b12fb35414607b8871e9534546c128a11965fd4a7002ccfbee/pillow-12.2.0-cp313-cp313-win32.whl", hash = "sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e", size = 6375896, upload-time = "2026-04-01T14:44:11.197Z" }, - { url = "https://files.pythonhosted.org/packages/3f/82/8a3739a5e470b3c6cbb1d21d315800d8e16bff503d1f16b03a4ec3212786/pillow-12.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b", size = 7081266, upload-time = "2026-04-01T14:44:13.947Z" }, - { url = "https://files.pythonhosted.org/packages/c3/25/f968f618a062574294592f668218f8af564830ccebdd1fa6200f598e65c5/pillow-12.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06", size = 2463508, upload-time = "2026-04-01T14:44:16.312Z" }, - { url = "https://files.pythonhosted.org/packages/4d/a4/b342930964e3cb4dce5038ae34b0eab4653334995336cd486c5a8c25a00c/pillow-12.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b", size = 5309927, upload-time = "2026-04-01T14:44:18.89Z" }, - { url = "https://files.pythonhosted.org/packages/9f/de/23198e0a65a9cf06123f5435a5d95cea62a635697f8f03d134d3f3a96151/pillow-12.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f", size = 4698624, upload-time = "2026-04-01T14:44:21.115Z" }, - { url = "https://files.pythonhosted.org/packages/01/a6/1265e977f17d93ea37aa28aa81bad4fa597933879fac2520d24e021c8da3/pillow-12.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612", size = 6321252, upload-time = "2026-04-01T14:44:23.663Z" }, - { url = "https://files.pythonhosted.org/packages/3c/83/5982eb4a285967baa70340320be9f88e57665a387e3a53a7f0db8231a0cd/pillow-12.2.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c", size = 8126550, upload-time = "2026-04-01T14:44:26.772Z" }, - { url = "https://files.pythonhosted.org/packages/4e/48/6ffc514adce69f6050d0753b1a18fd920fce8cac87620d5a31231b04bfc5/pillow-12.2.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea", size = 6433114, upload-time = "2026-04-01T14:44:29.615Z" }, - { url = "https://files.pythonhosted.org/packages/36/a3/f9a77144231fb8d40ee27107b4463e205fa4677e2ca2548e14da5cf18dce/pillow-12.2.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4", size = 7115667, upload-time = "2026-04-01T14:44:32.773Z" }, - { url = "https://files.pythonhosted.org/packages/c1/fc/ac4ee3041e7d5a565e1c4fd72a113f03b6394cc72ab7089d27608f8aaccb/pillow-12.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4", size = 6538966, upload-time = "2026-04-01T14:44:35.252Z" }, - { url = "https://files.pythonhosted.org/packages/c0/a8/27fb307055087f3668f6d0a8ccb636e7431d56ed0750e07a60547b1e083e/pillow-12.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea", size = 7238241, upload-time = "2026-04-01T14:44:37.875Z" }, - { url = "https://files.pythonhosted.org/packages/ad/4b/926ab182c07fccae9fcb120043464e1ff1564775ec8864f21a0ebce6ac25/pillow-12.2.0-cp313-cp313t-win32.whl", hash = "sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24", size = 6379592, upload-time = "2026-04-01T14:44:40.336Z" }, - { url = "https://files.pythonhosted.org/packages/c2/c4/f9e476451a098181b30050cc4c9a3556b64c02cf6497ea421ac047e89e4b/pillow-12.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98", size = 7085542, upload-time = "2026-04-01T14:44:43.251Z" }, - { url = "https://files.pythonhosted.org/packages/00/a4/285f12aeacbe2d6dc36c407dfbbe9e96d4a80b0fb710a337f6d2ad978c75/pillow-12.2.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453", size = 2465765, upload-time = "2026-04-01T14:44:45.996Z" }, - { url = "https://files.pythonhosted.org/packages/bf/98/4595daa2365416a86cb0d495248a393dfc84e96d62ad080c8546256cb9c0/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8", size = 4100848, upload-time = "2026-04-01T14:44:48.48Z" }, - { url = "https://files.pythonhosted.org/packages/0b/79/40184d464cf89f6663e18dfcf7ca21aae2491fff1a16127681bf1fa9b8cf/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b", size = 4176515, upload-time = "2026-04-01T14:44:51.353Z" }, - { url = "https://files.pythonhosted.org/packages/b0/63/703f86fd4c422a9cf722833670f4f71418fb116b2853ff7da722ea43f184/pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295", size = 3640159, upload-time = "2026-04-01T14:44:53.588Z" }, - { url = "https://files.pythonhosted.org/packages/71/e0/fb22f797187d0be2270f83500aab851536101b254bfa1eae10795709d283/pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed", size = 5312185, upload-time = "2026-04-01T14:44:56.039Z" }, - { url = "https://files.pythonhosted.org/packages/ba/8c/1a9e46228571de18f8e28f16fabdfc20212a5d019f3e3303452b3f0a580d/pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae", size = 4695386, upload-time = "2026-04-01T14:44:58.663Z" }, - { url = "https://files.pythonhosted.org/packages/70/62/98f6b7f0c88b9addd0e87c217ded307b36be024d4ff8869a812b241d1345/pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601", size = 6280384, upload-time = "2026-04-01T14:45:01.5Z" }, - { url = "https://files.pythonhosted.org/packages/5e/03/688747d2e91cfbe0e64f316cd2e8005698f76ada3130d0194664174fa5de/pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be", size = 8091599, upload-time = "2026-04-01T14:45:04.5Z" }, - { url = "https://files.pythonhosted.org/packages/f6/35/577e22b936fcdd66537329b33af0b4ccfefaeabd8aec04b266528cddb33c/pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f", size = 6396021, upload-time = "2026-04-01T14:45:07.117Z" }, - { url = "https://files.pythonhosted.org/packages/11/8d/d2532ad2a603ca2b93ad9f5135732124e57811d0168155852f37fbce2458/pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286", size = 7083360, upload-time = "2026-04-01T14:45:09.763Z" }, - { url = "https://files.pythonhosted.org/packages/5e/26/d325f9f56c7e039034897e7380e9cc202b1e368bfd04d4cbe6a441f02885/pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50", size = 6507628, upload-time = "2026-04-01T14:45:12.378Z" }, - { url = "https://files.pythonhosted.org/packages/5f/f7/769d5632ffb0988f1c5e7660b3e731e30f7f8ec4318e94d0a5d674eb65a4/pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104", size = 7209321, upload-time = "2026-04-01T14:45:15.122Z" }, - { url = "https://files.pythonhosted.org/packages/6a/7a/c253e3c645cd47f1aceea6a8bacdba9991bf45bb7dfe927f7c893e89c93c/pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7", size = 6479723, upload-time = "2026-04-01T14:45:17.797Z" }, - { url = "https://files.pythonhosted.org/packages/cd/8b/601e6566b957ca50e28725cb6c355c59c2c8609751efbecd980db44e0349/pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150", size = 7217400, upload-time = "2026-04-01T14:45:20.529Z" }, - { url = "https://files.pythonhosted.org/packages/d6/94/220e46c73065c3e2951bb91c11a1fb636c8c9ad427ac3ce7d7f3359b9b2f/pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1", size = 2554835, upload-time = "2026-04-01T14:45:23.162Z" }, - { url = "https://files.pythonhosted.org/packages/b6/ab/1b426a3974cb0e7da5c29ccff4807871d48110933a57207b5a676cccc155/pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463", size = 5314225, upload-time = "2026-04-01T14:45:25.637Z" }, - { url = "https://files.pythonhosted.org/packages/19/1e/dce46f371be2438eecfee2a1960ee2a243bbe5e961890146d2dee1ff0f12/pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3", size = 4698541, upload-time = "2026-04-01T14:45:28.355Z" }, - { url = "https://files.pythonhosted.org/packages/55/c3/7fbecf70adb3a0c33b77a300dc52e424dc22ad8cdc06557a2e49523b703d/pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166", size = 6322251, upload-time = "2026-04-01T14:45:30.924Z" }, - { url = "https://files.pythonhosted.org/packages/1c/3c/7fbc17cfb7e4fe0ef1642e0abc17fc6c94c9f7a16be41498e12e2ba60408/pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe", size = 8127807, upload-time = "2026-04-01T14:45:33.908Z" }, - { url = "https://files.pythonhosted.org/packages/ff/c3/a8ae14d6defd2e448493ff512fae903b1e9bd40b72efb6ec55ce0048c8ce/pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd", size = 6433935, upload-time = "2026-04-01T14:45:36.623Z" }, - { url = "https://files.pythonhosted.org/packages/6e/32/2880fb3a074847ac159d8f902cb43278a61e85f681661e7419e6596803ed/pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e", size = 7116720, upload-time = "2026-04-01T14:45:39.258Z" }, - { url = "https://files.pythonhosted.org/packages/46/87/495cc9c30e0129501643f24d320076f4cc54f718341df18cc70ec94c44e1/pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06", size = 6540498, upload-time = "2026-04-01T14:45:41.879Z" }, - { url = "https://files.pythonhosted.org/packages/18/53/773f5edca692009d883a72211b60fdaf8871cbef075eaa9d577f0a2f989e/pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43", size = 7239413, upload-time = "2026-04-01T14:45:44.705Z" }, - { url = "https://files.pythonhosted.org/packages/c9/e4/4b64a97d71b2a83158134abbb2f5bd3f8a2ea691361282f010998f339ec7/pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354", size = 6482084, upload-time = "2026-04-01T14:45:47.568Z" }, - { url = "https://files.pythonhosted.org/packages/ba/13/306d275efd3a3453f72114b7431c877d10b1154014c1ebbedd067770d629/pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1", size = 7225152, upload-time = "2026-04-01T14:45:50.032Z" }, - { url = "https://files.pythonhosted.org/packages/ff/6e/cf826fae916b8658848d7b9f38d88da6396895c676e8086fc0988073aaf8/pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb", size = 2556579, upload-time = "2026-04-01T14:45:52.529Z" }, +version = "12.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, + { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, + { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, + { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, + { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, + { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, + { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, + { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, + { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ac/31fb64e1e7efb5a4b50cd3d92049ba89ac6e4d8d3bb6a74e15048ca3353e/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89", size = 4161684, upload-time = "2026-07-01T11:54:25.934Z" }, + { url = "https://files.pythonhosted.org/packages/87/b4/9805e23d2b4d77842b468513841fda254ee42f0289d25088340e4ff46e2d/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace", size = 4255487, upload-time = "2026-07-01T11:54:27.935Z" }, + { url = "https://files.pythonhosted.org/packages/df/39/ecf519435a200c693fe053a6ee4d835b41cf963a4dfc2551c4e637cb2a71/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec", size = 3696433, upload-time = "2026-07-01T11:54:29.813Z" }, + { url = "https://files.pythonhosted.org/packages/42/92/2fc3ffad878ae8dd5469ec1bc8eb83b71f48e13efdf68f02709003982a32/pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66", size = 5345889, upload-time = "2026-07-01T11:54:31.97Z" }, + { url = "https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35", size = 4780109, upload-time = "2026-07-01T11:54:34.026Z" }, + { url = "https://files.pythonhosted.org/packages/1f/01/e18aff37cb0b4aac47ac90f016d347a49aca667ef97f190b06ac2aabc928/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65", size = 6263736, upload-time = "2026-07-01T11:54:36.131Z" }, + { url = "https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3", size = 6937129, upload-time = "2026-07-01T11:54:38.216Z" }, + { url = "https://files.pythonhosted.org/packages/70/4d/105627a13300c5e0df1d174230b32fd1273062c96f7745fd552b945d1e1d/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a", size = 6339562, upload-time = "2026-07-01T11:54:40.354Z" }, + { url = "https://files.pythonhosted.org/packages/6b/1d/f13de01a553988ab895ba1c722e06cf3144d4f57656fd5b81b6d881f1179/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e", size = 7049439, upload-time = "2026-07-01T11:54:42.489Z" }, + { url = "https://files.pythonhosted.org/packages/c9/f9/066794cca041b969964f779ee5fa66a9498bbf34248ac39c5d7954e4198f/pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f", size = 6473287, upload-time = "2026-07-01T11:54:44.9Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", size = 7239691, upload-time = "2026-07-01T11:54:47.141Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b0/c4ed4f0ef8f8fa5ee8351537db6650bb8189f7e118842978dd6589065692/pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b", size = 2568185, upload-time = "2026-07-01T11:54:49.137Z" }, + { url = "https://files.pythonhosted.org/packages/dc/01/001f65b68192f0228cc1dbbc8d2530ab5d58b61037ba0587f946fea607cd/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330", size = 4161736, upload-time = "2026-07-01T11:54:51.156Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d2/0219746d0fd16fc8a84498e79452375be3797d3ce4044596ce565164b84f/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217", size = 4255435, upload-time = "2026-07-01T11:54:53.414Z" }, + { url = "https://files.pythonhosted.org/packages/c8/02/8d0bc62ef0302318c46ff2a512822d2610e81c7aa46c9b3abe6cbaca5ad0/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930", size = 3696262, upload-time = "2026-07-01T11:54:55.739Z" }, + { url = "https://files.pythonhosted.org/packages/85/e2/73c77d218410b14f5f2d565e8a998d5317b7b9c75368d29985139f7a46f0/pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8", size = 5350344, upload-time = "2026-07-01T11:54:57.657Z" }, + { url = "https://files.pythonhosted.org/packages/c7/da/32c752228ae345f489e3a42499d817b6c3996da7e8a3bc7a04fc806b243b/pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0", size = 4780131, upload-time = "2026-07-01T11:54:59.713Z" }, + { url = "https://files.pythonhosted.org/packages/b1/9d/8b2c807dbef61a5197c047afe99823787eb66f63daf9fb2432f91d6f0462/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321", size = 6263757, upload-time = "2026-07-01T11:55:01.778Z" }, + { url = "https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b", size = 6936962, upload-time = "2026-07-01T11:55:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/18/7e/e483414b35800b86b6f08dbbc7803fb5cd52c4d6f897f47d53ea2c7e6f65/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198", size = 6339171, upload-time = "2026-07-01T11:55:05.989Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f4/68c491844841ede6bed70189546b3ee9731cf9f2cbad396faff5e1ccba45/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130", size = 7048116, upload-time = "2026-07-01T11:55:08.131Z" }, + { url = "https://files.pythonhosted.org/packages/a3/34/77f3f793fed8efc7d243f21b33c5a3f0d1c97ee70346d3db855587e155ff/pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a", size = 6467209, upload-time = "2026-07-01T11:55:10.408Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e0/492879f69d94f91f60fc8cd05ba03650e9520afebb2fb7aa12777d7c7f38/pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d", size = 7237707, upload-time = "2026-07-01T11:55:12.745Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ac/6b11f2875f1c2ac040d84e1bbf9cf22a88038f901ca1037898b280b38365/pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838", size = 2565995, upload-time = "2026-07-01T11:55:14.736Z" }, + { url = "https://files.pythonhosted.org/packages/52/69/c2208e56af9bfc1913afb24020297a691eb1d4ef688474c8a04913f65e04/pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e", size = 5352503, upload-time = "2026-07-01T11:55:17.076Z" }, + { url = "https://files.pythonhosted.org/packages/07/70/e5686d753e898a45d778ff1718dba8516ead6ab6b95d85fc8c4b70650cf2/pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17", size = 4782956, upload-time = "2026-07-01T11:55:19.448Z" }, + { url = "https://files.pythonhosted.org/packages/d5/37/25c6692f06927ee973ff18c8d9ee98ad0b4d84ee67a09610c2dd1447958e/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385", size = 6322855, upload-time = "2026-07-01T11:55:21.613Z" }, + { url = "https://files.pythonhosted.org/packages/cc/91/420637fcb8f1bc11029e403b4538e6694744428d8246118e45719f944556/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c", size = 6989642, upload-time = "2026-07-01T11:55:24.006Z" }, + { url = "https://files.pythonhosted.org/packages/10/08/b94d7811281ccf0d143a1cf768d1c49e1e54af63e7b708ab2ee3eb87face/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d", size = 6391281, upload-time = "2026-07-01T11:55:26.252Z" }, + { url = "https://files.pythonhosted.org/packages/d2/87/24233f785f55474dc02ce3e739c5528a77e3a862e9333d1dd7a25cc31f70/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931", size = 7096716, upload-time = "2026-07-01T11:55:28.318Z" }, + { url = "https://files.pythonhosted.org/packages/23/26/fcb2f6e37175b04f53570b59937867e2b80ee1685e744023153028fc14f9/pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7", size = 6474125, upload-time = "2026-07-01T11:55:30.956Z" }, + { url = "https://files.pythonhosted.org/packages/90/de/3634abee5f1c9e13c56787b7d5517b0ba8d6de51700b95578cf338349c9f/pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c", size = 7242939, upload-time = "2026-07-01T11:55:34.044Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2a/fd13f8eb24de5714a6eb444a3d67e2842c6c576e159a43793adf23051351/pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45", size = 2567506, upload-time = "2026-07-01T11:55:35.988Z" }, + { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" }, + { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" }, + { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" }, + { url = "https://files.pythonhosted.org/packages/12/40/d306fc2c8e4d45d7f175c77edca7063be7b86fe7fe6e68f4353bf71d808c/pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f", size = 5350370, upload-time = "2026-07-01T11:55:44.028Z" }, + { url = "https://files.pythonhosted.org/packages/dd/44/668fb1437e8ce420f62d6106eb66e44a5971602a4d794615bdf79315d82d/pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701", size = 4780147, upload-time = "2026-07-01T11:55:46.073Z" }, + { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" }, + { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" }, + { url = "https://files.pythonhosted.org/packages/54/76/a09cc3ccc8d773a7283d34c38bec1708f9e3cc932093cbc4c5e71ac4060b/pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827", size = 6467375, upload-time = "2026-07-01T11:55:57.769Z" }, + { url = "https://files.pythonhosted.org/packages/3e/03/1846c49ba3b1d5550392a4bbd06d6fb4578e1cd91a803198b5c90f5f7d53/pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5", size = 7237048, upload-time = "2026-07-01T11:55:59.975Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bb/89f35dcc79610423f9f195504d7def7f0d1416a711541b42867e25fe3412/pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658", size = 2566006, upload-time = "2026-07-01T11:56:02.143Z" }, + { url = "https://files.pythonhosted.org/packages/30/88/707027ba09942dfa2c28759b5c222d769290a41c6d20ea60ec250801941f/pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf", size = 5352509, upload-time = "2026-07-01T11:56:04.2Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6d/00352fa25332c2569cd387851f568cc5a4b75a9adbfb37ac4fbce4c02eec/pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64", size = 4783167, upload-time = "2026-07-01T11:56:06.631Z" }, + { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" }, + { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" }, + { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, + { url = "https://files.pythonhosted.org/packages/18/f0/a5595c1e8c3ae44b9828cb2f0fa8155e5095ef04d6327b8f61cf44a3df85/pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8", size = 6474384, upload-time = "2026-07-01T11:56:18.855Z" }, + { url = "https://files.pythonhosted.org/packages/e4/04/62bcd9f844984c5938d3b05264a61d797a29d3e0812341a8204af70bbdee/pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418", size = 7243537, upload-time = "2026-07-01T11:56:21.214Z" }, + { url = "https://files.pythonhosted.org/packages/3d/68/1f3066acedf37673694a7141381d8f811ae97f30d34413d236abe7d489f1/pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59", size = 2567491, upload-time = "2026-07-01T11:56:23.506Z" }, ] [[package]] From 8acbbb12dbc533a086e225908158e2dfb25dc49a Mon Sep 17 00:00:00 2001 From: Rohit Malhotra Date: Tue, 18 Aug 2026 19:33:17 -0400 Subject: [PATCH 04/14] feat(telemetry): identify automation conversations (#4529) Co-authored-by: openhands --- .../agent_server/conversation_service.py | 9 +++ .../agent_server/telemetry/models.py | 2 + .../agent_server/telemetry/subscriber.py | 2 + .../telemetry/test_telemetry_subscriber.py | 61 ++++++++++++++++++- 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/openhands-agent-server/openhands/agent_server/conversation_service.py b/openhands-agent-server/openhands/agent_server/conversation_service.py index 9d727f4131..7b93bc5a10 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_service.py +++ b/openhands-agent-server/openhands/agent_server/conversation_service.py @@ -78,6 +78,9 @@ from openhands.sdk.subagent.schema import AgentDefinition +_AUTOMATION_TAG_KEYS = ("automationtrigger", "automationid", "automationrunid") + + class CredentialBindingActivationRequired(RuntimeError): pass @@ -2334,6 +2337,11 @@ def _build_telemetry_context( agent explicitly. When ``agent`` is ``None`` (no live conversation), the agent-derived fields simply degrade to ``unknown``. """ + tags = getattr(stored, "tags", None) + is_automation = isinstance(tags, dict) and any( + bool(tags.get(key)) for key in _AUTOMATION_TAG_KEYS + ) + llm = getattr(agent, "llm", None) workspace = getattr(stored, "workspace", None) @@ -2357,6 +2365,7 @@ def _build_telemetry_context( confirmation_policy=safe_token( type(getattr(stored, "confirmation_policy", None)).__name__.lower() ), + is_automation=is_automation, ) diff --git a/openhands-agent-server/openhands/agent_server/telemetry/models.py b/openhands-agent-server/openhands/agent_server/telemetry/models.py index 7f498f66ff..14a163b1b5 100644 --- a/openhands-agent-server/openhands/agent_server/telemetry/models.py +++ b/openhands-agent-server/openhands/agent_server/telemetry/models.py @@ -169,6 +169,7 @@ class ConversationStartedProperties(_BaseProperties): has_agent_profile: bool workspace_kind: SafeToken confirmation_policy: SafeToken + is_automation: bool = False class ConversationOutcomeProperties(_BaseProperties): @@ -304,6 +305,7 @@ def to_payload(self) -> dict[str, object]: "has_agent_profile", "workspace_kind", "confirmation_policy", + "is_automation", "terminal_status", "duration_bucket", "event_count_bucket", diff --git a/openhands-agent-server/openhands/agent_server/telemetry/subscriber.py b/openhands-agent-server/openhands/agent_server/telemetry/subscriber.py index d7b3f7bfe8..cc45c7f8db 100644 --- a/openhands-agent-server/openhands/agent_server/telemetry/subscriber.py +++ b/openhands-agent-server/openhands/agent_server/telemetry/subscriber.py @@ -77,6 +77,7 @@ class ConversationTelemetryContext: has_agent_profile: bool workspace_kind: str confirmation_policy: str + is_automation: bool = False @dataclass(slots=True) @@ -146,6 +147,7 @@ def emit_started(self) -> None: has_agent_profile=self.context.has_agent_profile, workspace_kind=self.context.workspace_kind, confirmation_policy=self.context.confirmation_policy, + is_automation=self.context.is_automation, ) self.sink.emit( self.factory.build( diff --git a/tests/agent_server/telemetry/test_telemetry_subscriber.py b/tests/agent_server/telemetry/test_telemetry_subscriber.py index 08846ac4a2..4d1295c37f 100644 --- a/tests/agent_server/telemetry/test_telemetry_subscriber.py +++ b/tests/agent_server/telemetry/test_telemetry_subscriber.py @@ -56,7 +56,13 @@ def factory() -> DiagnosticEventFactory: ) -def make_subscriber(sink, factory, user_id: str | None = "user-1"): +def make_subscriber( + sink, + factory, + user_id: str | None = "user-1", + *, + is_automation: bool = False, +): conversation_id = uuid.uuid4() return TelemetrySubscriber( conversation_id=conversation_id, @@ -72,6 +78,7 @@ def make_subscriber(sink, factory, user_id: str | None = "user-1"): has_agent_profile=False, workspace_kind="localworkspace", confirmation_policy="neverconfirm", + is_automation=is_automation, ), ) @@ -87,6 +94,15 @@ async def test_emits_exactly_one_created_event(factory): assert sink.names == [m.EventName.CONVERSATION_CREATED] +async def test_created_event_identifies_automation_conversations(factory): + sink = CollectingSink() + sub = make_subscriber(sink, factory, is_automation=True) + + sub.emit_started() + + assert sink.events[0].to_payload()["is_automation"] is True + + def test_started_is_only_emitted_for_genuinely_new_conversations(): """Regression: ``_start_event_service`` also runs on rehydration. @@ -601,3 +617,46 @@ def test_confirmation_policy_is_read_from_the_field_that_exists(): fields = asdict(ctx) assert "unknown" not in fields.values() assert "secret-project" not in repr(fields) + + +@pytest.mark.parametrize( + ("tags", "is_automation"), + [ + ({"automationtrigger": "cron"}, True), + ({"automationid": "auto-1"}, True), + ({"automationrunid": "run-1"}, True), + ({"automationtrigger": ""}, False), + ({"automationname": "Nightly Audit"}, False), + ({"source": "automation"}, False), + ({}, False), + ], +) +def test_is_automation_is_derived_from_allowlisted_tags(tags, is_automation): + from openhands.agent_server.conversation_service import _build_telemetry_context + from openhands.agent_server.models import StoredConversation + from openhands.agent_server.telemetry.factory import ( + DiagnosticEventFactory, + build_runtime_properties, + ) + from openhands.sdk.agent import Agent + from openhands.sdk.llm import LLM + from openhands.sdk.workspace import LocalWorkspace + + agent = Agent(llm=LLM(model="anthropic/claude-sonnet-5", usage_id="t"), tools=[]) + stored = StoredConversation( + id=uuid.uuid4(), + workspace=LocalWorkspace(working_dir="/tmp"), + user_id="canvas-user-42", + tags=tags, + ) + + context = _build_telemetry_context( + stored, + DiagnosticEventFactory( + runtime=build_runtime_properties(deferred_init=False), + salt="s", + ), + agent=agent, + ) + + assert context.is_automation is is_automation From ddf1cc2af9790624007f1231964e1a5a506e70fb Mon Sep 17 00:00:00 2001 From: Engel Nyst Date: Wed, 19 Aug 2026 15:51:33 +0200 Subject: [PATCH 05/14] Weekly test sweep: remove low-value tests + simplify (#4484) Co-authored-by: smolpaws --- AGENTS.md | 1 + .../conversation/impl/remote_conversation.py | 12 - .../openhands/tools/browser_use/impl.py | 30 -- .../telemetry/test_telemetry_concurrency.py | 87 ----- tests/sdk/agent/test_agent_immutability.py | 146 +-------- tests/sdk/agent/test_agent_serialization.py | 91 ------ .../conversation/remote/test_remote_state.py | 18 -- .../test_agent_state_reassignment.py | 268 --------------- tests/sdk/llm/auth/test_openai.py | 73 +---- tests/sdk/llm/test_llm_metrics.py | 266 --------------- tests/sdk/tool/test_response_schema.py | 23 -- tests/sdk/tool/test_schema_immutability.py | 306 +----------------- tests/sdk/tool/test_tool.py | 160 +-------- .../browser_use/test_chromium_detection.py | 81 +---- .../tools/file_editor/utils/test_encoding.py | 9 - .../file_editor/utils/test_file_cache.py | 31 -- tests/tools/test_tool_name_consistency.py | 69 ---- 17 files changed, 22 insertions(+), 1649 deletions(-) delete mode 100644 tests/sdk/conversation/test_agent_state_reassignment.py delete mode 100644 tests/tools/test_tool_name_consistency.py diff --git a/AGENTS.md b/AGENTS.md index 9dd0571be1..9ada73a1d0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -283,6 +283,7 @@ gh run rerun --repo / --failed - DON'T write TEST CLASSES unless absolutely necessary! - If you find yourself duplicating logics in preparing mocks, loading data etc, these logic should be fixtures in conftest.py! - Please test only the logic implemented in the current codebase. Do not test functionality (e.g., BaseModel.model_dumps()) that is not implemented in this repository. +- Assert observable behavior rather than source text, static implementation lists, private helpers or state, generic framework behavior, exhaustive default/export mirrors, or mock wiring. Tests should survive behavior-preserving refactors. - For changes to prompt templates, tool descriptions, or agent decision logic, add the `integration-test` label to trigger integration tests and verify no unexpected impact on benchmark performance. # Stress Tests diff --git a/openhands-sdk/openhands/sdk/conversation/impl/remote_conversation.py b/openhands-sdk/openhands/sdk/conversation/impl/remote_conversation.py index 740b97b334..0c6f1d3dfd 100644 --- a/openhands-sdk/openhands/sdk/conversation/impl/remote_conversation.py +++ b/openhands-sdk/openhands/sdk/conversation/impl/remote_conversation.py @@ -584,18 +584,6 @@ def execution_status(self) -> ConversationExecutionStatus: ) return ConversationExecutionStatus(status_str) - @execution_status.setter - def execution_status(self, value: ConversationExecutionStatus) -> None: - """Set execution status is No-OP for RemoteConversation. - - # For remote conversations, execution status is managed server-side - # This setter is provided for test compatibility but doesn't actually change remote state # noqa: E501 - """ # noqa: E501 - raise NotImplementedError( - f"Setting execution_status on RemoteState has no effect. " - f"Remote execution status is managed server-side. Attempted to set: {value}" - ) - @property def confirmation_policy(self) -> ConfirmationPolicyBase: """The confirmation policy.""" diff --git a/openhands-tools/openhands/tools/browser_use/impl.py b/openhands-tools/openhands/tools/browser_use/impl.py index d7e6e37b45..c63a646fe5 100644 --- a/openhands-tools/openhands/tools/browser_use/impl.py +++ b/openhands-tools/openhands/tools/browser_use/impl.py @@ -8,7 +8,6 @@ import logging import os import shutil -import subprocess import sys import threading from collections.abc import Callable, Coroutine @@ -21,7 +20,6 @@ from openhands.sdk.logger import DEBUG, get_logger from openhands.sdk.tool import ToolExecutor -from openhands.sdk.utils import sanitized_env from openhands.sdk.utils.async_executor import AsyncExecutor from openhands.tools.browser_use.definition import ( BROWSER_RECORDING_OUTPUT_DIR, @@ -225,34 +223,6 @@ def _format_browser_operation_error( return f"Browser operation failed: {error_detail}" -def _install_chromium() -> bool: - """Attempt to install Chromium via uvx playwright install.""" - try: - # Check if uvx is available - if not shutil.which("uvx"): - logger.warning("uvx not found - cannot auto-install Chromium") - return False - - logger.info("Attempting to install Chromium via uvx...") - result = subprocess.run( - ["uvx", "playwright", "install", "chromium", "--with-deps", "--no-shell"], - capture_output=True, - text=True, - timeout=300, # 5 minutes timeout for installation - env=sanitized_env(), - ) - - if result.returncode == 0: - logger.info("Chromium installation completed successfully") - return True - else: - logger.error(f"Chromium installation failed: {result.stderr}") - return False - except (subprocess.TimeoutExpired, FileNotFoundError, Exception) as e: - logger.error(f"Error during Chromium installation: {e}") - return False - - def _get_chromium_error_message() -> str: """Get the error message for when Chromium is not available.""" return ( diff --git a/tests/agent_server/telemetry/test_telemetry_concurrency.py b/tests/agent_server/telemetry/test_telemetry_concurrency.py index f946d017c4..4a95113cf0 100644 --- a/tests/agent_server/telemetry/test_telemetry_concurrency.py +++ b/tests/agent_server/telemetry/test_telemetry_concurrency.py @@ -17,9 +17,7 @@ """ import asyncio -import inspect -from openhands.agent_server.telemetry.sink import BufferedTelemetrySink from tests.agent_server.telemetry.test_telemetry_sink import ( RecordingExporter, build_sink, @@ -103,17 +101,6 @@ async def test_emit_after_close_is_a_no_op_and_does_not_raise(): assert len(sink._queue) == 0 -async def test_double_start_creates_only_one_drain_task(): - sink = build_sink(RecordingExporter()) - sink.start() - first = sink._drain_task - sink.start() - try: - assert sink._drain_task is first - finally: - await asyncio.wait_for(sink.aclose(), timeout=TIMEOUT) - - # ── concurrent mutation ─────────────────────────────────────────────────── @@ -150,26 +137,6 @@ async def flipper(): await asyncio.wait_for(sink.aclose(), timeout=TIMEOUT) -async def test_many_concurrent_emitters_do_not_exceed_the_bound(): - exporter = RecordingExporter() - sink = build_sink(exporter, max_queue_size=50, flush_delay=3600) - try: - - async def emitter(n: int): - for i in range(200): - sink.emit(make_event(n * 1000 + i)) - if i % 25 == 0: - await asyncio.sleep(0) - - await asyncio.wait_for( - asyncio.gather(*(emitter(n) for n in range(8))), timeout=TIMEOUT - ) - assert len(sink._queue) == 50 - assert sink._queue.maxlen == 50 - finally: - await asyncio.wait_for(sink.aclose(), timeout=TIMEOUT) - - async def test_drain_survives_a_consent_reader_that_always_raises(): calls = {"n": 0} @@ -185,65 +152,11 @@ async def broken_reader(): await asyncio.sleep(0.3) assert calls["n"] >= 1, "consent reader was never exercised" - assert sink._drain_task is not None - assert not sink._drain_task.done(), "drain task died on a reader failure" - # Delivery continues on the last known decision. assert len(exporter.sent) >= 1 finally: await asyncio.wait_for(sink.aclose(), timeout=TIMEOUT) -# ── blocking-call discipline ────────────────────────────────────────────── - - -def test_emit_and_on_decision_changed_are_sync_and_lock_free(): - """Neither may await or touch the settings store on the hot path.""" - for fn in (BufferedTelemetrySink.emit, BufferedTelemetrySink.on_decision_changed): - assert not asyncio.iscoroutinefunction(fn) - src = inspect.getsource(fn) - assert "await" not in src, f"{fn.__name__} must not await" - assert "get_settings_store" not in src, f"{fn.__name__} must not read settings" - - -def test_consent_is_never_read_from_inside_a_settings_lock(): - """store.update() takes a non-reentrant flock; nesting would self-deadlock. - - The consent endpoint must finish its update() before touching the sink, and - the sink must never call back into the store. - """ - import openhands.agent_server.settings_router as router_mod - - src = inspect.getsource(router_mod._apply_settings_update) - update_at = src.index("store.update(") - notify_at = src.index("notify_misc_settings_changed(") - assert update_at < notify_at, ( - "notify_misc_settings_changed must run after store.update() returns, " - "not inside it" - ) - - sink_src = inspect.getsource(BufferedTelemetrySink) - assert "store.update(" not in sink_src, "the sink must never write settings" - - -def test_settings_load_does_not_take_the_file_lock(): - """The drain task reads consent on a worker thread. - - If load() took the same flock that update() does, a consent write on the - event loop and a consent read on the worker thread would contend on every - refresh. It does not — this test pins that assumption. - """ - from openhands.agent_server.persistence.store import FileSettingsStore - - load_src = inspect.getsource(FileSettingsStore.load) - update_src = inspect.getsource(FileSettingsStore.update) - - assert "_file_lock" in update_src, "update() is expected to lock" - assert "_file_lock" not in load_src, ( - "load() started taking the file lock; telemetry's background consent " - "refresh now contends with settings writes" - ) - - async def test_sink_never_blocks_the_loop_under_load(): """A heartbeat coroutine must keep ticking while telemetry is saturated.""" ticks = {"n": 0} diff --git a/tests/sdk/agent/test_agent_immutability.py b/tests/sdk/agent/test_agent_immutability.py index 9b7499f121..1246e84f4f 100644 --- a/tests/sdk/agent/test_agent_immutability.py +++ b/tests/sdk/agent/test_agent_immutability.py @@ -1,148 +1,14 @@ -"""Tests for Agent immutability and statelessness.""" +"""Tests for the Agent immutability contract.""" import pytest -from pydantic import SecretStr, ValidationError +from pydantic import ValidationError from openhands.sdk.agent.agent import Agent from openhands.sdk.llm import LLM -class TestAgentImmutability: - """Test Agent immutability and statelessness.""" +def test_agent_is_frozen(): + agent = Agent(llm=LLM(model="gpt-4o-mini", usage_id="test-llm"), tools=[]) - def setup_method(self): - """Set up test environment.""" - self.llm: LLM = LLM( - model="gpt-4o-mini", api_key=SecretStr("test-key"), usage_id="test-llm" - ) - - def test_agent_is_frozen(self): - """Test that Agent instances are frozen (immutable).""" - agent = Agent(llm=self.llm, tools=[]) - - # Test that we cannot modify core fields after creation - with pytest.raises(ValidationError, match="Instance is frozen"): - agent.llm = "new_value" # type: ignore[assignment] - - with pytest.raises(ValidationError, match="Instance is frozen"): - agent.agent_context = None - - # Verify the agent remains functional after failed modification attempts - assert agent.llm == self.llm - assert isinstance(agent.static_system_message, str) - assert len(agent.static_system_message) > 0 - - def test_system_message_is_computed_property(self): - """Test that system_message is computed on-demand, not stored.""" - agent = Agent(llm=self.llm, tools=[]) - - # Get system message multiple times - should be consistent - msg1 = agent.static_system_message - msg2 = agent.static_system_message - - # Should be the same content and valid - assert msg1 == msg2 - assert isinstance(msg1, str) - assert len(msg1) > 0 - - # Verify it's computed, not stored - assert not hasattr(agent, "_system_message") - assert "system_message" not in agent.__dict__ - - # Basic content validation - should look like a system message - assert any( - keyword in msg1.lower() for keyword in ["assistant", "help", "task", "user"] - ) - - def test_condenser_property_access(self): - """Test that condenser property works correctly.""" - # Test with None condenser - agent1 = Agent(llm=self.llm, tools=[], condenser=None) - assert agent1.condenser is None - - # For testing with a condenser, we'll just test that the property works - # We don't need to test with a real condenser since that would require - # importing and setting up the actual Condenser class - - def test_agent_properties_are_accessible(self): - """Test that all Agent properties are accessible and return expected types.""" - agent = Agent(llm=self.llm, tools=[]) - - # Test inherited properties from AgentBase - assert agent.llm == self.llm - - assert isinstance(agent.tools, list) - assert agent.agent_context is None - assert agent.name == "Agent" - assert isinstance(agent.prompt_dir, str) - - # Test Agent-specific properties - assert isinstance(agent.static_system_message, str) - assert agent.condenser is None - assert agent.system_prompt_filename == "system_prompt.j2" - - def test_agent_is_truly_stateless(self): - """Test that Agent doesn't store computed state.""" - agent = Agent(llm=self.llm, tools=[]) - - # Access system_message multiple times - for _ in range(3): - msg = agent.static_system_message - assert isinstance(msg, str) - assert len(msg) > 0 - - # The only fields should be the ones we explicitly defined -- i.e., those - # in the model definition. But since some are optional (and may not be set), - # and some are computed when models are dumped, we check that no extra - # attributes are present beyond the defined model fields. - expected_fields = set(Agent.model_fields.keys()) - actual_fields = set(agent.model_dump(mode="python").keys()) - computed_fields = set(Agent.model_computed_fields.keys()) - assert actual_fields - computed_fields <= expected_fields - - # Verify no additional attributes are stored - assert not hasattr(agent, "_system_message") - assert not hasattr(agent, "_computed_system_message") - - def test_multiple_agents_are_independent(self): - """Test that multiple Agent instances are independent.""" - agent1 = Agent( - llm=self.llm, tools=[], system_prompt_filename="system_prompt.j2" - ) - agent2 = Agent( - llm=self.llm, tools=[], system_prompt_filename="system_prompt.j2" - ) - - # Compare via model_dump() because direct equality (agent1 == agent2) - # fails: each agent has its own ParallelToolExecutor instance via - # PrivateAttr(default_factory=...), and Pydantic frozen models include - # private attrs in __eq__. - assert agent1.model_dump() == agent2.model_dump() - assert agent1.system_prompt_filename == agent2.system_prompt_filename - - # But they should be different instances - assert agent1 is not agent2 - - # And their system messages should be identical (same config) - assert agent1.static_system_message == agent2.static_system_message - - def test_agent_model_copy_creates_new_instance(self): - """Test that model_copy creates a new Agent instance with modified fields.""" - original_agent = Agent( - llm=self.llm, - tools=[], - system_prompt_kwargs={"cli_mode": True}, - ) - - # Create a copy with modified fields - modified_agent = original_agent.model_copy( - update={"system_prompt_kwargs": {"cli_mode": False}} - ) - - # Verify that a new instance was created - assert modified_agent is not original_agent - - # Verify that system messages are different due to different configs - assert ( - original_agent.static_system_message != modified_agent.static_system_message - ) + with pytest.raises(ValidationError, match="Instance is frozen"): + agent.agent_context = None diff --git a/tests/sdk/agent/test_agent_serialization.py b/tests/sdk/agent/test_agent_serialization.py index 331b0e9ad4..eb328c49e2 100644 --- a/tests/sdk/agent/test_agent_serialization.py +++ b/tests/sdk/agent/test_agent_serialization.py @@ -435,94 +435,3 @@ def test_include_default_tools_serialization_default() -> None: # Default should include both FinishTool and ThinkTool as strings assert "include_default_tools" in agent_dict assert set(agent_dict["include_default_tools"]) == {"FinishTool", "ThinkTool"} - - -def test_include_default_tools_serialization_empty() -> None: - """Test that include_default_tools serializes correctly when empty.""" - llm = LLM(model="test-model", usage_id="test-llm") - agent = Agent(llm=llm, tools=[], include_default_tools=[]) - - # Serialize to JSON - agent_json = agent.model_dump_json() - agent_dict = json.loads(agent_json) - - # Should be empty list - assert agent_dict["include_default_tools"] == [] - - -def test_include_default_tools_serialization_partial() -> None: - """Test that include_default_tools serializes correctly with partial list.""" - llm = LLM(model="test-model", usage_id="test-llm") - agent = Agent(llm=llm, tools=[], include_default_tools=["FinishTool"]) - - # Serialize to JSON - agent_json = agent.model_dump_json() - agent_dict = json.loads(agent_json) - - # Should be serialized as string - assert agent_dict["include_default_tools"] == ["FinishTool"] - - -def test_include_default_tools_deserialization_roundtrip() -> None: - """Test that include_default_tools deserializes correctly after round-trip.""" - llm = LLM(model="test-model", usage_id="test-llm") - agent = Agent(llm=llm, tools=[], include_default_tools=["FinishTool"]) - - # Serialize to JSON - agent_json = agent.model_dump_json() - - # Deserialize from JSON - deserialized_agent = AgentBase.model_validate_json(agent_json) - - # Should have the same include_default_tools - assert isinstance(deserialized_agent, Agent) - assert deserialized_agent.include_default_tools == ["FinishTool"] - - -def test_include_default_tools_deserialization_all_tools() -> None: - """Test that include_default_tools deserializes correctly with all tools.""" - llm = LLM(model="test-model", usage_id="test-llm") - agent = Agent(llm=llm, tools=[], include_default_tools=["FinishTool", "ThinkTool"]) - - # Serialize to JSON - agent_json = agent.model_dump_json() - - # Deserialize from JSON - deserialized_agent = AgentBase.model_validate_json(agent_json) - - # Should have both tools - assert isinstance(deserialized_agent, Agent) - assert set(deserialized_agent.include_default_tools) == {"FinishTool", "ThinkTool"} - - -def test_include_default_tools_deserialization_empty() -> None: - """Test that include_default_tools deserializes correctly when empty.""" - llm = LLM(model="test-model", usage_id="test-llm") - agent = Agent(llm=llm, tools=[], include_default_tools=[]) - - # Serialize to JSON - agent_json = agent.model_dump_json() - - # Deserialize from JSON - deserialized_agent = AgentBase.model_validate_json(agent_json) - - # Should be empty - assert isinstance(deserialized_agent, Agent) - assert deserialized_agent.include_default_tools == [] - - -def test_include_default_tools_deserialization_from_dict() -> None: - """Test that include_default_tools deserializes correctly from dict.""" - agent_dict = { - "llm": {"model": "test-model", "usage_id": "test-llm"}, - "tools": [], - "include_default_tools": ["ThinkTool"], - "kind": "Agent", - } - - # Deserialize from dict - agent = AgentBase.model_validate(agent_dict) - - # Should have ThinkTool - assert isinstance(agent, Agent) - assert agent.include_default_tools == ["ThinkTool"] diff --git a/tests/sdk/conversation/remote/test_remote_state.py b/tests/sdk/conversation/remote/test_remote_state.py index 13201f2eca..7ce35f1c02 100644 --- a/tests/sdk/conversation/remote/test_remote_state.py +++ b/tests/sdk/conversation/remote/test_remote_state.py @@ -106,24 +106,6 @@ def test_remote_state_execution_status( assert state.execution_status == expected -def test_remote_state_execution_status_setter_not_implemented( - mock_client, conversation_id -): - """Test that setting execution_status raises NotImplementedError.""" - mock_events_response = Mock() - mock_events_response.raise_for_status.return_value = None - mock_events_response.json.return_value = {"items": [], "next_page_id": None} - mock_client.request.return_value = mock_events_response - - state = RemoteState(mock_client, conversation_id) - - with pytest.raises( - NotImplementedError, - match="Setting execution_status on RemoteState has no effect", - ): - state.execution_status = ConversationExecutionStatus.PAUSED - - def test_remote_state_confirmation_policy(mock_client, conversation_id, mock_agent): """Test confirmation_policy property.""" conversation_info = create_mock_conversation_info( diff --git a/tests/sdk/conversation/test_agent_state_reassignment.py b/tests/sdk/conversation/test_agent_state_reassignment.py deleted file mode 100644 index fdca78acfe..0000000000 --- a/tests/sdk/conversation/test_agent_state_reassignment.py +++ /dev/null @@ -1,268 +0,0 @@ -"""Test that all writes to agent_state use the reassignment pattern. - -The agent_state field in ConversationState requires reassignment to trigger autosave. -In-place mutations like `state.agent_state[key] = value` will NOT trigger autosave. -The correct pattern is: `state.agent_state = {**state.agent_state, key: value}` - -This test scans the SDK codebase to ensure all writes to agent_state follow -this pattern. -""" - -import ast -from pathlib import Path - -import pytest - - -class AgentStateWriteVisitor(ast.NodeVisitor): - """AST visitor that detects in-place mutations to agent_state.""" - - def __init__(self, filepath: str): - self.filepath = filepath - self.violations: list[tuple[int, str]] = [] - - def visit_Subscript(self, node: ast.Subscript) -> None: - """Detect agent_state[key] = value patterns.""" - # Check if this is an assignment target (left side of =) - # We need to check the parent context, which is tricky with AST - # Instead, we'll check in visit_Assign - self.generic_visit(node) - - def visit_Assign(self, node: ast.Assign) -> None: - """Detect assignments to agent_state subscripts.""" - for target in node.targets: - if isinstance(target, ast.Subscript): - # Check if it's agent_state[...] - if self._is_agent_state_subscript(target): - self.violations.append( - ( - node.lineno, - "In-place mutation: agent_state[...] = ... " - "(use reassignment pattern instead)", - ) - ) - self.generic_visit(node) - - def visit_AugAssign(self, node: ast.AugAssign) -> None: - """Detect augmented assignments like agent_state[key] += value.""" - if isinstance(node.target, ast.Subscript): - if self._is_agent_state_subscript(node.target): - self.violations.append( - ( - node.lineno, - f"In-place mutation: agent_state[...] {ast.dump(node.op)}= ... " - f"(use reassignment pattern instead)", - ) - ) - self.generic_visit(node) - - def visit_Call(self, node: ast.Call) -> None: - """Detect method calls that mutate agent_state in-place.""" - if isinstance(node.func, ast.Attribute): - # Check for agent_state.update(), agent_state.setdefault(), etc. - mutating_methods = { - "update", - "setdefault", - "pop", - "popitem", - "clear", - "__setitem__", - "__delitem__", - } - if node.func.attr in mutating_methods: - if self._is_agent_state_attr(node.func.value): - self.violations.append( - ( - node.lineno, - f"In-place mutation: agent_state.{node.func.attr}() " - f"(use reassignment pattern instead)", - ) - ) - self.generic_visit(node) - - def visit_Delete(self, node: ast.Delete) -> None: - """Detect del agent_state[key] patterns.""" - for target in node.targets: - if isinstance(target, ast.Subscript): - if self._is_agent_state_subscript(target): - self.violations.append( - ( - node.lineno, - "In-place mutation: del agent_state[...] " - "(use reassignment pattern instead)", - ) - ) - self.generic_visit(node) - - def _is_agent_state_subscript(self, node: ast.Subscript) -> bool: - """Check if a subscript is accessing agent_state.""" - return self._is_agent_state_attr(node.value) - - def _is_agent_state_attr(self, node: ast.AST) -> bool: - """Check if a node refers to agent_state.""" - # Direct name: agent_state[...] - if isinstance(node, ast.Name) and node.id == "agent_state": - return True - # Attribute access: state.agent_state[...] or self.state.agent_state[...] - if isinstance(node, ast.Attribute) and node.attr == "agent_state": - return True - return False - - -def get_sdk_python_files() -> list[Path]: - """Get all Python files in the SDK source directory.""" - sdk_dir = Path(__file__).parent.parent.parent.parent / "openhands-sdk" - if not sdk_dir.exists(): - pytest.skip(f"SDK directory not found: {sdk_dir}") - - python_files = [] - for py_file in sdk_dir.rglob("*.py"): - # Skip __pycache__ and test files - if "__pycache__" in str(py_file): - continue - python_files.append(py_file) - - return python_files - - -def test_agent_state_writes_use_reassignment_pattern(): - """Verify all writes to agent_state use the reassignment pattern. - - The agent_state field requires reassignment to trigger autosave: - - WRONG: state.agent_state[key] = value (no autosave) - - WRONG: state.agent_state.update({key: value}) (no autosave) - - RIGHT: state.agent_state = {**state.agent_state, key: value} (triggers autosave) - - This test scans all SDK Python files and fails if any in-place mutations - to agent_state are found. - """ - python_files = get_sdk_python_files() - all_violations: list[tuple[Path, int, str]] = [] - - for py_file in python_files: - try: - source = py_file.read_text(encoding="utf-8") - tree = ast.parse(source, filename=str(py_file)) - except SyntaxError: - continue - - visitor = AgentStateWriteVisitor(str(py_file)) - visitor.visit(tree) - - for lineno, message in visitor.violations: - all_violations.append((py_file, lineno, message)) - - if all_violations: - error_msg = "Found in-place mutations to agent_state:\n" - for filepath, lineno, message in all_violations: - error_msg += f" {filepath}:{lineno}: {message}\n" - error_msg += ( - "\nTo trigger autosave, use the reassignment pattern:\n" - " state.agent_state = {**state.agent_state, key: value}" - ) - pytest.fail(error_msg) - - -def test_agent_state_reassignment_triggers_autosave(): - """Verify that reassigning agent_state triggers autosave. - - This is a runtime test that verifies the autosave mechanism works - correctly when agent_state is reassigned. - """ - import uuid - - from pydantic import SecretStr - - from openhands.sdk import Agent - from openhands.sdk.conversation.state import ConversationState - from openhands.sdk.io import InMemoryFileStore - from openhands.sdk.llm import LLM - from openhands.sdk.workspace import LocalWorkspace - - # Create a state with autosave enabled - llm = LLM(model="gpt-4o-mini", api_key=SecretStr("test-key"), usage_id="test-llm") - agent = Agent(llm=llm) - workspace = LocalWorkspace(working_dir="/tmp/test") - - state = ConversationState( - id=uuid.uuid4(), - workspace=workspace, - persistence_dir="/tmp/test/.state", - agent=agent, - ) - - # Set up filestore and enable autosave - fs = InMemoryFileStore() - state._fs = fs - state._autosave_enabled = True - - # Track saves - save_count = 0 - original_save = state._save_base_state - - def counting_save(fs): - nonlocal save_count - save_count += 1 - original_save(fs) - - state._save_base_state = counting_save - - # Reassign agent_state - should trigger autosave - with state: - state.agent_state = {**state.agent_state, "test_key": "test_value"} - - assert save_count == 1, "Reassigning agent_state should trigger autosave" - assert state.agent_state.get("test_key") == "test_value" - - -def test_agent_state_inplace_mutation_does_not_trigger_autosave(): - """Verify that in-place mutation of agent_state does NOT trigger autosave. - - This test demonstrates why the reassignment pattern is required. - """ - import uuid - - from pydantic import SecretStr - - from openhands.sdk import Agent - from openhands.sdk.conversation.state import ConversationState - from openhands.sdk.io import InMemoryFileStore - from openhands.sdk.llm import LLM - from openhands.sdk.workspace import LocalWorkspace - - # Create a state with autosave enabled - llm = LLM(model="gpt-4o-mini", api_key=SecretStr("test-key"), usage_id="test-llm") - agent = Agent(llm=llm) - workspace = LocalWorkspace(working_dir="/tmp/test") - - state = ConversationState( - id=uuid.uuid4(), - workspace=workspace, - persistence_dir="/tmp/test/.state", - agent=agent, - ) - - # Set up filestore and enable autosave - fs = InMemoryFileStore() - state._fs = fs - state._autosave_enabled = True - - # Track saves - save_count = 0 - original_save = state._save_base_state - - def counting_save(fs): - nonlocal save_count - save_count += 1 - original_save(fs) - - state._save_base_state = counting_save - - # In-place mutation - should NOT trigger autosave (this is the problem!) - with state: - state.agent_state["test_key"] = "test_value" - - # This demonstrates the problem: in-place mutation doesn't trigger autosave - assert save_count == 0, "In-place mutation should NOT trigger autosave" - # But the value is still set in memory - assert state.agent_state.get("test_key") == "test_value" diff --git a/tests/sdk/llm/auth/test_openai.py b/tests/sdk/llm/auth/test_openai.py index 889668a822..fc5b361172 100644 --- a/tests/sdk/llm/auth/test_openai.py +++ b/tests/sdk/llm/auth/test_openai.py @@ -70,8 +70,8 @@ def test_build_authorize_url(): assert "response_type=code" in url -def test_openai_codex_models(): - """Test that OPENAI_CODEX_MODELS contains expected models.""" +def test_openai_codex_models_include_acp_models(): + """Subscription auth supports every model exposed by the Codex provider.""" from openhands.sdk.settings.acp_providers import get_acp_provider codex_provider = get_acp_provider("codex") @@ -79,20 +79,6 @@ def test_openai_codex_models(): assert OPENAI_CODEX_MODELS.issuperset( model.id for model in codex_provider.available_models ) - assert "gpt-5.6" in OPENAI_CODEX_MODELS - assert "gpt-5.6-sol" in OPENAI_CODEX_MODELS - assert "gpt-5.6-terra" in OPENAI_CODEX_MODELS - assert "gpt-5.6-luna" in OPENAI_CODEX_MODELS - assert "gpt-5.5" in OPENAI_CODEX_MODELS - assert "gpt-5.4" in OPENAI_CODEX_MODELS - assert "gpt-5.4-mini" in OPENAI_CODEX_MODELS - assert "gpt-5.3-codex" not in OPENAI_CODEX_MODELS - - -def test_openai_subscription_auth_vendor(): - """Test OpenAISubscriptionAuth vendor property.""" - auth = OpenAISubscriptionAuth() - assert auth.vendor == "openai" def test_openai_subscription_auth_get_credentials(tmp_path): @@ -571,61 +557,6 @@ def test_display_consent_eof_error(self, tmp_path): assert result is False -# ========================================================================= -# Tests for joserfc migration (no authlib.jose deprecation warning) -# ========================================================================= - - -def test_no_authlib_jose_import(): - """Verify that the openai auth module does not import from authlib.jose. - - The authlib.jose module is deprecated and should be replaced by joserfc. - """ - import importlib - import sys - - # Remove cached module to force re-import - mod_name = "openhands.sdk.llm.auth.openai" - if mod_name in sys.modules: - importlib.reload(sys.modules[mod_name]) - - import inspect - - from openhands.sdk.llm.auth import openai as openai_auth_mod - - source = inspect.getsource(openai_auth_mod) - assert "from authlib.jose" not in source, ( - "Module still imports from the deprecated authlib.jose; use joserfc instead" - ) - - -def test_joserfc_keyset_import(): - """Test that joserfc KeySet can import a JWKS structure.""" - from joserfc.jwk import KeySetSerialization - - # Minimal valid RSA JWK for testing (RFC 7517 example modulus) - rsa_n = ( - "0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4" - "cbbfAAtVT86zwu1RK7aPFFxuhDR1L6tSoc_BJECPebWKRXjBZCiF" - "V4n3oknjhMstn64tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6C" - "f0h4QyQ5v-65YGjQR0_FDW2QvzqY368QQMicAtaSqzs8KJZgnYb9" - "c7d0zgdAZHzu6qMQvRL5hajrn1n91CbOpbISD08qNLyrdkt-bFTWh" - "AI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINHaQ-G_xBniIqbw0Ls1j" - "F44-csFCur-kEgU8awapJzKnqDKgw" - ) - test_jwks: KeySetSerialization = { - "keys": [ - {"kty": "RSA", "kid": "test-key-1", "use": "sig", "n": rsa_n, "e": "AQAB"} - ] - } - - key_set = KeySet.import_key_set(test_jwks) - assert key_set is not None - # Should have imported one key - keys = list(key_set) - assert len(keys) == 1 - - # ========================================================================= # End-to-end tests for _extract_chatgpt_account_id with joserfc # ========================================================================= diff --git a/tests/sdk/llm/test_llm_metrics.py b/tests/sdk/llm/test_llm_metrics.py index 7b92c1d893..3c7fffcf91 100644 --- a/tests/sdk/llm/test_llm_metrics.py +++ b/tests/sdk/llm/test_llm_metrics.py @@ -37,22 +37,6 @@ def test_cost_creation_negative_fails(): assert "cost" in errors[0]["loc"] -def test_cost_pydantic_features(): - """Test Pydantic features work correctly.""" - cost = Cost(cost=2.5, model="gpt-3.5") - - # Test model_dump - data = cost.model_dump() - assert data["cost"] == 2.5 - assert data["model"] == "gpt-3.5" - assert "timestamp" in data - - # Test model_validate - cost2 = Cost.model_validate(data) - assert cost2.cost == cost.cost - assert cost2.model == cost.model - - def test_response_latency_creation_valid(): """Test creating a valid ResponseLatency instance.""" latency = ResponseLatency(model="gpt-4o-mini", latency=1.5, response_id="test-123") @@ -78,21 +62,6 @@ def test_response_latency_creation_negative_fails(): assert "latency" in errors[0]["loc"] -def test_response_latency_pydantic_features(): - """Test Pydantic features work correctly.""" - latency = ResponseLatency(model="gpt-4o-mini", latency=2.3, response_id="test-789") - - # Test model_dump - data = latency.model_dump() - expected = {"model": "gpt-4o-mini", "latency": 2.3, "response_id": "test-789"} - assert data == expected - - # Test model_validate - latency2 = ResponseLatency.model_validate(data) - assert latency2.latency == latency.latency - assert latency2.response_id == latency.response_id - - def test_token_usage_creation_valid(): """Test creating a valid TokenUsage instance.""" usage = TokenUsage( @@ -238,41 +207,6 @@ def test_token_usage_addition(): assert combined.response_id == "test-1" # Should keep first response_id -def test_token_usage_pydantic_features(): - """Test Pydantic features work correctly.""" - usage = TokenUsage( - model="gpt-3.5", - prompt_tokens=75, - completion_tokens=25, - cache_read_tokens=5, - cache_write_tokens=2, - context_window=2048, - per_turn_token=102, - response_id="test-456", - ) - - # Test model_dump - data = usage.model_dump() - expected = { - "model": "gpt-3.5", - "prompt_tokens": 75, - "completion_tokens": 25, - "cache_read_tokens": 5, - "cache_write_tokens": 2, - "reasoning_tokens": 0, - "context_window": 2048, - "per_turn_token": 102, - "response_id": "test-456", - } - assert data == expected - - # Test model_validate - usage2 = TokenUsage.model_validate(data) - assert usage2.model == usage.model - assert usage2.prompt_tokens == usage.prompt_tokens - assert usage2.completion_tokens == usage.completion_tokens - - def test_metrics_creation_empty(): """Test creating an empty Metrics instance.""" metrics = Metrics() @@ -503,61 +437,6 @@ def test_metrics_deep_copy(): assert metrics.accumulated_cost == 5.0 -def test_metrics_pydantic_features(): - """Test Pydantic features work correctly.""" - metrics = Metrics(model_name="gpt-4o-mini") - metrics.add_cost(5.0) - metrics.add_token_usage(100, 50, 10, 5, 4096, "test-123") - - # Test model_dump - data = metrics.model_dump() - assert data["accumulated_cost"] == 5.0 - assert data["accumulated_token_usage"]["prompt_tokens"] == 100 - - # Test model_validate - metrics2 = Metrics.model_validate(data) - assert metrics2.model_name == metrics.model_name - assert metrics2.accumulated_cost == metrics.accumulated_cost - assert metrics2.accumulated_token_usage is not None - assert metrics.accumulated_token_usage is not None - assert ( - metrics2.accumulated_token_usage.prompt_tokens - == metrics.accumulated_token_usage.prompt_tokens - ) - - -def test_metrics_validation_errors(): - """Test that validation errors are properly raised.""" - # Test that we can't create metrics with invalid nested data - with pytest.raises(ValidationError): - Metrics.model_validate( - { - "accumulated_cost": -1.0, # Should be caught by validation - "accumulated_token_usage": None, - "costs": [], - "response_latencies": [], - "token_usages": [], - } - ) - - -def test_metrics_model_validator(): - """Test the model validator for accumulated_cost consistency.""" - # This should work - cost matches sum of costs - data = { - "accumulated_cost": 8.0, - "accumulated_token_usage": None, - "costs": [ - {"cost": 5.0, "model": "gpt-4o-mini", "response_id": "test-1"}, - {"cost": 3.0, "model": "gpt-4o-mini", "response_id": "test-2"}, - ], - "response_latencies": [], - "token_usages": [], - } - metrics = Metrics.model_validate(data) - assert metrics.accumulated_cost == 8.0 - - def test_metrics_empty_state_operations(): """Test operations on empty metrics work correctly.""" metrics = Metrics() @@ -579,39 +458,6 @@ def test_metrics_empty_state_operations(): assert metrics.accumulated_token_usage is not None -def test_metrics_as_pydantic_field(): - """Test that Metrics can be used as a field in another Pydantic class.""" - from pydantic import BaseModel - - class TestModel(BaseModel): - name: str - metrics: Metrics - - # Create a metrics instance - metrics = Metrics(model_name="gpt-4o-mini") - metrics.add_cost(5.0) - - # Use it in another model - test_model = TestModel(name="test", metrics=metrics) - assert test_model.name == "test" - assert test_model.metrics.model_name == "gpt-4o-mini" - assert test_model.metrics.accumulated_cost == 5.0 - - # Test serialization/deserialization - data = test_model.model_dump() - test_model2 = TestModel.model_validate(data) - assert test_model2.metrics.accumulated_cost == 5.0 - - -def test_metrics_cost_negative_validation(): - """Test Cost validation with negative values (line 17).""" - # Test negative cost validation - Pydantic validation happens first - with pytest.raises( - ValidationError, match="Input should be greater than or equal to 0" - ): - Cost(model="test-model", cost=-1.0) - - def test_metrics_accumulated_cost_negative_validation(): """Test Metrics accumulated cost validation with negative values (line 105).""" # Create a metrics instance with negative accumulated cost @@ -621,28 +467,6 @@ def test_metrics_accumulated_cost_negative_validation(): Metrics(accumulated_cost=-1.0) -def test_metrics_add_token_usage_none_accumulated(): - """Test adding token usage when accumulated_token_usage is None (line 172).""" - # Create metrics - it auto-initializes accumulated_token_usage - metrics = Metrics() - assert metrics.accumulated_token_usage is not None - assert metrics.accumulated_token_usage.prompt_tokens == 0 - - # Add token usage - should update accumulated_token_usage (line 172) - metrics.add_token_usage( - prompt_tokens=10, - completion_tokens=5, - cache_read_tokens=0, - cache_write_tokens=0, - context_window=100, - response_id="test-response", - ) - - assert metrics.accumulated_token_usage is not None - assert metrics.accumulated_token_usage.prompt_tokens == 10 - assert metrics.accumulated_token_usage.completion_tokens == 5 - - def test_metrics_merge_max_budget_from_other(): """Test merging when max_budget_per_task is None in self but set in other.""" # Create metrics with no max_budget_per_task @@ -681,37 +505,6 @@ def test_metrics_merge_accumulated_token_usage_none_self(): assert metrics1.accumulated_token_usage.completion_tokens == 5 -def test_metrics_diff_current_usage_not_none(): - """Test diff method when current_usage is not None (lines 274-275).""" - # Create metrics with accumulated token usage - metrics1 = Metrics() - metrics1.add_token_usage( - prompt_tokens=20, - completion_tokens=10, - cache_read_tokens=0, - cache_write_tokens=0, - context_window=100, - response_id="test1", - ) - - # Create another metrics with different usage - metrics2 = Metrics() - metrics2.add_token_usage( - prompt_tokens=10, - completion_tokens=5, - cache_read_tokens=0, - cache_write_tokens=0, - context_window=100, - response_id="test2", - ) - - # Calculate diff - should handle current_usage not None (lines 274-275) - diff = metrics1.diff(metrics2) - assert diff.accumulated_token_usage is not None - assert diff.accumulated_token_usage.prompt_tokens == 10 - assert diff.accumulated_token_usage.completion_tokens == 5 - - def test_metrics_diff_both_usage_none(): """Test diff method when both accumulated_token_usage are None (lines 276-277).""" # Create metrics and manually set accumulated_token_usage to None @@ -725,50 +518,6 @@ def test_metrics_diff_both_usage_none(): assert diff.accumulated_token_usage is None -def test_cost_positive_validation(): - """Test Cost model with positive cost (line 17 - positive case).""" - # Should not raise error for positive cost - cost = Cost(model="test-model", cost=10.5) - assert cost.cost == 10.5 - assert cost.model == "test-model" - - -def test_metrics_accumulated_cost_positive_validation(): - """Test Metrics model with positive accumulated_cost (line 105 - positive case).""" - # Should not raise error for positive accumulated_cost - metrics = Metrics(accumulated_cost=15.0) - assert metrics.accumulated_cost == 15.0 - - -def test_metrics_add_token_usage_with_existing_accumulated(): - """Test add_token_usage when accumulated_token_usage already exists.""" - # Create metrics and add initial usage - metrics = Metrics() - metrics.add_token_usage( - prompt_tokens=10, - completion_tokens=5, - cache_read_tokens=0, - cache_write_tokens=0, - context_window=100, - response_id="test1", - ) - - # Add more usage - should trigger line 174 (else branch) - metrics.add_token_usage( - prompt_tokens=20, - completion_tokens=10, - cache_read_tokens=0, - cache_write_tokens=0, - context_window=100, - response_id="test2", - ) - - # Should have accumulated the usage - assert metrics.accumulated_token_usage is not None - assert metrics.accumulated_token_usage.prompt_tokens == 30 - assert metrics.accumulated_token_usage.completion_tokens == 15 - - def test_metrics_add_token_usage_none_accumulated_initial(): """Test add_token_usage when accumulated_token_usage is None initially.""" # Create metrics and manually set accumulated_token_usage to None @@ -791,21 +540,6 @@ def test_metrics_add_token_usage_none_accumulated_initial(): assert metrics.accumulated_token_usage.completion_tokens == 5 -def test_cost_validator_positive_path(): - """Test Cost validator positive path.""" - # Create Cost using Pydantic validation to trigger validator - cost = Cost(model="test-model", cost=5.0) - assert cost.cost == 5.0 - assert cost.model == "test-model" - - -def test_metrics_accumulated_cost_validator_positive_path(): - """Test Metrics accumulated_cost validator positive path.""" - # Create Metrics using Pydantic validation to trigger validator - metrics = Metrics(accumulated_cost=10.0) - assert metrics.accumulated_cost == 10.0 - - def test_metrics_diff_current_only_not_none(): """Test diff method when current has usage but baseline doesn't (line 275).""" # Create metrics with usage diff --git a/tests/sdk/tool/test_response_schema.py b/tests/sdk/tool/test_response_schema.py index d47462c983..44057ceb87 100644 --- a/tests/sdk/tool/test_response_schema.py +++ b/tests/sdk/tool/test_response_schema.py @@ -436,29 +436,6 @@ def test_mcp_tool_supports_response_schema(): assert action.structured_output["success"] is True -def test_response_schema_json_is_cached_per_class(): - """The Pydantic model_json_schema() result is cached by the immutable class so - repeated _response_schema_json calls reuse the cached schema.""" - from openhands.sdk.tool.tool import _response_schema_json_cache - - # Prime the cache by resolving a tool with a schema. - _finish_with_schema(TaskResult) - assert TaskResult in _response_schema_json_cache - - cached = _response_schema_json_cache[TaskResult] - # Subsequent calls return deep copies of the same cached schema. - again = _finish_with_schema(TaskResult) - assert _response_schema_json_cache[TaskResult] == cached - # Multiple action_from_arguments calls do not regenerate the cache. - again.action_from_arguments( - {"message": "m", "success": True, "summary_text": "s", "files_changed": []} - ) - again.action_from_arguments( - {"message": "m2", "success": False, "summary_text": "s2", "files_changed": []} - ) - assert _response_schema_json_cache[TaskResult] == cached - - def test_response_schema_cache_does_not_go_stale_on_model_copy(): """A tool rebuilt via model_copy (bypassing set_response_schema) with a different schema must not reuse a stale cache.""" diff --git a/tests/sdk/tool/test_schema_immutability.py b/tests/sdk/tool/test_schema_immutability.py index eff2aca134..801b68ae3f 100644 --- a/tests/sdk/tool/test_schema_immutability.py +++ b/tests/sdk/tool/test_schema_immutability.py @@ -1,311 +1,17 @@ -"""Tests for schema immutability in openhands.sdk.tool.schema.""" - -from collections.abc import Sequence -from typing import Any +"""Tests for the Schema immutability contract.""" import pytest from pydantic import Field, ValidationError -from openhands.sdk.llm import ImageContent, TextContent -from openhands.sdk.mcp.definition import MCPToolAction -from openhands.sdk.tool.schema import ( - Action, - Observation, - Schema, -) +from openhands.sdk.tool.schema import Schema class MockSchema(Schema): - """Mock schema class for testing.""" - - name: str = Field(description="Name field") - value: int = Field(description="Value field") - optional_field: str | None = Field(default=None, description="Optional field") - - -class SchemaImmutabilityMockAction(Action): - """Mock action class for testing.""" - - command: str = Field(description="Command to execute") - args: list[str] = Field(default_factory=list, description="Command arguments") - metadata: dict[str, Any] = Field(default_factory=dict, description="Metadata") - - -class MockMCPAction(MCPToolAction): - """Mock MCP action class for testing.""" - - operation: str = Field(description="Operation to perform") - parameters: dict[str, str] = Field( - default_factory=dict, description="Operation parameters" - ) - - -class SchemaImmutabilityMockObservation(Observation): - """Mock observation class for testing.""" - - result: str = Field(description="Result of the action") - status: str = Field(default="success", description="Status of the operation") - data: dict[str, Any | None] | None = Field(default=None, description="Result data") - - @property - def to_llm_content(self) -> Sequence[TextContent | ImageContent]: - """Get the observation string to show to the agent.""" - return [TextContent(text=f"Result: {self.result}, Status: {self.status}")] - + value: str = Field(description="Test value") -class _SchemaImmutabilityCustomAction(Action): - """Custom action for testing schema inheritance immutability. - This class is defined at module level (rather than inside a test function) to - ensure it's importable by Pydantic during serialization/deserialization. - Defining it inside a test function causes test pollution when running tests - in parallel with pytest-xdist. - """ - - custom_field: str = Field(description="Custom field") - - -class _SchemaImmutabilityCustomObservation(Observation): - """Custom observation for testing schema inheritance immutability. - - This class is defined at module level (rather than inside a test function) to - ensure it's importable by Pydantic during serialization/deserialization. - Defining it inside a test function causes test pollution when running tests - in parallel with pytest-xdist. - """ - - custom_result: str = Field(description="Custom result") - - @property - def to_llm_content(self) -> Sequence[TextContent | ImageContent]: - return [TextContent(text=self.custom_result)] - - -def test_schema_is_frozen(): - """Test that Schema instances are frozen and cannot be modified.""" - schema = MockSchema(name="test", value=42) - - # Test that we cannot modify any field - with pytest.raises(ValidationError, match="Instance is frozen"): - schema.name = "modified" - - with pytest.raises(ValidationError, match="Instance is frozen"): - schema.value = 100 - - with pytest.raises(ValidationError, match="Instance is frozen"): - schema.optional_field = "new_value" - - -def test_action_base_is_frozen(): - """Test that Action instances are frozen and cannot be modified.""" - action = SchemaImmutabilityMockAction(command="test_command", args=["arg1", "arg2"]) - - # Test that we cannot modify any field - with pytest.raises(ValidationError, match="Instance is frozen"): - action.command = "modified_command" - - with pytest.raises(ValidationError, match="Instance is frozen"): - action.args = ["new_arg"] - - with pytest.raises(ValidationError, match="Instance is frozen"): - action.metadata = {"new": "data"} - - -def test_mcp_action_base_is_frozen(): - """Test that MCPToolAction instances are frozen and cannot be modified.""" - action = MockMCPAction(operation="test_op", parameters={"key": "value"}) - - # Test that we cannot modify any field - with pytest.raises(ValidationError, match="Instance is frozen"): - action.operation = "modified_op" - - with pytest.raises(ValidationError, match="Instance is frozen"): - action.parameters = {"new": "params"} - - -def test_observation_base_is_frozen(): - """Test that Observation instances are frozen and cannot be modified.""" - observation = SchemaImmutabilityMockObservation( - result="test_result", status="completed" - ) - - # Test that we cannot modify any field - with pytest.raises(ValidationError, match="Instance is frozen"): - observation.result = "modified_result" - - with pytest.raises(ValidationError, match="Instance is frozen"): - observation.status = "failed" - - with pytest.raises(ValidationError, match="Instance is frozen"): - observation.data = {"new": "data"} - - -def test_schema_model_copy_creates_new_instance(): - """Test that model_copy creates a new instance with updated fields.""" - original = MockSchema(name="original", value=10) - - # Create a copy with updated fields - updated = original.model_copy(update={"name": "updated", "value": 20}) - - # Verify original is unchanged - assert original.name == "original" - assert original.value == 10 - - # Verify updated instance has new values - assert updated.name == "updated" - assert updated.value == 20 - - # Verify they are different instances - assert original is not updated - - -def test_action_model_copy_creates_new_instance(): - """Test that Action model_copy creates a new instance with updated fields.""" - original = SchemaImmutabilityMockAction(command="original_cmd", args=["arg1"]) - - # Create a copy with updated fields - updated = original.model_copy( - update={"command": "updated_cmd", "args": ["arg1", "arg2"]} - ) - - # Verify original is unchanged - assert original.command == "original_cmd" - assert original.args == ["arg1"] - - # Verify updated instance has new values - assert updated.command == "updated_cmd" - assert updated.args == ["arg1", "arg2"] - - # Verify they are different instances - assert original is not updated - - -def test_mcp_action_model_copy_creates_new_instance(): - """Test that MCPToolAction model_copy creates a new instance with updated fields.""" - original = MockMCPAction(operation="original_op", parameters={"key": "value"}) - - # Create a copy with updated fields - updated = original.model_copy( - update={"operation": "updated_op", "parameters": {"new_key": "new_value"}} - ) - - # Verify original is unchanged - assert original.operation == "original_op" - assert original.parameters == {"key": "value"} - - # Verify updated instance has new values - assert updated.operation == "updated_op" - assert updated.parameters == {"new_key": "new_value"} - - # Verify they are different instances - assert original is not updated - - -def test_observation_model_copy_creates_new_instance(): - """Test that Observation model_copy creates a new instance. - - Creates a new instance with updated fields. - """ - original = SchemaImmutabilityMockObservation( - result="original_result", status="pending" - ) - - # Create a copy with updated fields - updated = original.model_copy( - update={"result": "updated_result", "status": "completed"} - ) - - # Verify original is unchanged - assert original.result == "original_result" - assert original.status == "pending" - - # Verify updated instance has new values - assert updated.result == "updated_result" - assert updated.status == "completed" - - # Verify they are different instances - assert original is not updated - - -def test_schema_immutability_prevents_mutation_bugs(): - """Test a practical scenario where immutability prevents mutation bugs.""" - # Create an action that might be shared across multiple contexts - shared_action = SchemaImmutabilityMockAction( - command="shared_cmd", args=["shared_arg"] - ) - - # Simulate two different contexts trying to modify the action - def context_a_processing( - action: SchemaImmutabilityMockAction, - ) -> SchemaImmutabilityMockAction: - # Context A wants to reassign the args field - this should fail - with pytest.raises(ValidationError, match="Instance is frozen"): - action.args = action.args + ["context_a_arg"] - - # Context A should use model_copy instead - return action.model_copy(update={"args": action.args + ["context_a_arg"]}) - - def context_b_processing( - action: SchemaImmutabilityMockAction, - ) -> SchemaImmutabilityMockAction: - # Context B wants to change the command - this should fail - with pytest.raises(ValidationError, match="Instance is frozen"): - action.command = "context_b_cmd" - - # Context B should use model_copy instead - return action.model_copy(update={"command": "context_b_cmd"}) - - # Process the action in both contexts - action_a = context_a_processing(shared_action) - action_b = context_b_processing(shared_action) - - # Verify the original action is unchanged - assert shared_action.command == "shared_cmd" - assert shared_action.args == ["shared_arg"] - - # Verify each context got its own modified version - assert action_a.command == "shared_cmd" - assert action_a.args == ["shared_arg", "context_a_arg"] - - assert action_b.command == "context_b_cmd" - assert action_b.args == ["shared_arg"] - - # Verify all instances are different - assert shared_action is not action_a - assert shared_action is not action_b - assert action_a is not action_b - - -def test_all_schema_classes_are_frozen(): - """Test that all schema base classes are properly frozen.""" - # Test Schema - schema = MockSchema(name="test", value=1) - with pytest.raises(ValidationError, match="Instance is frozen"): - schema.name = "changed" - - # Test Action - action = SchemaImmutabilityMockAction(command="test") - with pytest.raises(ValidationError, match="Instance is frozen"): - action.command = "changed" - - # Test MCPToolAction - mcp_action = MockMCPAction(operation="test") - with pytest.raises(ValidationError, match="Instance is frozen"): - mcp_action.operation = "changed" - - # Test Observation - observation = SchemaImmutabilityMockObservation(result="test") - with pytest.raises(ValidationError, match="Instance is frozen"): - observation.result = "changed" - - -def test_schema_inheritance_preserves_immutability(): - """Test that classes inheriting from schema bases are also immutable.""" - # Test that custom classes are also frozen - custom_action = _SchemaImmutabilityCustomAction(custom_field="test") - with pytest.raises(ValidationError, match="Instance is frozen"): - custom_action.custom_field = "changed" +def test_schema_subclasses_are_frozen(): + schema = MockSchema(value="original") - custom_obs = _SchemaImmutabilityCustomObservation(custom_result="test") with pytest.raises(ValidationError, match="Instance is frozen"): - custom_obs.custom_result = "changed" + schema.value = "changed" diff --git a/tests/sdk/tool/test_tool.py b/tests/sdk/tool/test_tool.py index 97c7db918b..46d9df9a37 100644 --- a/tests/sdk/tool/test_tool.py +++ b/tests/sdk/tool/test_tool.py @@ -35,147 +35,10 @@ class _Bug2642ActionC(Action, ABC): tab_id: int = Field(description="tab id") -def test_tool_minimal(): - """Test creating Tool with minimal required fields.""" - tool = Tool(name="TestTool") +def test_tool_json_round_trip(): + tool = Tool(name="TestTool", params={"working_dir": "/test", "timeout": 45}) - assert tool.name == "TestTool" - assert tool.params == {} - - -def test_tool_with_params(): - """Test creating Tool with parameters.""" - params = {"working_dir": "/workspace", "timeout": 30} - tool = Tool(name="TestTool", params=params) - - assert tool.name == "TestTool" - assert tool.params == params - - -def test_tool_complex_params(): - """Test creating Tool with complex parameters.""" - params = { - "working_dir": "/workspace", - "env_vars": {"PATH": "/usr/bin", "HOME": "/home/user"}, - "timeout": 60, - "shell": "/bin/bash", - "debug": True, - } - - tool = Tool(name="TestTool", params=params) - - assert tool.name == "TestTool" - assert tool.params == params - assert tool.params["env_vars"]["PATH"] == "/usr/bin" - assert tool.params["debug"] is True - - -def test_tool_serialization(): - """Test Tool serialization and deserialization.""" - params = {"working_dir": "/test", "timeout": 45} - tool = Tool(name="TestTool", params=params) - - # Test model_dump - tool_dict = tool.model_dump() - assert tool_dict["name"] == "TestTool" - assert tool_dict["params"] == params - - # Test model_dump_json - tool_json = tool.model_dump_json() - assert isinstance(tool_json, str) - - # Test deserialization - tool_restored = Tool.model_validate_json(tool_json) - assert tool_restored.name == "TestTool" - assert tool_restored.params == params - - -def test_tool_validation_requires_name(): - """Test that Tool requires a name.""" - with pytest.raises(ValidationError): - Tool() # type: ignore - - -def test_tool_examples_from_docstring(): - """Test the examples provided in Tool docstring.""" - # Test the examples from the docstring - examples = ["TestTool", "AnotherTool", "TaskTrackerTool"] - - for example_name in examples: - spec = Tool(name=example_name) - assert spec.name == example_name - assert spec.params == {} - - # Test with params example - spec_with_params = Tool(name="TestTool", params={"custom_param": "/workspace"}) - assert spec_with_params.name == "TestTool" - assert spec_with_params.params == {"custom_param": "/workspace"} - - -def test_tool_different_tool_types(): - """Test creating Tool for different tool types.""" - # TestTool - test_tool = Tool( - name="TestTool", params={"custom_dir": "/workspace", "timeout": 30} - ) - assert test_tool.name == "TestTool" - assert test_tool.params["custom_dir"] == "/workspace" - - # AnotherTool - another_tool = Tool(name="AnotherTool") - assert another_tool.name == "AnotherTool" - assert another_tool.params == {} - - # TaskTrackerTool - tracker_tool = Tool( - name="TaskTrackerTool", params={"save_dir": "/workspace/.openhands"} - ) - assert tracker_tool.name == "TaskTrackerTool" - assert tracker_tool.params["save_dir"] == "/workspace/.openhands" - - -def test_tool_nested_params(): - """Test Tool with nested parameter structures.""" - params = { - "config": { - "timeout": 30, - "retries": 3, - "options": {"verbose": True, "debug": False}, - }, - "paths": ["/usr/bin", "/usr/local/bin"], - "env": {"LANG": "en_US.UTF-8"}, - } - - tool = Tool(name="ComplexTool", params=params) - - assert tool.name == "ComplexTool" - assert tool.params["config"]["timeout"] == 30 - assert tool.params["config"]["options"]["verbose"] is True - assert tool.params["paths"] == ["/usr/bin", "/usr/local/bin"] - assert tool.params["env"]["LANG"] == "en_US.UTF-8" - - -def test_tool_field_descriptions(): - """Test that Tool fields have proper descriptions.""" - fields = Tool.model_fields - - assert "name" in fields - assert fields["name"].description is not None - assert "Name of the tool class" in fields["name"].description - assert ( - "Import it from an `openhands.tools.` subpackage." - in fields["name"].description - ) - - assert "params" in fields - assert fields["params"].description is not None - assert "Parameters for the tool's .create() method" in fields["params"].description - - -def test_tool_default_params(): - """Test that Tool has correct default for params.""" - tool = Tool(name="TestTool") - assert tool.params == {} + assert Tool.model_validate_json(tool.model_dump_json()) == tool def test_tool_immutability(): @@ -188,25 +51,10 @@ def test_tool_immutability(): assert tool.params["test_param"] == "/workspace" -def test_tool_validation_edge_cases(): - """Test Tool validation with edge cases.""" - # Empty string name should be invalid +def test_tool_rejects_empty_name(): with pytest.raises(ValidationError): Tool(name="") - # None params should use default empty dict (handled by validator) - tool = Tool(name="TestTool") - assert tool.params == {} - - -def test_tool_repr(): - """Test Tool string representation.""" - tool = Tool(name="TerminalTool", params={"test_param": "/test"}) - repr_str = repr(tool) - - assert "Tool" in repr_str - assert "TerminalTool" in repr_str - def test_issue_2199_1(request): """Reproduce issue #2199: duplicate dynamic Action wrapper classes. diff --git a/tests/tools/browser_use/test_chromium_detection.py b/tests/tools/browser_use/test_chromium_detection.py index 6c2dbe7a5d..bffe71aa62 100644 --- a/tests/tools/browser_use/test_chromium_detection.py +++ b/tests/tools/browser_use/test_chromium_detection.py @@ -1,12 +1,11 @@ -"""Tests for Chromium detection and installation functionality.""" +"""Tests for Chromium detection and availability errors.""" -import subprocess from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import patch import pytest -from openhands.tools.browser_use.impl import BrowserToolExecutor, _install_chromium +from openhands.tools.browser_use.impl import BrowserToolExecutor @pytest.fixture(autouse=True) @@ -226,80 +225,6 @@ def test_check_chromium_available_not_found(self): result = executor.check_chromium_available() assert result is None - def test_check_chromium_available_playwright_cache_not_found(self): - """Test when Playwright cache directory doesn't exist.""" - executor = BrowserToolExecutor.__new__(BrowserToolExecutor) - with ( - patch("openhands.tools.browser_use.impl.sys.platform", "linux"), - patch("shutil.which", return_value=None), - patch("pathlib.Path.home", return_value=Path("/home/user")), - patch.object(Path, "exists", return_value=False), - ): - result = executor.check_chromium_available() - assert result is None - - -class TestChromiumInstallation: - """Test Chromium installation functionality.""" - - def test_install_chromium_success(self): - """Test successful Chromium installation.""" - mock_result = MagicMock() - mock_result.returncode = 0 - - with ( - patch("shutil.which", return_value="/usr/bin/uvx"), - patch("subprocess.run", return_value=mock_result), - ): - result = _install_chromium() - assert result is True - - def test_install_chromium_uvx_not_found(self): - """Test Chromium installation when uvx is not available.""" - with patch("shutil.which", return_value=None): - result = _install_chromium() - assert result is False - - def test_install_chromium_subprocess_failure(self): - """Test Chromium installation when subprocess fails.""" - mock_result = MagicMock() - mock_result.returncode = 1 - mock_result.stderr = "Installation failed" - - with ( - patch("shutil.which", return_value="/usr/bin/uvx"), - patch("subprocess.run", return_value=mock_result), - ): - result = _install_chromium() - assert result is False - - def test_install_chromium_timeout(self): - """Test Chromium installation timeout.""" - with ( - patch("shutil.which", return_value="/usr/bin/uvx"), - patch("subprocess.run", side_effect=subprocess.TimeoutExpired("uvx", 300)), - ): - result = _install_chromium() - assert result is False - - def test_install_chromium_file_not_found(self): - """Test Chromium installation when uvx command is not found.""" - with ( - patch("shutil.which", return_value="/usr/bin/uvx"), - patch("subprocess.run", side_effect=FileNotFoundError("uvx not found")), - ): - result = _install_chromium() - assert result is False - - def test_install_chromium_generic_exception(self): - """Test Chromium installation with generic exception.""" - with ( - patch("shutil.which", return_value="/usr/bin/uvx"), - patch("subprocess.run", side_effect=Exception("Generic error")), - ): - result = _install_chromium() - assert result is False - class TestEnsureChromiumAvailable: """Test ensure Chromium available functionality.""" diff --git a/tests/tools/file_editor/utils/test_encoding.py b/tests/tools/file_editor/utils/test_encoding.py index 5cd46f6d79..045c74b131 100644 --- a/tests/tools/file_editor/utils/test_encoding.py +++ b/tests/tools/file_editor/utils/test_encoding.py @@ -7,7 +7,6 @@ from unittest.mock import patch import pytest -from cachetools import LRUCache from openhands.tools.file_editor import file_editor from openhands.tools.file_editor.editor import FileEditor @@ -35,14 +34,6 @@ def encoding_manager(): return EncodingManager() -def test_init(encoding_manager): - """Test initialization of EncodingManager.""" - assert isinstance(encoding_manager, EncodingManager) - assert isinstance(encoding_manager._encoding_cache, LRUCache) - assert encoding_manager.default_encoding == "utf-8" - assert encoding_manager.confidence_threshold == 0.9 - - def test_detect_encoding_nonexistent_file(encoding_manager): """Test detecting encoding for a nonexistent file.""" nonexistent_path = Path("/nonexistent/file.txt") diff --git a/tests/tools/file_editor/utils/test_file_cache.py b/tests/tools/file_editor/utils/test_file_cache.py index 9cfcabcf51..787d76ba98 100644 --- a/tests/tools/file_editor/utils/test_file_cache.py +++ b/tests/tools/file_editor/utils/test_file_cache.py @@ -15,12 +15,6 @@ def file_cache(): cache.clear() -def test_init(file_cache): - assert isinstance(file_cache, FileCache) - assert file_cache.directory.exists() - assert file_cache.directory.is_dir() - - def test_set_and_get(file_cache): file_cache.set("test_key", "test_value") assert file_cache.get("test_key") == "test_value" @@ -31,11 +25,6 @@ def test_get_nonexistent_key(file_cache): assert file_cache.get("nonexistent_key", "default") == "default" -def test_set_nested_key(file_cache): - file_cache.set("folder/nested/key", "nested_value") - assert file_cache.get("folder/nested/key") == "nested_value" - - def test_set_overwrite(file_cache): file_cache.set("test_key", "initial_value") file_cache.set("test_key", "new_value") @@ -52,23 +41,6 @@ def test_delete_nonexistent_key(file_cache): file_cache.delete("nonexistent_key") # Should not raise an exception -def test_delete_nested_key(file_cache): - file_cache.set("folder/nested/key", "nested_value") - file_cache.delete("folder/nested/key") - assert file_cache.get("folder/nested/key") is None - - -def test_clear(file_cache): - file_cache.set("key1", "value1") - file_cache.set("key2", "value2") - file_cache.set("folder/key3", "value3") - file_cache.clear() - assert len(file_cache) == 0 - assert file_cache.get("key1") is None - assert file_cache.get("key2") is None - assert file_cache.get("folder/key3") is None - - def test_contains(file_cache): file_cache.set("test_key", "test_value") assert "test_key" in file_cache @@ -149,9 +121,6 @@ def test_size_limit(): cache.set("key1", val1) cache.set("key2", val2) - assert len(val1.encode("utf-8")) <= 100 - assert len(val1.encode("utf-8") + val2.encode("utf-8")) > 100 - val3 = "z" * 40 # This should cause key1 to be evicted cache.set("key3", val3) # 40 bytes diff --git a/tests/tools/test_tool_name_consistency.py b/tests/tools/test_tool_name_consistency.py deleted file mode 100644 index 98607fd891..0000000000 --- a/tests/tools/test_tool_name_consistency.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Test that tool_name class variables are consistent with automatic naming.""" - -from openhands.tools.browser_use import BrowserToolSet -from openhands.tools.file_editor import FileEditorTool -from openhands.tools.glob import GlobTool -from openhands.tools.grep import GrepTool -from openhands.tools.planning_file_editor import PlanningFileEditorTool -from openhands.tools.task_tracker import TaskTrackerTool -from openhands.tools.terminal import TerminalTool - - -def test_tool_name_attributes_exist(): - """Test that all tool classes have name class variables.""" - tools = [ - TerminalTool, - FileEditorTool, - TaskTrackerTool, - BrowserToolSet, - GrepTool, - GlobTool, - PlanningFileEditorTool, - ] - - for tool_class in tools: - assert hasattr(tool_class, "name"), ( - f"{tool_class.__name__} missing name attribute" - ) - assert isinstance(tool_class.name, str), ( - f"{tool_class.__name__}.name is not a string" - ) - # name should be snake_case version of class name - assert tool_class.name.islower(), ( - f"{tool_class.__name__}.name should be snake_case" - ) - # Allow single words without underscores (e.g., "terminal", "grep") - assert "_" in tool_class.name or len(tool_class.name) <= 10, ( - f"{tool_class.__name__}.name should contain underscores for " - "multi-word names or be a short single word" - ) - - -def test_tool_name_consistency(): - """Test that name matches the expected snake_case conversion.""" - expected_names = { - TerminalTool: "terminal", - FileEditorTool: "file_editor", - TaskTrackerTool: "task_tracker", - BrowserToolSet: "browser_tool_set", - GrepTool: "grep", - GlobTool: "glob", - PlanningFileEditorTool: "planning_file_editor", - } - - for tool_class, expected_name in expected_names.items(): - assert tool_class.name == expected_name, ( - f"{tool_class.__name__}.name should be '{expected_name}'" - ) - - -def test_tool_name_accessible_at_class_level(): - """Test that name can be accessed at the class level without instantiation.""" - # This should not raise any errors and should return snake_case names - assert TerminalTool.name == "terminal" - assert FileEditorTool.name == "file_editor" - assert TaskTrackerTool.name == "task_tracker" - assert BrowserToolSet.name == "browser_tool_set" - assert GrepTool.name == "grep" - assert GlobTool.name == "glob" - assert PlanningFileEditorTool.name == "planning_file_editor" From bb26768cc501e63d6e9733990a184e0612d55c20 Mon Sep 17 00:00:00 2001 From: george larson Date: Wed, 19 Aug 2026 10:02:32 -0400 Subject: [PATCH 06/14] test(sdk): pin events_to_messages boundaries + fix responses_reasoning_item batch drop (#4526) Co-authored-by: enyst Co-authored-by: openhands --- .pr/gpt-5-nano-integration-results.md | 74 ++++++++++ openhands-sdk/openhands/sdk/event/base.py | 10 +- tests/sdk/event/test_events_to_messages.py | 159 ++++++++++++++++++++- 3 files changed, 237 insertions(+), 6 deletions(-) create mode 100644 .pr/gpt-5-nano-integration-results.md diff --git a/.pr/gpt-5-nano-integration-results.md b/.pr/gpt-5-nano-integration-results.md new file mode 100644 index 0000000000..3cda307865 --- /dev/null +++ b/.pr/gpt-5-nano-integration-results.md @@ -0,0 +1,74 @@ +# GPT-5-nano integration results + +Tested PR head `cf0fd2e11e3e26ded2ed1eb31fc9178c567d6ba6` on 2026-08-18 with `litellm_proxy/openai/gpt-5-nano`, `reasoning_effort=high`, and `https://llm-proxy.eval.all-hands.dev`. + +No `b*` behavior tests were run. + +## Result summary + +| Test | Result | +| --- | --- | +| Focused unit suite, `tests/sdk/event/test_events_to_messages.py` | 23 passed | +| New reasoning-item regression against `origin/main` | Failed as expected: the combined message had `responses_reasoning_item=None` | +| `t*` integration suite | 8 passed, 1 failed | +| `c*` condenser suite | 2 passed, 2 failed, 1 skipped | +| Isolated rerun of `c02` and `c05` | `c05` passed; `c02` failed again | +| Purpose-built parallel-tool replay probe | Passed | + +## Integration suite details + +### `t*` + +Passed: `t01`, `t02`, `t03`, `t04`, `t06`, `t07`, `t08`, and `t09`. + +`t05_simple_browsing` failed twice, including an isolated retry. Chromium launched and the agent navigated the test site, but GPT-5-nano stopped after saying it would fetch the answer instead of reporting it. This is model behavior and does not exercise action batching. + +### `c*` + +- `c01_thinking_block_condenser`: skipped as designed because GPT-5-nano produces Responses API reasoning items rather than Anthropic thinking blocks. +- `c03_delayed_condensation`: passed with five condensations. +- `c04_token_condenser`: passed. +- `c05_size_condenser`: failed initially because the model emitted only one tool call and stopped before enough events existed; it passed on an isolated rerun, confirming model-dependent flakiness. +- `c02_hard_context_reset`: failed twice because GPT-5-nano answered calculation requests directly instead of creating enough tool-loop events for the second condensation to become a normal condensation. + +The `c02` and initial `c05` failures did not produce parallel sibling `ActionEvent`s, so they did not exercise the code changed by this PR. + +## Direct parallel-tool replay validation + +A first live attempt asked GPT-5-nano to issue two calls to the same terminal tool in parallel. The model instead emitted them in two separate LLM responses, confirming that a generic integration task does not reliably cover this regression. + +A second probe exposed two distinct independent tools, `get_alpha` and `get_beta`, and required both before a final response. GPT-5-nano then produced: + +1. two `ActionEvent`s with the same `llm_response_id`; +2. a Responses API reasoning item only on the first action; +3. a recombined assistant message containing both tool calls in order; +4. a reasoning item exactly equal to the first action's item; and +5. a successful follow-up Responses API turn ending with `alpha beta verified`. + +This exercises the PR's changed path end to end: the reasoning item is retained on the recombined message and accepted when the tool-call batch is sent back to GPT-5-nano. + +## Commands + +```bash +uv run --frozen pytest tests/sdk/event/test_events_to_messages.py + +LLM_API_KEY=... \ +LLM_BASE_URL=https://llm-proxy.eval.all-hands.dev \ +IN_DOCKER=true \ +uv run --frozen python tests/integration/run_infer.py \ + --llm-config '{"model":"litellm_proxy/openai/gpt-5-nano","reasoning_effort":"high"}' \ + --num-workers 4 \ + --test-type integration + +LLM_API_KEY=... \ +LLM_BASE_URL=https://llm-proxy.eval.all-hands.dev \ +IN_DOCKER=true \ +uv run --frozen python tests/integration/run_infer.py \ + --llm-config '{"model":"litellm_proxy/openai/gpt-5-nano","reasoning_effort":"high"}' \ + --num-workers 4 \ + --test-type condenser +``` + +## Assessment + +The focused regression and the live parallel-tool replay both validate the fix. The remaining integration failures are explained by GPT-5-nano task compliance and did not execute the changed parallel-action reconstruction path. diff --git a/openhands-sdk/openhands/sdk/event/base.py b/openhands-sdk/openhands/sdk/event/base.py index 0c3502e7e8..7c464d661d 100644 --- a/openhands-sdk/openhands/sdk/event/base.py +++ b/openhands-sdk/openhands/sdk/event/base.py @@ -106,8 +106,12 @@ def __str__(self) -> str: @staticmethod def events_to_messages(events: list["LLMConvertibleEvent"]) -> list[Message]: - """Convert event stream to LLM message stream, handling multi-action batches""" - # TODO: We should add extensive tests for this + """Convert event stream to LLM message stream, handling multi-action batches. + + This is a read-only projection over the event log: events are + immutable once created and appended, so merges build new content + lists rather than mutating the events' own messages. + """ from openhands.sdk.event.llm_convertible import ActionEvent messages = [] @@ -190,4 +194,6 @@ def _combine_action_events(events: list["ActionEvent"]) -> Message: tool_calls=[event.tool_call for event in events], reasoning_content=events[0].reasoning_content, # Shared reasoning content thinking_blocks=events[0].thinking_blocks, # Shared thinking blocks + # Shared responses reasoning item + responses_reasoning_item=events[0].responses_reasoning_item, ) diff --git a/tests/sdk/event/test_events_to_messages.py b/tests/sdk/event/test_events_to_messages.py index 148c144e89..441f8c9072 100644 --- a/tests/sdk/event/test_events_to_messages.py +++ b/tests/sdk/event/test_events_to_messages.py @@ -7,6 +7,7 @@ import pytest from openhands.sdk.event.base import LLMConvertibleEvent +from openhands.sdk.event.condenser import CondensationSummaryEvent from openhands.sdk.event.llm_convertible import ( ActionEvent, AgentErrorEvent, @@ -18,7 +19,9 @@ ImageContent, Message, MessageToolCall, + ReasoningItemModel, TextContent, + ThinkingBlock, ) from openhands.sdk.tool import Action, Observation @@ -137,6 +140,85 @@ def test_consecutive_user_message_events_are_merged(self): "Relevant project context", ] + def test_user_message_merge_cascades_across_three_events(self): + """A merged message stays a plain user turn, so the merge cascades.""" + first = MessageEvent( + source="user", + llm_message=Message(role="user", content=[TextContent(text="one")]), + ) + second = MessageEvent( + source="user", + llm_message=Message(role="user", content=[TextContent(text="two")]), + ) + third = MessageEvent( + source="user", + llm_message=Message(role="user", content=[TextContent(text="three")]), + ) + + events = cast(list[LLMConvertibleEvent], [first, second, third]) + messages = LLMConvertibleEvent.events_to_messages(events) + + assert [message.role for message in messages] == ["user"] + assert [ + content.text + for content in messages[0].content + if isinstance(content, TextContent) + ] == ["one", "two", "three"] + + def test_user_message_merge_preserves_mixed_content_order(self): + """Merged content concatenates in log order, text and images alike.""" + text_first = MessageEvent( + source="user", + llm_message=Message( + role="user", content=[TextContent(text="what is in this image?")] + ), + ) + image_second = MessageEvent( + source="user", + llm_message=Message( + role="user", + content=[ImageContent(image_urls=["http://example.com/a.png"])], + ), + ) + + events = cast(list[LLMConvertibleEvent], [text_first, image_second]) + messages = LLMConvertibleEvent.events_to_messages(events) + + assert len(messages) == 1 + assert [type(content) for content in messages[0].content] == [ + TextContent, + ImageContent, + ] + + def test_condensation_summary_merges_with_adjacent_user_message(self): + """A summary event converts to a plain user message, so it coalesces. + + The summarizing condenser inserts the summary where the forgotten + block began (summary_offset == forgetting_start), and the kept suffix + can legally begin with a user message — so this adjacency is + reachable, and the fusion erases the summary's structural boundary. + """ + summary_event = CondensationSummaryEvent( + summary="Earlier we refactored the module." + ) + user_after = MessageEvent( + source="user", + llm_message=Message( + role="user", content=[TextContent(text="Now continue.")] + ), + ) + + events = cast(list[LLMConvertibleEvent], [summary_event, user_after]) + messages = LLMConvertibleEvent.events_to_messages(events) + + assert len(messages) == 1 + assert messages[0].role == "user" + assert [ + content.text + for content in messages[0].content + if isinstance(content, TextContent) + ] == ["Earlier we refactored the module.", "Now continue."] + def test_user_messages_with_tool_call_id_not_merged(self): """Tool-result user messages must not be coalesced.""" result = MessageEvent( @@ -320,15 +402,84 @@ def test_parallel_function_calling_same_response_id(self): assert len(tool_calls) == 3 # Verify tool call details - tool_call_ids = [tc.id for tc in tool_calls] - assert "call_SF" in tool_call_ids - assert "call_Tokyo" in tool_call_ids - assert "call_Paris" in tool_call_ids + # Exact sequence, not membership: order within the batch is log order. + assert [tc.id for tc in tool_calls] == ["call_SF", "call_Tokyo", "call_Paris"] # All should be weather function calls for tool_call in tool_calls: assert tool_call.name == "get_current_weather" + def test_parallel_batch_preserves_first_event_reasoning(self): + """The combined message keeps the first event's reasoning state. + + ThinkingBlock's own docstring requires these blocks be preserved and + passed back to the API for tool use scenarios — so a batched message + must carry them exactly like a singleton does. + """ + first = ActionEvent( + source="agent", + thought=[TextContent(text="batching two calls")], + reasoning_content="step-by-step reasoning", + thinking_blocks=[ThinkingBlock(thinking="let me think", signature="sig-1")], + action=EventsToMessagesMockAction(command="one"), + tool_name="terminal", + tool_call_id="call_1", + tool_call=create_tool_call("call_1", "terminal", {"command": "one"}), + llm_response_id="response_reasoning", + ) + second = ActionEvent( + source="agent", + thought=[], + action=EventsToMessagesMockAction(command="two"), + tool_name="terminal", + tool_call_id="call_2", + tool_call=create_tool_call("call_2", "terminal", {"command": "two"}), + llm_response_id="response_reasoning", + ) + + events = [first, second] + messages = LLMConvertibleEvent.events_to_messages(events) # type: ignore + + assert len(messages) == 1 + combined = messages[0] + assert combined.reasoning_content == "step-by-step reasoning" + assert len(combined.thinking_blocks) == 1 + assert combined.thinking_blocks[0].thinking == "let me think" # type: ignore + + def test_parallel_batch_preserves_responses_reasoning_item(self): + """A singleton keeps its responses reasoning item; a batch must too.""" + first = ActionEvent( + source="agent", + thought=[TextContent(text="batching two calls")], + responses_reasoning_item=ReasoningItemModel( + id="rs_1", summary=["reasoned about the batch"] + ), + action=EventsToMessagesMockAction(command="one"), + tool_name="terminal", + tool_call_id="call_1", + tool_call=create_tool_call("call_1", "terminal", {"command": "one"}), + llm_response_id="response_reasoning_item", + ) + second = ActionEvent( + source="agent", + thought=[], + action=EventsToMessagesMockAction(command="two"), + tool_name="terminal", + tool_call_id="call_2", + tool_call=create_tool_call("call_2", "terminal", {"command": "two"}), + llm_response_id="response_reasoning_item", + ) + + events = [first, second] + messages = LLMConvertibleEvent.events_to_messages(events) # type: ignore + + assert len(messages) == 1 + combined = messages[0] + # Whole-item equality, not just the id — future field loss fails here. + assert combined.responses_reasoning_item == ReasoningItemModel( + id="rs_1", summary=["reasoned about the batch"] + ) + def test_multiple_separate_action_events(self): """Test multiple ActionEvents with different response_ids (separate calls).""" action1 = create_action_event( From 73fabfd76491940fcb1a042289a18ad618ec89d7 Mon Sep 17 00:00:00 2001 From: Juan Pedro Michelini Jorge Date: Wed, 19 Aug 2026 14:21:46 -0300 Subject: [PATCH 07/14] Add read-at-use LLM provider connections (#4492) Co-authored-by: openhands --- .../agent_server/_secrets_exposure.py | 10 + .../openhands/agent_server/api.py | 4 + .../agent_server/persistence/__init__.py | 10 + .../agent_server/persistence/store.py | 33 +- .../openhands/agent_server/profiles_router.py | 40 +- .../provider_connections_router.py | 280 ++++++++++ .../openhands/agent_server/settings_router.py | 3 + openhands-sdk/openhands/sdk/llm/llm.py | 10 + .../openhands/sdk/llm/llm_profile_store.py | 145 ++++- .../sdk/llm/provider_connection_store.py | 292 +++++++++++ tests/agent_server/test_profiles_router.py | 496 +++++++++++++++++- tests/sdk/llm/test_llm_profile_store.py | 48 ++ .../sdk/llm/test_provider_connection_store.py | 253 +++++++++ 13 files changed, 1610 insertions(+), 14 deletions(-) create mode 100644 openhands-agent-server/openhands/agent_server/provider_connections_router.py create mode 100644 openhands-sdk/openhands/sdk/llm/provider_connection_store.py create mode 100644 tests/sdk/llm/test_provider_connection_store.py diff --git a/openhands-agent-server/openhands/agent_server/_secrets_exposure.py b/openhands-agent-server/openhands/agent_server/_secrets_exposure.py index c3f2f79458..779fc6d3fc 100644 --- a/openhands-agent-server/openhands/agent_server/_secrets_exposure.py +++ b/openhands-agent-server/openhands/agent_server/_secrets_exposure.py @@ -10,6 +10,7 @@ from openhands.sdk.llm import LLM from openhands.sdk.llm.llm import LLM_SECRET_FIELDS +from openhands.sdk.llm.provider_connection_store import ProviderConnectionNotFound from openhands.sdk.utils.cipher import FERNET_TOKEN_PREFIX, Cipher from openhands.sdk.utils.pydantic_secrets import MissingCipherError @@ -134,6 +135,15 @@ def store_errors() -> Iterator[None]: status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Profile store is busy. Please retry.", ) + except ProviderConnectionNotFound as e: + # Subclass of ValueError, so it must be handled before the generic + # ValueError arm below. A dangling provider reference is a resolvable + # config problem (recreate the connection or edit the profile), hence + # 422 rather than a generic 400. + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=str(e), + ) except ValueError as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, diff --git a/openhands-agent-server/openhands/agent_server/api.py b/openhands-agent-server/openhands/agent_server/api.py index 541edf0829..03871848db 100644 --- a/openhands-agent-server/openhands/agent_server/api.py +++ b/openhands-agent-server/openhands/agent_server/api.py @@ -57,6 +57,9 @@ ) from openhands.agent_server.plugins_router import plugins_router from openhands.agent_server.profiles_router import profiles_router +from openhands.agent_server.provider_connections_router import ( + provider_connections_router, +) from openhands.agent_server.server_details_router import ( get_server_info, mark_initialization_complete, @@ -440,6 +443,7 @@ def _add_api_routes(app: FastAPI) -> None: api_router.include_router(plugins_router) api_router.include_router(hooks_router) api_router.include_router(llm_router) + api_router.include_router(provider_connections_router) api_router.include_router(mcp_router) api_router.include_router(settings_router) api_router.include_router(workspaces_router) diff --git a/openhands-agent-server/openhands/agent_server/persistence/__init__.py b/openhands-agent-server/openhands/agent_server/persistence/__init__.py index b41360a259..3123be11d7 100644 --- a/openhands-agent-server/openhands/agent_server/persistence/__init__.py +++ b/openhands-agent-server/openhands/agent_server/persistence/__init__.py @@ -27,11 +27,17 @@ WorkspacesStore, get_agent_profile_store, get_llm_profile_store, + get_provider_connections_store, get_secrets_store, get_settings_store, get_workspaces_store, reset_stores, ) +from openhands.sdk.llm.provider_connection_store import ( + PersistedProviderConnections, + ProviderConnection, + ProviderConnectionStore, +) __all__ = [ @@ -41,8 +47,10 @@ "WORKSPACES_SCHEMA_VERSION", # Models "CustomSecret", + "PersistedProviderConnections", "PersistedSettings", "PersistedWorkspaces", + "ProviderConnection", "Secrets", "SettingsUpdatePayload", "WorkspaceItem", @@ -51,11 +59,13 @@ "FileSecretsStore", "FileSettingsStore", "FileWorkspacesStore", + "ProviderConnectionStore", "SecretsStore", "SettingsStore", "WorkspacesStore", "get_agent_profile_store", "get_llm_profile_store", + "get_provider_connections_store", "get_secrets_store", "get_settings_store", "get_workspaces_store", diff --git a/openhands-agent-server/openhands/agent_server/persistence/store.py b/openhands-agent-server/openhands/agent_server/persistence/store.py index d661a0cd87..c798569a8e 100644 --- a/openhands-agent-server/openhands/agent_server/persistence/store.py +++ b/openhands-agent-server/openhands/agent_server/persistence/store.py @@ -29,6 +29,7 @@ Secrets, ) from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.llm.provider_connection_store import ProviderConnectionStore from openhands.sdk.logger import get_logger from openhands.sdk.profiles.agent_profile_store import AgentProfileStore from openhands.sdk.utils.cipher import Cipher @@ -790,6 +791,7 @@ def update( _settings_store: FileSettingsStore | None = None _secrets_store: FileSecretsStore | None = None +_provider_connections_store: ProviderConnectionStore | None = None _workspaces_store: FileWorkspacesStore | None = None _llm_profile_store: LLMProfileStore | None = None _agent_profile_store: AgentProfileStore | None = None @@ -895,6 +897,27 @@ def get_secrets_store(config: Config | None = None) -> FileSecretsStore: return _secrets_store +def get_provider_connections_store( + config: Config | None = None, # noqa: ARG001 +) -> ProviderConnectionStore: + """Get the global provider connections store instance (thread-safe). + + Stored at ``/provider-connections`` alongside ``profiles`` / + ``agent-profiles``. The store itself is cipher-agnostic; callers pass the + request cipher per call so keys are encrypted at rest. + """ + global _provider_connections_store + if _provider_connections_store is not None: + return _provider_connections_store + + with _store_lock: + if _provider_connections_store is None: + _provider_connections_store = ProviderConnectionStore( + base_dir=_get_profile_persistence_dir() / "provider-connections", + ) + return _provider_connections_store + + def get_workspaces_store(config: Config | None = None) -> FileWorkspacesStore: """Get the global workspaces store instance (thread-safe). @@ -927,10 +950,15 @@ def get_llm_profile_store() -> LLMProfileStore: if _llm_profile_store is not None: return _llm_profile_store + # Resolve the provider store first: it takes ``_store_lock`` itself, and + # ``_store_lock`` is non-reentrant, so calling it while already holding the + # lock below would deadlock. + provider_store = get_provider_connections_store() with _store_lock: if _llm_profile_store is None: _llm_profile_store = LLMProfileStore( base_dir=_get_profile_persistence_dir() / "profiles", + provider_store=provider_store, ) return _llm_profile_store @@ -956,11 +984,12 @@ def get_agent_profile_store() -> AgentProfileStore: def reset_stores() -> None: """Reset global store instances (for testing).""" - global _settings_store, _secrets_store, _workspaces_store - global _llm_profile_store, _agent_profile_store + global _settings_store, _secrets_store, _provider_connections_store + global _workspaces_store, _llm_profile_store, _agent_profile_store with _store_lock: _settings_store = None _secrets_store = None + _provider_connections_store = None _workspaces_store = None _llm_profile_store = None _agent_profile_store = None diff --git a/openhands-agent-server/openhands/agent_server/profiles_router.py b/openhands-agent-server/openhands/agent_server/profiles_router.py index 74e85a4ec2..eb08fa7c3a 100644 --- a/openhands-agent-server/openhands/agent_server/profiles_router.py +++ b/openhands-agent-server/openhands/agent_server/profiles_router.py @@ -18,6 +18,7 @@ PersistedSettings, get_agent_profile_store, get_llm_profile_store, + get_provider_connections_store, get_settings_store, ) from openhands.sdk.llm import LLM, Message, TextContent @@ -56,6 +57,8 @@ class ProfileInfo(BaseModel): name: str model: str | None = None base_url: str | None = None + provider_connection_id: str | None = None + provider_connection_broken: bool = False api_key_set: bool = False @@ -100,6 +103,28 @@ def _has_api_key(llm: LLM) -> bool: return bool(llm.api_key.get_secret_value().strip()) +def _profile_api_key_set(request: Request, llm: LLM) -> bool: + """Effective key presence: the profile's own key, or its provider's. + + A profile linked to a provider connection carries no inline key (cleared on + save), so its key presence lives on the connection. + """ + if _has_api_key(llm): + return True + connection_id = llm.provider_connection_id + if not connection_id: + return False + config = get_config(request) + cipher = get_cipher(request) + # The provider store read can raise on a corrupted file; map it instead of + # letting it surface as an unhandled 500 on GET /profiles/{name}. + with store_errors(): + connection = get_provider_connections_store(config).get( + connection_id, cipher=cipher + ) + return connection is not None and connection.api_key_value() is not None + + def _set_active_profile_if_matches( request: Request, old_name: str, new_name: str | None ) -> bool: @@ -154,7 +179,10 @@ async def get_profile(request: Request, name: ProfileName) -> ProfileDetailRespo store = get_llm_profile_store() try: with store_errors(): - llm = store.load(name, cipher=cipher) + # Display the profile exactly as stored: don't inject the linked + # provider's credentials, and don't fail a read when the reference + # dangles. Effective key presence is reported via ``api_key_set``. + llm = store.load(name, cipher=cipher, resolve_provider=False) except FileNotFoundError: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -170,7 +198,7 @@ async def get_profile(request: Request, name: ProfileName) -> ProfileDetailRespo config["api_key"] = None return ProfileDetailResponse( - name=name, config=config, api_key_set=_has_api_key(llm) + name=name, config=config, api_key_set=_profile_api_key_set(request, llm) ) @@ -417,8 +445,13 @@ async def activate_profile( cipher = get_cipher(request) config = get_config(request) - # Load the profile + # Load the profile. ``load`` resolves any referenced provider connection + # (read-at-use), so the LLM applied to settings already carries the shared + # api_key / base_url; ``provider_connection_id`` is retained so a later + # rotation re-resolves on the next activation or launch. profile_store = get_llm_profile_store() + # A dangling provider_connection_id raises ProviderConnectionNotFound, which + # store_errors() maps to 422. try: with store_errors(): llm = profile_store.load(name, cipher=cipher) @@ -428,7 +461,6 @@ async def activate_profile( detail=f"Profile '{name}' not found", ) - # Apply the LLM config to settings and record active profile settings_store = get_settings_store(config) def apply_profile(settings: PersistedSettings) -> PersistedSettings: diff --git a/openhands-agent-server/openhands/agent_server/provider_connections_router.py b/openhands-agent-server/openhands/agent_server/provider_connections_router.py new file mode 100644 index 0000000000..9f86fe6f22 --- /dev/null +++ b/openhands-agent-server/openhands/agent_server/provider_connections_router.py @@ -0,0 +1,280 @@ +"""Provider connection endpoints for sharing LLM credentials across profiles. + +A provider connection is a shared ``api_key`` + optional ``base_url`` that one +or more LLM profiles reference by id. The credential is resolved into a runnable +:class:`~openhands.sdk.llm.llm.LLM` lazily, at profile-load time +(:meth:`LLMProfileStore.load`) — this router only performs CRUD over the stored +connections. Because resolution is read-at-use, rotating a key here takes effect +the next time a linked profile is activated or launched; nothing is copied into +active settings, so there is no separate refresh path to keep in sync. +""" + +from __future__ import annotations + +import time +import uuid +from typing import Any + +from fastapi import APIRouter, HTTPException, Request, status +from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator + +from openhands.agent_server._secrets_exposure import ( + get_cipher, + get_config, + store_errors, +) +from openhands.agent_server.persistence import ( + ProviderConnection, + get_llm_profile_store, + get_provider_connections_store, + get_settings_store, +) +from openhands.sdk.llm.provider_connection_store import ( + ProviderConnectionLimitExceeded, + ProviderConnectionNotFound, +) +from openhands.sdk.logger import get_logger + + +logger = get_logger(__name__) + +provider_connections_router = APIRouter( + prefix="/llm/provider-connections", tags=["LLM Provider Connections"] +) + + +def _now() -> int: + return int(time.time()) + + +class ProviderConnectionCreateRequest(BaseModel): + display_name: str = Field(..., min_length=1, max_length=128) + provider: str = Field(default="custom", min_length=1, max_length=128) + api_key: SecretStr = Field(..., min_length=1) + base_url: str | None = Field(default=None, max_length=2048) + + model_config = ConfigDict(extra="forbid") + + +class ProviderConnectionUpdateRequest(BaseModel): + display_name: str | None = Field(default=None, min_length=1, max_length=128) + provider: str | None = Field(default=None, min_length=1, max_length=128) + api_key: SecretStr | None = None + base_url: str | None = Field(default=None, max_length=2048) + + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="after") + def _reject_null_required_fields(self) -> ProviderConnectionUpdateRequest: + # Only `base_url` may be set to null (to clear it). `display_name` and + # `provider` are required on the stored model; accepting explicit null on + # PATCH would persist a null that poisons every subsequent store read. + for field in ("display_name", "provider"): + if field in self.model_fields_set and getattr(self, field) is None: + raise ValueError(f"{field} cannot be set to null") + return self + + +class ProviderConnectionResponse(BaseModel): + id: str + display_name: str + provider: str + base_url: str | None = None + created_at: int + updated_at: int + api_key_set: bool = False + + +def _to_response(connection: ProviderConnection) -> ProviderConnectionResponse: + return ProviderConnectionResponse( + id=connection.id, + display_name=connection.display_name, + provider=connection.provider, + base_url=connection.base_url, + created_at=connection.created_at, + updated_at=connection.updated_at, + api_key_set=connection.api_key_value() is not None, + ) + + +def _linked_profile_names(connection_id: str) -> list[str]: + return sorted( + str(summary["name"]) + for summary in get_llm_profile_store().list_summaries() + if summary.get("provider_connection_id") == connection_id + ) + + +def _active_settings_references_connection(config, connection_id: str) -> bool: + settings = get_settings_store(config).load() + if settings is None: + return False + return settings.agent_settings.llm.provider_connection_id == connection_id + + +def _raise_if_connection_is_referenced(config, connection_id: str) -> None: + """Block deletion while any profile or the active settings still point here. + + Deleting a referenced connection would leave those references dangling; a + linked profile with no inline key raises on its next load. So the delete is + rejected until the references are removed first. + """ + profile_names = _linked_profile_names(connection_id) + active_reference = _active_settings_references_connection(config, connection_id) + if not profile_names and not active_reference: + return + + reasons = [] + if profile_names: + reasons.append(f"referenced by LLM profile(s): {', '.join(profile_names)}") + if active_reference: + reasons.append("referenced by the active agent settings") + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "Provider connection cannot be deleted while it is " + + " and ".join(reasons) + + ". Update those references before deleting it." + ), + ) + + +@provider_connections_router.get("", response_model=list[ProviderConnectionResponse]) +async def list_provider_connections( + request: Request, +) -> list[ProviderConnectionResponse]: + cipher = get_cipher(request) + store = get_provider_connections_store(get_config(request)) + with store_errors(): + connections = store.list(cipher=cipher) + return [_to_response(c) for c in connections] + + +@provider_connections_router.post( + "", response_model=ProviderConnectionResponse, status_code=status.HTTP_201_CREATED +) +async def create_provider_connection( + request: Request, body: ProviderConnectionCreateRequest +) -> ProviderConnectionResponse: + cipher = get_cipher(request) + store = get_provider_connections_store(get_config(request)) + now = _now() + connection = ProviderConnection( + id=uuid.uuid4().hex, + display_name=body.display_name, + provider=body.provider, + api_key=body.api_key, + base_url=body.base_url, + created_at=now, + updated_at=now, + ) + try: + with store_errors(): + store.create(connection, cipher=cipher) + except ProviderConnectionLimitExceeded as e: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"{e} Delete one before adding another.", + ) + logger.info( + "Created LLM provider connection", extra={"connection_id": connection.id} + ) + return _to_response(connection) + + +@provider_connections_router.patch( + "/{connection_id}", response_model=ProviderConnectionResponse +) +async def update_provider_connection( + request: Request, connection_id: str, body: ProviderConnectionUpdateRequest +) -> ProviderConnectionResponse: + fields = body.model_fields_set + if not fields: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="Provide at least one provider connection field to update", + ) + + cipher = get_cipher(request) + store = get_provider_connections_store(get_config(request)) + with store_errors(): + connection = store.get(connection_id, cipher=cipher) + if connection is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Provider connection '{connection_id}' not found", + ) + + # A connection must always have a key, so clearing it is not a valid + # update. Reject api_key: null explicitly instead of silently dropping it. + if "api_key" in fields and body.api_key is None: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="api_key cannot be cleared; provide a new key to rotate it", + ) + + updates: dict[str, Any] = {"updated_at": _now()} + for field in ("display_name", "provider", "base_url"): + if field in fields: + updates[field] = getattr(body, field) + if "api_key" in fields: + updates["api_key"] = body.api_key + updated = connection.model_copy(update=updates) + + # store_errors() maps infra failures (lock timeout -> 503, corrupted/ + # wrong-cipher file -> 4xx); the inner except keeps the deleted-between- + # get-and-update race as a 404 rather than the 422 store_errors would give + # ProviderConnectionNotFound. + with store_errors(): + try: + store.update(updated, cipher=cipher) + except ProviderConnectionNotFound: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Provider connection '{connection_id}' not found", + ) + logger.info( + "Updated LLM provider connection", extra={"connection_id": connection_id} + ) + return _to_response(updated) + + +@provider_connections_router.delete( + "/{connection_id}", response_model=ProviderConnectionResponse +) +async def delete_provider_connection( + request: Request, connection_id: str +) -> ProviderConnectionResponse: + config = get_config(request) + cipher = get_cipher(request) + store = get_provider_connections_store(config) + with store_errors(): + connection = store.get(connection_id, cipher=cipher) + if connection is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Provider connection '{connection_id}' not found", + ) + _raise_if_connection_is_referenced(config, connection_id) + + # See update handler: keep the delete-race as 404, map infra errors. + with store_errors(): + try: + store.delete(connection_id, cipher=cipher) + except ProviderConnectionNotFound: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Provider connection '{connection_id}' not found", + ) + logger.info( + "Deleted LLM provider connection", extra={"connection_id": connection_id} + ) + return ProviderConnectionResponse( + id=connection.id, + display_name=connection.display_name, + provider=connection.provider, + base_url=connection.base_url, + created_at=connection.created_at, + updated_at=connection.updated_at, + api_key_set=False, + ) diff --git a/openhands-agent-server/openhands/agent_server/settings_router.py b/openhands-agent-server/openhands/agent_server/settings_router.py index 6024b1b254..a00dcd6a4c 100644 --- a/openhands-agent-server/openhands/agent_server/settings_router.py +++ b/openhands-agent-server/openhands/agent_server/settings_router.py @@ -246,6 +246,9 @@ def _resolve_active_profile_llm( cipher = get_cipher(request) profile_store = get_llm_profile_store() + # ``load`` resolves any referenced provider connection (read-at-use); a + # dangling reference raises ProviderConnectionNotFound, which + # store_errors() maps to 422. try: with store_errors(): llm = profile_store.load(profile_name, cipher=cipher) diff --git a/openhands-sdk/openhands/sdk/llm/llm.py b/openhands-sdk/openhands/sdk/llm/llm.py index dd33c971cf..2dd49d7164 100644 --- a/openhands-sdk/openhands/sdk/llm/llm.py +++ b/openhands-sdk/openhands/sdk/llm/llm.py @@ -262,6 +262,16 @@ class LLM(BaseModel, RetryMixin, NonNativeToolCallingMixin): label="API Key", ), ) + provider_connection_id: str | None = Field( + default=None, + description=( + "Optional provider connection whose shared API key and base URL " + "are resolved and applied each time this LLM profile is loaded " + "(read-at-use). When set, the profile stores no inline api_key or " + "base_url of its own." + ), + json_schema_extra=field_meta(SettingProminence.MAJOR), + ) auth_type: Literal["api_key", "subscription"] = Field( default="api_key", description="Authentication mode for the LLM.", diff --git a/openhands-sdk/openhands/sdk/llm/llm_profile_store.py b/openhands-sdk/openhands/sdk/llm/llm_profile_store.py index ed9a1ce505..5001e4b9c4 100644 --- a/openhands-sdk/openhands/sdk/llm/llm_profile_store.py +++ b/openhands-sdk/openhands/sdk/llm/llm_profile_store.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: from openhands.sdk.llm.llm import LLM + from openhands.sdk.llm.provider_connection_store import ProviderConnectionStore from openhands.sdk.utils.cipher import Cipher _DEFAULT_PROFILE_DIR: Final[Path] = Path.home() / ".openhands" / "profiles" @@ -39,6 +40,17 @@ class ProfileLimitExceeded(Exception): """Raised when saving would exceed the configured profile limit.""" +def _api_key_present(llm: LLM) -> bool: + """True when ``llm`` carries a non-empty, non-redacted API key.""" + from pydantic import SecretStr + + api_key = llm.api_key + if api_key is None: + return False + value = api_key.get_secret_value() if isinstance(api_key, SecretStr) else api_key + return bool(value.strip()) and value != REDACTED_SECRET_VALUE + + @runtime_checkable class LLMProfileLoader(Protocol): """Minimal load-only contract consumed by ``resolve_agent_profile``. @@ -69,18 +81,49 @@ def rename(self, old_name: str, new_name: str) -> None: ... class LLMProfileStore: """Standalone utility for persisting LLM configurations.""" - def __init__(self, base_dir: Path | str | None = None) -> None: + def __init__( + self, + base_dir: Path | str | None = None, + *, + provider_store: ProviderConnectionStore | None = None, + ) -> None: """Initialize the profile store. Args: base_dir: Path to the directory where the profiles are stored. If `None` is provided, the default directory is used, i.e., `~/.openhands/profiles`. + provider_store: Store of shared provider connections used to + resolve a profile's ``provider_connection_id`` at load time. + When `None` (the default), a :class:`ProviderConnectionStore` + is created in a ``provider-connections`` directory *sibling to* + ``base_dir`` so that every ``LLMProfileStore`` — including + custom-directory and bare standalone SDK instances — resolves + connections from the same location it reads profiles from. Pass + an explicit store to use an unrelated directory (e.g. the + agent-server's config-scoped directory) or a test double. """ self.base_dir = Path(base_dir) if base_dir is not None else _DEFAULT_PROFILE_DIR # ensure directory existence self.base_dir.mkdir(parents=True, exist_ok=True) self._file_lock = FileLock(self.base_dir / ".profiles.lock") + if provider_store is None: + from openhands.sdk.llm.provider_connection_store import ( + ProviderConnectionStore, + ) + + # Derive the connections directory from base_dir rather than $HOME, + # so a custom-directory profile store reads its linked credentials + # from the same location it reads profiles from. For the default + # ~/.openhands/profiles this resolves to ~/.openhands/provider- + # connections, matching ProviderConnectionStore's own default. + self._provider_store: ProviderConnectionStore | None = ( + ProviderConnectionStore( + base_dir=self.base_dir.parent / "provider-connections" + ) + ) + else: + self._provider_store = provider_store @contextmanager def _acquire_lock(self, timeout: float = _LOCK_TIMEOUT_SECONDS) -> Iterator[None]: @@ -177,6 +220,13 @@ def save( f"[Profile Store] Profile `{name}` already exists. Overwriting." ) + # Rule 5c: a profile that references a provider connection owns no + # inline credentials — the connection is the single source of truth, + # so clear any api_key / base_url before persisting to avoid a stale + # copy that could later disagree with the connection. + if llm.provider_connection_id: + llm = llm.model_copy(update={"api_key": None, "base_url": None}) + context: dict[str, Any] = {} if include_secrets: if cipher: @@ -199,20 +249,34 @@ def save( raise logger.info(f"[Profile Store] Saved profile `{name}` at {profile_path}") - def load(self, name: str, *, cipher: Cipher | None = None) -> LLM: + def load( + self, + name: str, + *, + cipher: Cipher | None = None, + resolve_provider: bool = True, + ) -> LLM: """Load an LLM instance from the given profile name. Args: name: Name of the profile to load. cipher: Optional cipher for decrypting secrets stored at rest. When provided, encrypted secrets are decrypted during load. + resolve_provider: When True (default) and the profile references a + provider connection, its shared ``api_key`` / ``base_url`` are + applied to the returned LLM (read-at-use). Set False to inspect + the profile as stored without touching the provider store — used + by display paths so a dangling reference never fails a read. Returns: - An LLM instance constructed from the profile configuration. + An LLM instance constructed from the profile configuration, with + provider-connection credentials applied when ``resolve_provider``. Raises: FileNotFoundError: If the profile name does not exist. ValueError: If the profile file is corrupted or invalid. + ProviderConnectionNotFound: If the profile references a provider + connection that no longer exists and carries no inline key. TimeoutError: If the lock cannot be acquired. """ profile_path = self._get_profile_path(name) @@ -236,7 +300,63 @@ def load(self, name: str, *, cipher: Cipher | None = None) -> LLM: raise ValueError(f"Failed to load profile `{name}`: {e}") from e logger.info(f"[Profile Store] Loaded profile `{name}` from {profile_path}") - return llm_instance + + if resolve_provider: + llm_instance = self._resolve_provider_connection( + name, llm_instance, cipher=cipher + ) + return llm_instance + + def _resolve_provider_connection( + self, profile_name: str, llm: LLM, *, cipher: Cipher | None + ) -> LLM: + """Apply a referenced provider connection's credentials to ``llm``. + + Rules: + - no ``provider_connection_id`` -> unchanged (byte-identical old path). + - no provider store configured -> unchanged (inert field). + - connection found -> its ``api_key`` / ``base_url`` win (``base_url`` + is applied as-is, including ``None``). + - connection missing -> raise :class:`ProviderConnectionNotFound`. + + The inline-key fallback below is not a recovery path for the usual + "linked profile, connection later deleted" case: :meth:`save` strips + inline creds from any linked profile, so on disk there is no inline key + to fall back to and this raises. It only applies to an LLM whose + ``provider_connection_id`` was set without going through :meth:`save` + (e.g. constructed in memory). + """ + connection_id = llm.provider_connection_id + if not connection_id or self._provider_store is None: + return llm + + from openhands.sdk.llm.provider_connection_store import ( + ProviderConnectionNotFound, + ) + + connection = self._provider_store.get(connection_id, cipher=cipher) + if connection is None: + if _api_key_present(llm): + logger.warning( + "[Profile Store] Profile %r references missing provider " + "connection %r; falling back to the profile's inline key.", + profile_name, + connection_id, + ) + return llm + raise ProviderConnectionNotFound( + f"Profile {profile_name!r} references provider connection " + f"{connection_id!r}, which does not exist. Update the profile or " + "recreate the connection." + ) + + updates: dict[str, Any] = {"base_url": connection.base_url} + api_key = connection.api_key_value() + if api_key is not None: + from pydantic import SecretStr + + updates["api_key"] = SecretStr(api_key) + return llm.model_copy(update=updates) def delete(self, name: str) -> None: """Delete an existing profile. @@ -285,6 +405,10 @@ def list_summaries(self) -> list[dict[str, Any]]: Reads JSON directly to avoid ``LLM._set_env_side_effects`` mutating ``os.environ``. Files with invalid names, corrupted JSON, or non-dict top-level values are skipped with a warning. + + A linked provider connection's key presence is read without a cipher: + this method only reports whether a key is set, never its plaintext, so + an encrypted-at-rest key still reads as present without decryption. """ summaries: list[dict[str, Any]] = [] with self._acquire_lock(): @@ -314,11 +438,24 @@ def list_summaries(self) -> list[dict[str, Any]]: and bool(api_key.strip()) and api_key != REDACTED_SECRET_VALUE ) + connection_id = data.get("provider_connection_id") + # A profile linked to a provider connection carries no inline + # key (cleared on save), so its effective key presence lives on + # the connection. + provider_connection_broken = False + if isinstance(connection_id, str) and self._provider_store is not None: + connection = self._provider_store.get(connection_id) + if connection is None: + provider_connection_broken = True + elif not api_key_set: + api_key_set = connection.api_key_value() is not None summaries.append( { "name": name, "model": data.get("model"), "base_url": data.get("base_url"), + "provider_connection_id": connection_id, + "provider_connection_broken": provider_connection_broken, "api_key_set": api_key_set, } ) diff --git a/openhands-sdk/openhands/sdk/llm/provider_connection_store.py b/openhands-sdk/openhands/sdk/llm/provider_connection_store.py new file mode 100644 index 0000000000..32536844dc --- /dev/null +++ b/openhands-sdk/openhands/sdk/llm/provider_connection_store.py @@ -0,0 +1,292 @@ +"""Persistence for shared LLM provider connections. + +A *provider connection* is a small, named bundle of the credential material an +LLM profile would otherwise carry inline: an ``api_key`` and an optional +``base_url``. Several LLM profiles can reference one connection by id, so +rotating the shared key in one place updates every profile that points at it. + +This lives in the SDK — next to :class:`LLMProfileStore` — on purpose. The +profile store resolves a profile's ``provider_connection_id`` into concrete +credentials at load time (see :meth:`LLMProfileStore.load`), and it can only do +that if the connection store is reachable from the same layer. Keeping it here +(rather than in the agent-server) means every path that turns a stored profile +into a runnable :class:`~openhands.sdk.llm.llm.LLM` — named-profile activation +*and* the default/seed launch path — resolves identically. + +The credential is encrypted at rest with the same cipher machinery LLM profiles +use, so the connection file never holds a plaintext key when ``OH_SECRET_KEY`` +is configured. +""" + +from __future__ import annotations + +import json +import re +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING, Any, Final + +from filelock import FileLock, Timeout +from pydantic import ( + BaseModel, + ConfigDict, + Field, + SecretStr, + field_serializer, + field_validator, +) + +from openhands.sdk.logger import get_logger +from openhands.sdk.utils.pydantic_secrets import is_redacted_secret, serialize_secret + + +if TYPE_CHECKING: + from openhands.sdk.utils.cipher import Cipher + + +_DEFAULT_DIR: Final[Path] = Path.home() / ".openhands" / "provider-connections" +_FILENAME: Final[str] = "provider_connections.json" +_LOCK_TIMEOUT_SECONDS: Final[float] = 30.0 + +PROVIDER_CONNECTIONS_SCHEMA_VERSION: Final[int] = 1 +MAX_PROVIDER_CONNECTIONS: Final[int] = 64 + +# Connection ids: 1-128 chars, alphanumeric start, then alphanumeric/._-. +# Same shape as profile names — blocks path separators and leading dots. +CONNECTION_ID_PATTERN: Final[str] = r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" +CONNECTION_ID_REGEX: Final[re.Pattern[str]] = re.compile(CONNECTION_ID_PATTERN) + +logger = get_logger(__name__) + + +class ProviderConnectionLimitExceeded(Exception): + """Raised when creating a connection would exceed the configured limit.""" + + +class ProviderConnectionNotFound(ValueError): + """A referenced provider connection id does not exist. + + Raised by :meth:`LLMProfileStore.load` when a profile points at a connection + that has been deleted and the profile has no usable inline key to fall back + on. + + Subclasses :class:`ValueError` so every ``load()`` caller degrades sensibly + even without a dedicated ``except``. The profile-activation and settings + endpoints (via ``store_errors``) map it to 422, and the OpenAI-compatible + gateway's ``except ValueError`` turns it into a 400 instead of an opaque + 500. The agent-profile launch path already funnels load failures through + its resolver's ``ValueError`` handling; that path can't actually hit a + dangling reference, since deleting a referenced connection is blocked while + any profile points at it. + """ + + +class ProviderConnection(BaseModel): + """A shared credential bundle reused by one or more LLM profiles.""" + + id: str = Field(..., min_length=1, max_length=128) + display_name: str = Field(..., min_length=1, max_length=128) + provider: str = Field(default="custom", min_length=1, max_length=128) + api_key: SecretStr | None = None + base_url: str | None = Field(default=None, max_length=2048) + created_at: int = Field(..., description="Unix epoch seconds.") + updated_at: int = Field(..., description="Unix epoch seconds.") + + model_config = ConfigDict(populate_by_name=True) + + @field_validator("api_key", mode="before") + @classmethod + def _validate_api_key(cls, v: str | SecretStr | None, info) -> SecretStr | None: + # Provider connections use a strict decryption path: a Fernet-encrypted key + # that cannot be decrypted raises instead of silently becoming None. A + # wrong-cipher read→modify→write would otherwise rewrite the collection + # with api_key=null, permanently destroying the stored ciphertext. + from openhands.sdk.utils.cipher import FERNET_TOKEN_PREFIX + + if v is None: + return None + secret_value = v.get_secret_value() if isinstance(v, SecretStr) else v + if ( + not secret_value + or not secret_value.strip() + or is_redacted_secret(secret_value) + ): + return None + cipher = (info.context or {}).get("cipher") + if cipher is not None and secret_value.startswith(FERNET_TOKEN_PREFIX): + decrypted = cipher.decrypt(secret_value) + if decrypted is None: + raise ValueError( + "api_key is encrypted but cannot be decrypted with the current " + "cipher. Verify that OH_SECRET_KEY matches the key used when " + "this connection was saved." + ) + return decrypted + return v if isinstance(v, SecretStr) else SecretStr(secret_value) + + @field_serializer("api_key", when_used="always") + def _serialize_api_key(self, v: SecretStr | None, info): + return serialize_secret(v, info) + + def api_key_value(self) -> str | None: + """Return the plaintext key, or ``None`` when unset/empty.""" + if self.api_key is None: + return None + value = self.api_key.get_secret_value() + return value if value.strip() else None + + +class PersistedProviderConnections(BaseModel): + """Container for the saved provider connections file.""" + + schema_version: int = Field(default=PROVIDER_CONNECTIONS_SCHEMA_VERSION) + connections: list[ProviderConnection] = Field(default_factory=list) + + model_config = ConfigDict(populate_by_name=True) + + +class ProviderConnectionStore: + """File-backed store for shared LLM provider connections. + + All connections live in a single JSON file guarded by a file lock, so + concurrent create/update/delete calls in-process are serialized. The + ``api_key`` of each connection is encrypted at rest when a cipher is + supplied, matching :class:`LLMProfileStore`'s secret handling. + """ + + def __init__(self, base_dir: Path | str | None = None) -> None: + self.base_dir = Path(base_dir) if base_dir is not None else _DEFAULT_DIR + self.base_dir.mkdir(parents=True, exist_ok=True) + self._path = self.base_dir / _FILENAME + self._file_lock = FileLock(self.base_dir / ".provider-connections.lock") + + @contextmanager + def _acquire_lock(self, timeout: float = _LOCK_TIMEOUT_SECONDS) -> Iterator[None]: + try: + with self._file_lock.acquire(timeout=timeout): + yield + except Timeout: + logger.error( + f"[Provider Connections] Failed to acquire lock within {timeout}s" + ) + raise TimeoutError( + f"Provider connection store lock acquisition timed out after {timeout}s" + ) + + def _read(self, *, cipher: Cipher | None) -> PersistedProviderConnections: + """Read the file without locking. Missing file -> empty container. + + A corrupted file raises ``ValueError`` rather than being silently + replaced, so a bad key or truncated write never destroys stored + credentials. + """ + if not self._path.exists(): + return PersistedProviderConnections() + try: + raw = json.loads(self._path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + raise ValueError(f"Provider connections file is unreadable: {e}") from e + if not isinstance(raw, dict): + raise ValueError("Provider connections file must contain a JSON object") + version = raw.get("schema_version", PROVIDER_CONNECTIONS_SCHEMA_VERSION) + if not isinstance(version, int): + raise ValueError("schema_version must be an integer") + if version > PROVIDER_CONNECTIONS_SCHEMA_VERSION: + raise ValueError( + f"schema_version {version} is newer than supported " + f"{PROVIDER_CONNECTIONS_SCHEMA_VERSION}" + ) + raw["schema_version"] = PROVIDER_CONNECTIONS_SCHEMA_VERSION + context = {"cipher": cipher} if cipher else None + return PersistedProviderConnections.model_validate(raw, context=context) + + def _write( + self, persisted: PersistedProviderConnections, *, cipher: Cipher | None + ) -> None: + context: dict[str, Any] = {} + if cipher is not None: + context["cipher"] = cipher + context["expose_secrets"] = "encrypted" + else: + context["expose_secrets"] = True + data = persisted.model_dump(mode="json", context=context) + payload = json.dumps(data, indent=2) + with tempfile.NamedTemporaryFile( + mode="w", dir=self.base_dir, suffix=".tmp", delete=False + ) as tmp: + tmp.write(payload) + tmp_path = Path(tmp.name) + try: + Path.replace(tmp_path, self._path) + except Exception: + tmp_path.unlink(missing_ok=True) + raise + + def list(self, *, cipher: Cipher | None = None) -> list[ProviderConnection]: + with self._acquire_lock(): + return list(self._read(cipher=cipher).connections) + + def get( + self, connection_id: str, *, cipher: Cipher | None = None + ) -> ProviderConnection | None: + with self._acquire_lock(): + for connection in self._read(cipher=cipher).connections: + if connection.id == connection_id: + return connection + return None + + def create( + self, connection: ProviderConnection, *, cipher: Cipher | None = None + ) -> ProviderConnection: + if not CONNECTION_ID_REGEX.match(connection.id): + raise ValueError(f"Invalid provider connection id: {connection.id!r}") + with self._acquire_lock(): + persisted = self._read(cipher=cipher) + if any(c.id == connection.id for c in persisted.connections): + raise ValueError( + f"Provider connection {connection.id!r} already exists" + ) + if len(persisted.connections) >= MAX_PROVIDER_CONNECTIONS: + raise ProviderConnectionLimitExceeded( + f"Provider connection limit reached ({MAX_PROVIDER_CONNECTIONS})." + ) + persisted.connections.append(connection) + self._write(persisted, cipher=cipher) + logger.info( + "[Provider Connections] Created connection", + extra={"connection_id": connection.id}, + ) + return connection + + def update( + self, connection: ProviderConnection, *, cipher: Cipher | None = None + ) -> ProviderConnection: + with self._acquire_lock(): + persisted = self._read(cipher=cipher) + if not any(c.id == connection.id for c in persisted.connections): + raise ProviderConnectionNotFound(connection.id) + persisted.connections = [ + connection if c.id == connection.id else c + for c in persisted.connections + ] + self._write(persisted, cipher=cipher) + logger.info( + "[Provider Connections] Updated connection", + extra={"connection_id": connection.id}, + ) + return connection + + def delete(self, connection_id: str, *, cipher: Cipher | None = None) -> None: + with self._acquire_lock(): + persisted = self._read(cipher=cipher) + remaining = [c for c in persisted.connections if c.id != connection_id] + if len(remaining) == len(persisted.connections): + raise ProviderConnectionNotFound(connection_id) + persisted.connections = remaining + self._write(persisted, cipher=cipher) + logger.info( + "[Provider Connections] Deleted connection", + extra={"connection_id": connection_id}, + ) diff --git a/tests/agent_server/test_profiles_router.py b/tests/agent_server/test_profiles_router.py index 28ce663200..64a8948054 100644 --- a/tests/agent_server/test_profiles_router.py +++ b/tests/agent_server/test_profiles_router.py @@ -14,6 +14,7 @@ from openhands.agent_server.persistence import reset_stores from openhands.sdk.llm import LLM from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.llm.provider_connection_store import ProviderConnectionStore from openhands.sdk.profiles import AgentProfileStore, OpenHandsAgentProfile @@ -57,17 +58,46 @@ def client(temp_profiles_dir, temp_agent_profiles_dir, temp_settings_dir, monkey config = Config(static_files_path=None, session_api_keys=[], secret_key=None) app = create_app(config) - # Patch both stores to use temp directories (AgentProfileStore is hit by the - # FK guard on delete/rename). + # Patch stores to use temp directories (AgentProfileStore is hit by the + # FK guard on delete/rename). The LLM profile store is wired to a shared + # provider-connection store so ``load`` resolves provider references, and + # the router's own ``get_provider_connections_store`` returns the same + # instance so CRUD and resolution see one file. + provider_store = ProviderConnectionStore( + base_dir=temp_profiles_dir.parent / "provider-connections" + ) + + def make_llm_store(): + return LLMProfileStore( + base_dir=temp_profiles_dir, provider_store=provider_store + ) + with ( patch( "openhands.agent_server.profiles_router.get_llm_profile_store", - lambda: LLMProfileStore(base_dir=temp_profiles_dir), + make_llm_store, ), patch( "openhands.agent_server.profiles_router.get_agent_profile_store", lambda: AgentProfileStore(base_dir=temp_agent_profiles_dir), ), + patch( + "openhands.agent_server.profiles_router.get_provider_connections_store", + lambda config=None: provider_store, + ), + patch( + "openhands.agent_server.settings_router.get_llm_profile_store", + make_llm_store, + ), + patch( + "openhands.agent_server.provider_connections_router.get_llm_profile_store", + make_llm_store, + ), + patch( + "openhands.agent_server.provider_connections_router." + "get_provider_connections_store", + lambda config=None: provider_store, + ), ): yield TestClient(app) @@ -170,6 +200,308 @@ def test_list_profiles_returns_saved_profiles(client, store): # ── Get Profile ──────────────────────────────────────────────────────────── +def test_provider_connection_key_shared_by_linked_profiles(client): + """Profiles stay runnable, while provider fields are shared by reference.""" + connection = client.post( + "/api/llm/provider-connections", + json={ + "display_name": "Anthropic Work", + "provider": "anthropic", + "api_key": "sk-ant-old", + "base_url": "https://api.anthropic.com", + }, + ) + assert connection.status_code == 201 + connection_body = connection.json() + connection_id = connection_body["id"] + assert connection_body["api_key_set"] is True + assert "api_key" not in connection_body + assert "secret_name" not in connection_body + + for name, model in { + "sonnet-4": "anthropic/claude-sonnet-4", + "sonnet-5": "anthropic/claude-sonnet-5", + }.items(): + response = client.post( + f"/api/profiles/{name}", + json={ + "llm": { + "model": model, + "provider_connection_id": connection_id, + }, + "include_secrets": False, + }, + ) + assert response.status_code == 201 + + profiles = client.get("/api/profiles").json()["profiles"] + linked = {p["name"]: p for p in profiles} + assert linked["sonnet-4"]["provider_connection_id"] == connection_id + assert linked["sonnet-4"]["api_key_set"] is True + assert linked["sonnet-5"]["api_key_set"] is True + + detail = client.get("/api/profiles/sonnet-4").json() + assert detail["config"]["api_key"] is None + assert detail["api_key_set"] is True + + activated = client.post("/api/profiles/sonnet-4/activate") + assert activated.status_code == 200 + settings = client.get( + "/api/settings", headers={"X-Expose-Secrets": "plaintext"} + ).json() + llm = settings["agent_settings"]["llm"] + assert llm["model"] == "anthropic/claude-sonnet-4" + assert llm["api_key"] == "sk-ant-old" + assert llm["base_url"] == "https://api.anthropic.com" + + delete = client.delete(f"/api/llm/provider-connections/{connection_id}") + assert delete.status_code == 409 + assert "referenced by LLM profile" in delete.json()["detail"] + + # Rotate the shared key. Read-at-use: active settings keep the previously + # resolved key until the profile is activated again (nothing is auto-copied). + rotated = client.patch( + f"/api/llm/provider-connections/{connection_id}", + json={"api_key": "sk-ant-new"}, + ) + assert rotated.status_code == 200 + settings = client.get( + "/api/settings", headers={"X-Expose-Secrets": "plaintext"} + ).json() + assert settings["agent_settings"]["llm"]["api_key"] == "sk-ant-old" + + # Re-activating re-resolves the connection and applies the rotated key. + activated = client.post("/api/profiles/sonnet-4/activate") + assert activated.status_code == 200 + settings = client.get( + "/api/settings", headers={"X-Expose-Secrets": "plaintext"} + ).json() + assert settings["agent_settings"]["llm"]["api_key"] == "sk-ant-new" + + activated = client.post("/api/profiles/sonnet-5/activate") + assert activated.status_code == 200 + settings = client.get( + "/api/settings", headers={"X-Expose-Secrets": "plaintext"} + ).json() + llm = settings["agent_settings"]["llm"] + assert llm["model"] == "anthropic/claude-sonnet-5" + assert llm["api_key"] == "sk-ant-new" + + +def test_settings_active_profile_resolves_provider_connection(client): + connection_id = client.post( + "/api/llm/provider-connections", + json={ + "display_name": "OpenAI", + "provider": "openai", + "api_key": "sk-openai-provider", + }, + ).json()["id"] + client.post( + "/api/profiles/gpt-provider", + json={ + "llm": { + "model": "openai/gpt-5.5", + "provider_connection_id": connection_id, + }, + "include_secrets": False, + }, + ) + + response = client.patch("/api/settings", json={"active_profile": "gpt-provider"}) + + assert response.status_code == 200 + settings = client.get( + "/api/settings", headers={"X-Expose-Secrets": "plaintext"} + ).json() + llm = settings["agent_settings"]["llm"] + assert llm["model"] == "openai/gpt-5.5" + assert llm["api_key"] == "sk-openai-provider" + + +def test_provider_connection_delete_rejects_active_settings_reference(client): + connection_id = client.post( + "/api/llm/provider-connections", + json={ + "display_name": "Anthropic Work", + "provider": "anthropic", + "api_key": "sk-ant-old", + }, + ).json()["id"] + client.post( + "/api/profiles/temporary-provider-profile", + json={ + "llm": { + "model": "anthropic/claude-sonnet-4", + "provider_connection_id": connection_id, + }, + "include_secrets": False, + }, + ) + assert ( + client.post("/api/profiles/temporary-provider-profile/activate").status_code + == 200 + ) + assert client.delete("/api/profiles/temporary-provider-profile").status_code == 200 + + delete = client.delete(f"/api/llm/provider-connections/{connection_id}") + + assert delete.status_code == 409 + assert "referenced by the active agent settings" in delete.json()["detail"] + settings = client.get( + "/api/settings", headers={"X-Expose-Secrets": "plaintext"} + ).json() + assert settings["agent_settings"]["llm"]["api_key"] == "sk-ant-old" + + +def test_provider_connection_rotation_not_copied_into_active_settings(client): + """Read-at-use: rotating a key does not rewrite the resolved active settings. + + The active ``agent_settings.llm`` keeps the key resolved at activation time + until the profile is activated again — nothing is auto-copied on rotation. + """ + connection_id = client.post( + "/api/llm/provider-connections", + json={ + "display_name": "Anthropic Work", + "provider": "anthropic", + "api_key": "sk-ant-old", + }, + ).json()["id"] + client.post( + "/api/profiles/provider-profile", + json={ + "llm": { + "model": "anthropic/claude-sonnet-4", + "provider_connection_id": connection_id, + }, + "include_secrets": False, + }, + ) + assert client.post("/api/profiles/provider-profile/activate").status_code == 200 + + rotated = client.patch( + f"/api/llm/provider-connections/{connection_id}", + json={"api_key": "sk-ant-new"}, + ) + assert rotated.status_code == 200 + + # Active settings still hold the key resolved at activation. + settings = client.get( + "/api/settings", headers={"X-Expose-Secrets": "plaintext"} + ).json() + assert settings["agent_settings"]["llm"]["api_key"] == "sk-ant-old" + + # Re-activating re-resolves and picks up the rotated key. + assert client.post("/api/profiles/provider-profile/activate").status_code == 200 + settings = client.get( + "/api/settings", headers={"X-Expose-Secrets": "plaintext"} + ).json() + assert settings["agent_settings"]["llm"]["api_key"] == "sk-ant-new" + + +def test_provider_connection_base_url_authoritative_on_activation(client): + """A connection's ``base_url`` wins on activation, including when cleared.""" + connection_id = client.post( + "/api/llm/provider-connections", + json={ + "display_name": "Proxy", + "provider": "custom", + "api_key": "sk-provider", + "base_url": "https://old.example", + }, + ).json()["id"] + client.post( + "/api/profiles/provider-profile", + json={ + "llm": { + "model": "openai/gpt-5.5", + "base_url": "https://profile.example", + "provider_connection_id": connection_id, + }, + "include_secrets": False, + }, + ) + assert client.post("/api/profiles/provider-profile/activate").status_code == 200 + settings = client.get( + "/api/settings", headers={"X-Expose-Secrets": "plaintext"} + ).json() + # Connection base_url wins over the profile's own. + assert settings["agent_settings"]["llm"]["base_url"] == "https://old.example" + + updated = client.patch( + f"/api/llm/provider-connections/{connection_id}", + json={"base_url": None}, + ) + assert updated.status_code == 200 + assert updated.json()["base_url"] is None + + # base_url is authoritative including None: re-activation clears it. + assert client.post("/api/profiles/provider-profile/activate").status_code == 200 + settings = client.get( + "/api/settings", headers={"X-Expose-Secrets": "plaintext"} + ).json() + assert settings["agent_settings"]["llm"]["base_url"] is None + + +def test_activate_profile_with_dangling_provider_connection_fails(client): + """A profile whose connection no longer exists fails loudly on activation.""" + connection_id = client.post( + "/api/llm/provider-connections", + json={ + "display_name": "Anthropic Work", + "provider": "anthropic", + "api_key": "sk-ant-old", + }, + ).json()["id"] + client.post( + "/api/profiles/provider-profile", + json={ + "llm": { + "model": "anthropic/claude-sonnet-4", + "provider_connection_id": connection_id, + }, + "include_secrets": False, + }, + ) + # Drop the profile so the connection is unreferenced and can be deleted, + # then re-create the profile to leave a dangling provider reference. + assert client.delete("/api/profiles/provider-profile").status_code == 200 + assert ( + client.delete(f"/api/llm/provider-connections/{connection_id}").status_code + == 200 + ) + client.post( + "/api/profiles/provider-profile", + json={ + "llm": { + "model": "anthropic/claude-sonnet-4", + "provider_connection_id": connection_id, + }, + "include_secrets": False, + }, + ) + + activated = client.post("/api/profiles/provider-profile/activate") + + assert activated.status_code == 422 + assert connection_id in activated.json()["detail"] + + +def test_provider_connection_rejects_extra_headers(client): + response = client.post( + "/api/llm/provider-connections", + json={ + "display_name": "Proxy", + "provider": "custom", + "api_key": "sk-provider", + "extra_headers": {"Authorization": "Bearer proxy-secret"}, + }, + ) + + assert response.status_code == 422 + + def test_get_profile_returns_config(client, store): """GET /api/profiles/{name} returns profile config with api_key nulled.""" llm = LLM(model="gpt-4o", api_key="sk-secret-key", temperature=0.7) @@ -538,7 +870,7 @@ def test_get_profile_timeout_returns_503(client, store, monkeypatch): """Get endpoint surfaces TimeoutError as 503.""" store.save("present", LLM(model="gpt-4o")) - def boom(self, name, *, cipher=None): + def boom(self, name, *, cipher=None, resolve_provider=True): raise TimeoutError("locked") monkeypatch.setattr(LLMProfileStore, "load", boom) @@ -1250,6 +1582,162 @@ def test_list_profiles_no_auto_create_after_deleting_active_profile(client, stor assert body["active_profile"] is None +def test_patch_provider_connection_rejects_clearing_api_key(client): + """PATCH api_key: null is a 422, not a silently-ignored no-op.""" + connection_id = client.post( + "/api/llm/provider-connections", + json={ + "display_name": "Anthropic", + "provider": "anthropic", + "api_key": "sk-ant-old", + }, + ).json()["id"] + + cleared = client.patch( + f"/api/llm/provider-connections/{connection_id}", + json={"api_key": None}, + ) + assert cleared.status_code == 422 + assert "api_key cannot be cleared" in cleared.json()["detail"] + + # The stored key is untouched: a linked profile still resolves it. + client.post( + "/api/profiles/linked", + json={ + "llm": { + "model": "anthropic/claude-sonnet-4", + "provider_connection_id": connection_id, + }, + "include_secrets": False, + }, + ) + assert client.get("/api/profiles/linked").json()["api_key_set"] is True + + +def test_get_profile_maps_corrupted_provider_file(client, temp_profiles_dir): + """A corrupted provider file yields a mapped 4xx, not an unhandled 500. + + ``_profile_api_key_set`` reads the provider store while rendering a linked + profile; that read must go through ``store_errors()`` so a corrupt file + does not escape as a 500 on GET /profiles/{name}. + """ + connection_id = client.post( + "/api/llm/provider-connections", + json={ + "display_name": "Anthropic", + "provider": "anthropic", + "api_key": "sk-ant-old", + }, + ).json()["id"] + client.post( + "/api/profiles/linked", + json={ + "llm": { + "model": "anthropic/claude-sonnet-4", + "provider_connection_id": connection_id, + }, + "include_secrets": False, + }, + ) + + provider_file = ( + temp_profiles_dir.parent / "provider-connections" / "provider_connections.json" + ) + provider_file.write_text("{ not valid json", encoding="utf-8") + + response = client.get("/api/profiles/linked") + + assert response.status_code == 400 + assert response.status_code != 500 + + +def test_patch_provider_connection_rejects_null_display_name(client): + """Fix: PATCH display_name=null must return 422, not persist null.""" + connection_id = client.post( + "/api/llm/provider-connections", + json={ + "display_name": "Anthropic", + "provider": "anthropic", + "api_key": "sk-ant", + }, + ).json()["id"] + + response = client.patch( + f"/api/llm/provider-connections/{connection_id}", + json={"display_name": None}, + ) + assert response.status_code == 422 + + # The connection must be intact after the rejected PATCH. + conn = client.get("/api/llm/provider-connections").json()[0] + assert conn["display_name"] == "Anthropic" + + +def test_patch_provider_connection_rejects_null_provider(client): + """Fix: PATCH provider=null must return 422.""" + connection_id = client.post( + "/api/llm/provider-connections", + json={ + "display_name": "Anthropic", + "provider": "anthropic", + "api_key": "sk-ant", + }, + ).json()["id"] + + response = client.patch( + f"/api/llm/provider-connections/{connection_id}", + json={"provider": None}, + ) + assert response.status_code == 422 + + +def test_list_provider_connections_maps_corrupted_file(client, temp_profiles_dir): + """GET provider-connections maps a corrupted file to 400, not 500.""" + client.post( + "/api/llm/provider-connections", + json={ + "display_name": "Anthropic", + "provider": "anthropic", + "api_key": "sk-ant", + }, + ) + + provider_file = ( + temp_profiles_dir.parent / "provider-connections" / "provider_connections.json" + ) + provider_file.write_text("{ not valid json", encoding="utf-8") + + response = client.get("/api/llm/provider-connections") + + assert response.status_code == 400 + assert response.status_code != 500 + + +def test_patch_provider_connection_maps_corrupted_file(client, temp_profiles_dir): + """PATCH provider-connections maps a corrupted file to 400, not 500.""" + connection_id = client.post( + "/api/llm/provider-connections", + json={ + "display_name": "Anthropic", + "provider": "anthropic", + "api_key": "sk-ant", + }, + ).json()["id"] + + provider_file = ( + temp_profiles_dir.parent / "provider-connections" / "provider_connections.json" + ) + provider_file.write_text("{ not valid json", encoding="utf-8") + + response = client.patch( + f"/api/llm/provider-connections/{connection_id}", + json={"display_name": "Renamed"}, + ) + + assert response.status_code == 400 + assert response.status_code != 500 + + # ── Pre-flight validation: POST /api/profiles/{name}/validate ──────────────── diff --git a/tests/sdk/llm/test_llm_profile_store.py b/tests/sdk/llm/test_llm_profile_store.py index b3d5615917..6c429c165a 100644 --- a/tests/sdk/llm/test_llm_profile_store.py +++ b/tests/sdk/llm/test_llm_profile_store.py @@ -187,6 +187,8 @@ def test_list_summaries_migrates_legacy_openhands_proxy_profile( "name": "legacy", "model": "openhands/claude-opus-4-8", "base_url": None, + "provider_connection_id": None, + "provider_connection_broken": False, "api_key_set": False, } ] @@ -739,3 +741,49 @@ def test_multiple_profiles(profile_store: LLMProfileStore) -> None: profile_store.delete("gpt4") assert len(profile_store.list()) == 2 assert "gpt4.json" not in profile_store.list() + + +def test_default_provider_store_is_sibling_of_base_dir(tmp_path: Path) -> None: + """A custom-dir profile store resolves connections under its own base_dir. + + Regression: the default ProviderConnectionStore must be derived from + ``base_dir`` (a sibling ``provider-connections`` directory), not from + ``$HOME``. Otherwise a custom-directory profile store reads profiles from + ``base_dir`` but credentials from ``~/.openhands`` — the wrong source. + """ + from openhands.sdk.llm.provider_connection_store import ( + ProviderConnection, + ProviderConnectionStore, + ) + + store = LLMProfileStore(base_dir=tmp_path) + + provider_store = store._provider_store + assert provider_store is not None + assert provider_store.base_dir == tmp_path.parent / "provider-connections" + assert Path.home() not in provider_store.base_dir.parents + + connections = ProviderConnectionStore( + base_dir=tmp_path.parent / "provider-connections" + ) + now = 1_000 + connections.create( + ProviderConnection( + id="conn1", + display_name="Anthropic", + provider="anthropic", + api_key=SecretStr("sk-shared"), + created_at=now, + updated_at=now, + ) + ) + llm = LLM( + usage_id="linked", + model="anthropic/claude-sonnet-4", + provider_connection_id="conn1", + ) + store.save("linked", llm) + + resolved = store.load("linked") + assert isinstance(resolved.api_key, SecretStr) + assert resolved.api_key.get_secret_value() == "sk-shared" diff --git a/tests/sdk/llm/test_provider_connection_store.py b/tests/sdk/llm/test_provider_connection_store.py new file mode 100644 index 0000000000..e77c1aa059 --- /dev/null +++ b/tests/sdk/llm/test_provider_connection_store.py @@ -0,0 +1,253 @@ +"""Tests for ProviderConnectionStore and its resolution in LLMProfileStore.""" + +import json +import time + +import pytest +from pydantic import SecretStr + +from openhands.sdk.llm import LLM +from openhands.sdk.llm.llm_profile_store import LLMProfileStore +from openhands.sdk.llm.provider_connection_store import ( + PROVIDER_CONNECTIONS_SCHEMA_VERSION, + ProviderConnection, + ProviderConnectionNotFound, + ProviderConnectionStore, +) +from openhands.sdk.utils.cipher import Cipher + + +def _connection(**overrides) -> ProviderConnection: + now = int(time.time()) + data = { + "id": "conn1", + "display_name": "Anthropic", + "provider": "anthropic", + "api_key": "sk-shared", + "base_url": "https://api.anthropic.com", + "created_at": now, + "updated_at": now, + } + data.update(overrides) + return ProviderConnection(**data) + + +def test_provider_connection_not_found_is_value_error(): + """The agent-server relies on this subclassing so generic ``ValueError`` + handlers (OpenAI gateway, profile resolver) degrade a dangling reference to + a 4xx instead of an opaque 500.""" + assert issubclass(ProviderConnectionNotFound, ValueError) + + +def test_crud_roundtrip(tmp_path): + store = ProviderConnectionStore(base_dir=tmp_path) + assert store.list() == [] + + store.create(_connection()) + got = store.get("conn1") + assert got is not None + assert got.api_key_value() == "sk-shared" + + store.update(_connection(display_name="Renamed", api_key="sk-new")) + renamed = store.get("conn1") + assert renamed is not None + assert renamed.display_name == "Renamed" + assert renamed.api_key_value() == "sk-new" + + store.delete("conn1") + assert store.get("conn1") is None + with pytest.raises(ProviderConnectionNotFound): + store.delete("conn1") + + +def test_api_key_encrypted_at_rest(tmp_path): + cipher = Cipher("unit-test-secret-key") + store = ProviderConnectionStore(base_dir=tmp_path) + store.create(_connection(), cipher=cipher) + + raw = json.loads((tmp_path / "provider_connections.json").read_text()) + stored_key = raw["connections"][0]["api_key"] + assert stored_key != "sk-shared" # not plaintext + assert raw["schema_version"] == PROVIDER_CONNECTIONS_SCHEMA_VERSION + + # Round-trips back to plaintext with the same cipher. + loaded = store.get("conn1", cipher=cipher) + assert loaded is not None + assert loaded.api_key_value() == "sk-shared" + + +def test_corrupted_file_raises_not_clobbers(tmp_path): + path = tmp_path / "provider_connections.json" + path.write_text("{ not valid json") + store = ProviderConnectionStore(base_dir=tmp_path) + with pytest.raises(ValueError): + store.list() + # The corrupt file is left intact, not silently replaced. + assert path.read_text() == "{ not valid json" + + +# ── Resolution in LLMProfileStore ────────────────────────────────────────── + + +def test_load_without_provider_is_unchanged(tmp_path): + """Rule 1/4: no provider reference -> byte-identical old behavior.""" + store = LLMProfileStore(base_dir=tmp_path / "profiles") + store.save("p", LLM(model="gpt-4o", api_key="sk-own"), include_secrets=True) + llm = store.load("p") + assert isinstance(llm.api_key, SecretStr) + assert llm.api_key.get_secret_value() == "sk-own" + + +def test_save_clears_inline_credentials_when_linked(tmp_path): + """Rule 5c: a linked profile persists no inline api_key / base_url.""" + provider = ProviderConnectionStore(base_dir=tmp_path / "conns") + profiles = LLMProfileStore(base_dir=tmp_path / "profiles", provider_store=provider) + profiles.save( + "p", + LLM( + model="anthropic/claude-sonnet-4", + api_key="sk-should-be-dropped", + base_url="https://should.drop", + provider_connection_id="conn1", + ), + include_secrets=True, + ) + raw = json.loads((tmp_path / "profiles" / "p.json").read_text()) + assert raw.get("api_key") in (None, "") + assert raw.get("base_url") is None + assert raw["provider_connection_id"] == "conn1" + + +def test_load_resolves_provider_credentials(tmp_path): + """Rule 5: connection api_key / base_url are applied at load (read-at-use).""" + provider = ProviderConnectionStore(base_dir=tmp_path / "conns") + provider.create(_connection()) + profiles = LLMProfileStore(base_dir=tmp_path / "profiles", provider_store=provider) + profiles.save( + "p", + LLM(model="anthropic/claude-sonnet-4", provider_connection_id="conn1"), + ) + + llm = profiles.load("p") + assert isinstance(llm.api_key, SecretStr) + assert llm.api_key.get_secret_value() == "sk-shared" + assert llm.base_url == "https://api.anthropic.com" + + # Rotation takes effect on the next load, nothing cached. + provider.update(_connection(api_key="sk-rotated")) + rotated = profiles.load("p") + assert isinstance(rotated.api_key, SecretStr) + assert rotated.api_key.get_secret_value() == "sk-rotated" + + +def test_load_base_url_authoritative_including_none(tmp_path): + provider = ProviderConnectionStore(base_dir=tmp_path / "conns") + provider.create(_connection(base_url=None)) + profiles = LLMProfileStore(base_dir=tmp_path / "profiles", provider_store=provider) + profiles.save( + "p", + LLM( + model="openai/gpt-5.5", + base_url="https://profile.example", + provider_connection_id="conn1", + ), + ) + assert profiles.load("p").base_url is None + + +def test_load_missing_connection_raises(tmp_path): + """Rule 5b: dangling reference with no inline key fails loudly.""" + provider = ProviderConnectionStore(base_dir=tmp_path / "conns") + profiles = LLMProfileStore(base_dir=tmp_path / "profiles", provider_store=provider) + profiles.save( + "p", + LLM(model="anthropic/claude-sonnet-4", provider_connection_id="ghost"), + ) + with pytest.raises(ProviderConnectionNotFound): + profiles.load("p") + + # resolve_provider=False inspects the stored profile without resolving. + llm = profiles.load("p", resolve_provider=False) + assert llm.provider_connection_id == "ghost" + + +def test_list_summaries_reports_linked_key_presence(tmp_path): + provider = ProviderConnectionStore(base_dir=tmp_path / "conns") + provider.create(_connection()) + profiles = LLMProfileStore(base_dir=tmp_path / "profiles", provider_store=provider) + profiles.save( + "p", + LLM(model="anthropic/claude-sonnet-4", provider_connection_id="conn1"), + ) + summary = profiles.list_summaries()[0] + assert summary["provider_connection_id"] == "conn1" + assert summary["api_key_set"] is True + assert summary["provider_connection_broken"] is False + + +def test_list_summaries_marks_broken_when_connection_deleted(tmp_path): + """Fix (enyst): deleting a provider connection must surface as + ``provider_connection_broken=True`` on every profile that referenced it, + so the UI can show a 'Broken link' indicator rather than silently failing + on activation.""" + provider = ProviderConnectionStore(base_dir=tmp_path / "conns") + provider.create(_connection()) + profiles = LLMProfileStore(base_dir=tmp_path / "profiles", provider_store=provider) + profiles.save( + "p", LLM(model="anthropic/claude-sonnet-4", provider_connection_id="conn1") + ) + + # Confirm not broken before deletion. + assert profiles.list_summaries()[0]["provider_connection_broken"] is False + + # Delete the connection; the profile on disk still references it. + provider.delete("conn1") + + summary = profiles.list_summaries()[0] + assert summary["provider_connection_broken"] is True + assert summary["provider_connection_id"] == "conn1" + # api_key_set must also be False (no inline key, no connection). + assert summary["api_key_set"] is False + + +def test_bare_profile_store_auto_wires_provider_store(tmp_path): + """Fix: bare LLMProfileStore() auto-creates a ProviderConnectionStore so that + standalone SDK paths (LocalConversation, FallbackStrategy, switch_llm) can + resolve linked profiles saved by the agent-server. + + The auto-wired store lives in a ``provider-connections`` directory sibling + to the profile store's ``base_dir`` (not under ``$HOME``), so a custom-dir + profile store reads its credentials from the same location. + """ + # The auto-wired provider store is a sibling of base_dir. + provider = ProviderConnectionStore(base_dir=tmp_path / "provider-connections") + provider.create(_connection()) + + # Construct with only base_dir — no explicit provider_store arg. + profiles = LLMProfileStore(base_dir=tmp_path / "profiles") + profiles.save("p", LLM(model="gpt-4o", provider_connection_id="conn1")) + + llm = profiles.load("p") + assert isinstance(llm.api_key, SecretStr) + assert llm.api_key.get_secret_value() == "sk-shared" + assert llm.base_url == "https://api.anthropic.com" + + +def test_wrong_cipher_update_raises_not_destroys_key(tmp_path): + """Fix: a read-modify-write with a wrong cipher must raise a ValidationError, + not silently persist api_key=null and destroy the stored ciphertext.""" + from pydantic import ValidationError + + cipher_a = Cipher("key-a") + cipher_b = Cipher("key-b") + store = ProviderConnectionStore(base_dir=tmp_path) + store.create(_connection(), cipher=cipher_a) + + # Attempting to update with the wrong cipher must fail before any write. + with pytest.raises((ValidationError, ValueError)): + store.update(_connection(display_name="updated"), cipher=cipher_b) + + # The stored key must still be intact — readable with the original cipher. + restored = store.get("conn1", cipher=cipher_a) + assert restored is not None + assert restored.api_key_value() == "sk-shared" From d98fd95005fba2faf41983c3cec13282e27e8264 Mon Sep 17 00:00:00 2001 From: george larson Date: Wed, 19 Aug 2026 17:42:39 -0400 Subject: [PATCH 08/14] test(sdk): pin send_message skill-activation wiring (#4536) --- .../conversation/impl/local_conversation.py | 1 - .../local/test_conversation_send_message.py | 162 +++++++++++++++++- 2 files changed, 160 insertions(+), 3 deletions(-) diff --git a/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py b/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py index ffe3e7b864..fb4316046d 100644 --- a/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py +++ b/openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py @@ -1837,7 +1837,6 @@ def send_message(self, message: str | Message, sender: str | None = None) -> Non ConversationExecutionStatus.IDLE ) # new message resets terminal states - # TODO: We should add test cases for all these scenarios activated_skill_names: list[str] = [] extended_content: list[TextContent] = [] diff --git a/tests/sdk/conversation/local/test_conversation_send_message.py b/tests/sdk/conversation/local/test_conversation_send_message.py index ec124f2da8..56095d943f 100644 --- a/tests/sdk/conversation/local/test_conversation_send_message.py +++ b/tests/sdk/conversation/local/test_conversation_send_message.py @@ -6,6 +6,7 @@ from openhands.sdk.agent.acp_agent import ACPAgent from openhands.sdk.agent.base import AgentBase +from openhands.sdk.context.agent_context import AgentContext from openhands.sdk.conversation import Conversation, LocalConversation from openhands.sdk.conversation.impl.local_conversation import ( ACP_INFLIGHT_PROMPT_USER_MESSAGE_ID, @@ -21,14 +22,15 @@ ) from openhands.sdk.event.llm_convertible import MessageEvent, SystemPromptEvent from openhands.sdk.llm import LLM, Message, TextContent +from openhands.sdk.skills import KeywordTrigger, Skill class SendMessageDummyAgent(AgentBase): - def __init__(self): + def __init__(self, agent_context: AgentContext | None = None): llm = LLM( model="gpt-4o-mini", api_key=SecretStr("test-key"), usage_id="test-llm" ) - super().__init__(llm=llm, tools=[]) + super().__init__(llm=llm, tools=[], agent_context=agent_context) def init_state( self, state: ConversationState, on_event: ConversationCallbackType @@ -849,3 +851,159 @@ async def blocking_astep( assert prompts_seen == ["initial request"] assert conversation.state.execution_status == ConversationExecutionStatus.IDLE + + +# --- Skill activation wiring ------------------------------------------------- +# Characterization tests for the skill-activation block in +# LocalConversation.send_message(). AgentContext.get_user_message_suffix() has +# its own unit tests; these pin the conversation-level contract: which fields +# of the MessageEvent carry the activation, what happens to the user's own +# text, and how ConversationState remembers activations across turns. + + +def _python_tips_skill() -> Skill: + return Skill( + name="python_tips", + content="Use list comprehensions for better performance.", + source="python-tips.md", + trigger=KeywordTrigger(keywords=["python", "performance"]), + ) + + +def test_send_message_activates_skill_on_trigger_match(): + """A keyword match wires the skill into the event and conversation state.""" + agent = SendMessageDummyAgent( + agent_context=AgentContext(skills=[_python_tips_skill()]) + ) + conversation = Conversation(agent=agent) + + user_text = "How can I improve my Python code performance?" + conversation.send_message(user_text) + + user_event = conversation.state.events[-1] + assert isinstance(user_event, MessageEvent) + + # The user's own message is not rewritten ... + assert len(user_event.llm_message.content) == 1 + assert isinstance(user_event.llm_message.content[0], TextContent) + assert user_event.llm_message.content[0].text == user_text + + # ... the recalled knowledge rides alongside it in extended_content. + assert len(user_event.extended_content) == 1 + assert "Use list comprehensions" in user_event.extended_content[0].text + assert user_event.activated_skills == ["python_tips"] + + # The state records the activation so later turns can skip the skill. + assert conversation.state.activated_knowledge_skills == ["python_tips"] + + # The fold into the LLM payload happens at to_llm_message() time. + llm_message = user_event.to_llm_message() + assert len(llm_message.content) == 2 + assert isinstance(llm_message.content[0], TextContent) + assert llm_message.content[0].text == user_text + assert isinstance(llm_message.content[1], TextContent) + assert llm_message.content[1].text == user_event.extended_content[0].text + + +def test_send_message_skips_already_activated_skill(): + """A skill fires once per conversation, not once per matching message.""" + agent = SendMessageDummyAgent( + agent_context=AgentContext(skills=[_python_tips_skill()]) + ) + conversation = Conversation(agent=agent) + + conversation.send_message("Tell me about python performance") + conversation.send_message("More python tips please") + + user_events = [ + e + for e in conversation.state.events + if isinstance(e, MessageEvent) and e.source == "user" + ] + first_event, second_event = user_events + + assert first_event.activated_skills == ["python_tips"] + assert len(first_event.extended_content) == 1 + + assert second_event.activated_skills == [] + assert second_event.extended_content == [] + + # Recorded once, not duplicated. + assert conversation.state.activated_knowledge_skills == ["python_tips"] + + +def test_send_message_without_trigger_match_leaves_event_plain(): + """No keyword match: the event carries no activation and state stays empty.""" + agent = SendMessageDummyAgent( + agent_context=AgentContext(skills=[_python_tips_skill()]) + ) + conversation = Conversation(agent=agent) + + conversation.send_message("How do I write JavaScript code?") + + user_event = conversation.state.events[-1] + assert isinstance(user_event, MessageEvent) + assert user_event.activated_skills == [] + assert user_event.extended_content == [] + assert conversation.state.activated_knowledge_skills == [] + + +def test_send_message_without_agent_context_leaves_event_plain(): + """No agent_context: the activation block is a no-op.""" + agent = SendMessageDummyAgent() + conversation = Conversation(agent=agent) + + conversation.send_message("Tell me about python performance") + + user_event = conversation.state.events[-1] + assert isinstance(user_event, MessageEvent) + assert user_event.activated_skills == [] + assert user_event.extended_content == [] + assert conversation.state.activated_knowledge_skills == [] + + +def test_send_message_activates_all_matching_skills(): + """Every matching skill activates on the same message.""" + testing_skill = Skill( + name="testing_framework", + content="Use pytest for comprehensive testing with fixtures.", + source="testing-framework.md", + trigger=KeywordTrigger(keywords=["testing", "pytest"]), + ) + agent = SendMessageDummyAgent( + agent_context=AgentContext(skills=[_python_tips_skill(), testing_skill]) + ) + conversation = Conversation(agent=agent) + + conversation.send_message("I need help with python testing using pytest") + + user_event = conversation.state.events[-1] + assert isinstance(user_event, MessageEvent) + # Activation order is deterministic: AgentContext.skills declaration order. + assert user_event.activated_skills == ["python_tips", "testing_framework"] + # All matched skills are merged into a single extended_content block. + assert len(user_event.extended_content) == 1 + assert "Use list comprehensions" in user_event.extended_content[0].text + assert "Use pytest" in user_event.extended_content[0].text + assert conversation.state.activated_knowledge_skills == [ + "python_tips", + "testing_framework", + ] + + +def test_send_message_user_message_suffix_without_skill_match(): + """A static user_message_suffix reaches extended_content with no activation.""" + agent = SendMessageDummyAgent( + agent_context=AgentContext(user_message_suffix="Always cite your sources.") + ) + conversation = Conversation(agent=agent) + + conversation.send_message("How do I write JavaScript code?") + + user_event = conversation.state.events[-1] + assert isinstance(user_event, MessageEvent) + assert user_event.activated_skills == [] + assert [c.text for c in user_event.extended_content] == [ + "Always cite your sources." + ] + assert conversation.state.activated_knowledge_skills == [] From 1de2e6d1bfcf70c7c3d4eb13616811943f33dd75 Mon Sep 17 00:00:00 2001 From: Vasco Schiavo <115561717+VascoSch92@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:46:14 +0200 Subject: [PATCH 09/14] fix(agent-server): propagate out-of-band run failures as ConversationErrorEvent (#16686) (#4535) Co-authored-by: vasco --- .../agent_server/conversation_service.py | 11 ++- .../openhands/agent_server/event_service.py | 32 ++++++++- .../openhands/sdk/conversation/title_utils.py | 34 +++++++-- .../agent_server/test_conversation_service.py | 23 ++++++ tests/agent_server/test_event_service.py | 71 +++++++++++++++++++ tests/sdk/conversation/test_generate_title.py | 17 +++++ 6 files changed, 179 insertions(+), 9 deletions(-) diff --git a/openhands-agent-server/openhands/agent_server/conversation_service.py b/openhands-agent-server/openhands/agent_server/conversation_service.py index 7b93bc5a10..32f9e30f59 100644 --- a/openhands-agent-server/openhands/agent_server/conversation_service.py +++ b/openhands-agent-server/openhands/agent_server/conversation_service.py @@ -2388,7 +2388,7 @@ async def __call__(self, _event: Event): @observe( name="conversation.generate_title", - ignore_inputs=["conversation", "llm"], + ignore_inputs=["conversation", "llm", "on_error"], metadata={OPERATION_METADATA_KEY: "title_generation"}, ) def _generate_title_traced( @@ -2398,8 +2398,9 @@ def _generate_title_traced( message: str, llm: LLM | None, max_length: int, + on_error: Callable[[Exception], None] | None = None, ) -> str: - return generate_title_from_message(message, llm, max_length) + return generate_title_from_message(message, llm, max_length, on_error=on_error) @dataclass @@ -2429,6 +2430,11 @@ async def __call__(self, event: Event) -> None: if title_llm is None: title_llm = conversation.agent.llm if conversation else None + # Surface an LLM failure during auto-titling to the UI (issue #16686); + # generation itself stays non-fatal and falls back to truncation. + def _on_title_error(exc: Exception) -> None: + self.service._publish_error_event_sync(exc) + async def _generate_and_save() -> None: try: loop = asyncio.get_running_loop() @@ -2439,6 +2445,7 @@ async def _generate_and_save() -> None: message_text, title_llm, 50, + _on_title_error, ) if title and self.service.stored.title is None: self.service.stored.title = title diff --git a/openhands-agent-server/openhands/agent_server/event_service.py b/openhands-agent-server/openhands/agent_server/event_service.py index 6b06cd790e..8b87487319 100644 --- a/openhands-agent-server/openhands/agent_server/event_service.py +++ b/openhands-agent-server/openhands/agent_server/event_service.py @@ -31,6 +31,7 @@ ) from openhands.sdk.conversation.base import BaseConversation from openhands.sdk.conversation.events_list_base import EventsListBase +from openhands.sdk.conversation.exceptions import ConversationRunError from openhands.sdk.conversation.goal import ( GoalController, GoalDone, @@ -64,6 +65,7 @@ ObservationBaseEvent, StreamingDeltaEvent, ) +from openhands.sdk.event.conversation_error import ConversationErrorEvent from openhands.sdk.event.conversation_state import ConversationStateUpdateEvent from openhands.sdk.event.error_classification import ErrorClassification, FailureKind from openhands.sdk.event.llm_completion_log import LLMCompletionLogEvent @@ -656,6 +658,26 @@ def _mark_error_status_sync(self) -> None: if state.execution_status != ConversationExecutionStatus.ERROR: state.execution_status = ConversationExecutionStatus.ERROR + def _publish_error_event_sync(self, exc: BaseException) -> None: + """Emit a ConversationErrorEvent so the UI sees the failure detail. + + For LLM/runtime failures that would otherwise only reach the logs — the + run-loop backstop and auto-title generation (issue #16686). Best-effort: + never raises (the caller is an error handler). + """ + if not self._conversation: + return + try: + error_event = ConversationErrorEvent( + source="environment", + code=type(exc).__name__, + detail=str(exc), + ) + with self._conversation._state: + self._conversation._on_event(error_event) + except Exception: + logger.exception("Failed to publish backstop ConversationErrorEvent") + def _create_state_update_event_sync(self) -> ConversationStateUpdateEvent: if not self._conversation: raise ValueError("inactive_service") @@ -1226,7 +1248,7 @@ async def _run_and_publish(): await conversation.arun() else: await loop.run_in_executor(self._run_executor, conversation.run) - except Exception: + except Exception as exc: logger.exception("Error during conversation run") # Backstop: a run that raised before reaching its own error # handling (e.g. an ACP cold-start failure in init_state, @@ -1234,6 +1256,14 @@ async def _run_and_publish(): # status at IDLE/RUNNING. Force ERROR so the finally's # _publish_state_update() surfaces the failure instead of a # misleading non-error state. + # + # Also surface the detail to the UI (issue #16686). A + # ConversationRunError means run()/arun() already emitted its + # own event, so skip it there to avoid duplicating the error. + if not isinstance(exc, ConversationRunError): + await loop.run_in_executor( + None, self._publish_error_event_sync, exc + ) await loop.run_in_executor(None, self._mark_error_status_sync) finally: # Wait for all pending events to be published via diff --git a/openhands-sdk/openhands/sdk/conversation/title_utils.py b/openhands-sdk/openhands/sdk/conversation/title_utils.py index d9e2642710..4e98dd015e 100644 --- a/openhands-sdk/openhands/sdk/conversation/title_utils.py +++ b/openhands-sdk/openhands/sdk/conversation/title_utils.py @@ -1,6 +1,6 @@ """Utility functions for generating conversation titles.""" -from collections.abc import Sequence +from collections.abc import Callable, Sequence from openhands.sdk.event import MessageEvent from openhands.sdk.event.base import Event @@ -59,13 +59,21 @@ def extract_first_user_message(events: Sequence[Event]) -> str | None: return None -def generate_title_with_llm(message: str, llm: LLM, max_length: int = 50) -> str | None: +def generate_title_with_llm( + message: str, + llm: LLM, + max_length: int = 50, + on_error: Callable[[Exception], None] | None = None, +) -> str | None: """Generate a conversation title using LLM. Args: message: The first user message to generate title from. llm: The LLM to use for title generation. max_length: Maximum length of the generated title. + on_error: Optional callback invoked with the exception when the LLM + call fails. Title generation still falls back (returns None); the + callback lets callers surface the otherwise-swallowed error. Returns: Generated title, or None if LLM fails or returns empty response. @@ -142,6 +150,10 @@ def generate_title_with_llm(message: str, llm: LLM, max_length: int = 50) -> str except Exception as e: logger.warning(f"Error generating conversation title with LLM: {e}") + # Non-fatal (we fall back to truncation), but let callers surface the + # otherwise-invisible LLM error to the UI (issue #16686). + if on_error is not None: + on_error(e) return None @@ -162,7 +174,10 @@ def generate_fallback_title(message: str, max_length: int = 50) -> str: def generate_title_from_message( - message: str, llm: LLM | None = None, max_length: int = 50 + message: str, + llm: LLM | None = None, + max_length: int = 50, + on_error: Callable[[Exception], None] | None = None, ) -> str: """Generate a title from an already-extracted user message.""" # Skip the ACP sentinel LLM — it has no credentials and cannot be @@ -171,7 +186,9 @@ def generate_title_from_message( llm_to_use = None if llm and llm.usage_id == "acp-managed" else llm if llm_to_use: - llm_title = generate_title_with_llm(message, llm_to_use, max_length) + llm_title = generate_title_with_llm( + message, llm_to_use, max_length, on_error=on_error + ) if llm_title: return llm_title @@ -179,7 +196,10 @@ def generate_title_from_message( def generate_conversation_title( - events: Sequence[Event], llm: LLM | None = None, max_length: int = 50 + events: Sequence[Event], + llm: LLM | None = None, + max_length: int = 50, + on_error: Callable[[Exception], None] | None = None, ) -> str: """Generate a title for a conversation based on the first user message. @@ -205,4 +225,6 @@ def generate_conversation_title( if not first_user_message: raise ValueError("No user messages found in conversation events") - return generate_title_from_message(first_user_message, llm, max_length) + return generate_title_from_message( + first_user_message, llm, max_length, on_error=on_error + ) diff --git a/tests/agent_server/test_conversation_service.py b/tests/agent_server/test_conversation_service.py index 1ee9a7f386..4fb3b79784 100644 --- a/tests/agent_server/test_conversation_service.py +++ b/tests/agent_server/test_conversation_service.py @@ -3114,6 +3114,29 @@ async def test_autotitle_handles_generate_title_failure(self): assert service.stored.title is None service.save_meta.assert_not_called() + @pytest.mark.asyncio + async def test_autotitle_surfaces_llm_error_to_ui(self): + """When the title LLM call fails, the error is surfaced to the UI via + the EventService error-event helper (issue #16686) — while auto-titling + stays non-fatal and falls back to truncation.""" + service = self._make_service() + + # Let the real title utils run; only the LLM call fails, so the error + # is swallowed into a fallback title and reported through on_error. + with patch( + "openhands.sdk.llm.llm.LLM.completion", + side_effect=Exception("model does not exist"), + ): + subscriber = AutoTitleSubscriber(service=service) + await subscriber(self._user_message_event()) + await self._drain_title_task( + lambda: service._publish_error_event_sync.called + ) + + service._publish_error_event_sync.assert_called_once() + (exc,) = service._publish_error_event_sync.call_args.args + assert str(exc) == "model does not exist" + @pytest.mark.asyncio async def test_autotitle_skips_empty_message(self): """No title generation if the user message has no text content.""" diff --git a/tests/agent_server/test_event_service.py b/tests/agent_server/test_event_service.py index 48eb49d103..74f55baac5 100644 --- a/tests/agent_server/test_event_service.py +++ b/tests/agent_server/test_event_service.py @@ -27,6 +27,7 @@ from openhands.sdk import LLM, Agent, AgentBase, Conversation, Message from openhands.sdk.agent import ACPAgent from openhands.sdk.conversation.event_store import EventLog +from openhands.sdk.conversation.exceptions import ConversationRunError from openhands.sdk.conversation.fifo_lock import FIFOLock from openhands.sdk.conversation.impl.local_conversation import ( ACP_INFLIGHT_PROMPT_USER_MESSAGE_ID, @@ -39,6 +40,7 @@ ) from openhands.sdk.credential import CredentialSyncError from openhands.sdk.event import AgentErrorEvent, Event +from openhands.sdk.event.conversation_error import ConversationErrorEvent from openhands.sdk.event.conversation_state import ConversationStateUpdateEvent from openhands.sdk.event.llm_convertible import ( ActionEvent, @@ -1323,6 +1325,75 @@ async def test_run_exception_preserves_existing_error_status(self, event_service assert state.execution_status == ConversationExecutionStatus.ERROR + @pytest.mark.asyncio + async def test_run_exception_emits_conversation_error_event(self, event_service): + """A failure that escapes run()/arun()'s own emission must be surfaced + by the backstop as a ConversationErrorEvent (issue #16686).""" + conversation = MagicMock() + state = MagicMock() + state.execution_status = ConversationExecutionStatus.IDLE + state.__enter__ = MagicMock(return_value=state) + state.__exit__ = MagicMock(return_value=None) + conversation.state = state + conversation._state = state + conversation.send_message = MagicMock() + conversation._on_event = MagicMock() + conversation.run = MagicMock(side_effect=RuntimeError("model does not exist")) + + event_service._conversation = conversation + event_service._publish_state_update = AsyncMock() + + await event_service.send_message(Message(role="user", content=[]), run=True) + assert event_service._run_task is not None + await event_service._run_task + + # A single ConversationErrorEvent was emitted through _on_event, carrying + # the exception type and message so the UI can render the detail. + error_events = [ + call.args[0] + for call in conversation._on_event.call_args_list + if isinstance(call.args[0], ConversationErrorEvent) + ] + assert len(error_events) == 1 + assert error_events[0].code == "RuntimeError" + assert error_events[0].detail == "model does not exist" + assert error_events[0].source == "environment" + assert state.execution_status == ConversationExecutionStatus.ERROR + + @pytest.mark.asyncio + async def test_run_conversation_run_error_does_not_double_emit(self, event_service): + """A ConversationRunError is already surfaced by run()/arun(), so the + backstop must not emit a duplicate ConversationErrorEvent.""" + conversation = MagicMock() + state = MagicMock() + state.execution_status = ConversationExecutionStatus.ERROR + state.__enter__ = MagicMock(return_value=state) + state.__exit__ = MagicMock(return_value=None) + conversation.state = state + conversation._state = state + conversation.send_message = MagicMock() + conversation._on_event = MagicMock() + conversation.run = MagicMock( + side_effect=ConversationRunError( + conversation_id=uuid4(), + original_exception=RuntimeError("already surfaced"), + ) + ) + + event_service._conversation = conversation + event_service._publish_state_update = AsyncMock() + + await event_service.send_message(Message(role="user", content=[]), run=True) + assert event_service._run_task is not None + await event_service._run_task + + error_events = [ + call.args[0] + for call in conversation._on_event.call_args_list + if isinstance(call.args[0], ConversationErrorEvent) + ] + assert error_events == [] + @pytest.mark.asyncio async def test_send_message_with_different_message_types(self, event_service): """Test send_message with different message types.""" diff --git a/tests/sdk/conversation/test_generate_title.py b/tests/sdk/conversation/test_generate_title.py index 55afc8a385..858404dd40 100644 --- a/tests/sdk/conversation/test_generate_title.py +++ b/tests/sdk/conversation/test_generate_title.py @@ -10,6 +10,7 @@ from openhands.sdk.agent import Agent from openhands.sdk.conversation import Conversation +from openhands.sdk.conversation.title_utils import generate_title_with_llm from openhands.sdk.event.llm_convertible import MessageEvent from openhands.sdk.llm import LLM, LLMResponse, Message, MetricsSnapshot, TextContent @@ -124,6 +125,22 @@ def test_generate_title_llm_error_fallback(mock_completion): assert title == "Fix the bug in my application" +@patch("openhands.sdk.llm.llm.LLM.completion") +def test_generate_title_with_llm_invokes_on_error(mock_completion): + """generate_title_with_llm reports the swallowed LLM error via on_error + (the opt-in seam used to surface it to clients — issue #16686) while still + returning None so callers fall back to truncation.""" + custom_llm = LLM(model="gpt-4o-mini", api_key=SecretStr("key"), usage_id="err") + mock_completion.side_effect = Exception("model does not exist") + + seen: list[Exception] = [] + result = generate_title_with_llm("Fix the bug", custom_llm, on_error=seen.append) + + assert result is None + assert len(seen) == 1 + assert str(seen[0]) == "model does not exist" + + @patch("openhands.sdk.llm.llm.LLM.completion") def test_generate_title_truncation_respects_max_length(mock_completion): """When LLM fails, truncation fallback respects max_length.""" From 3e38fade8fe9333d645b8b2a513782cb29613cd9 Mon Sep 17 00:00:00 2001 From: Rohit Malhotra Date: Fri, 21 Aug 2026 02:33:38 -0700 Subject: [PATCH 10/14] fix(sdk): normalize Kimi K3 vision metadata (#4567) Co-authored-by: openhands --- openhands-sdk/openhands/sdk/llm/utils/model_features.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/openhands-sdk/openhands/sdk/llm/utils/model_features.py b/openhands-sdk/openhands/sdk/llm/utils/model_features.py index cd4236d7cc..063e287a54 100644 --- a/openhands-sdk/openhands/sdk/llm/utils/model_features.py +++ b/openhands-sdk/openhands/sdk/llm/utils/model_features.py @@ -88,6 +88,9 @@ def _normalize_model_for_litellm(model: str | None) -> str | None: normalized = normalized.removeprefix(prefix) break + if normalized == "kimi-k3": + return "moonshot/kimi-k3" + return normalized @@ -217,14 +220,12 @@ def _normalized_supported_openai_params(model: str | None) -> frozenset[str]: ] # Match token -> canonical LiteLLM ID for vision metadata overrides. -VISION_MODEL_OVERRIDES = {"kimi-k3": "moonshot/kimi-k3"} +VISION_MODEL_OVERRIDES: dict[str, str] = {} @cache def _model_supports_vision(model: str | None) -> bool: - """Return whether LiteLLM or our override list marks the model as visual.""" - if model and model_matches(model, VISION_MODEL_OVERRIDES.keys()): - return True + """Return whether LiteLLM marks the model as visual.""" normalized = _normalize_model_for_litellm(model) with warnings.catch_warnings(): warnings.simplefilter("ignore") From 4c1237f391fe394e9f67505fe3a0bd2d81f84188 Mon Sep 17 00:00:00 2001 From: OpenHands Bot Date: Fri, 21 Aug 2026 04:51:01 -0500 Subject: [PATCH 11/14] Release v1.43.0 (#4553) Co-authored-by: github-actions[bot] Co-authored-by: openhands Co-authored-by: allhands-bot Co-authored-by: Rohit Malhotra --- .pr/gpt-5-nano-integration-results.md | 74 --------------------------- openhands-agent-server/pyproject.toml | 2 +- openhands-sdk/pyproject.toml | 2 +- openhands-tools/pyproject.toml | 2 +- openhands-workspace/pyproject.toml | 2 +- uv.lock | 8 +-- 6 files changed, 8 insertions(+), 82 deletions(-) delete mode 100644 .pr/gpt-5-nano-integration-results.md diff --git a/.pr/gpt-5-nano-integration-results.md b/.pr/gpt-5-nano-integration-results.md deleted file mode 100644 index 3cda307865..0000000000 --- a/.pr/gpt-5-nano-integration-results.md +++ /dev/null @@ -1,74 +0,0 @@ -# GPT-5-nano integration results - -Tested PR head `cf0fd2e11e3e26ded2ed1eb31fc9178c567d6ba6` on 2026-08-18 with `litellm_proxy/openai/gpt-5-nano`, `reasoning_effort=high`, and `https://llm-proxy.eval.all-hands.dev`. - -No `b*` behavior tests were run. - -## Result summary - -| Test | Result | -| --- | --- | -| Focused unit suite, `tests/sdk/event/test_events_to_messages.py` | 23 passed | -| New reasoning-item regression against `origin/main` | Failed as expected: the combined message had `responses_reasoning_item=None` | -| `t*` integration suite | 8 passed, 1 failed | -| `c*` condenser suite | 2 passed, 2 failed, 1 skipped | -| Isolated rerun of `c02` and `c05` | `c05` passed; `c02` failed again | -| Purpose-built parallel-tool replay probe | Passed | - -## Integration suite details - -### `t*` - -Passed: `t01`, `t02`, `t03`, `t04`, `t06`, `t07`, `t08`, and `t09`. - -`t05_simple_browsing` failed twice, including an isolated retry. Chromium launched and the agent navigated the test site, but GPT-5-nano stopped after saying it would fetch the answer instead of reporting it. This is model behavior and does not exercise action batching. - -### `c*` - -- `c01_thinking_block_condenser`: skipped as designed because GPT-5-nano produces Responses API reasoning items rather than Anthropic thinking blocks. -- `c03_delayed_condensation`: passed with five condensations. -- `c04_token_condenser`: passed. -- `c05_size_condenser`: failed initially because the model emitted only one tool call and stopped before enough events existed; it passed on an isolated rerun, confirming model-dependent flakiness. -- `c02_hard_context_reset`: failed twice because GPT-5-nano answered calculation requests directly instead of creating enough tool-loop events for the second condensation to become a normal condensation. - -The `c02` and initial `c05` failures did not produce parallel sibling `ActionEvent`s, so they did not exercise the code changed by this PR. - -## Direct parallel-tool replay validation - -A first live attempt asked GPT-5-nano to issue two calls to the same terminal tool in parallel. The model instead emitted them in two separate LLM responses, confirming that a generic integration task does not reliably cover this regression. - -A second probe exposed two distinct independent tools, `get_alpha` and `get_beta`, and required both before a final response. GPT-5-nano then produced: - -1. two `ActionEvent`s with the same `llm_response_id`; -2. a Responses API reasoning item only on the first action; -3. a recombined assistant message containing both tool calls in order; -4. a reasoning item exactly equal to the first action's item; and -5. a successful follow-up Responses API turn ending with `alpha beta verified`. - -This exercises the PR's changed path end to end: the reasoning item is retained on the recombined message and accepted when the tool-call batch is sent back to GPT-5-nano. - -## Commands - -```bash -uv run --frozen pytest tests/sdk/event/test_events_to_messages.py - -LLM_API_KEY=... \ -LLM_BASE_URL=https://llm-proxy.eval.all-hands.dev \ -IN_DOCKER=true \ -uv run --frozen python tests/integration/run_infer.py \ - --llm-config '{"model":"litellm_proxy/openai/gpt-5-nano","reasoning_effort":"high"}' \ - --num-workers 4 \ - --test-type integration - -LLM_API_KEY=... \ -LLM_BASE_URL=https://llm-proxy.eval.all-hands.dev \ -IN_DOCKER=true \ -uv run --frozen python tests/integration/run_infer.py \ - --llm-config '{"model":"litellm_proxy/openai/gpt-5-nano","reasoning_effort":"high"}' \ - --num-workers 4 \ - --test-type condenser -``` - -## Assessment - -The focused regression and the live parallel-tool replay both validate the fix. The remaining integration failures are explained by GPT-5-nano task compliance and did not execute the changed parallel-action reconstruction path. diff --git a/openhands-agent-server/pyproject.toml b/openhands-agent-server/pyproject.toml index 315056996a..3087c34a0a 100644 --- a/openhands-agent-server/pyproject.toml +++ b/openhands-agent-server/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openhands-agent-server" -version = "1.42.1" +version = "1.43.0" description = "OpenHands Agent Server - REST/WebSocket interface for OpenHands AI Agent" requires-python = ">=3.12" diff --git a/openhands-sdk/pyproject.toml b/openhands-sdk/pyproject.toml index 505afe45d9..86ac2e90e0 100644 --- a/openhands-sdk/pyproject.toml +++ b/openhands-sdk/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openhands-sdk" -version = "1.42.1" +version = "1.43.0" description = "OpenHands SDK - Core functionality for building AI agents" requires-python = ">=3.12" diff --git a/openhands-tools/pyproject.toml b/openhands-tools/pyproject.toml index be86db0aff..794d0c2939 100644 --- a/openhands-tools/pyproject.toml +++ b/openhands-tools/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openhands-tools" -version = "1.42.1" +version = "1.43.0" description = "OpenHands Tools - Runtime tools for AI agents" requires-python = ">=3.12" diff --git a/openhands-workspace/pyproject.toml b/openhands-workspace/pyproject.toml index 375ab4ae31..44d1dc2c74 100644 --- a/openhands-workspace/pyproject.toml +++ b/openhands-workspace/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openhands-workspace" -version = "1.42.1" +version = "1.43.0" description = "OpenHands Workspace - Docker and container-based workspace implementations" requires-python = ">=3.12" diff --git a/uv.lock b/uv.lock index 10772d6889..3065648ab3 100644 --- a/uv.lock +++ b/uv.lock @@ -2719,7 +2719,7 @@ wheels = [ [[package]] name = "openhands-agent-server" -version = "1.42.1" +version = "1.43.0" source = { editable = "openhands-agent-server" } dependencies = [ { name = "aiosqlite" }, @@ -2759,7 +2759,7 @@ provides-extras = ["posthog"] [[package]] name = "openhands-sdk" -version = "1.42.1" +version = "1.43.0" source = { editable = "openhands-sdk" } dependencies = [ { name = "agent-client-protocol" }, @@ -2821,7 +2821,7 @@ provides-extras = ["boto3", "toolshield", "vertex"] [[package]] name = "openhands-tools" -version = "1.42.1" +version = "1.43.0" source = { editable = "openhands-tools" } dependencies = [ { name = "binaryornot" }, @@ -2852,7 +2852,7 @@ requires-dist = [ [[package]] name = "openhands-workspace" -version = "1.42.1" +version = "1.43.0" source = { editable = "openhands-workspace" } dependencies = [ { name = "openhands-agent-server" }, From 6899f00b0f352635b68cb50b5be0ec1d69885432 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:04:00 +0200 Subject: [PATCH 12/14] chore(deps): bump gitpython from 3.1.50 to 3.1.58 (#4545) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: allhands-bot Co-authored-by: aivong-openhands --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 3065648ab3..6018be413a 100644 --- a/uv.lock +++ b/uv.lock @@ -1238,14 +1238,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.50" +version = "3.1.58" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } +sdist = { url = "https://files.pythonhosted.org/packages/26/d6/5f358ff283325580c2003a6d953aea18cfe10ae87b46f5ebc80fa3a386dc/gitpython-3.1.58.tar.gz", hash = "sha256:621416df10ef3fd0e19fabf9172ddeed0fa704d353d04f194eec56a625a95b22", size = 228498, upload-time = "2026-08-04T15:05:49.47Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, + { url = "https://files.pythonhosted.org/packages/ec/0c/9d8752098bc442f0726e64aa6135940b3a96809915d1aa4206c1bb97881d/gitpython-3.1.58-py3-none-any.whl", hash = "sha256:d331e722577f0fd7fc1f857419b3ecc07af66282b933d2a4d95f84a042fdd50f", size = 220183, upload-time = "2026-08-04T15:05:48.025Z" }, ] [[package]] From 3fa2a00affc597db3d99b6de42d7db315052c873 Mon Sep 17 00:00:00 2001 From: Graham Neubig Date: Fri, 21 Aug 2026 08:22:21 -0400 Subject: [PATCH 13/14] feat(prompt): mention local conversation history (#4527) Co-authored-by: openhands Co-authored-by: neubig --- openhands-sdk/openhands/sdk/context/prompts/sections/static.py | 1 + .../snapshots/anthropic__browser-off__secana-off__cli-on.txt | 1 + .../snapshots/anthropic__browser-off__secana-on__cli-off.txt | 1 + .../snapshots/anthropic__browser-off__secana-on__cli-on.txt | 1 + .../snapshots/anthropic__browser-on__secana-off__cli-on.txt | 1 + .../snapshots/anthropic__browser-on__secana-on__cli-off.txt | 1 + .../snapshots/anthropic__browser-on__secana-on__cli-on.txt | 1 + .../anthropic__browser-on__secana-on__cli-on__win32.txt | 1 + .../snapshots/gemini__browser-off__secana-off__cli-on.txt | 1 + .../snapshots/gemini__browser-off__secana-on__cli-off.txt | 1 + .../prompts/snapshots/gemini__browser-off__secana-on__cli-on.txt | 1 + .../prompts/snapshots/gemini__browser-on__secana-off__cli-on.txt | 1 + .../prompts/snapshots/gemini__browser-on__secana-on__cli-off.txt | 1 + .../prompts/snapshots/gemini__browser-on__secana-on__cli-on.txt | 1 + .../snapshots/openai__browser-off__secana-off__cli-on.txt | 1 + .../snapshots/openai__browser-off__secana-on__cli-off.txt | 1 + .../prompts/snapshots/openai__browser-off__secana-on__cli-on.txt | 1 + .../prompts/snapshots/openai__browser-on__secana-off__cli-on.txt | 1 + .../prompts/snapshots/openai__browser-on__secana-on__cli-off.txt | 1 + .../prompts/snapshots/openai__browser-on__secana-on__cli-on.txt | 1 + .../prompts/snapshots/other__browser-off__secana-off__cli-on.txt | 1 + .../prompts/snapshots/other__browser-off__secana-on__cli-off.txt | 1 + .../prompts/snapshots/other__browser-off__secana-on__cli-on.txt | 1 + .../prompts/snapshots/other__browser-on__secana-off__cli-on.txt | 1 + .../prompts/snapshots/other__browser-on__secana-on__cli-off.txt | 1 + .../prompts/snapshots/other__browser-on__secana-on__cli-on.txt | 1 + tests/sdk/context/prompts/snapshots/soul__custom.txt | 1 + tests/sdk/context/prompts/snapshots/soul__default.txt | 1 + 28 files changed, 28 insertions(+) diff --git a/openhands-sdk/openhands/sdk/context/prompts/sections/static.py b/openhands-sdk/openhands/sdk/context/prompts/sections/static.py index cfedfe1fb5..9012d346f2 100644 --- a/openhands-sdk/openhands/sdk/context/prompts/sections/static.py +++ b/openhands-sdk/openhands/sdk/context/prompts/sections/static.py @@ -108,6 +108,7 @@ class MemorySection(_StaticTextSection): _AGENTS_MD_GUIDANCE = """\ * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills""" diff --git a/tests/sdk/context/prompts/snapshots/anthropic__browser-off__secana-off__cli-on.txt b/tests/sdk/context/prompts/snapshots/anthropic__browser-off__secana-off__cli-on.txt index 7a68629b14..86c34aedfe 100644 --- a/tests/sdk/context/prompts/snapshots/anthropic__browser-off__secana-off__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/anthropic__browser-off__secana-off__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/anthropic__browser-off__secana-on__cli-off.txt b/tests/sdk/context/prompts/snapshots/anthropic__browser-off__secana-on__cli-off.txt index b4040f9dc3..765b1b45ee 100644 --- a/tests/sdk/context/prompts/snapshots/anthropic__browser-off__secana-on__cli-off.txt +++ b/tests/sdk/context/prompts/snapshots/anthropic__browser-off__secana-on__cli-off.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/anthropic__browser-off__secana-on__cli-on.txt b/tests/sdk/context/prompts/snapshots/anthropic__browser-off__secana-on__cli-on.txt index f8e030f842..5560bbf96d 100644 --- a/tests/sdk/context/prompts/snapshots/anthropic__browser-off__secana-on__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/anthropic__browser-off__secana-on__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-off__cli-on.txt b/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-off__cli-on.txt index 5134dba6ee..9831c50961 100644 --- a/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-off__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-off__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-on__cli-off.txt b/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-on__cli-off.txt index d4ab7ddb04..a328e9dbcb 100644 --- a/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-on__cli-off.txt +++ b/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-on__cli-off.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-on__cli-on.txt b/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-on__cli-on.txt index 87319c50f0..3ed60e3b40 100644 --- a/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-on__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-on__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-on__cli-on__win32.txt b/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-on__cli-on__win32.txt index 0980f050f4..aca0d4cf70 100644 --- a/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-on__cli-on__win32.txt +++ b/tests/sdk/context/prompts/snapshots/anthropic__browser-on__secana-on__cli-on__win32.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/gemini__browser-off__secana-off__cli-on.txt b/tests/sdk/context/prompts/snapshots/gemini__browser-off__secana-off__cli-on.txt index 02b530e22d..cf297adc72 100644 --- a/tests/sdk/context/prompts/snapshots/gemini__browser-off__secana-off__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/gemini__browser-off__secana-off__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/gemini__browser-off__secana-on__cli-off.txt b/tests/sdk/context/prompts/snapshots/gemini__browser-off__secana-on__cli-off.txt index dc2c55902d..e2e75bac59 100644 --- a/tests/sdk/context/prompts/snapshots/gemini__browser-off__secana-on__cli-off.txt +++ b/tests/sdk/context/prompts/snapshots/gemini__browser-off__secana-on__cli-off.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/gemini__browser-off__secana-on__cli-on.txt b/tests/sdk/context/prompts/snapshots/gemini__browser-off__secana-on__cli-on.txt index 30a9aab7a4..3179097b4d 100644 --- a/tests/sdk/context/prompts/snapshots/gemini__browser-off__secana-on__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/gemini__browser-off__secana-on__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/gemini__browser-on__secana-off__cli-on.txt b/tests/sdk/context/prompts/snapshots/gemini__browser-on__secana-off__cli-on.txt index 5855dcc644..140eaa9b50 100644 --- a/tests/sdk/context/prompts/snapshots/gemini__browser-on__secana-off__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/gemini__browser-on__secana-off__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/gemini__browser-on__secana-on__cli-off.txt b/tests/sdk/context/prompts/snapshots/gemini__browser-on__secana-on__cli-off.txt index 13086708cb..e27d7d0c69 100644 --- a/tests/sdk/context/prompts/snapshots/gemini__browser-on__secana-on__cli-off.txt +++ b/tests/sdk/context/prompts/snapshots/gemini__browser-on__secana-on__cli-off.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/gemini__browser-on__secana-on__cli-on.txt b/tests/sdk/context/prompts/snapshots/gemini__browser-on__secana-on__cli-on.txt index e496d33898..70c1606e3c 100644 --- a/tests/sdk/context/prompts/snapshots/gemini__browser-on__secana-on__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/gemini__browser-on__secana-on__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/openai__browser-off__secana-off__cli-on.txt b/tests/sdk/context/prompts/snapshots/openai__browser-off__secana-off__cli-on.txt index 5489163f19..3a4ee5579c 100644 --- a/tests/sdk/context/prompts/snapshots/openai__browser-off__secana-off__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/openai__browser-off__secana-off__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/openai__browser-off__secana-on__cli-off.txt b/tests/sdk/context/prompts/snapshots/openai__browser-off__secana-on__cli-off.txt index 8040fe291b..c60d0a56db 100644 --- a/tests/sdk/context/prompts/snapshots/openai__browser-off__secana-on__cli-off.txt +++ b/tests/sdk/context/prompts/snapshots/openai__browser-off__secana-on__cli-off.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/openai__browser-off__secana-on__cli-on.txt b/tests/sdk/context/prompts/snapshots/openai__browser-off__secana-on__cli-on.txt index d93956d919..9b936b038e 100644 --- a/tests/sdk/context/prompts/snapshots/openai__browser-off__secana-on__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/openai__browser-off__secana-on__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/openai__browser-on__secana-off__cli-on.txt b/tests/sdk/context/prompts/snapshots/openai__browser-on__secana-off__cli-on.txt index e56d781731..4b167f926f 100644 --- a/tests/sdk/context/prompts/snapshots/openai__browser-on__secana-off__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/openai__browser-on__secana-off__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/openai__browser-on__secana-on__cli-off.txt b/tests/sdk/context/prompts/snapshots/openai__browser-on__secana-on__cli-off.txt index a7467f2cbe..fb4daeaf9a 100644 --- a/tests/sdk/context/prompts/snapshots/openai__browser-on__secana-on__cli-off.txt +++ b/tests/sdk/context/prompts/snapshots/openai__browser-on__secana-on__cli-off.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/openai__browser-on__secana-on__cli-on.txt b/tests/sdk/context/prompts/snapshots/openai__browser-on__secana-on__cli-on.txt index 9833883b45..2e1e537b75 100644 --- a/tests/sdk/context/prompts/snapshots/openai__browser-on__secana-on__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/openai__browser-on__secana-on__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/other__browser-off__secana-off__cli-on.txt b/tests/sdk/context/prompts/snapshots/other__browser-off__secana-off__cli-on.txt index dc8f8efaa1..f5f8d850bd 100644 --- a/tests/sdk/context/prompts/snapshots/other__browser-off__secana-off__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/other__browser-off__secana-off__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/other__browser-off__secana-on__cli-off.txt b/tests/sdk/context/prompts/snapshots/other__browser-off__secana-on__cli-off.txt index d79fefae0c..f8dda420ae 100644 --- a/tests/sdk/context/prompts/snapshots/other__browser-off__secana-on__cli-off.txt +++ b/tests/sdk/context/prompts/snapshots/other__browser-off__secana-on__cli-off.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/other__browser-off__secana-on__cli-on.txt b/tests/sdk/context/prompts/snapshots/other__browser-off__secana-on__cli-on.txt index 034f4620db..6517f3c45e 100644 --- a/tests/sdk/context/prompts/snapshots/other__browser-off__secana-on__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/other__browser-off__secana-on__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/other__browser-on__secana-off__cli-on.txt b/tests/sdk/context/prompts/snapshots/other__browser-on__secana-off__cli-on.txt index 4503444806..3bf3803fc1 100644 --- a/tests/sdk/context/prompts/snapshots/other__browser-on__secana-off__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/other__browser-on__secana-off__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/other__browser-on__secana-on__cli-off.txt b/tests/sdk/context/prompts/snapshots/other__browser-on__secana-on__cli-off.txt index ee10071755..06a3b8bead 100644 --- a/tests/sdk/context/prompts/snapshots/other__browser-on__secana-on__cli-off.txt +++ b/tests/sdk/context/prompts/snapshots/other__browser-on__secana-on__cli-off.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/other__browser-on__secana-on__cli-on.txt b/tests/sdk/context/prompts/snapshots/other__browser-on__secana-on__cli-on.txt index 1857e8c8a7..709ba83793 100644 --- a/tests/sdk/context/prompts/snapshots/other__browser-on__secana-on__cli-on.txt +++ b/tests/sdk/context/prompts/snapshots/other__browser-on__secana-on__cli-on.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/soul__custom.txt b/tests/sdk/context/prompts/snapshots/soul__custom.txt index 0681ee2614..f788618fc7 100644 --- a/tests/sdk/context/prompts/snapshots/soul__custom.txt +++ b/tests/sdk/context/prompts/snapshots/soul__custom.txt @@ -10,6 +10,7 @@ You are a tiny cat agent with toe beans. * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills diff --git a/tests/sdk/context/prompts/snapshots/soul__default.txt b/tests/sdk/context/prompts/snapshots/soul__default.txt index 3bcb0fd83a..220a72bbca 100644 --- a/tests/sdk/context/prompts/snapshots/soul__default.txt +++ b/tests/sdk/context/prompts/snapshots/soul__default.txt @@ -10,6 +10,7 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute * Use `AGENTS.md` under the repository root as your persistent memory for repository-specific knowledge and context. * Add important insights, patterns, and learnings to this file to improve future task performance. +* When asked to find a previous local OpenHands conversation, search the workspace's `workspace/conversations/` directory for its event history. * This repository skill is automatically loaded for every conversation and helps maintain context across sessions. * For more information about skills, see: https://docs.openhands.dev/overview/skills From 013fba46279888f6f0040cd3a6affb2831482cbd Mon Sep 17 00:00:00 2001 From: Graham Neubig Date: Fri, 21 Aug 2026 08:41:30 -0400 Subject: [PATCH 14/14] fix(sdk): resolve workspace default from active LLM profile (#4497) Co-authored-by: openhands Co-authored-by: neubig Co-authored-by: hieptl --- .../openhands/sdk/workspace/remote/base.py | 71 +++++---- .../test_remote_conversation_live_server.py | 79 ++++++++++ .../workspace/remote/test_remote_workspace.py | 135 ++++++++++++++++-- 3 files changed, 250 insertions(+), 35 deletions(-) diff --git a/openhands-sdk/openhands/sdk/workspace/remote/base.py b/openhands-sdk/openhands/sdk/workspace/remote/base.py index 8620fad286..b935e13456 100644 --- a/openhands-sdk/openhands/sdk/workspace/remote/base.py +++ b/openhands-sdk/openhands/sdk/workspace/remote/base.py @@ -321,27 +321,30 @@ def __exit__( # settings endpoints. Subclasses like OpenHandsCloudWorkspace may override # to use alternative endpoints (e.g., Cloud API). + def _fetch_settings_response( + self, *, expose_secrets: bool = True + ) -> SettingsResponse: + """Call ``GET /api/settings`` and return the validated response.""" + headers = dict(self._headers) + if expose_secrets: + headers["X-Expose-Secrets"] = "plaintext" + + response = self.client.get("/api/settings", headers=headers) + response.raise_for_status() + return SettingsResponse.model_validate(response.json()) + def _fetch_agent_settings( self, ) -> "OpenHandsAgentSettings | LLMAgentSettings | ACPAgentSettings": - """Call ``GET /api/settings`` and return a validated settings model. + """Return the validated agent settings from ``GET /api/settings``. Uses ``X-Expose-Secrets: plaintext`` so secret fields (e.g. LLM - api_key) are returned as plain strings. The outer response is - validated via :class:`SettingsResponse`, then the ``agent_settings`` - dict is validated through :meth:`SettingsResponse.get_agent_settings`, - which applies the persisted settings migration entry point before - picking the correct discriminated-union variant - (``OpenHandsAgentSettings`` or ``ACPAgentSettings``). + api_key) are returned as plain strings. The validated + ``SettingsResponse`` is narrowed through + :meth:`SettingsResponse.get_agent_settings`, which selects the correct + discriminated-union variant. """ - headers = dict(self._headers) - headers["X-Expose-Secrets"] = "plaintext" - - response = self.client.get("/api/settings", headers=headers) - response.raise_for_status() - - data = SettingsResponse.model_validate(response.json()) - return data.get_agent_settings() + return self._fetch_settings_response().get_agent_settings() def _fetch_llm_profile_config(self, profile_name: str) -> dict[str, Any]: """Call ``GET /api/profiles/{name}`` and return plaintext LLM config.""" @@ -368,16 +371,23 @@ def _fetch_llm_profile_config(self, profile_name: str) -> dict[str, Any]: reraise=True, ) def get_llm(self, profile_name: str | None = None, **llm_kwargs: Any) -> "LLM": - """Fetch LLM settings from persisted settings or a named profile. + """Fetch the active or explicitly named LLM profile. + + When no ``profile_name`` is given, the persisted ``active_profile`` + pointer is resolved first (so the UI-advertised default is honored). + If no ``active_profile`` is configured, the legacy + ``agent_settings.llm`` payload is used as a fallback (preserving + backward compatibility for servers that have not adopted named + profiles). Args: profile_name: Optional LLM profile name. When provided, loads that - named profile instead of the active persisted LLM settings. - **llm_kwargs: Additional keyword arguments that override persisted - or profile values (e.g., ``model``, ``temperature``). + named profile instead of resolving the active profile. + **llm_kwargs: Additional keyword arguments that override profile + values (e.g., ``model``, ``temperature``). Returns: - An LLM instance configured with the persisted settings or profile. + An LLM instance configured with the active or named profile. Raises: FileNotFoundError: If ``profile_name`` does not exist. @@ -394,14 +404,23 @@ def get_llm(self, profile_name: str | None = None, **llm_kwargs: Any) -> "LLM": if not self.host or self.host == "undefined": raise RuntimeError("Workspace host is not set") - if profile_name: + if profile_name is None: + settings_response = self._fetch_settings_response(expose_secrets=False) + resolved_profile_name = settings_response.active_profile + if resolved_profile_name in (None, ""): + settings_response = self._fetch_settings_response() + agent_settings = settings_response.get_agent_settings() + if not llm_kwargs: + return agent_settings.llm + llm_data = agent_settings.llm.model_dump( + context={"expose_secrets": "plaintext"} + ) + else: + llm_data = self._fetch_llm_profile_config(resolved_profile_name) + llm_data["usage_id"] = f"profile:{resolved_profile_name}" + else: llm_data = self._fetch_llm_profile_config(profile_name) llm_data["usage_id"] = f"profile:{profile_name}" - else: - settings = self._fetch_agent_settings() - if not llm_kwargs: - return settings.llm - llm_data = settings.llm.model_dump(context={"expose_secrets": "plaintext"}) llm_data.update(llm_kwargs) return LLM(**llm_data) diff --git a/tests/cross/test_remote_conversation_live_server.py b/tests/cross/test_remote_conversation_live_server.py index c8d370b26c..13db6842ca 100644 --- a/tests/cross/test_remote_conversation_live_server.py +++ b/tests/cross/test_remote_conversation_live_server.py @@ -164,6 +164,14 @@ def live_server_env( shutil.rmtree(cwd_conversations) +def _assert_secret(value: "str | SecretStr", expected: str) -> None: + """Assert a SecretStr-or-str api_key matches the expected plaintext.""" + if isinstance(value, SecretStr): + assert value.get_secret_value() == expected + else: + assert value == expected + + def test_health_endpoints_return_ok_json(server_env): with httpx.Client() as client: for endpoint in ("/alive", "/health"): @@ -2133,6 +2141,77 @@ async def fake_acompletion(self, messages, tools=None, **kwargs): # type: ignor conv.close() +def test_workspace_default_llm_resolves_active_profile_despite_settings_drift( + tmp_path, monkeypatch +): + """An unpinned automation gets the UI-advertised active named profile. + + Reproduces the production drift through real HTTP endpoints: activate GLM, + then patch only legacy agent_settings.llm to keyless GPT while leaving the + active pointer untouched. RemoteWorkspace.get_llm() must resolve GLM and its + named-profile credential. Explicit profile selection remains an override. + """ + with live_server_env(tmp_path, monkeypatch) as env: + with httpx.Client(base_url=env["host"], timeout=10.0) as client: + save_glm = client.post( + "/api/profiles/glm-default", + json={ + "llm": { + "model": "openhands/glm-5.2", + "api_key": "sk-glm-key", + }, + "include_secrets": True, + }, + ) + assert save_glm.status_code == 201 + assert client.post("/api/profiles/glm-default/activate").status_code == 200 + + save_explicit = client.post( + "/api/profiles/explicit-model", + json={ + "llm": { + "model": "openrouter/explicit-model", + "api_key": "sk-explicit-key", + }, + "include_secrets": True, + }, + ) + assert save_explicit.status_code == 201 + + drift = client.patch( + "/api/settings", + json={ + "agent_settings_diff": { + "llm": {"model": "gpt-5.5", "api_key": None} + } + }, + ) + assert drift.status_code == 200 + + profiles = client.get("/api/profiles").json() + settings = client.get("/api/settings").json() + assert profiles["active_profile"] == "glm-default" + assert settings["active_profile"] == "glm-default" + assert settings["agent_settings"]["llm"]["model"] == "gpt-5.5" + + workspace = RemoteWorkspace( + host=env["host"], + working_dir=str(env["workspace_path"]), + ) + default_llm = workspace.get_llm() + assert default_llm.model == "openhands/glm-5.2" + assert default_llm.api_key is not None + _assert_secret(default_llm.api_key, "sk-glm-key") + assert default_llm.usage_id == "profile:glm-default" + + # Mirrors an explicit AUTOMATION_MODEL/profile_name override. + explicit_llm = workspace.get_llm(profile_name="explicit-model") + assert explicit_llm.model == "openrouter/explicit-model" + assert explicit_llm.api_key is not None + _assert_secret(explicit_llm.api_key, "sk-explicit-key") + assert explicit_llm.usage_id == "profile:explicit-model" + + def test_settings_and_secrets_api_with_live_server(server_env): """End-to-end test for settings and secrets API endpoints. diff --git a/tests/sdk/workspace/remote/test_remote_workspace.py b/tests/sdk/workspace/remote/test_remote_workspace.py index aeaee2ef21..dec14c9dfa 100644 --- a/tests/sdk/workspace/remote/test_remote_workspace.py +++ b/tests/sdk/workspace/remote/test_remote_workspace.py @@ -431,9 +431,21 @@ def test_get_llm_returns_configured_llm(monkeypatch): }, "conversation_settings": {}, "llm_api_key_is_set": True, + "active_profile": "default", } mock_response.raise_for_status = Mock() - mock_client.get.return_value = mock_response + profile_response = Mock() + profile_response.status_code = 200 + profile_response.raise_for_status = Mock() + profile_response.json.return_value = { + "name": "default", + "config": { + "model": "gpt-4", + "api_key": "sk-test-key", + "base_url": "https://api.openai.com/v1", + }, + } + mock_client.get.side_effect = [mock_response, profile_response] workspace._client = mock_client llm = workspace.get_llm() @@ -448,12 +460,106 @@ def test_get_llm_returns_configured_llm(monkeypatch): assert llm.api_key == "sk-test-key" assert llm.base_url == "https://api.openai.com/v1" - # Verify API was called with correct headers - mock_client.get.assert_called_once() - call_args = mock_client.get.call_args - assert call_args[0][0] == "/api/settings" - assert call_args[1]["headers"]["X-Expose-Secrets"] == "plaintext" - assert call_args[1]["headers"]["X-Session-API-Key"] == "test-key" + assert [call.args[0] for call in mock_client.get.call_args_list] == [ + "/api/settings", + "/api/profiles/default", + ] + + +def test_get_llm_without_name_resolves_active_profile(monkeypatch): + """Default resolution honors active_profile, not stale agent settings.""" + from pydantic import SecretStr + + monkeypatch.setenv("ALLOW_SHORT_CONTEXT_WINDOWS", "true") + workspace = RemoteWorkspace( + host="http://localhost:8000", working_dir="/tmp", api_key="test-key" + ) + + settings_response = Mock() + settings_response.json.return_value = { + "agent_settings": { + "llm": {"model": "gpt-5.5", "api_key": None}, + }, + "conversation_settings": {}, + "llm_api_key_is_set": False, + "active_profile": "glm-default", + } + settings_response.raise_for_status = Mock() + profile_response = Mock() + profile_response.status_code = 200 + profile_response.json.return_value = { + "name": "glm-default", + "config": { + "model": "openhands/glm-5.2", + "api_key": "sk-glm-key", + "usage_id": "default", + }, + "api_key_set": True, + } + profile_response.raise_for_status = Mock() + + client = MagicMock() + client.get.side_effect = [settings_response, profile_response] + workspace._client = client + + llm = workspace.get_llm() + + assert llm.model == "openhands/glm-5.2" + assert isinstance(llm.api_key, SecretStr) + assert llm.api_key.get_secret_value() == "sk-glm-key" + assert llm.usage_id == "profile:glm-default" + assert [call.args[0] for call in client.get.call_args_list] == [ + "/api/settings", + "/api/profiles/glm-default", + ] + + +@pytest.mark.parametrize("active_profile", [None, ""]) +def test_get_llm_without_active_profile_falls_back_to_legacy( + monkeypatch, active_profile +): + """Falsy active_profile values use the legacy agent_settings.llm fallback.""" + from pydantic import SecretStr + + monkeypatch.setenv("ALLOW_SHORT_CONTEXT_WINDOWS", "true") + workspace = RemoteWorkspace( + host="http://localhost:8000", working_dir="/tmp", api_key="test-key" + ) + + settings_response = Mock() + settings_response.json.return_value = { + "agent_settings": { + "llm": {"model": "gpt-4", "api_key": "sk-legacy"}, + }, + "conversation_settings": {}, + "llm_api_key_is_set": True, + "active_profile": active_profile, + } + settings_response.raise_for_status = Mock() + + client = MagicMock() + client.get.return_value = settings_response + workspace._client = client + + llm = workspace.get_llm() + + assert llm.model == "gpt-4" + assert isinstance(llm.api_key, SecretStr) + assert llm.api_key.get_secret_value() == "sk-legacy" + + # Discovery avoids exposing legacy credentials; the fallback fetches them only + # when active_profile is absent. + assert [call.args[0] for call in client.get.call_args_list] == [ + "/api/settings", + "/api/settings", + ] + assert client.get.call_args_list[0].kwargs["headers"] == { + "X-Session-API-Key": "test-key" + } + assert client.get.call_args_list[1].kwargs["headers"] == { + "X-Session-API-Key": "test-key", + "X-Expose-Secrets": "plaintext", + } def test_get_llm_with_kwargs_override(monkeypatch): @@ -478,12 +584,23 @@ def test_get_llm_with_kwargs_override(monkeypatch): }, "conversation_settings": {}, "llm_api_key_is_set": True, + "active_profile": "default", } mock_response.raise_for_status = Mock() - mock_client.get.return_value = mock_response + profile_response = Mock() + profile_response.status_code = 200 + profile_response.raise_for_status = Mock() + profile_response.json.return_value = { + "name": "default", + "config": { + "model": "gpt-3.5-turbo", + "api_key": "sk-persisted-key", + }, + } + mock_client.get.side_effect = [mock_response, profile_response] workspace._client = mock_client - # Override model but use persisted API key + # Override model but use the active profile API key llm = workspace.get_llm(model="gpt-4o") assert llm.model == "gpt-4o" # Overridden