diff --git a/src/cgis/guardian/providers/base.py b/src/cgis/guardian/providers/base.py index b9156cc0..3bd8d1af 100644 --- a/src/cgis/guardian/providers/base.py +++ b/src/cgis/guardian/providers/base.py @@ -130,18 +130,28 @@ async def _sleep(self, seconds: float) -> None: """Wait between retries. Overridden in tests so they do not actually wait.""" await asyncio.sleep(seconds) + def _retryable_exceptions(self) -> tuple[type[Exception], ...]: + """The exception types `_retry` treats as transient. + + A provider whose gateway signals a retryable failure with an HTTP status + — opencode-go's 400 `inference_failed` — widens this instead of changing + the loop, so the base set stays the transport failures every provider + shares and every other 4xx still fails on the first attempt. + """ + return RETRYABLE_EXCEPTIONS + async def _retry(self, call: Callable[[], Awaitable[str]]) -> str: - """Run call, retrying transient transport failures with exponential backoff. + """Run call, retrying transient failures with exponential backoff. - Retries only RETRYABLE_EXCEPTIONS: an auth or validation failure must - fail on the first attempt rather than burn MAX_ATTEMPTS calls. The final - exception propagates unchanged so callers' existing degradation paths - still see a real error (#275). + Retries only `_retryable_exceptions()`: an auth or validation failure + must fail on the first attempt rather than burn MAX_ATTEMPTS calls. The + final exception propagates unchanged so callers' existing degradation + paths still see a real error (#275). """ for attempt in range(1, MAX_ATTEMPTS + 1): try: return await call() - except RETRYABLE_EXCEPTIONS as exc: + except self._retryable_exceptions() as exc: if attempt == MAX_ATTEMPTS: log.warning( "Provider call failed; retries exhausted.", diff --git a/src/cgis/guardian/providers/opencode.py b/src/cgis/guardian/providers/opencode.py index 30eb09cd..b1d01f4a 100644 --- a/src/cgis/guardian/providers/opencode.py +++ b/src/cgis/guardian/providers/opencode.py @@ -25,13 +25,14 @@ because `…/v1/` + `/chat/completions` would otherwise double the separator. """ +import json import uuid from typing import Any, ClassVar, Final, override import httpx from cgis import __version__ -from cgis.guardian.providers.base import ProviderUsage +from cgis.guardian.providers.base import RETRYABLE_EXCEPTIONS, ProviderUsage from cgis.guardian.providers.openai_compatible import OpenAICompatibleProvider #: The model this provider exists for: the finder's default, priced at @@ -60,9 +61,21 @@ #: gateway, where `raise_for_status` keeps the 4xx message. DEFAULT_OPENCODE_EFFORT: Final = "low" +_BAD_REQUEST = 400 _TOO_MANY_REQUESTS = 429 +class OpenCodeTransientError(RuntimeError): + """A 400 the gateway marks as an upstream failure it could not complete (#779). + + `inference_failed` is not a request-validation error: the same request + succeeds on the next attempt when the gateway retries it itself, and it does + so only sometimes (the console log shows `attemptCount` 1 on the runs that + failed and 2 on one that succeeded). Retried through `_retry`, never + returned and never degraded to "no judgement". + """ + + class OpenCodeQuotaError(RuntimeError): """Raised on HTTP 429 — the gateway's quota for this key is spent. @@ -82,6 +95,28 @@ class OpenCodeReplyError(RuntimeError): """ +def _is_transient_upstream_failure(body_text: str) -> bool: + """Whether the gateway marked this failure as a transient upstream error. + + The code is a field in the gateway's JSON error, so a JSON body is read as + one: a validation message that merely mentions `inference_failed` is not a + transient failure and must not be retried. A non-JSON body (a proxy page, an + empty error) is the fallback the raw text can still answer, and there a false + positive costs only the retries this file already allows — never a verdict. + """ + try: + parsed: object = json.loads(body_text) + except ValueError: + return "inference_failed" in body_text + if not isinstance(parsed, dict): + return False + error = parsed.get("error") + candidates: list[object] = [parsed.get("code"), parsed.get("errorCode"), error] + if isinstance(error, dict): + candidates += [error.get("code"), error.get("type")] + return any(candidate == "inference_failed" for candidate in candidates) + + class OpenCodeProvider(OpenAICompatibleProvider): """A DeepSeek model reached through opencode-go's OpenAI-compatible API.""" @@ -122,6 +157,17 @@ def effort(self) -> str: """The reasoning depth sent on every call, for the run record (#713).""" return self._effort + @override + def _retryable_exceptions(self) -> tuple[type[Exception], ...]: + """Retry the gateway's transient 400 beside the transport failures. + + `inference_failed` is an upstream failure, not a request-validation + error: the same request succeeds on the next attempt when the gateway + retries it itself (#779). Every other 4xx still fails on the first + attempt, as the base class documents. + """ + return (*RETRYABLE_EXCEPTIONS, OpenCodeTransientError) + async def _post(self, payload: dict[str, Any]) -> str: """One completion call; return the message content. @@ -143,19 +189,8 @@ async def _post(self, payload: dict[str, Any]) -> str: response = await client.post( f"{self._base_url}/chat/completions", headers=headers, json=payload ) - if response.status_code == _TOO_MANY_REQUESTS: - # The body is carried, not summarised: it says whether the limit is - # per-minute or the account's allowance, and those have different - # remedies — wait, or stop. Dropping it would leave the operator - # guessing, the same defect as an exception that swallows stderr. - _msg = ( - f"OpenCode returned 429 for {self._model_name}: " - f"{response.text.strip()[:400]}. A run that continued would " - f"record unanswered findings, which is indistinguishable from a " - f"skeptic that refutes nothing." - ) - raise OpenCodeQuotaError(_msg) - response.raise_for_status() + if response.is_error: + self._raise_for_http_error(response) body = response.json() # An error object with HTTP 200 is how some gateways report an upstream # refusal; `body["choices"]` would raise KeyError with nothing in the @@ -196,6 +231,31 @@ async def _post(self, payload: dict[str, Any]) -> str: raise OpenCodeReplyError(_msg) return content + def _raise_for_http_error(self, response: httpx.Response) -> None: + """Raise for an HTTP failure, carrying the gateway's body. + + `429` keeps its own quota error: it must never be swallowed into "the + finder had nothing to say". A `400` marked `inference_failed` is a + transient upstream failure and is retried; every other error fails at + once with the status and body, because dropping the body is what made + #779 read as a bare "400 Bad Request". + """ + body_text = response.text.strip() + if response.status_code == _TOO_MANY_REQUESTS: + _msg = ( + f"OpenCode returned 429 for {self._model_name}: " + f"{body_text[:400]}. A run that continued would record unanswered " + f"findings, which is indistinguishable from a skeptic that refutes nothing." + ) + raise OpenCodeQuotaError(_msg) + _msg = ( + f"OpenCode returned HTTP {response.status_code} for {self._model_name}: " + f"{body_text[:400] or '(empty body)'}" + ) + if response.status_code == _BAD_REQUEST and _is_transient_upstream_failure(body_text): + raise OpenCodeTransientError(_msg) + raise RuntimeError(_msg) + @override def _payload(self, system_prompt: str, user_prompt: str) -> dict[str, Any]: """The request body; this gateway takes reasoning as an effort level. diff --git a/tests/unit/test_guardian_opencode.py b/tests/unit/test_guardian_opencode.py index 977f4fda..d65d3f01 100644 --- a/tests/unit/test_guardian_opencode.py +++ b/tests/unit/test_guardian_opencode.py @@ -17,6 +17,7 @@ from pydantic import BaseModel from cgis.guardian.providers import opencode as mod +from cgis.guardian.providers.base import MAX_ATTEMPTS from cgis.guardian.providers.opencode import ( DEFAULT_MAX_TOKENS, DEFAULT_OPENCODE_BASE_URL, @@ -26,6 +27,7 @@ OpenCodeProvider, OpenCodeQuotaError, OpenCodeReplyError, + OpenCodeTransientError, ) _ENDPOINT = f"{DEFAULT_OPENCODE_BASE_URL}/chat/completions" @@ -55,14 +57,19 @@ def _reply( return httpx.Response(status, json=payload, request=httpx.Request("POST", _ENDPOINT)) -def _serve(monkeypatch: pytest.MonkeyPatch, response: httpx.Response) -> dict[str, list[Any]]: - """Route every POST to `response`; return the URLs, headers, bodies and timeouts sent.""" +def _serve( + monkeypatch: pytest.MonkeyPatch, + response: httpx.Response | list[httpx.Response], +) -> dict[str, list[Any]]: + """Route POSTs to `response` (or the next of a list); return what was sent.""" captured: dict[str, list[Any]] = { "urls": [], "headers": [], "bodies": [], "timeouts": [], } + single: httpx.Response | None = response if isinstance(response, httpx.Response) else None + queue: list[httpx.Response] = [] if single is not None else list(response) class _Client: def __init__(self, **kw: object) -> None: @@ -78,12 +85,28 @@ async def post(self, url: str, **kwargs: object) -> httpx.Response: captured["urls"].append(url) captured["headers"].append(kwargs["headers"]) captured["bodies"].append(kwargs["json"]) - return response + if single is not None: + return single + return queue.pop(0) monkeypatch.setattr(mod.httpx, "AsyncClient", _Client) return captured +class _NoSleepProvider(OpenCodeProvider): + """A provider that records retry backoff instead of waiting through it.""" + + slept: list[float] + + def __init__(self, api_key: str, model_name: str = DEFAULT_OPENCODE_MODEL) -> None: + super().__init__(api_key=api_key, model_name=model_name) + self.slept = [] + + async def _sleep(self, seconds: float) -> None: + """Record the backoff interval.""" + self.slept.append(seconds) + + def _provider(**overrides: object) -> OpenCodeProvider: settings: dict[str, Any] = {"api_key": "k", "model_name": DEFAULT_OPENCODE_MODEL} settings.update(overrides) @@ -104,14 +127,100 @@ async def test_a_spent_quota_raises_rather_than_returning_nothing( await provider.generate_content("s", "u") @pytest.mark.asyncio - async def test_any_other_http_error_is_raised_not_degraded( + async def test_any_other_http_error_is_raised_with_its_body_on_the_first_attempt( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """An auth, model or effort mistake fails loudly, first time, with the gateway's text.""" + captured = _serve( + monkeypatch, _reply(status=400, body={"error": {"message": "bad effort"}}) + ) + provider = _provider() + with pytest.raises(RuntimeError, match=r"HTTP 400.*bad effort"): + await provider.generate_content("s", "u") + assert len(captured["urls"]) == 1 + + @pytest.mark.asyncio + async def test_a_server_error_carries_its_status_and_body( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + captured = _serve( + monkeypatch, _reply(status=502, body={"error": {"message": "upstream down"}}) + ) + provider = _provider() + with pytest.raises(RuntimeError, match=r"HTTP 502.*upstream down"): + await provider.generate_content("s", "u") + assert len(captured["urls"]) == 1 + + @pytest.mark.asyncio + async def test_a_transient_inference_failed_400_is_retried_until_it_succeeds( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The gateway marks a transient upstream failure as `400 inference_failed`. + + It retries that itself only sometimes; the console log shows attemptCount + 1 on the runs that failed and 2 on one that succeeded (#779). + """ + captured = _serve( + monkeypatch, + [ + _reply(status=400, body={"error": {"code": "inference_failed"}}), + _reply(), + ], + ) + provider = _NoSleepProvider(api_key="k", model_name=DEFAULT_OPENCODE_MODEL) + content = await provider.generate_content("s", "u") + assert content.startswith("{") + assert len(captured["urls"]) == 2 + assert provider.slept == [2.0] + + @pytest.mark.asyncio + async def test_a_transient_400_that_never_clears_raises_after_the_attempts( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + captured = _serve( + monkeypatch, + [_reply(status=400, body={"error": {"code": "inference_failed"}})] * MAX_ATTEMPTS, + ) + provider = _NoSleepProvider(api_key="k", model_name=DEFAULT_OPENCODE_MODEL) + with pytest.raises(OpenCodeTransientError, match="inference_failed"): + await provider.generate_content("s", "u") + assert len(captured["urls"]) == MAX_ATTEMPTS + assert provider.slept == [2.0, 4.0] + + @pytest.mark.asyncio + async def test_a_mention_of_the_code_is_not_a_gateway_code( self, monkeypatch: pytest.MonkeyPatch ) -> None: - """An auth, model or effort mistake fails loudly, with the gateway's message.""" - _serve(monkeypatch, _reply(status=400, body={"error": {"message": "bad effort"}})) + """A validation message that merely names `inference_failed` is not retried.""" + captured = _serve( + monkeypatch, + _reply(status=400, body={"error": {"message": "previous inference_failed attempt"}}), + ) provider = _provider() - with pytest.raises(httpx.HTTPStatusError): + with pytest.raises(RuntimeError, match="previous inference_failed attempt"): await provider.generate_content("s", "u") + assert len(captured["urls"]) == 1 + + @pytest.mark.asyncio + async def test_a_non_json_body_with_the_code_is_still_retried( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A proxy page that carries the code must not turn a transient failure fatal.""" + _serve( + monkeypatch, + [ + httpx.Response( + 400, + text="inference_failed", + request=httpx.Request("POST", _ENDPOINT), + ), + _reply(), + ], + ) + provider = _NoSleepProvider(api_key="k", model_name=DEFAULT_OPENCODE_MODEL) + content = await provider.generate_content("s", "u") + assert content.startswith("{") + assert provider.slept == [2.0] @pytest.mark.asyncio async def test_hitting_the_token_budget_raises_rather_than_returning_reasoning(