diff --git a/llm_router_host.py b/llm_router_host.py index f75a446..ff3fe27 100644 --- a/llm_router_host.py +++ b/llm_router_host.py @@ -16,6 +16,7 @@ from __future__ import annotations import asyncio +import contextvars import json import os import time @@ -87,6 +88,12 @@ _AUTH_UNCONFIGURED = "auth_unconfigured" # Host-side backstop for the engine's per-execution cap (core DEFAULTS). _MAX_PROVIDER_ATTEMPTS = 32 +# Provider attempts of the current request, recorded host-side. The engine's +# decision_path dies with a run the shim's outer deadline cancels; this list +# survives it, so a deadline error can still name the route it waited on. The +# shim sets a list per request; the default None records nothing. +attempt_log: contextvars.ContextVar[list | None] = contextvars.ContextVar( + "router_attempt_log", default=None) _SECRET_PLACEHOLDERS = {"", "CHANGE_ME", "TODO", "TODO_CHANGE_ME", "PLACEHOLDER"} @@ -687,7 +694,17 @@ async def execute_async(self, contract: dict, call_override=None) -> dict: if (contract.get("first_token_timeout_ms") is not None and req.get("first_token_timeout_ms") is None): req["first_token_timeout_ms"] = contract["first_token_timeout_ms"] + log = attempt_log.get() + entry = None + if log is not None and len(log) < _MAX_PROVIDER_ATTEMPTS: + entry = {"provider_id": req.get("provider_id"), + "model_family": req.get("model_family"), + "started": time.monotonic()} + log.append(entry) resp = await self._resolve_call_async(req, call_override, session=session) + if entry is not None: + entry.update({key: resp.get(key) for key in ( + "ok", "error_kind", "http_status", "error_message", "latency_ms")}) diagnostic = bounded_diagnostics(resp.get("diagnostics")) if diagnostic and len(provider_diagnostics) < 32: diagnostic.update(bounded_diagnostics({ diff --git a/shim.py b/shim.py index d1defa5..e3fb9ea 100644 --- a/shim.py +++ b/shim.py @@ -42,6 +42,7 @@ import control_plane_client from env_coerce import env_int +from llm_router_host import attempt_log import host_store from policy_templates import ( PolicyTemplateError, @@ -297,6 +298,8 @@ async def _execute_with_deadline(awaitable): so an expired request does not leave an orphan fallback chain running after the HTTP response has finished. """ + attempts: list = [] + log_token = attempt_log.set(attempts) deadline = asyncio.timeout(request_deadline_s) try: async with deadline: @@ -306,21 +309,29 @@ async def _execute_with_deadline(awaitable): # timeout context's own expiry is the outer request deadline. if not deadline.expired(): raise + decision_path = _deadline_decision_path(attempts, request_deadline_ms) + waiting_on = [ + f"{e.get('provider_id') or '?'}/{e.get('model_family') or '?'}" + for e in decision_path if e.get("in_flight") + ] _log.warning( - "router request deadline exceeded after %d ms", - request_deadline_ms, + "router request deadline exceeded after %d ms (%d attempts, waiting on %s)", + request_deadline_ms, len(decision_path), + ", ".join(waiting_on) or "no provider", ) return { "ok": False, "error": "timeout", "trace": { - "decision_path": [], + "decision_path": decision_path, "request_deadline_exceeded": True, "request_deadline_ms": request_deadline_ms, "total_latency_ms": request_deadline_ms, **automatic_trace(), }, } + finally: + attempt_log.reset(log_token) from saas_routes import ScopedHost, automatic_trace, install as install_saas host = ScopedHost(host) @@ -2067,6 +2078,37 @@ def _attempt_summary(trace: dict) -> str | None: return "; ".join(parts) or None +def _deadline_decision_path(attempts: list, deadline_ms: int) -> list: + """Rebuild decision_path for a run the outer request deadline cancelled, + from the host's attempt log. An attempt still waiting at the deadline is + reported as an in-flight timeout, so the error names the route that hung.""" + now = time.monotonic() + path: list[dict] = [] + for number, attempt in enumerate(attempts, 1): + event = { + "event": "attempted", + "provider_id": attempt.get("provider_id"), + "model_family": attempt.get("model_family"), + "attempt": number, + } + if "ok" not in attempt: + event.update({ + "error_kind": "timeout", + "in_flight": True, + "latency_ms": int((now - attempt["started"]) * 1000), + "error_message": f"no response before the {deadline_ms} ms request deadline", + }) + else: + event["latency_ms"] = attempt.get("latency_ms") + if not attempt.get("ok"): + event["error_kind"] = attempt.get("error_kind") or "unknown" + event["http_status"] = attempt.get("http_status") + if attempt.get("error_message") is not None: + event["error_message"] = str(attempt["error_message"])[:300] + path.append({k: v for k, v in event.items() if v is not None}) + return path + + def _openai_error_from_router(result: dict) -> JSONResponse: error_kind = str(result.get("error") or "unknown") @@ -2087,18 +2129,27 @@ def _openai_error_from_router(result: dict) -> JSONResponse: elif kind in ("bad_request", "context_overflow"): status = 400 elif kind == "timeout": - status = 504 # upstream (or a flow node's seller) timed out — Gateway Timeout + # Upstream (or a flow node's seller) timed out, or the request deadline + # expired. Not 504: Cloudflare in front of the public router replaces an + # origin 502/504 body with its own plain-text page, and the caller loses + # this JSON error. error.code still carries "timeout". + status = 503 else: status = 502 + trace = result.get("trace") or {} # router's error strings already start with "exhausted: " when candidates # were tried — don't double-prefix. - message = error_kind if error_kind.startswith("exhausted:") else f"router: {error_kind}" + if trace.get("request_deadline_exceeded"): + message = f"router: request deadline of {trace.get('request_deadline_ms')} ms exceeded" + elif error_kind.startswith("exhausted:"): + message = error_kind + else: + message = f"router: {error_kind}" # Surface the per-attempt detail: append a provider-by-provider summary to # the message and ship the full trace in x_router (same field the ingress # proxy already extracts and the dashboard stores as decision_trace). - trace = result.get("trace") or {} summary = _attempt_summary(trace) if summary: message = f"{message} — {summary}" diff --git a/tests/test_flow.py b/tests/test_flow.py index 148a725..806a0c8 100644 --- a/tests/test_flow.py +++ b/tests/test_flow.py @@ -299,9 +299,10 @@ async def backend(req): assert "HAS_SECRET" in r.json()["choices"][0]["message"]["content"] -def test_flow_node_timeout_maps_to_504_not_502(): - """A flow node whose seller times out must surface a clean 504, not the - generic 502 catch-all (the observed flow 502 was a node timeout).""" +def test_flow_node_timeout_maps_to_503_not_502(): + """A flow node whose seller times out must surface a clean timeout (503, + since Cloudflare rewrites origin 502/504 bodies), not the generic 502 + catch-all (the observed flow 502 was a node timeout).""" async def backend(req): sys_txt = " ".join(m.get("content") or "" for m in (req.get("messages") or []) if m.get("role") == "system") @@ -315,7 +316,8 @@ async def backend(req): "model": "", "flow_ir": MOA(), "messages": [{"role": "user", "content": "hi"}], }) - assert r.status_code == 504, r.text + assert r.status_code == 503, r.text + assert r.json()["error"]["code"].endswith("timeout"), r.text def test_flow_with_multi_turn_conversation_succeeds(): diff --git a/tests/test_provider_diagnostics.py b/tests/test_provider_diagnostics.py index 4cc3a7e..450edfa 100644 --- a/tests/test_provider_diagnostics.py +++ b/tests/test_provider_diagnostics.py @@ -160,7 +160,7 @@ async def call(req): host.set_async_call_hook(call) response = TestClient(create_app(host)).post("/v1/chat/completions", json={ "policy_ir": _PIN, "messages": [{"role": "user", "content": "fixture"}]}) - assert response.status_code == (504 if failed else 200) + assert response.status_code == (503 if failed else 200) data = response.json() d = data["x_router"]["decision_trace"]["provider_diagnostics"][0] assert d["upstream_id"] == "gen-proof" and d["attempt"] == 1 diff --git a/tests/test_shim.py b/tests/test_shim.py index 3b3b7f6..49811f8 100644 --- a/tests/test_shim.py +++ b/tests/test_shim.py @@ -76,7 +76,7 @@ def test_healthz(client): assert r.json() == {"ok": True, "initialized": True} -def test_request_deadline_returns_504_and_cancels_host(host, monkeypatch): +def test_request_deadline_returns_503_and_cancels_host(host, monkeypatch): import asyncio cancelled = [] @@ -94,15 +94,63 @@ async def _never_finishes(contract, call_override=None): r = deadline_client.post("/v1/chat/completions", json={ "model": "", "messages": [{"role": "user", "content": "hi"}]}) - assert r.status_code == 504 + # Not 504: Cloudflare replaces an origin 502/504 body with plain text. + assert r.status_code == 503 body = r.json() assert body["error"]["code"] == "timeout" + assert body["error"]["message"] == "router: request deadline of 10 ms exceeded" trace = body["x_router"]["decision_trace"] assert trace["request_deadline_exceeded"] is True assert trace["request_deadline_ms"] == 10 + assert trace["decision_path"] == [] assert cancelled == [True] +def test_request_deadline_names_the_route_it_was_waiting_on(): + import asyncio + + host = LLMRouterHost( + router_path=ROOT / "core" / "router.lua", + config_path=ROOT / "core" / "config.example.lua", + metrics_path=ROOT / "core" / "metrics.example.lua", + now_ms=lambda: 1_000_000, + enforce_provider_auth=False, + ) + host.init() + calls = [] + + async def call(req): + calls.append(req) + if len(calls) == 1: + return {"ok": False, "error_kind": "timeout", "http_status": 504, + "latency_ms": 1, "error_message": "upstream timed out"} + await asyncio.sleep(60) + + host.set_async_call_hook(call) + deadline_client = TestClient(create_app( + host, default_profile="default", request_deadline_ms=1_000)) + + r = deadline_client.post("/v1/chat/completions", json={ + "model": "", "messages": [{"role": "user", "content": "hi"}]}) + + assert r.status_code == 503 + assert len(calls) == 2 + body = r.json() + path = body["x_router"]["decision_trace"]["decision_path"] + assert [e["attempt"] for e in path] == [1, 2] + failed, hung = path + assert failed["provider_id"] == calls[0]["provider_id"] + assert failed["error_kind"] == "timeout" + assert failed["http_status"] == 504 + assert "in_flight" not in failed + assert hung["provider_id"] == calls[1]["provider_id"] + assert hung["model_family"] == calls[1]["model_family"] + assert hung["error_kind"] == "timeout" and hung["in_flight"] is True + message = body["error"]["message"] + assert message.startswith("router: request deadline of 1000 ms exceeded — ") + assert f"{calls[1]['provider_id']}/{calls[1]['model_family']}=timeout" in message + + # ---- wallet control endpoints ------------------------------------------ def test_wallet_deposit_503_without_control(client, monkeypatch):