Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions llm_router_host.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from __future__ import annotations

import asyncio
import contextvars
import json
import os
import time
Expand Down Expand Up @@ -87,6 +88,12 @@
_AUTH_UNCONFIGURED = "auth_unconfigured"
# Host-side backstop for the engine's per-execution cap (core DEFAULTS).
_MAX_PROVIDER_ATTEMPTS = 32
# Provider attempts of the current request, recorded host-side. The engine's
# decision_path dies with a run the shim's outer deadline cancels; this list
# survives it, so a deadline error can still name the route it waited on. The
# shim sets a list per request; the default None records nothing.
attempt_log: contextvars.ContextVar[list | None] = contextvars.ContextVar(
"router_attempt_log", default=None)
_SECRET_PLACEHOLDERS = {"", "CHANGE_ME", "TODO", "TODO_CHANGE_ME", "PLACEHOLDER"}


Expand Down Expand Up @@ -687,7 +694,17 @@ async def execute_async(self, contract: dict, call_override=None) -> dict:
if (contract.get("first_token_timeout_ms") is not None
and req.get("first_token_timeout_ms") is None):
req["first_token_timeout_ms"] = contract["first_token_timeout_ms"]
log = attempt_log.get()
entry = None
if log is not None and len(log) < _MAX_PROVIDER_ATTEMPTS:
entry = {"provider_id": req.get("provider_id"),
"model_family": req.get("model_family"),
"started": time.monotonic()}
log.append(entry)
resp = await self._resolve_call_async(req, call_override, session=session)
if entry is not None:
entry.update({key: resp.get(key) for key in (
"ok", "error_kind", "http_status", "error_message", "latency_ms")})
diagnostic = bounded_diagnostics(resp.get("diagnostics"))
if diagnostic and len(provider_diagnostics) < 32:
diagnostic.update(bounded_diagnostics({
Expand Down
63 changes: 57 additions & 6 deletions shim.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@

import control_plane_client
from env_coerce import env_int
from llm_router_host import attempt_log
import host_store
from policy_templates import (
PolicyTemplateError,
Expand Down Expand Up @@ -297,6 +298,8 @@ async def _execute_with_deadline(awaitable):
so an expired request does not leave an orphan fallback chain running
after the HTTP response has finished.
"""
attempts: list = []
log_token = attempt_log.set(attempts)
deadline = asyncio.timeout(request_deadline_s)
try:
async with deadline:
Expand All @@ -306,21 +309,29 @@ async def _execute_with_deadline(awaitable):
# timeout context's own expiry is the outer request deadline.
if not deadline.expired():
raise
decision_path = _deadline_decision_path(attempts, request_deadline_ms)
waiting_on = [
f"{e.get('provider_id') or '?'}/{e.get('model_family') or '?'}"
for e in decision_path if e.get("in_flight")
]
_log.warning(
"router request deadline exceeded after %d ms",
request_deadline_ms,
"router request deadline exceeded after %d ms (%d attempts, waiting on %s)",
request_deadline_ms, len(decision_path),
", ".join(waiting_on) or "no provider",
)
return {
"ok": False,
"error": "timeout",
"trace": {
"decision_path": [],
"decision_path": decision_path,
"request_deadline_exceeded": True,
"request_deadline_ms": request_deadline_ms,
"total_latency_ms": request_deadline_ms,
**automatic_trace(),
},
}
finally:
attempt_log.reset(log_token)

from saas_routes import ScopedHost, automatic_trace, install as install_saas
host = ScopedHost(host)
Expand Down Expand Up @@ -2067,6 +2078,37 @@ def _attempt_summary(trace: dict) -> str | None:
return "; ".join(parts) or None


def _deadline_decision_path(attempts: list, deadline_ms: int) -> list:
"""Rebuild decision_path for a run the outer request deadline cancelled,
from the host's attempt log. An attempt still waiting at the deadline is
reported as an in-flight timeout, so the error names the route that hung."""
now = time.monotonic()
path: list[dict] = []
for number, attempt in enumerate(attempts, 1):
event = {
"event": "attempted",
"provider_id": attempt.get("provider_id"),
"model_family": attempt.get("model_family"),
"attempt": number,
}
if "ok" not in attempt:
event.update({
"error_kind": "timeout",
"in_flight": True,
"latency_ms": int((now - attempt["started"]) * 1000),
"error_message": f"no response before the {deadline_ms} ms request deadline",
})
else:
event["latency_ms"] = attempt.get("latency_ms")
if not attempt.get("ok"):
event["error_kind"] = attempt.get("error_kind") or "unknown"
event["http_status"] = attempt.get("http_status")
if attempt.get("error_message") is not None:
event["error_message"] = str(attempt["error_message"])[:300]
path.append({k: v for k, v in event.items() if v is not None})
return path


def _openai_error_from_router(result: dict) -> JSONResponse:
error_kind = str(result.get("error") or "unknown")

Expand All @@ -2087,18 +2129,27 @@ def _openai_error_from_router(result: dict) -> JSONResponse:
elif kind in ("bad_request", "context_overflow"):
status = 400
elif kind == "timeout":
status = 504 # upstream (or a flow node's seller) timed out — Gateway Timeout
# Upstream (or a flow node's seller) timed out, or the request deadline
# expired. Not 504: Cloudflare in front of the public router replaces an
# origin 502/504 body with its own plain-text page, and the caller loses
# this JSON error. error.code still carries "timeout".
status = 503
else:
status = 502

trace = result.get("trace") or {}
# router's error strings already start with "exhausted: " when candidates
# were tried — don't double-prefix.
message = error_kind if error_kind.startswith("exhausted:") else f"router: {error_kind}"
if trace.get("request_deadline_exceeded"):
message = f"router: request deadline of {trace.get('request_deadline_ms')} ms exceeded"
elif error_kind.startswith("exhausted:"):
message = error_kind
else:
message = f"router: {error_kind}"

# Surface the per-attempt detail: append a provider-by-provider summary to
# the message and ship the full trace in x_router (same field the ingress
# proxy already extracts and the dashboard stores as decision_trace).
trace = result.get("trace") or {}
summary = _attempt_summary(trace)
if summary:
message = f"{message} — {summary}"
Expand Down
10 changes: 6 additions & 4 deletions tests/test_flow.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,9 +299,10 @@ async def backend(req):
assert "HAS_SECRET" in r.json()["choices"][0]["message"]["content"]


def test_flow_node_timeout_maps_to_504_not_502():
"""A flow node whose seller times out must surface a clean 504, not the
generic 502 catch-all (the observed flow 502 was a node timeout)."""
def test_flow_node_timeout_maps_to_503_not_502():
"""A flow node whose seller times out must surface a clean timeout (503,
since Cloudflare rewrites origin 502/504 bodies), not the generic 502
catch-all (the observed flow 502 was a node timeout)."""
async def backend(req):
sys_txt = " ".join(m.get("content") or "" for m in (req.get("messages") or [])
if m.get("role") == "system")
Expand All @@ -315,7 +316,8 @@ async def backend(req):
"model": "", "flow_ir": MOA(),
"messages": [{"role": "user", "content": "hi"}],
})
assert r.status_code == 504, r.text
assert r.status_code == 503, r.text
assert r.json()["error"]["code"].endswith("timeout"), r.text


def test_flow_with_multi_turn_conversation_succeeds():
Expand Down
2 changes: 1 addition & 1 deletion tests/test_provider_diagnostics.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ async def call(req):
host.set_async_call_hook(call)
response = TestClient(create_app(host)).post("/v1/chat/completions", json={
"policy_ir": _PIN, "messages": [{"role": "user", "content": "fixture"}]})
assert response.status_code == (504 if failed else 200)
assert response.status_code == (503 if failed else 200)
data = response.json()
d = data["x_router"]["decision_trace"]["provider_diagnostics"][0]
assert d["upstream_id"] == "gen-proof" and d["attempt"] == 1
Expand Down
52 changes: 50 additions & 2 deletions tests/test_shim.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def test_healthz(client):
assert r.json() == {"ok": True, "initialized": True}


def test_request_deadline_returns_504_and_cancels_host(host, monkeypatch):
def test_request_deadline_returns_503_and_cancels_host(host, monkeypatch):
import asyncio

cancelled = []
Expand All @@ -94,15 +94,63 @@ async def _never_finishes(contract, call_override=None):
r = deadline_client.post("/v1/chat/completions", json={
"model": "", "messages": [{"role": "user", "content": "hi"}]})

assert r.status_code == 504
# Not 504: Cloudflare replaces an origin 502/504 body with plain text.
assert r.status_code == 503
body = r.json()
assert body["error"]["code"] == "timeout"
assert body["error"]["message"] == "router: request deadline of 10 ms exceeded"
trace = body["x_router"]["decision_trace"]
assert trace["request_deadline_exceeded"] is True
assert trace["request_deadline_ms"] == 10
assert trace["decision_path"] == []
assert cancelled == [True]


def test_request_deadline_names_the_route_it_was_waiting_on():
import asyncio

host = LLMRouterHost(
router_path=ROOT / "core" / "router.lua",
config_path=ROOT / "core" / "config.example.lua",
metrics_path=ROOT / "core" / "metrics.example.lua",
now_ms=lambda: 1_000_000,
enforce_provider_auth=False,
)
host.init()
calls = []

async def call(req):
calls.append(req)
if len(calls) == 1:
return {"ok": False, "error_kind": "timeout", "http_status": 504,
"latency_ms": 1, "error_message": "upstream timed out"}
await asyncio.sleep(60)

host.set_async_call_hook(call)
deadline_client = TestClient(create_app(
host, default_profile="default", request_deadline_ms=1_000))

r = deadline_client.post("/v1/chat/completions", json={
"model": "", "messages": [{"role": "user", "content": "hi"}]})

assert r.status_code == 503
assert len(calls) == 2
body = r.json()
path = body["x_router"]["decision_trace"]["decision_path"]
assert [e["attempt"] for e in path] == [1, 2]
failed, hung = path
assert failed["provider_id"] == calls[0]["provider_id"]
assert failed["error_kind"] == "timeout"
assert failed["http_status"] == 504
assert "in_flight" not in failed
assert hung["provider_id"] == calls[1]["provider_id"]
assert hung["model_family"] == calls[1]["model_family"]
assert hung["error_kind"] == "timeout" and hung["in_flight"] is True
message = body["error"]["message"]
assert message.startswith("router: request deadline of 1000 ms exceeded — ")
assert f"{calls[1]['provider_id']}/{calls[1]['model_family']}=timeout" in message


# ---- wallet control endpoints ------------------------------------------

def test_wallet_deposit_503_without_control(client, monkeypatch):
Expand Down
Loading