From acbac6132ba228b8233b3127b764fe5ffc26bfa1 Mon Sep 17 00:00:00 2001 From: Rafal Rabenda Date: Wed, 28 Oct 2026 15:15:19 +0100 Subject: [PATCH] Bound first output on the default profile so fallback fits the deadline The default profile set no attempt bounds, so every attempt got the engine's 30 s timeout_ms. A route that never answered used 30 s of the 50 s request deadline, and the next candidate had only 20 s before the caller received a timeout. Set first_token_timeout_ms to 15 s on the default profile, matching how the agent profile bounds attempts. A silent route now falls through after 15 s and the next candidate keeps its full 30 s budget. A slow generation that is already streaming is not affected. --- config.live.lua | 8 ++++++- policy_templates.py | 14 +++++++++--- tests/test_policy_templates.py | 42 ++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/config.live.lua b/config.live.lua index 8c07ad7..1e86aaf 100644 --- a/config.live.lua +++ b/config.live.lua @@ -45,6 +45,12 @@ local DEFAULT_MAX_PRICE_OUT = 25.0 local DEFAULT_QUALITY_TOP_N = 5 local DEFAULT_COST_WEIGHT = 0.75 local DEFAULT_INTELLIGENCE_WEIGHT = 0.25 +-- A route that sends nothing for this long falls through to the next +-- candidate. Only the wait for the first output is bounded: a slow generation +-- that is already streaming keeps the engine's 30 s attempt budget, while a +-- silent route no longer leaves the next candidate too little of the 50 s +-- request deadline to answer before the caller gets a timeout. +local DEFAULT_FIRST_TOKEN_TIMEOUT_MS = 15000 -- Quality-first defaults for long-running tool agents. Unlike the vanilla -- chat policy, each provider attempt is bounded inside the policy itself so a @@ -188,7 +194,7 @@ local function default_policy_ir() }, }, selector, - { "id" }, + { "set_param", "first_token_timeout_ms", DEFAULT_FIRST_TOKEN_TIMEOUT_MS }, fail_plan(BALANCED_RETRY), } end diff --git a/policy_templates.py b/policy_templates.py index 314453c..ff809ea 100644 --- a/policy_templates.py +++ b/policy_templates.py @@ -40,6 +40,10 @@ class PolicyTemplateError(ValueError): DEFAULT_QUALITY_TOP_N = 5 DEFAULT_COST_WEIGHT = 0.75 DEFAULT_INTELLIGENCE_WEIGHT = 0.25 +# Bound only the wait for first output, so a silent route falls through while +# the request deadline still has room for the fallback cascade. Generations +# that are already streaming keep the full attempt budget. +DEFAULT_FIRST_TOKEN_TIMEOUT_MS = 15_000 # Long-running tool agents need a different posture from vanilla chat. Keep # the quality floor hard, put subscription/direct routes ahead of gateways and @@ -214,8 +218,9 @@ def template_catalog() -> list[dict[str, Any]]: "description": ( "Safe policy used by OpenAI-compatible callers that send no " "policy: Codex, AntSeed, Bedrock, then OpenRouter; prefer a " - "top-five intelligence model and use a cost-dominant value " - "score inside each provider." + "top-five intelligence model, use a cost-dominant value " + "score inside each provider, and fall through a route that " + "sends no output within 15 seconds." ), "required": [], "defaults": { @@ -226,6 +231,7 @@ def template_catalog() -> list[dict[str, Any]]: "max_price_out": DEFAULT_MAX_PRICE_OUT, "cost_weight": DEFAULT_COST_WEIGHT, "intelligence_weight": DEFAULT_INTELLIGENCE_WEIGHT, + "first_token_timeout_ms": DEFAULT_FIRST_TOKEN_TIMEOUT_MS, "provider_order": [list(group) for group in DEFAULT_PROVIDER_ORDER], }, }, @@ -484,6 +490,7 @@ def _default() -> tuple[list, dict[str, Any]]: "top_n": DEFAULT_QUALITY_TOP_N, "cost_weight": DEFAULT_COST_WEIGHT, "intelligence_weight": DEFAULT_INTELLIGENCE_WEIGHT, + "first_token_timeout_ms": DEFAULT_FIRST_TOKEN_TIMEOUT_MS, "provider_order": [list(group) for group in provider_order], }) value_score = [ @@ -495,7 +502,8 @@ def _default() -> tuple[list, dict[str, Any]]: ["normalize", ["field", "bench_intelligence"]], ], ] - return _policy(_and(parts), value_score, selector), intent + xform = ["set_param", "first_token_timeout_ms", DEFAULT_FIRST_TOKEN_TIMEOUT_MS] + return _policy(_and(parts), value_score, selector, xform), intent def _trusted_antseed_gate() -> list: diff --git a/tests/test_policy_templates.py b/tests/test_policy_templates.py index 58e2c33..275b441 100644 --- a/tests/test_policy_templates.py +++ b/tests/test_policy_templates.py @@ -398,6 +398,48 @@ def test_live_default_profile_is_exactly_the_published_default_template(): assert configured_pairs[0] == ("openai_codex", "gpt-5.5") +def test_live_default_profile_bounds_first_output_so_fallback_fits_the_deadline(): + import asyncio + + host = LLMRouterHost( + router_path=ROOT / "core" / "router.lua", + config_path=ROOT / "config.live.lua", + metrics_path=ROOT / "metrics.live.lua", + env={ + "OPENAI_API_KEY": "test", + "OPENROUTER_API_KEY": "test", + }, + now_ms=lambda: 1_000, + enforce_provider_auth=False, + ) + host.init() + _, intent = build_policy_template("default") + assert intent["first_token_timeout_ms"] == 15_000 + + seen = [] + + async def override(request): + seen.append(request) + if len(seen) == 1: + return {"ok": False, "error_kind": "timeout", "latency_ms": 1} + return { + "ok": True, + "latency_ms": 1, + "response": {"text": "fallback", "finish_reason": "stop"}, + } + + result = asyncio.run(host.execute_async({ + "prompt": "hi", + "profile": "default", + }, call_override=override)) + assert result["ok"] and result["response"]["text"] == "fallback" + assert len(seen) == 2 + assert all(req["first_token_timeout_ms"] == 15_000 for req in seen) + # Only the first output is bounded; the whole attempt keeps the engine's + # 30 s default. + assert all(req["timeout_ms"] == 30_000 for req in seen) + + def test_live_agent_profile_is_exactly_the_published_agent_template(): import asyncio