Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion config.live.lua
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ local DEFAULT_MAX_PRICE_OUT = 25.0
local DEFAULT_QUALITY_TOP_N = 5
local DEFAULT_COST_WEIGHT = 0.75
local DEFAULT_INTELLIGENCE_WEIGHT = 0.25
-- A route that sends nothing for this long falls through to the next
-- candidate. Only the wait for the first output is bounded: a slow generation
-- that is already streaming keeps the engine's 30 s attempt budget, while a
-- silent route no longer leaves the next candidate too little of the 50 s
-- request deadline to answer before the caller gets a timeout.
local DEFAULT_FIRST_TOKEN_TIMEOUT_MS = 15000

-- Quality-first defaults for long-running tool agents. Unlike the vanilla
-- chat policy, each provider attempt is bounded inside the policy itself so a
Expand Down Expand Up @@ -188,7 +194,7 @@ local function default_policy_ir()
},
},
selector,
{ "id" },
{ "set_param", "first_token_timeout_ms", DEFAULT_FIRST_TOKEN_TIMEOUT_MS },
fail_plan(BALANCED_RETRY),
}
end
Expand Down
14 changes: 11 additions & 3 deletions policy_templates.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ class PolicyTemplateError(ValueError):
DEFAULT_QUALITY_TOP_N = 5
DEFAULT_COST_WEIGHT = 0.75
DEFAULT_INTELLIGENCE_WEIGHT = 0.25
# Bound only the wait for first output, so a silent route falls through while
# the request deadline still has room for the fallback cascade. Generations
# that are already streaming keep the full attempt budget.
DEFAULT_FIRST_TOKEN_TIMEOUT_MS = 15_000

# Long-running tool agents need a different posture from vanilla chat. Keep
# the quality floor hard, put subscription/direct routes ahead of gateways and
Expand Down Expand Up @@ -214,8 +218,9 @@ def template_catalog() -> list[dict[str, Any]]:
"description": (
"Safe policy used by OpenAI-compatible callers that send no "
"policy: Codex, AntSeed, Bedrock, then OpenRouter; prefer a "
"top-five intelligence model and use a cost-dominant value "
"score inside each provider."
"top-five intelligence model, use a cost-dominant value "
"score inside each provider, and fall through a route that "
"sends no output within 15 seconds."
),
"required": [],
"defaults": {
Expand All @@ -226,6 +231,7 @@ def template_catalog() -> list[dict[str, Any]]:
"max_price_out": DEFAULT_MAX_PRICE_OUT,
"cost_weight": DEFAULT_COST_WEIGHT,
"intelligence_weight": DEFAULT_INTELLIGENCE_WEIGHT,
"first_token_timeout_ms": DEFAULT_FIRST_TOKEN_TIMEOUT_MS,
"provider_order": [list(group) for group in DEFAULT_PROVIDER_ORDER],
},
},
Expand Down Expand Up @@ -484,6 +490,7 @@ def _default() -> tuple[list, dict[str, Any]]:
"top_n": DEFAULT_QUALITY_TOP_N,
"cost_weight": DEFAULT_COST_WEIGHT,
"intelligence_weight": DEFAULT_INTELLIGENCE_WEIGHT,
"first_token_timeout_ms": DEFAULT_FIRST_TOKEN_TIMEOUT_MS,
"provider_order": [list(group) for group in provider_order],
})
value_score = [
Expand All @@ -495,7 +502,8 @@ def _default() -> tuple[list, dict[str, Any]]:
["normalize", ["field", "bench_intelligence"]],
],
]
return _policy(_and(parts), value_score, selector), intent
xform = ["set_param", "first_token_timeout_ms", DEFAULT_FIRST_TOKEN_TIMEOUT_MS]
return _policy(_and(parts), value_score, selector, xform), intent


def _trusted_antseed_gate() -> list:
Expand Down
42 changes: 42 additions & 0 deletions tests/test_policy_templates.py
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,48 @@ def test_live_default_profile_is_exactly_the_published_default_template():
assert configured_pairs[0] == ("openai_codex", "gpt-5.5")


def test_live_default_profile_bounds_first_output_so_fallback_fits_the_deadline():
import asyncio

host = LLMRouterHost(
router_path=ROOT / "core" / "router.lua",
config_path=ROOT / "config.live.lua",
metrics_path=ROOT / "metrics.live.lua",
env={
"OPENAI_API_KEY": "test",
"OPENROUTER_API_KEY": "test",
},
now_ms=lambda: 1_000,
enforce_provider_auth=False,
)
host.init()
_, intent = build_policy_template("default")
assert intent["first_token_timeout_ms"] == 15_000

seen = []

async def override(request):
seen.append(request)
if len(seen) == 1:
return {"ok": False, "error_kind": "timeout", "latency_ms": 1}
return {
"ok": True,
"latency_ms": 1,
"response": {"text": "fallback", "finish_reason": "stop"},
}

result = asyncio.run(host.execute_async({
"prompt": "hi",
"profile": "default",
}, call_override=override))
assert result["ok"] and result["response"]["text"] == "fallback"
assert len(seen) == 2
assert all(req["first_token_timeout_ms"] == 15_000 for req in seen)
# Only the first output is bounded; the whole attempt keeps the engine's
# 30 s default.
assert all(req["timeout_ms"] == 30_000 for req in seen)


def test_live_agent_profile_is_exactly_the_published_agent_template():
import asyncio

Expand Down
Loading