From 34690c60d4d1319af126ac639204f918ca43933b Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Tue, 8 Sep 2026 09:22:29 -0700 Subject: [PATCH 1/2] fix(benchmark): guarantee supports_parallel_tool_calls in codex model catalog Signed-off-by: Sabhatina Selvam --- benchmark/codex_model_catalog_lib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/codex_model_catalog_lib.py b/benchmark/codex_model_catalog_lib.py index bcefe8530..dd3bc0190 100644 --- a/benchmark/codex_model_catalog_lib.py +++ b/benchmark/codex_model_catalog_lib.py @@ -108,6 +108,7 @@ def _build_codex_model_catalog( """Build Codex catalog JSON for Switchyard route ids.""" template = _load_codex_model_template(codex_bin) template.setdefault("supports_reasoning_summaries", True) + template.setdefault("supports_parallel_tool_calls", True) models: list[dict[str, Any]] = [] for priority, (model_id, display_name, description) in enumerate(entries): model = copy.deepcopy(template) From e524201471e307b7cb2dfb1fe03d30add37f1335 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Tue, 8 Sep 2026 09:22:40 -0700 Subject: [PATCH 2/2] feat(libsy): add LLM-driven tool-signal discovery with warm-up gate Signed-off-by: Sabhatina Selvam --- crates/libsy/src/algorithms/stage.rs | 128 +++- crates/libsy/src/algorithms/util.rs | 1 + .../algorithms/util/tool_signal_discovery.rs | 579 ++++++++++++++++++ .../libsy/src/algorithms/util/tool_signals.rs | 1 + crates/libsy/src/core/state.rs | 5 + crates/libsy/src/lib.rs | 2 +- .../prompts/tool_signal_discovery/prompt.md | 107 ++++ crates/switchyard-py/src/libsy_bindings.rs | 13 +- crates/switchyard-runner/src/algorithm.rs | 26 +- 9 files changed, 851 insertions(+), 11 deletions(-) create mode 100644 crates/libsy/src/algorithms/util/tool_signal_discovery.rs create mode 100644 crates/libsy/src/prompts/tool_signal_discovery/prompt.md diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index 483a46282..b08c1d48c 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -24,6 +24,7 @@ use super::util::stage::{ DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, StageTargets, Tier, fall_open_tier, record_decision_source, record_routing_decision, }; +use super::util::tool_signal_discovery::LlmToolSignalProcessor; use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor}; use crate::core::algorithm::{Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier, Score}; @@ -87,6 +88,47 @@ impl Classifier for FallOpen { } } +/// Forces the default tier for a session's first `warmup_turns` assistant turns, +/// regardless of what the signal source reports — long enough for a judge-driven +/// source to accumulate real signal (vocabulary, windowed counts) before its +/// output is trusted to decide routing. Abstains once warm-up has elapsed, so +/// the normal cascade (`StageClassifier`, `llm_fallback`, `FallOpen`) takes over +/// unchanged. The signal source itself still runs every turn during warm-up — +/// only the routing *decision* is held back, not signal collection. +struct WarmupGate { + targets: StageTargets, + default_tier: Tier, + warmup_turns: u32, +} + +#[async_trait] +impl Classifier for WarmupGate { + async fn score( + &self, + state: &mut State, + _request: &mut Request, + _driver: Option<&Driver>, + ) -> Result<(Classification, Option)> { + let assistant_turn_count = state + .tool_signals + .as_ref() + .map(|signal| signal.assistant_turn_count) + .unwrap_or(0); + if assistant_turn_count >= self.warmup_turns { + return Ok((Classification::Ambiguous(vec![]), None)); + } + let tier = fall_open_tier(state).unwrap_or(self.default_tier); + let target = self.targets.name(tier).clone(); + Ok(( + Classification::Scores(vec![Score { + target, + confidence: 0.0, + }]), + None, + )) + } +} + /// The capability judge a stage router falls through to. pub struct LlmFallback { /// Target the judge model is called through. It is not a routing @@ -118,6 +160,24 @@ pub struct StageRouterConfig { /// judge's own target, plus the same configuration the standalone capability /// route takes. pub llm_fallback: Option, + /// What populates `State::tool_signals`: the regex/name-table extractor in + /// `tool_signals.rs`, or an LLM judge that discovers its own buckets. Both + /// write the same `ToolSignals` shape, so `StageClassifier` is unaffected + /// by the choice. + pub tool_signal_source: ToolSignalSource, + /// Assistant turns to force the default tier for before routing decisions + /// are trusted, regardless of `confidence_threshold`. `0` disables warm-up — + /// routing decides from the first turn, as before. + pub warmup_turns: u32, +} + +/// Chooses what populates `State::tool_signals` for a `stage_router`. +#[derive(Clone)] +pub enum ToolSignalSource { + /// `tool_signals::ToolSignalProcessor` — fixed regex/name tables. + Static, + /// `tool_signal_discovery::LlmToolSignalProcessor`, called through this target. + Llm(ModelId), } impl StageRouterConfig { @@ -131,6 +191,8 @@ impl StageRouterConfig { handoff_notes: None, tier_prompts: TargetPrompts::default(), llm_fallback: None, + tool_signal_source: ToolSignalSource::Static, + warmup_turns: 0, } } } @@ -198,14 +260,30 @@ pub(crate) fn build_stage_route( if let Some(notes) = config.handoff_notes { classifier = classifier.with_handoff_notes(notes); } - let signals = ToolSignalProcessor { - recent_window: config.recent_window.unwrap_or(DEFAULT_RECENT_WINDOW), - }; + let recent_window = config.recent_window.unwrap_or(DEFAULT_RECENT_WINDOW); let target_set = vec![capable.clone(), efficient.clone()]; - let mut router = FallThrough::::new_with_state(target_set) - .with_name(STAGE_ROUTER) - .with_processor(Arc::new(signals)) - .with_classifier(Arc::new(classifier)); + let mut router = FallThrough::::new_with_state(target_set).with_name(STAGE_ROUTER); + if config.warmup_turns > 0 { + // Ahead of StageClassifier: first classifier to decide wins, so this + // holds routing on the default tier until warm-up elapses. + router = router.with_classifier(Arc::new(SourceStamp { + inner: Arc::new(WarmupGate { + targets: StageTargets::new(capable.clone(), efficient.clone()), + default_tier, + warmup_turns: config.warmup_turns, + }), + source: DecisionSource::FallOpen, + })); + } + router = router.with_classifier(Arc::new(classifier)); + router = match config.tool_signal_source { + ToolSignalSource::Static => { + router.with_processor(Arc::new(ToolSignalProcessor { recent_window })) + } + ToolSignalSource::Llm(judge_target) => router.with_processor(Arc::new( + LlmToolSignalProcessor::new(judge_target, recent_window), + )), + }; if let Some(fallback) = config.llm_fallback { // The capability judge takes its tiers in the same order the capability // route passes them: efficient first, capable second. @@ -561,4 +639,40 @@ mod tests { ); Ok(()) } + + fn state_with_assistant_turns(assistant_turn_count: u32) -> State { + let mut state = State::default(); + state.tool_signals = Some(crate::algorithms::util::tool_signals::ToolSignals { + assistant_turn_count, + ..Default::default() + }); + state + } + + #[tokio::test] + async fn warmup_gate_forces_default_tier_before_warmup_elapses() -> Result<()> { + let gate = WarmupGate { + targets: StageTargets::new("capable", "efficient"), + default_tier: Tier::Efficient, + warmup_turns: 3, + }; + let mut state = state_with_assistant_turns(1); + let (classification, _) = gate.score(&mut state, &mut Request::default(), None).await?; + let winner = classification.argmax(false)?.expect("should be decisive"); + assert_eq!(winner.target, ModelId::from("efficient")); + Ok(()) + } + + #[tokio::test] + async fn warmup_gate_abstains_once_warmup_elapses() -> Result<()> { + let gate = WarmupGate { + targets: StageTargets::new("capable", "efficient"), + default_tier: Tier::Efficient, + warmup_turns: 3, + }; + let mut state = state_with_assistant_turns(3); + let (classification, _) = gate.score(&mut state, &mut Request::default(), None).await?; + assert!(classification.argmax(false)?.is_none(), "should abstain"); + Ok(()) + } } diff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rs index 3f1202e26..a1e06dd96 100644 --- a/crates/libsy/src/algorithms/util.rs +++ b/crates/libsy/src/algorithms/util.rs @@ -12,6 +12,7 @@ pub mod subagent; pub(crate) mod target_selector; #[cfg(test)] pub(crate) mod tier_fixtures; +pub mod tool_signal_discovery; pub(crate) mod tool_signals; use switchyard_protocol::ModelId; diff --git a/crates/libsy/src/algorithms/util/tool_signal_discovery.rs b/crates/libsy/src/algorithms/util/tool_signal_discovery.rs new file mode 100644 index 000000000..16793d3a4 --- /dev/null +++ b/crates/libsy/src/algorithms/util/tool_signal_discovery.rs @@ -0,0 +1,579 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Judge-derived analogue of [`super::tool_signals`]: an LLM discovers its own +//! severity/category buckets from raw tool activity instead of regex tables, and +//! writes the same [`ToolSignals`] shape into [`State::tool_signals`]. A +//! `stage_router` can therefore be configured to source its signal from either +//! extractor — [`crate::algorithms::stage::StageRouterConfig`] picks one, and +//! [`crate::algorithms::stage::StageClassifier`] is unaffected by the choice. + +use std::collections::{HashMap, VecDeque}; +use std::sync::Arc; + +use async_trait::async_trait; +use parking_lot::Mutex; +use serde::Deserialize; +use serde_json::{Value, json}; +use switchyard_protocol::{ + AggLlmResponse, ContentBlock, InstructionBlock, LlmRequest, Message, ModelId, OutputParams, + Request, Role, completion_text, +}; + +use crate::core::algorithm::Driver; +use crate::core::processor::{Event, Processor}; +use crate::core::state::State; +use crate::{LibsyError, Result}; + +use super::tool_signals::ToolSignals; + +const SYSTEM_PROMPT: &str = include_str!("../../prompts/tool_signal_discovery/prompt.md"); +const MAX_OUTPUT_TOKENS: u64 = 1024; +/// Same marker `tool_signals.rs` latches on; duplicated rather than made +/// cross-module-visible since it is the only piece of that module this needs. +const COMPACTION_MARKER: &str = "session is being continued"; + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +enum DiscoveredCategory { + Write, + Edit, + Read, + Plan, + Other, +} + +impl DiscoveredCategory { + fn as_str(self) -> &'static str { + match self { + Self::Write => "write", + Self::Edit => "edit", + Self::Read => "read", + Self::Plan => "plan", + Self::Other => "other", + } + } +} + +#[derive(Debug, Deserialize)] +struct DiscoveredToolCall { + category: DiscoveredCategory, + pattern_label: String, +} + +#[derive(Debug, Deserialize)] +struct DiscoveredToolResult { + severity: f32, + pattern_label: Option, + tests_passed: bool, +} + +#[derive(Debug, Default, Deserialize)] +struct DiscoveryVerdict { + #[serde(default)] + tool_calls: Vec, + #[serde(default)] + tool_results: Vec, +} + +/// Discovered vocabulary shared across every session this route serves — the +/// calibration a judge-driven signal source is meant to accumulate over many +/// tasks, not just one. Owned by [`LlmToolSignalProcessor`] itself (a singleton +/// for the route's lifetime), never by per-session [`State`]. +#[derive(Clone, Debug, Default)] +pub struct SharedVocabulary { + /// Discovered `pattern_label` -> cumulative occurrences, across every + /// session, for both calls and results. + pub pattern_counts: HashMap, + /// Tool-call `pattern_label` -> the category it was first (and should stay) + /// associated with, so vocabulary feedback can catch cross-category reuse. + label_categories: HashMap, +} + +impl SharedVocabulary { + /// Vocabulary fed back into the prompt: call labels annotated with the + /// category they were first seen under, so the judge can catch itself + /// reusing a label across a category boundary; result labels stay bare. + fn vocabulary(&self) -> Vec { + self.pattern_counts + .keys() + .map(|label| match self.label_categories.get(label) { + Some(category) => format!("{label} ({})", category.as_str()), + None => label.clone(), + }) + .collect() + } + + fn record_call_label(&mut self, label: &str, category: DiscoveredCategory) { + self.label_categories + .entry(label.to_string()) + .or_insert(category); + *self.pattern_counts.entry(label.to_string()).or_default() += 1; + } + + fn record_result_label(&mut self, label: &str) { + *self.pattern_counts.entry(label.to_string()).or_default() += 1; + } +} + +/// Judge-discovered analogue of [`ToolSignals`], accumulated incrementally across +/// this session's own turns — reset per session, unlike [`SharedVocabulary`]. +#[derive(Clone, Debug, Default)] +pub struct LlmToolSignals { + last_processed_message_index: usize, + recent_categories: VecDeque, + recent_severities: VecDeque, + recent_tests_passed: VecDeque, + no_error_streak: u32, + pure_bash_streak: u32, + edit_count: u32, + write_count: u32, + read_count: u32, + todowrite_count: u32, +} + +impl LlmToolSignals { + fn record_call(&mut self, category: DiscoveredCategory, window: usize) { + match category { + DiscoveredCategory::Edit => self.edit_count += 1, + DiscoveredCategory::Write => self.write_count += 1, + DiscoveredCategory::Read => self.read_count += 1, + DiscoveredCategory::Plan => self.todowrite_count += 1, + DiscoveredCategory::Other => {} + } + self.pure_bash_streak = if category == DiscoveredCategory::Other { + self.pure_bash_streak + 1 + } else { + 0 + }; + push_bounded(&mut self.recent_categories, category, window); + } + + fn record_result(&mut self, severity: f32, tests_passed: bool, window: usize) { + self.no_error_streak = if severity > 0.0 { 0 } else { self.no_error_streak + 1 }; + push_bounded(&mut self.recent_severities, severity, window); + push_bounded(&mut self.recent_tests_passed, tests_passed, window); + } + + fn recent_count(&self, category: DiscoveredCategory) -> u32 { + self.recent_categories.iter().filter(|c| **c == category).count() as u32 + } + + /// Synthesizes the same [`ToolSignals`] shape `tool_signals.rs` produces, from + /// this accumulator's running windows plus this request's structural counts. + fn to_tool_signals( + &self, + turn_depth: u32, + assistant_turn_count: u32, + tool_result_count: u32, + compacted: bool, + ) -> ToolSignals { + ToolSignals { + severity: self.recent_severities.iter().cloned().fold(0.0, f32::max), + no_error_streak: self.no_error_streak, + edit_count: self.edit_count, + write_count: self.write_count, + read_count: self.read_count, + todowrite_count: self.todowrite_count, + recent_edit_count: self.recent_count(DiscoveredCategory::Edit), + recent_write_count: self.recent_count(DiscoveredCategory::Write), + recent_read_count: self.recent_count(DiscoveredCategory::Read), + recent_todowrite_count: self.recent_count(DiscoveredCategory::Plan), + pure_bash_streak: self.pure_bash_streak, + tests_passed: self.recent_tests_passed.iter().any(|passed| *passed), + tool_result_count, + assistant_turn_count, + turn_depth, + compacted, + } + } +} + +fn push_bounded(buffer: &mut VecDeque, value: T, window: usize) { + buffer.push_back(value); + while buffer.len() > window.max(1) { + buffer.pop_front(); + } +} + +/// Runs the discovery judge on each request's new tool activity, accumulates its +/// output into [`State::llm_tool_signals`], and writes the synthesized +/// [`ToolSignals`] into [`State::tool_signals`] — the same field +/// `super::tool_signals::ToolSignalProcessor` writes, so the picker cannot tell +/// which extractor produced it. +pub struct LlmToolSignalProcessor { + pub judge_target: ModelId, + /// Window the accumulator's `recent_*` counts and windowed severity are + /// computed over. Mirrors `ToolSignalProcessor::recent_window`. + pub recent_window: usize, + /// Discovered vocabulary, shared and cumulative across every session this + /// route serves — this processor is one singleton for the route's whole + /// lifetime, so this is where calibration actually accumulates. + pub vocabulary: Arc>, +} + +impl LlmToolSignalProcessor { + /// A fresh processor with its own empty, unshared vocabulary. + pub fn new(judge_target: ModelId, recent_window: usize) -> Self { + Self { + judge_target, + recent_window, + vocabulary: Arc::new(Mutex::new(SharedVocabulary::default())), + } + } +} + +#[async_trait] +impl Processor for LlmToolSignalProcessor { + async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> { + let Event::Request { request, driver } = event else { + return Ok(()); + }; + let messages = &request.llm_request.messages; + let (turn_depth, assistant_turn_count, tool_result_count, compacted) = + structural_signals(messages); + + let verdict = match driver { + Some(driver) => { + let signals = state.llm_tool_signals.get_or_insert_with(LlmToolSignals::default); + let start = signals.last_processed_message_index; + signals.last_processed_message_index = messages.len(); + if start < messages.len() { + let (calls, results) = new_tool_activity(&messages[start..]); + if calls.is_empty() && results.is_empty() { + None + } else { + let vocab = self.vocabulary.lock().vocabulary(); + consult_judge(driver, &self.judge_target, &calls, &results, &vocab).await + } + } else { + None + } + } + None => None, + }; + + let signals = state.llm_tool_signals.get_or_insert_with(LlmToolSignals::default); + if let Some(verdict) = verdict { + let mut vocabulary = self.vocabulary.lock(); + for call in &verdict.tool_calls { + tracing::info!(target: "libsy", category = ?call.category, label = call.pattern_label, "discovered tool-call pattern"); + signals.record_call(call.category, self.recent_window); + vocabulary.record_call_label(&call.pattern_label, call.category); + } + for result in &verdict.tool_results { + tracing::info!( + target: "libsy", + severity = result.severity, + tests_passed = result.tests_passed, + label = result.pattern_label.as_deref(), + "discovered tool-result pattern" + ); + signals.record_result(result.severity, result.tests_passed, self.recent_window); + if let Some(label) = &result.pattern_label { + vocabulary.record_result_label(label); + } + } + } + let tool_signal = signals.to_tool_signals( + turn_depth, + assistant_turn_count, + tool_result_count, + compacted, + ); + tracing::info!(target: "libsy", ?tool_signal, "synthesized llm tool signals"); + state.tool_signals = Some(tool_signal); + Ok(()) + } +} + +/// Calls the judge and parses its verdict. Fails open (`None`) and logs a warning +/// on any transport, stream, or parse failure — a judge outage should not stall +/// routing, and the caller keeps the accumulator's last-known-good signal. +async fn consult_judge( + driver: &Driver, + judge_target: &ModelId, + calls: &[(String, Value)], + results: &[String], + vocab: &[String], +) -> Option { + let vocab_refs: Vec<&str> = vocab.iter().map(String::as_str).collect(); + let (judge_request, activity) = build_request(judge_target, calls, results, &vocab_refs); + tracing::info!( + target: "libsy", + judge_target = %judge_target, + new_tool_calls = calls.len(), + new_tool_results = results.len(), + prior_vocabulary_size = vocab.len(), + prior_vocabulary = %vocab.join(", "), + request = %activity, + "signal-discovery judge request" + ); + + let response = match driver + .call_model(judge_request, vec![judge_target.clone()]) + .await + { + Ok(response) => response, + Err(error) => { + tracing::warn!(target: "libsy", %error, "signal-discovery judge call failed"); + return None; + } + }; + let aggregate = match response.llm_response.into_agg().await { + Ok(aggregate) => aggregate, + Err(error) => { + tracing::warn!(target: "libsy", %error, "signal-discovery judge stream failed"); + return None; + } + }; + tracing::info!(target: "libsy", response = %completion_text(&aggregate), "signal-discovery judge response"); + match parse_verdict(&aggregate) { + Ok(verdict) => Some(verdict), + Err(error) => { + tracing::warn!(target: "libsy", %error, "signal-discovery judge verdict did not parse"); + None + } + } +} + +/// Raw tool calls and tool-result text found in `messages`, in document order. +fn new_tool_activity(messages: &[Message]) -> (Vec<(String, Value)>, Vec) { + let mut calls = Vec::new(); + let mut results = Vec::new(); + for message in messages { + for block in &message.content { + match block { + ContentBlock::ToolCall(call) => calls.push((call.name.clone(), call.arguments.clone())), + ContentBlock::ToolResult(result) => { + let text = result + .content + .iter() + .filter_map(|b| match b { + ContentBlock::Text { text } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"); + if !text.is_empty() { + results.push(text); + } + } + _ => {} + } + } + } + (calls, results) +} + +/// Counts that need no judge call: message-count depth, assistant-turn count, +/// total tool-result blocks, and compaction — computed fresh every request, the +/// same way `tool_signals.rs` computes its equivalents. +fn structural_signals(messages: &[Message]) -> (u32, u32, u32, bool) { + let mut assistant_turn_count = 0u32; + let mut tool_result_count = 0u32; + let mut compacted = false; + for message in messages { + if message.role == Role::Assistant { + assistant_turn_count += 1; + } + for block in &message.content { + match block { + ContentBlock::ToolResult(_) => tool_result_count += 1, + ContentBlock::Text { text } => { + compacted |= text.to_lowercase().contains(COMPACTION_MARKER); + } + _ => {} + } + } + } + ( + messages.len() as u32, + assistant_turn_count, + tool_result_count, + compacted, + ) +} + +fn build_request( + target: &ModelId, + calls: &[(String, Value)], + results: &[String], + vocab: &[&str], +) -> (Request, String) { + let activity = json!({ + "tool_calls": calls.iter().map(|(name, args)| json!({"name": name, "arguments": args})).collect::>(), + "tool_results": results, + }) + .to_string(); + let prior_vocabulary = if vocab.is_empty() { + "(none yet)".to_string() + } else { + vocab.join(", ") + }; + let prompt = SYSTEM_PROMPT.replace("{prior_vocabulary}", &prior_vocabulary); + + let request = Request { + llm_request: LlmRequest { + model: Some(target.to_string()), + instructions: vec![InstructionBlock { + role: Role::System, + content: Message::text(Role::System, prompt).content, + }], + messages: vec![Message::text(Role::User, activity.clone())], + output: OutputParams { + max_output_tokens: Some(MAX_OUTPUT_TOKENS), + response_format: Some(json!({"type": "json_object"})), + }, + ..LlmRequest::default() + }, + raw_request: None, + metadata: None, + }; + (request, activity) +} + +fn parse_verdict(response: &AggLlmResponse) -> Result { + let text = completion_text(response); + let trimmed = text + .trim() + .trim_start_matches("```json") + .trim_start_matches("```") + .trim_end_matches("```") + .trim(); + serde_json::from_str(trimmed).map_err(|err| LibsyError::AlgorithmError { + message: format!("discovery verdict did not parse: {err}"), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use switchyard_protocol::{ContentBlock, Role, ToolCall, ToolResult}; + + fn tc(name: &str) -> Message { + Message { + role: Role::Assistant, + content: vec![ContentBlock::ToolCall(ToolCall { + id: String::new(), + name: name.to_string(), + arguments: json!({}), + })], + } + } + + fn tr(text: &str) -> Message { + Message { + role: Role::User, + content: vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: String::new(), + content: vec![ContentBlock::Text { + text: text.to_string(), + }], + is_error: None, + })], + } + } + + #[test] + fn extracts_calls_and_results_in_order() { + let messages = vec![tc("write_file"), tr("wrote ok"), tc("terminal")]; + let (calls, results) = new_tool_activity(&messages); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].0, "write_file"); + assert_eq!(results, vec!["wrote ok".to_string()]); + } + + #[test] + fn parses_fenced_verdict() { + let raw = "```json\n{\"tool_calls\":[{\"category\":\"write\",\"pattern_label\":\"write_file_tool\"}],\"tool_results\":[]}\n```"; + let trimmed = raw + .trim() + .trim_start_matches("```json") + .trim_start_matches("```") + .trim_end_matches("```") + .trim(); + let verdict: DiscoveryVerdict = serde_json::from_str(trimmed).unwrap(); + assert_eq!(verdict.tool_calls.len(), 1); + assert_eq!(verdict.tool_calls[0].pattern_label, "write_file_tool"); + } + + #[test] + fn accumulates_pattern_counts_in_shared_vocabulary() { + let mut vocabulary = SharedVocabulary::default(); + let verdict = DiscoveryVerdict { + tool_calls: vec![DiscoveredToolCall { + category: DiscoveredCategory::Write, + pattern_label: "write_file_tool".to_string(), + }], + tool_results: vec![DiscoveredToolResult { + severity: 0.7, + pattern_label: Some("missing_python_module".to_string()), + tests_passed: false, + }], + }; + for call in &verdict.tool_calls { + vocabulary.record_call_label(&call.pattern_label, call.category); + } + for result in &verdict.tool_results { + if let Some(label) = &result.pattern_label { + vocabulary.record_result_label(label); + } + } + assert_eq!(vocabulary.pattern_counts["write_file_tool"], 1); + assert_eq!(vocabulary.pattern_counts["missing_python_module"], 1); + } + + #[test] + fn synthesizes_tool_signals_shape_from_accumulator() { + let mut signals = LlmToolSignals::default(); + signals.record_call(DiscoveredCategory::Write, 3); + signals.record_result(0.7, false, 3); + let synthesized = signals.to_tool_signals(4, 2, 1, false); + assert_eq!(synthesized.severity, 0.7); + assert_eq!(synthesized.write_count, 1); + assert_eq!(synthesized.recent_write_count, 1); + assert_eq!(synthesized.no_error_streak, 0); + assert_eq!(synthesized.turn_depth, 4); + assert_eq!(synthesized.assistant_turn_count, 2); + assert_eq!(synthesized.tool_result_count, 1); + } + + #[test] + fn recent_window_bounds_accumulator_history() { + let mut signals = LlmToolSignals::default(); + for _ in 0..5 { + signals.record_call(DiscoveredCategory::Edit, 3); + } + assert_eq!(signals.edit_count, 5); + assert_eq!(signals.recent_count(DiscoveredCategory::Edit), 3); + } + + #[test] + fn vocabulary_annotates_call_labels_with_their_first_category() { + let mut vocabulary = SharedVocabulary::default(); + vocabulary.record_call_label("heredoc_write", DiscoveredCategory::Write); + vocabulary.record_result_label("import_error"); + let vocab = vocabulary.vocabulary(); + assert!(vocab.contains(&"heredoc_write (write)".to_string())); + assert!(vocab.contains(&"import_error".to_string())); + } + + #[test] + fn vocabulary_persists_across_sessions_via_shared_processor() { + let processor = LlmToolSignalProcessor::new(ModelId::from("judge"), 3); + processor + .vocabulary + .lock() + .record_call_label("heredoc_write", DiscoveredCategory::Write); + // A second, independent session's LlmToolSignals starts empty... + let session_two_signals = LlmToolSignals::default(); + assert_eq!(session_two_signals.write_count, 0); + // ...but the processor's shared vocabulary — not session state — still has it. + assert_eq!( + processor.vocabulary.lock().pattern_counts["heredoc_write"], + 1 + ); + } +} diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index 3427357be..7ee49fc6d 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -308,6 +308,7 @@ impl Processor for ToolSignalProcessor { async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> { if let Event::Request { request: req, .. } = event { let tool_signal = ToolSignals::from_request(req, Some(self.recent_window)); + tracing::info!(target: "libsy", ?tool_signal, "deterministic tool signals"); state.tool_signals = Some(tool_signal); } Ok(()) diff --git a/crates/libsy/src/core/state.rs b/crates/libsy/src/core/state.rs index 01ccb08a9..5bf1f3194 100644 --- a/crates/libsy/src/core/state.rs +++ b/crates/libsy/src/core/state.rs @@ -5,6 +5,7 @@ use std::collections::HashMap; +use crate::algorithms::util::tool_signal_discovery::LlmToolSignals; use crate::algorithms::util::tool_signals::ToolSignals; /// A value in a session's [`State`]. @@ -29,6 +30,10 @@ pub struct State { /// processor. `None` until it runs or when the request has no tool activity, /// so routers must treat absence as "no signal yet". pub tool_signals: Option, + /// Judge-discovered analogue of `tool_signals`, accumulated by + /// [`crate::algorithms::util::tool_signal_discovery::LlmToolSignalProcessor`]. + /// Observation-only: no classifier reads this field yet. + pub llm_tool_signals: Option, /// Algorithm-specific state keyed by stable internal names. pub extra: HashMap, } diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 3907feb3c..38ec957df 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -23,7 +23,7 @@ pub use algorithms::llm_class::{ pub use algorithms::noop::Noop; pub use algorithms::passthrough::Passthrough; pub use algorithms::rand::{Random, RandomClassifier}; -pub use algorithms::stage::{LlmFallback, StageRouter, StageRouterConfig}; +pub use algorithms::stage::{LlmFallback, StageRouter, StageRouterConfig, ToolSignalSource}; pub use algorithms::subagent::{SubagentRouter, SubagentRouterConfig}; pub use algorithms::util::affinity::{AffinityRouter, ClassifyTrigger}; pub use algorithms::util::classifier_contract::{ diff --git a/crates/libsy/src/prompts/tool_signal_discovery/prompt.md b/crates/libsy/src/prompts/tool_signal_discovery/prompt.md new file mode 100644 index 000000000..c014623d4 --- /dev/null +++ b/crates/libsy/src/prompts/tool_signal_discovery/prompt.md @@ -0,0 +1,107 @@ +You are extracting structured signals from raw coding-agent tool activity, one turn +at a time. You maintain state across turns: labels you invent must stay consistent +with what you invented in earlier turns of this conversation (shown to you below as +prior vocabulary). + +Below is the baseline rule set a separate deterministic extractor uses for common +coding-agent harnesses. Use it as your starting point — match tool names and shell +commands against it first. This harness may also use tools or command shapes the +baseline does not cover (new tool names, unusual shell idioms, harness-specific +conventions) — for those, infer a bucket yourself and invent a new pattern_label, +the way a human reading terminal output would. Both paths matter: staying aligned +with the baseline where it applies, and extending it where it does not. + +═══ BASELINE: ACTION CATEGORY ═══ + +Tool-name matches (case-insensitive): + write: write, create_file, new_file, write_file + edit: edit, multiedit, notebookedit, str_replace, str_replace_based_edit_tool, + apply_patch, text_editor, patch + read: read, view, read_file, search_files + plan: todowrite, todo_write, todo, update_plan + +Shell/terminal tool names (bash, shell_command, shell, local_shell_call, terminal, +exec_command) do not classify by name — classify by what the command line does: + write patterns: cat >, cat >>, echo >, echo >>, tee, printf >, printf >>, > /, + >> /, heredoc redirection (< /tmp/out` + is a write, not a read). + Anything else (build commands, test runners, servers, generic scripts) -> other. + +═══ BASELINE: SEVERITY (for each NEW tool result) ═══ + +If the result text contains one of these substrings (case-insensitive), use that +severity AND reuse that exact name as the pattern_label: + 1.0 critical: "out of memory" / "memoryerror" / "cannot allocate memory" -> pattern_label=oom + "connection refused" / "connectionrefusederror" / "econnrefused" -> pattern_label=connection_refused + 0.7 hard: "traceback (most recent call last)" -> pattern_label=traceback + "modulenotfounderror:" / "importerror:" / "no module named " -> pattern_label=import_error + "command not found" / "not found" (own line) / "/usr/bin/env: " -> pattern_label=cmd_not_found + "assertionerror" -> pattern_label=assertion + "valueerror:" -> pattern_label=value_error + "syntaxerror:" -> pattern_label=syntax_error + "timed out" / "timeouterror" / "timeout expired" / "deadline exceeded" -> pattern_label=timeout + "filenotfounderror:" / "no such file or directory" / "file does not exist" -> pattern_label=no_such_file + 0.3 soft: "exit code 1" / "exit code 2" / "exit status 1" / "returned non-zero" / + "exited with code" (and no other pattern above also fires) -> pattern_label=exit_nonzero + 0.0 clean: none of the above fire -> pattern_label=null + +If the result text contains MULTIPLE of the above, use the highest severity, but +still name the pattern_label after whichever single one is most specific to what +actually failed (prefer a concrete cause like import_error/assertion over the +generic exit_nonzero when both are present). + +If nothing above fires but the result still clearly reports a failure or crash in a +way this list doesn't cover, use your judgment on severity (0.3/0.7/1.0 by how bad +it looks) and invent a new pattern_label — do not force it into a baseline name that +doesn't really fit. + +Also judge, for each NEW tool result: does it look like it reports a test suite (or +equivalent verification step) that ran and PASSED — not partially, not with any +failures, not merely attempted? Trip this only on phrases like " passed", "passed in", +"tests passed", "all tests passed", "test ok", "tests pass", a bare "\nok " line, or +"✓ " — and only when the same result does NOT also contain a failure marker such as +"✗ ", "fatal:", "assertionerror", "error:", or an explicit nonzero failure count like +"2 failed" / "3 errors" (a clean "0 failed" / "0 errors" summary does not count as a +failure). Be conservative: prefer a false "no" over a false "yes". + +═══ pattern_label FORMAT ═══ + +Lowercase snake_case, 2-4 words, naming the specific pattern (not a restatement of +the category/severity tier). Reuse the same slug every time you see the same kind of +thing again — for baseline hits, reuse the exact baseline name given above; for +anything you're inferring yourself, invent one and then stay consistent with it for +the rest of this conversation. + +A `pattern_label` on a tool call must map to exactly ONE category, always. If you +are tempted to reuse a call label across two different categories (e.g. the same +kind of inline script sometimes writes a file and sometimes only inspects one), stop +and split it into two distinct labels instead — one per category (for example +`python_inline_write` vs `python_inline_check`, not `python_inline_script` for both). +Before emitting a label, check your prior vocabulary below: if that label was used +with a different category earlier in this conversation, mint a more specific label +rather than reusing it as-is. + +═══ OUTPUT ═══ + +Output ONLY this JSON for the NEW activity in this turn — never re-classify tool +calls/results you already labeled in a prior turn: + +{ + "tool_calls": [ + { "category": "write" | "edit" | "read" | "plan" | "other", "pattern_label": "slug" } + ], + "tool_results": [ + { "severity": 0.0 | 0.3 | 0.7 | 1.0, "pattern_label": "slug_or_null", "tests_passed": true | false } + ] +} + +Arrays are positional and must match the order the new tool calls/results appear in +below. Your own vocabulary so far this conversation: +{prior_vocabulary} diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index 0571a3e12..5e15162f7 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -16,7 +16,7 @@ use switchyard_libsy::{ CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, HandoffNoteConfig, LibsyError as RustLibsyError, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, StageRouter, StageRouterConfig, Step as RustStep, - StepStream, TaskClassifierConfig, + StepStream, TaskClassifierConfig, ToolSignalSource, }; use switchyard_protocol::{ LlmClientError, LlmResponse, LlmResponseStream, LlmResponseStreamEvent, Metadata, ModelId, @@ -778,7 +778,9 @@ fn build_llm_classifier(config: LlmClassifierConfig) -> PyResult { only_on_wrong_signal_escalation=true, capable_system_prompt=None, efficient_system_prompt=None, - classifier=None + classifier=None, + signal_discovery_judge_target=None, + warmup_turns=0 ))] #[allow(clippy::too_many_arguments)] fn stage_router_algorithm( @@ -794,6 +796,8 @@ fn stage_router_algorithm( capable_system_prompt: Option, efficient_system_prompt: Option, classifier: Option>, + signal_discovery_judge_target: Option, + warmup_turns: u32, ) -> PyResult { let mode = match picker { "capable_first" => PickerMode::CapableFirst, @@ -830,6 +834,11 @@ fn stage_router_algorithm( config.llm_fallback = classifier .map(|classifier| classifier.bind(py).try_borrow()?.clone_core(py)) .transpose()?; + config.tool_signal_source = match signal_discovery_judge_target { + Some(target) => ToolSignalSource::Llm(ModelId::new(target)), + None => ToolSignalSource::Static, + }; + config.warmup_turns = warmup_turns; let algorithm = StageRouter::new(capable, efficient, config) .map_err(|error| PyValueError::new_err(error.to_string()))?; diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index a601ab94a..f48d04ea1 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -14,7 +14,7 @@ use libsy::{ CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, TargetPrompts, - TaskClassifierConfig, + TaskClassifierConfig, ToolSignalSource, }; use serde::Deserialize; use switchyard_protocol::ModelId; @@ -254,6 +254,17 @@ pub enum AlgorithmSpec { /// Judge consulted for turns the tool signals cannot decide. #[serde(default)] classifier: Option, + /// When set, an LLM judge discovers its own severity/category buckets from + /// tool activity and drives routing instead of the fixed regex/name tables — + /// target name to call the judge through. Unset keeps the static extractor. + #[serde(default)] + signal_discovery_judge_target: Option, + /// Assistant turns to force the default tier for before routing decisions + /// are trusted, regardless of `confidence_threshold` — lets a judge-driven + /// signal source accumulate real signal before it can affect routing. `0` + /// (default) disables warm-up. + #[serde(default)] + warmup_turns: u32, /// Separate policy for delegated sub-agent work. #[serde(default)] subagents: Option, @@ -488,12 +499,16 @@ impl AlgorithmSpec { } => names.extend(subagents.classifier_target_name()), Self::StageRouter { classifier, + signal_discovery_judge_target, subagents, .. } => { if let Some(classifier) = classifier { names.push(&classifier.target); } + if let Some(target) = signal_discovery_judge_target { + names.push(target); + } if let Some(subagents) = subagents { names.extend(subagents.classifier_target_name()); } @@ -936,6 +951,8 @@ fn build_algorithm( tiers, picker, classifier, + signal_discovery_judge_target, + warmup_turns, subagents, .. } => { @@ -977,6 +994,13 @@ fn build_algorithm( ) }) .transpose()?; + config.tool_signal_source = match signal_discovery_judge_target.as_ref() { + Some(target) => { + ToolSignalSource::Llm(resolve_target_model_id(route_name, target, targets)?) + } + None => ToolSignalSource::Static, + }; + config.warmup_turns = *warmup_turns; let algorithm = StageRouter::new(capable, efficient, config).map_err(|error| { AlgorithmConfigError::with_source( format!("stage_router route {route_name}: {error}"),