From 5a73391ff487ea238e09099b01e88c1b9b0fdd4f Mon Sep 17 00:00:00 2001 From: Shangxin Date: Sat, 26 Sep 2026 17:12:51 +0000 Subject: [PATCH] fix: the desktop speaks up only when the agent acted, not when it narrates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The resident agent wakes on every observed event, and a small model often narrates "no user request; nothing to act on" instead of staying silent as its prompt asks. The watch loop toasted on any non-empty reply, so under a home-dir file-I/O flood that became a popup every few seconds — the anti-pattern the notify path already guarded against for failed turns. Gate the toast on whether the turn ran a world-changing tool. Pi fires `tool_call` before each tool; a new `turnActionTracker` latches only the side-effecting built-ins (bash/edit/write) and ignores read-only orientation — `query_recent_events` (whose own description invites the model to look back) and Pi's `read`. The real session resets it at the top of each `prompt()` and exposes it as `ResidentSession.actedInLastTurn()`. O(1) per event, deliberately not `getSessionStats().toolCalls`, which rescans the whole unbounded continue-d history on every read. The panel and the JSON log still record every reply — only the desktop's one interruption is gated. A pure-narration or read-only-orientation turn stays on the panel and in the log, where a no-op is read; it no longer rings the doorbell. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/resident-agent.ts | 93 +++++++++++++++++++++++++++++++++--- src/watch-session.ts | 20 ++++++-- tests/resident-agent.test.ts | 62 ++++++++++++++++++++++++ tests/watch-desktop.test.ts | 3 ++ tests/watch-session.test.ts | 46 +++++++++++++++++- 5 files changed, 212 insertions(+), 12 deletions(-) diff --git a/src/resident-agent.ts b/src/resident-agent.ts index f17d68b..92b2287 100644 --- a/src/resident-agent.ts +++ b/src/resident-agent.ts @@ -41,6 +41,14 @@ export interface ResidentSession { waitForIdle(): Promise; abort(): Promise; getLastAssistantText(): string | undefined; + /** + * True when the agent ran a world-changing tool — a command or a file edit, + * not a read-only orientation call — during the most recent `prompt()` turn. + * A turn that only produced assistant text, or that only read to orient + * before narrating "nothing to do", is false. Lets a caller tell an action + * worth announcing apart from an idle narration. + */ + actedInLastTurn(): boolean; /** * Append an observation to the session history without triggering a turn. * The agent sees it as context on its next prompt — this is how senses feed @@ -50,6 +58,61 @@ export interface ResidentSession { dispose(): void; } +/** + * Remembers whether the current prompt turn changed anything. + * + * Squid's watch loop announces a reply on the desktop, but the resident model + * often wakes to background file I/O, finds nothing to do, and narrates that + * anyway — a toast for every no-op turn. Whether the agent *acted* is the + * distinction that gate needs, and Pi already fires `tool_call` before each + * tool runs. So this counts nothing: it only latches that at least one + * *world-changing* call fired since the last `reset()`, which the real session + * calls at the start of every `prompt()`. O(1) per event — deliberately not + * `getSessionStats().toolCalls`, which rescans the whole (unbounded, + * `continue`d) history on every read. + * + * Read-only calls do not count. Orienting — `query_recent_events` (whose own + * description invites "check what happened while you were thinking"), or Pi's + * `read`/`grep`/`find`/`ls` — is the model looking around, not doing something + * worth pulling a person back to a window they closed. Only a call that runs a + * command or edits a file earns the desktop's one interruption. + */ +export interface TurnActionTracker { + readonly extension: InlineExtension; + acted(): boolean; + reset(): void; +} + +/** + * The tool names that count as acting: Pi's side-effecting built-ins. Anything + * else — Squid's read-only `query_recent_events`, Pi's `read`, and any future + * sensory tool — is orientation, not action, so the default is "does not toast". + * + * ponytail: hardcoded to Pi's built-in names; if the agent gains another + * world-changing tool (or Pi renames one), add it here — a call not in this set + * fails safe toward silence, which is the behavior this whole change is for. + */ +const ACTING_TOOL_NAMES: ReadonlySet = new Set(["bash", "edit", "write"]); + +export function turnActionTracker(): TurnActionTracker { + let acted = false; + return { + extension: { + name: "squid-turn-action", + hidden: true, + factory: (pi) => { + pi.on("tool_call", (event) => { + if (ACTING_TOOL_NAMES.has(event.toolName)) acted = true; + }); + }, + }, + acted: () => acted, + reset: () => { + acted = false; + }, + }; +} + /** * Creates the real Pi session for a resident agent. Exported as a seam so tests * can substitute a fake — the whole repo avoids live `ModelRuntime`, and this @@ -79,6 +142,10 @@ export const createDefaultResidentSession: ResidentSessionFactory = async (spec) if (model === null) { throw new Error(`No resident-agent model is configured: ${provider}/${modelId}`); } + // First among the extensions so its `tool_call` hook sees every call. The + // real session owns the tracker; the adapter resets it per prompt and reads + // it back through `actedInLastTurn`. + const tracker = turnActionTracker(); const handle = await createPiSession({ model, modelRuntime, @@ -88,13 +155,11 @@ export const createDefaultResidentSession: ResidentSessionFactory = async (spec) // why is in there — so a real run persists it. `continue` rather than // `persist`: a restarted watch should resume the agent it already had. history: spec.history ?? "continue", - ...(spec.extensionFactories !== undefined - ? { extensionFactories: spec.extensionFactories } - : {}), + extensionFactories: [tracker.extension, ...(spec.extensionFactories ?? [])], ...(spec.customTools !== undefined ? { customTools: spec.customTools } : {}), ...(spec.noTools === true ? { noTools: "all" as const } : {}), }); - return adaptHandle(handle); + return adaptHandle(handle, tracker); }; /** @@ -113,12 +178,18 @@ export function residentAgentConfigured(): boolean { } /** Adapts a `PiSessionHandle` to the narrowed `ResidentSession` surface. */ -function adaptHandle(handle: PiSessionHandle): ResidentSession { +function adaptHandle(handle: PiSessionHandle, tracker: TurnActionTracker): ResidentSession { return { - prompt: (text) => handle.session.prompt(text), + prompt: (text) => { + // Cleared before the turn so `actedInLastTurn` reflects this prompt only; + // the tracker's `tool_call` hook latches it back on as the turn runs. + tracker.reset(); + return handle.session.prompt(text); + }, waitForIdle: () => handle.session.waitForIdle(), abort: () => handle.session.abort(), getLastAssistantText: () => handle.session.getLastAssistantText(), + actedInLastTurn: () => tracker.acted(), observe: (customType, content) => handle.session.sendCustomMessage( { customType, content, display: false }, @@ -212,6 +283,12 @@ export interface ResidentAgent { abort(): Promise; /** The agent's last assistant reply, if any. */ getLastAssistantText(): string | undefined; + /** + * True when the agent ran a world-changing tool during the most recent + * `prompt()` turn — the signal that tells a real action apart from the model + * reading to orient and then narrating that it found nothing to do. + */ + actedInLastTurn(): boolean; /** * Feed the agent a new world-state snapshot. Updates the live sense buffer * (so the next LLM call sees it) and appends a one-line observation to the @@ -278,6 +355,10 @@ export async function createResidentAgent( guardLive(); return session.getLastAssistantText(); }, + actedInLastTurn(): boolean { + guardLive(); + return session.actedInLastTurn(); + }, async observe(state: WorldState): Promise { guardLive(); senseBuffer.state = state; diff --git a/src/watch-session.ts b/src/watch-session.ts index 71968d8..d510fd1 100644 --- a/src/watch-session.ts +++ b/src/watch-session.ts @@ -707,6 +707,13 @@ export async function startWatchSession( await live.prompt(AGENT_TURN_PROMPT); await live.waitForIdle(); const reply = live.getLastAssistantText(); + // Whether the turn did anything the user would want pulled to the front. + // The resident model wakes on background file I/O it did not ask for, and + // a small model often narrates "nothing to act on" — sometimes after a + // read-only orientation call — rather than staying silent as its prompt + // asks. Neither pure narration nor a read runs a world-changing tool; + // `actedInLastTurn` is how a real action is told apart from both. + const acted = live.actedInLastTurn(); // Reported, not interpreted. Whether the agent acted, and how, is in its // session history; this line is the operator's window into a loop that is // otherwise silent. @@ -719,11 +726,14 @@ export async function startWatchSession( // what the agent said. `panelAgentLine` caps and redacts it and strips // the control characters, which is also why the body is safely one line. // - // Replies only. A failed turn broadcasts an agent line too, and - // announcing those would turn one misconfigured model into a popup every - // few seconds — a loud way to say something the log already said. The - // panel and the log stay the place a failure is read. - if (line.text !== undefined && line.text.length > 0) { + // Gated twice, for the same reason. A failed turn broadcasts an agent + // line too, and so does an idle turn the model narrated instead of + // ending in silence; announcing either would turn one misconfigured or + // chatty model into a popup every few seconds — a loud way to say + // something the log already said. Only a turn that acted earns the + // desktop's one interruption. The panel and the log stay the place a + // failure, or a no-op, is read. + if (acted && line.text !== undefined && line.text.length > 0) { notifier?.notify({ title: "Squid replied", body: line.text }); } } diff --git a/tests/resident-agent.test.ts b/tests/resident-agent.test.ts index 0c02f3c..c6474b9 100644 --- a/tests/resident-agent.test.ts +++ b/tests/resident-agent.test.ts @@ -3,6 +3,7 @@ import { createResidentAgent, residentAgentSystemPrompt, senseExtension, + turnActionTracker, type ResidentSession, type SenseBuffer, } from "../src/resident-agent.js"; @@ -35,6 +36,7 @@ function extensionFactory(extension: InlineExtension): (pi: ExtensionAPI) => voi function fakeSession(opts: { readonly assistantText?: string; readonly promptError?: Error; + readonly acted?: boolean; }): ResidentSession & { readonly calls: { prompt: string[]; @@ -72,6 +74,9 @@ function fakeSession(opts: { getLastAssistantText(): string | undefined { return opts.assistantText; }, + actedInLastTurn(): boolean { + return opts.acted ?? false; + }, dispose(): void { disposed = true; calls.dispose += 1; @@ -88,6 +93,48 @@ describe("residentAgentSystemPrompt", () => { }); }); +describe("turnActionTracker", () => { + function wireTracker() { + const tracker = turnActionTracker(); + const handlers: Record void> = {}; + const pi = { + on: (name: string, handler: (event: unknown) => void) => { + handlers[name] = handler; + }, + }; + extensionFactory(tracker.extension)(pi as unknown as ExtensionAPI); + const toolCall = (toolName: string) => + handlers["tool_call"]!({ type: "tool_call", toolName, input: {}, toolCallId: "1" }); + return { tracker, toolCall }; + } + + it("latches on a world-changing tool call and clears on reset", () => { + const { tracker, toolCall } = wireTracker(); + // A turn that never touched a tool: pure narration. + expect(tracker.acted()).toBe(false); + // Running a command or editing a file is the action worth a toast. + toolCall("bash"); + expect(tracker.acted()).toBe(true); + // Reset is what the real session calls at the top of each prompt, so the + // flag reports this turn and not a stale one. + tracker.reset(); + expect(tracker.acted()).toBe(false); + }); + + it("does not latch on read-only orientation calls", () => { + const { tracker, toolCall } = wireTracker(); + // The model looking around — Squid's history query, or Pi's file read — + // before narrating "nothing to do" is not an action. Counting these would + // reinstate the toast spam this whole change removes, one read behind it. + toolCall("query_recent_events"); + toolCall("read"); + expect(tracker.acted()).toBe(false); + // A real edit in the same turn still counts. + toolCall("edit"); + expect(tracker.acted()).toBe(true); + }); +}); + describe("createResidentAgent", () => { it("prompts the session and surfaces the assistant reply", async () => { const session = fakeSession({ assistantText: "ok" }); @@ -105,6 +152,21 @@ describe("createResidentAgent", () => { expect(session.calls.dispose).toBe(1); }); + it("reports actedInLastTurn straight from the session", async () => { + const acting = await createResidentAgent({ + systemPrompt: "p", + cwd: ".", + createSession: async () => fakeSession({ acted: true }), + }); + const idle = await createResidentAgent({ + systemPrompt: "p", + cwd: ".", + createSession: async () => fakeSession({ acted: false }), + }); + expect(acting.actedInLastTurn()).toBe(true); + expect(idle.actedInLastTurn()).toBe(false); + }); + it("preserves history: the second prompt reaches the same session", async () => { const session = fakeSession({ assistantText: "second" }); const agent = await createResidentAgent({ diff --git a/tests/watch-desktop.test.ts b/tests/watch-desktop.test.ts index 6802fa7..339c4eb 100644 --- a/tests/watch-desktop.test.ts +++ b/tests/watch-desktop.test.ts @@ -37,6 +37,9 @@ function silentAgent(): ResidentSession { getLastAssistantText(): string | undefined { return undefined; }, + actedInLastTurn(): boolean { + return false; + }, async observe(): Promise {}, dispose(): void {}, }; diff --git a/tests/watch-session.test.ts b/tests/watch-session.test.ts index a54d29a..04aeccc 100644 --- a/tests/watch-session.test.ts +++ b/tests/watch-session.test.ts @@ -76,6 +76,8 @@ interface AgentLog { interface FakeAgentOptions { readonly promptError?: Error; readonly reply?: string; + /** What `actedInLastTurn` reports: whether the turn invoked a tool. */ + readonly acted?: boolean; /** A session whose turn never settles — the wedge the watchdog exists to break. */ readonly neverIdle?: boolean; /** @@ -131,6 +133,9 @@ function fakeAgentFactory( getLastAssistantText(): string | undefined { return options.reply; }, + actedInLastTurn(): boolean { + return options.acted ?? false; + }, dispose(): void { log.calls.push("dispose"); log.disposed += 1; @@ -1010,6 +1015,7 @@ describe("the control panel on a watch session", () => { waitForIdle: session.waitForIdle, abort: session.abort, getLastAssistantText: session.getLastAssistantText, + actedInLastTurn: session.actedInLastTurn, observe: session.observe, dispose: () => { if (started) { @@ -1163,7 +1169,7 @@ describe("the control panel on a watch session", () => { const source = new ScriptedSource("alpha", []); const { result } = await startSession({ sources: [source], - agentOptions: { reply: "ran the tests; they pass" }, + agentOptions: { reply: "ran the tests; they pass", acted: true }, notifierFactory: () => ({ notify: (notice) => notices.push(notice), close: async () => undefined, @@ -1181,6 +1187,44 @@ describe("the control panel on a watch session", () => { } }); + /** + * A turn that acted on nothing reaches the panel and the log, and stops there. + * + * The resident model wakes on background file I/O and, being small, often + * narrates "nothing to act on" instead of ending in the silence its prompt + * asks for. That reply called no tool, so it is not worth the desktop's one + * interruption — the same restraint a failed turn already gets. The panel row + * (asserted separately) and the log line remain where a no-op turn is read. + */ + it("stays quiet on the desktop when the turn acted on nothing", async () => { + const notices: DesktopNotice[] = []; + const diagnostics: string[] = []; + const logged: string[] = []; + const source = new ScriptedSource("alpha", []); + const { result } = await startSession({ + sources: [source], + log: (line) => logged.push(line), + logError: (line) => diagnostics.push(line), + agentOptions: { reply: "no user request; nothing to act on", acted: false }, + notifierFactory: () => ({ + notify: (notice) => notices.push(notice), + close: async () => undefined, + }), + }); + if ("configError" in result) throw new Error(result.configError); + try { + await source.emit!(fileSaved("fs-1", "src/x.test.ts")); + // Wait for the reply to be logged: that is what turns the silence below + // into an assertion rather than a race the notifier happened to lose. + await waitForLine(() => + logged.some((line) => line.includes("nothing to act on")) ? { seen: true } : undefined, + ); + expect(notices).toEqual([]); + } finally { + await result.session.stop(); + } + }); + /** * A failed turn reaches the panel and the log, and stops there. *