From 9bac12f3b44d994d33cf525ad999621727af4cda Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 18:33:21 -0500 Subject: [PATCH 01/20] fix: detect reasoning loops by hashing stream segments Replace the between-calls loop-detection middleware, which could never see repetition inside a single generation, with a streaming detector. Reasoning and message deltas are split into sentence segments and hashed into a rolling window; when a segment repeats past the threshold the stream is abandoned and the turn re-run with a looping nudge, compacting context after streamMaxNudges failed nudges. --- config.yaml | 5 +- coverage.txt | 16 +- index.js | 55 +++- src/agent/deepAgents.js | 26 -- src/config/schemas/loopDetection.js | 30 +-- src/provider/loopDetectionMiddleware.js | 242 ------------------ src/stream/consume.js | 15 +- src/stream/loopDetector.js | 105 ++++++++ .../provider/loopDetectionMiddleware.test.js | 242 ------------------ tests/unit/stream/loopDetector.test.js | 118 +++++++++ 10 files changed, 305 insertions(+), 549 deletions(-) delete mode 100644 src/provider/loopDetectionMiddleware.js create mode 100644 src/stream/loopDetector.js delete mode 100644 tests/unit/provider/loopDetectionMiddleware.test.js create mode 100644 tests/unit/stream/loopDetector.test.js diff --git a/config.yaml b/config.yaml index d1ea211b..dbfa6663 100644 --- a/config.yaml +++ b/config.yaml @@ -57,11 +57,10 @@ jira: token: "" # personal access token, sent as Bearer (JIRA_TOKEN) loopDetection: enabled: true - escalationCount: 3 - escalationWindowMs: 60000 + streamMaxNudges: 2 threshold: 2 ttlMs: 60000 - windowSize: 10 + windowSize: 50 memory: checkpointsDir: memory/checkpoints/ contextDir: memory/context/ diff --git a/coverage.txt b/coverage.txt index 50293f88..4df63beb 100644 --- a/coverage.txt +++ b/coverage.txt @@ -8,7 +8,7 @@ ℹ compaction.js | 98.84 | 78.43 | 100.00 | 89-90 ℹ contextBackend.js | 100.00 | 100.00 | 100.00 | ℹ coreBackend.js | 100.00 | 100.00 | 100.00 | -ℹ deepAgents.js | 94.93 | 67.65 | 60.87 | 157-158 182 198-200 310-317 346-347 352-353 460-462 512 548-550 597-598 608-610 617-618 +ℹ deepAgents.js | 93.72 | 68.00 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 ℹ evictionCeiling.js | 100.00 | 100.00 | 100.00 | ℹ skillsMiddleware.js | 100.00 | 87.50 | 100.00 | ℹ config | | | | @@ -60,9 +60,8 @@ ℹ copilotAuth.js | 96.80 | 78.87 | 90.00 | 12-13 131-132 196-197 237-239 352-353 378-379 391-393 484 565-566 575-576 ℹ copilotRetryMiddleware.js | 88.52 | 79.49 | 83.33 | 75-88 ℹ index.js | 100.00 | 100.00 | 100.00 | -ℹ loopDetectionMiddleware.js | 99.59 | 83.08 | 100.00 | 32 -ℹ modelInfo.js | 98.17 | 96.39 | 100.00 | 128-129 179-180 -ℹ openai.js | 91.60 | 73.61 | 78.57 | 65 90-92 138-139 145-146 181-184 233 236-238 257-259 267-271 279-280 292-295 +ℹ modelInfo.js | 97.25 | 93.98 | 100.00 | 128-129 131-132 179-180 +ℹ openai.js | 91.60 | 74.65 | 78.57 | 65 90-92 138-139 145-146 181-184 233 236-238 257-259 267-271 279-280 292-295 ℹ summarizationMiddleware.js | 97.01 | 94.00 | 83.33 | 107 126-128 149 ℹ tokenBudget.js | 97.14 | 85.71 | 82.35 | 50-51 83-84 ℹ tokenBudgetMiddleware.js | 98.10 | 78.67 | 87.50 | 94-95 97-98 @@ -96,8 +95,9 @@ ℹ types.js | 100.00 | 100.00 | 100.00 | ℹ validator.js | 100.00 | 100.00 | 100.00 | ℹ stream | | | | -ℹ consume.js | 85.51 | 74.29 | 100.00 | 52-58 88-93 104-110 +ℹ consume.js | 95.97 | 96.88 | 100.00 | 95-100 ℹ interrupt.js | 100.00 | 100.00 | 100.00 | +ℹ loopDetector.js | 100.00 | 86.21 | 100.00 | ℹ namespace.js | 100.00 | 100.00 | 100.00 | ℹ transformers | | | | ℹ index.js | 100.00 | 100.00 | 100.00 | @@ -144,7 +144,7 @@ ℹ confluence | | | | ℹ index.js | 98.59 | 82.50 | 83.33 | 89 191-192 ℹ cron | | | | -ℹ index.js | 93.59 | 84.53 | 82.50 | 47-48 87-92 199-200 255-256 278-280 294-295 367-368 517-519 548-549 559-562 569-570 585-589 600-602 614-616 627-628 +ℹ index.js | 93.59 | 85.08 | 82.50 | 47-48 87-92 199-200 255-256 278-280 294-295 367-368 517-519 548-549 559-562 569-570 585-589 600-602 614-616 627-628 ℹ data | | | | ℹ index.js | 98.90 | 96.49 | 100.00 | 158 240-241 ℹ date | | | | @@ -185,7 +185,7 @@ ℹ xlsxParser.js | 82.21 | 83.33 | 88.89 | 47-48 93-112 130-137 150-156 ℹ zipExtractor.js | 82.32 | 77.78 | 66.67 | 62-63 104-107 124-142 149-152 ℹ git | | | | -ℹ index.js | 87.41 | 64.13 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 +ℹ index.js | 87.41 | 64.55 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 ℹ graphql | | | | ℹ index.js | 97.85 | 98.13 | 93.75 | 57-68 ℹ image | | | | @@ -284,6 +284,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.34 | 86.41 | 88.20 | +ℹ all files | 93.33 | 86.55 | 88.25 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/index.js b/index.js index e01d29f7..1a038395 100644 --- a/index.js +++ b/index.js @@ -19,6 +19,7 @@ import { import { resumeSession } from "./src/session/resume.js"; import { logger, flush as flushLogger } from "./src/shared/logger.js"; import { consumeStream } from "./src/stream/consume.js"; +import { createStreamLoopDetector } from "./src/stream/loopDetector.js"; // --------------------------------------------------------------------------- // CLI arguments @@ -296,21 +297,57 @@ async function doBootstrap() { * @param {AbortSignal} [signal] - Abort signal */ async function runAgent(input, threadExtras, streamingCallback, signal) { - const stream = await agent.stream(input, { + const loopConfig = config.loopDetection ?? {}; + const loopDetector = + loopConfig.enabled === false + ? undefined + : createStreamLoopDetector({ + windowSize: loopConfig.windowSize, + threshold: loopConfig.threshold, + ttlMs: loopConfig.ttlMs, + }); + const maxNudges = loopConfig.streamMaxNudges ?? 2; + const threadConfig = { configurable: { thread_id: sessionState.getSessionId(), ...threadExtras }, maxTokens: providerConfig.maxTokens, signal, recursionLimit: config.agent?.recursionLimit, streamMode: ["messages", "tools", "values"], subgraphs: true, - }); - const { content, reasoning, interrupted, lastEventType } = await consumeStream( - stream, - streamingCallback, - { - showToolResults, - }, - ); + }; + + let currentInput = input; + let nudges = 0; + let result; + let content = ""; + let reasoning = ""; + + // A loop inside a single generation is caught while streaming: abandon the + // stream, then re-run with a nudge (or compact once nudges are exhausted). + while (true) { + const stream = await agent.stream(currentInput, threadConfig); + result = await consumeStream(stream, streamingCallback, { showToolResults, loopDetector }); + content += result.content; + reasoning += result.reasoning; + + if (!result.loopDetected || signal?.aborted) break; + + loopDetector?.reset(); + if (nudges >= maxNudges) { + logger.warn("[loopDetection] stream loop persists; compacting context"); + try { + await agent.compactContext(threadConfig); + } catch (err) { + logger.warn({ error: err?.message }, "[loopDetection] compaction failed"); + } + break; + } + nudges++; + logger.warn({ nudges }, "[loopDetection] stream loop detected; aborting and nudging"); + currentInput = { messages: [new HumanMessage("You're looping")] }; + } + + const { interrupted, lastEventType } = result; return { provider: providerName, content, diff --git a/src/agent/deepAgents.js b/src/agent/deepAgents.js index 6d96ebcc..57769489 100644 --- a/src/agent/deepAgents.js +++ b/src/agent/deepAgents.js @@ -16,7 +16,6 @@ import { getModelContextLength } from "../provider/modelInfo.js"; import { getActiveProviderConfig, getActiveProviderName } from "../provider/index.js"; import { createTokenBudgetMiddleware } from "../provider/tokenBudgetMiddleware.js"; import { createCopilotRetryMiddleware } from "../provider/copilotRetryMiddleware.js"; -import { createLoopDetectionMiddleware } from "../provider/loopDetectionMiddleware.js"; import { createSummarizationMiddlewareFromConfig } from "../provider/summarizationMiddleware.js"; import { createEvictionAwareFilesystemMiddleware } from "./evictionCeiling.js"; import { createSkillsMiddleware } from "./skillsMiddleware.js"; @@ -535,22 +534,6 @@ export async function createDeepAgentsOrchestrator(checkpointer = null) { ? createCopilotRetryMiddleware({ maxRetries: 3 }) : null; - // Loop-detection middleware. Detects when the model repeats the same message - // or reasoning content and injects a "You're looping" nudge, escalating to the - // agent's compaction path when the loop persists. The `onCompact` callback is - // wired via closure after the agent exists (mirroring - // `compactOnContextWindowExceeded`), so it can call `agent.compactContext`. - // Returns null when disabled, so unset config is a true no-op. - let loopDetectionOnCompact; - const loopDetectionMiddleware = createLoopDetectionMiddleware({ - config: config.loopDetection, - onCompact: async (threadId) => { - if (typeof loopDetectionOnCompact === "function") { - await loopDetectionOnCompact(threadId); - } - }, - }); - // Replacement for deepagents' FilesystemMiddleware with an eviction ceiling // fixed to fit image payloads. The library evicts any ToolMessage whose // stringified content exceeds `toolTokenLimitBeforeEvict * 4` characters @@ -583,7 +566,6 @@ export async function createDeepAgentsOrchestrator(checkpointer = null) { : []), ...(summarizationMiddleware ? [summarizationMiddleware] : []), ...(copilotRetryMiddleware ? [copilotRetryMiddleware] : []), - ...(loopDetectionMiddleware ? [loopDetectionMiddleware] : []), ...(tokenBudgetMiddleware ? [tokenBudgetMiddleware] : []), ], streamTransformers: [() => createTurnTransformer()], @@ -610,14 +592,6 @@ export async function createDeepAgentsOrchestrator(checkpointer = null) { await agent.compactContext(config); }; - // Wire the loop-detection middleware's escalation to the agent's compaction - // path. The middleware is created before the agent, so this closure is set - // once the agent exists. It compacts the current thread's context. - loopDetectionOnCompact = async (threadId) => { - const config = threadId ? { configurable: { thread_id: threadId } } : undefined; - await agent.compactContext(config); - }; - // Expose the MCP adapter (if any) so the caller can close it on shutdown. // `allTools` is an array with the adapter attached as a property when MCP // servers are configured. diff --git a/src/config/schemas/loopDetection.js b/src/config/schemas/loopDetection.js index 40d8c532..66afcff9 100644 --- a/src/config/schemas/loopDetection.js +++ b/src/config/schemas/loopDetection.js @@ -1,30 +1,26 @@ import { z } from "zod"; /** - * Configuration for the `LoopDetection` middleware. + * Configuration for streaming loop detection. * - * When `enabled` is false (or the section is absent), madz does not register - * the loop-detection middleware and no loop detection occurs. When enabled, - * the middleware hashes the most recent model message text and reasoning - * content independently per thread, and injects a "You're looping" nudge when - * a hash repeats more than `threshold` times within a rolling `windowSize` - * window. If `escalationCount` nudges occur within `escalationWindowMs`, the - * middleware invokes the agent's compaction path instead of nudging again. + * When `enabled` is false, no loop detection occurs. When enabled, streamed + * reasoning and message text are split into segments (sentences/lines), each + * segment is hashed into a rolling `windowSize` window, and a loop is flagged + * when a segment hash repeats more than `threshold` times. The stream is then + * abandoned and the turn re-run with a "You're looping" nudge; after + * `streamMaxNudges` failed nudges the context is compacted instead. */ export const LoopDetectionSchema = z .object({ enabled: z.boolean().default(true), - // Number of recent content hashes retained per thread for loop detection. - windowSize: z.number().int().positive().default(10), - // A hash older than this many ms is pruned from the window, so past - // content cannot cause false positives as new content streams in. + // Number of recent segment (sentence) hashes retained for loop detection. + windowSize: z.number().int().positive().default(50), + // A hash older than this many ms is pruned from the window. ttlMs: z.number().int().positive().default(60000), - // A hash must appear more than this many times within the window to flag a loop. + // A segment hash must appear more than this many times within the window to flag a loop. threshold: z.number().int().positive().default(2), - // Number of loop nudges within the escalation window before compacting. - escalationCount: z.number().int().positive().default(3), - // Rolling window (ms) for counting loop nudges before escalation. - escalationWindowMs: z.number().int().positive().default(60000), + // Max times a single streamed turn is aborted and re-nudged before compacting instead. + streamMaxNudges: z.number().int().nonnegative().default(2), }) .strict() .default({}); diff --git a/src/provider/loopDetectionMiddleware.js b/src/provider/loopDetectionMiddleware.js deleted file mode 100644 index f50e189b..00000000 --- a/src/provider/loopDetectionMiddleware.js +++ /dev/null @@ -1,242 +0,0 @@ -import { createHash } from "node:crypto"; -import { createMiddleware } from "langchain"; -import { HumanMessage } from "@langchain/core/messages"; -import { logger } from "../shared/logger.js"; - -/** The literal nudge injected into the model state when a loop is detected. */ -const LOOP_NUDGE = "You're looping"; - -/** - * Extract the message text and reasoning content from a LangChain message. - * - * Mirrors the content-extraction patterns in `src/stream/transformers/turn.js` - * so message text and reasoning content are captured independently. Returns - * `{ text, reasoning }` where each is a string (possibly empty). - * - * @param {Object} msg - A LangChain message - * @returns {{ text: string, reasoning: string }} Extracted content - */ -export function extractMessageContent(msg) { - if (!msg || typeof msg !== "object") return { text: "", reasoning: "" }; - - let text = ""; - let reasoning = ""; - - // String content is plain text. - if (typeof msg.content === "string") { - text = msg.content; - } else if (Array.isArray(msg.content)) { - // Content blocks (Responses API / block format). - for (const block of msg.content) { - if (typeof block === "string") { - text += block; - } else if (block && typeof block === "object") { - if (block.type === "text") text += block.text ?? ""; - else if (block.type === "reasoning") reasoning += block.reasoning ?? ""; - } - } - } - - // Reasoning content in additional_kwargs (Chat Completions API). - if (msg.additional_kwargs && typeof msg.additional_kwargs === "object") { - const rc = msg.additional_kwargs.reasoning_content; - if (typeof rc === "string") reasoning += rc; - } - - return { text, reasoning }; -} - -/** - * Compute a stable sha256 hash of a string. Returns `undefined` for empty or - * non-string input so empty content is not recorded in the hash window. - * @param {string} value - The content to hash - * @returns {string | undefined} The hex hash, or `undefined` when empty - */ -export function hashContent(value) { - if (typeof value !== "string" || value.length === 0) return undefined; - return createHash("sha256").update(value).digest("hex"); -} - -/** - * Create the `LoopDetection` middleware. - * - * Detects when the model repeats the same message text or reasoning content - * across successive calls. On each `beforeModel` hook, the most recent message - * content is hashed independently (text and reasoning) and recorded in a - * per-thread rolling window. When a hash appears more than the configured - * threshold within the window, the middleware injects a `HumanMessage` with - * the text "You're looping" so the model observes the nudge on its next turn. - * - * If the escalation count of nudges is reached within the escalation window, - * the middleware invokes the `onCompact` callback (wired to the agent's - * compaction path) instead of injecting another nudge. - * - * Returns `null` when disabled, so unset config is a true no-op. - * - * @param {Object} options - Middleware options - * @param {Object} [options.config] - The resolved `loopDetection` config section - * @param {Function} [options.onCompact] - Callback invoked on escalation (e.g. `agent.compactContext`) - * @param {Function} [options.now] - Clock function returning ms (injectable for tests) - * @returns {Object|null} The middleware, or `null` when disabled - */ -export function createLoopDetectionMiddleware(options = {}) { - const config = options.config ?? {}; - if (config.enabled === false) return null; - - const windowSize = config.windowSize ?? 10; - const ttlMs = config.ttlMs ?? 60000; - const threshold = config.threshold ?? 2; - const escalationCount = config.escalationCount ?? 3; - const escalationWindowMs = config.escalationWindowMs ?? 60000; - const onCompact = options.onCompact; - const now = options.now ?? (() => Date.now()); - - /** Per-thread state: hash windows (with timestamps) and nudge timestamps. */ - const threadState = new Map(); - - /** - * Get (or create) the state for a thread. - * @param {string} threadKey - The thread_id - * @returns {{ messageHashes: {hash: string, ts: number}[], reasoningHashes: {hash: string, ts: number}[], nudges: number[] }} State - */ - function getThreadState(threadKey) { - let state = threadState.get(threadKey); - if (!state) { - state = { messageHashes: [], reasoningHashes: [], nudges: [] }; - threadState.set(threadKey, state); - } - return state; - } - - /** - * Prune hashes older than the TTL from a window. The window is - * insertion-ordered, so the oldest entries are at the front. - * @param {{hash: string, ts: number}[]} window - The hash window array - * @param {number} nowMs - Current time in ms - * @returns {void} - */ - function pruneHashes(window, nowMs) { - while (window.length > 0 && nowMs - window[0].ts > ttlMs) window.shift(); - } - - /** - * Clear both hash windows for a thread. Called when a nudge is injected (or - * on escalation) so the model gets a fresh slate — otherwise it could repeat - * the same phrase on its next turn and immediately re-trigger detection. - * @param {{messageHashes: {hash: string, ts: number}[], reasoningHashes: {hash: string, ts: number}[]}} thread - The thread state - * @returns {void} - */ - function clearHashes(thread) { - thread.messageHashes = []; - thread.reasoningHashes = []; - } - - /** - * Record a hash with a timestamp in a bounded rolling window, evicting the - * oldest entry when the window exceeds `windowSize` or the TTL. - * @param {{hash: string, ts: number}[]} window - The hash window array - * @param {string} hash - The hash to record - * @param {number} nowMs - Current time in ms - * @returns {void} - */ - function recordHash(window, hash, nowMs) { - pruneHashes(window, nowMs); - window.push({ hash, ts: nowMs }); - while (window.length > windowSize) window.shift(); - } - - /** - * Count occurrences of a hash within a window. - * @param {{hash: string, ts: number}[]} window - The hash window array - * @param {string} hash - The hash to count - * @returns {number} Occurrence count - */ - function countOccurrences(window, hash) { - let count = 0; - for (const entry of window) if (entry.hash === hash) count++; - return count; - } - - /** - * Decide whether a hash exceeds the loop threshold within its window. - * @param {{hash: string, ts: number}[]} window - The hash window array - * @param {string} hash - The hash to check - * @returns {boolean} True when the hash appears more than `threshold` times - */ - function isLooping(window, hash) { - return countOccurrences(window, hash) > threshold; - } - - /** - * Prune nudge timestamps older than the escalation window. - * @param {number[]} nudges - Nudge timestamps - * @param {number} nowMs - Current time in ms - * @returns {void} - */ - function pruneNudges(nudges, nowMs) { - while (nudges.length > 0 && nowMs - nudges[0] > escalationWindowMs) nudges.shift(); - } - - return createMiddleware({ - name: "LoopDetection", - beforeModel: { - canJumpTo: [], - hook: async (state, runtime) => { - const messages = state?.messages ?? []; - if (messages.length === 0) return undefined; - - const threadKey = runtime?.configurable?.thread_id ?? ""; - const thread = getThreadState(threadKey); - const lastMsg = messages[messages.length - 1]; - const { text, reasoning } = extractMessageContent(lastMsg); - - const nowMs = now(); - const textHash = hashContent(text); - const reasoningHash = hashContent(reasoning); - - let looped = false; - if (textHash) { - recordHash(thread.messageHashes, textHash, nowMs); - if (isLooping(thread.messageHashes, textHash)) looped = true; - } - if (reasoningHash) { - recordHash(thread.reasoningHashes, reasoningHash, nowMs); - if (isLooping(thread.reasoningHashes, reasoningHash)) looped = true; - } - - if (!looped) return undefined; - - pruneNudges(thread.nudges, nowMs); - - // Escalate to compaction when the nudge count reaches the threshold. - if (thread.nudges.length >= escalationCount) { - logger.warn( - { thread: threadKey, nudges: thread.nudges.length, escalationCount }, - "[loopDetection] escalating to context compaction", - ); - thread.nudges = []; - clearHashes(thread); - if (typeof onCompact === "function") { - try { - await onCompact(threadKey); - } catch (err) { - logger.warn({ error: err?.message }, "[loopDetection] onCompact callback failed"); - } - } - return undefined; - } - - // Otherwise inject the nudge and record it. Clear the hash windows - // so the model gets a fresh slate — otherwise it could repeat the - // same phrase on its next turn and immediately re-trigger. - thread.nudges.push(nowMs); - clearHashes(thread); - logger.warn( - { thread: threadKey, nudges: thread.nudges.length }, - "[loopDetection] detected loop; injecting nudge", - ); - return { messages: [new HumanMessage(LOOP_NUDGE)] }; - }, - }, - }); -} diff --git a/src/stream/consume.js b/src/stream/consume.js index 1de4f12a..7618d1f2 100644 --- a/src/stream/consume.js +++ b/src/stream/consume.js @@ -13,7 +13,10 @@ import { isSubagentNamespace } from "./namespace.js"; * @param {Function} [streamingCallback] - Callback for streamed chunks * @param {Object} [options] - Options * @param {boolean} [options.showToolResults=true] - Whether to emit `tool` events - * @returns {Promise<{content: string, reasoning: string, interrupted: boolean, lastEventType: string | null}>} + * @param {{push: Function, reset: Function}} [options.loopDetector] - Streaming repetition + * detector. When it flags a loop the stream is abandoned (closing the underlying + * iterator) and `loopDetected` is returned true. + * @returns {Promise<{content: string, reasoning: string, interrupted: boolean, lastEventType: string | null, loopDetected: boolean}>} */ export async function consumeStream(stream, streamingCallback, options = {}) { const showToolResults = options.showToolResults; @@ -21,6 +24,8 @@ export async function consumeStream(stream, streamingCallback, options = {}) { let reasoning = ""; let interrupted = false; let lastEventType = null; + let loopDetected = false; + const loopDetector = options.loopDetector; for await (const [namespace, mode, payload] of stream) { // Subagent runs are nested under a `tools:` namespace segment. Their @@ -62,6 +67,7 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (!isSubagent) { content += text; + if (loopDetector?.push("message", text)) loopDetected = true; } if (streamingCallback) { @@ -83,6 +89,7 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (reasoningContent) { if (!isSubagent) { reasoning += reasoningContent; + if (loopDetector?.push("reasoning", reasoningContent)) loopDetected = true; } if (streamingCallback) { streamingCallback({ @@ -99,6 +106,7 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (block?.type === "reasoning" && block.reasoning) { if (!isSubagent) { reasoning += block.reasoning; + if (loopDetector?.push("reasoning", block.reasoning)) loopDetected = true; } if (streamingCallback) { streamingCallback({ @@ -132,7 +140,10 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (!isSubagent) lastEventType = "on_tool_end"; } } + + // Abandon the stream; leaving the for-await closes the iterator. + if (loopDetected) break; } - return { content, reasoning, interrupted, lastEventType }; + return { content, reasoning, interrupted, lastEventType, loopDetected }; } diff --git a/src/stream/loopDetector.js b/src/stream/loopDetector.js new file mode 100644 index 00000000..54441681 --- /dev/null +++ b/src/stream/loopDetector.js @@ -0,0 +1,105 @@ +import { createHash } from "node:crypto"; + +const BOUNDARY = /(?<=[.!?])\s+|\n+/; + +/** + * Create a segment-hash repetition detector. + * + * Text (reasoning and message text, tracked separately by `kind`) is buffered + * and split into sentences/lines. Each completed segment is normalized and + * hashed into a rolling window; when one hash occurs more than `threshold` + * times the detector reports a loop. It is used both on live stream deltas + * (catching repetition within a single generation) and on completed assistant + * messages (catching repetition across turns), so one mechanism serves both. + * + * @param {Object} [options] - Options + * @param {number} [options.windowSize=50] - Segment hashes retained per kind + * @param {number} [options.threshold=2] - Occurrences above which a loop is flagged + * @param {number} [options.minLength=24] - Segments shorter than this are ignored + * @param {number} [options.maxBuffer=500] - Force a split when no boundary appears + * @param {number} [options.ttlMs] - Hashes older than this are pruned (no expiry when unset) + * @param {Function} [options.now] - Clock returning ms (injectable for tests) + * @returns {{ push: (kind: string, delta: string) => boolean, flush: (kind?: string) => boolean, reset: () => void }} Detector + */ +export function createStreamLoopDetector(options = {}) { + const windowSize = options.windowSize ?? 50; + const threshold = options.threshold ?? 2; + const minLength = options.minLength ?? 24; + const maxBuffer = options.maxBuffer ?? 500; + const ttlMs = options.ttlMs; + const now = options.now ?? (() => Date.now()); + + /** @type {Map} */ + const kinds = new Map(); + + function getKind(kind) { + let state = kinds.get(kind); + if (!state) { + state = { buffer: "", hashes: [] }; + kinds.set(kind, state); + } + return state; + } + + function record(state, segment) { + const normalized = segment.replace(/\s+/g, " ").trim().toLowerCase(); + if (normalized.length < minLength) return false; + const nowMs = now(); + if (ttlMs !== undefined) { + while (state.hashes.length > 0 && nowMs - state.hashes[0].ts > ttlMs) state.hashes.shift(); + } + const hash = createHash("sha256").update(normalized).digest("hex"); + state.hashes.push({ hash, ts: nowMs }); + while (state.hashes.length > windowSize) state.hashes.shift(); + let count = 0; + for (const entry of state.hashes) if (entry.hash === hash) count++; + return count > threshold; + } + + return { + /** + * Feed a delta (or a whole message). + * @param {string} kind - Kind, e.g. "reasoning" or "message" + * @param {string} delta - New text + * @returns {boolean} True when a loop has been detected + */ + push(kind, delta) { + if (typeof delta !== "string" || delta.length === 0) return false; + const state = getKind(kind); + state.buffer += delta; + + const parts = state.buffer.split(BOUNDARY); + state.buffer = parts.pop() ?? ""; + if (state.buffer.length > maxBuffer) { + parts.push(state.buffer); + state.buffer = ""; + } + + let looped = false; + for (const part of parts) if (record(state, part)) looped = true; + return looped; + }, + + /** + * Hash any buffered trailing text as a final segment. Call at the end of a + * complete message so its last (unterminated) segment is counted. + * @param {string} [kind] - Kind to flush; all kinds when omitted + * @returns {boolean} True when the flushed segment completes a loop + */ + flush(kind) { + let looped = false; + const targets = kind === undefined ? [...kinds.values()] : [getKind(kind)]; + for (const state of targets) { + const rest = state.buffer; + state.buffer = ""; + if (rest && record(state, rest)) looped = true; + } + return looped; + }, + + /** Clear all state (call after a nudge so the next attempt starts fresh). */ + reset() { + kinds.clear(); + }, + }; +} diff --git a/tests/unit/provider/loopDetectionMiddleware.test.js b/tests/unit/provider/loopDetectionMiddleware.test.js deleted file mode 100644 index d55519b0..00000000 --- a/tests/unit/provider/loopDetectionMiddleware.test.js +++ /dev/null @@ -1,242 +0,0 @@ -import { describe, it } from "node:test"; -import assert from "node:assert"; -import { AIMessage } from "@langchain/core/messages"; -import { - createLoopDetectionMiddleware, - extractMessageContent, - hashContent, -} from "../../../src/provider/loopDetectionMiddleware.js"; - -const config = (extra = {}) => ({ - enabled: true, - windowSize: 10, - threshold: 2, - escalationCount: 3, - escalationWindowMs: 60000, - ...extra, -}); - -const run = (mw, state, thread = "t") => - mw.beforeModel.hook(state, { configurable: { thread_id: thread } }); - -const ai = (text, reasoning) => { - const msg = new AIMessage(text); - if (reasoning !== undefined) { - msg.additional_kwargs = { reasoning_content: reasoning }; - } - return msg; -}; - -describe("extractMessageContent", () => { - it("extracts string content as text", () => { - assert.deepStrictEqual(extractMessageContent(new AIMessage("hello")), { - text: "hello", - reasoning: "", - }); - }); - - it("extracts reasoning from additional_kwargs", () => { - const msg = new AIMessage("hello"); - msg.additional_kwargs = { reasoning_content: "think" }; - assert.deepStrictEqual(extractMessageContent(msg), { text: "hello", reasoning: "think" }); - }); - - it("extracts reasoning from content blocks", () => { - const msg = new AIMessage([ - { type: "text", text: "hello" }, - { type: "reasoning", reasoning: "think" }, - ]); - assert.deepStrictEqual(extractMessageContent(msg), { text: "hello", reasoning: "think" }); - }); - - it("returns empty strings for null/empty input", () => { - assert.deepStrictEqual(extractMessageContent(null), { text: "", reasoning: "" }); - assert.deepStrictEqual(extractMessageContent({}), { text: "", reasoning: "" }); - }); -}); - -describe("hashContent", () => { - it("hashes identical content identically", () => { - assert.strictEqual(hashContent("abc"), hashContent("abc")); - }); - - it("returns undefined for empty or non-string content", () => { - assert.strictEqual(hashContent(""), undefined); - assert.strictEqual(hashContent(undefined), undefined); - }); -}); - -describe("createLoopDetectionMiddleware", () => { - it("returns null when disabled", () => { - assert.strictEqual(createLoopDetectionMiddleware({ config: { enabled: false } }), null); - }); - - it("is named LoopDetection and uses beforeModel", () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - assert.strictEqual(mw.name, "LoopDetection"); - assert.strictEqual(typeof mw.beforeModel.hook, "function"); - assert.strictEqual(mw.wrapModelCall, undefined); - }); - - it("does nothing for empty state", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - assert.strictEqual(await run(mw, { messages: [] }), undefined); - }); - - it("does not flag a loop below the threshold", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - // Two identical messages: threshold is 2, so 2 occurrences is NOT > 2. - assert.strictEqual(await run(mw, { messages: [ai("same")] }), undefined); - assert.strictEqual(await run(mw, { messages: [ai("same")] }), undefined); - }); - - it("flags a loop above the threshold and injects a nudge", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - await run(mw, { messages: [ai("same")] }); - await run(mw, { messages: [ai("same")] }); - const out = await run(mw, { messages: [ai("same")] }); - assert.ok(out); - assert.strictEqual(out.messages.length, 1); - assert.strictEqual(out.messages[0].content, "You're looping"); - }); - - it("clears hash windows after injecting a nudge, preventing immediate re-trigger", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - // Build up a loop and inject a nudge. - await run(mw, { messages: [ai("same")] }); - await run(mw, { messages: [ai("same")] }); - const out = await run(mw, { messages: [ai("same")] }); - assert.ok(out); - // The model repeats the same phrase on its next turn. Because the hash - // windows were cleared on injection, this must NOT immediately re-trigger. - assert.strictEqual(await run(mw, { messages: [ai("same")] }), undefined); - }); - - it("tracks message and reasoning content independently", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - // Same reasoning, different text: reasoning loop should trigger. - await run(mw, { messages: [ai("a", "think")] }); - await run(mw, { messages: [ai("b", "think")] }); - const out = await run(mw, { messages: [ai("c", "think")] }); - assert.ok(out); - assert.strictEqual(out.messages[0].content, "You're looping"); - }); - - it("does not false-positive on changing content", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - for (const text of ["a", "b", "c", "d", "e"]) { - assert.strictEqual(await run(mw, { messages: [ai(text)] }), undefined); - } - }); - - it("evicts the oldest hash when the window is exceeded", async () => { - const mw = createLoopDetectionMiddleware({ config: config({ windowSize: 3 }) }); - // Fill the window with distinct hashes, then repeat the first one. - // With windowSize 3, only the last 3 hashes are retained. - await run(mw, { messages: [ai("x")] }); - await run(mw, { messages: [ai("y")] }); - await run(mw, { messages: [ai("z")] }); - // "x" was evicted; repeating it should not trigger. - assert.strictEqual(await run(mw, { messages: [ai("x")] }), undefined); - }); - - it("prunes hashes older than the TTL, preventing false positives", async () => { - let clock = 0; - const mw = createLoopDetectionMiddleware({ - config: config({ ttlMs: 100 }), - now: () => clock, - }); - // Build up 3 occurrences of "same" at t=0. - await run(mw, { messages: [ai("same")] }); - await run(mw, { messages: [ai("same")] }); - await run(mw, { messages: [ai("same")] }); - // Advance past the TTL; the old hashes are pruned. - clock = 200; - // "same" again should NOT trigger, because the old occurrences expired. - assert.strictEqual(await run(mw, { messages: [ai("same")] }), undefined); - }); - - // Trigger a single loop detection (3 identical messages, since threshold is 2). - const triggerLoop = async (mw, text = "same") => { - await run(mw, { messages: [ai(text)] }); - await run(mw, { messages: [ai(text)] }); - return run(mw, { messages: [ai(text)] }); - }; - - it("escalates to onCompact after the escalation count within the window", async () => { - let compacted = 0; - const mw = createLoopDetectionMiddleware({ - config: config(), - onCompact: () => { - compacted++; - }, - }); - // 3 nudges injected (each clears the hash window), then the 4th loop - // detection escalates to compaction. - for (let i = 0; i < 3; i++) { - assert.ok(await triggerLoop(mw)); - } - // The 4th loop detection reaches the escalation count and compacts. - assert.strictEqual(await triggerLoop(mw), undefined); - assert.strictEqual(compacted, 1); - }); - - it("resets the escalation counter when the window expires", async () => { - let compacted = 0; - let clock = 0; - const mw = createLoopDetectionMiddleware({ - config: config({ escalationWindowMs: 100 }), - onCompact: () => { - compacted++; - }, - now: () => clock, - }); - // First nudge at t=0. - assert.ok(await triggerLoop(mw)); - // Advance clock past the window; nudges should be pruned. - clock = 200; - // Two more nudges in the new window (below escalationCount 3), so no compaction. - assert.ok(await triggerLoop(mw)); - assert.ok(await triggerLoop(mw)); - assert.strictEqual(compacted, 0); - }); - - it("does not compact below the escalation count", async () => { - let compacted = 0; - const mw = createLoopDetectionMiddleware({ - config: config(), - onCompact: () => { - compacted++; - }, - }); - // Only 2 nudges (below escalationCount 3). - assert.ok(await triggerLoop(mw)); - assert.ok(await triggerLoop(mw)); - assert.strictEqual(compacted, 0); - }); - - it("survives a throwing onCompact callback", async () => { - const mw = createLoopDetectionMiddleware({ - config: config(), - onCompact: () => { - throw new Error("boom"); - }, - }); - // 3 nudges, then the 4th loop detection escalates; the throwing - // onCompact must not propagate. - for (let i = 0; i < 3; i++) { - assert.ok(await triggerLoop(mw)); - } - await triggerLoop(mw); - }); - - it("maintains independent state per thread", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - // Thread "a" loops; thread "b" does not. - await run(mw, { messages: [ai("same")] }, "a"); - await run(mw, { messages: [ai("same")] }, "a"); - const outA = await run(mw, { messages: [ai("same")] }, "a"); - assert.ok(outA); - assert.strictEqual(await run(mw, { messages: [ai("same")] }, "b"), undefined); - }); -}); diff --git a/tests/unit/stream/loopDetector.test.js b/tests/unit/stream/loopDetector.test.js new file mode 100644 index 00000000..e8a0843c --- /dev/null +++ b/tests/unit/stream/loopDetector.test.js @@ -0,0 +1,118 @@ +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { createStreamLoopDetector } from "../../../src/stream/loopDetector.js"; +import { consumeStream } from "../../../src/stream/consume.js"; + +const SENTENCE = "I need to verify the config schemas are set up correctly. "; + +describe("createStreamLoopDetector", () => { + it("flags a sentence repeated beyond the threshold", () => { + const d = createStreamLoopDetector({ threshold: 2 }); + assert.strictEqual(d.push("reasoning", SENTENCE), false); + assert.strictEqual(d.push("reasoning", SENTENCE), false); + // Third completed sentence (> threshold 2) is flagged. + assert.strictEqual(d.push("reasoning", SENTENCE), true); + }); + + it("handles deltas split mid-sentence", () => { + const d = createStreamLoopDetector({ threshold: 2 }); + const text = SENTENCE.repeat(4); + let flagged = false; + for (let i = 0; i < text.length; i += 7) { + if (d.push("reasoning", text.slice(i, i + 7))) flagged = true; + } + assert.ok(flagged); + }); + + it("does not flag distinct sentences", () => { + const d = createStreamLoopDetector(); + for (let i = 0; i < 20; i++) { + assert.strictEqual(d.push("reasoning", `This is unique sentence number ${i} here. `), false); + } + }); + + it("ignores short sentences", () => { + const d = createStreamLoopDetector(); + for (let i = 0; i < 10; i++) assert.strictEqual(d.push("message", "Ok. "), false); + }); + + it("tracks reasoning and message independently", () => { + const d = createStreamLoopDetector({ threshold: 2 }); + d.push("reasoning", SENTENCE); + d.push("reasoning", SENTENCE); + assert.strictEqual(d.push("message", SENTENCE), false); + }); + + it("flags a long repeat with no sentence boundary via maxBuffer", () => { + const d = createStreamLoopDetector({ threshold: 2, maxBuffer: 40 }); + const chunk = "x".repeat(41); + d.push("reasoning", chunk); + d.push("reasoning", chunk); + assert.strictEqual(d.push("reasoning", chunk), true); + }); + + it("flush counts a trailing unterminated segment", () => { + const d = createStreamLoopDetector({ threshold: 2 }); + const tail = "a trailing segment with no terminal punctuation"; + for (let i = 0; i < 2; i++) { + d.push("message", tail); + assert.strictEqual(d.flush("message"), false); + } + d.push("message", tail); + assert.strictEqual(d.flush("message"), true); + }); + + it("prunes hashes older than ttlMs", () => { + let clock = 0; + const d = createStreamLoopDetector({ threshold: 2, ttlMs: 100, now: () => clock }); + d.push("reasoning", SENTENCE); + d.push("reasoning", SENTENCE); + clock = 500; + assert.strictEqual(d.push("reasoning", SENTENCE), false); + }); + + it("reset clears state", () => { + const d = createStreamLoopDetector({ threshold: 2 }); + d.push("reasoning", SENTENCE); + d.push("reasoning", SENTENCE); + d.reset(); + assert.strictEqual(d.push("reasoning", SENTENCE), false); + }); +}); + +describe("consumeStream with loopDetector", () => { + const reasoningChunk = (text) => [ + [], + "messages", + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: text } }], + ]; + + it("stops consuming and reports loopDetected", async () => { + let consumed = 0; + let closed = false; + const stream = (async function* () { + try { + for (let i = 0; i < 100; i++) { + consumed++; + yield reasoningChunk(SENTENCE); + } + } finally { + closed = true; + } + })(); + const result = await consumeStream(stream, undefined, { + loopDetector: createStreamLoopDetector({ threshold: 2 }), + }); + assert.strictEqual(result.loopDetected, true); + assert.strictEqual(consumed, 3); + assert.ok(closed, "underlying stream is closed"); + }); + + it("reports loopDetected false without a detector", async () => { + const stream = (async function* () { + yield reasoningChunk(SENTENCE); + })(); + const result = await consumeStream(stream); + assert.strictEqual(result.loopDetected, false); + }); +}); From 979c4c1aa265a3c026b75425268ab361e31e87b1 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 18:42:04 -0500 Subject: [PATCH 02/20] fix: hash app stream segments in loop detector Build the detector on coalesceSegments so a segment is the same message/reasoning block the TUI renders (docs/STREAMING.md), instead of a custom sentence split. Drop maxBuffer, minLength and flush; short segments such as 'ok' are now counted. --- coverage.txt | 10 +- index.js | 4 +- src/config/schemas/loopDetection.js | 9 +- src/stream/consume.js | 33 ++++- src/stream/loopDetector.js | 118 +++++++---------- tests/unit/stream/loopDetector.test.js | 173 +++++++++++++++++-------- 6 files changed, 202 insertions(+), 145 deletions(-) diff --git a/coverage.txt b/coverage.txt index 4df63beb..4b9cc0d5 100644 --- a/coverage.txt +++ b/coverage.txt @@ -61,7 +61,7 @@ ℹ copilotRetryMiddleware.js | 88.52 | 79.49 | 83.33 | 75-88 ℹ index.js | 100.00 | 100.00 | 100.00 | ℹ modelInfo.js | 97.25 | 93.98 | 100.00 | 128-129 131-132 179-180 -ℹ openai.js | 91.60 | 74.65 | 78.57 | 65 90-92 138-139 145-146 181-184 233 236-238 257-259 267-271 279-280 292-295 +ℹ openai.js | 92.72 | 75.71 | 78.57 | 65 90-92 138-139 145-146 233 236-238 257-259 267-271 279-280 292-295 ℹ summarizationMiddleware.js | 97.01 | 94.00 | 83.33 | 107 126-128 149 ℹ tokenBudget.js | 97.14 | 85.71 | 82.35 | 50-51 83-84 ℹ tokenBudgetMiddleware.js | 98.10 | 78.67 | 87.50 | 94-95 97-98 @@ -95,9 +95,9 @@ ℹ types.js | 100.00 | 100.00 | 100.00 | ℹ validator.js | 100.00 | 100.00 | 100.00 | ℹ stream | | | | -ℹ consume.js | 95.97 | 96.88 | 100.00 | 95-100 +ℹ consume.js | 92.35 | 89.19 | 100.00 | 108-113 133-139 ℹ interrupt.js | 100.00 | 100.00 | 100.00 | -ℹ loopDetector.js | 100.00 | 86.21 | 100.00 | +ℹ loopDetector.js | 100.00 | 100.00 | 100.00 | ℹ namespace.js | 100.00 | 100.00 | 100.00 | ℹ transformers | | | | ℹ index.js | 100.00 | 100.00 | 100.00 | @@ -185,7 +185,7 @@ ℹ xlsxParser.js | 82.21 | 83.33 | 88.89 | 47-48 93-112 130-137 150-156 ℹ zipExtractor.js | 82.32 | 77.78 | 66.67 | 62-63 104-107 124-142 149-152 ℹ git | | | | -ℹ index.js | 87.41 | 64.55 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 +ℹ index.js | 87.41 | 64.37 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 ℹ graphql | | | | ℹ index.js | 97.85 | 98.13 | 93.75 | 57-68 ℹ image | | | | @@ -284,6 +284,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.33 | 86.55 | 88.25 | +ℹ all files | 93.32 | 86.56 | 88.24 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/index.js b/index.js index 1a038395..d99b3771 100644 --- a/index.js +++ b/index.js @@ -19,7 +19,7 @@ import { import { resumeSession } from "./src/session/resume.js"; import { logger, flush as flushLogger } from "./src/shared/logger.js"; import { consumeStream } from "./src/stream/consume.js"; -import { createStreamLoopDetector } from "./src/stream/loopDetector.js"; +import { createSegmentLoopDetector } from "./src/stream/loopDetector.js"; // --------------------------------------------------------------------------- // CLI arguments @@ -301,7 +301,7 @@ async function runAgent(input, threadExtras, streamingCallback, signal) { const loopDetector = loopConfig.enabled === false ? undefined - : createStreamLoopDetector({ + : createSegmentLoopDetector({ windowSize: loopConfig.windowSize, threshold: loopConfig.threshold, ttlMs: loopConfig.ttlMs, diff --git a/src/config/schemas/loopDetection.js b/src/config/schemas/loopDetection.js index 66afcff9..df0d3e27 100644 --- a/src/config/schemas/loopDetection.js +++ b/src/config/schemas/loopDetection.js @@ -4,16 +4,17 @@ import { z } from "zod"; * Configuration for streaming loop detection. * * When `enabled` is false, no loop detection occurs. When enabled, streamed - * reasoning and message text are split into segments (sentences/lines), each - * segment is hashed into a rolling `windowSize` window, and a loop is flagged - * when a segment hash repeats more than `threshold` times. The stream is then + * events are coalesced into the app's message/reasoning segments (see + * docs/STREAMING.md), each completed segment is hashed into a rolling + * `windowSize` window, and a loop is flagged when a segment hash repeats more + * than `threshold` times. The stream is then * abandoned and the turn re-run with a "You're looping" nudge; after * `streamMaxNudges` failed nudges the context is compacted instead. */ export const LoopDetectionSchema = z .object({ enabled: z.boolean().default(true), - // Number of recent segment (sentence) hashes retained for loop detection. + // Number of recent completed-segment hashes retained for loop detection. windowSize: z.number().int().positive().default(50), // A hash older than this many ms is pruned from the window. ttlMs: z.number().int().positive().default(60000), diff --git a/src/stream/consume.js b/src/stream/consume.js index 7618d1f2..62e5395d 100644 --- a/src/stream/consume.js +++ b/src/stream/consume.js @@ -13,9 +13,11 @@ import { isSubagentNamespace } from "./namespace.js"; * @param {Function} [streamingCallback] - Callback for streamed chunks * @param {Object} [options] - Options * @param {boolean} [options.showToolResults=true] - Whether to emit `tool` events - * @param {{push: Function, reset: Function}} [options.loopDetector] - Streaming repetition - * detector. When it flags a loop the stream is abandoned (closing the underlying - * iterator) and `loopDetected` is returned true. + * @param {{push: Function, reset: Function}} [options.loopDetector] - Segment loop + * detector. Each non-subagent message/reasoning event is pushed as a segment + * (`{ type, content, id? }`, the shape the TUI coalesces). When it flags a loop + * the stream is abandoned (closing the underlying iterator) and `loopDetected` + * is returned true. * @returns {Promise<{content: string, reasoning: string, interrupted: boolean, lastEventType: string | null, loopDetected: boolean}>} */ export async function consumeStream(stream, streamingCallback, options = {}) { @@ -67,7 +69,16 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (!isSubagent) { content += text; - if (loopDetector?.push("message", text)) loopDetected = true; + const blockIndex = msg?.content?.[0]?.index; + if ( + loopDetector?.push({ + type: "message", + content: text, + ...(blockIndex != null ? { id: blockIndex } : {}), + }) + ) { + loopDetected = true; + } } if (streamingCallback) { @@ -89,7 +100,9 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (reasoningContent) { if (!isSubagent) { reasoning += reasoningContent; - if (loopDetector?.push("reasoning", reasoningContent)) loopDetected = true; + if (loopDetector?.push({ type: "reasoning", content: reasoningContent })) { + loopDetected = true; + } } if (streamingCallback) { streamingCallback({ @@ -106,7 +119,15 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (block?.type === "reasoning" && block.reasoning) { if (!isSubagent) { reasoning += block.reasoning; - if (loopDetector?.push("reasoning", block.reasoning)) loopDetected = true; + if ( + loopDetector?.push({ + type: "reasoning", + content: block.reasoning, + ...(block.id != null ? { id: block.id } : {}), + }) + ) { + loopDetected = true; + } } if (streamingCallback) { streamingCallback({ diff --git a/src/stream/loopDetector.js b/src/stream/loopDetector.js index 54441681..dfaff11a 100644 --- a/src/stream/loopDetector.js +++ b/src/stream/loopDetector.js @@ -1,105 +1,79 @@ import { createHash } from "node:crypto"; - -const BOUNDARY = /(?<=[.!?])\s+|\n+/; +import { coalesceSegments } from "../tui/messages.js"; /** - * Create a segment-hash repetition detector. + * Create a streaming loop detector that hashes stream segments. + * + * Stream events are folded into segments with the same `coalesceSegments` + * routine the TUI uses (see docs/STREAMING.md), so a segment here is exactly + * a `message` or `reasoning` block as the app defines it: chunks sharing an + * `id` merge into one block, and id-less reasoning splits on a grammatical + * sentence boundary. When a new segment starts, the previous one is complete; + * its content is hashed into a rolling window. A loop is flagged when the same + * segment (same type and content) completes more than `threshold` times. * - * Text (reasoning and message text, tracked separately by `kind`) is buffered - * and split into sentences/lines. Each completed segment is normalized and - * hashed into a rolling window; when one hash occurs more than `threshold` - * times the detector reports a loop. It is used both on live stream deltas - * (catching repetition within a single generation) and on completed assistant - * messages (catching repetition across turns), so one mechanism serves both. + * Tool segments are not hashed. A segment is never counted until it closes, so + * a single unbroken block cannot be flagged by this detector. * * @param {Object} [options] - Options - * @param {number} [options.windowSize=50] - Segment hashes retained per kind - * @param {number} [options.threshold=2] - Occurrences above which a loop is flagged - * @param {number} [options.minLength=24] - Segments shorter than this are ignored - * @param {number} [options.maxBuffer=500] - Force a split when no boundary appears + * @param {number} [options.windowSize=50] - Segment hashes retained + * @param {number} [options.threshold=2] - Completions above which a loop is flagged * @param {number} [options.ttlMs] - Hashes older than this are pruned (no expiry when unset) * @param {Function} [options.now] - Clock returning ms (injectable for tests) - * @returns {{ push: (kind: string, delta: string) => boolean, flush: (kind?: string) => boolean, reset: () => void }} Detector + * @returns {{ push: (segment: {type: string, content: string, id?: string | number}) => boolean, reset: () => void }} Detector */ -export function createStreamLoopDetector(options = {}) { +export function createSegmentLoopDetector(options = {}) { const windowSize = options.windowSize ?? 50; const threshold = options.threshold ?? 2; - const minLength = options.minLength ?? 24; - const maxBuffer = options.maxBuffer ?? 500; const ttlMs = options.ttlMs; const now = options.now ?? (() => Date.now()); - /** @type {Map} */ - const kinds = new Map(); + let segments = []; + /** @type {{hash: string, ts: number}[]} */ + let hashes = []; - function getKind(kind) { - let state = kinds.get(kind); - if (!state) { - state = { buffer: "", hashes: [] }; - kinds.set(kind, state); - } - return state; - } + /** + * Hash a completed segment into the window and report whether it repeats + * more than `threshold` times. + * @param {{type: string, content: string}} segment - The completed segment + * @returns {boolean} True when the segment is looping + */ + function record(segment) { + if (segment.type !== "message" && segment.type !== "reasoning") return false; + const content = segment.content.trim(); + if (content.length === 0) return false; - function record(state, segment) { - const normalized = segment.replace(/\s+/g, " ").trim().toLowerCase(); - if (normalized.length < minLength) return false; const nowMs = now(); if (ttlMs !== undefined) { - while (state.hashes.length > 0 && nowMs - state.hashes[0].ts > ttlMs) state.hashes.shift(); + while (hashes.length > 0 && nowMs - hashes[0].ts > ttlMs) hashes.shift(); } - const hash = createHash("sha256").update(normalized).digest("hex"); - state.hashes.push({ hash, ts: nowMs }); - while (state.hashes.length > windowSize) state.hashes.shift(); + const hash = createHash("sha256").update(`${segment.type}\0${content}`).digest("hex"); + hashes.push({ hash, ts: nowMs }); + while (hashes.length > windowSize) hashes.shift(); + let count = 0; - for (const entry of state.hashes) if (entry.hash === hash) count++; + for (const entry of hashes) if (entry.hash === hash) count++; return count > threshold; } return { /** - * Feed a delta (or a whole message). - * @param {string} kind - Kind, e.g. "reasoning" or "message" - * @param {string} delta - New text - * @returns {boolean} True when a loop has been detected - */ - push(kind, delta) { - if (typeof delta !== "string" || delta.length === 0) return false; - const state = getKind(kind); - state.buffer += delta; - - const parts = state.buffer.split(BOUNDARY); - state.buffer = parts.pop() ?? ""; - if (state.buffer.length > maxBuffer) { - parts.push(state.buffer); - state.buffer = ""; - } - - let looped = false; - for (const part of parts) if (record(state, part)) looped = true; - return looped; - }, - - /** - * Hash any buffered trailing text as a final segment. Call at the end of a - * complete message so its last (unterminated) segment is counted. - * @param {string} [kind] - Kind to flush; all kinds when omitted - * @returns {boolean} True when the flushed segment completes a loop + * Fold a stream event's segment in. If it starts a new segment, the + * previous one is complete and is hashed. + * @param {{type: string, content: string, id?: string | number}} segment - Incoming segment + * @returns {boolean} True when a completed segment is looping */ - flush(kind) { - let looped = false; - const targets = kind === undefined ? [...kinds.values()] : [getKind(kind)]; - for (const state of targets) { - const rest = state.buffer; - state.buffer = ""; - if (rest && record(state, rest)) looped = true; - } - return looped; + push(segment) { + const before = segments.length; + segments = coalesceSegments(segments, segment); + if (before > 0 && segments.length > before) return record(segments[before - 1]); + return false; }, /** Clear all state (call after a nudge so the next attempt starts fresh). */ reset() { - kinds.clear(); + segments = []; + hashes = []; }, }; } diff --git a/tests/unit/stream/loopDetector.test.js b/tests/unit/stream/loopDetector.test.js index e8a0843c..0776ffca 100644 --- a/tests/unit/stream/loopDetector.test.js +++ b/tests/unit/stream/loopDetector.test.js @@ -1,116 +1,177 @@ import { describe, it } from "node:test"; import assert from "node:assert"; -import { createStreamLoopDetector } from "../../../src/stream/loopDetector.js"; +import { createSegmentLoopDetector } from "../../../src/stream/loopDetector.js"; import { consumeStream } from "../../../src/stream/consume.js"; -const SENTENCE = "I need to verify the config schemas are set up correctly. "; +const R = (content, id) => ({ type: "reasoning", content, ...(id !== undefined ? { id } : {}) }); +const M = (content, id) => ({ type: "message", content, ...(id !== undefined ? { id } : {}) }); -describe("createStreamLoopDetector", () => { - it("flags a sentence repeated beyond the threshold", () => { - const d = createStreamLoopDetector({ threshold: 2 }); - assert.strictEqual(d.push("reasoning", SENTENCE), false); - assert.strictEqual(d.push("reasoning", SENTENCE), false); - // Third completed sentence (> threshold 2) is flagged. - assert.strictEqual(d.push("reasoning", SENTENCE), true); +/** Push a reasoning block with its own id, then a different one to close it. */ +const complete = (d, content, id) => d.push(R(content, id)); + +describe("createSegmentLoopDetector", () => { + it("does not count a segment until it is complete", () => { + const d = createSegmentLoopDetector({ threshold: 2 }); + // Chunks sharing an id are one segment; nothing has closed yet. + assert.strictEqual(d.push(R("same ", "a")), false); + assert.strictEqual(d.push(R("thought", "a")), false); + }); + + it("flags a segment that completes more than threshold times", () => { + const d = createSegmentLoopDetector({ threshold: 2 }); + // Each new id closes the previous block. Blocks 1..4 are identical; the + // 3rd completion of the same content exceeds threshold 2. + assert.strictEqual(complete(d, "same thought", "1"), false); // opens 1 + assert.strictEqual(complete(d, "same thought", "2"), false); // closes 1 (count 1) + assert.strictEqual(complete(d, "same thought", "3"), false); // closes 2 (count 2) + assert.strictEqual(complete(d, "same thought", "4"), true); // closes 3 (count 3) + }); + + it("merges chunks that share an id into one segment before hashing", () => { + const d = createSegmentLoopDetector({ threshold: 2 }); + const feed = (id) => { + const closed = d.push(R("same ", id)); // a new id closes the previous segment + d.push(R("thought", id)); + return closed; + }; + assert.strictEqual(feed("1"), false); + assert.strictEqual(feed("2"), false); // closes 1 ("same thought") + assert.strictEqual(feed("3"), false); // closes 2 + assert.strictEqual(feed("4"), true); // closes 3 }); - it("handles deltas split mid-sentence", () => { - const d = createStreamLoopDetector({ threshold: 2 }); - const text = SENTENCE.repeat(4); + it("splits id-less reasoning on a sentence boundary followed by a capital", () => { + const d = createSegmentLoopDetector({ threshold: 2 }); + // "Check it." then a capitalised chunk starts a new segment, closing the first. let flagged = false; - for (let i = 0; i < text.length; i += 7) { - if (d.push("reasoning", text.slice(i, i + 7))) flagged = true; + for (let i = 0; i < 4; i++) { + if (d.push(R("Check it."))) flagged = true; + if (d.push(R(" "))) flagged = true; + if (d.push(R("Again"))) flagged = true; + if (d.push(R(" now."))) flagged = true; } + // Segments alternate; the first sentence repeats past the threshold. assert.ok(flagged); }); - it("does not flag distinct sentences", () => { - const d = createStreamLoopDetector(); - for (let i = 0; i < 20; i++) { - assert.strictEqual(d.push("reasoning", `This is unique sentence number ${i} here. `), false); - } + it("does not flag distinct segments", () => { + const d = createSegmentLoopDetector(); + for (let i = 0; i < 20; i++) + assert.strictEqual(complete(d, `thought number ${i}`, String(i)), false); }); - it("ignores short sentences", () => { - const d = createStreamLoopDetector(); - for (let i = 0; i < 10; i++) assert.strictEqual(d.push("message", "Ok. "), false); + it("flags a very short segment such as 'ok'", () => { + const d = createSegmentLoopDetector({ threshold: 2 }); + const out = []; + for (let i = 0; i < 5; i++) out.push(complete(d, "ok", String(i))); + assert.ok(out.includes(true)); }); - it("tracks reasoning and message independently", () => { - const d = createStreamLoopDetector({ threshold: 2 }); - d.push("reasoning", SENTENCE); - d.push("reasoning", SENTENCE); - assert.strictEqual(d.push("message", SENTENCE), false); + it("ignores empty and whitespace-only segments", () => { + const d = createSegmentLoopDetector({ threshold: 2 }); + for (let i = 0; i < 10; i++) assert.strictEqual(complete(d, " \n ", String(i)), false); }); - it("flags a long repeat with no sentence boundary via maxBuffer", () => { - const d = createStreamLoopDetector({ threshold: 2, maxBuffer: 40 }); - const chunk = "x".repeat(41); - d.push("reasoning", chunk); - d.push("reasoning", chunk); - assert.strictEqual(d.push("reasoning", chunk), true); + it("tracks reasoning and message separately", () => { + const d = createSegmentLoopDetector({ threshold: 2 }); + d.push(R("same", "r1")); + d.push(R("same", "r2")); // closes r1: reasoning "same" x1 + // These identical-text messages close r2 (reasoning x2) and then m1. If + // type were ignored, the shared hash would reach 3 here and flag a loop. + assert.strictEqual(d.push(M("same", "m1")), false); + assert.strictEqual(d.push(M("same", "m2")), false); + assert.strictEqual(d.push(M("other", "m3")), false); }); - it("flush counts a trailing unterminated segment", () => { - const d = createStreamLoopDetector({ threshold: 2 }); - const tail = "a trailing segment with no terminal punctuation"; - for (let i = 0; i < 2; i++) { - d.push("message", tail); - assert.strictEqual(d.flush("message"), false); + it("does not hash tool segments", () => { + const d = createSegmentLoopDetector({ threshold: 2 }); + for (let i = 0; i < 10; i++) { + assert.strictEqual(d.push({ type: "tool", content: "result", name: "x" }), false); } - d.push("message", tail); - assert.strictEqual(d.flush("message"), true); + }); + + it("evicts the oldest hash when the window is exceeded", () => { + const d = createSegmentLoopDetector({ threshold: 2, windowSize: 3 }); + complete(d, "x", "1"); + complete(d, "y", "2"); + complete(d, "z", "3"); + complete(d, "x", "4"); + // "x" was evicted from the 3-entry window; repeating it does not trigger. + assert.strictEqual(complete(d, "w", "5"), false); }); it("prunes hashes older than ttlMs", () => { let clock = 0; - const d = createStreamLoopDetector({ threshold: 2, ttlMs: 100, now: () => clock }); - d.push("reasoning", SENTENCE); - d.push("reasoning", SENTENCE); + const d = createSegmentLoopDetector({ threshold: 2, ttlMs: 100, now: () => clock }); + complete(d, "same", "1"); + complete(d, "same", "2"); + complete(d, "same", "3"); clock = 500; - assert.strictEqual(d.push("reasoning", SENTENCE), false); + complete(d, "same", "4"); + assert.strictEqual(complete(d, "same", "5"), false); }); it("reset clears state", () => { - const d = createStreamLoopDetector({ threshold: 2 }); - d.push("reasoning", SENTENCE); - d.push("reasoning", SENTENCE); + const d = createSegmentLoopDetector({ threshold: 2 }); + complete(d, "same", "1"); + complete(d, "same", "2"); + complete(d, "same", "3"); d.reset(); - assert.strictEqual(d.push("reasoning", SENTENCE), false); + assert.strictEqual(complete(d, "same", "4"), false); + assert.strictEqual(complete(d, "same", "5"), false); }); }); describe("consumeStream with loopDetector", () => { - const reasoningChunk = (text) => [ + const reasoningBlock = (text, id) => [ [], "messages", - [{ type: "ai", text: "", additional_kwargs: { reasoning_content: text } }], + [{ type: "ai", text: "", content: [{ type: "reasoning", reasoning: text, id }] }], ]; - it("stops consuming and reports loopDetected", async () => { + it("stops consuming, closes the stream and reports loopDetected", async () => { let consumed = 0; let closed = false; const stream = (async function* () { try { for (let i = 0; i < 100; i++) { consumed++; - yield reasoningChunk(SENTENCE); + yield reasoningBlock("same thought", `b${i}`); } } finally { closed = true; } })(); const result = await consumeStream(stream, undefined, { - loopDetector: createStreamLoopDetector({ threshold: 2 }), + loopDetector: createSegmentLoopDetector({ threshold: 2 }), }); assert.strictEqual(result.loopDetected, true); - assert.strictEqual(consumed, 3); + assert.strictEqual(consumed, 4); assert.ok(closed, "underlying stream is closed"); }); + it("feeds Chat Completions reasoning_content as id-less segments", async () => { + const chunk = (text) => [ + [], + "messages", + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: text } }], + ]; + const stream = (async function* () { + for (let i = 0; i < 6; i++) { + yield chunk("Same thought here."); + yield chunk(" Next"); + yield chunk(" part."); + } + })(); + const result = await consumeStream(stream, undefined, { + loopDetector: createSegmentLoopDetector({ threshold: 2 }), + }); + assert.strictEqual(result.loopDetected, true); + }); + it("reports loopDetected false without a detector", async () => { const stream = (async function* () { - yield reasoningChunk(SENTENCE); + yield reasoningBlock("hello", "a"); })(); const result = await consumeStream(stream); assert.strictEqual(result.loopDetected, false); From 3c1091a1ee06f9b3f8948f257d31c8b7a4451104 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 18:48:01 -0500 Subject: [PATCH 03/20] test: replay captured reasoning loop through the loop detector Stream the repeating paragraph from madz_loop.md as multi-character reasoning_content deltas (1-20 chars, cut mid-word, no ids) through consumeStream with the segment detector. Covers one paragraph, two paragraphs, two sentences and one sentence repeating, with paragraph, line and space separators. These tests currently fail: the detector never closes a segment on an id-less delta stream. --- tests/fixtures/loop-reasoning.md | 94 +++++++ tests/unit/stream/loopDetector.sample.test.js | 230 ++++++++++++++++++ 2 files changed, 324 insertions(+) create mode 100644 tests/fixtures/loop-reasoning.md create mode 100644 tests/unit/stream/loopDetector.sample.test.js diff --git a/tests/fixtures/loop-reasoning.md b/tests/fixtures/loop-reasoning.md new file mode 100644 index 00000000..0ffb1b43 --- /dev/null +++ b/tests/fixtures/loop-reasoning.md @@ -0,0 +1,94 @@ +For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I'm realizing the core tension: the user wants a module-level cached config like jira uses, but the existing tests call `sqlImpl` without passing a policy, which would + trigger a real `loadConfig()` call and potentially fail or connect to an actual database. I need to figure out how to make the tests work with this pattern—either by having the tests provide a policy, or by + making the config loading safe when no policy is passed. + + The cleanest approach is to have `sqlImpl` read from `options.policy` if provided, otherwise fall back to the cached config, and then update the tests to pass a policy for the connection-related cases. + This keeps the implementation consistent with jira while making the test behavior explicit and predictable. I'm settling on a pragmatic approach: keep the existing schema structure with `postgres`, `mysql`, + `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so + environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the + + ⚡ 3 tool calls: file ×3 + + ⏱ 8m 26s + +───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + + [8:23 a.m.] You: + + you're looping + +───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + + [8:23 a.m.] Madz: + + 💭 Thinking… → \ No newline at end of file diff --git a/tests/unit/stream/loopDetector.sample.test.js b/tests/unit/stream/loopDetector.sample.test.js new file mode 100644 index 00000000..02d72feb --- /dev/null +++ b/tests/unit/stream/loopDetector.sample.test.js @@ -0,0 +1,230 @@ +/** + * Replays a real reasoning loop (tests/fixtures/loop-reasoning.md) through the + * loop detector the way it arrives in production: as a stream of small, + * multi-character `reasoning_content` deltas that split mid-word and never + * align to sentence or paragraph boundaries, with no block ids. + * + * The fixture is a TUI capture of the model repeating the same paragraph. When + * the loop happens it is one paragraph repeating; sometimes two sentences, + * sometimes one, sometimes more. Each shape is replayed here. + */ + +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { consumeStream } from "../../../src/stream/consume.js"; +import { createSegmentLoopDetector } from "../../../src/stream/loopDetector.js"; + +const FIXTURE = fileURLToPath(new URL("../../fixtures/loop-reasoning.md", import.meta.url)); + +/** + * Extract the reasoning paragraphs from the TUI capture. The capture is + * hard-wrapped and indented; a blank line separates paragraphs. Everything from + * the tool-call footer onward is TUI chrome, not model output. + * @returns {string[]} Paragraphs with wrapping collapsed + */ +function loadParagraphs() { + const text = readFileSync(FIXTURE, "utf8"); + const end = text.indexOf("⚡"); + const body = end === -1 ? text : text.slice(0, end); + return body + .split(/\n\s*\n/) + .map((p) => + p + .replace(/\s*\n\s*/g, " ") + .replace(/\s+/g, " ") + .trim(), + ) + .filter(Boolean); +} + +/** @returns {string[]} Sentences of a paragraph */ +const sentences = (paragraph) => paragraph.split(/(?<=[.!?])\s+/).filter(Boolean); + +/** + * Deterministic PRNG (mulberry32) so chunking is reproducible. + * @param {number} seed - Seed + * @returns {() => number} Function returning floats in [0, 1) + */ +function rng(seed) { + let a = seed; + return () => { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** + * Cut text into multi-character deltas of `min`..`max` characters. Cuts land + * anywhere, including mid-word, like real token streaming. + * @param {string} text - Text to stream + * @param {number} min - Minimum delta length + * @param {number} max - Maximum delta length + * @param {number} seed - PRNG seed + * @returns {string[]} Deltas whose concatenation equals `text` + */ +function toDeltas(text, min, max, seed) { + const rand = rng(seed); + const deltas = []; + for (let i = 0; i < text.length;) { + const size = min + Math.floor(rand() * (max - min + 1)); + deltas.push(text.slice(i, i + size)); + i += size; + } + return deltas; +} + +/** + * Build the agent stream tuples for reasoning deltas (Chat Completions shape: + * `additional_kwargs.reasoning_content`, no ids). + * @param {string[]} deltas - Reasoning deltas + * @param {{ consumed: number, closed: boolean }} probe - Records consumption + * @returns {AsyncIterable>} The stream + */ +function reasoningStream(deltas, probe) { + return (async function* () { + try { + for (const delta of deltas) { + probe.consumed++; + yield [ + [], + "messages", + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: delta } }], + ]; + } + } finally { + probe.closed = true; + } + })(); +} + +/** + * Stream `unit` repeated `repeats` times through consumeStream with a fresh detector. + * @param {string} unit - The repeating unit of reasoning + * @param {Object} opts - Options + * @param {number} opts.repeats - Times the unit repeats + * @param {string} opts.separator - Text between repeats + * @param {number} opts.min - Minimum delta length + * @param {number} opts.max - Maximum delta length + * @param {number} opts.seed - PRNG seed + * @returns {Promise<{ result: Object, probe: Object, total: number }>} Outcome + */ +async function replay(unit, { repeats, separator, min, max, seed }) { + const text = Array.from({ length: repeats }, () => unit).join(separator); + const deltas = toDeltas(text, min, max, seed); + const probe = { consumed: 0, closed: false }; + const detector = createSegmentLoopDetector({ threshold: 2 }); + const result = await consumeStream(reasoningStream(deltas, probe), undefined, { + loopDetector: detector, + }); + return { result, probe, total: deltas.length }; +} + +const paragraphs = loadParagraphs(); +const loopParagraph = paragraphs[0]; +const twoParagraphs = `${paragraphs[0]}\n\n${paragraphs[1]}`; +const oneSentence = sentences(paragraphs[0])[0]; +const twoSentences = sentences(paragraphs[0]).slice(0, 2).join(" "); + +// Delta sizes in characters: tiny, typical, and large token groups. +const DELTA_SIZES = [ + [1, 3], + [2, 8], + [5, 20], +]; +// How the model separates repeats: paragraph break, line break, or just a space. +const SEPARATORS = ["\n\n", "\n", " "]; + +describe("loop detector replaying the captured reasoning loop", () => { + it("loads the fixture", () => { + assert.ok(paragraphs.length >= 6, `expected several paragraphs, got ${paragraphs.length}`); + assert.ok(loopParagraph.length > 200, "first paragraph is the long repeating one"); + assert.ok( + paragraphs.filter((p) => p === loopParagraph).length >= 3, + "the fixture really repeats the paragraph", + ); + }); + + const shapes = [ + ["one paragraph", loopParagraph], + ["two paragraphs", twoParagraphs], + ["two sentences", twoSentences], + ["one sentence", oneSentence], + ]; + + for (const [label, unit] of shapes) { + for (const [min, max] of DELTA_SIZES) { + for (const separator of SEPARATORS) { + const sep = JSON.stringify(separator); + it(`detects ${label} repeating (deltas ${min}-${max} chars, separator ${sep})`, async () => { + const { result, probe, total } = await replay(unit, { + repeats: 8, + separator, + min, + max, + seed: 42, + }); + assert.strictEqual(result.loopDetected, true, "loop must be detected"); + assert.ok(probe.consumed < total, `stopped early (${probe.consumed} of ${total} deltas)`); + assert.ok(probe.closed, "the underlying stream is closed"); + }); + } + } + } + + it("detects the loop before the stream is mostly consumed", async () => { + const { probe, total, result } = await replay(loopParagraph, { + repeats: 10, + separator: "\n\n", + min: 2, + max: 8, + seed: 7, + }); + assert.strictEqual(result.loopDetected, true); + // threshold 2 means the 3rd repeat trips it; allow for the unit boundary. + assert.ok(probe.consumed <= total * 0.5, `consumed ${probe.consumed} of ${total}`); + }); + + it("does not flag the distinct paragraphs of the capture (no false positive)", async () => { + const unique = [...new Set(paragraphs)]; + assert.ok(unique.length >= 4); + const text = unique.join("\n\n"); + const probe = { consumed: 0, closed: false }; + const result = await consumeStream( + reasoningStream(toDeltas(text, 2, 8, 11), probe), + undefined, + { + loopDetector: createSegmentLoopDetector({ threshold: 2 }), + }, + ); + assert.strictEqual(result.loopDetected, false); + }); + + it("does not flag a paragraph that repeats only twice", async () => { + const { result } = await replay(loopParagraph, { + repeats: 2, + separator: "\n\n", + min: 2, + max: 8, + seed: 3, + }); + assert.strictEqual(result.loopDetected, false); + }); + + it("reproduces the same outcome regardless of chunking seed", async () => { + for (const seed of [1, 2, 3, 4, 5]) { + const { result } = await replay(loopParagraph, { + repeats: 8, + separator: "\n\n", + min: 1, + max: 12, + seed, + }); + assert.strictEqual(result.loopDetected, true, `seed ${seed}`); + } + }); +}); From 92316f909855ba4c77802c4e078b4e53ae32a643 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 18:53:36 -0500 Subject: [PATCH 04/20] fix: hash streamed deltas to detect looping segments A segment is one streamed delta (a small group of characters), not a sentence or block. Hash each delta into a bounded per-kind list and flag a loop when the tail is one unit repeated back-to-back more than threshold times, at any period. Default windowSize is now 1000 so three copies of a paragraph fit; ttlMs is removed. Replay lines 14-21 of the captured loop as random 4-10 char segments repeated, plus 1-20 char delta variants across paragraph, sentence and separator shapes. --- config.yaml | 3 +- coverage.txt | 4 +- index.js | 1 - src/config/schemas/loopDetection.js | 24 +-- src/stream/consume.js | 32 +-- src/stream/loopDetector.js | 96 ++++----- tests/unit/stream/loopDetector.sample.test.js | 24 ++- .../unit/stream/loopDetector.segments.test.js | 154 ++++++++++++++ tests/unit/stream/loopDetector.test.js | 195 +++++++++--------- 9 files changed, 341 insertions(+), 192 deletions(-) create mode 100644 tests/unit/stream/loopDetector.segments.test.js diff --git a/config.yaml b/config.yaml index dbfa6663..770361f6 100644 --- a/config.yaml +++ b/config.yaml @@ -59,8 +59,7 @@ loopDetection: enabled: true streamMaxNudges: 2 threshold: 2 - ttlMs: 60000 - windowSize: 50 + windowSize: 1000 memory: checkpointsDir: memory/checkpoints/ contextDir: memory/context/ diff --git a/coverage.txt b/coverage.txt index 4b9cc0d5..29398c48 100644 --- a/coverage.txt +++ b/coverage.txt @@ -95,7 +95,7 @@ ℹ types.js | 100.00 | 100.00 | 100.00 | ℹ validator.js | 100.00 | 100.00 | 100.00 | ℹ stream | | | | -ℹ consume.js | 92.35 | 89.19 | 100.00 | 108-113 133-139 +ℹ consume.js | 96.00 | 97.14 | 100.00 | 96-101 ℹ interrupt.js | 100.00 | 100.00 | 100.00 | ℹ loopDetector.js | 100.00 | 100.00 | 100.00 | ℹ namespace.js | 100.00 | 100.00 | 100.00 | @@ -284,6 +284,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.32 | 86.56 | 88.24 | +ℹ all files | 93.33 | 86.59 | 88.24 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/index.js b/index.js index d99b3771..51a5ce16 100644 --- a/index.js +++ b/index.js @@ -304,7 +304,6 @@ async function runAgent(input, threadExtras, streamingCallback, signal) { : createSegmentLoopDetector({ windowSize: loopConfig.windowSize, threshold: loopConfig.threshold, - ttlMs: loopConfig.ttlMs, }); const maxNudges = loopConfig.streamMaxNudges ?? 2; const threadConfig = { diff --git a/src/config/schemas/loopDetection.js b/src/config/schemas/loopDetection.js index df0d3e27..40db9dd4 100644 --- a/src/config/schemas/loopDetection.js +++ b/src/config/schemas/loopDetection.js @@ -3,22 +3,22 @@ import { z } from "zod"; /** * Configuration for streaming loop detection. * - * When `enabled` is false, no loop detection occurs. When enabled, streamed - * events are coalesced into the app's message/reasoning segments (see - * docs/STREAMING.md), each completed segment is hashed into a rolling - * `windowSize` window, and a loop is flagged when a segment hash repeats more - * than `threshold` times. The stream is then - * abandoned and the turn re-run with a "You're looping" nudge; after - * `streamMaxNudges` failed nudges the context is compacted instead. + * When `enabled` is false, no loop detection occurs. When enabled, every + * streamed delta (a segment: a small group of characters as it arrives) is + * hashed into a bounded per-kind list of `windowSize` hashes. A loop is flagged + * when the tail of that list is one unit repeated back-to-back more than + * `threshold` times. The stream is then abandoned and the turn re-run with a + * "You're looping" nudge; after `streamMaxNudges` failed nudges the context is + * compacted instead. */ export const LoopDetectionSchema = z .object({ enabled: z.boolean().default(true), - // Number of recent completed-segment hashes retained for loop detection. - windowSize: z.number().int().positive().default(50), - // A hash older than this many ms is pruned from the window. - ttlMs: z.number().int().positive().default(60000), - // A segment hash must appear more than this many times within the window to flag a loop. + // Number of recent segment (delta) hashes retained. Must hold + // `threshold + 1` copies of the longest repeating unit; a paragraph is on + // the order of 100 deltas. + windowSize: z.number().int().positive().default(1000), + // A unit must repeat back-to-back more than this many times to flag a loop. threshold: z.number().int().positive().default(2), // Max times a single streamed turn is aborted and re-nudged before compacting instead. streamMaxNudges: z.number().int().nonnegative().default(2), diff --git a/src/stream/consume.js b/src/stream/consume.js index 62e5395d..ee178ad5 100644 --- a/src/stream/consume.js +++ b/src/stream/consume.js @@ -14,10 +14,9 @@ import { isSubagentNamespace } from "./namespace.js"; * @param {Object} [options] - Options * @param {boolean} [options.showToolResults=true] - Whether to emit `tool` events * @param {{push: Function, reset: Function}} [options.loopDetector] - Segment loop - * detector. Each non-subagent message/reasoning event is pushed as a segment - * (`{ type, content, id? }`, the shape the TUI coalesces). When it flags a loop - * the stream is abandoned (closing the underlying iterator) and `loopDetected` - * is returned true. + * detector. Each non-subagent message/reasoning delta is pushed as a segment. + * When it flags a loop the stream is abandoned (closing the underlying iterator) + * and `loopDetected` is returned true. * @returns {Promise<{content: string, reasoning: string, interrupted: boolean, lastEventType: string | null, loopDetected: boolean}>} */ export async function consumeStream(stream, streamingCallback, options = {}) { @@ -69,16 +68,7 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (!isSubagent) { content += text; - const blockIndex = msg?.content?.[0]?.index; - if ( - loopDetector?.push({ - type: "message", - content: text, - ...(blockIndex != null ? { id: blockIndex } : {}), - }) - ) { - loopDetected = true; - } + if (loopDetector?.push("message", text)) loopDetected = true; } if (streamingCallback) { @@ -100,9 +90,7 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (reasoningContent) { if (!isSubagent) { reasoning += reasoningContent; - if (loopDetector?.push({ type: "reasoning", content: reasoningContent })) { - loopDetected = true; - } + if (loopDetector?.push("reasoning", reasoningContent)) loopDetected = true; } if (streamingCallback) { streamingCallback({ @@ -119,15 +107,7 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (block?.type === "reasoning" && block.reasoning) { if (!isSubagent) { reasoning += block.reasoning; - if ( - loopDetector?.push({ - type: "reasoning", - content: block.reasoning, - ...(block.id != null ? { id: block.id } : {}), - }) - ) { - loopDetected = true; - } + if (loopDetector?.push("reasoning", block.reasoning)) loopDetected = true; } if (streamingCallback) { streamingCallback({ diff --git a/src/stream/loopDetector.js b/src/stream/loopDetector.js index dfaff11a..31fda4e4 100644 --- a/src/stream/loopDetector.js +++ b/src/stream/loopDetector.js @@ -1,79 +1,73 @@ import { createHash } from "node:crypto"; -import { coalesceSegments } from "../tui/messages.js"; /** * Create a streaming loop detector that hashes stream segments. * - * Stream events are folded into segments with the same `coalesceSegments` - * routine the TUI uses (see docs/STREAMING.md), so a segment here is exactly - * a `message` or `reasoning` block as the app defines it: chunks sharing an - * `id` merge into one block, and id-less reasoning splits on a grammatical - * sentence boundary. When a new segment starts, the previous one is complete; - * its content is hashed into a rolling window. A loop is flagged when the same - * segment (same type and content) completes more than `threshold` times. + * A segment is one streamed delta: a small group of characters exactly as it + * arrives (not a sentence or paragraph). Each delta is hashed and appended to + * a bounded list, per stream kind (reasoning, message). A looping model + * re-emits the same sequence of segments, so a loop is a trailing run of the + * hash list that repeats back-to-back more than `threshold` times, whatever + * the length of the repeating unit (a word, a sentence, a paragraph). * - * Tool segments are not hashed. A segment is never counted until it closes, so - * a single unbroken block cannot be flagged by this detector. + * `windowSize` must be large enough to hold `threshold + 1` copies of the + * longest unit to be caught: a paragraph is on the order of 100 deltas. * * @param {Object} [options] - Options - * @param {number} [options.windowSize=50] - Segment hashes retained - * @param {number} [options.threshold=2] - Completions above which a loop is flagged - * @param {number} [options.ttlMs] - Hashes older than this are pruned (no expiry when unset) - * @param {Function} [options.now] - Clock returning ms (injectable for tests) - * @returns {{ push: (segment: {type: string, content: string, id?: string | number}) => boolean, reset: () => void }} Detector + * @param {number} [options.windowSize=1000] - Segment hashes retained per kind + * @param {number} [options.threshold=2] - Back-to-back repeats above which a loop is flagged + * @returns {{ push: (kind: string, delta: string) => boolean, reset: () => void }} Detector */ export function createSegmentLoopDetector(options = {}) { - const windowSize = options.windowSize ?? 50; + const windowSize = options.windowSize ?? 1000; const threshold = options.threshold ?? 2; - const ttlMs = options.ttlMs; - const now = options.now ?? (() => Date.now()); + const needed = threshold + 1; - let segments = []; - /** @type {{hash: string, ts: number}[]} */ - let hashes = []; + /** @type {Map} */ + const lists = new Map(); /** - * Hash a completed segment into the window and report whether it repeats - * more than `threshold` times. - * @param {{type: string, content: string}} segment - The completed segment - * @returns {boolean} True when the segment is looping + * Check whether the tail of the list is one unit repeated `needed` times. + * @param {string[]} list - Segment hashes, oldest first + * @returns {boolean} True when a repeating unit is found */ - function record(segment) { - if (segment.type !== "message" && segment.type !== "reasoning") return false; - const content = segment.content.trim(); - if (content.length === 0) return false; - - const nowMs = now(); - if (ttlMs !== undefined) { - while (hashes.length > 0 && nowMs - hashes[0].ts > ttlMs) hashes.shift(); + function isLooping(list) { + const n = list.length; + const last = list[n - 1]; + for (let period = 1; period * needed <= n; period++) { + // Cheap reject: the unit's last segment must recur exactly one period back. + if (list[n - 1 - period] !== last) continue; + let match = true; + for (let i = 1; i < period * (needed - 1) && match; i++) { + if (list[n - 1 - i] !== list[n - 1 - i - period]) match = false; + } + if (match) return true; } - const hash = createHash("sha256").update(`${segment.type}\0${content}`).digest("hex"); - hashes.push({ hash, ts: nowMs }); - while (hashes.length > windowSize) hashes.shift(); - - let count = 0; - for (const entry of hashes) if (entry.hash === hash) count++; - return count > threshold; + return false; } return { /** - * Fold a stream event's segment in. If it starts a new segment, the - * previous one is complete and is hashed. - * @param {{type: string, content: string, id?: string | number}} segment - Incoming segment - * @returns {boolean} True when a completed segment is looping + * Record a streamed segment. + * @param {string} kind - Stream kind, e.g. "reasoning" or "message" + * @param {string} delta - The segment exactly as streamed + * @returns {boolean} True when the stream is looping */ - push(segment) { - const before = segments.length; - segments = coalesceSegments(segments, segment); - if (before > 0 && segments.length > before) return record(segments[before - 1]); - return false; + push(kind, delta) { + if (typeof delta !== "string" || delta.length === 0) return false; + let list = lists.get(kind); + if (!list) { + list = []; + lists.set(kind, list); + } + list.push(createHash("sha256").update(delta).digest("hex")); + while (list.length > windowSize) list.shift(); + return isLooping(list); }, /** Clear all state (call after a nudge so the next attempt starts fresh). */ reset() { - segments = []; - hashes = []; + lists.clear(); }, }; } diff --git a/tests/unit/stream/loopDetector.sample.test.js b/tests/unit/stream/loopDetector.sample.test.js index 02d72feb..23afa646 100644 --- a/tests/unit/stream/loopDetector.sample.test.js +++ b/tests/unit/stream/loopDetector.sample.test.js @@ -4,8 +4,10 @@ * multi-character `reasoning_content` deltas that split mid-word and never * align to sentence or paragraph boundaries, with no block ids. * - * The fixture is a TUI capture of the model repeating the same paragraph. When - * the loop happens it is one paragraph repeating; sometimes two sentences, + * A segment is one streamed delta. A looping model re-emits the same tokens, + * so the unit is cut into deltas once and that same delta sequence repeats. + * The fixture is a TUI capture of the model repeating the same paragraph; when + * the loop happens it is one paragraph repeating, sometimes two sentences, * sometimes one, sometimes more. Each shape is replayed here. */ @@ -114,10 +116,22 @@ function reasoningStream(deltas, probe) { * @returns {Promise<{ result: Object, probe: Object, total: number }>} Outcome */ async function replay(unit, { repeats, separator, min, max, seed }) { - const text = Array.from({ length: repeats }, () => unit).join(separator); - const deltas = toDeltas(text, min, max, seed); + // A looping model re-emits the same tokens: chunk the unit once, then + // repeat that identical delta sequence. + const unitDeltas = toDeltas(unit, min, max, seed); + const sepDeltas = separator ? [separator] : []; + const deltas = []; + for (let i = 0; i < repeats; i++) { + if (i > 0) deltas.push(...sepDeltas); + deltas.push(...unitDeltas); + } const probe = { consumed: 0, closed: false }; - const detector = createSegmentLoopDetector({ threshold: 2 }); + // The window must hold threshold + 1 (3) back-to-back copies of the unit; + // size it to the unit so tiny deltas on long units are still testable. + const detector = createSegmentLoopDetector({ + threshold: 2, + windowSize: Math.max(1000, unitDeltas.length * 3 + sepDeltas.length * 2 + 10), + }); const result = await consumeStream(reasoningStream(deltas, probe), undefined, { loopDetector: detector, }); diff --git a/tests/unit/stream/loopDetector.segments.test.js b/tests/unit/stream/loopDetector.segments.test.js new file mode 100644 index 00000000..d4278b50 --- /dev/null +++ b/tests/unit/stream/loopDetector.segments.test.js @@ -0,0 +1,154 @@ +/** + * A segment is one streamed delta: a small group of characters. This test cuts + * lines 14-21 of the captured loop (tests/fixtures/loop-reasoning.md) once into + * random 4-10 character segments, then repeats that same segment array, which is + * what a looping model does (it re-emits the same tokens). The detector must flag + * the loop while the stream is still running. + */ + +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { consumeStream } from "../../../src/stream/consume.js"; +import { createSegmentLoopDetector } from "../../../src/stream/loopDetector.js"; + +// Lines 14-21 of the capture: two paragraphs, hard-wrapped by the TUI. +const TEXT = readFileSync( + fileURLToPath(new URL("../../fixtures/loop-reasoning.md", import.meta.url)), + "utf8", +) + .split("\n") + .slice(13, 21) + .map((line) => line.trim()) + .join(" ") + .replace(/\s+/g, " ") + .trim(); + +/** + * Deterministic PRNG (mulberry32). + * @param {number} seed - Seed + * @returns {() => number} Floats in [0, 1) + */ +function rng(seed) { + let a = seed; + return () => { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** + * Break text into random segments of `min`..`max` characters. + * @param {string} text - Text to break up + * @param {number} seed - PRNG seed + * @param {number} [min=4] - Minimum segment length + * @param {number} [max=10] - Maximum segment length + * @returns {string[]} Segments whose concatenation equals `text` + */ +function toSegments(text, seed, min = 4, max = 10) { + const rand = rng(seed); + const out = []; + for (let i = 0; i < text.length;) { + const size = min + Math.floor(rand() * (max - min + 1)); + out.push(text.slice(i, i + size)); + i += size; + } + return out; +} + +/** + * Wrap segments as a Chat Completions reasoning stream. + * @param {string[]} segments - Segments to emit + * @param {{ consumed: number, closed: boolean }} probe - Records consumption + * @returns {AsyncIterable>} The stream + */ +function reasoningStream(segments, probe) { + return (async function* () { + try { + for (const s of segments) { + probe.consumed++; + yield [ + [], + "messages", + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: s } }], + ]; + } + } finally { + probe.closed = true; + } + })(); +} + +const repeat = (segments, times) => Array.from({ length: times }, () => segments).flat(); + +describe("loop detector with the passage broken into 4-10 char segments", () => { + it("breaks the passage into segments that rebuild the text", () => { + const segments = toSegments(TEXT, 1); + assert.strictEqual(segments.join(""), TEXT); + assert.ok(segments.every((s) => s.length >= 1 && s.length <= 10)); + assert.ok( + segments.slice(0, -1).every((s) => s.length >= 4), + "all but the last are 4-10 chars", + ); + }); + + for (const seed of [1, 2, 3, 4, 5]) { + it(`flags the repeating segment array mid-stream (seed ${seed})`, async () => { + const segments = toSegments(TEXT, seed); + const stream = repeat(segments, 6); + const probe = { consumed: 0, closed: false }; + const result = await consumeStream(reasoningStream(stream, probe), undefined, { + loopDetector: createSegmentLoopDetector({ threshold: 2 }), + }); + assert.strictEqual(result.loopDetected, true); + // threshold 2: the third back-to-back copy completes the loop. + assert.strictEqual(probe.consumed, segments.length * 3); + assert.ok(probe.closed, "underlying stream is closed"); + }); + } + + it("does not flag the passage streamed once", async () => { + const probe = { consumed: 0, closed: false }; + const result = await consumeStream(reasoningStream(toSegments(TEXT, 1), probe), undefined, { + loopDetector: createSegmentLoopDetector({ threshold: 2 }), + }); + assert.strictEqual(result.loopDetected, false); + }); + + it("does not flag the passage repeated only twice", async () => { + const probe = { consumed: 0, closed: false }; + const result = await consumeStream( + reasoningStream(repeat(toSegments(TEXT, 1), 2), probe), + undefined, + { + loopDetector: createSegmentLoopDetector({ threshold: 2 }), + }, + ); + assert.strictEqual(result.loopDetected, false); + }); + + it("flags a repeat of only the first paragraph's segments", async () => { + const first = TEXT.split(". Writing")[0]; + const segments = toSegments(first, 9); + const probe = { consumed: 0, closed: false }; + const result = await consumeStream(reasoningStream(repeat(segments, 6), probe), undefined, { + loopDetector: createSegmentLoopDetector({ threshold: 2 }), + }); + assert.strictEqual(result.loopDetected, true); + assert.strictEqual(probe.consumed, segments.length * 3); + }); + + it("sizes the default window to hold three copies of the passage", () => { + const n = toSegments(TEXT, 1).length; + // Three back-to-back copies must fit in the default window to be detectable. + const defaultWindow = 1000; + assert.ok( + n * 3 <= defaultWindow, + `3 copies = ${n * 3} segments, default window ${defaultWindow}`, + ); + }); +}); diff --git a/tests/unit/stream/loopDetector.test.js b/tests/unit/stream/loopDetector.test.js index 0776ffca..a41b7f6d 100644 --- a/tests/unit/stream/loopDetector.test.js +++ b/tests/unit/stream/loopDetector.test.js @@ -3,131 +3,118 @@ import assert from "node:assert"; import { createSegmentLoopDetector } from "../../../src/stream/loopDetector.js"; import { consumeStream } from "../../../src/stream/consume.js"; -const R = (content, id) => ({ type: "reasoning", content, ...(id !== undefined ? { id } : {}) }); -const M = (content, id) => ({ type: "message", content, ...(id !== undefined ? { id } : {}) }); +/** Push every segment; return whether any push flagged a loop. */ +const feed = (d, segments, kind = "reasoning") => { + let flagged = false; + for (const s of segments) if (d.push(kind, s)) flagged = true; + return flagged; +}; -/** Push a reasoning block with its own id, then a different one to close it. */ -const complete = (d, content, id) => d.push(R(content, id)); +const repeat = (unit, times) => Array.from({ length: times }, () => unit).flat(); describe("createSegmentLoopDetector", () => { - it("does not count a segment until it is complete", () => { + it("flags a unit repeated back-to-back more than threshold times", () => { const d = createSegmentLoopDetector({ threshold: 2 }); - // Chunks sharing an id are one segment; nothing has closed yet. - assert.strictEqual(d.push(R("same ", "a")), false); - assert.strictEqual(d.push(R("thought", "a")), false); + const unit = ["alpha ", "beta ", "gamma "]; + assert.strictEqual(feed(d, repeat(unit, 2)), false); + assert.strictEqual(feed(createSegmentLoopDetector({ threshold: 2 }), repeat(unit, 3)), true); }); - it("flags a segment that completes more than threshold times", () => { + it("flags on the segment that completes the final repeat", () => { const d = createSegmentLoopDetector({ threshold: 2 }); - // Each new id closes the previous block. Blocks 1..4 are identical; the - // 3rd completion of the same content exceeds threshold 2. - assert.strictEqual(complete(d, "same thought", "1"), false); // opens 1 - assert.strictEqual(complete(d, "same thought", "2"), false); // closes 1 (count 1) - assert.strictEqual(complete(d, "same thought", "3"), false); // closes 2 (count 2) - assert.strictEqual(complete(d, "same thought", "4"), true); // closes 3 (count 3) + const unit = ["a1", "b2", "c3", "d4"]; + const stream = repeat(unit, 3); + const results = stream.map((s) => d.push("reasoning", s)); + assert.strictEqual(results.lastIndexOf(true), stream.length - 1); + assert.strictEqual(results.indexOf(true), stream.length - 1); }); - it("merges chunks that share an id into one segment before hashing", () => { + it("flags a single segment repeating, such as 'ok'", () => { const d = createSegmentLoopDetector({ threshold: 2 }); - const feed = (id) => { - const closed = d.push(R("same ", id)); // a new id closes the previous segment - d.push(R("thought", id)); - return closed; - }; - assert.strictEqual(feed("1"), false); - assert.strictEqual(feed("2"), false); // closes 1 ("same thought") - assert.strictEqual(feed("3"), false); // closes 2 - assert.strictEqual(feed("4"), true); // closes 3 - }); - - it("splits id-less reasoning on a sentence boundary followed by a capital", () => { - const d = createSegmentLoopDetector({ threshold: 2 }); - // "Check it." then a capitalised chunk starts a new segment, closing the first. - let flagged = false; - for (let i = 0; i < 4; i++) { - if (d.push(R("Check it."))) flagged = true; - if (d.push(R(" "))) flagged = true; - if (d.push(R("Again"))) flagged = true; - if (d.push(R(" now."))) flagged = true; + assert.strictEqual(d.push("message", "ok"), false); + assert.strictEqual(d.push("message", "ok"), false); + assert.strictEqual(d.push("message", "ok"), true); + }); + + it("flags units of different lengths", () => { + for (const len of [1, 2, 7, 40]) { + const unit = Array.from({ length: len }, (_, i) => `s${i}`); + assert.strictEqual( + feed(createSegmentLoopDetector({ threshold: 2 }), repeat(unit, 3)), + true, + `len ${len}`, + ); } - // Segments alternate; the first sentence repeats past the threshold. - assert.ok(flagged); }); it("does not flag distinct segments", () => { const d = createSegmentLoopDetector(); - for (let i = 0; i < 20; i++) - assert.strictEqual(complete(d, `thought number ${i}`, String(i)), false); - }); - - it("flags a very short segment such as 'ok'", () => { - const d = createSegmentLoopDetector({ threshold: 2 }); - const out = []; - for (let i = 0; i < 5; i++) out.push(complete(d, "ok", String(i))); - assert.ok(out.includes(true)); + const stream = Array.from({ length: 500 }, (_, i) => `segment-${i}`); + assert.strictEqual(feed(d, stream), false); }); - it("ignores empty and whitespace-only segments", () => { + it("does not flag a repeat that is not back-to-back", () => { const d = createSegmentLoopDetector({ threshold: 2 }); - for (let i = 0; i < 10; i++) assert.strictEqual(complete(d, " \n ", String(i)), false); + const unit = ["a1", "b2", "c3"]; + const stream = [...unit, "x1", ...unit, "x2", ...unit, "x3"]; + assert.strictEqual(feed(d, stream), false); }); it("tracks reasoning and message separately", () => { const d = createSegmentLoopDetector({ threshold: 2 }); - d.push(R("same", "r1")); - d.push(R("same", "r2")); // closes r1: reasoning "same" x1 - // These identical-text messages close r2 (reasoning x2) and then m1. If - // type were ignored, the shared hash would reach 3 here and flag a loop. - assert.strictEqual(d.push(M("same", "m1")), false); - assert.strictEqual(d.push(M("same", "m2")), false); - assert.strictEqual(d.push(M("other", "m3")), false); + // Interleaved kinds: each kind alone never repeats back-to-back 3 times. + assert.strictEqual(d.push("reasoning", "same"), false); + assert.strictEqual(d.push("message", "same"), false); + assert.strictEqual(d.push("reasoning", "same"), false); + assert.strictEqual(d.push("message", "same"), false); + assert.strictEqual(d.push("reasoning", "same"), true); }); - it("does not hash tool segments", () => { + it("ignores empty and non-string deltas", () => { const d = createSegmentLoopDetector({ threshold: 2 }); for (let i = 0; i < 10; i++) { - assert.strictEqual(d.push({ type: "tool", content: "result", name: "x" }), false); + assert.strictEqual(d.push("reasoning", ""), false); + assert.strictEqual(d.push("reasoning", undefined), false); + assert.strictEqual(d.push("reasoning", null), false); } }); - it("evicts the oldest hash when the window is exceeded", () => { - const d = createSegmentLoopDetector({ threshold: 2, windowSize: 3 }); - complete(d, "x", "1"); - complete(d, "y", "2"); - complete(d, "z", "3"); - complete(d, "x", "4"); - // "x" was evicted from the 3-entry window; repeating it does not trigger. - assert.strictEqual(complete(d, "w", "5"), false); + it("cannot see a repeat that does not fit in the window", () => { + const unit = Array.from({ length: 10 }, (_, i) => `s${i}`); + // Window of 25 cannot hold 3 copies (30 segments). + assert.strictEqual( + feed(createSegmentLoopDetector({ threshold: 2, windowSize: 25 }), repeat(unit, 6)), + false, + ); + assert.strictEqual( + feed(createSegmentLoopDetector({ threshold: 2, windowSize: 30 }), repeat(unit, 6)), + true, + ); }); - it("prunes hashes older than ttlMs", () => { - let clock = 0; - const d = createSegmentLoopDetector({ threshold: 2, ttlMs: 100, now: () => clock }); - complete(d, "same", "1"); - complete(d, "same", "2"); - complete(d, "same", "3"); - clock = 500; - complete(d, "same", "4"); - assert.strictEqual(complete(d, "same", "5"), false); + it("honours a higher threshold", () => { + const unit = ["a1", "b2"]; + assert.strictEqual(feed(createSegmentLoopDetector({ threshold: 4 }), repeat(unit, 4)), false); + assert.strictEqual(feed(createSegmentLoopDetector({ threshold: 4 }), repeat(unit, 5)), true); }); it("reset clears state", () => { const d = createSegmentLoopDetector({ threshold: 2 }); - complete(d, "same", "1"); - complete(d, "same", "2"); - complete(d, "same", "3"); + const unit = ["a1", "b2"]; + feed(d, repeat(unit, 2)); d.reset(); - assert.strictEqual(complete(d, "same", "4"), false); - assert.strictEqual(complete(d, "same", "5"), false); + assert.strictEqual(feed(d, unit), false); + assert.strictEqual(feed(d, repeat(unit, 2)), true); }); }); describe("consumeStream with loopDetector", () => { - const reasoningBlock = (text, id) => [ + const reasoning = (text) => [ [], "messages", - [{ type: "ai", text: "", content: [{ type: "reasoning", reasoning: text, id }] }], + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: text } }], ]; + const message = (text) => [[], "messages", [{ type: "ai", text, content: [] }]]; it("stops consuming, closes the stream and reports loopDetected", async () => { let consumed = 0; @@ -136,7 +123,7 @@ describe("consumeStream with loopDetector", () => { try { for (let i = 0; i < 100; i++) { consumed++; - yield reasoningBlock("same thought", `b${i}`); + yield reasoning("same"); } } finally { closed = true; @@ -146,22 +133,28 @@ describe("consumeStream with loopDetector", () => { loopDetector: createSegmentLoopDetector({ threshold: 2 }), }); assert.strictEqual(result.loopDetected, true); - assert.strictEqual(consumed, 4); + assert.strictEqual(consumed, 3); assert.ok(closed, "underlying stream is closed"); }); - it("feeds Chat Completions reasoning_content as id-less segments", async () => { - const chunk = (text) => [ + it("detects a looping message stream", async () => { + const stream = (async function* () { + for (let i = 0; i < 20; i++) yield message("again "); + })(); + const result = await consumeStream(stream, undefined, { + loopDetector: createSegmentLoopDetector({ threshold: 2 }), + }); + assert.strictEqual(result.loopDetected, true); + }); + + it("detects a looping Responses API reasoning block stream", async () => { + const block = (text) => [ [], "messages", - [{ type: "ai", text: "", additional_kwargs: { reasoning_content: text } }], + [{ type: "ai", text: "", content: [{ type: "reasoning", reasoning: text, id: "r1" }] }], ]; const stream = (async function* () { - for (let i = 0; i < 6; i++) { - yield chunk("Same thought here."); - yield chunk(" Next"); - yield chunk(" part."); - } + for (let i = 0; i < 20; i++) yield block("think "); })(); const result = await consumeStream(stream, undefined, { loopDetector: createSegmentLoopDetector({ threshold: 2 }), @@ -169,9 +162,25 @@ describe("consumeStream with loopDetector", () => { assert.strictEqual(result.loopDetected, true); }); + it("does not feed subagent events to the detector", async () => { + const stream = (async function* () { + for (let i = 0; i < 20; i++) { + yield [ + ["tools:abc"], + "messages", + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: "same" } }], + ]; + } + })(); + const result = await consumeStream(stream, undefined, { + loopDetector: createSegmentLoopDetector({ threshold: 2 }), + }); + assert.strictEqual(result.loopDetected, false); + }); + it("reports loopDetected false without a detector", async () => { const stream = (async function* () { - yield reasoningBlock("hello", "a"); + yield reasoning("hello"); })(); const result = await consumeStream(stream); assert.strictEqual(result.loopDetected, false); From 347e7e1c62fced709bf8ac6c4a6909e2b7e67d02 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 18:55:39 -0500 Subject: [PATCH 05/20] fix: hash streamed deltas to detect looping segments A segment is one streamed delta (a small group of characters), not a sentence or block. Hash each delta into a bounded per-kind list and flag a loop when the tail is one unit repeated back-to-back more than threshold times, at any period. Default windowSize is now 1000 so three copies of a paragraph fit; ttlMs is removed. Replay lines 14-21 of the captured loop as random 4-10 char segments repeated through Chat Completions reasoning_content, Responses API reasoning blocks and message text, asserting identical detection. --- coverage.txt | 6 +- .../unit/stream/loopDetector.segments.test.js | 149 +++++++++++------- 2 files changed, 99 insertions(+), 56 deletions(-) diff --git a/coverage.txt b/coverage.txt index 29398c48..3fb2e8f8 100644 --- a/coverage.txt +++ b/coverage.txt @@ -61,7 +61,7 @@ ℹ copilotRetryMiddleware.js | 88.52 | 79.49 | 83.33 | 75-88 ℹ index.js | 100.00 | 100.00 | 100.00 | ℹ modelInfo.js | 97.25 | 93.98 | 100.00 | 128-129 131-132 179-180 -ℹ openai.js | 92.72 | 75.71 | 78.57 | 65 90-92 138-139 145-146 233 236-238 257-259 267-271 279-280 292-295 +ℹ openai.js | 91.60 | 74.65 | 78.57 | 65 90-92 138-139 145-146 181-184 233 236-238 257-259 267-271 279-280 292-295 ℹ summarizationMiddleware.js | 97.01 | 94.00 | 83.33 | 107 126-128 149 ℹ tokenBudget.js | 97.14 | 85.71 | 82.35 | 50-51 83-84 ℹ tokenBudgetMiddleware.js | 98.10 | 78.67 | 87.50 | 94-95 97-98 @@ -144,7 +144,7 @@ ℹ confluence | | | | ℹ index.js | 98.59 | 82.50 | 83.33 | 89 191-192 ℹ cron | | | | -ℹ index.js | 93.59 | 85.08 | 82.50 | 47-48 87-92 199-200 255-256 278-280 294-295 367-368 517-519 548-549 559-562 569-570 585-589 600-602 614-616 627-628 +ℹ index.js | 93.59 | 84.53 | 82.50 | 47-48 87-92 199-200 255-256 278-280 294-295 367-368 517-519 548-549 559-562 569-570 585-589 600-602 614-616 627-628 ℹ data | | | | ℹ index.js | 98.90 | 96.49 | 100.00 | 158 240-241 ℹ date | | | | @@ -284,6 +284,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.33 | 86.59 | 88.24 | +ℹ all files | 93.32 | 86.57 | 88.24 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/tests/unit/stream/loopDetector.segments.test.js b/tests/unit/stream/loopDetector.segments.test.js index d4278b50..b13a3f30 100644 --- a/tests/unit/stream/loopDetector.segments.test.js +++ b/tests/unit/stream/loopDetector.segments.test.js @@ -4,6 +4,10 @@ * random 4-10 character segments, then repeats that same segment array, which is * what a looping model does (it re-emits the same tokens). The detector must flag * the loop while the stream is still running. + * + * The detector must behave identically for every kind of stream, so the same + * segment array is replayed through each: Chat Completions `reasoning_content`, + * Responses API reasoning blocks, and message text. */ import { describe, it } from "node:test"; @@ -61,21 +65,36 @@ function toSegments(text, seed, min = 4, max = 10) { } /** - * Wrap segments as a Chat Completions reasoning stream. + * The kinds of stream the detector sees, each wrapping a segment in the event + * shape `consumeStream` receives for that kind. + */ +const STREAM_KINDS = { + "reasoning_content (Chat Completions)": (s) => [ + [], + "messages", + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: s } }], + ], + "reasoning block (Responses API)": (s) => [ + [], + "messages", + [{ type: "ai", text: "", content: [{ type: "reasoning", reasoning: s, id: "r1" }] }], + ], + "message text": (s) => [[], "messages", [{ type: "ai", text: s, content: [] }]], +}; + +/** + * Wrap segments as a stream of the given kind. * @param {string[]} segments - Segments to emit + * @param {(s: string) => Array} wrap - Builds the event for one segment * @param {{ consumed: number, closed: boolean }} probe - Records consumption * @returns {AsyncIterable>} The stream */ -function reasoningStream(segments, probe) { +function streamOf(segments, wrap, probe) { return (async function* () { try { for (const s of segments) { probe.consumed++; - yield [ - [], - "messages", - [{ type: "ai", text: "", additional_kwargs: { reasoning_content: s } }], - ]; + yield wrap(s); } } finally { probe.closed = true; @@ -83,6 +102,20 @@ function reasoningStream(segments, probe) { })(); } +/** + * Stream segments through consumeStream with a fresh detector. + * @param {string[]} segments - Segments to emit + * @param {(s: string) => Array} wrap - Event builder for the stream kind + * @returns {Promise<{ result: Object, probe: Object }>} Outcome + */ +async function run(segments, wrap) { + const probe = { consumed: 0, closed: false }; + const result = await consumeStream(streamOf(segments, wrap, probe), undefined, { + loopDetector: createSegmentLoopDetector({ threshold: 2 }), + }); + return { result, probe }; +} + const repeat = (segments, times) => Array.from({ length: times }, () => segments).flat(); describe("loop detector with the passage broken into 4-10 char segments", () => { @@ -96,52 +129,6 @@ describe("loop detector with the passage broken into 4-10 char segments", () => ); }); - for (const seed of [1, 2, 3, 4, 5]) { - it(`flags the repeating segment array mid-stream (seed ${seed})`, async () => { - const segments = toSegments(TEXT, seed); - const stream = repeat(segments, 6); - const probe = { consumed: 0, closed: false }; - const result = await consumeStream(reasoningStream(stream, probe), undefined, { - loopDetector: createSegmentLoopDetector({ threshold: 2 }), - }); - assert.strictEqual(result.loopDetected, true); - // threshold 2: the third back-to-back copy completes the loop. - assert.strictEqual(probe.consumed, segments.length * 3); - assert.ok(probe.closed, "underlying stream is closed"); - }); - } - - it("does not flag the passage streamed once", async () => { - const probe = { consumed: 0, closed: false }; - const result = await consumeStream(reasoningStream(toSegments(TEXT, 1), probe), undefined, { - loopDetector: createSegmentLoopDetector({ threshold: 2 }), - }); - assert.strictEqual(result.loopDetected, false); - }); - - it("does not flag the passage repeated only twice", async () => { - const probe = { consumed: 0, closed: false }; - const result = await consumeStream( - reasoningStream(repeat(toSegments(TEXT, 1), 2), probe), - undefined, - { - loopDetector: createSegmentLoopDetector({ threshold: 2 }), - }, - ); - assert.strictEqual(result.loopDetected, false); - }); - - it("flags a repeat of only the first paragraph's segments", async () => { - const first = TEXT.split(". Writing")[0]; - const segments = toSegments(first, 9); - const probe = { consumed: 0, closed: false }; - const result = await consumeStream(reasoningStream(repeat(segments, 6), probe), undefined, { - loopDetector: createSegmentLoopDetector({ threshold: 2 }), - }); - assert.strictEqual(result.loopDetected, true); - assert.strictEqual(probe.consumed, segments.length * 3); - }); - it("sizes the default window to hold three copies of the passage", () => { const n = toSegments(TEXT, 1).length; // Three back-to-back copies must fit in the default window to be detectable. @@ -151,4 +138,60 @@ describe("loop detector with the passage broken into 4-10 char segments", () => `3 copies = ${n * 3} segments, default window ${defaultWindow}`, ); }); + + for (const [label, wrap] of Object.entries(STREAM_KINDS)) { + describe(label, () => { + for (const seed of [1, 2, 3, 4, 5]) { + it(`flags the repeating segment array mid-stream (seed ${seed})`, async () => { + const segments = toSegments(TEXT, seed); + const { result, probe } = await run(repeat(segments, 6), wrap); + assert.strictEqual(result.loopDetected, true); + // threshold 2: the third back-to-back copy completes the loop. + assert.strictEqual(probe.consumed, segments.length * 3); + assert.ok(probe.closed, "underlying stream is closed"); + }); + } + + it("does not flag the passage streamed once", async () => { + const { result } = await run(toSegments(TEXT, 1), wrap); + assert.strictEqual(result.loopDetected, false); + }); + + it("does not flag the passage repeated only twice", async () => { + const { result } = await run(repeat(toSegments(TEXT, 1), 2), wrap); + assert.strictEqual(result.loopDetected, false); + }); + + it("flags a repeat of only the first paragraph's segments", async () => { + const segments = toSegments(TEXT.split(". Writing")[0], 9); + const { result, probe } = await run(repeat(segments, 6), wrap); + assert.strictEqual(result.loopDetected, true); + assert.strictEqual(probe.consumed, segments.length * 3); + }); + }); + } + + it("detects the loop at the same segment in every kind of stream", async () => { + for (const seed of [1, 2, 3]) { + const stream = repeat(toSegments(TEXT, seed), 6); + const outcomes = []; + for (const wrap of Object.values(STREAM_KINDS)) { + const { result, probe } = await run(stream, wrap); + outcomes.push({ detected: result.loopDetected, consumed: probe.consumed }); + } + assert.ok(outcomes[0].detected, `seed ${seed}: loop detected`); + for (const o of outcomes) assert.deepStrictEqual(o, outcomes[0], `seed ${seed}`); + } + }); + + it("detects the loop at the same segment when pushed straight to the detector", () => { + const stream = repeat(toSegments(TEXT, 1), 6); + const at = {}; + for (const kind of ["reasoning", "message"]) { + const d = createSegmentLoopDetector({ threshold: 2 }); + at[kind] = stream.findIndex((s) => d.push(kind, s)); + } + assert.ok(at.reasoning > 0); + assert.strictEqual(at.message, at.reasoning); + }); }); From 55b4a83edcb9a76d3afb2d2496bc323940974f88 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 19:02:42 -0500 Subject: [PATCH 06/20] fix: send loop nudge from the TUI like the silent Continue Stop retrying inside runAgent. It now returns loopDetected from a single stream, and the conversation area dispatches a silent "You're looping" via handleChat with silentUser, mirroring the reasoning-only Continue. shouldNudgeLoop gates it on the last bubble and a nudge count bounded by loopDetection.streamMaxNudges, reset per session. --- coverage.txt | 8 ++-- index.js | 49 +++++++----------------- src/tui/conversationArea.js | 50 +++++++++++++++++++++++++ tests/unit/tui/conversationArea.test.js | 37 ++++++++++++++++++ 4 files changed, 104 insertions(+), 40 deletions(-) diff --git a/coverage.txt b/coverage.txt index 3fb2e8f8..99554658 100644 --- a/coverage.txt +++ b/coverage.txt @@ -61,7 +61,7 @@ ℹ copilotRetryMiddleware.js | 88.52 | 79.49 | 83.33 | 75-88 ℹ index.js | 100.00 | 100.00 | 100.00 | ℹ modelInfo.js | 97.25 | 93.98 | 100.00 | 128-129 131-132 179-180 -ℹ openai.js | 91.60 | 74.65 | 78.57 | 65 90-92 138-139 145-146 181-184 233 236-238 257-259 267-271 279-280 292-295 +ℹ openai.js | 92.72 | 75.71 | 78.57 | 65 90-92 138-139 145-146 233 236-238 257-259 267-271 279-280 292-295 ℹ summarizationMiddleware.js | 97.01 | 94.00 | 83.33 | 107 126-128 149 ℹ tokenBudget.js | 97.14 | 85.71 | 82.35 | 50-51 83-84 ℹ tokenBudgetMiddleware.js | 98.10 | 78.67 | 87.50 | 94-95 97-98 @@ -144,7 +144,7 @@ ℹ confluence | | | | ℹ index.js | 98.59 | 82.50 | 83.33 | 89 191-192 ℹ cron | | | | -ℹ index.js | 93.59 | 84.53 | 82.50 | 47-48 87-92 199-200 255-256 278-280 294-295 367-368 517-519 548-549 559-562 569-570 585-589 600-602 614-616 627-628 +ℹ index.js | 93.59 | 85.08 | 82.50 | 47-48 87-92 199-200 255-256 278-280 294-295 367-368 517-519 548-549 559-562 569-570 585-589 600-602 614-616 627-628 ℹ data | | | | ℹ index.js | 98.90 | 96.49 | 100.00 | 158 240-241 ℹ date | | | | @@ -212,7 +212,7 @@ ℹ reflection | | | | ℹ index.js | 94.98 | 83.02 | 90.00 | 56-60 125-126 142-143 197-198 ℹ sampling | | | | -ℹ index.js | 99.52 | 83.33 | 100.00 | 27 +ℹ index.js | 99.52 | 85.71 | 100.00 | 27 ℹ session | | | | ℹ index.js | 95.76 | 70.69 | 100.00 | 72-73 99-100 116-117 155-156 177-178 ℹ skills | | | | @@ -284,6 +284,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.32 | 86.57 | 88.24 | +ℹ all files | 93.33 | 86.60 | 88.24 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/index.js b/index.js index 51a5ce16..6116598d 100644 --- a/index.js +++ b/index.js @@ -305,48 +305,24 @@ async function runAgent(input, threadExtras, streamingCallback, signal) { windowSize: loopConfig.windowSize, threshold: loopConfig.threshold, }); - const maxNudges = loopConfig.streamMaxNudges ?? 2; - const threadConfig = { + const stream = await agent.stream(input, { configurable: { thread_id: sessionState.getSessionId(), ...threadExtras }, maxTokens: providerConfig.maxTokens, signal, recursionLimit: config.agent?.recursionLimit, streamMode: ["messages", "tools", "values"], subgraphs: true, - }; - - let currentInput = input; - let nudges = 0; - let result; - let content = ""; - let reasoning = ""; - - // A loop inside a single generation is caught while streaming: abandon the - // stream, then re-run with a nudge (or compact once nudges are exhausted). - while (true) { - const stream = await agent.stream(currentInput, threadConfig); - result = await consumeStream(stream, streamingCallback, { showToolResults, loopDetector }); - content += result.content; - reasoning += result.reasoning; - - if (!result.loopDetected || signal?.aborted) break; - - loopDetector?.reset(); - if (nudges >= maxNudges) { - logger.warn("[loopDetection] stream loop persists; compacting context"); - try { - await agent.compactContext(threadConfig); - } catch (err) { - logger.warn({ error: err?.message }, "[loopDetection] compaction failed"); - } - break; - } - nudges++; - logger.warn({ nudges }, "[loopDetection] stream loop detected; aborting and nudging"); - currentInput = { messages: [new HumanMessage("You're looping")] }; - } - - const { interrupted, lastEventType } = result; + }); + // A detected loop abandons the stream; the caller (the TUI) sends the silent + // nudge, the same way it sends the reasoning-only "Continue". + const { content, reasoning, interrupted, lastEventType, loopDetected } = await consumeStream( + stream, + streamingCallback, + { + showToolResults, + loopDetector, + }, + ); return { provider: providerName, content, @@ -354,6 +330,7 @@ async function runAgent(input, threadExtras, streamingCallback, signal) { tokens: { input: 0, output: 0 }, interrupted, lastEventType, + loopDetected, }; } diff --git a/src/tui/conversationArea.js b/src/tui/conversationArea.js index 51588168..2b0f1c55 100644 --- a/src/tui/conversationArea.js +++ b/src/tui/conversationArea.js @@ -69,6 +69,33 @@ export function shouldAutoContinue({ ); } +/** + * Decide whether to dispatch a silent loop nudge after a turn ends. + * + * Mirrors `shouldAutoContinue`: it fires only when the streaming bubble is the + * last in the message list AND the stream was abandoned because the loop + * detector flagged it AND the nudge limit is not exhausted. + * + * @param {Object} params - Decision inputs + * @param {boolean | undefined} params.loopDetected - Whether the loop detector flagged the stream + * @param {string | undefined} params.streamingMsgId - The streaming bubble's message ID + * @param {string | undefined} params.lastMessageId - The last message ID in the list + * @param {number} params.loopNudgeCount - Current loop nudge count + * @param {number} params.loopNudgeLimit - Max loop nudge count + * @returns {boolean} True when a silent loop nudge should be dispatched + */ +export function shouldNudgeLoop({ + loopDetected, + streamingMsgId, + lastMessageId, + loopNudgeCount, + loopNudgeLimit, +}) { + return ( + loopDetected === true && streamingMsgId === lastMessageId && loopNudgeCount < loopNudgeLimit + ); +} + /** * ConversationArea — owns all conversation and streaming state. * Communicates with InputArea exclusively via stable App-provided callbacks. @@ -126,6 +153,9 @@ const ConversationArea = forwardRef(function ConversationArea( // Counts silent "Continue" nudges dispatched when a turn ends on reasoning. // Bounded by config.agent.autoContinueLimit to prevent an infinite loop. const autoContinueCountRef = useRef(0); + // Counts silent loop nudges dispatched when the loop detector abandons a + // stream. Bounded by config.loopDetection.streamMaxNudges. + const loopNudgeCountRef = useRef(0); const toolMessageTokensRef = useRef(0); const contextUpdateTimerRef = useRef(null); const pendingContextRef = useRef({ content: "" }); @@ -493,6 +523,25 @@ const ConversationArea = forwardRef(function ConversationArea( completedToolCalls, ); + // Loop nudge: when the loop detector abandoned the stream, nudge the model + // with a silent "You're looping", the same way the reasoning-only + // "Continue" below is dispatched. Scoped to the LAST bubble and bounded by + // config.loopDetection.streamMaxNudges. + const loopNudgeLimit = config?.loopDetection?.streamMaxNudges ?? 2; + if ( + shouldNudgeLoop({ + loopDetected: response?.loopDetected, + streamingMsgId: streamingMsgIdRef.current, + lastMessageId: messageListRef.current?.getLastMessageId(), + loopNudgeCount: loopNudgeCountRef.current, + loopNudgeLimit, + }) + ) { + loopNudgeCountRef.current += 1; + await handleChat("You're looping", { silentUser: true }); + return; + } + // Reasoning-only auto-continue: when the last bubble ends on reasoning // (the model produced no message content), nudge it with a silent // "Continue" so it produces an actual response. Scoped to the LAST @@ -574,6 +623,7 @@ const ConversationArea = forwardRef(function ConversationArea( // not per-process. A reasoning loop is a per-thread phenomenon; a fresh // session must not inherit a counter exhausted by a prior thread. autoContinueCountRef.current = 0; + loopNudgeCountRef.current = 0; messageListRef.current?.clear(); // Recompute the context size for the fresh thread rather than hard-setting // 0. The system prompt (SYSTEM_PROMPT.md + AGENTS.md + memory context) is diff --git a/tests/unit/tui/conversationArea.test.js b/tests/unit/tui/conversationArea.test.js index 465c7118..12591047 100644 --- a/tests/unit/tui/conversationArea.test.js +++ b/tests/unit/tui/conversationArea.test.js @@ -41,6 +41,43 @@ describe("countToolMessageTokens", () => { }); }); +describe("shouldNudgeLoop", () => { + const base = { + loopDetected: true, + streamingMsgId: "3", + lastMessageId: "3", + loopNudgeCount: 0, + loopNudgeLimit: 2, + }; + + it("returns true when the detector flagged the last bubble and the limit is not exhausted", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual(mod.shouldNudgeLoop(base), true); + }); + + it("returns false when no loop was detected", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual(mod.shouldNudgeLoop({ ...base, loopDetected: false }), false); + assert.strictEqual(mod.shouldNudgeLoop({ ...base, loopDetected: undefined }), false); + }); + + it("returns false when the streaming bubble is not the last message", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual(mod.shouldNudgeLoop({ ...base, lastMessageId: "4" }), false); + }); + + it("returns false once the nudge limit is exhausted", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual(mod.shouldNudgeLoop({ ...base, loopNudgeCount: 2 }), false); + assert.strictEqual(mod.shouldNudgeLoop({ ...base, loopNudgeCount: 1 }), true); + }); + + it("never nudges when the limit is 0", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual(mod.shouldNudgeLoop({ ...base, loopNudgeLimit: 0 }), false); + }); +}); + describe("shouldAutoContinue", () => { it("returns true when the last bubble ends on reasoning and the guard is not exhausted", async () => { const mod = await import("../../../src/tui/conversationArea.js"); From 8668c7926bbaebe78ef6fcbd2c95e06d1055c3a8 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 19:05:53 -0500 Subject: [PATCH 07/20] fix: read loop nudge limit once from config Compute loopNudgeLimit once per component instead of per turn, and drop the hardcoded ?? 2 fallback; the schema already defaults streamMaxNudges. --- src/tui/conversationArea.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tui/conversationArea.js b/src/tui/conversationArea.js index 2b0f1c55..b0757ce3 100644 --- a/src/tui/conversationArea.js +++ b/src/tui/conversationArea.js @@ -156,6 +156,7 @@ const ConversationArea = forwardRef(function ConversationArea( // Counts silent loop nudges dispatched when the loop detector abandons a // stream. Bounded by config.loopDetection.streamMaxNudges. const loopNudgeCountRef = useRef(0); + const loopNudgeLimit = config?.loopDetection?.streamMaxNudges; const toolMessageTokensRef = useRef(0); const contextUpdateTimerRef = useRef(null); const pendingContextRef = useRef({ content: "" }); @@ -527,7 +528,6 @@ const ConversationArea = forwardRef(function ConversationArea( // with a silent "You're looping", the same way the reasoning-only // "Continue" below is dispatched. Scoped to the LAST bubble and bounded by // config.loopDetection.streamMaxNudges. - const loopNudgeLimit = config?.loopDetection?.streamMaxNudges ?? 2; if ( shouldNudgeLoop({ loopDetected: response?.loopDetected, From c06c05a4a918addf45eb5e72628d957fa1eb1bb3 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 19:07:38 -0500 Subject: [PATCH 08/20] fix: read autoContinueLimit once from config Compute autoContinueLimit once per component instead of per turn, and drop the hardcoded ?? 1000 fallback; the agent schema already defaults it. --- coverage.txt | 4 ++-- src/tui/conversationArea.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/coverage.txt b/coverage.txt index 99554658..fe4d3309 100644 --- a/coverage.txt +++ b/coverage.txt @@ -187,7 +187,7 @@ ℹ git | | | | ℹ index.js | 87.41 | 64.37 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 ℹ graphql | | | | -ℹ index.js | 97.85 | 98.13 | 93.75 | 57-68 +ℹ index.js | 97.50 | 97.17 | 93.75 | 47-48 57-68 ℹ image | | | | ℹ index.js | 100.00 | 100.00 | 50.00 | ℹ readImage.js | 94.16 | 72.22 | 100.00 | 79-80 88-89 95-96 103-104 @@ -284,6 +284,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.33 | 86.60 | 88.24 | +ℹ all files | 93.33 | 86.59 | 88.24 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/src/tui/conversationArea.js b/src/tui/conversationArea.js index b0757ce3..470ecee7 100644 --- a/src/tui/conversationArea.js +++ b/src/tui/conversationArea.js @@ -153,6 +153,7 @@ const ConversationArea = forwardRef(function ConversationArea( // Counts silent "Continue" nudges dispatched when a turn ends on reasoning. // Bounded by config.agent.autoContinueLimit to prevent an infinite loop. const autoContinueCountRef = useRef(0); + const autoContinueLimit = config?.agent?.autoContinueLimit; // Counts silent loop nudges dispatched when the loop detector abandons a // stream. Bounded by config.loopDetection.streamMaxNudges. const loopNudgeCountRef = useRef(0); @@ -547,7 +548,6 @@ const ConversationArea = forwardRef(function ConversationArea( // "Continue" so it produces an actual response. Scoped to the LAST // bubble only — earlier incomplete bubbles must not trigger it. Bounded // by config.agent.autoContinueLimit to prevent an infinite loop. - const autoContinueLimit = config?.agent?.autoContinueLimit ?? 1000; if ( shouldAutoContinue({ lastEventType: response?.lastEventType, From c5c9897dac051fdfab485bed90757e116360111e Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 19:12:55 -0500 Subject: [PATCH 09/20] fix: restore loop detection escalation and ttl config Put escalationCount, escalationWindowMs and ttlMs back in the schema and config.yaml, and wire them up. The detector prunes segment hashes older than ttlMs. The TUI tracks nudge timestamps and compacts the context once escalationCount nudges fall within escalationWindowMs, replacing streamMaxNudges. --- config.yaml | 4 +- coverage.txt | 6 +- index.js | 1 + src/config/schemas/loopDetection.js | 15 +++-- src/stream/loopDetector.js | 22 ++++++-- src/tui/conversationArea.js | 75 +++++++++++++++---------- tests/unit/stream/loopDetector.test.js | 19 +++++++ tests/unit/tui/conversationArea.test.js | 58 ++++++++++++++----- 8 files changed, 141 insertions(+), 59 deletions(-) diff --git a/config.yaml b/config.yaml index 770361f6..538ecbe7 100644 --- a/config.yaml +++ b/config.yaml @@ -57,8 +57,10 @@ jira: token: "" # personal access token, sent as Bearer (JIRA_TOKEN) loopDetection: enabled: true - streamMaxNudges: 2 + escalationCount: 3 + escalationWindowMs: 60000 threshold: 2 + ttlMs: 60000 windowSize: 1000 memory: checkpointsDir: memory/checkpoints/ diff --git a/coverage.txt b/coverage.txt index fe4d3309..840a37f1 100644 --- a/coverage.txt +++ b/coverage.txt @@ -8,7 +8,7 @@ ℹ compaction.js | 98.84 | 78.43 | 100.00 | 89-90 ℹ contextBackend.js | 100.00 | 100.00 | 100.00 | ℹ coreBackend.js | 100.00 | 100.00 | 100.00 | -ℹ deepAgents.js | 93.72 | 68.00 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 +ℹ deepAgents.js | 93.72 | 69.39 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 ℹ evictionCeiling.js | 100.00 | 100.00 | 100.00 | ℹ skillsMiddleware.js | 100.00 | 87.50 | 100.00 | ℹ config | | | | @@ -187,7 +187,7 @@ ℹ git | | | | ℹ index.js | 87.41 | 64.37 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 ℹ graphql | | | | -ℹ index.js | 97.50 | 97.17 | 93.75 | 47-48 57-68 +ℹ index.js | 97.85 | 98.13 | 93.75 | 57-68 ℹ image | | | | ℹ index.js | 100.00 | 100.00 | 50.00 | ℹ readImage.js | 94.16 | 72.22 | 100.00 | 79-80 88-89 95-96 103-104 @@ -284,6 +284,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.33 | 86.59 | 88.24 | +ℹ all files | 93.34 | 86.63 | 88.24 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/index.js b/index.js index 6116598d..006fb3da 100644 --- a/index.js +++ b/index.js @@ -304,6 +304,7 @@ async function runAgent(input, threadExtras, streamingCallback, signal) { : createSegmentLoopDetector({ windowSize: loopConfig.windowSize, threshold: loopConfig.threshold, + ttlMs: loopConfig.ttlMs, }); const stream = await agent.stream(input, { configurable: { thread_id: sessionState.getSessionId(), ...threadExtras }, diff --git a/src/config/schemas/loopDetection.js b/src/config/schemas/loopDetection.js index 40db9dd4..b4a7965b 100644 --- a/src/config/schemas/loopDetection.js +++ b/src/config/schemas/loopDetection.js @@ -7,9 +7,9 @@ import { z } from "zod"; * streamed delta (a segment: a small group of characters as it arrives) is * hashed into a bounded per-kind list of `windowSize` hashes. A loop is flagged * when the tail of that list is one unit repeated back-to-back more than - * `threshold` times. The stream is then abandoned and the turn re-run with a - * "You're looping" nudge; after `streamMaxNudges` failed nudges the context is - * compacted instead. + * `threshold` times. The stream is then abandoned and the TUI sends a silent + * "You're looping" nudge. If `escalationCount` nudges occur within + * `escalationWindowMs`, the context is compacted instead of nudging again. */ export const LoopDetectionSchema = z .object({ @@ -18,10 +18,15 @@ export const LoopDetectionSchema = z // `threshold + 1` copies of the longest repeating unit; a paragraph is on // the order of 100 deltas. windowSize: z.number().int().positive().default(1000), + // A hash older than this many ms is pruned from the window, so past + // content cannot cause false positives as new content streams in. + ttlMs: z.number().int().positive().default(60000), // A unit must repeat back-to-back more than this many times to flag a loop. threshold: z.number().int().positive().default(2), - // Max times a single streamed turn is aborted and re-nudged before compacting instead. - streamMaxNudges: z.number().int().nonnegative().default(2), + // Number of loop nudges within the escalation window before compacting. + escalationCount: z.number().int().positive().default(3), + // Rolling window (ms) for counting loop nudges before escalation. + escalationWindowMs: z.number().int().positive().default(60000), }) .strict() .default({}); diff --git a/src/stream/loopDetector.js b/src/stream/loopDetector.js index 31fda4e4..6239daec 100644 --- a/src/stream/loopDetector.js +++ b/src/stream/loopDetector.js @@ -12,34 +12,40 @@ import { createHash } from "node:crypto"; * * `windowSize` must be large enough to hold `threshold + 1` copies of the * longest unit to be caught: a paragraph is on the order of 100 deltas. + * Hashes older than `ttlMs` are pruned so past content cannot form a false + * positive with new content. * * @param {Object} [options] - Options * @param {number} [options.windowSize=1000] - Segment hashes retained per kind * @param {number} [options.threshold=2] - Back-to-back repeats above which a loop is flagged + * @param {number} [options.ttlMs] - Hashes older than this are pruned (no expiry when unset) + * @param {Function} [options.now] - Clock returning ms (injectable for tests) * @returns {{ push: (kind: string, delta: string) => boolean, reset: () => void }} Detector */ export function createSegmentLoopDetector(options = {}) { const windowSize = options.windowSize ?? 1000; const threshold = options.threshold ?? 2; + const ttlMs = options.ttlMs; + const now = options.now ?? (() => Date.now()); const needed = threshold + 1; - /** @type {Map} */ + /** @type {Map} */ const lists = new Map(); /** * Check whether the tail of the list is one unit repeated `needed` times. - * @param {string[]} list - Segment hashes, oldest first + * @param {{hash: string, ts: number}[]} list - Segment hashes, oldest first * @returns {boolean} True when a repeating unit is found */ function isLooping(list) { const n = list.length; - const last = list[n - 1]; + const last = list[n - 1].hash; for (let period = 1; period * needed <= n; period++) { // Cheap reject: the unit's last segment must recur exactly one period back. - if (list[n - 1 - period] !== last) continue; + if (list[n - 1 - period].hash !== last) continue; let match = true; for (let i = 1; i < period * (needed - 1) && match; i++) { - if (list[n - 1 - i] !== list[n - 1 - i - period]) match = false; + if (list[n - 1 - i].hash !== list[n - 1 - i - period].hash) match = false; } if (match) return true; } @@ -60,7 +66,11 @@ export function createSegmentLoopDetector(options = {}) { list = []; lists.set(kind, list); } - list.push(createHash("sha256").update(delta).digest("hex")); + const nowMs = now(); + if (ttlMs !== undefined) { + while (list.length > 0 && nowMs - list[0].ts > ttlMs) list.shift(); + } + list.push({ hash: createHash("sha256").update(delta).digest("hex"), ts: nowMs }); while (list.length > windowSize) list.shift(); return isLooping(list); }, diff --git a/src/tui/conversationArea.js b/src/tui/conversationArea.js index 470ecee7..f781eff9 100644 --- a/src/tui/conversationArea.js +++ b/src/tui/conversationArea.js @@ -70,30 +70,39 @@ export function shouldAutoContinue({ } /** - * Decide whether to dispatch a silent loop nudge after a turn ends. + * Decide what to do after a turn when the loop detector may have abandoned the stream. * - * Mirrors `shouldAutoContinue`: it fires only when the streaming bubble is the + * Mirrors `shouldAutoContinue`: it acts only when the streaming bubble is the * last in the message list AND the stream was abandoned because the loop - * detector flagged it AND the nudge limit is not exhausted. + * detector flagged it. If `escalationCount` nudges already happened within + * `escalationWindowMs`, the context is compacted instead of nudging again. * * @param {Object} params - Decision inputs * @param {boolean | undefined} params.loopDetected - Whether the loop detector flagged the stream * @param {string | undefined} params.streamingMsgId - The streaming bubble's message ID * @param {string | undefined} params.lastMessageId - The last message ID in the list - * @param {number} params.loopNudgeCount - Current loop nudge count - * @param {number} params.loopNudgeLimit - Max loop nudge count - * @returns {boolean} True when a silent loop nudge should be dispatched + * @param {number[]} params.nudges - Timestamps (ms) of previous loop nudges + * @param {number} params.now - Current time in ms + * @param {number} params.escalationCount - Nudges within the window before compacting + * @param {number} params.escalationWindowMs - Rolling window (ms) for counting nudges + * @returns {{ action: "none" | "nudge" | "compact", nudges: number[] }} The action and the + * updated nudge timestamps (pruned to the window) */ -export function shouldNudgeLoop({ +export function decideLoopAction({ loopDetected, streamingMsgId, lastMessageId, - loopNudgeCount, - loopNudgeLimit, + nudges, + now, + escalationCount, + escalationWindowMs, }) { - return ( - loopDetected === true && streamingMsgId === lastMessageId && loopNudgeCount < loopNudgeLimit - ); + if (loopDetected !== true || streamingMsgId !== lastMessageId) { + return { action: "none", nudges }; + } + const recent = nudges.filter((ts) => now - ts <= escalationWindowMs); + if (recent.length >= escalationCount) return { action: "compact", nudges: [] }; + return { action: "nudge", nudges: [...recent, now] }; } /** @@ -154,10 +163,12 @@ const ConversationArea = forwardRef(function ConversationArea( // Bounded by config.agent.autoContinueLimit to prevent an infinite loop. const autoContinueCountRef = useRef(0); const autoContinueLimit = config?.agent?.autoContinueLimit; - // Counts silent loop nudges dispatched when the loop detector abandons a - // stream. Bounded by config.loopDetection.streamMaxNudges. - const loopNudgeCountRef = useRef(0); - const loopNudgeLimit = config?.loopDetection?.streamMaxNudges; + // Timestamps of silent loop nudges dispatched when the loop detector abandons + // a stream. `escalationCount` nudges within `escalationWindowMs` compact the + // context instead of nudging again. + const loopNudgesRef = useRef([]); + const escalationCount = config?.loopDetection?.escalationCount; + const escalationWindowMs = config?.loopDetection?.escalationWindowMs; const toolMessageTokensRef = useRef(0); const contextUpdateTimerRef = useRef(null); const pendingContextRef = useRef({ content: "" }); @@ -527,18 +538,24 @@ const ConversationArea = forwardRef(function ConversationArea( // Loop nudge: when the loop detector abandoned the stream, nudge the model // with a silent "You're looping", the same way the reasoning-only - // "Continue" below is dispatched. Scoped to the LAST bubble and bounded by - // config.loopDetection.streamMaxNudges. - if ( - shouldNudgeLoop({ - loopDetected: response?.loopDetected, - streamingMsgId: streamingMsgIdRef.current, - lastMessageId: messageListRef.current?.getLastMessageId(), - loopNudgeCount: loopNudgeCountRef.current, - loopNudgeLimit, - }) - ) { - loopNudgeCountRef.current += 1; + // "Continue" below is dispatched. Scoped to the LAST bubble. After + // `escalationCount` nudges within `escalationWindowMs`, compact the + // context instead of nudging again. + const loop = decideLoopAction({ + loopDetected: response?.loopDetected, + streamingMsgId: streamingMsgIdRef.current, + lastMessageId: messageListRef.current?.getLastMessageId(), + nudges: loopNudgesRef.current, + now: Date.now(), + escalationCount, + escalationWindowMs, + }); + loopNudgesRef.current = loop.nudges; + if (loop.action === "compact") { + await handleCompact(); + return; + } + if (loop.action === "nudge") { await handleChat("You're looping", { silentUser: true }); return; } @@ -623,7 +640,7 @@ const ConversationArea = forwardRef(function ConversationArea( // not per-process. A reasoning loop is a per-thread phenomenon; a fresh // session must not inherit a counter exhausted by a prior thread. autoContinueCountRef.current = 0; - loopNudgeCountRef.current = 0; + loopNudgesRef.current = []; messageListRef.current?.clear(); // Recompute the context size for the fresh thread rather than hard-setting // 0. The system prompt (SYSTEM_PROMPT.md + AGENTS.md + memory context) is diff --git a/tests/unit/stream/loopDetector.test.js b/tests/unit/stream/loopDetector.test.js index a41b7f6d..1be1fad3 100644 --- a/tests/unit/stream/loopDetector.test.js +++ b/tests/unit/stream/loopDetector.test.js @@ -98,6 +98,25 @@ describe("createSegmentLoopDetector", () => { assert.strictEqual(feed(createSegmentLoopDetector({ threshold: 4 }), repeat(unit, 5)), true); }); + it("prunes hashes older than ttlMs", () => { + let clock = 0; + const d = createSegmentLoopDetector({ threshold: 2, ttlMs: 100, now: () => clock }); + const unit = ["a1", "b2"]; + for (const s of [...unit, ...unit]) d.push("reasoning", s); + clock = 500; + // The earlier copies expired, so the third copy alone is not a loop. + assert.strictEqual(feed(d, unit), false); + }); + + it("does not expire hashes when ttlMs is unset", () => { + let clock = 0; + const d = createSegmentLoopDetector({ threshold: 2, now: () => clock }); + const unit = ["a1", "b2"]; + for (const s of [...unit, ...unit]) d.push("reasoning", s); + clock = 10_000_000; + assert.strictEqual(feed(d, unit), true); + }); + it("reset clears state", () => { const d = createSegmentLoopDetector({ threshold: 2 }); const unit = ["a1", "b2"]; diff --git a/tests/unit/tui/conversationArea.test.js b/tests/unit/tui/conversationArea.test.js index 12591047..a96e7370 100644 --- a/tests/unit/tui/conversationArea.test.js +++ b/tests/unit/tui/conversationArea.test.js @@ -41,40 +41,68 @@ describe("countToolMessageTokens", () => { }); }); -describe("shouldNudgeLoop", () => { +describe("decideLoopAction", () => { const base = { loopDetected: true, streamingMsgId: "3", lastMessageId: "3", - loopNudgeCount: 0, - loopNudgeLimit: 2, + nudges: [], + now: 100000, + escalationCount: 3, + escalationWindowMs: 60000, }; - it("returns true when the detector flagged the last bubble and the limit is not exhausted", async () => { + it("nudges when the detector flagged the last bubble and there are no recent nudges", async () => { const mod = await import("../../../src/tui/conversationArea.js"); - assert.strictEqual(mod.shouldNudgeLoop(base), true); + assert.deepStrictEqual(mod.decideLoopAction(base), { action: "nudge", nudges: [100000] }); }); - it("returns false when no loop was detected", async () => { + it("does nothing when no loop was detected", async () => { const mod = await import("../../../src/tui/conversationArea.js"); - assert.strictEqual(mod.shouldNudgeLoop({ ...base, loopDetected: false }), false); - assert.strictEqual(mod.shouldNudgeLoop({ ...base, loopDetected: undefined }), false); + assert.strictEqual(mod.decideLoopAction({ ...base, loopDetected: false }).action, "none"); + assert.strictEqual(mod.decideLoopAction({ ...base, loopDetected: undefined }).action, "none"); }); - it("returns false when the streaming bubble is not the last message", async () => { + it("does nothing when the streaming bubble is not the last message", async () => { const mod = await import("../../../src/tui/conversationArea.js"); - assert.strictEqual(mod.shouldNudgeLoop({ ...base, lastMessageId: "4" }), false); + assert.strictEqual(mod.decideLoopAction({ ...base, lastMessageId: "4" }).action, "none"); }); - it("returns false once the nudge limit is exhausted", async () => { + it("records each nudge and keeps nudging below the escalation count", async () => { const mod = await import("../../../src/tui/conversationArea.js"); - assert.strictEqual(mod.shouldNudgeLoop({ ...base, loopNudgeCount: 2 }), false); - assert.strictEqual(mod.shouldNudgeLoop({ ...base, loopNudgeCount: 1 }), true); + const out = mod.decideLoopAction({ ...base, nudges: [90000, 95000] }); + assert.strictEqual(out.action, "nudge"); + assert.deepStrictEqual(out.nudges, [90000, 95000, 100000]); }); - it("never nudges when the limit is 0", async () => { + it("compacts instead of nudging once escalationCount nudges fall within the window", async () => { const mod = await import("../../../src/tui/conversationArea.js"); - assert.strictEqual(mod.shouldNudgeLoop({ ...base, loopNudgeLimit: 0 }), false); + const out = mod.decideLoopAction({ ...base, nudges: [80000, 90000, 95000] }); + assert.strictEqual(out.action, "compact"); + assert.deepStrictEqual(out.nudges, [], "counter resets after escalation"); + }); + + it("drops nudges older than the window so a slow loop does not escalate", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + // Two of the three are outside the 60s window. + const out = mod.decideLoopAction({ ...base, nudges: [10000, 20000, 95000] }); + assert.strictEqual(out.action, "nudge"); + assert.deepStrictEqual(out.nudges, [95000, 100000]); + }); + + it("treats a nudge exactly at the window edge as still inside it", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + const out = mod.decideLoopAction({ ...base, nudges: [40000, 41000, 42000] }); + assert.strictEqual(out.action, "compact"); + }); + + it("leaves the nudge timestamps untouched when it does nothing", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + const nudges = [90000]; + assert.strictEqual( + mod.decideLoopAction({ ...base, loopDetected: false, nudges }).nudges, + nudges, + ); }); }); From 9f6906fa281ea23cecdc845158f8515a064b0c21 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 19:24:37 -0500 Subject: [PATCH 10/20] fix: make the loop nudge visible Dispatch "You're looping" without silentUser so it renders as a user message in the TUI and it is obvious when the loop detector fires. --- src/tui/conversationArea.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tui/conversationArea.js b/src/tui/conversationArea.js index f781eff9..72ccca23 100644 --- a/src/tui/conversationArea.js +++ b/src/tui/conversationArea.js @@ -537,8 +537,8 @@ const ConversationArea = forwardRef(function ConversationArea( ); // Loop nudge: when the loop detector abandoned the stream, nudge the model - // with a silent "You're looping", the same way the reasoning-only - // "Continue" below is dispatched. Scoped to the LAST bubble. After + // with "You're looping". Rendered as a visible user message (not silent) so + // it is obvious when the detector fires. Scoped to the LAST bubble. After // `escalationCount` nudges within `escalationWindowMs`, compact the // context instead of nudging again. const loop = decideLoopAction({ @@ -556,7 +556,7 @@ const ConversationArea = forwardRef(function ConversationArea( return; } if (loop.action === "nudge") { - await handleChat("You're looping", { silentUser: true }); + await handleChat("You're looping"); return; } From 232e833f0b5d62453bb43baef25e87491ae68ab7 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 20:12:24 -0500 Subject: [PATCH 11/20] fix: detect loops from accumulated text in a worker thread Hashing each streamed delta missed loops whenever the repeats were cut at different positions, which is how a provider really streams: only single-character deltas were detected. Accumulate the streamed text per kind and check the text instead: a sentence repeated within the last windowSize completed sentences, or a run of words repeated back-to-back for loops with no punctuation such as 'ok ok ok'. The check runs in a Piscina worker so it stays off the main loop. At most one check per kind is in flight, text that arrives meanwhile is re-checked, a failing worker is treated as 'not looping', and the pool is destroyed on shutdown. windowSize now means recent sentences (default 20). Tests cut every repeat at its own random positions. --- config.yaml | 2 +- coverage.txt | 10 +- index.js | 10 +- src/config/schemas/loopDetection.js | 10 +- src/stream/consume.js | 13 +- src/stream/loopCheck.js | 80 +++++ src/stream/loopDetector.js | 157 ++++++--- src/stream/loopDetectorWorker.js | 17 + tests/unit/stream/loopCheck.test.js | 98 ++++++ tests/unit/stream/loopDetector.sample.test.js | 323 ++++++++---------- .../unit/stream/loopDetector.segments.test.js | 197 ----------- tests/unit/stream/loopDetector.test.js | 268 ++++++++------- 12 files changed, 621 insertions(+), 564 deletions(-) create mode 100644 src/stream/loopCheck.js create mode 100644 src/stream/loopDetectorWorker.js create mode 100644 tests/unit/stream/loopCheck.test.js delete mode 100644 tests/unit/stream/loopDetector.segments.test.js diff --git a/config.yaml b/config.yaml index 538ecbe7..6436c3c6 100644 --- a/config.yaml +++ b/config.yaml @@ -61,7 +61,7 @@ loopDetection: escalationWindowMs: 60000 threshold: 2 ttlMs: 60000 - windowSize: 1000 + windowSize: 20 memory: checkpointsDir: memory/checkpoints/ contextDir: memory/context/ diff --git a/coverage.txt b/coverage.txt index 840a37f1..b2fe2756 100644 --- a/coverage.txt +++ b/coverage.txt @@ -95,9 +95,11 @@ ℹ types.js | 100.00 | 100.00 | 100.00 | ℹ validator.js | 100.00 | 100.00 | 100.00 | ℹ stream | | | | -ℹ consume.js | 96.00 | 97.14 | 100.00 | 96-101 +ℹ consume.js | 91.61 | 92.50 | 100.00 | 97-102 114-120 ℹ interrupt.js | 100.00 | 100.00 | 100.00 | +ℹ loopCheck.js | 100.00 | 100.00 | 100.00 | ℹ loopDetector.js | 100.00 | 100.00 | 100.00 | +ℹ loopDetectorWorker.js | 100.00 | 100.00 | 100.00 | ℹ namespace.js | 100.00 | 100.00 | 100.00 | ℹ transformers | | | | ℹ index.js | 100.00 | 100.00 | 100.00 | @@ -185,7 +187,7 @@ ℹ xlsxParser.js | 82.21 | 83.33 | 88.89 | 47-48 93-112 130-137 150-156 ℹ zipExtractor.js | 82.32 | 77.78 | 66.67 | 62-63 104-107 124-142 149-152 ℹ git | | | | -ℹ index.js | 87.41 | 64.37 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 +ℹ index.js | 87.41 | 64.55 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 ℹ graphql | | | | ℹ index.js | 97.85 | 98.13 | 93.75 | 57-68 ℹ image | | | | @@ -212,7 +214,7 @@ ℹ reflection | | | | ℹ index.js | 94.98 | 83.02 | 90.00 | 56-60 125-126 142-143 197-198 ℹ sampling | | | | -ℹ index.js | 99.52 | 85.71 | 100.00 | 27 +ℹ index.js | 99.52 | 83.33 | 100.00 | 27 ℹ session | | | | ℹ index.js | 95.76 | 70.69 | 100.00 | 72-73 99-100 116-117 155-156 177-178 ℹ skills | | | | @@ -284,6 +286,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.34 | 86.63 | 88.24 | +ℹ all files | 93.35 | 86.67 | 88.33 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/index.js b/index.js index 006fb3da..acbdcd0f 100644 --- a/index.js +++ b/index.js @@ -19,7 +19,10 @@ import { import { resumeSession } from "./src/session/resume.js"; import { logger, flush as flushLogger } from "./src/shared/logger.js"; import { consumeStream } from "./src/stream/consume.js"; -import { createSegmentLoopDetector } from "./src/stream/loopDetector.js"; +import { + createSegmentLoopDetector, + destroyPool as destroyLoopPool, +} from "./src/stream/loopDetector.js"; // --------------------------------------------------------------------------- // CLI arguments @@ -314,8 +317,8 @@ async function runAgent(input, threadExtras, streamingCallback, signal) { streamMode: ["messages", "tools", "values"], subgraphs: true, }); - // A detected loop abandons the stream; the caller (the TUI) sends the silent - // nudge, the same way it sends the reasoning-only "Continue". + // A detected loop abandons the stream; the caller (the TUI) sends the nudge, + // the same way it sends the reasoning-only "Continue". const { content, reasoning, interrupted, lastEventType, loopDetected } = await consumeStream( stream, streamingCallback, @@ -424,6 +427,7 @@ async function runShutdown() { await handleShutdown({ onShutdown: async () => { if (gcManager) gcManager.stop(); + await destroyLoopPool(); if (mcpAdapter) await mcpAdapter.close(); if (shutdownFn) await shutdownFn(); }, diff --git a/src/config/schemas/loopDetection.js b/src/config/schemas/loopDetection.js index b4a7965b..0b9370c0 100644 --- a/src/config/schemas/loopDetection.js +++ b/src/config/schemas/loopDetection.js @@ -14,14 +14,12 @@ import { z } from "zod"; export const LoopDetectionSchema = z .object({ enabled: z.boolean().default(true), - // Number of recent segment (delta) hashes retained. Must hold - // `threshold + 1` copies of the longest repeating unit; a paragraph is on - // the order of 100 deltas. - windowSize: z.number().int().positive().default(1000), - // A hash older than this many ms is pruned from the window, so past + // Number of recent completed sentences compared for duplicates. + windowSize: z.number().int().positive().default(20), + // Accumulated stream text older than this many ms is dropped, so past // content cannot cause false positives as new content streams in. ttlMs: z.number().int().positive().default(60000), - // A unit must repeat back-to-back more than this many times to flag a loop. + // A sentence (or word run) must repeat more than this many times to flag a loop. threshold: z.number().int().positive().default(2), // Number of loop nudges within the escalation window before compacting. escalationCount: z.number().int().positive().default(3), diff --git a/src/stream/consume.js b/src/stream/consume.js index ee178ad5..dc7f1ea4 100644 --- a/src/stream/consume.js +++ b/src/stream/consume.js @@ -13,10 +13,11 @@ import { isSubagentNamespace } from "./namespace.js"; * @param {Function} [streamingCallback] - Callback for streamed chunks * @param {Object} [options] - Options * @param {boolean} [options.showToolResults=true] - Whether to emit `tool` events - * @param {{push: Function, reset: Function}} [options.loopDetector] - Segment loop - * detector. Each non-subagent message/reasoning delta is pushed as a segment. - * When it flags a loop the stream is abandoned (closing the underlying iterator) - * and `loopDetected` is returned true. + * @param {{push: Function, settle?: Function, reset: Function}} [options.loopDetector] - Loop + * detector. Each non-subagent message/reasoning delta is pushed to it. When it + * flags a loop the stream is abandoned (closing the underlying iterator) and + * `loopDetected` is returned true. Checks run in a worker, so a loop can be + * flagged after the stream ends; `settle()` is awaited to collect it. * @returns {Promise<{content: string, reasoning: string, interrupted: boolean, lastEventType: string | null, loopDetected: boolean}>} */ export async function consumeStream(stream, streamingCallback, options = {}) { @@ -146,5 +147,9 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (loopDetected) break; } + // Worker checks may still be in flight when the stream ends; collect them so + // a loop in the final deltas is not missed. + if (!loopDetected && loopDetector?.settle) loopDetected = await loopDetector.settle(); + return { content, reasoning, interrupted, lastEventType, loopDetected }; } diff --git a/src/stream/loopCheck.js b/src/stream/loopCheck.js new file mode 100644 index 00000000..162f208e --- /dev/null +++ b/src/stream/loopCheck.js @@ -0,0 +1,80 @@ +/** + * Pure loop checks over accumulated stream text. They look at the text itself, + * never at how the provider happened to split it into deltas, so a repeat is + * found however each copy was chunked. Kept free of I/O so the worker thread + * and the unit tests run the exact same code. + */ + +const SENTENCE_BOUNDARY = /(?<=[.!?])\s+|\n+/; + +/** + * Split text into completed sentences. A sentence ends at `.`, `?` or `!` + * followed by whitespace, or at a newline. The trailing piece has not ended + * yet, so it is dropped. + * @param {string} text - Accumulated stream text + * @returns {string[]} Completed sentences, whitespace collapsed + */ +export function completedSentences(text) { + const parts = text.split(SENTENCE_BOUNDARY); + parts.pop(); + const out = []; + for (const part of parts) { + const sentence = part.replace(/\s+/g, " ").trim(); + if (sentence) out.push(sentence); + } + return out; +} + +/** + * Look for a sentence that repeats within the last `windowSize` completed + * sentences. + * @param {string} text - Accumulated stream text + * @param {number} windowSize - How many recent sentences to compare + * @param {number} threshold - Occurrences above which a sentence is looping + * @returns {boolean} True when a sentence occurs more than `threshold` times + */ +export function hasDuplicateSentence(text, windowSize, threshold) { + const counts = new Map(); + for (const sentence of completedSentences(text).slice(-windowSize)) { + const count = (counts.get(sentence) ?? 0) + 1; + if (count > threshold) return true; + counts.set(sentence, count); + } + return false; +} + +/** + * Look for a run of words repeated back-to-back at the end of the text. This + * covers loops with no sentence terminator at all, such as "ok ok ok ok". + * @param {string} text - Accumulated stream text + * @param {number} threshold - Back-to-back repeats above which a loop is flagged + * @returns {boolean} True when the tail is one word run repeated more than `threshold` times + */ +export function hasRepeatedWords(text, threshold) { + const words = text.split(/\s+/).filter(Boolean); + const needed = threshold + 1; + const n = words.length; + const last = words[n - 1]; + for (let period = 1; period * needed <= n; period++) { + // Cheap reject: the unit's last word must recur exactly one period back. + if (words[n - 1 - period] !== last) continue; + let match = true; + for (let i = 1; i < period * (needed - 1) && match; i++) { + if (words[n - 1 - i] !== words[n - 1 - i - period]) match = false; + } + if (match) return true; + } + return false; +} + +/** + * Check accumulated stream text for a loop. + * @param {string} text - Accumulated stream text + * @param {Object} options - Options + * @param {number} options.windowSize - Recent completed sentences to compare + * @param {number} options.threshold - Repeats above which a loop is flagged + * @returns {boolean} True when the text is looping + */ +export function checkLoop(text, { windowSize, threshold }) { + return hasDuplicateSentence(text, windowSize, threshold) || hasRepeatedWords(text, threshold); +} diff --git a/src/stream/loopDetector.js b/src/stream/loopDetector.js index 6239daec..47000615 100644 --- a/src/stream/loopDetector.js +++ b/src/stream/loopDetector.js @@ -1,83 +1,140 @@ -import { createHash } from "node:crypto"; +import { Piscina } from "piscina"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +/** @type {Piscina | null} */ +let pool = null; + +/** + * Piscina worker pool for loop detection. Created lazily on first use and then + * reused, like the indexCode pool, so threads are not spawned per call and + * nothing is spawned at import time. + * @returns {Piscina} The shared pool + */ +export function getPool() { + if (!pool) pool = new Piscina({ filename: join(__dirname, "loopDetectorWorker.js") }); + return pool; +} /** - * Create a streaming loop detector that hashes stream segments. + * Destroy the shared pool (shutdown and tests). A later `getPool()` makes a new one. + * @returns {Promise} + */ +export async function destroyPool() { + const current = pool; + pool = null; + if (current) await current.destroy(); +} + +/** + * Create a streaming loop detector. * - * A segment is one streamed delta: a small group of characters exactly as it - * arrives (not a sentence or paragraph). Each delta is hashed and appended to - * a bounded list, per stream kind (reasoning, message). A looping model - * re-emits the same sequence of segments, so a loop is a trailing run of the - * hash list that repeats back-to-back more than `threshold` times, whatever - * the length of the repeating unit (a word, a sentence, a paragraph). + * Streamed deltas are accumulated into text per kind (reasoning, message). The + * text, not the deltas, is checked, so it does not matter how the provider + * chunks the stream. A loop is either a sentence that repeats within the last + * `windowSize` completed sentences, or a run of words repeated back-to-back + * (for loops with no sentence terminator, such as "ok ok ok"). * - * `windowSize` must be large enough to hold `threshold + 1` copies of the - * longest unit to be caught: a paragraph is on the order of 100 deltas. - * Hashes older than `ttlMs` are pruned so past content cannot form a false - * positive with new content. + * The check runs in a worker thread so it never blocks the main loop. `push` + * returns immediately with the latest known result and starts a check in the + * background when none is in flight, so a loop is reported on the first `push` + * after its check completes. At most one check per kind is in flight. * * @param {Object} [options] - Options - * @param {number} [options.windowSize=1000] - Segment hashes retained per kind - * @param {number} [options.threshold=2] - Back-to-back repeats above which a loop is flagged - * @param {number} [options.ttlMs] - Hashes older than this are pruned (no expiry when unset) + * @param {number} [options.windowSize=20] - Recent completed sentences compared + * @param {number} [options.threshold=2] - Repeats above which a loop is flagged + * @param {number} [options.ttlMs] - Accumulated text older than this is dropped (no expiry when unset) * @param {Function} [options.now] - Clock returning ms (injectable for tests) - * @returns {{ push: (kind: string, delta: string) => boolean, reset: () => void }} Detector + * @param {{ run: Function }} [options.runner] - Runs a check; defaults to the shared pool, injectable for tests + * @returns {{ push: (kind: string, delta: string) => boolean, settle: () => Promise, reset: () => void }} Detector */ export function createSegmentLoopDetector(options = {}) { - const windowSize = options.windowSize ?? 1000; + const windowSize = options.windowSize ?? 20; const threshold = options.threshold ?? 2; const ttlMs = options.ttlMs; const now = options.now ?? (() => Date.now()); - const needed = threshold + 1; + const runner = options.runner ?? { run: (task) => getPool().run(task) }; - /** @type {Map} */ - const lists = new Map(); + /** @type {Map | null, version: number}>} */ + const kinds = new Map(); + let generation = 0; + + function getKind(kind) { + let state = kinds.get(kind); + if (!state) { + state = { text: "", ts: now(), flagged: false, inflight: null, version: 0 }; + kinds.set(kind, state); + } + return state; + } /** - * Check whether the tail of the list is one unit repeated `needed` times. - * @param {{hash: string, ts: number}[]} list - Segment hashes, oldest first - * @returns {boolean} True when a repeating unit is found + * Start a worker check for a kind's current text, unless one is running. + * @param {{text: string, flagged: boolean, inflight: Promise | null, version: number}} state - Kind state + * @returns {void} */ - function isLooping(list) { - const n = list.length; - const last = list[n - 1].hash; - for (let period = 1; period * needed <= n; period++) { - // Cheap reject: the unit's last segment must recur exactly one period back. - if (list[n - 1 - period].hash !== last) continue; - let match = true; - for (let i = 1; i < period * (needed - 1) && match; i++) { - if (list[n - 1 - i].hash !== list[n - 1 - i - period].hash) match = false; - } - if (match) return true; - } - return false; + function startCheck(state) { + if (state.inflight) return; + const forGeneration = generation; + const checkedVersion = state.version; + state.inflight = Promise.resolve(runner.run({ text: state.text, windowSize, threshold })) + .then((looping) => { + // A reset since this check started makes its answer stale. + if (forGeneration === generation && looping) state.flagged = true; + }) + .catch(() => { + // A failed check must never break streaming: treat as "not looping". + }) + .finally(() => { + state.inflight = null; + // Text arrived while checking: check again so the tail is covered. + if (forGeneration === generation && !state.flagged && state.version !== checkedVersion) { + startCheck(state); + } + }); } return { /** - * Record a streamed segment. + * Record a streamed delta. * @param {string} kind - Stream kind, e.g. "reasoning" or "message" - * @param {string} delta - The segment exactly as streamed - * @returns {boolean} True when the stream is looping + * @param {string} delta - The delta exactly as streamed + * @returns {boolean} True when a completed check has flagged a loop */ push(kind, delta) { if (typeof delta !== "string" || delta.length === 0) return false; - let list = lists.get(kind); - if (!list) { - list = []; - lists.set(kind, list); - } + const state = getKind(kind); const nowMs = now(); - if (ttlMs !== undefined) { - while (list.length > 0 && nowMs - list[0].ts > ttlMs) list.shift(); + if (ttlMs !== undefined && nowMs - state.ts > ttlMs) { + state.text = ""; + state.flagged = false; + } + state.ts = nowMs; + state.text += delta; + state.version++; + if (!state.flagged) startCheck(state); + return state.flagged; + }, + + /** + * Wait for in-flight checks and report whether any kind is flagged. + * @returns {Promise} True when a loop has been flagged + */ + async settle() { + let pending = [...kinds.values()].map((s) => s.inflight).filter(Boolean); + while (pending.length > 0) { + await Promise.all(pending); + pending = [...kinds.values()].map((s) => s.inflight).filter(Boolean); } - list.push({ hash: createHash("sha256").update(delta).digest("hex"), ts: nowMs }); - while (list.length > windowSize) list.shift(); - return isLooping(list); + return [...kinds.values()].some((s) => s.flagged); }, /** Clear all state (call after a nudge so the next attempt starts fresh). */ reset() { - lists.clear(); + generation++; + kinds.clear(); }, }; } diff --git a/src/stream/loopDetectorWorker.js b/src/stream/loopDetectorWorker.js new file mode 100644 index 00000000..736c5413 --- /dev/null +++ b/src/stream/loopDetectorWorker.js @@ -0,0 +1,17 @@ +import { checkLoop } from "./loopCheck.js"; + +/** + * Piscina worker entry point for loop detection. + * + * Runs the text checks off the main event loop. Stateless: each task carries + * the text to check, so any worker thread can serve any task. + * + * @param {Object} task - The check to run + * @param {string} task.text - Accumulated stream text + * @param {number} task.windowSize - Recent completed sentences to compare + * @param {number} task.threshold - Repeats above which a loop is flagged + * @returns {boolean} True when the text is looping + */ +export default function loopWorker({ text, windowSize, threshold }) { + return checkLoop(text, { windowSize, threshold }); +} diff --git a/tests/unit/stream/loopCheck.test.js b/tests/unit/stream/loopCheck.test.js new file mode 100644 index 00000000..c6a3abcd --- /dev/null +++ b/tests/unit/stream/loopCheck.test.js @@ -0,0 +1,98 @@ +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { + checkLoop, + completedSentences, + hasDuplicateSentence, + hasRepeatedWords, +} from "../../../src/stream/loopCheck.js"; + +const OPTS = { windowSize: 20, threshold: 2 }; + +describe("completedSentences", () => { + it("splits on sentence ends and newlines and drops the unfinished tail", () => { + assert.deepStrictEqual(completedSentences("One. Two? Three!\nFour and"), [ + "One.", + "Two?", + "Three!", + ]); + }); + + it("collapses whitespace and skips blanks", () => { + assert.deepStrictEqual(completedSentences(" a b. \n\n c d. tail"), ["a b.", "c d."]); + }); + + it("returns nothing when no sentence has ended", () => { + assert.deepStrictEqual(completedSentences("ok ok ok ok"), []); + assert.deepStrictEqual(completedSentences(""), []); + }); +}); + +describe("hasDuplicateSentence", () => { + it("flags a sentence repeated more than threshold times", () => { + assert.strictEqual(hasDuplicateSentence("Same thing. ".repeat(3) + "tail", 20, 2), true); + assert.strictEqual(hasDuplicateSentence("Same thing. ".repeat(2) + "tail", 20, 2), false); + }); + + it("does not need the repeats to be back-to-back", () => { + const text = "Alpha one. Beta two. Alpha one. Gamma three. Alpha one. tail"; + assert.strictEqual(hasDuplicateSentence(text, 20, 2), true); + }); + + it("only compares the last windowSize sentences", () => { + const text = + "Same thing. " + "Filler a. Filler b. Filler c. ".repeat(1) + "Same thing. Same thing. tail"; + // 3 copies exist, but a window of 3 only sees the last 3 sentences. + assert.strictEqual(hasDuplicateSentence(text, 3, 2), false); + assert.strictEqual(hasDuplicateSentence(text, 20, 2), true); + }); + + it("does not flag distinct sentences", () => { + const text = Array.from({ length: 30 }, (_, i) => `Sentence number ${i}.`).join(" ") + " tail"; + assert.strictEqual(hasDuplicateSentence(text, 20, 2), false); + }); + + it("does not lowercase: a differently-cased sentence is a different sentence", () => { + assert.strictEqual(hasDuplicateSentence("Ok then. ok then. OK THEN. tail", 20, 2), false); + }); +}); + +describe("hasRepeatedWords", () => { + it("flags 'ok' repeated with no punctuation", () => { + assert.strictEqual(hasRepeatedWords("ok ok ok", 2), true); + assert.strictEqual(hasRepeatedWords("ok ok", 2), false); + }); + + it("flags a multi-word run repeated back-to-back", () => { + assert.strictEqual(hasRepeatedWords("so then we go so then we go so then we go", 2), true); + }); + + it("flags a repeat at the tail even after unrelated text", () => { + assert.strictEqual(hasRepeatedWords("something else entirely ok ok ok", 2), true); + }); + + it("does not flag a repeat that is not at the tail", () => { + assert.strictEqual(hasRepeatedWords("ok ok ok and then it moved on", 2), false); + }); + + it("does not flag text with no repeat", () => { + assert.strictEqual(hasRepeatedWords("the quick brown fox jumps over the lazy dog", 2), false); + assert.strictEqual(hasRepeatedWords("", 2), false); + }); + + it("honours a higher threshold", () => { + assert.strictEqual(hasRepeatedWords("ok ok ok ok", 4), false); + assert.strictEqual(hasRepeatedWords("ok ok ok ok ok", 4), true); + }); +}); + +describe("checkLoop", () => { + it("is true for either kind of repeat and false otherwise", () => { + assert.strictEqual(checkLoop("ok ".repeat(20), OPTS), true); + assert.strictEqual(checkLoop("This is a test. ".repeat(5), OPTS), true); + assert.strictEqual( + checkLoop("A normal paragraph. With different sentences. Nothing repeats.", OPTS), + false, + ); + }); +}); diff --git a/tests/unit/stream/loopDetector.sample.test.js b/tests/unit/stream/loopDetector.sample.test.js index 23afa646..af68537d 100644 --- a/tests/unit/stream/loopDetector.sample.test.js +++ b/tests/unit/stream/loopDetector.sample.test.js @@ -1,53 +1,45 @@ /** - * Replays a real reasoning loop (tests/fixtures/loop-reasoning.md) through the - * loop detector the way it arrives in production: as a stream of small, - * multi-character `reasoning_content` deltas that split mid-word and never - * align to sentence or paragraph boundaries, with no block ids. + * Replays looping text through the real loop detector (worker pool included) the + * way a provider streams it: as multi-character deltas that split mid-word. Every + * repeat is cut at its OWN random positions, so the repeats do not share delta + * boundaries (a looping model does not re-emit identical chunks). * - * A segment is one streamed delta. A looping model re-emits the same tokens, - * so the unit is cut into deltas once and that same delta sequence repeats. - * The fixture is a TUI capture of the model repeating the same paragraph; when + * The fixture is a TUI capture of the model repeating the same paragraph. When * the loop happens it is one paragraph repeating, sometimes two sentences, - * sometimes one, sometimes more. Each shape is replayed here. + * sometimes one, sometimes more, and sometimes just "ok ok ok" with no + * punctuation. Each shape is replayed here. */ -import { describe, it } from "node:test"; +import { describe, it, after } from "node:test"; import assert from "node:assert"; import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { consumeStream } from "../../../src/stream/consume.js"; -import { createSegmentLoopDetector } from "../../../src/stream/loopDetector.js"; +import { createSegmentLoopDetector, destroyPool } from "../../../src/stream/loopDetector.js"; -const FIXTURE = fileURLToPath(new URL("../../fixtures/loop-reasoning.md", import.meta.url)); - -/** - * Extract the reasoning paragraphs from the TUI capture. The capture is - * hard-wrapped and indented; a blank line separates paragraphs. Everything from - * the tool-call footer onward is TUI chrome, not model output. - * @returns {string[]} Paragraphs with wrapping collapsed - */ -function loadParagraphs() { - const text = readFileSync(FIXTURE, "utf8"); - const end = text.indexOf("⚡"); - const body = end === -1 ? text : text.slice(0, end); - return body - .split(/\n\s*\n/) - .map((p) => - p - .replace(/\s*\n\s*/g, " ") - .replace(/\s+/g, " ") - .trim(), - ) - .filter(Boolean); -} +after(async () => { + await destroyPool(); +}); -/** @returns {string[]} Sentences of a paragraph */ -const sentences = (paragraph) => paragraph.split(/(?<=[.!?])\s+/).filter(Boolean); +// Lines 14-21 of the capture: two paragraphs, hard-wrapped by the TUI. +const PASSAGE = readFileSync( + fileURLToPath(new URL("../../fixtures/loop-reasoning.md", import.meta.url)), + "utf8", +) + .split("\n") + .slice(13, 21) + .map((line) => line.trim()) + .join(" ") + .replace(/\s+/g, " ") + .trim(); + +const sentences = (text) => text.split(/(?<=[.!?])\s+/).filter(Boolean); +const FIRST_PARAGRAPH = PASSAGE.split(". Writing")[0] + "."; /** - * Deterministic PRNG (mulberry32) so chunking is reproducible. + * Deterministic PRNG (mulberry32). * @param {number} seed - Seed - * @returns {() => number} Function returning floats in [0, 1) + * @returns {() => number} Floats in [0, 1) */ function rng(seed) { let a = seed; @@ -61,184 +53,155 @@ function rng(seed) { } /** - * Cut text into multi-character deltas of `min`..`max` characters. Cuts land - * anywhere, including mid-word, like real token streaming. - * @param {string} text - Text to stream + * Cut text into deltas of `min`..`max` characters at random positions. + * @param {string} text - Text to cut * @param {number} min - Minimum delta length * @param {number} max - Maximum delta length - * @param {number} seed - PRNG seed + * @param {() => number} rand - Random source * @returns {string[]} Deltas whose concatenation equals `text` */ -function toDeltas(text, min, max, seed) { - const rand = rng(seed); - const deltas = []; +function cut(text, min, max, rand) { + const out = []; for (let i = 0; i < text.length;) { const size = min + Math.floor(rand() * (max - min + 1)); - deltas.push(text.slice(i, i + size)); + out.push(text.slice(i, i + size)); i += size; } - return deltas; + return out; } /** - * Build the agent stream tuples for reasoning deltas (Chat Completions shape: - * `additional_kwargs.reasoning_content`, no ids). - * @param {string[]} deltas - Reasoning deltas - * @param {{ consumed: number, closed: boolean }} probe - Records consumption - * @returns {AsyncIterable>} The stream + * Repeat `unit` `repeats` times, cutting each copy independently. + * @param {string} unit - The repeating text + * @param {number} repeats - Number of copies + * @param {string} separator - Text between copies + * @param {number} min - Minimum delta length + * @param {number} max - Maximum delta length + * @param {number} seed - PRNG seed + * @returns {string[]} Deltas */ -function reasoningStream(deltas, probe) { - return (async function* () { +function loopDeltas(unit, repeats, separator, min, max, seed) { + const rand = rng(seed); + const out = []; + for (let r = 0; r < repeats; r++) { + out.push(...cut(r === 0 ? unit : separator + unit, min, max, rand)); + } + return out; +} + +/** + * Wrap deltas as an async stream of the given kind. A short real delay between + * deltas stands in for network latency: a worker check takes a round trip, so + * deltas arriving faster than that would finish the stream before any check + * could return. + */ +const STREAM_KINDS = { + "reasoning_content (Chat Completions)": (s) => [ + [], + "messages", + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: s } }], + ], + "reasoning block (Responses API)": (s) => [ + [], + "messages", + [{ type: "ai", text: "", content: [{ type: "reasoning", reasoning: s, id: "r1" }] }], + ], + "message text": (s) => [[], "messages", [{ type: "ai", text: s, content: [] }]], +}; + +async function run(deltas, wrap) { + const probe = { consumed: 0, closed: false }; + const stream = (async function* () { try { - for (const delta of deltas) { + for (const d of deltas) { probe.consumed++; - yield [ - [], - "messages", - [{ type: "ai", text: "", additional_kwargs: { reasoning_content: delta } }], - ]; + await new Promise((r) => setTimeout(r, 1)); + yield wrap(d); } } finally { probe.closed = true; } })(); -} - -/** - * Stream `unit` repeated `repeats` times through consumeStream with a fresh detector. - * @param {string} unit - The repeating unit of reasoning - * @param {Object} opts - Options - * @param {number} opts.repeats - Times the unit repeats - * @param {string} opts.separator - Text between repeats - * @param {number} opts.min - Minimum delta length - * @param {number} opts.max - Maximum delta length - * @param {number} opts.seed - PRNG seed - * @returns {Promise<{ result: Object, probe: Object, total: number }>} Outcome - */ -async function replay(unit, { repeats, separator, min, max, seed }) { - // A looping model re-emits the same tokens: chunk the unit once, then - // repeat that identical delta sequence. - const unitDeltas = toDeltas(unit, min, max, seed); - const sepDeltas = separator ? [separator] : []; - const deltas = []; - for (let i = 0; i < repeats; i++) { - if (i > 0) deltas.push(...sepDeltas); - deltas.push(...unitDeltas); - } - const probe = { consumed: 0, closed: false }; - // The window must hold threshold + 1 (3) back-to-back copies of the unit; - // size it to the unit so tiny deltas on long units are still testable. - const detector = createSegmentLoopDetector({ - threshold: 2, - windowSize: Math.max(1000, unitDeltas.length * 3 + sepDeltas.length * 2 + 10), - }); - const result = await consumeStream(reasoningStream(deltas, probe), undefined, { - loopDetector: detector, + const result = await consumeStream(stream, undefined, { + loopDetector: createSegmentLoopDetector({ windowSize: 20, threshold: 2 }), }); return { result, probe, total: deltas.length }; } -const paragraphs = loadParagraphs(); -const loopParagraph = paragraphs[0]; -const twoParagraphs = `${paragraphs[0]}\n\n${paragraphs[1]}`; -const oneSentence = sentences(paragraphs[0])[0]; -const twoSentences = sentences(paragraphs[0]).slice(0, 2).join(" "); - -// Delta sizes in characters: tiny, typical, and large token groups. -const DELTA_SIZES = [ +const SHAPES = [ + ["one paragraph", FIRST_PARAGRAPH, " "], + ["two paragraphs", PASSAGE, " "], + ["two sentences", sentences(FIRST_PARAGRAPH).slice(0, 2).join(" "), " "], + ["one sentence", sentences(FIRST_PARAGRAPH)[0], " "], + ["one sentence (newline separated)", sentences(FIRST_PARAGRAPH)[0], "\n"], + ["'ok ' with no punctuation", "ok", " "], + ["a short phrase with no punctuation", "let me check that", " "], +]; +const SIZES = [ [1, 3], - [2, 8], - [5, 20], + [4, 10], ]; -// How the model separates repeats: paragraph break, line break, or just a space. -const SEPARATORS = ["\n\n", "\n", " "]; - -describe("loop detector replaying the captured reasoning loop", () => { - it("loads the fixture", () => { - assert.ok(paragraphs.length >= 6, `expected several paragraphs, got ${paragraphs.length}`); - assert.ok(loopParagraph.length > 200, "first paragraph is the long repeating one"); - assert.ok( - paragraphs.filter((p) => p === loopParagraph).length >= 3, - "the fixture really repeats the paragraph", - ); + +describe("loop detector on the captured loop, each repeat cut independently", () => { + it("loads the passage", () => { + assert.ok(PASSAGE.length > 600, `passage is ${PASSAGE.length} chars`); + assert.ok(FIRST_PARAGRAPH.length > 200); }); - const shapes = [ - ["one paragraph", loopParagraph], - ["two paragraphs", twoParagraphs], - ["two sentences", twoSentences], - ["one sentence", oneSentence], - ]; - - for (const [label, unit] of shapes) { - for (const [min, max] of DELTA_SIZES) { - for (const separator of SEPARATORS) { - const sep = JSON.stringify(separator); - it(`detects ${label} repeating (deltas ${min}-${max} chars, separator ${sep})`, async () => { - const { result, probe, total } = await replay(unit, { - repeats: 8, - separator, - min, - max, - seed: 42, + for (const [kindLabel, wrap] of Object.entries(STREAM_KINDS)) { + describe(kindLabel, () => { + for (const [label, unit, separator] of SHAPES) { + for (const [min, max] of SIZES) { + it(`detects ${label} repeating (deltas ${min}-${max} chars)`, async () => { + for (const seed of [1]) { + const deltas = loopDeltas(unit, 5, separator, min, max, seed); + const { result, probe, total } = await run(deltas, wrap); + assert.strictEqual(result.loopDetected, true, `seed ${seed}`); + assert.ok(probe.closed, `seed ${seed}: stream closed`); + // Tiny streams end before a worker round trip can stop them early. + if (total > 60) { + assert.ok(probe.consumed < total, `seed ${seed}: stopped early`); + } + } }); - assert.strictEqual(result.loopDetected, true, "loop must be detected"); - assert.ok(probe.consumed < total, `stopped early (${probe.consumed} of ${total} deltas)`); - assert.ok(probe.closed, "the underlying stream is closed"); - }); + } } - } - } - - it("detects the loop before the stream is mostly consumed", async () => { - const { probe, total, result } = await replay(loopParagraph, { - repeats: 10, - separator: "\n\n", - min: 2, - max: 8, - seed: 7, - }); - assert.strictEqual(result.loopDetected, true); - // threshold 2 means the 3rd repeat trips it; allow for the unit boundary. - assert.ok(probe.consumed <= total * 0.5, `consumed ${probe.consumed} of ${total}`); - }); - it("does not flag the distinct paragraphs of the capture (no false positive)", async () => { - const unique = [...new Set(paragraphs)]; - assert.ok(unique.length >= 4); - const text = unique.join("\n\n"); - const probe = { consumed: 0, closed: false }; - const result = await consumeStream( - reasoningStream(toDeltas(text, 2, 8, 11), probe), - undefined, - { - loopDetector: createSegmentLoopDetector({ threshold: 2 }), - }, - ); - assert.strictEqual(result.loopDetected, false); - }); + it("does not flag the passage streamed once", async () => { + const { result } = await run(loopDeltas(PASSAGE, 1, " ", 4, 10, 1), wrap); + assert.strictEqual(result.loopDetected, false); + }); - it("does not flag a paragraph that repeats only twice", async () => { - const { result } = await replay(loopParagraph, { - repeats: 2, - separator: "\n\n", - min: 2, - max: 8, - seed: 3, + it("does not flag a sentence that repeats only twice", async () => { + const { result } = await run( + loopDeltas(sentences(FIRST_PARAGRAPH)[0], 2, " ", 4, 10, 1), + wrap, + ); + assert.strictEqual(result.loopDetected, false); + }); }); - assert.strictEqual(result.loopDetected, false); + } + + it("flags 'ok ' repeated 20 times, the case that failed in practice", async () => { + for (const [min, max] of [...SIZES, [5, 20]]) { + for (const seed of [1, 2]) { + const deltas = cut("ok ".repeat(20), min, max, rng(seed)); + const { result } = await run(deltas, STREAM_KINDS["reasoning_content (Chat Completions)"]); + assert.strictEqual(result.loopDetected, true, `deltas ${min}-${max}, seed ${seed}`); + } + } }); - it("reproduces the same outcome regardless of chunking seed", async () => { - for (const seed of [1, 2, 3, 4, 5]) { - const { result } = await replay(loopParagraph, { - repeats: 8, - separator: "\n\n", - min: 1, - max: 12, - seed, - }); - assert.strictEqual(result.loopDetected, true, `seed ${seed}`); + it("flags a sentence repeated 5 times, the other case that failed in practice", async () => { + const text = "This is a test of streamed repetition. ".repeat(5); + for (const [min, max] of [...SIZES, [5, 20]]) { + for (const seed of [1, 2]) { + const { result } = await run( + cut(text, min, max, rng(seed)), + STREAM_KINDS["reasoning_content (Chat Completions)"], + ); + assert.strictEqual(result.loopDetected, true, `deltas ${min}-${max}, seed ${seed}`); + } } }); }); diff --git a/tests/unit/stream/loopDetector.segments.test.js b/tests/unit/stream/loopDetector.segments.test.js deleted file mode 100644 index b13a3f30..00000000 --- a/tests/unit/stream/loopDetector.segments.test.js +++ /dev/null @@ -1,197 +0,0 @@ -/** - * A segment is one streamed delta: a small group of characters. This test cuts - * lines 14-21 of the captured loop (tests/fixtures/loop-reasoning.md) once into - * random 4-10 character segments, then repeats that same segment array, which is - * what a looping model does (it re-emits the same tokens). The detector must flag - * the loop while the stream is still running. - * - * The detector must behave identically for every kind of stream, so the same - * segment array is replayed through each: Chat Completions `reasoning_content`, - * Responses API reasoning blocks, and message text. - */ - -import { describe, it } from "node:test"; -import assert from "node:assert"; -import { readFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; -import { consumeStream } from "../../../src/stream/consume.js"; -import { createSegmentLoopDetector } from "../../../src/stream/loopDetector.js"; - -// Lines 14-21 of the capture: two paragraphs, hard-wrapped by the TUI. -const TEXT = readFileSync( - fileURLToPath(new URL("../../fixtures/loop-reasoning.md", import.meta.url)), - "utf8", -) - .split("\n") - .slice(13, 21) - .map((line) => line.trim()) - .join(" ") - .replace(/\s+/g, " ") - .trim(); - -/** - * Deterministic PRNG (mulberry32). - * @param {number} seed - Seed - * @returns {() => number} Floats in [0, 1) - */ -function rng(seed) { - let a = seed; - return () => { - a |= 0; - a = (a + 0x6d2b79f5) | 0; - let t = Math.imul(a ^ (a >>> 15), 1 | a); - t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; - return ((t ^ (t >>> 14)) >>> 0) / 4294967296; - }; -} - -/** - * Break text into random segments of `min`..`max` characters. - * @param {string} text - Text to break up - * @param {number} seed - PRNG seed - * @param {number} [min=4] - Minimum segment length - * @param {number} [max=10] - Maximum segment length - * @returns {string[]} Segments whose concatenation equals `text` - */ -function toSegments(text, seed, min = 4, max = 10) { - const rand = rng(seed); - const out = []; - for (let i = 0; i < text.length;) { - const size = min + Math.floor(rand() * (max - min + 1)); - out.push(text.slice(i, i + size)); - i += size; - } - return out; -} - -/** - * The kinds of stream the detector sees, each wrapping a segment in the event - * shape `consumeStream` receives for that kind. - */ -const STREAM_KINDS = { - "reasoning_content (Chat Completions)": (s) => [ - [], - "messages", - [{ type: "ai", text: "", additional_kwargs: { reasoning_content: s } }], - ], - "reasoning block (Responses API)": (s) => [ - [], - "messages", - [{ type: "ai", text: "", content: [{ type: "reasoning", reasoning: s, id: "r1" }] }], - ], - "message text": (s) => [[], "messages", [{ type: "ai", text: s, content: [] }]], -}; - -/** - * Wrap segments as a stream of the given kind. - * @param {string[]} segments - Segments to emit - * @param {(s: string) => Array} wrap - Builds the event for one segment - * @param {{ consumed: number, closed: boolean }} probe - Records consumption - * @returns {AsyncIterable>} The stream - */ -function streamOf(segments, wrap, probe) { - return (async function* () { - try { - for (const s of segments) { - probe.consumed++; - yield wrap(s); - } - } finally { - probe.closed = true; - } - })(); -} - -/** - * Stream segments through consumeStream with a fresh detector. - * @param {string[]} segments - Segments to emit - * @param {(s: string) => Array} wrap - Event builder for the stream kind - * @returns {Promise<{ result: Object, probe: Object }>} Outcome - */ -async function run(segments, wrap) { - const probe = { consumed: 0, closed: false }; - const result = await consumeStream(streamOf(segments, wrap, probe), undefined, { - loopDetector: createSegmentLoopDetector({ threshold: 2 }), - }); - return { result, probe }; -} - -const repeat = (segments, times) => Array.from({ length: times }, () => segments).flat(); - -describe("loop detector with the passage broken into 4-10 char segments", () => { - it("breaks the passage into segments that rebuild the text", () => { - const segments = toSegments(TEXT, 1); - assert.strictEqual(segments.join(""), TEXT); - assert.ok(segments.every((s) => s.length >= 1 && s.length <= 10)); - assert.ok( - segments.slice(0, -1).every((s) => s.length >= 4), - "all but the last are 4-10 chars", - ); - }); - - it("sizes the default window to hold three copies of the passage", () => { - const n = toSegments(TEXT, 1).length; - // Three back-to-back copies must fit in the default window to be detectable. - const defaultWindow = 1000; - assert.ok( - n * 3 <= defaultWindow, - `3 copies = ${n * 3} segments, default window ${defaultWindow}`, - ); - }); - - for (const [label, wrap] of Object.entries(STREAM_KINDS)) { - describe(label, () => { - for (const seed of [1, 2, 3, 4, 5]) { - it(`flags the repeating segment array mid-stream (seed ${seed})`, async () => { - const segments = toSegments(TEXT, seed); - const { result, probe } = await run(repeat(segments, 6), wrap); - assert.strictEqual(result.loopDetected, true); - // threshold 2: the third back-to-back copy completes the loop. - assert.strictEqual(probe.consumed, segments.length * 3); - assert.ok(probe.closed, "underlying stream is closed"); - }); - } - - it("does not flag the passage streamed once", async () => { - const { result } = await run(toSegments(TEXT, 1), wrap); - assert.strictEqual(result.loopDetected, false); - }); - - it("does not flag the passage repeated only twice", async () => { - const { result } = await run(repeat(toSegments(TEXT, 1), 2), wrap); - assert.strictEqual(result.loopDetected, false); - }); - - it("flags a repeat of only the first paragraph's segments", async () => { - const segments = toSegments(TEXT.split(". Writing")[0], 9); - const { result, probe } = await run(repeat(segments, 6), wrap); - assert.strictEqual(result.loopDetected, true); - assert.strictEqual(probe.consumed, segments.length * 3); - }); - }); - } - - it("detects the loop at the same segment in every kind of stream", async () => { - for (const seed of [1, 2, 3]) { - const stream = repeat(toSegments(TEXT, seed), 6); - const outcomes = []; - for (const wrap of Object.values(STREAM_KINDS)) { - const { result, probe } = await run(stream, wrap); - outcomes.push({ detected: result.loopDetected, consumed: probe.consumed }); - } - assert.ok(outcomes[0].detected, `seed ${seed}: loop detected`); - for (const o of outcomes) assert.deepStrictEqual(o, outcomes[0], `seed ${seed}`); - } - }); - - it("detects the loop at the same segment when pushed straight to the detector", () => { - const stream = repeat(toSegments(TEXT, 1), 6); - const at = {}; - for (const kind of ["reasoning", "message"]) { - const d = createSegmentLoopDetector({ threshold: 2 }); - at[kind] = stream.findIndex((s) => d.push(kind, s)); - } - assert.ok(at.reasoning > 0); - assert.strictEqual(at.message, at.reasoning); - }); -}); diff --git a/tests/unit/stream/loopDetector.test.js b/tests/unit/stream/loopDetector.test.js index 1be1fad3..68df2514 100644 --- a/tests/unit/stream/loopDetector.test.js +++ b/tests/unit/stream/loopDetector.test.js @@ -1,77 +1,49 @@ -import { describe, it } from "node:test"; +import { describe, it, after } from "node:test"; import assert from "node:assert"; -import { createSegmentLoopDetector } from "../../../src/stream/loopDetector.js"; +import { createSegmentLoopDetector, destroyPool } from "../../../src/stream/loopDetector.js"; +import { checkLoop } from "../../../src/stream/loopCheck.js"; import { consumeStream } from "../../../src/stream/consume.js"; -/** Push every segment; return whether any push flagged a loop. */ -const feed = (d, segments, kind = "reasoning") => { - let flagged = false; - for (const s of segments) if (d.push(kind, s)) flagged = true; - return flagged; -}; - -const repeat = (unit, times) => Array.from({ length: times }, () => unit).flat(); - -describe("createSegmentLoopDetector", () => { - it("flags a unit repeated back-to-back more than threshold times", () => { - const d = createSegmentLoopDetector({ threshold: 2 }); - const unit = ["alpha ", "beta ", "gamma "]; - assert.strictEqual(feed(d, repeat(unit, 2)), false); - assert.strictEqual(feed(createSegmentLoopDetector({ threshold: 2 }), repeat(unit, 3)), true); - }); - - it("flags on the segment that completes the final repeat", () => { - const d = createSegmentLoopDetector({ threshold: 2 }); - const unit = ["a1", "b2", "c3", "d4"]; - const stream = repeat(unit, 3); - const results = stream.map((s) => d.push("reasoning", s)); - assert.strictEqual(results.lastIndexOf(true), stream.length - 1); - assert.strictEqual(results.indexOf(true), stream.length - 1); - }); - - it("flags a single segment repeating, such as 'ok'", () => { - const d = createSegmentLoopDetector({ threshold: 2 }); - assert.strictEqual(d.push("message", "ok"), false); - assert.strictEqual(d.push("message", "ok"), false); - assert.strictEqual(d.push("message", "ok"), true); - }); - - it("flags units of different lengths", () => { - for (const len of [1, 2, 7, 40]) { - const unit = Array.from({ length: len }, (_, i) => `s${i}`); - assert.strictEqual( - feed(createSegmentLoopDetector({ threshold: 2 }), repeat(unit, 3)), - true, - `len ${len}`, - ); - } - }); +after(async () => { + await destroyPool(); +}); - it("does not flag distinct segments", () => { - const d = createSegmentLoopDetector(); - const stream = Array.from({ length: 500 }, (_, i) => `segment-${i}`); - assert.strictEqual(feed(d, stream), false); +/** Runner that executes the check inline, so unit tests need no thread. */ +const inline = { run: async (task) => checkLoop(task.text, task) }; + +const detector = (extra = {}) => createSegmentLoopDetector({ runner: inline, ...extra }); + +/** Push every delta, letting checks finish between pushes; return the first index flagged. */ +async function firstFlag(d, deltas, kind = "reasoning") { + for (let i = 0; i < deltas.length; i++) { + if (d.push(kind, deltas[i])) return i; + await d.settle(); + if (d.push(kind, "")) return i; // no-op push, keeps shape symmetric + } + return (await d.settle()) ? deltas.length - 1 : -1; +} + +describe("createSegmentLoopDetector (worker-backed)", () => { + it("flags 'ok' repeated with no punctuation", async () => { + const d = detector(); + const deltas = Array.from({ length: 20 }, () => "ok "); + assert.ok((await firstFlag(d, deltas)) >= 0); }); - it("does not flag a repeat that is not back-to-back", () => { - const d = createSegmentLoopDetector({ threshold: 2 }); - const unit = ["a1", "b2", "c3"]; - const stream = [...unit, "x1", ...unit, "x2", ...unit, "x3"]; - assert.strictEqual(feed(d, stream), false); + it("flags a repeated sentence", async () => { + const d = detector(); + const deltas = Array.from({ length: 5 }, () => "This is a test of streamed repetition. "); + assert.ok((await firstFlag(d, deltas)) >= 0); }); - it("tracks reasoning and message separately", () => { - const d = createSegmentLoopDetector({ threshold: 2 }); - // Interleaved kinds: each kind alone never repeats back-to-back 3 times. - assert.strictEqual(d.push("reasoning", "same"), false); - assert.strictEqual(d.push("message", "same"), false); - assert.strictEqual(d.push("reasoning", "same"), false); - assert.strictEqual(d.push("message", "same"), false); - assert.strictEqual(d.push("reasoning", "same"), true); + it("does not flag distinct text", async () => { + const d = detector(); + const deltas = Array.from({ length: 40 }, (_, i) => `Sentence number ${i} is unique. `); + assert.strictEqual(await firstFlag(d, deltas), -1); }); it("ignores empty and non-string deltas", () => { - const d = createSegmentLoopDetector({ threshold: 2 }); + const d = detector(); for (let i = 0; i < 10; i++) { assert.strictEqual(d.push("reasoning", ""), false); assert.strictEqual(d.push("reasoning", undefined), false); @@ -79,51 +51,107 @@ describe("createSegmentLoopDetector", () => { } }); - it("cannot see a repeat that does not fit in the window", () => { - const unit = Array.from({ length: 10 }, (_, i) => `s${i}`); - // Window of 25 cannot hold 3 copies (30 segments). - assert.strictEqual( - feed(createSegmentLoopDetector({ threshold: 2, windowSize: 25 }), repeat(unit, 6)), - false, - ); - assert.strictEqual( - feed(createSegmentLoopDetector({ threshold: 2, windowSize: 30 }), repeat(unit, 6)), - true, - ); - }); - - it("honours a higher threshold", () => { - const unit = ["a1", "b2"]; - assert.strictEqual(feed(createSegmentLoopDetector({ threshold: 4 }), repeat(unit, 4)), false); - assert.strictEqual(feed(createSegmentLoopDetector({ threshold: 4 }), repeat(unit, 5)), true); - }); - - it("prunes hashes older than ttlMs", () => { + it("tracks reasoning and message text separately", async () => { + const d = detector(); + // Each kind alone has 2 copies; combined text would have 4. + for (const kind of ["reasoning", "message"]) { + d.push(kind, "Same thing. "); + d.push(kind, "Same thing. "); + } + assert.strictEqual(await d.settle(), false); + }); + + it("reports a flag on the first push after the check completes", async () => { + const d = detector(); + d.push("reasoning", "ok ok ok "); + assert.strictEqual(await d.settle(), true); + assert.strictEqual(d.push("reasoning", "more "), true); + }); + + it("keeps at most one check in flight per kind", async () => { + let running = 0; + let peak = 0; + const slow = { + run: async (task) => { + running++; + peak = Math.max(peak, running); + await new Promise((r) => setTimeout(r, 5)); + running--; + return checkLoop(task.text, task); + }, + }; + const d = createSegmentLoopDetector({ runner: slow }); + for (let i = 0; i < 30; i++) d.push("reasoning", `word${i} `); + await d.settle(); + assert.strictEqual(peak, 1); + }); + + it("re-checks text that arrived while a check was running", async () => { + const seen = []; + const slow = { + run: async (task) => { + seen.push(task.text); + await new Promise((r) => setTimeout(r, 5)); + return checkLoop(task.text, task); + }, + }; + const d = createSegmentLoopDetector({ runner: slow }); + d.push("reasoning", "ok "); + d.push("reasoning", "ok "); + d.push("reasoning", "ok "); + assert.strictEqual(await d.settle(), true); + assert.ok(seen.length >= 2, "the tail was checked again"); + }); + + it("treats a failing check as not looping and keeps streaming", async () => { + const broken = { + run: async () => { + throw new Error("worker died"); + }, + }; + const d = createSegmentLoopDetector({ runner: broken }); + assert.strictEqual(d.push("reasoning", "ok ok ok ok "), false); + assert.strictEqual(await d.settle(), false); + }); + + it("drops accumulated text older than ttlMs", async () => { let clock = 0; - const d = createSegmentLoopDetector({ threshold: 2, ttlMs: 100, now: () => clock }); - const unit = ["a1", "b2"]; - for (const s of [...unit, ...unit]) d.push("reasoning", s); + const d = detector({ ttlMs: 100, now: () => clock }); + d.push("reasoning", "ok ok "); + await d.settle(); clock = 500; - // The earlier copies expired, so the third copy alone is not a loop. - assert.strictEqual(feed(d, unit), false); + d.push("reasoning", "ok "); + assert.strictEqual(await d.settle(), false); }); - it("does not expire hashes when ttlMs is unset", () => { + it("does not expire text when ttlMs is unset", async () => { let clock = 0; - const d = createSegmentLoopDetector({ threshold: 2, now: () => clock }); - const unit = ["a1", "b2"]; - for (const s of [...unit, ...unit]) d.push("reasoning", s); + const d = detector({ now: () => clock }); + d.push("reasoning", "ok ok "); + await d.settle(); clock = 10_000_000; - assert.strictEqual(feed(d, unit), true); + d.push("reasoning", "ok "); + assert.strictEqual(await d.settle(), true); + }); + + it("reset clears state and discards a stale in-flight result", async () => { + const slow = { + run: async (task) => { + await new Promise((r) => setTimeout(r, 10)); + return checkLoop(task.text, task); + }, + }; + const d = createSegmentLoopDetector({ runner: slow }); + d.push("reasoning", "ok ok ok "); + d.reset(); + assert.strictEqual(await d.settle(), false); + assert.strictEqual(d.push("reasoning", "fresh start "), false); }); - it("reset clears state", () => { - const d = createSegmentLoopDetector({ threshold: 2 }); - const unit = ["a1", "b2"]; - feed(d, repeat(unit, 2)); - d.reset(); - assert.strictEqual(feed(d, unit), false); - assert.strictEqual(feed(d, repeat(unit, 2)), true); + it("runs the check in a real worker thread by default", async () => { + const d = createSegmentLoopDetector(); + d.push("reasoning", "ok ok ok ok "); + assert.strictEqual(await d.settle(), true); }); }); @@ -136,33 +164,40 @@ describe("consumeStream with loopDetector", () => { const message = (text) => [[], "messages", [{ type: "ai", text, content: [] }]]; it("stops consuming, closes the stream and reports loopDetected", async () => { - let consumed = 0; let closed = false; + let consumed = 0; const stream = (async function* () { try { - for (let i = 0; i < 100; i++) { + for (let i = 0; i < 400; i++) { consumed++; - yield reasoning("same"); + await new Promise((r) => setImmediate(r)); + yield reasoning("ok "); } } finally { closed = true; } })(); - const result = await consumeStream(stream, undefined, { - loopDetector: createSegmentLoopDetector({ threshold: 2 }), - }); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); assert.strictEqual(result.loopDetected, true); - assert.strictEqual(consumed, 3); + assert.ok(consumed < 400, `stopped early (${consumed})`); assert.ok(closed, "underlying stream is closed"); }); + it("collects a loop flagged by a check still running when the stream ends", async () => { + const stream = (async function* () { + yield reasoning("ok "); + yield reasoning("ok "); + yield reasoning("ok "); + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, true); + }); + it("detects a looping message stream", async () => { const stream = (async function* () { for (let i = 0; i < 20; i++) yield message("again "); })(); - const result = await consumeStream(stream, undefined, { - loopDetector: createSegmentLoopDetector({ threshold: 2 }), - }); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); assert.strictEqual(result.loopDetected, true); }); @@ -175,9 +210,7 @@ describe("consumeStream with loopDetector", () => { const stream = (async function* () { for (let i = 0; i < 20; i++) yield block("think "); })(); - const result = await consumeStream(stream, undefined, { - loopDetector: createSegmentLoopDetector({ threshold: 2 }), - }); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); assert.strictEqual(result.loopDetected, true); }); @@ -187,13 +220,11 @@ describe("consumeStream with loopDetector", () => { yield [ ["tools:abc"], "messages", - [{ type: "ai", text: "", additional_kwargs: { reasoning_content: "same" } }], + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: "same " } }], ]; } })(); - const result = await consumeStream(stream, undefined, { - loopDetector: createSegmentLoopDetector({ threshold: 2 }), - }); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); assert.strictEqual(result.loopDetected, false); }); @@ -201,7 +232,6 @@ describe("consumeStream with loopDetector", () => { const stream = (async function* () { yield reasoning("hello"); })(); - const result = await consumeStream(stream); - assert.strictEqual(result.loopDetected, false); + assert.strictEqual((await consumeStream(stream)).loopDetected, false); }); }); From db37a03cd46e5a6dd58717d6b2a7b06be89692f7 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 20:23:41 -0500 Subject: [PATCH 12/20] fix: make the loop nudge silent again The detector is confirmed firing, so dispatch "You're looping" with silentUser like the reasoning-only Continue. --- coverage.txt | 4 ++-- index.js | 4 ++-- src/tui/conversationArea.js | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/coverage.txt b/coverage.txt index b2fe2756..71dde95a 100644 --- a/coverage.txt +++ b/coverage.txt @@ -187,7 +187,7 @@ ℹ xlsxParser.js | 82.21 | 83.33 | 88.89 | 47-48 93-112 130-137 150-156 ℹ zipExtractor.js | 82.32 | 77.78 | 66.67 | 62-63 104-107 124-142 149-152 ℹ git | | | | -ℹ index.js | 87.41 | 64.55 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 +ℹ index.js | 87.41 | 64.37 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 ℹ graphql | | | | ℹ index.js | 97.85 | 98.13 | 93.75 | 57-68 ℹ image | | | | @@ -286,6 +286,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.35 | 86.67 | 88.33 | +ℹ all files | 93.35 | 86.65 | 88.33 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/index.js b/index.js index acbdcd0f..2b47cee5 100644 --- a/index.js +++ b/index.js @@ -317,8 +317,8 @@ async function runAgent(input, threadExtras, streamingCallback, signal) { streamMode: ["messages", "tools", "values"], subgraphs: true, }); - // A detected loop abandons the stream; the caller (the TUI) sends the nudge, - // the same way it sends the reasoning-only "Continue". + // A detected loop abandons the stream; the caller (the TUI) sends the silent + // nudge, the same way it sends the reasoning-only "Continue". const { content, reasoning, interrupted, lastEventType, loopDetected } = await consumeStream( stream, streamingCallback, diff --git a/src/tui/conversationArea.js b/src/tui/conversationArea.js index 72ccca23..f781eff9 100644 --- a/src/tui/conversationArea.js +++ b/src/tui/conversationArea.js @@ -537,8 +537,8 @@ const ConversationArea = forwardRef(function ConversationArea( ); // Loop nudge: when the loop detector abandoned the stream, nudge the model - // with "You're looping". Rendered as a visible user message (not silent) so - // it is obvious when the detector fires. Scoped to the LAST bubble. After + // with a silent "You're looping", the same way the reasoning-only + // "Continue" below is dispatched. Scoped to the LAST bubble. After // `escalationCount` nudges within `escalationWindowMs`, compact the // context instead of nudging again. const loop = decideLoopAction({ @@ -556,7 +556,7 @@ const ConversationArea = forwardRef(function ConversationArea( return; } if (loop.action === "nudge") { - await handleChat("You're looping"); + await handleChat("You're looping", { silentUser: true }); return; } From b5058eee0869a991a6f58ecce2aee4e7a79c4f78 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 20:34:45 -0500 Subject: [PATCH 13/20] perf: split each streamed delta once and keep a bounded window The detector re-split the whole accumulated text for each kind on every check and copied it to the worker, so cost grew with stream length: about 7 ms per check at 3 MB, quadratic over a long run. Split only the new text as it arrives and keep the last windowSize sentences and wordWindowSize words. The worker now receives those bounded lists, and a check only starts when a delta completed a sentence or word. Per-push cost is flat (about 1 us from 1k to 200k deltas). Verdicts are unchanged however the stream is chunked. Adds loopLoop wordWindowSize handling in the detector only; no config change. --- coverage.txt | 4 +- src/stream/loopCheck.js | 93 +++++++++------ src/stream/loopDetector.js | 105 ++++++++++++---- src/stream/loopDetectorWorker.js | 14 +-- tests/unit/stream/loopCheck.test.js | 158 ++++++++++++++++++------- tests/unit/stream/loopDetector.test.js | 96 ++++++++++++++- 6 files changed, 352 insertions(+), 118 deletions(-) diff --git a/coverage.txt b/coverage.txt index 71dde95a..2e3f5f1f 100644 --- a/coverage.txt +++ b/coverage.txt @@ -8,7 +8,7 @@ ℹ compaction.js | 98.84 | 78.43 | 100.00 | 89-90 ℹ contextBackend.js | 100.00 | 100.00 | 100.00 | ℹ coreBackend.js | 100.00 | 100.00 | 100.00 | -ℹ deepAgents.js | 93.72 | 69.39 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 +ℹ deepAgents.js | 93.72 | 68.00 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 ℹ evictionCeiling.js | 100.00 | 100.00 | 100.00 | ℹ skillsMiddleware.js | 100.00 | 87.50 | 100.00 | ℹ config | | | | @@ -286,6 +286,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.35 | 86.65 | 88.33 | +ℹ all files | 93.36 | 86.66 | 88.35 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/src/stream/loopCheck.js b/src/stream/loopCheck.js index 162f208e..80761884 100644 --- a/src/stream/loopCheck.js +++ b/src/stream/loopCheck.js @@ -1,41 +1,59 @@ /** - * Pure loop checks over accumulated stream text. They look at the text itself, - * never at how the provider happened to split it into deltas, so a repeat is - * found however each copy was chunked. Kept free of I/O so the worker thread - * and the unit tests run the exact same code. + * Pure loop checks. They work on already-split sentences and words, never on + * the raw stream, so how the provider cut the stream into deltas does not + * matter, and the cost of a check depends on the window, not on how long the + * stream has been running. Kept free of I/O so the worker thread and the unit + * tests run the exact same code. */ -const SENTENCE_BOUNDARY = /(?<=[.!?])\s+|\n+/; - /** - * Split text into completed sentences. A sentence ends at `.`, `?` or `!` - * followed by whitespace, or at a newline. The trailing piece has not ended - * yet, so it is dropped. - * @param {string} text - Accumulated stream text - * @returns {string[]} Completed sentences, whitespace collapsed + * Take the sentences that have completed in `pending`. A sentence ends at `.`, + * `?` or `!` followed by whitespace, or at a newline. Only the text from `from` + * on is scanned, so a caller that appends a delta and passes the old length + * pays for the delta, not for everything seen so far. + * @param {string} pending - Text not yet consumed (an unfinished sentence plus new text) + * @param {number} [from=0] - Index to start scanning at (the length before the new text was appended) + * @returns {{ sentences: string[], rest: string }} Completed sentences (whitespace collapsed) + * and the unfinished remainder to carry forward */ -export function completedSentences(text) { - const parts = text.split(SENTENCE_BOUNDARY); - parts.pop(); - const out = []; - for (const part of parts) { - const sentence = part.replace(/\s+/g, " ").trim(); - if (sentence) out.push(sentence); +export function takeSentences(pending, from = 0) { + const boundary = /(?<=[.!?])\s+|\n+/g; + boundary.lastIndex = from; + const sentences = []; + let start = 0; + let match; + while ((match = boundary.exec(pending)) !== null) { + const sentence = pending.slice(start, match.index).replace(/\s+/g, " ").trim(); + if (sentence) sentences.push(sentence); + start = match.index + match[0].length; } - return out; + return { sentences, rest: pending.slice(start) }; +} + +/** + * Take the words that have completed once `delta` is appended. A word ends at + * whitespace; a trailing piece with no whitespace after it may continue in the + * next delta, so it is carried forward. + * @param {string} partial - Unfinished word carried from the previous delta + * @param {string} delta - New text + * @returns {{ words: string[], rest: string }} Completed words and the unfinished remainder + */ +export function takeWords(partial, delta) { + const text = partial + delta; + const parts = text.split(/\s+/); + const rest = /\s$/.test(text) ? "" : parts.pop(); + return { words: parts.filter(Boolean), rest }; } /** - * Look for a sentence that repeats within the last `windowSize` completed - * sentences. - * @param {string} text - Accumulated stream text - * @param {number} windowSize - How many recent sentences to compare + * Look for a sentence that occurs more than `threshold` times in `sentences`. + * @param {string[]} sentences - The recent completed sentences * @param {number} threshold - Occurrences above which a sentence is looping - * @returns {boolean} True when a sentence occurs more than `threshold` times + * @returns {boolean} True when a sentence repeats */ -export function hasDuplicateSentence(text, windowSize, threshold) { +export function hasDuplicateSentence(sentences, threshold) { const counts = new Map(); - for (const sentence of completedSentences(text).slice(-windowSize)) { + for (const sentence of sentences) { const count = (counts.get(sentence) ?? 0) + 1; if (count > threshold) return true; counts.set(sentence, count); @@ -44,14 +62,13 @@ export function hasDuplicateSentence(text, windowSize, threshold) { } /** - * Look for a run of words repeated back-to-back at the end of the text. This + * Look for a run of words repeated back-to-back at the end of `words`. This * covers loops with no sentence terminator at all, such as "ok ok ok ok". - * @param {string} text - Accumulated stream text + * @param {string[]} words - The recent completed words * @param {number} threshold - Back-to-back repeats above which a loop is flagged * @returns {boolean} True when the tail is one word run repeated more than `threshold` times */ -export function hasRepeatedWords(text, threshold) { - const words = text.split(/\s+/).filter(Boolean); +export function hasRepeatedWords(words, threshold) { const needed = threshold + 1; const n = words.length; const last = words[n - 1]; @@ -68,13 +85,13 @@ export function hasRepeatedWords(text, threshold) { } /** - * Check accumulated stream text for a loop. - * @param {string} text - Accumulated stream text - * @param {Object} options - Options - * @param {number} options.windowSize - Recent completed sentences to compare - * @param {number} options.threshold - Repeats above which a loop is flagged - * @returns {boolean} True when the text is looping + * Check the recent sentences and words for a loop. + * @param {Object} task - The check + * @param {string[]} task.sentences - Recent completed sentences + * @param {string[]} task.words - Recent completed words + * @param {number} task.threshold - Repeats above which a loop is flagged + * @returns {boolean} True when the stream is looping */ -export function checkLoop(text, { windowSize, threshold }) { - return hasDuplicateSentence(text, windowSize, threshold) || hasRepeatedWords(text, threshold); +export function checkLoop({ sentences, words, threshold }) { + return hasDuplicateSentence(sentences, threshold) || hasRepeatedWords(words, threshold); } diff --git a/src/stream/loopDetector.js b/src/stream/loopDetector.js index 47000615..fe92666d 100644 --- a/src/stream/loopDetector.js +++ b/src/stream/loopDetector.js @@ -1,6 +1,7 @@ import { Piscina } from "piscina"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; +import { takeSentences, takeWords } from "./loopCheck.js"; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -25,61 +26,100 @@ export function getPool() { export async function destroyPool() { const current = pool; pool = null; - if (current) await current.destroy(); + await current?.destroy(); } /** * Create a streaming loop detector. * - * Streamed deltas are accumulated into text per kind (reasoning, message). The - * text, not the deltas, is checked, so it does not matter how the provider - * chunks the stream. A loop is either a sentence that repeats within the last - * `windowSize` completed sentences, or a run of words repeated back-to-back - * (for loops with no sentence terminator, such as "ok ok ok"). + * Each streamed delta is split once, as it arrives, into completed sentences + * and words per kind (reasoning, message). Only the last `windowSize` sentences + * and `wordWindowSize` words are kept, so memory and the cost of a check stay + * bounded however long the stream runs, and the provider's delta boundaries do + * not matter. A loop is a sentence repeated within the sentence window, or a + * run of words repeated back-to-back (for loops with no sentence terminator, + * such as "ok ok ok"). * * The check runs in a worker thread so it never blocks the main loop. `push` - * returns immediately with the latest known result and starts a check in the - * background when none is in flight, so a loop is reported on the first `push` - * after its check completes. At most one check per kind is in flight. + * returns immediately with the latest known result. A check starts only when a + * delta completed a new sentence or word, and at most one check per kind is in + * flight. * * @param {Object} [options] - Options - * @param {number} [options.windowSize=20] - Recent completed sentences compared + * @param {number} [options.windowSize=20] - Recent completed sentences kept and compared + * @param {number} [options.wordWindowSize=200] - Recent completed words kept and compared * @param {number} [options.threshold=2] - Repeats above which a loop is flagged - * @param {number} [options.ttlMs] - Accumulated text older than this is dropped (no expiry when unset) + * @param {number} [options.ttlMs] - State older than this is dropped (no expiry when unset) * @param {Function} [options.now] - Clock returning ms (injectable for tests) * @param {{ run: Function }} [options.runner] - Runs a check; defaults to the shared pool, injectable for tests * @returns {{ push: (kind: string, delta: string) => boolean, settle: () => Promise, reset: () => void }} Detector */ export function createSegmentLoopDetector(options = {}) { const windowSize = options.windowSize ?? 20; + const wordWindowSize = options.wordWindowSize ?? 200; const threshold = options.threshold ?? 2; const ttlMs = options.ttlMs; const now = options.now ?? (() => Date.now()); const runner = options.runner ?? { run: (task) => getPool().run(task) }; - /** @type {Map | null, version: number}>} */ + /** + * @typedef {Object} KindState + * @property {string} pending - Unfinished sentence text + * @property {string} partialWord - Unfinished word + * @property {string[]} sentences - Last `windowSize` completed sentences + * @property {string[]} words - Last `wordWindowSize` completed words + * @property {number} ts - Time of the last delta + * @property {boolean} flagged - A check has flagged a loop + * @property {Promise | null} inflight - The running check, if any + * @property {number} version - Bumped when a sentence or word completes + */ + /** @type {Map} */ const kinds = new Map(); let generation = 0; + function freshState() { + return { + pending: "", + partialWord: "", + sentences: [], + words: [], + ts: now(), + flagged: false, + inflight: null, + version: 0, + }; + } + function getKind(kind) { let state = kinds.get(kind); if (!state) { - state = { text: "", ts: now(), flagged: false, inflight: null, version: 0 }; + state = freshState(); kinds.set(kind, state); } return state; } /** - * Start a worker check for a kind's current text, unless one is running. - * @param {{text: string, flagged: boolean, inflight: Promise | null, version: number}} state - Kind state + * Keep only the last `size` entries, trimming in place. + * @param {string[]} list - List to trim + * @param {number} size - Entries to keep + * @returns {void} + */ + function trim(list, size) { + if (list.length > size) list.splice(0, list.length - size); + } + + /** + * Start a worker check on a kind's recent sentences and words, unless one is running. + * @param {KindState} state - Kind state * @returns {void} */ function startCheck(state) { if (state.inflight) return; const forGeneration = generation; const checkedVersion = state.version; - state.inflight = Promise.resolve(runner.run({ text: state.text, windowSize, threshold })) + const task = { sentences: state.sentences.slice(), words: state.words.slice(), threshold }; + state.inflight = Promise.resolve(runner.run(task)) .then((looping) => { // A reset since this check started makes its answer stale. if (forGeneration === generation && looping) state.flagged = true; @@ -89,7 +129,7 @@ export function createSegmentLoopDetector(options = {}) { }) .finally(() => { state.inflight = null; - // Text arrived while checking: check again so the tail is covered. + // Sentences or words completed while checking: check again so the tail is covered. if (forGeneration === generation && !state.flagged && state.version !== checkedVersion) { startCheck(state); } @@ -105,16 +145,35 @@ export function createSegmentLoopDetector(options = {}) { */ push(kind, delta) { if (typeof delta !== "string" || delta.length === 0) return false; - const state = getKind(kind); + let state = getKind(kind); const nowMs = now(); if (ttlMs !== undefined && nowMs - state.ts > ttlMs) { - state.text = ""; - state.flagged = false; + // Expired: start this kind over. Bumping the generation drops any stale in-flight answer. + generation++; + state = freshState(); + kinds.set(kind, state); } state.ts = nowMs; - state.text += delta; - state.version++; - if (!state.flagged) startCheck(state); + + // Split only the new text. The unfinished tail is carried forward. + const joined = state.pending + delta; + const { sentences, rest } = takeSentences(joined, state.pending.length); + state.pending = rest; + const { words, rest: partialWord } = takeWords(state.partialWord, delta); + state.partialWord = partialWord; + + if (sentences.length > 0) { + state.sentences.push(...sentences); + trim(state.sentences, windowSize); + } + if (words.length > 0) { + state.words.push(...words); + trim(state.words, wordWindowSize); + } + if (sentences.length > 0 || words.length > 0) { + state.version++; + if (!state.flagged) startCheck(state); + } return state.flagged; }, diff --git a/src/stream/loopDetectorWorker.js b/src/stream/loopDetectorWorker.js index 736c5413..a371d030 100644 --- a/src/stream/loopDetectorWorker.js +++ b/src/stream/loopDetectorWorker.js @@ -3,15 +3,15 @@ import { checkLoop } from "./loopCheck.js"; /** * Piscina worker entry point for loop detection. * - * Runs the text checks off the main event loop. Stateless: each task carries - * the text to check, so any worker thread can serve any task. + * Runs the check off the main event loop. Stateless: each task carries the + * recent sentences and words, so any worker thread can serve any task. * * @param {Object} task - The check to run - * @param {string} task.text - Accumulated stream text - * @param {number} task.windowSize - Recent completed sentences to compare + * @param {string[]} task.sentences - Recent completed sentences + * @param {string[]} task.words - Recent completed words * @param {number} task.threshold - Repeats above which a loop is flagged - * @returns {boolean} True when the text is looping + * @returns {boolean} True when the stream is looping */ -export default function loopWorker({ text, windowSize, threshold }) { - return checkLoop(text, { windowSize, threshold }); +export default function loopWorker(task) { + return checkLoop(task); } diff --git a/tests/unit/stream/loopCheck.test.js b/tests/unit/stream/loopCheck.test.js index c6a3abcd..f4b6d49b 100644 --- a/tests/unit/stream/loopCheck.test.js +++ b/tests/unit/stream/loopCheck.test.js @@ -2,96 +2,168 @@ import { describe, it } from "node:test"; import assert from "node:assert"; import { checkLoop, - completedSentences, hasDuplicateSentence, hasRepeatedWords, + takeSentences, + takeWords, } from "../../../src/stream/loopCheck.js"; -const OPTS = { windowSize: 20, threshold: 2 }; - -describe("completedSentences", () => { - it("splits on sentence ends and newlines and drops the unfinished tail", () => { - assert.deepStrictEqual(completedSentences("One. Two? Three!\nFour and"), [ - "One.", - "Two?", - "Three!", - ]); +describe("takeSentences", () => { + it("splits on sentence ends and newlines and returns the unfinished tail", () => { + assert.deepStrictEqual(takeSentences("One. Two? Three!\nFour and"), { + sentences: ["One.", "Two?", "Three!"], + rest: "Four and", + }); }); it("collapses whitespace and skips blanks", () => { - assert.deepStrictEqual(completedSentences(" a b. \n\n c d. tail"), ["a b.", "c d."]); + assert.deepStrictEqual(takeSentences(" a b. \n\n c d. tail"), { + sentences: ["a b.", "c d."], + rest: "tail", + }); + }); + + it("returns nothing completed when no sentence has ended", () => { + assert.deepStrictEqual(takeSentences("ok ok ok ok"), { sentences: [], rest: "ok ok ok ok" }); + assert.deepStrictEqual(takeSentences(""), { sentences: [], rest: "" }); + }); + + it("scans only from the given index, so a boundary split across deltas is still found", () => { + const first = takeSentences("Hello there."); + assert.deepStrictEqual(first, { sentences: [], rest: "Hello there." }); + // The next delta starts with the whitespace that completes the sentence. + const joined = first.rest + " Next"; + assert.deepStrictEqual(takeSentences(joined, first.rest.length), { + sentences: ["Hello there."], + rest: "Next", + }); + }); + + it("gives the same sentences however the text is chunked", () => { + const text = "First one. Second one? Third one!\nFourth line\nFifth one. tail"; + const whole = takeSentences(text).sentences; + for (const size of [1, 2, 3, 5, 7, 11]) { + let pending = ""; + const got = []; + for (let i = 0; i < text.length; i += size) { + const delta = text.slice(i, i + size); + const out = takeSentences(pending + delta, pending.length); + pending = out.rest; + got.push(...out.sentences); + } + assert.deepStrictEqual(got, whole, `chunk size ${size}`); + } + }); +}); + +describe("takeWords", () => { + it("returns completed words and carries the unfinished one", () => { + assert.deepStrictEqual(takeWords("", "ok ok o"), { words: ["ok", "ok"], rest: "o" }); + }); + + it("joins a word split across deltas", () => { + const a = takeWords("", "hel"); + const b = takeWords(a.rest, "lo wor"); + assert.deepStrictEqual(a, { words: [], rest: "hel" }); + assert.deepStrictEqual(b, { words: ["hello"], rest: "wor" }); + }); + + it("completes the word when the delta ends in whitespace", () => { + assert.deepStrictEqual(takeWords("", "ok "), { words: ["ok"], rest: "" }); }); - it("returns nothing when no sentence has ended", () => { - assert.deepStrictEqual(completedSentences("ok ok ok ok"), []); - assert.deepStrictEqual(completedSentences(""), []); + it("gives the same words however the text is chunked", () => { + const text = "the quick brown fox jumps over the lazy dog "; + const whole = takeWords("", text).words; + for (const size of [1, 2, 3, 5, 7]) { + let partial = ""; + const got = []; + for (let i = 0; i < text.length; i += size) { + const out = takeWords(partial, text.slice(i, i + size)); + partial = out.rest; + got.push(...out.words); + } + assert.deepStrictEqual(got, whole, `chunk size ${size}`); + } }); }); describe("hasDuplicateSentence", () => { it("flags a sentence repeated more than threshold times", () => { - assert.strictEqual(hasDuplicateSentence("Same thing. ".repeat(3) + "tail", 20, 2), true); - assert.strictEqual(hasDuplicateSentence("Same thing. ".repeat(2) + "tail", 20, 2), false); + assert.strictEqual( + hasDuplicateSentence(["Same thing.", "Same thing.", "Same thing."], 2), + true, + ); + assert.strictEqual(hasDuplicateSentence(["Same thing.", "Same thing."], 2), false); }); it("does not need the repeats to be back-to-back", () => { - const text = "Alpha one. Beta two. Alpha one. Gamma three. Alpha one. tail"; - assert.strictEqual(hasDuplicateSentence(text, 20, 2), true); - }); - - it("only compares the last windowSize sentences", () => { - const text = - "Same thing. " + "Filler a. Filler b. Filler c. ".repeat(1) + "Same thing. Same thing. tail"; - // 3 copies exist, but a window of 3 only sees the last 3 sentences. - assert.strictEqual(hasDuplicateSentence(text, 3, 2), false); - assert.strictEqual(hasDuplicateSentence(text, 20, 2), true); + assert.strictEqual( + hasDuplicateSentence(["Alpha.", "Beta.", "Alpha.", "Gamma.", "Alpha."], 2), + true, + ); }); it("does not flag distinct sentences", () => { - const text = Array.from({ length: 30 }, (_, i) => `Sentence number ${i}.`).join(" ") + " tail"; - assert.strictEqual(hasDuplicateSentence(text, 20, 2), false); + const list = Array.from({ length: 30 }, (_, i) => `Sentence number ${i}.`); + assert.strictEqual(hasDuplicateSentence(list, 2), false); }); it("does not lowercase: a differently-cased sentence is a different sentence", () => { - assert.strictEqual(hasDuplicateSentence("Ok then. ok then. OK THEN. tail", 20, 2), false); + assert.strictEqual(hasDuplicateSentence(["Ok then.", "ok then.", "OK THEN."], 2), false); }); }); describe("hasRepeatedWords", () => { + const w = (s) => s.split(" "); + it("flags 'ok' repeated with no punctuation", () => { - assert.strictEqual(hasRepeatedWords("ok ok ok", 2), true); - assert.strictEqual(hasRepeatedWords("ok ok", 2), false); + assert.strictEqual(hasRepeatedWords(w("ok ok ok"), 2), true); + assert.strictEqual(hasRepeatedWords(w("ok ok"), 2), false); }); it("flags a multi-word run repeated back-to-back", () => { - assert.strictEqual(hasRepeatedWords("so then we go so then we go so then we go", 2), true); + assert.strictEqual(hasRepeatedWords(w("so then we go so then we go so then we go"), 2), true); }); - it("flags a repeat at the tail even after unrelated text", () => { - assert.strictEqual(hasRepeatedWords("something else entirely ok ok ok", 2), true); + it("flags a repeat at the tail even after unrelated words", () => { + assert.strictEqual(hasRepeatedWords(w("something else entirely ok ok ok"), 2), true); }); it("does not flag a repeat that is not at the tail", () => { - assert.strictEqual(hasRepeatedWords("ok ok ok and then it moved on", 2), false); + assert.strictEqual(hasRepeatedWords(w("ok ok ok and then it moved on"), 2), false); }); - it("does not flag text with no repeat", () => { - assert.strictEqual(hasRepeatedWords("the quick brown fox jumps over the lazy dog", 2), false); - assert.strictEqual(hasRepeatedWords("", 2), false); + it("does not flag words with no repeat, or an empty list", () => { + assert.strictEqual( + hasRepeatedWords(w("the quick brown fox jumps over the lazy dog"), 2), + false, + ); + assert.strictEqual(hasRepeatedWords([], 2), false); }); it("honours a higher threshold", () => { - assert.strictEqual(hasRepeatedWords("ok ok ok ok", 4), false); - assert.strictEqual(hasRepeatedWords("ok ok ok ok ok", 4), true); + assert.strictEqual(hasRepeatedWords(w("ok ok ok ok"), 4), false); + assert.strictEqual(hasRepeatedWords(w("ok ok ok ok ok"), 4), true); }); }); describe("checkLoop", () => { it("is true for either kind of repeat and false otherwise", () => { - assert.strictEqual(checkLoop("ok ".repeat(20), OPTS), true); - assert.strictEqual(checkLoop("This is a test. ".repeat(5), OPTS), true); assert.strictEqual( - checkLoop("A normal paragraph. With different sentences. Nothing repeats.", OPTS), + checkLoop({ sentences: [], words: Array(20).fill("ok"), threshold: 2 }), + true, + ); + assert.strictEqual( + checkLoop({ sentences: Array(5).fill("This is a test."), words: [], threshold: 2 }), + true, + ); + assert.strictEqual( + checkLoop({ + sentences: ["A one.", "B two.", "C three."], + words: ["a", "b", "c"], + threshold: 2, + }), false, ); }); diff --git a/tests/unit/stream/loopDetector.test.js b/tests/unit/stream/loopDetector.test.js index 68df2514..0209ca22 100644 --- a/tests/unit/stream/loopDetector.test.js +++ b/tests/unit/stream/loopDetector.test.js @@ -9,7 +9,7 @@ after(async () => { }); /** Runner that executes the check inline, so unit tests need no thread. */ -const inline = { run: async (task) => checkLoop(task.text, task) }; +const inline = { run: async (task) => checkLoop(task) }; const detector = (extra = {}) => createSegmentLoopDetector({ runner: inline, ...extra }); @@ -77,7 +77,7 @@ describe("createSegmentLoopDetector (worker-backed)", () => { peak = Math.max(peak, running); await new Promise((r) => setTimeout(r, 5)); running--; - return checkLoop(task.text, task); + return checkLoop(task); }, }; const d = createSegmentLoopDetector({ runner: slow }); @@ -90,9 +90,9 @@ describe("createSegmentLoopDetector (worker-backed)", () => { const seen = []; const slow = { run: async (task) => { - seen.push(task.text); + seen.push(task); await new Promise((r) => setTimeout(r, 5)); - return checkLoop(task.text, task); + return checkLoop(task); }, }; const d = createSegmentLoopDetector({ runner: slow }); @@ -101,6 +101,7 @@ describe("createSegmentLoopDetector (worker-backed)", () => { d.push("reasoning", "ok "); assert.strictEqual(await d.settle(), true); assert.ok(seen.length >= 2, "the tail was checked again"); + assert.ok(seen.at(-1).words.length > seen[0].words.length, "the later check saw newer words"); }); it("treats a failing check as not looping and keeps streaming", async () => { @@ -134,11 +135,96 @@ describe("createSegmentLoopDetector (worker-backed)", () => { assert.strictEqual(await d.settle(), true); }); + it("keeps only the last windowSize sentences and wordWindowSize words", async () => { + const seen = []; + const spy = { + run: async (task) => { + seen.push(task); + return false; + }, + }; + const d = createSegmentLoopDetector({ runner: spy, windowSize: 5, wordWindowSize: 12 }); + for (let i = 0; i < 100; i++) { + d.push("reasoning", `Unique sentence number ${i} here. `); + await d.settle(); + } + const last = seen.at(-1); + assert.ok(last.sentences.length <= 5, `sentences ${last.sentences.length}`); + assert.ok(last.words.length <= 12, `words ${last.words.length}`); + assert.strictEqual(last.sentences.at(-1), "Unique sentence number 99 here."); + }); + + it("sends the worker a bounded task however long the stream gets", async () => { + const sizes = []; + const spy = { + run: async (task) => { + sizes.push(task.sentences.length + task.words.length); + return false; + }, + }; + const d = createSegmentLoopDetector({ runner: spy, windowSize: 20, wordWindowSize: 200 }); + for (let i = 0; i < 2000; i++) { + d.push("reasoning", `Distinct sentence ${i} goes here. `); + if (i % 50 === 0) await d.settle(); + } + await d.settle(); + assert.ok(Math.max(...sizes) <= 220, `max task size ${Math.max(...sizes)}`); + }); + + it("does not start a check for a delta that completes no sentence or word", async () => { + let calls = 0; + const spy = { + run: async () => { + calls++; + return false; + }, + }; + const d = createSegmentLoopDetector({ runner: spy }); + d.push("reasoning", "hel"); + d.push("reasoning", "lo"); + d.push("reasoning", " wor"); + await d.settle(); + const afterFirstWord = calls; + d.push("reasoning", "l"); + d.push("reasoning", "d"); + await d.settle(); + assert.strictEqual(calls, afterFirstWord, "no new check until a word completes"); + }); + + it("finds a sentence whose boundary falls across two deltas", async () => { + const d = detector(); + for (const delta of ["Same thing.", " ", "Same thing.", " ", "Same thing.", " ", "x"]) { + d.push("reasoning", delta); + } + assert.strictEqual(await d.settle(), true); + }); + + it("reaches the same verdict however the stream is chunked", async () => { + const text = "This is a test of streamed repetition. ".repeat(5); + for (const size of [1, 2, 3, 5, 8, 13]) { + const d = detector(); + for (let i = 0; i < text.length; i += size) d.push("reasoning", text.slice(i, i + size)); + assert.strictEqual(await d.settle(), true, `chunk size ${size}`); + } + }); + + it("drops state older than ttlMs for that kind only", async () => { + let clock = 0; + const d = detector({ ttlMs: 100, now: () => clock }); + d.push("reasoning", "ok ok "); + await d.settle(); + clock = 50; + d.push("message", "fresh "); + clock = 500; + d.push("reasoning", "ok "); + assert.strictEqual(await d.settle(), false); + }); + it("reset clears state and discards a stale in-flight result", async () => { const slow = { run: async (task) => { await new Promise((r) => setTimeout(r, 10)); - return checkLoop(task.text, task); + return checkLoop(task); }, }; const d = createSegmentLoopDetector({ runner: slow }); From 56033280fe8c6c5775d051bd73d1120c1cf5e7e4 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 20:37:45 -0500 Subject: [PATCH 14/20] feat: make the word window configurable and large enough Add loopDetection.wordWindowSize (default 1000) to the schema, config.yaml and index.js. The back-to-back word check has to hold threshold + 1 copies of the longest repeating unit: the two-paragraph capture is 190 words, so three copies need 570 and the previous hardcoded 200 could not see it. Measured on a no-match worst case, the check stays at a few microseconds and the task copy is about 19 us at 1000 words. Also correct the schema header, which still described hashing deltas. --- config.yaml | 1 + coverage.txt | 6 +- index.js | 1 + src/config/schemas/loopDetection.js | 21 +++-- src/stream/loopDetector.js | 4 +- tests/unit/stream/loopDetectorWindow.test.js | 87 ++++++++++++++++++++ 6 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 tests/unit/stream/loopDetectorWindow.test.js diff --git a/config.yaml b/config.yaml index 6436c3c6..883507cc 100644 --- a/config.yaml +++ b/config.yaml @@ -62,6 +62,7 @@ loopDetection: threshold: 2 ttlMs: 60000 windowSize: 20 + wordWindowSize: 1000 memory: checkpointsDir: memory/checkpoints/ contextDir: memory/context/ diff --git a/coverage.txt b/coverage.txt index 2e3f5f1f..0da691d5 100644 --- a/coverage.txt +++ b/coverage.txt @@ -8,7 +8,7 @@ ℹ compaction.js | 98.84 | 78.43 | 100.00 | 89-90 ℹ contextBackend.js | 100.00 | 100.00 | 100.00 | ℹ coreBackend.js | 100.00 | 100.00 | 100.00 | -ℹ deepAgents.js | 93.72 | 68.00 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 +ℹ deepAgents.js | 93.72 | 69.39 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 ℹ evictionCeiling.js | 100.00 | 100.00 | 100.00 | ℹ skillsMiddleware.js | 100.00 | 87.50 | 100.00 | ℹ config | | | | @@ -189,7 +189,7 @@ ℹ git | | | | ℹ index.js | 87.41 | 64.37 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 ℹ graphql | | | | -ℹ index.js | 97.85 | 98.13 | 93.75 | 57-68 +ℹ index.js | 97.50 | 97.17 | 93.75 | 47-48 57-68 ℹ image | | | | ℹ index.js | 100.00 | 100.00 | 50.00 | ℹ readImage.js | 94.16 | 72.22 | 100.00 | 79-80 88-89 95-96 103-104 @@ -286,6 +286,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.36 | 86.66 | 88.35 | +ℹ all files | 93.35 | 86.66 | 88.35 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/index.js b/index.js index 2b47cee5..8ac9ab32 100644 --- a/index.js +++ b/index.js @@ -306,6 +306,7 @@ async function runAgent(input, threadExtras, streamingCallback, signal) { ? undefined : createSegmentLoopDetector({ windowSize: loopConfig.windowSize, + wordWindowSize: loopConfig.wordWindowSize, threshold: loopConfig.threshold, ttlMs: loopConfig.ttlMs, }); diff --git a/src/config/schemas/loopDetection.js b/src/config/schemas/loopDetection.js index 0b9370c0..9180383c 100644 --- a/src/config/schemas/loopDetection.js +++ b/src/config/schemas/loopDetection.js @@ -3,19 +3,26 @@ import { z } from "zod"; /** * Configuration for streaming loop detection. * - * When `enabled` is false, no loop detection occurs. When enabled, every - * streamed delta (a segment: a small group of characters as it arrives) is - * hashed into a bounded per-kind list of `windowSize` hashes. A loop is flagged - * when the tail of that list is one unit repeated back-to-back more than - * `threshold` times. The stream is then abandoned and the TUI sends a silent - * "You're looping" nudge. If `escalationCount` nudges occur within - * `escalationWindowMs`, the context is compacted instead of nudging again. + * When `enabled` is false, no loop detection occurs. When enabled, each + * streamed delta is split once, as it arrives, into completed sentences and + * words per kind (reasoning, message), and only the last `windowSize` sentences + * and `wordWindowSize` words are kept. A loop is a sentence repeated more than + * `threshold` times within the sentence window, or a run of words repeated + * back-to-back more than `threshold` times (loops with no punctuation). The + * stream is then abandoned and the TUI sends a silent "You're looping" nudge. + * If `escalationCount` nudges occur within `escalationWindowMs`, the context is + * compacted instead of nudging again. */ export const LoopDetectionSchema = z .object({ enabled: z.boolean().default(true), // Number of recent completed sentences compared for duplicates. windowSize: z.number().int().positive().default(20), + // Number of recent completed words compared for a back-to-back repeat (loops + // with no sentence terminator, such as "ok ok ok"). Must hold `threshold + 1` + // copies of the longest repeating unit: the two-paragraph capture is 190 + // words, so 3 copies need 570. + wordWindowSize: z.number().int().positive().default(1000), // Accumulated stream text older than this many ms is dropped, so past // content cannot cause false positives as new content streams in. ttlMs: z.number().int().positive().default(60000), diff --git a/src/stream/loopDetector.js b/src/stream/loopDetector.js index fe92666d..91cfd210 100644 --- a/src/stream/loopDetector.js +++ b/src/stream/loopDetector.js @@ -47,7 +47,7 @@ export async function destroyPool() { * * @param {Object} [options] - Options * @param {number} [options.windowSize=20] - Recent completed sentences kept and compared - * @param {number} [options.wordWindowSize=200] - Recent completed words kept and compared + * @param {number} [options.wordWindowSize=1000] - Recent completed words kept and compared * @param {number} [options.threshold=2] - Repeats above which a loop is flagged * @param {number} [options.ttlMs] - State older than this is dropped (no expiry when unset) * @param {Function} [options.now] - Clock returning ms (injectable for tests) @@ -56,7 +56,7 @@ export async function destroyPool() { */ export function createSegmentLoopDetector(options = {}) { const windowSize = options.windowSize ?? 20; - const wordWindowSize = options.wordWindowSize ?? 200; + const wordWindowSize = options.wordWindowSize ?? 1000; const threshold = options.threshold ?? 2; const ttlMs = options.ttlMs; const now = options.now ?? (() => Date.now()); diff --git a/tests/unit/stream/loopDetectorWindow.test.js b/tests/unit/stream/loopDetectorWindow.test.js new file mode 100644 index 00000000..d003d664 --- /dev/null +++ b/tests/unit/stream/loopDetectorWindow.test.js @@ -0,0 +1,87 @@ +import { describe, it, after } from "node:test"; +import assert from "node:assert"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { createSegmentLoopDetector, destroyPool } from "../../../src/stream/loopDetector.js"; +import { LoopDetectionSchema } from "../../../src/config/schemas/loopDetection.js"; + +after(async () => { + await destroyPool(); +}); + +// Lines 14-21 of the capture: two paragraphs, hard-wrapped by the TUI. +const PASSAGE = readFileSync( + fileURLToPath(new URL("../../fixtures/loop-reasoning.md", import.meta.url)), + "utf8", +) + .split("\n") + .slice(13, 21) + .map((line) => line.trim()) + .join(" ") + .replace(/\s+/g, " ") + .trim(); + +const WORDS = PASSAGE.split(" "); + +/** Stream a unit of WORDS-only text (no sentence ends) so only the word check can catch it. */ +async function flaggedWithWordWindow(wordWindowSize, copies) { + const d = createSegmentLoopDetector({ wordWindowSize }); + // Strip sentence terminators and newlines so the sentence check cannot fire. + const unit = WORDS.map((w) => w.replace(/[.!?]/g, "")).join(" ") + " "; + for (let c = 0; c < copies; c++) d.push("reasoning", unit); + return d.settle(); +} + +describe("loopDetection config schema", () => { + it("defaults wordWindowSize to 1000", () => { + assert.strictEqual(LoopDetectionSchema.parse({}).wordWindowSize, 1000); + }); + + it("accepts a configured wordWindowSize", () => { + assert.strictEqual(LoopDetectionSchema.parse({ wordWindowSize: 2500 }).wordWindowSize, 2500); + }); + + it("rejects a zero, negative or fractional wordWindowSize", () => { + for (const bad of [0, -5, 1.5]) { + assert.throws( + () => LoopDetectionSchema.parse({ wordWindowSize: bad }), + `wordWindowSize ${bad}`, + ); + } + }); + + it("still rejects unknown keys (strict)", () => { + assert.throws(() => LoopDetectionSchema.parse({ wordWindow: 5 })); + }); + + it("keeps the other defaults", () => { + const cfg = LoopDetectionSchema.parse({}); + assert.strictEqual(cfg.windowSize, 20); + assert.strictEqual(cfg.threshold, 2); + assert.strictEqual(cfg.ttlMs, 60000); + assert.strictEqual(cfg.escalationCount, 3); + assert.strictEqual(cfg.escalationWindowMs, 60000); + }); +}); + +describe("wordWindowSize is big enough for the captured loop", () => { + it("the two-paragraph passage needs more words than the old 200-word window", () => { + assert.ok(WORDS.length * 3 > 200, `3 copies = ${WORDS.length * 3} words`); + }); + + it("the default 1000 holds three copies of the two-paragraph passage", () => { + assert.ok(WORDS.length * 3 <= 1000, `3 copies = ${WORDS.length * 3} words`); + }); + + it("catches the two-paragraph loop with no punctuation at the default size", async () => { + assert.strictEqual(await flaggedWithWordWindow(1000, 4), true); + }); + + it("misses the same loop when the window is too small to hold three copies", async () => { + assert.strictEqual(await flaggedWithWordWindow(200, 4), false); + }); + + it("catches it again once the window just holds three copies", async () => { + assert.strictEqual(await flaggedWithWordWindow(WORDS.length * 3, 4), true); + }); +}); From d47bf59d2a14954b2e9b4078b5c3752559977e15 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 20:44:07 -0500 Subject: [PATCH 15/20] fix: restrict loop detection to a single stream block id State was keyed by kind alone, so concurrent blocks interleaved into one sentence and word list. That could hide a real loop in one block, or invent one from two blocks that merely alternate. This regressed an earlier version that passed the block id through to the detector. Key state by kind plus the block id: the content-block index for message text and block.id for Responses API reasoning. Chat Completions reasoning_content carries no id and stays keyed by kind. Each block gets its own lists and its own in-flight check. At most maxBlocks (16) blocks are tracked and the least recently updated is evicted, so memory stays bounded. Stale-answer handling moves from a global generation counter to a per-block dead flag, so expiring or evicting one block no longer discards another block's in-flight result. --- coverage.txt | 6 +- src/stream/consume.js | 10 +- src/stream/loopDetector.js | 64 +++-- tests/unit/stream/loopDetectorBlocks.test.js | 250 +++++++++++++++++++ 4 files changed, 302 insertions(+), 28 deletions(-) create mode 100644 tests/unit/stream/loopDetectorBlocks.test.js diff --git a/coverage.txt b/coverage.txt index 0da691d5..45f9f444 100644 --- a/coverage.txt +++ b/coverage.txt @@ -95,7 +95,7 @@ ℹ types.js | 100.00 | 100.00 | 100.00 | ℹ validator.js | 100.00 | 100.00 | 100.00 | ℹ stream | | | | -ℹ consume.js | 91.61 | 92.50 | 100.00 | 97-102 114-120 +ℹ consume.js | 91.82 | 92.50 | 100.00 | 101-106 118-124 ℹ interrupt.js | 100.00 | 100.00 | 100.00 | ℹ loopCheck.js | 100.00 | 100.00 | 100.00 | ℹ loopDetector.js | 100.00 | 100.00 | 100.00 | @@ -189,7 +189,7 @@ ℹ git | | | | ℹ index.js | 87.41 | 64.37 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 ℹ graphql | | | | -ℹ index.js | 97.50 | 97.17 | 93.75 | 47-48 57-68 +ℹ index.js | 97.85 | 98.13 | 93.75 | 57-68 ℹ image | | | | ℹ index.js | 100.00 | 100.00 | 50.00 | ℹ readImage.js | 94.16 | 72.22 | 100.00 | 79-80 88-89 95-96 103-104 @@ -286,6 +286,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.35 | 86.66 | 88.35 | +ℹ all files | 93.36 | 86.69 | 88.35 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/src/stream/consume.js b/src/stream/consume.js index dc7f1ea4..06a8dc47 100644 --- a/src/stream/consume.js +++ b/src/stream/consume.js @@ -14,7 +14,10 @@ import { isSubagentNamespace } from "./namespace.js"; * @param {Object} [options] - Options * @param {boolean} [options.showToolResults=true] - Whether to emit `tool` events * @param {{push: Function, settle?: Function, reset: Function}} [options.loopDetector] - Loop - * detector. Each non-subagent message/reasoning delta is pushed to it. When it + * detector. Each non-subagent message/reasoning delta is pushed to it together with + * the stream block's id (the content-block index for messages, `block.id` for + * Responses API reasoning), so detection is restricted to one block and concurrent + * blocks never mix. Chat Completions `reasoning_content` carries no id. When it * flags a loop the stream is abandoned (closing the underlying iterator) and * `loopDetected` is returned true. Checks run in a worker, so a loop can be * flagged after the stream ends; `settle()` is awaited to collect it. @@ -69,7 +72,8 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (!isSubagent) { content += text; - if (loopDetector?.push("message", text)) loopDetected = true; + const blockIndex = msg?.content?.[0]?.index; + if (loopDetector?.push("message", text, blockIndex)) loopDetected = true; } if (streamingCallback) { @@ -108,7 +112,7 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (block?.type === "reasoning" && block.reasoning) { if (!isSubagent) { reasoning += block.reasoning; - if (loopDetector?.push("reasoning", block.reasoning)) loopDetected = true; + if (loopDetector?.push("reasoning", block.reasoning, block.id)) loopDetected = true; } if (streamingCallback) { streamingCallback({ diff --git a/src/stream/loopDetector.js b/src/stream/loopDetector.js index 91cfd210..3d68a978 100644 --- a/src/stream/loopDetector.js +++ b/src/stream/loopDetector.js @@ -33,7 +33,12 @@ export async function destroyPool() { * Create a streaming loop detector. * * Each streamed delta is split once, as it arrives, into completed sentences - * and words per kind (reasoning, message). Only the last `windowSize` sentences + * and words. State is kept per block: the key is the kind (reasoning, message) + * plus the stream block's id, so concurrent blocks never mix and detection is + * restricted to the one block a delta belongs to. A delta with no id (Chat + * Completions `reasoning_content`) is keyed by kind alone. At most `maxBlocks` + * blocks are tracked; the least recently updated is evicted. Only the last + * `windowSize` sentences * and `wordWindowSize` words are kept, so memory and the cost of a check stay * bounded however long the stream runs, and the provider's delta boundaries do * not matter. A loop is a sentence repeated within the sentence window, or a @@ -42,22 +47,24 @@ export async function destroyPool() { * * The check runs in a worker thread so it never blocks the main loop. `push` * returns immediately with the latest known result. A check starts only when a - * delta completed a new sentence or word, and at most one check per kind is in + * delta completed a new sentence or word, and at most one check per block is in * flight. * * @param {Object} [options] - Options * @param {number} [options.windowSize=20] - Recent completed sentences kept and compared * @param {number} [options.wordWindowSize=1000] - Recent completed words kept and compared * @param {number} [options.threshold=2] - Repeats above which a loop is flagged + * @param {number} [options.maxBlocks=16] - Concurrent blocks tracked; the least recently updated is evicted * @param {number} [options.ttlMs] - State older than this is dropped (no expiry when unset) * @param {Function} [options.now] - Clock returning ms (injectable for tests) * @param {{ run: Function }} [options.runner] - Runs a check; defaults to the shared pool, injectable for tests - * @returns {{ push: (kind: string, delta: string) => boolean, settle: () => Promise, reset: () => void }} Detector + * @returns {{ push: (kind: string, delta: string, id?: string | number) => boolean, settle: () => Promise, reset: () => void }} Detector */ export function createSegmentLoopDetector(options = {}) { const windowSize = options.windowSize ?? 20; const wordWindowSize = options.wordWindowSize ?? 1000; const threshold = options.threshold ?? 2; + const maxBlocks = options.maxBlocks ?? 16; const ttlMs = options.ttlMs; const now = options.now ?? (() => Date.now()); const runner = options.runner ?? { run: (task) => getPool().run(task) }; @@ -72,10 +79,10 @@ export function createSegmentLoopDetector(options = {}) { * @property {boolean} flagged - A check has flagged a loop * @property {Promise | null} inflight - The running check, if any * @property {number} version - Bumped when a sentence or word completes + * @property {boolean} dead - The state was replaced, evicted or reset; in-flight answers are stale */ - /** @type {Map} */ + /** @type {Map} Insertion order is recency order, oldest first. */ const kinds = new Map(); - let generation = 0; function freshState() { return { @@ -87,14 +94,27 @@ export function createSegmentLoopDetector(options = {}) { flagged: false, inflight: null, version: 0, + dead: false, }; } - function getKind(kind) { - let state = kinds.get(kind); - if (!state) { - state = freshState(); - kinds.set(kind, state); + /** + * Get (or create) the state for a block and mark it most recently used. The + * least recently used block is evicted once more than `maxBlocks` are tracked. + * @param {string} key - Block key (kind, or kind plus id) + * @param {KindState} [replacement] - State to install in place of the existing one + * @returns {KindState} The block's state + */ + function touch(key, replacement) { + const existing = kinds.get(key); + const state = replacement ?? existing ?? freshState(); + if (existing && existing !== state) existing.dead = true; + kinds.delete(key); + kinds.set(key, state); + if (kinds.size > maxBlocks) { + const oldest = kinds.keys().next().value; + kinds.get(oldest).dead = true; + kinds.delete(oldest); } return state; } @@ -116,13 +136,12 @@ export function createSegmentLoopDetector(options = {}) { */ function startCheck(state) { if (state.inflight) return; - const forGeneration = generation; const checkedVersion = state.version; const task = { sentences: state.sentences.slice(), words: state.words.slice(), threshold }; state.inflight = Promise.resolve(runner.run(task)) .then((looping) => { - // A reset since this check started makes its answer stale. - if (forGeneration === generation && looping) state.flagged = true; + // A state that was replaced, evicted or reset makes its answer stale. + if (!state.dead && looping) state.flagged = true; }) .catch(() => { // A failed check must never break streaming: treat as "not looping". @@ -130,7 +149,7 @@ export function createSegmentLoopDetector(options = {}) { .finally(() => { state.inflight = null; // Sentences or words completed while checking: check again so the tail is covered. - if (forGeneration === generation && !state.flagged && state.version !== checkedVersion) { + if (!state.dead && !state.flagged && state.version !== checkedVersion) { startCheck(state); } }); @@ -141,17 +160,18 @@ export function createSegmentLoopDetector(options = {}) { * Record a streamed delta. * @param {string} kind - Stream kind, e.g. "reasoning" or "message" * @param {string} delta - The delta exactly as streamed - * @returns {boolean} True when a completed check has flagged a loop + * @param {string | number} [id] - The stream block's id; a delta with no id is keyed by kind alone + * @returns {boolean} True when a completed check has flagged a loop for this block */ - push(kind, delta) { + push(kind, delta, id) { if (typeof delta !== "string" || delta.length === 0) return false; - let state = getKind(kind); + const key = id == null ? kind : `${kind}\0${id}`; + let state = touch(key); const nowMs = now(); if (ttlMs !== undefined && nowMs - state.ts > ttlMs) { - // Expired: start this kind over. Bumping the generation drops any stale in-flight answer. - generation++; - state = freshState(); - kinds.set(kind, state); + // Expired: start this block over. touch() marks the old state dead, so + // its in-flight answer is dropped without affecting any other block. + state = touch(key, freshState()); } state.ts = nowMs; @@ -192,7 +212,7 @@ export function createSegmentLoopDetector(options = {}) { /** Clear all state (call after a nudge so the next attempt starts fresh). */ reset() { - generation++; + for (const state of kinds.values()) state.dead = true; kinds.clear(); }, }; diff --git a/tests/unit/stream/loopDetectorBlocks.test.js b/tests/unit/stream/loopDetectorBlocks.test.js new file mode 100644 index 00000000..f51c0964 --- /dev/null +++ b/tests/unit/stream/loopDetectorBlocks.test.js @@ -0,0 +1,250 @@ +/** + * Detection is restricted to the single stream block a delta belongs to. Blocks + * can stream concurrently, so deltas from different blocks interleave; state + * keyed only by kind would mix them. These tests interleave blocks on purpose. + */ + +import { describe, it, after } from "node:test"; +import assert from "node:assert"; +import { createSegmentLoopDetector, destroyPool } from "../../../src/stream/loopDetector.js"; +import { checkLoop } from "../../../src/stream/loopCheck.js"; +import { consumeStream } from "../../../src/stream/consume.js"; + +after(async () => { + await destroyPool(); +}); + +const inline = { run: async (task) => checkLoop(task) }; +const detector = (extra = {}) => createSegmentLoopDetector({ runner: inline, ...extra }); + +describe("per-block detection", () => { + it("flags a loop inside one block", async () => { + const d = detector(); + for (let i = 0; i < 5; i++) d.push("reasoning", "ok ", "a"); + assert.strictEqual(await d.settle(), true); + }); + + it("does not flag when each block repeats only below the threshold", async () => { + const d = detector(); + // 2 copies in block a and 2 in block b: 4 combined would trip a shared list. + for (const id of ["a", "b"]) { + d.push("reasoning", "Same thing. ", id); + d.push("reasoning", "Same thing. ", id); + } + assert.strictEqual(await d.settle(), false); + }); + + it("is not hidden by another block interleaving into the stream", async () => { + const d = detector(); + // Block a loops on 'ok'; block b streams unrelated words between every delta. + // A shared word list would see 'ok w1 ok w2 ok w3 ...' and never find a repeat. + for (let i = 0; i < 6; i++) { + d.push("reasoning", "ok ", "a"); + d.push("reasoning", `unrelated${i} `, "b"); + } + assert.strictEqual(await d.settle(), true); + }); + + it("does not invent a loop from two blocks that merely alternate", async () => { + const d = detector(); + // a says 'x y' and b says 'x y', interleaved, so a shared list reads 'x x y y x x y y'. + for (let i = 0; i < 4; i++) { + d.push("reasoning", "x ", `a${i}`); + d.push("reasoning", "y ", `b${i}`); + } + assert.strictEqual(await d.settle(), false); + }); + + it("keeps reasoning and message with the same id separate", async () => { + const d = detector(); + for (let i = 0; i < 2; i++) { + d.push("reasoning", "Same thing. ", "0"); + d.push("message", "Same thing. ", "0"); + } + assert.strictEqual(await d.settle(), false); + }); + + it("keys deltas with no id by kind alone", async () => { + const d = detector(); + for (let i = 0; i < 5; i++) d.push("reasoning", "ok "); + assert.strictEqual(await d.settle(), true); + }); + + it("treats a numeric id and its string form as the same block", async () => { + const d = detector(); + d.push("message", "ok ", 0); + d.push("message", "ok ", "0"); + d.push("message", "ok ", 0); + assert.strictEqual(await d.settle(), true); + }); + + it("treats id 0 as a real block, not as 'no id'", async () => { + const d = detector(); + for (let i = 0; i < 3; i++) d.push("reasoning", "ok ", 0); + for (let i = 0; i < 3; i++) d.push("reasoning", "different "); + assert.strictEqual(await d.settle(), true, "block 0 looped"); + }); + + it("a word split across deltas stays inside its own block", async () => { + const d = detector(); + // Block a is mid-word when block b's delta arrives. + d.push("reasoning", "ok", "a"); + d.push("reasoning", "zzz ", "b"); + d.push("reasoning", " ok", "a"); + d.push("reasoning", " ok ", "a"); + assert.strictEqual(await d.settle(), true); + }); + + it("runs one check per block, so concurrent blocks check at the same time", async () => { + let running = 0; + let peak = 0; + const slow = { + run: async (task) => { + running++; + peak = Math.max(peak, running); + await new Promise((r) => setTimeout(r, 10)); + running--; + return checkLoop(task); + }, + }; + const d = createSegmentLoopDetector({ runner: slow }); + d.push("reasoning", "alpha ", "a"); + d.push("reasoning", "beta ", "b"); + d.push("reasoning", "gamma ", "c"); + await d.settle(); + assert.strictEqual(peak, 3); + }); + + it("still allows only one check at a time within a block", async () => { + let running = 0; + let peak = 0; + const slow = { + run: async (task) => { + running++; + peak = Math.max(peak, running); + await new Promise((r) => setTimeout(r, 5)); + running--; + return checkLoop(task); + }, + }; + const d = createSegmentLoopDetector({ runner: slow }); + for (let i = 0; i < 30; i++) d.push("reasoning", `word${i} `, "a"); + await d.settle(); + assert.strictEqual(peak, 1); + }); +}); + +describe("tracked block limit", () => { + it("evicts the least recently updated block past maxBlocks", async () => { + const d = detector({ maxBlocks: 2 }); + d.push("reasoning", "ok ok ", "old"); + d.push("reasoning", "x ", "b"); + d.push("reasoning", "y ", "c"); // evicts 'old' + d.push("reasoning", "ok ", "old"); // starts over: only one 'ok' + assert.strictEqual(await d.settle(), false); + }); + + it("does not evict a block that keeps receiving deltas", async () => { + const d = detector({ maxBlocks: 2 }); + d.push("reasoning", "ok ", "busy"); + d.push("reasoning", "x ", "b"); + d.push("reasoning", "ok ", "busy"); // refreshes 'busy' + d.push("reasoning", "y ", "c"); // evicts 'b', not 'busy' + d.push("reasoning", "ok ", "busy"); + assert.strictEqual(await d.settle(), true); + }); + + it("bounds memory across many short-lived blocks", async () => { + const seen = new Set(); + const spy = { + run: async (task) => { + seen.add(task); + return false; + }, + }; + const d = createSegmentLoopDetector({ runner: spy, maxBlocks: 16 }); + for (let i = 0; i < 5000; i++) d.push("message", "hello there ", i); + await d.settle(); + assert.ok(seen.size > 0); + }); + + it("drops a block's state after ttlMs without touching the others", async () => { + let clock = 0; + const d = detector({ ttlMs: 100, now: () => clock }); + d.push("reasoning", "ok ok ", "a"); + clock = 50; + d.push("reasoning", "ok ok ", "b"); + clock = 120; // a is stale (120 > 100), b is not (70) + d.push("reasoning", "ok ", "a"); + d.push("reasoning", "ok ", "b"); + assert.strictEqual(await d.settle(), true, "b kept its history and reached 3"); + }); +}); + +describe("consumeStream passes each block's id to the detector", () => { + const reasoningBlock = (id, text) => [ + [], + "messages", + [{ type: "ai", text: "", content: [{ type: "reasoning", reasoning: text, id }] }], + ]; + const message = (index, text) => [[], "messages", [{ type: "ai", text, content: [{ index }] }]]; + const tick = () => new Promise((r) => setTimeout(r, 2)); + + it("flags a loop in one Responses API block while another block interleaves", async () => { + const stream = (async function* () { + for (let i = 0; i < 20; i++) { + await tick(); + yield reasoningBlock("r1", "ok "); + yield reasoningBlock("r2", `different${i} `); + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, true); + }); + + it("does not flag two blocks whose combined text would repeat", async () => { + const stream = (async function* () { + for (let i = 0; i < 4; i++) { + yield reasoningBlock(`a${i}`, "x "); + yield reasoningBlock(`b${i}`, "y "); + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, false); + }); + + it("keys message text by its content-block index", async () => { + const stream = (async function* () { + for (let i = 0; i < 20; i++) { + await tick(); + yield message(0, "again "); + yield message(1, `other${i} `); + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, true); + }); + + it("does not flag message text split across different block indexes", async () => { + const stream = (async function* () { + for (let i = 0; i < 6; i++) yield message(i, "same "); + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, false); + }); + + it("still detects a Chat Completions reasoning_content loop that has no id", async () => { + const stream = (async function* () { + for (let i = 0; i < 20; i++) { + await tick(); + yield [ + [], + "messages", + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: "ok " } }], + ]; + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, true); + }); +}); From 1000e6fb0e8392822e67175cd059ec82b050a055 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 20:49:48 -0500 Subject: [PATCH 16/20] refactor: rename maxBlocks to maxStreams and add it to config A stream is a kind plus an id, not a block, and a long turn can hold hundreds to thousands of them. Rename the option to maxStreams and set the default to 1000, backed by a measurement: a stream with both windows full holds about 30-40 KB, so 1000 is roughly 30-40 MB worst case. Add maxStreams to the loopDetection schema, config.yaml and index.js, and document that a looping stream is always receiving deltas so eviction, which drops the least recently updated stream, never removes it. Tests and wording updated from block to stream. --- config.yaml | 1 + coverage.txt | 4 +- index.js | 1 + src/config/schemas/loopDetection.js | 10 +++- src/stream/loopDetector.js | 38 +++++++------ ...ks.test.js => loopDetectorStreams.test.js} | 56 +++++++++---------- tests/unit/stream/loopDetectorWindow.test.js | 25 +++++++++ 7 files changed, 86 insertions(+), 49 deletions(-) rename tests/unit/stream/{loopDetectorBlocks.test.js => loopDetectorStreams.test.js} (77%) diff --git a/config.yaml b/config.yaml index 883507cc..c905508d 100644 --- a/config.yaml +++ b/config.yaml @@ -59,6 +59,7 @@ loopDetection: enabled: true escalationCount: 3 escalationWindowMs: 60000 + maxStreams: 1000 threshold: 2 ttlMs: 60000 windowSize: 20 diff --git a/coverage.txt b/coverage.txt index 45f9f444..b526b381 100644 --- a/coverage.txt +++ b/coverage.txt @@ -8,7 +8,7 @@ ℹ compaction.js | 98.84 | 78.43 | 100.00 | 89-90 ℹ contextBackend.js | 100.00 | 100.00 | 100.00 | ℹ coreBackend.js | 100.00 | 100.00 | 100.00 | -ℹ deepAgents.js | 93.72 | 69.39 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 +ℹ deepAgents.js | 93.72 | 68.00 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 ℹ evictionCeiling.js | 100.00 | 100.00 | 100.00 | ℹ skillsMiddleware.js | 100.00 | 87.50 | 100.00 | ℹ config | | | | @@ -187,7 +187,7 @@ ℹ xlsxParser.js | 82.21 | 83.33 | 88.89 | 47-48 93-112 130-137 150-156 ℹ zipExtractor.js | 82.32 | 77.78 | 66.67 | 62-63 104-107 124-142 149-152 ℹ git | | | | -ℹ index.js | 87.41 | 64.37 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 +ℹ index.js | 87.41 | 64.87 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 ℹ graphql | | | | ℹ index.js | 97.85 | 98.13 | 93.75 | 57-68 ℹ image | | | | diff --git a/index.js b/index.js index 8ac9ab32..5f501ce6 100644 --- a/index.js +++ b/index.js @@ -307,6 +307,7 @@ async function runAgent(input, threadExtras, streamingCallback, signal) { : createSegmentLoopDetector({ windowSize: loopConfig.windowSize, wordWindowSize: loopConfig.wordWindowSize, + maxStreams: loopConfig.maxStreams, threshold: loopConfig.threshold, ttlMs: loopConfig.ttlMs, }); diff --git a/src/config/schemas/loopDetection.js b/src/config/schemas/loopDetection.js index 9180383c..2ba34a97 100644 --- a/src/config/schemas/loopDetection.js +++ b/src/config/schemas/loopDetection.js @@ -8,7 +8,8 @@ import { z } from "zod"; * words per kind (reasoning, message), and only the last `windowSize` sentences * and `wordWindowSize` words are kept. A loop is a sentence repeated more than * `threshold` times within the sentence window, or a run of words repeated - * back-to-back more than `threshold` times (loops with no punctuation). The + * back-to-back more than `threshold` times (loops with no punctuation). State + * is kept per stream (a kind plus an id), at most `maxStreams` at once. The * stream is then abandoned and the TUI sends a silent "You're looping" nudge. * If `escalationCount` nudges occur within `escalationWindowMs`, the context is * compacted instead of nudging again. @@ -23,6 +24,13 @@ export const LoopDetectionSchema = z // copies of the longest repeating unit: the two-paragraph capture is 190 // words, so 3 copies need 570. wordWindowSize: z.number().int().positive().default(1000), + // Maximum number of streams (a kind plus an id) tracked at once. A long turn + // can produce hundreds to thousands of reasoning and message streams, but only + // the ones still receiving deltas matter; when more than this many are live, + // the least recently updated is evicted. A looping stream is always actively + // receiving deltas, so it is never the one evicted. Each full stream holds + // about 30-40 KB (measured), so 1000 is on the order of 30-40 MB worst case. + maxStreams: z.number().int().positive().default(1000), // Accumulated stream text older than this many ms is dropped, so past // content cannot cause false positives as new content streams in. ttlMs: z.number().int().positive().default(60000), diff --git a/src/stream/loopDetector.js b/src/stream/loopDetector.js index 3d68a978..eb51108a 100644 --- a/src/stream/loopDetector.js +++ b/src/stream/loopDetector.js @@ -33,12 +33,14 @@ export async function destroyPool() { * Create a streaming loop detector. * * Each streamed delta is split once, as it arrives, into completed sentences - * and words. State is kept per block: the key is the kind (reasoning, message) - * plus the stream block's id, so concurrent blocks never mix and detection is - * restricted to the one block a delta belongs to. A delta with no id (Chat - * Completions `reasoning_content`) is keyed by kind alone. At most `maxBlocks` - * blocks are tracked; the least recently updated is evicted. Only the last - * `windowSize` sentences + * and words. State is kept per stream: the key is the kind (reasoning, message) + * plus the stream's id, so concurrent streams never mix and detection is + * restricted to the one stream a delta belongs to. A delta with no id (Chat + * Completions `reasoning_content`) is keyed by kind alone. At most `maxStreams` + * streams are tracked at once; the least recently updated is evicted. A looping + * stream is by definition receiving deltas, so it is never the one evicted; the + * cap only matters when more than `maxStreams` streams are live at the same + * moment. Only the last `windowSize` sentences * and `wordWindowSize` words are kept, so memory and the cost of a check stay * bounded however long the stream runs, and the provider's delta boundaries do * not matter. A loop is a sentence repeated within the sentence window, or a @@ -47,14 +49,14 @@ export async function destroyPool() { * * The check runs in a worker thread so it never blocks the main loop. `push` * returns immediately with the latest known result. A check starts only when a - * delta completed a new sentence or word, and at most one check per block is in + * delta completed a new sentence or word, and at most one check per stream is in * flight. * * @param {Object} [options] - Options * @param {number} [options.windowSize=20] - Recent completed sentences kept and compared * @param {number} [options.wordWindowSize=1000] - Recent completed words kept and compared * @param {number} [options.threshold=2] - Repeats above which a loop is flagged - * @param {number} [options.maxBlocks=16] - Concurrent blocks tracked; the least recently updated is evicted + * @param {number} [options.maxStreams=1000] - Streams tracked at once; the least recently updated is evicted * @param {number} [options.ttlMs] - State older than this is dropped (no expiry when unset) * @param {Function} [options.now] - Clock returning ms (injectable for tests) * @param {{ run: Function }} [options.runner] - Runs a check; defaults to the shared pool, injectable for tests @@ -64,7 +66,7 @@ export function createSegmentLoopDetector(options = {}) { const windowSize = options.windowSize ?? 20; const wordWindowSize = options.wordWindowSize ?? 1000; const threshold = options.threshold ?? 2; - const maxBlocks = options.maxBlocks ?? 16; + const maxStreams = options.maxStreams ?? 1000; const ttlMs = options.ttlMs; const now = options.now ?? (() => Date.now()); const runner = options.runner ?? { run: (task) => getPool().run(task) }; @@ -99,11 +101,11 @@ export function createSegmentLoopDetector(options = {}) { } /** - * Get (or create) the state for a block and mark it most recently used. The - * least recently used block is evicted once more than `maxBlocks` are tracked. - * @param {string} key - Block key (kind, or kind plus id) + * Get (or create) the state for a stream and mark it most recently used. The + * least recently used stream is evicted once more than `maxStreams` are tracked. + * @param {string} key - Stream key (kind, or kind plus id) * @param {KindState} [replacement] - State to install in place of the existing one - * @returns {KindState} The block's state + * @returns {KindState} The stream's state */ function touch(key, replacement) { const existing = kinds.get(key); @@ -111,7 +113,7 @@ export function createSegmentLoopDetector(options = {}) { if (existing && existing !== state) existing.dead = true; kinds.delete(key); kinds.set(key, state); - if (kinds.size > maxBlocks) { + if (kinds.size > maxStreams) { const oldest = kinds.keys().next().value; kinds.get(oldest).dead = true; kinds.delete(oldest); @@ -160,8 +162,8 @@ export function createSegmentLoopDetector(options = {}) { * Record a streamed delta. * @param {string} kind - Stream kind, e.g. "reasoning" or "message" * @param {string} delta - The delta exactly as streamed - * @param {string | number} [id] - The stream block's id; a delta with no id is keyed by kind alone - * @returns {boolean} True when a completed check has flagged a loop for this block + * @param {string | number} [id] - The stream's id; a delta with no id is keyed by kind alone + * @returns {boolean} True when a completed check has flagged a loop for this stream */ push(kind, delta, id) { if (typeof delta !== "string" || delta.length === 0) return false; @@ -169,8 +171,8 @@ export function createSegmentLoopDetector(options = {}) { let state = touch(key); const nowMs = now(); if (ttlMs !== undefined && nowMs - state.ts > ttlMs) { - // Expired: start this block over. touch() marks the old state dead, so - // its in-flight answer is dropped without affecting any other block. + // Expired: start this stream over. touch() marks the old state dead, so + // its in-flight answer is dropped without affecting any other stream. state = touch(key, freshState()); } state.ts = nowMs; diff --git a/tests/unit/stream/loopDetectorBlocks.test.js b/tests/unit/stream/loopDetectorStreams.test.js similarity index 77% rename from tests/unit/stream/loopDetectorBlocks.test.js rename to tests/unit/stream/loopDetectorStreams.test.js index f51c0964..4d127511 100644 --- a/tests/unit/stream/loopDetectorBlocks.test.js +++ b/tests/unit/stream/loopDetectorStreams.test.js @@ -1,7 +1,7 @@ /** - * Detection is restricted to the single stream block a delta belongs to. Blocks - * can stream concurrently, so deltas from different blocks interleave; state - * keyed only by kind would mix them. These tests interleave blocks on purpose. + * Detection is restricted to the single stream a delta belongs to. Streams + * can stream concurrently, so deltas from different streams interleave; state + * keyed only by kind would mix them. These tests interleave streams on purpose. */ import { describe, it, after } from "node:test"; @@ -17,16 +17,16 @@ after(async () => { const inline = { run: async (task) => checkLoop(task) }; const detector = (extra = {}) => createSegmentLoopDetector({ runner: inline, ...extra }); -describe("per-block detection", () => { - it("flags a loop inside one block", async () => { +describe("per-stream detection", () => { + it("flags a loop inside one stream", async () => { const d = detector(); for (let i = 0; i < 5; i++) d.push("reasoning", "ok ", "a"); assert.strictEqual(await d.settle(), true); }); - it("does not flag when each block repeats only below the threshold", async () => { + it("does not flag when each stream repeats only below the threshold", async () => { const d = detector(); - // 2 copies in block a and 2 in block b: 4 combined would trip a shared list. + // 2 copies in stream a and 2 in stream b: 4 combined would trip a shared list. for (const id of ["a", "b"]) { d.push("reasoning", "Same thing. ", id); d.push("reasoning", "Same thing. ", id); @@ -34,9 +34,9 @@ describe("per-block detection", () => { assert.strictEqual(await d.settle(), false); }); - it("is not hidden by another block interleaving into the stream", async () => { + it("is not hidden by another stream interleaving into the stream", async () => { const d = detector(); - // Block a loops on 'ok'; block b streams unrelated words between every delta. + // Stream a loops on 'ok'; stream b streams unrelated words between every delta. // A shared word list would see 'ok w1 ok w2 ok w3 ...' and never find a repeat. for (let i = 0; i < 6; i++) { d.push("reasoning", "ok ", "a"); @@ -45,7 +45,7 @@ describe("per-block detection", () => { assert.strictEqual(await d.settle(), true); }); - it("does not invent a loop from two blocks that merely alternate", async () => { + it("does not invent a loop from two streams that merely alternate", async () => { const d = detector(); // a says 'x y' and b says 'x y', interleaved, so a shared list reads 'x x y y x x y y'. for (let i = 0; i < 4; i++) { @@ -70,7 +70,7 @@ describe("per-block detection", () => { assert.strictEqual(await d.settle(), true); }); - it("treats a numeric id and its string form as the same block", async () => { + it("treats a numeric id and its string form as the same stream", async () => { const d = detector(); d.push("message", "ok ", 0); d.push("message", "ok ", "0"); @@ -78,16 +78,16 @@ describe("per-block detection", () => { assert.strictEqual(await d.settle(), true); }); - it("treats id 0 as a real block, not as 'no id'", async () => { + it("treats id 0 as a real stream, not as 'no id'", async () => { const d = detector(); for (let i = 0; i < 3; i++) d.push("reasoning", "ok ", 0); for (let i = 0; i < 3; i++) d.push("reasoning", "different "); - assert.strictEqual(await d.settle(), true, "block 0 looped"); + assert.strictEqual(await d.settle(), true, "stream 0 looped"); }); - it("a word split across deltas stays inside its own block", async () => { + it("a word split across deltas stays inside its own stream", async () => { const d = detector(); - // Block a is mid-word when block b's delta arrives. + // Stream a is mid-word when stream b's delta arrives. d.push("reasoning", "ok", "a"); d.push("reasoning", "zzz ", "b"); d.push("reasoning", " ok", "a"); @@ -95,7 +95,7 @@ describe("per-block detection", () => { assert.strictEqual(await d.settle(), true); }); - it("runs one check per block, so concurrent blocks check at the same time", async () => { + it("runs one check per stream, so concurrent streams check at the same time", async () => { let running = 0; let peak = 0; const slow = { @@ -115,7 +115,7 @@ describe("per-block detection", () => { assert.strictEqual(peak, 3); }); - it("still allows only one check at a time within a block", async () => { + it("still allows only one check at a time within a stream", async () => { let running = 0; let peak = 0; const slow = { @@ -134,9 +134,9 @@ describe("per-block detection", () => { }); }); -describe("tracked block limit", () => { - it("evicts the least recently updated block past maxBlocks", async () => { - const d = detector({ maxBlocks: 2 }); +describe("tracked stream limit", () => { + it("evicts the least recently updated stream past maxStreams", async () => { + const d = detector({ maxStreams: 2 }); d.push("reasoning", "ok ok ", "old"); d.push("reasoning", "x ", "b"); d.push("reasoning", "y ", "c"); // evicts 'old' @@ -144,8 +144,8 @@ describe("tracked block limit", () => { assert.strictEqual(await d.settle(), false); }); - it("does not evict a block that keeps receiving deltas", async () => { - const d = detector({ maxBlocks: 2 }); + it("does not evict a stream that keeps receiving deltas", async () => { + const d = detector({ maxStreams: 2 }); d.push("reasoning", "ok ", "busy"); d.push("reasoning", "x ", "b"); d.push("reasoning", "ok ", "busy"); // refreshes 'busy' @@ -154,7 +154,7 @@ describe("tracked block limit", () => { assert.strictEqual(await d.settle(), true); }); - it("bounds memory across many short-lived blocks", async () => { + it("bounds memory across many short-lived streams", async () => { const seen = new Set(); const spy = { run: async (task) => { @@ -162,13 +162,13 @@ describe("tracked block limit", () => { return false; }, }; - const d = createSegmentLoopDetector({ runner: spy, maxBlocks: 16 }); + const d = createSegmentLoopDetector({ runner: spy, maxStreams: 16 }); for (let i = 0; i < 5000; i++) d.push("message", "hello there ", i); await d.settle(); assert.ok(seen.size > 0); }); - it("drops a block's state after ttlMs without touching the others", async () => { + it("drops a stream's state after ttlMs without touching the others", async () => { let clock = 0; const d = detector({ ttlMs: 100, now: () => clock }); d.push("reasoning", "ok ok ", "a"); @@ -181,7 +181,7 @@ describe("tracked block limit", () => { }); }); -describe("consumeStream passes each block's id to the detector", () => { +describe("consumeStream passes each stream's id to the detector", () => { const reasoningBlock = (id, text) => [ [], "messages", @@ -190,7 +190,7 @@ describe("consumeStream passes each block's id to the detector", () => { const message = (index, text) => [[], "messages", [{ type: "ai", text, content: [{ index }] }]]; const tick = () => new Promise((r) => setTimeout(r, 2)); - it("flags a loop in one Responses API block while another block interleaves", async () => { + it("flags a loop in one Responses API stream while another stream interleaves", async () => { const stream = (async function* () { for (let i = 0; i < 20; i++) { await tick(); @@ -202,7 +202,7 @@ describe("consumeStream passes each block's id to the detector", () => { assert.strictEqual(result.loopDetected, true); }); - it("does not flag two blocks whose combined text would repeat", async () => { + it("does not flag two streams whose combined text would repeat", async () => { const stream = (async function* () { for (let i = 0; i < 4; i++) { yield reasoningBlock(`a${i}`, "x "); diff --git a/tests/unit/stream/loopDetectorWindow.test.js b/tests/unit/stream/loopDetectorWindow.test.js index d003d664..9d178aa4 100644 --- a/tests/unit/stream/loopDetectorWindow.test.js +++ b/tests/unit/stream/loopDetectorWindow.test.js @@ -56,6 +56,7 @@ describe("loopDetection config schema", () => { it("keeps the other defaults", () => { const cfg = LoopDetectionSchema.parse({}); + assert.strictEqual(cfg.maxStreams, 1000); assert.strictEqual(cfg.windowSize, 20); assert.strictEqual(cfg.threshold, 2); assert.strictEqual(cfg.ttlMs, 60000); @@ -64,6 +65,30 @@ describe("loopDetection config schema", () => { }); }); +describe("loopDetection maxStreams", () => { + it("defaults to 1000", () => { + assert.strictEqual(LoopDetectionSchema.parse({}).maxStreams, 1000); + }); + + it("accepts a configured value", () => { + assert.strictEqual(LoopDetectionSchema.parse({ maxStreams: 5000 }).maxStreams, 5000); + }); + + it("rejects zero, negative and fractional values", () => { + for (const bad of [0, -1, 2.5]) { + assert.throws(() => LoopDetectionSchema.parse({ maxStreams: bad }), `maxStreams ${bad}`); + } + }); + + it("is set in config.yaml to the schema default", () => { + const yaml = readFileSync( + fileURLToPath(new URL("../../../config.yaml", import.meta.url)), + "utf8", + ); + assert.match(yaml, /^\s+maxStreams: 1000$/m); + }); +}); + describe("wordWindowSize is big enough for the captured loop", () => { it("the two-paragraph passage needs more words than the old 200-word window", () => { assert.ok(WORDS.length * 3 > 200, `3 copies = ${WORDS.length * 3} words`); From fe9e4f5ae54e2637686d2ce76b79a1f8ec7a6df2 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 20:55:59 -0500 Subject: [PATCH 17/20] fix: release detector state when a turn ends and set maxStreams to 4 consumeStream now resets the loop detector in a finally block, so its word and sentence lists are cleared and nulled when the turn ends however it ends: finished, interrupted by a clarify, abandoned on a detected loop, or thrown. A released state is also marked dead, so a check still running in the worker cannot flag a stream that has been released. maxStreams default goes from 1000 to 4: two concurrent streams is the realistic edge case and four leaves headroom, since finished streams are released and a long turn of sequential messages never accumulates them. Adds size() to the detector so tests can assert on what is tracked. --- config.yaml | 2 +- coverage.txt | 10 +- src/config/schemas/loopDetection.js | 14 +- src/stream/consume.js | 21 +- src/stream/loopDetector.js | 43 +++- tests/unit/stream/loopDetectorTurnEnd.test.js | 193 ++++++++++++++++++ tests/unit/stream/loopDetectorWindow.test.js | 8 +- 7 files changed, 264 insertions(+), 27 deletions(-) create mode 100644 tests/unit/stream/loopDetectorTurnEnd.test.js diff --git a/config.yaml b/config.yaml index c905508d..48baaf1c 100644 --- a/config.yaml +++ b/config.yaml @@ -59,7 +59,7 @@ loopDetection: enabled: true escalationCount: 3 escalationWindowMs: 60000 - maxStreams: 1000 + maxStreams: 4 threshold: 2 ttlMs: 60000 windowSize: 20 diff --git a/coverage.txt b/coverage.txt index b526b381..a0fd4fbd 100644 --- a/coverage.txt +++ b/coverage.txt @@ -8,7 +8,7 @@ ℹ compaction.js | 98.84 | 78.43 | 100.00 | 89-90 ℹ contextBackend.js | 100.00 | 100.00 | 100.00 | ℹ coreBackend.js | 100.00 | 100.00 | 100.00 | -ℹ deepAgents.js | 93.72 | 68.00 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 +ℹ deepAgents.js | 93.72 | 69.39 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 ℹ evictionCeiling.js | 100.00 | 100.00 | 100.00 | ℹ skillsMiddleware.js | 100.00 | 87.50 | 100.00 | ℹ config | | | | @@ -95,7 +95,7 @@ ℹ types.js | 100.00 | 100.00 | 100.00 | ℹ validator.js | 100.00 | 100.00 | 100.00 | ℹ stream | | | | -ℹ consume.js | 91.82 | 92.50 | 100.00 | 101-106 118-124 +ℹ consume.js | 96.07 | 95.56 | 100.00 | 137-143 ℹ interrupt.js | 100.00 | 100.00 | 100.00 | ℹ loopCheck.js | 100.00 | 100.00 | 100.00 | ℹ loopDetector.js | 100.00 | 100.00 | 100.00 | @@ -187,9 +187,9 @@ ℹ xlsxParser.js | 82.21 | 83.33 | 88.89 | 47-48 93-112 130-137 150-156 ℹ zipExtractor.js | 82.32 | 77.78 | 66.67 | 62-63 104-107 124-142 149-152 ℹ git | | | | -ℹ index.js | 87.41 | 64.87 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 +ℹ index.js | 87.41 | 64.37 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 ℹ graphql | | | | -ℹ index.js | 97.85 | 98.13 | 93.75 | 57-68 +ℹ index.js | 97.50 | 97.17 | 93.75 | 47-48 57-68 ℹ image | | | | ℹ index.js | 100.00 | 100.00 | 50.00 | ℹ readImage.js | 94.16 | 72.22 | 100.00 | 79-80 88-89 95-96 103-104 @@ -286,6 +286,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.36 | 86.69 | 88.35 | +ℹ all files | 93.38 | 86.70 | 88.36 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/src/config/schemas/loopDetection.js b/src/config/schemas/loopDetection.js index 2ba34a97..e963a16d 100644 --- a/src/config/schemas/loopDetection.js +++ b/src/config/schemas/loopDetection.js @@ -24,13 +24,13 @@ export const LoopDetectionSchema = z // copies of the longest repeating unit: the two-paragraph capture is 190 // words, so 3 copies need 570. wordWindowSize: z.number().int().positive().default(1000), - // Maximum number of streams (a kind plus an id) tracked at once. A long turn - // can produce hundreds to thousands of reasoning and message streams, but only - // the ones still receiving deltas matter; when more than this many are live, - // the least recently updated is evicted. A looping stream is always actively - // receiving deltas, so it is never the one evicted. Each full stream holds - // about 30-40 KB (measured), so 1000 is on the order of 30-40 MB worst case. - maxStreams: z.number().int().positive().default(1000), + // Maximum number of streams (a kind plus an id) tracked at once. Two live at + // the same moment is the realistic edge case; 4 leaves headroom. Streams that + // are done are released, so a long turn with hundreds of sequential messages + // never holds more than this. When more are live, the least recently updated + // is evicted; a looping stream is always receiving deltas, so it is never the + // one evicted. A stream with both windows full holds about 30-40 KB (measured). + maxStreams: z.number().int().positive().default(4), // Accumulated stream text older than this many ms is dropped, so past // content cannot cause false positives as new content streams in. ttlMs: z.number().int().positive().default(60000), diff --git a/src/stream/consume.js b/src/stream/consume.js index 06a8dc47..7ee0fc5c 100644 --- a/src/stream/consume.js +++ b/src/stream/consume.js @@ -20,10 +20,29 @@ import { isSubagentNamespace } from "./namespace.js"; * blocks never mix. Chat Completions `reasoning_content` carries no id. When it * flags a loop the stream is abandoned (closing the underlying iterator) and * `loopDetected` is returned true. Checks run in a worker, so a loop can be - * flagged after the stream ends; `settle()` is awaited to collect it. + * flagged after the stream ends; `settle()` is awaited to collect it. When the + * turn ends, however it ends, the detector is reset so its word and sentence + * lists are cleared and released. * @returns {Promise<{content: string, reasoning: string, interrupted: boolean, lastEventType: string | null, loopDetected: boolean}>} */ export async function consumeStream(stream, streamingCallback, options = {}) { + try { + return await drainStream(stream, streamingCallback, options); + } finally { + // The turn is over (finished, interrupted, abandoned on a loop, or thrown): + // release the detector's lists so nothing from this turn is retained. + options.loopDetector?.reset(); + } +} + +/** + * Drain the agent stream. See `consumeStream`. + * @param {AsyncIterable} stream - The agent stream (subgraphs mode) + * @param {Function} [streamingCallback] - Callback for streamed chunks + * @param {Object} options - Options + * @returns {Promise<{content: string, reasoning: string, interrupted: boolean, lastEventType: string | null, loopDetected: boolean}>} + */ +async function drainStream(stream, streamingCallback, options) { const showToolResults = options.showToolResults; let content = ""; let reasoning = ""; diff --git a/src/stream/loopDetector.js b/src/stream/loopDetector.js index eb51108a..5334ea67 100644 --- a/src/stream/loopDetector.js +++ b/src/stream/loopDetector.js @@ -56,17 +56,17 @@ export async function destroyPool() { * @param {number} [options.windowSize=20] - Recent completed sentences kept and compared * @param {number} [options.wordWindowSize=1000] - Recent completed words kept and compared * @param {number} [options.threshold=2] - Repeats above which a loop is flagged - * @param {number} [options.maxStreams=1000] - Streams tracked at once; the least recently updated is evicted + * @param {number} [options.maxStreams=4] - Streams tracked at once; the least recently updated is evicted * @param {number} [options.ttlMs] - State older than this is dropped (no expiry when unset) * @param {Function} [options.now] - Clock returning ms (injectable for tests) * @param {{ run: Function }} [options.runner] - Runs a check; defaults to the shared pool, injectable for tests - * @returns {{ push: (kind: string, delta: string, id?: string | number) => boolean, settle: () => Promise, reset: () => void }} Detector + * @returns {{ push: (kind: string, delta: string, id?: string | number) => boolean, settle: () => Promise, reset: () => void, size: () => number }} Detector */ export function createSegmentLoopDetector(options = {}) { const windowSize = options.windowSize ?? 20; const wordWindowSize = options.wordWindowSize ?? 1000; const threshold = options.threshold ?? 2; - const maxStreams = options.maxStreams ?? 1000; + const maxStreams = options.maxStreams ?? 4; const ttlMs = options.ttlMs; const now = options.now ?? (() => Date.now()); const runner = options.runner ?? { run: (task) => getPool().run(task) }; @@ -75,8 +75,8 @@ export function createSegmentLoopDetector(options = {}) { * @typedef {Object} KindState * @property {string} pending - Unfinished sentence text * @property {string} partialWord - Unfinished word - * @property {string[]} sentences - Last `windowSize` completed sentences - * @property {string[]} words - Last `wordWindowSize` completed words + * @property {string[] | null} sentences - Last `windowSize` completed sentences (null once released) + * @property {string[] | null} words - Last `wordWindowSize` completed words (null once released) * @property {number} ts - Time of the last delta * @property {boolean} flagged - A check has flagged a loop * @property {Promise | null} inflight - The running check, if any @@ -100,6 +100,23 @@ export function createSegmentLoopDetector(options = {}) { }; } + /** + * Release a stream's state: clear its word and sentence lists and null them so + * the memory is freed even while a closure (an in-flight check) still holds the + * state object. A released state is dead, so any in-flight answer is ignored. + * @param {KindState} state - State to release + * @returns {void} + */ + function release(state) { + state.dead = true; + state.pending = ""; + state.partialWord = ""; + if (state.sentences) state.sentences.length = 0; + if (state.words) state.words.length = 0; + state.sentences = null; + state.words = null; + } + /** * Get (or create) the state for a stream and mark it most recently used. The * least recently used stream is evicted once more than `maxStreams` are tracked. @@ -110,12 +127,12 @@ export function createSegmentLoopDetector(options = {}) { function touch(key, replacement) { const existing = kinds.get(key); const state = replacement ?? existing ?? freshState(); - if (existing && existing !== state) existing.dead = true; + if (existing && existing !== state) release(existing); kinds.delete(key); kinds.set(key, state); if (kinds.size > maxStreams) { const oldest = kinds.keys().next().value; - kinds.get(oldest).dead = true; + release(kinds.get(oldest)); kinds.delete(oldest); } return state; @@ -212,10 +229,18 @@ export function createSegmentLoopDetector(options = {}) { return [...kinds.values()].some((s) => s.flagged); }, - /** Clear all state (call after a nudge so the next attempt starts fresh). */ + /** Release every stream's state (called when a turn ends, and after a nudge). */ reset() { - for (const state of kinds.values()) state.dead = true; + for (const state of kinds.values()) release(state); kinds.clear(); }, + + /** + * Number of streams currently tracked. + * @returns {number} Tracked stream count + */ + size() { + return kinds.size; + }, }; } diff --git a/tests/unit/stream/loopDetectorTurnEnd.test.js b/tests/unit/stream/loopDetectorTurnEnd.test.js new file mode 100644 index 00000000..51f9d4d9 --- /dev/null +++ b/tests/unit/stream/loopDetectorTurnEnd.test.js @@ -0,0 +1,193 @@ +/** + * When a message turn ends, the detector's word and sentence lists are cleared + * and nulled so nothing from the turn is retained, however the turn ends. + */ + +import { describe, it, after } from "node:test"; +import assert from "node:assert"; +import { createSegmentLoopDetector, destroyPool } from "../../../src/stream/loopDetector.js"; +import { checkLoop } from "../../../src/stream/loopCheck.js"; +import { consumeStream } from "../../../src/stream/consume.js"; + +after(async () => { + await destroyPool(); +}); + +const inline = { run: async (task) => checkLoop(task) }; + +/** Runner that captures the task lists it was handed, so tests can see what the detector released. */ +function capturing() { + const tasks = []; + return { + tasks, + run: async (task) => { + tasks.push(task); + return false; + }, + }; +} + +const reasoning = (text, id) => [ + [], + "messages", + [{ type: "ai", text: "", content: [{ type: "reasoning", reasoning: text, id }] }], +]; + +describe("release on reset", () => { + it("empties every stream and reports none tracked", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + d.push("reasoning", "alpha beta gamma. ", "a"); + d.push("message", "delta epsilon. ", 0); + await d.settle(); + assert.strictEqual(d.size(), 2); + d.reset(); + assert.strictEqual(d.size(), 0); + }); + + it("clears and nulls the lists held by an in-flight check's state", async () => { + let release; + const gate = new Promise((r) => { + release = r; + }); + let captured; + const held = { + run: async (task) => { + captured = task; + await gate; + return true; // would flag a loop if the answer were honoured + }, + }; + const d = createSegmentLoopDetector({ runner: held }); + d.push("reasoning", "one two three. ", "a"); + d.reset(); // turn ends while the check is still running + release(); + assert.strictEqual(await d.settle(), false, "a released stream's late answer is ignored"); + assert.ok(captured.words.length > 0, "the task copy the worker held is its own, untouched"); + }); + + it("does not carry state into the next turn", async () => { + const spy = capturing(); + const d = createSegmentLoopDetector({ runner: spy }); + d.push("reasoning", "first turn words here ", "a"); + await d.settle(); + d.reset(); + d.push("reasoning", "second ", "a"); + await d.settle(); + const last = spy.tasks.at(-1); + assert.deepStrictEqual(last.words, ["second"].slice(0, last.words.length)); + assert.ok(!last.words.includes("first"), "no words from the previous turn"); + }); + + it("can be reset more than once and reused", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + d.reset(); + d.reset(); + for (let i = 0; i < 5; i++) d.push("reasoning", "ok ", "a"); + assert.strictEqual(await d.settle(), true); + }); + + it("reset on an empty detector is a no-op", () => { + const d = createSegmentLoopDetector({ runner: inline }); + assert.doesNotThrow(() => d.reset()); + assert.strictEqual(d.size(), 0); + }); +}); + +describe("consumeStream resets the detector when the turn ends", () => { + it("after a normal turn", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + yield reasoning("a fine thought. ", "r1"); + yield reasoning("another one. ", "r2"); + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(d.size(), 0); + }); + + it("after the stream is abandoned on a detected loop", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + for (let i = 0; i < 200; i++) { + await new Promise((r) => setTimeout(r, 1)); + yield reasoning("ok ", "r1"); + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(result.loopDetected, true); + assert.strictEqual(d.size(), 0); + }); + + it("after a clarify interrupt", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + yield reasoning("thinking first. ", "r1"); + yield [ + [], + "values", + { __interrupt__: [{ value: { question: "?", choices: [] }, id: "i1" }] }, + ]; + })(); + const result = await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(result.interrupted, true); + assert.strictEqual(d.size(), 0); + }); + + it("when the stream throws", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + yield reasoning("partial thought. ", "r1"); + throw new Error("network dropped"); + })(); + await assert.rejects( + () => consumeStream(stream, undefined, { loopDetector: d }), + /network dropped/, + ); + assert.strictEqual(d.size(), 0); + }); + + it("still returns the loop verdict even though state is released", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + for (let i = 0; i < 5; i++) yield reasoning("ok ", "r1"); + })(); + const result = await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(result.loopDetected, true); + }); + + it("works when no detector is passed", async () => { + const stream = (async function* () { + yield reasoning("hello. ", "r1"); + })(); + const result = await consumeStream(stream); + assert.strictEqual(result.loopDetected, false); + }); +}); + +describe("maxStreams of 4", () => { + it("holds two concurrent streams, the realistic edge case, without eviction", async () => { + const d = createSegmentLoopDetector({ runner: inline, maxStreams: 4 }); + for (let i = 0; i < 6; i++) { + d.push("reasoning", "ok ", "a"); + d.push("message", `distinct${i} `, 0); + } + assert.strictEqual(await d.settle(), true, "the looping stream was never evicted"); + assert.strictEqual(d.size(), 2); + }); + + it("never tracks more than 4 across hundreds of sequential streams", async () => { + const d = createSegmentLoopDetector({ runner: inline, maxStreams: 4 }); + let peak = 0; + for (let i = 0; i < 500; i++) { + d.push("message", `message number ${i} here. `, i); + peak = Math.max(peak, d.size()); + } + assert.ok(peak <= 4, `peak ${peak}`); + }); + + it("still catches a loop in the stream that is active while old ones age out", async () => { + const d = createSegmentLoopDetector({ runner: inline, maxStreams: 4 }); + for (let i = 0; i < 50; i++) d.push("message", `old message ${i}. `, i); + for (let i = 0; i < 5; i++) d.push("message", "again ", 999); + assert.strictEqual(await d.settle(), true); + }); +}); diff --git a/tests/unit/stream/loopDetectorWindow.test.js b/tests/unit/stream/loopDetectorWindow.test.js index 9d178aa4..c4a7223b 100644 --- a/tests/unit/stream/loopDetectorWindow.test.js +++ b/tests/unit/stream/loopDetectorWindow.test.js @@ -56,7 +56,7 @@ describe("loopDetection config schema", () => { it("keeps the other defaults", () => { const cfg = LoopDetectionSchema.parse({}); - assert.strictEqual(cfg.maxStreams, 1000); + assert.strictEqual(cfg.maxStreams, 4); assert.strictEqual(cfg.windowSize, 20); assert.strictEqual(cfg.threshold, 2); assert.strictEqual(cfg.ttlMs, 60000); @@ -66,8 +66,8 @@ describe("loopDetection config schema", () => { }); describe("loopDetection maxStreams", () => { - it("defaults to 1000", () => { - assert.strictEqual(LoopDetectionSchema.parse({}).maxStreams, 1000); + it("defaults to 4", () => { + assert.strictEqual(LoopDetectionSchema.parse({}).maxStreams, 4); }); it("accepts a configured value", () => { @@ -85,7 +85,7 @@ describe("loopDetection maxStreams", () => { fileURLToPath(new URL("../../../config.yaml", import.meta.url)), "utf8", ); - assert.match(yaml, /^\s+maxStreams: 1000$/m); + assert.match(yaml, /^\s+maxStreams: 4$/m); }); }); From 3622662c2d6b8d0f504ade620200d1a93c118bfb Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 21:01:15 -0500 Subject: [PATCH 18/20] fix: release detector state at the end of each message turn A message turn is start, stream, end, so the detector's word and sentence lists are cleared and nulled when each message ends, not only when the whole agent run does. Without this, a run with hundreds of messages kept state for the last few until they were evicted. consumeStream now ends a message when its last chunk carries a finish_reason, when a different message id arrives, or when a tool result follows the AI message. end() reads the verdict of any check still running and then releases, so a loop in the final deltas is not lost. Subagent events are ignored for this. The stream-end reset stays as a backstop. Also stops the same short answer repeated across separate messages from being counted as one loop. --- coverage.txt | 8 +- src/stream/consume.js | 27 ++- src/stream/loopDetector.js | 17 +- .../stream/loopDetectorMessageEnd.test.js | 196 ++++++++++++++++++ 4 files changed, 239 insertions(+), 9 deletions(-) create mode 100644 tests/unit/stream/loopDetectorMessageEnd.test.js diff --git a/coverage.txt b/coverage.txt index a0fd4fbd..2eea2ea6 100644 --- a/coverage.txt +++ b/coverage.txt @@ -61,7 +61,7 @@ ℹ copilotRetryMiddleware.js | 88.52 | 79.49 | 83.33 | 75-88 ℹ index.js | 100.00 | 100.00 | 100.00 | ℹ modelInfo.js | 97.25 | 93.98 | 100.00 | 128-129 131-132 179-180 -ℹ openai.js | 92.72 | 75.71 | 78.57 | 65 90-92 138-139 145-146 233 236-238 257-259 267-271 279-280 292-295 +ℹ openai.js | 91.60 | 74.65 | 78.57 | 65 90-92 138-139 145-146 181-184 233 236-238 257-259 267-271 279-280 292-295 ℹ summarizationMiddleware.js | 97.01 | 94.00 | 83.33 | 107 126-128 149 ℹ tokenBudget.js | 97.14 | 85.71 | 82.35 | 50-51 83-84 ℹ tokenBudgetMiddleware.js | 98.10 | 78.67 | 87.50 | 94-95 97-98 @@ -95,7 +95,7 @@ ℹ types.js | 100.00 | 100.00 | 100.00 | ℹ validator.js | 100.00 | 100.00 | 100.00 | ℹ stream | | | | -ℹ consume.js | 96.07 | 95.56 | 100.00 | 137-143 +ℹ consume.js | 96.48 | 98.36 | 100.00 | 156-162 ℹ interrupt.js | 100.00 | 100.00 | 100.00 | ℹ loopCheck.js | 100.00 | 100.00 | 100.00 | ℹ loopDetector.js | 100.00 | 100.00 | 100.00 | @@ -189,7 +189,7 @@ ℹ git | | | | ℹ index.js | 87.41 | 64.37 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 ℹ graphql | | | | -ℹ index.js | 97.50 | 97.17 | 93.75 | 47-48 57-68 +ℹ index.js | 97.85 | 98.13 | 93.75 | 57-68 ℹ image | | | | ℹ index.js | 100.00 | 100.00 | 50.00 | ℹ readImage.js | 94.16 | 72.22 | 100.00 | 79-80 88-89 95-96 103-104 @@ -286,6 +286,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.38 | 86.70 | 88.36 | +ℹ all files | 93.38 | 86.74 | 88.37 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/src/stream/consume.js b/src/stream/consume.js index 7ee0fc5c..e6641599 100644 --- a/src/stream/consume.js +++ b/src/stream/consume.js @@ -20,9 +20,11 @@ import { isSubagentNamespace } from "./namespace.js"; * blocks never mix. Chat Completions `reasoning_content` carries no id. When it * flags a loop the stream is abandoned (closing the underlying iterator) and * `loopDetected` is returned true. Checks run in a worker, so a loop can be - * flagged after the stream ends; `settle()` is awaited to collect it. When the - * turn ends, however it ends, the detector is reset so its word and sentence - * lists are cleared and released. + * flagged after the stream ends; `settle()` is awaited to collect it. When a + * message turn ends (its last chunk carries a `finish_reason`, a different message + * id arrives, or a tool result follows) `end()` collects the verdict and releases + * the detector's word and sentence lists. The detector is also reset when the + * stream itself ends, however it ends. * @returns {Promise<{content: string, reasoning: string, interrupted: boolean, lastEventType: string | null, loopDetected: boolean}>} */ export async function consumeStream(stream, streamingCallback, options = {}) { @@ -50,6 +52,15 @@ async function drainStream(stream, streamingCallback, options) { let lastEventType = null; let loopDetected = false; const loopDetector = options.loopDetector; + /** The id shared by the chunks of the message currently streaming. */ + let currentMsgId = null; + + // A message turn is start, stream, end. When it ends, collect the verdict of + // any worker check still running and release the detector's lists. + const endMessage = async () => { + currentMsgId = null; + if (loopDetector?.end && (await loopDetector.end())) loopDetected = true; + }; for await (const [namespace, mode, payload] of stream) { // Subagent runs are nested under a `tools:` namespace segment. Their @@ -73,6 +84,14 @@ async function drainStream(stream, streamingCallback, options) { // never rendered into the assistant's streaming bubble. if (msgType === "human") continue; + if (!isSubagent) { + // A different message id means the previous message's turn has ended. + if (msg?.id != null && currentMsgId != null && msg.id !== currentMsgId) await endMessage(); + if (msg?.id != null) currentMsgId = msg.id; + // A tool result means the AI message that called the tool has ended. + if (msgType === "tool") await endMessage(); + } + if (text) { // ToolMessage text is emitted as a distinct `tool` event, never // folded into the message segment. @@ -145,6 +164,8 @@ async function drainStream(stream, streamingCallback, options) { } } } + // The chunk that carries a finish reason is the last of its message. + if (!isSubagent && msg?.response_metadata?.finish_reason) await endMessage(); } else if (mode === "tools" && streamingCallback) { if (payload?.event === "on_tool_start") { streamingCallback({ diff --git a/src/stream/loopDetector.js b/src/stream/loopDetector.js index 5334ea67..36a3ffd5 100644 --- a/src/stream/loopDetector.js +++ b/src/stream/loopDetector.js @@ -60,7 +60,7 @@ export async function destroyPool() { * @param {number} [options.ttlMs] - State older than this is dropped (no expiry when unset) * @param {Function} [options.now] - Clock returning ms (injectable for tests) * @param {{ run: Function }} [options.runner] - Runs a check; defaults to the shared pool, injectable for tests - * @returns {{ push: (kind: string, delta: string, id?: string | number) => boolean, settle: () => Promise, reset: () => void, size: () => number }} Detector + * @returns {{ push: (kind: string, delta: string, id?: string | number) => boolean, settle: () => Promise, end: () => Promise, reset: () => void, size: () => number }} Detector */ export function createSegmentLoopDetector(options = {}) { const windowSize = options.windowSize ?? 20; @@ -229,12 +229,25 @@ export function createSegmentLoopDetector(options = {}) { return [...kinds.values()].some((s) => s.flagged); }, - /** Release every stream's state (called when a turn ends, and after a nudge). */ + /** Release every stream's state (called when a message turn ends, and after a nudge). */ reset() { for (const state of kinds.values()) release(state); kinds.clear(); }, + /** + * End a message turn: collect the verdict of any check still running (the + * last deltas may have started one), then release every stream's lists. The + * verdict is read before the release, so a loop in the final deltas of the + * message is not lost. + * @returns {Promise} True when a loop was flagged during the message + */ + async end() { + const flagged = await this.settle(); + this.reset(); + return flagged; + }, + /** * Number of streams currently tracked. * @returns {number} Tracked stream count diff --git a/tests/unit/stream/loopDetectorMessageEnd.test.js b/tests/unit/stream/loopDetectorMessageEnd.test.js new file mode 100644 index 00000000..3940eab6 --- /dev/null +++ b/tests/unit/stream/loopDetectorMessageEnd.test.js @@ -0,0 +1,196 @@ +/** + * A message turn is: start, stream, end. When it ends the detector's word and + * sentence lists are cleared and nulled, so a long run of hundreds of messages + * never accumulates state. The end is the chunk carrying a `finish_reason`, a + * different message id arriving, or a tool result following the AI message. + */ + +import { describe, it, after } from "node:test"; +import assert from "node:assert"; +import { AIMessageChunk, ToolMessage } from "@langchain/core/messages"; +import { createSegmentLoopDetector, destroyPool } from "../../../src/stream/loopDetector.js"; +import { checkLoop } from "../../../src/stream/loopCheck.js"; +import { consumeStream } from "../../../src/stream/consume.js"; + +after(async () => { + await destroyPool(); +}); + +const inline = { run: async (task) => checkLoop(task) }; +const ev = (msg) => [[], "messages", [msg, {}]]; +const think = (id, text, extra = {}) => + new AIMessageChunk({ id, content: "", additional_kwargs: { reasoning_content: text }, ...extra }); +const say = (id, text, extra = {}) => new AIMessageChunk({ id, content: text, ...extra }); + +/** A detector that records how many streams it held each time something was pushed. */ +function watched() { + const d = createSegmentLoopDetector({ runner: inline }); + const sizes = []; + const push = d.push.bind(d); + d.push = (...args) => { + const out = push(...args); + sizes.push(d.size()); + return out; + }; + const resets = []; + const reset = d.reset.bind(d); + d.reset = () => { + resets.push(d.size()); + return reset(); + }; + return { d, sizes, resets }; +} + +describe("end of a message turn releases the lists", () => { + it("releases when the last chunk carries a finish_reason", async () => { + const { d, resets } = watched(); + const stream = (async function* () { + yield ev(think("A", "plan it. ")); + yield ev(say("A", "done", { response_metadata: { finish_reason: "stop" } })); + yield ev(think("B", "next message. ")); + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + assert.ok( + resets.length >= 2, + `released at the message end and at the stream end (${resets.length})`, + ); + assert.ok(resets[0] > 0, "the first release happened while the message's state existed"); + }); + + it("releases when a different message id arrives", async () => { + const { d, sizes } = watched(); + const stream = (async function* () { + yield ev(think("A", "first thought. ")); + yield ev(think("B", "second thought. ")); // new id, no finish_reason on A + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + // After B's delta the detector holds only B, because A was released at the id change. + assert.strictEqual(sizes.at(-1), 1); + }); + + it("releases when a tool result follows the AI message", async () => { + const { d, resets } = watched(); + const stream = (async function* () { + yield ev(think("A", "calling a tool. ")); + yield ev(new ToolMessage({ content: "result", tool_call_id: "t1" })); + yield ev(say("B", "answer")); + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + assert.ok(resets.length >= 2); + }); + + it("never accumulates across hundreds of sequential messages", async () => { + const { d, sizes } = watched(); + const stream = (async function* () { + for (let i = 0; i < 400; i++) { + yield ev(think(`m${i}`, `reasoning for message ${i}. `)); + yield ev(say(`m${i}`, `answer ${i}`, { response_metadata: { finish_reason: "stop" } })); + } + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + assert.ok(Math.max(...sizes) <= 2, `peak tracked ${Math.max(...sizes)}`); + }); + + it("does not carry one message's words into the next", async () => { + const tasks = []; + const spy = { + run: async (task) => { + tasks.push({ words: [...task.words] }); + return false; + }, + }; + const d = createSegmentLoopDetector({ runner: spy }); + const stream = (async function* () { + yield ev(think("A", "alpha beta gamma delta ")); + yield ev(say("A", "x", { response_metadata: { finish_reason: "stop" } })); + yield ev(think("B", "epsilon zeta ")); + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + const sawBoth = tasks.some((t) => t.words.includes("alpha") && t.words.includes("epsilon")); + assert.strictEqual(sawBoth, false, "no check mixed message A and message B"); + }); +}); + +describe("a loop is still caught across message boundaries", () => { + it("flags a loop inside one message", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + for (let i = 0; i < 200; i++) { + await new Promise((r) => setTimeout(r, 1)); + yield ev(think("A", "ok ")); + } + })(); + assert.strictEqual( + (await consumeStream(stream, undefined, { loopDetector: d })).loopDetected, + true, + ); + }); + + it("keeps the verdict of a loop in the last deltas of a message that ends right after", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + yield ev(think("A", "ok ")); + yield ev(think("A", "ok ")); + yield ev(think("A", "ok ", { response_metadata: { finish_reason: "stop" } })); + })(); + const result = await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(result.loopDetected, true, "end() read the verdict before releasing"); + }); + + it("does not treat the same short answer repeated across separate messages as one loop", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + for (let i = 0; i < 6; i++) { + yield ev(say(`m${i}`, "ok ", { response_metadata: { finish_reason: "stop" } })); + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(result.loopDetected, false); + }); + + it("ignores subagent messages when deciding where a message ends", async () => { + const { d, resets } = watched(); + const stream = (async function* () { + yield ev(think("A", "main thought. ")); + yield [ + ["tools:abc"], + "messages", + [say("S", "sub", { response_metadata: { finish_reason: "stop" } }), {}], + ]; + yield ev(think("A", "still the same message. ")); + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(resets.length, 1, "only the final stream-end release"); + }); + + it("still resets once at the end of the stream", async () => { + const { d, resets } = watched(); + const stream = (async function* () { + yield ev(think("A", "only message. ")); + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(resets.length, 1); + assert.strictEqual(d.size(), 0); + }); +}); + +describe("end()", () => { + it("returns the verdict and releases", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + for (let i = 0; i < 5; i++) d.push("reasoning", "ok ", "a"); + assert.strictEqual(await d.end(), true); + assert.strictEqual(d.size(), 0); + }); + + it("returns false and releases when nothing looped", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + d.push("reasoning", "a fine thought. ", "a"); + assert.strictEqual(await d.end(), false); + assert.strictEqual(d.size(), 0); + }); + + it("can be called on an empty detector", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + assert.strictEqual(await d.end(), false); + }); +}); From 083e5b578b4f49a2d4046c3838ebd5e2a6915ff5 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 21:07:32 -0500 Subject: [PATCH 19/20] fix: compact on the third loop strike, not after three nudges Three strikes and you're out. Each detected loop is a strike; strikes within escalationWindowMs are counted including the current one, and the escalationCount-th strike compacts the context instead of nudging. So strikes 1 and 2 nudge and strike 3 compacts. The previous code waited for three nudges to be on record, so it compacted on the fourth loop. Update the decideLoopAction tests to play out strikes 1 to 3 and to cover window pruning, the reset after compaction and other escalationCount values, and correct the schema and component comments. --- coverage.txt | 6 +-- src/config/schemas/loopDetection.js | 6 ++- src/tui/conversationArea.js | 26 +++++----- tests/unit/tui/conversationArea.test.js | 69 +++++++++++++++++++++---- 4 files changed, 79 insertions(+), 28 deletions(-) diff --git a/coverage.txt b/coverage.txt index 2eea2ea6..4437a48a 100644 --- a/coverage.txt +++ b/coverage.txt @@ -61,7 +61,7 @@ ℹ copilotRetryMiddleware.js | 88.52 | 79.49 | 83.33 | 75-88 ℹ index.js | 100.00 | 100.00 | 100.00 | ℹ modelInfo.js | 97.25 | 93.98 | 100.00 | 128-129 131-132 179-180 -ℹ openai.js | 91.60 | 74.65 | 78.57 | 65 90-92 138-139 145-146 181-184 233 236-238 257-259 267-271 279-280 292-295 +ℹ openai.js | 92.72 | 75.71 | 78.57 | 65 90-92 138-139 145-146 233 236-238 257-259 267-271 279-280 292-295 ℹ summarizationMiddleware.js | 97.01 | 94.00 | 83.33 | 107 126-128 149 ℹ tokenBudget.js | 97.14 | 85.71 | 82.35 | 50-51 83-84 ℹ tokenBudgetMiddleware.js | 98.10 | 78.67 | 87.50 | 94-95 97-98 @@ -262,7 +262,7 @@ ℹ openUrl.js | 100.00 | 100.00 | 100.00 | ℹ panels.js | 100.00 | 100.00 | 100.00 | ℹ projectsPanel.js | 26.83 | 100.00 | 80.00 | 34-123 -ℹ quotes.js | 100.00 | 100.00 | 100.00 | +ℹ quotes.js | 96.97 | 87.50 | 100.00 | 62-63 ℹ sanitize.js | 100.00 | 100.00 | 100.00 | ℹ scrollView.js | 98.83 | 97.73 | 73.91 | 197-199 ℹ selectComponents.js | 100.00 | 100.00 | 100.00 | @@ -286,6 +286,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.38 | 86.74 | 88.37 | +ℹ all files | 93.39 | 86.74 | 88.37 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/src/config/schemas/loopDetection.js b/src/config/schemas/loopDetection.js index e963a16d..8078a7a6 100644 --- a/src/config/schemas/loopDetection.js +++ b/src/config/schemas/loopDetection.js @@ -36,9 +36,11 @@ export const LoopDetectionSchema = z ttlMs: z.number().int().positive().default(60000), // A sentence (or word run) must repeat more than this many times to flag a loop. threshold: z.number().int().positive().default(2), - // Number of loop nudges within the escalation window before compacting. + // Strike, within the escalation window, that compacts the context instead of + // nudging: three strikes and you're out, so strikes 1 and 2 nudge and strike 3 + // compacts. escalationCount: z.number().int().positive().default(3), - // Rolling window (ms) for counting loop nudges before escalation. + // Rolling window (ms) for counting loop strikes before escalation. escalationWindowMs: z.number().int().positive().default(60000), }) .strict() diff --git a/src/tui/conversationArea.js b/src/tui/conversationArea.js index f781eff9..9f324651 100644 --- a/src/tui/conversationArea.js +++ b/src/tui/conversationArea.js @@ -74,19 +74,21 @@ export function shouldAutoContinue({ * * Mirrors `shouldAutoContinue`: it acts only when the streaming bubble is the * last in the message list AND the stream was abandoned because the loop - * detector flagged it. If `escalationCount` nudges already happened within - * `escalationWindowMs`, the context is compacted instead of nudging again. + * detector flagged it. Each detected loop is a strike. Strikes within + * `escalationWindowMs` are counted, this one included, and the `escalationCount`th + * strike compacts the context instead of nudging (three strikes and you're out: + * strikes 1 and 2 nudge, strike 3 compacts). * * @param {Object} params - Decision inputs * @param {boolean | undefined} params.loopDetected - Whether the loop detector flagged the stream * @param {string | undefined} params.streamingMsgId - The streaming bubble's message ID * @param {string | undefined} params.lastMessageId - The last message ID in the list - * @param {number[]} params.nudges - Timestamps (ms) of previous loop nudges + * @param {number[]} params.nudges - Timestamps (ms) of previous strikes that were nudged * @param {number} params.now - Current time in ms - * @param {number} params.escalationCount - Nudges within the window before compacting - * @param {number} params.escalationWindowMs - Rolling window (ms) for counting nudges + * @param {number} params.escalationCount - The strike, within the window, that compacts + * @param {number} params.escalationWindowMs - Rolling window (ms) for counting strikes * @returns {{ action: "none" | "nudge" | "compact", nudges: number[] }} The action and the - * updated nudge timestamps (pruned to the window) + * updated strike timestamps (pruned to the window) */ export function decideLoopAction({ loopDetected, @@ -101,7 +103,8 @@ export function decideLoopAction({ return { action: "none", nudges }; } const recent = nudges.filter((ts) => now - ts <= escalationWindowMs); - if (recent.length >= escalationCount) return { action: "compact", nudges: [] }; + // This loop is itself a strike, so it is counted before deciding. + if (recent.length + 1 >= escalationCount) return { action: "compact", nudges: [] }; return { action: "nudge", nudges: [...recent, now] }; } @@ -163,9 +166,8 @@ const ConversationArea = forwardRef(function ConversationArea( // Bounded by config.agent.autoContinueLimit to prevent an infinite loop. const autoContinueCountRef = useRef(0); const autoContinueLimit = config?.agent?.autoContinueLimit; - // Timestamps of silent loop nudges dispatched when the loop detector abandons - // a stream. `escalationCount` nudges within `escalationWindowMs` compact the - // context instead of nudging again. + // Timestamps of loop strikes that were nudged. The `escalationCount`th strike + // within `escalationWindowMs` compacts the context instead of nudging. const loopNudgesRef = useRef([]); const escalationCount = config?.loopDetection?.escalationCount; const escalationWindowMs = config?.loopDetection?.escalationWindowMs; @@ -539,8 +541,8 @@ const ConversationArea = forwardRef(function ConversationArea( // Loop nudge: when the loop detector abandoned the stream, nudge the model // with a silent "You're looping", the same way the reasoning-only // "Continue" below is dispatched. Scoped to the LAST bubble. After - // `escalationCount` nudges within `escalationWindowMs`, compact the - // context instead of nudging again. + // `escalationCount`th strike within `escalationWindowMs`, compact the + // context instead of nudging. const loop = decideLoopAction({ loopDetected: response?.loopDetected, streamingMsgId: streamingMsgIdRef.current, diff --git a/tests/unit/tui/conversationArea.test.js b/tests/unit/tui/conversationArea.test.js index a96e7370..1554952a 100644 --- a/tests/unit/tui/conversationArea.test.js +++ b/tests/unit/tui/conversationArea.test.js @@ -68,34 +68,81 @@ describe("decideLoopAction", () => { assert.strictEqual(mod.decideLoopAction({ ...base, lastMessageId: "4" }).action, "none"); }); - it("records each nudge and keeps nudging below the escalation count", async () => { + it("strike 1 nudges", async () => { const mod = await import("../../../src/tui/conversationArea.js"); - const out = mod.decideLoopAction({ ...base, nudges: [90000, 95000] }); + const out = mod.decideLoopAction({ ...base, nudges: [] }); + assert.strictEqual(out.action, "nudge"); + assert.deepStrictEqual(out.nudges, [100000]); + }); + + it("strike 2 nudges and records both strikes", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + const out = mod.decideLoopAction({ ...base, nudges: [95000] }); assert.strictEqual(out.action, "nudge"); - assert.deepStrictEqual(out.nudges, [90000, 95000, 100000]); + assert.deepStrictEqual(out.nudges, [95000, 100000]); }); - it("compacts instead of nudging once escalationCount nudges fall within the window", async () => { + it("strike 3 compacts instead of nudging: three strikes and you're out", async () => { const mod = await import("../../../src/tui/conversationArea.js"); - const out = mod.decideLoopAction({ ...base, nudges: [80000, 90000, 95000] }); + const out = mod.decideLoopAction({ ...base, nudges: [90000, 95000] }); assert.strictEqual(out.action, "compact"); - assert.deepStrictEqual(out.nudges, [], "counter resets after escalation"); + assert.deepStrictEqual(out.nudges, [], "the counter resets after compaction"); + }); + + it("plays out strikes 1 to 3 in order", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + let nudges = []; + const actions = []; + for (const now of [100000, 110000, 120000]) { + const out = mod.decideLoopAction({ ...base, nudges, now }); + actions.push(out.action); + nudges = out.nudges; + } + assert.deepStrictEqual(actions, ["nudge", "nudge", "compact"]); + }); + + it("starts counting again after a compaction", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + const afterCompact = mod.decideLoopAction({ ...base, nudges: [90000, 95000] }).nudges; + const next = mod.decideLoopAction({ ...base, nudges: afterCompact, now: 130000 }); + assert.strictEqual(next.action, "nudge", "the next loop is strike 1 again"); + }); + + it("drops strikes older than the window so a slow loop does not escalate", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + // Two earlier strikes are outside the 60s window, so this is strike 1 again. + const out = mod.decideLoopAction({ ...base, nudges: [10000, 20000] }); + assert.strictEqual(out.action, "nudge"); + assert.deepStrictEqual(out.nudges, [100000]); }); - it("drops nudges older than the window so a slow loop does not escalate", async () => { + it("counts only the strikes inside the window", async () => { const mod = await import("../../../src/tui/conversationArea.js"); - // Two of the three are outside the 60s window. - const out = mod.decideLoopAction({ ...base, nudges: [10000, 20000, 95000] }); + // One old strike, one recent: this loop is only strike 2. + const out = mod.decideLoopAction({ ...base, nudges: [10000, 95000] }); assert.strictEqual(out.action, "nudge"); assert.deepStrictEqual(out.nudges, [95000, 100000]); }); - it("treats a nudge exactly at the window edge as still inside it", async () => { + it("treats a strike exactly at the window edge as still inside it", async () => { const mod = await import("../../../src/tui/conversationArea.js"); - const out = mod.decideLoopAction({ ...base, nudges: [40000, 41000, 42000] }); + const out = mod.decideLoopAction({ ...base, nudges: [40000, 41000] }); assert.strictEqual(out.action, "compact"); }); + it("honours a different escalationCount", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual(mod.decideLoopAction({ ...base, escalationCount: 1 }).action, "compact"); + assert.strictEqual( + mod.decideLoopAction({ ...base, escalationCount: 2, nudges: [95000] }).action, + "compact", + ); + assert.strictEqual( + mod.decideLoopAction({ ...base, escalationCount: 5, nudges: [91000, 92000, 93000] }).action, + "nudge", + ); + }); + it("leaves the nudge timestamps untouched when it does nothing", async () => { const mod = await import("../../../src/tui/conversationArea.js"); const nudges = [90000]; From 6f50ec31a909239a3ac759045da2efdb659a60c0 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 17 Nov 2026 21:14:48 -0500 Subject: [PATCH 20/20] feat: send a silent Continue after a loop-triggered compaction When the third loop strike compacts the context, resume the model with a silent "Continue" so it does not sit idle until the user types. handleCompact now returns whether it succeeded, and shouldContinueAfterCompact gates the Continue on that and on the shared autoContinueLimit, so a failed compaction (the looping context is still in place) does not continue and a model that keeps looping cannot cycle through nudge, compact and continue forever. The /compact command ignores the return value. --- coverage.txt | 6 +-- src/tui/conversationArea.js | 43 +++++++++++++++++++--- tests/unit/tui/conversationArea.test.js | 49 +++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 8 deletions(-) diff --git a/coverage.txt b/coverage.txt index 4437a48a..7191149e 100644 --- a/coverage.txt +++ b/coverage.txt @@ -8,7 +8,7 @@ ℹ compaction.js | 98.84 | 78.43 | 100.00 | 89-90 ℹ contextBackend.js | 100.00 | 100.00 | 100.00 | ℹ coreBackend.js | 100.00 | 100.00 | 100.00 | -ℹ deepAgents.js | 93.72 | 69.39 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 +ℹ deepAgents.js | 93.72 | 68.00 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 ℹ evictionCeiling.js | 100.00 | 100.00 | 100.00 | ℹ skillsMiddleware.js | 100.00 | 87.50 | 100.00 | ℹ config | | | | @@ -262,7 +262,7 @@ ℹ openUrl.js | 100.00 | 100.00 | 100.00 | ℹ panels.js | 100.00 | 100.00 | 100.00 | ℹ projectsPanel.js | 26.83 | 100.00 | 80.00 | 34-123 -ℹ quotes.js | 96.97 | 87.50 | 100.00 | 62-63 +ℹ quotes.js | 100.00 | 100.00 | 100.00 | ℹ sanitize.js | 100.00 | 100.00 | 100.00 | ℹ scrollView.js | 98.83 | 97.73 | 73.91 | 197-199 ℹ selectComponents.js | 100.00 | 100.00 | 100.00 | @@ -286,6 +286,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.39 | 86.74 | 88.37 | +ℹ all files | 93.39 | 86.73 | 88.37 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/src/tui/conversationArea.js b/src/tui/conversationArea.js index 9f324651..a5d22d12 100644 --- a/src/tui/conversationArea.js +++ b/src/tui/conversationArea.js @@ -69,6 +69,24 @@ export function shouldAutoContinue({ ); } +/** + * Decide whether to send a silent "Continue" after a loop-triggered compaction. + * + * The model is resumed only when the compaction succeeded (a failed one leaves + * the looping context in place, so continuing would loop again) and the shared + * auto-continue guard is not exhausted, so a model that keeps looping cannot + * cycle through nudge, compact, continue forever. + * + * @param {Object} params - Decision inputs + * @param {boolean} params.compacted - Whether the compaction succeeded + * @param {number} params.autoContinueCount - Current auto-continue count + * @param {number} params.autoContinueLimit - Max auto-continue count + * @returns {boolean} True when a silent "Continue" should be dispatched + */ +export function shouldContinueAfterCompact({ compacted, autoContinueCount, autoContinueLimit }) { + return compacted === true && autoContinueCount < autoContinueLimit; +} + /** * Decide what to do after a turn when the loop detector may have abandoned the stream. * @@ -303,11 +321,12 @@ const ConversationArea = forwardRef(function ConversationArea( * Compact the context window on demand. * Invokes the agent's compaction routine, syncs the TUI conversation view, * recomputes the context size. + * @returns {Promise} True when the context was compacted */ const handleCompact = async () => { if (!compactContext) { addMessage({ role: "system", content: "Compaction is not available." }); - return; + return false; } const compactMsgId = addMessage({ role: "system", content: "Compacting context window..." }); try { @@ -325,13 +344,15 @@ const ConversationArea = forwardRef(function ConversationArea( messageListRef.current?.updateMessage(compactMsgId, { content: msg }); // Recompute the context size so the status bar reflects the reduced window. await updateContextSize(sessionState, config); - } else { - const msg = `Compaction failed: ${result?.error || "unknown error"}`; - messageListRef.current?.updateMessage(compactMsgId, { content: msg }); + return true; } + const msg = `Compaction failed: ${result?.error || "unknown error"}`; + messageListRef.current?.updateMessage(compactMsgId, { content: msg }); + return false; } catch (err) { const msg = `Compaction failed: ${err.message}`; messageListRef.current?.updateMessage(compactMsgId, { content: msg }); + return false; } }; @@ -554,7 +575,19 @@ const ConversationArea = forwardRef(function ConversationArea( }); loopNudgesRef.current = loop.nudges; if (loop.action === "compact") { - await handleCompact(); + const compacted = await handleCompact(); + // After a successful compaction, resume the model with a silent + // "Continue", counted against the same autoContinueLimit guard. + if ( + shouldContinueAfterCompact({ + compacted, + autoContinueCount: autoContinueCountRef.current, + autoContinueLimit, + }) + ) { + autoContinueCountRef.current += 1; + await handleChat("Continue", { silentUser: true }); + } return; } if (loop.action === "nudge") { diff --git a/tests/unit/tui/conversationArea.test.js b/tests/unit/tui/conversationArea.test.js index 1554952a..1e02a9be 100644 --- a/tests/unit/tui/conversationArea.test.js +++ b/tests/unit/tui/conversationArea.test.js @@ -41,6 +41,55 @@ describe("countToolMessageTokens", () => { }); }); +describe("shouldContinueAfterCompact", () => { + it("continues after a successful compaction", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual( + mod.shouldContinueAfterCompact({ + compacted: true, + autoContinueCount: 0, + autoContinueLimit: 1000, + }), + true, + ); + }); + + it("does not continue after a failed compaction", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + for (const compacted of [false, undefined, null]) { + assert.strictEqual( + mod.shouldContinueAfterCompact({ + compacted, + autoContinueCount: 0, + autoContinueLimit: 1000, + }), + false, + `compacted ${compacted}`, + ); + } + }); + + it("stops once the shared auto-continue limit is reached", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual( + mod.shouldContinueAfterCompact({ + compacted: true, + autoContinueCount: 999, + autoContinueLimit: 1000, + }), + true, + ); + assert.strictEqual( + mod.shouldContinueAfterCompact({ + compacted: true, + autoContinueCount: 1000, + autoContinueLimit: 1000, + }), + false, + ); + }); +}); + describe("decideLoopAction", () => { const base = { loopDetected: true,