diff --git a/config.yaml b/config.yaml index 6ba0f4638..8ea67273a 100644 --- a/config.yaml +++ b/config.yaml @@ -59,9 +59,11 @@ loopDetection: enabled: true escalationCount: 3 escalationWindowMs: 60000 + maxStreams: 4 threshold: 2 ttlMs: 60000 - windowSize: 10 + windowSize: 20 + wordWindowSize: 1000 memory: checkpointsDir: memory/checkpoints/ contextDir: memory/context/ diff --git a/coverage.txt b/coverage.txt index 50293f88c..d97fe82a5 100644 --- a/coverage.txt +++ b/coverage.txt @@ -8,7 +8,7 @@ ℹ compaction.js | 98.84 | 78.43 | 100.00 | 89-90 ℹ contextBackend.js | 100.00 | 100.00 | 100.00 | ℹ coreBackend.js | 100.00 | 100.00 | 100.00 | -ℹ deepAgents.js | 94.93 | 67.65 | 60.87 | 157-158 182 198-200 310-317 346-347 352-353 460-462 512 548-550 597-598 608-610 617-618 +ℹ deepAgents.js | 93.72 | 69.39 | 66.67 | 156-157 181 197-199 309-316 345-346 459-461 488-500 511 579-580 590-592 ℹ evictionCeiling.js | 100.00 | 100.00 | 100.00 | ℹ skillsMiddleware.js | 100.00 | 87.50 | 100.00 | ℹ config | | | | @@ -60,9 +60,8 @@ ℹ copilotAuth.js | 96.80 | 78.87 | 90.00 | 12-13 131-132 196-197 237-239 352-353 378-379 391-393 484 565-566 575-576 ℹ copilotRetryMiddleware.js | 88.52 | 79.49 | 83.33 | 75-88 ℹ index.js | 100.00 | 100.00 | 100.00 | -ℹ loopDetectionMiddleware.js | 99.59 | 83.08 | 100.00 | 32 -ℹ modelInfo.js | 98.17 | 96.39 | 100.00 | 128-129 179-180 -ℹ openai.js | 91.60 | 73.61 | 78.57 | 65 90-92 138-139 145-146 181-184 233 236-238 257-259 267-271 279-280 292-295 +ℹ modelInfo.js | 97.25 | 93.98 | 100.00 | 128-129 131-132 179-180 +ℹ openai.js | 92.72 | 75.71 | 78.57 | 65 90-92 138-139 145-146 233 236-238 257-259 267-271 279-280 292-295 ℹ summarizationMiddleware.js | 97.01 | 94.00 | 83.33 | 107 126-128 149 ℹ tokenBudget.js | 97.14 | 85.71 | 82.35 | 50-51 83-84 ℹ tokenBudgetMiddleware.js | 98.10 | 78.67 | 87.50 | 94-95 97-98 @@ -96,8 +95,11 @@ ℹ types.js | 100.00 | 100.00 | 100.00 | ℹ validator.js | 100.00 | 100.00 | 100.00 | ℹ stream | | | | -ℹ consume.js | 85.51 | 74.29 | 100.00 | 52-58 88-93 104-110 +ℹ consume.js | 96.48 | 98.36 | 100.00 | 156-162 ℹ interrupt.js | 100.00 | 100.00 | 100.00 | +ℹ loopCheck.js | 100.00 | 100.00 | 100.00 | +ℹ loopDetector.js | 100.00 | 100.00 | 100.00 | +ℹ loopDetectorWorker.js | 100.00 | 100.00 | 100.00 | ℹ namespace.js | 100.00 | 100.00 | 100.00 | ℹ transformers | | | | ℹ index.js | 100.00 | 100.00 | 100.00 | @@ -185,7 +187,7 @@ ℹ xlsxParser.js | 82.21 | 83.33 | 88.89 | 47-48 93-112 130-137 150-156 ℹ zipExtractor.js | 82.32 | 77.78 | 66.67 | 62-63 104-107 124-142 149-152 ℹ git | | | | -ℹ index.js | 87.41 | 64.13 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 +ℹ index.js | 87.41 | 64.37 | 83.69 | 324-326 379-380 393-396 481-483 515-516 626-643 689-691 722-723 755-757 821-824 840-843 866-872 906 908-912 914-917 929-933 976-980 984 989-991 1006-1015 1024 1038-1044 1046-1049 1057-1063 1086-1091 1093-1099 1101-1110 1112-1115 1122-1126 1136-1143 1191-1192 1194-1199 1291-1292 1314-1315 1323-1324 1331-1332 1334-1340 1362 1366-1368 1373-1376 1428-1435 1439-1447 1450-1453 1455-1458 ℹ graphql | | | | ℹ index.js | 97.85 | 98.13 | 93.75 | 57-68 ℹ image | | | | @@ -284,6 +286,6 @@ ℹ loadAgents.js | 74.29 | 78.57 | 50.00 | 25-26 41 56-70 ℹ project.js | 100.00 | 100.00 | 100.00 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -ℹ all files | 93.34 | 86.41 | 88.20 | +ℹ all files | 93.39 | 86.74 | 88.37 | ℹ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ℹ end of coverage report diff --git a/index.js b/index.js index e01d29f73..5f501ce69 100644 --- a/index.js +++ b/index.js @@ -19,6 +19,10 @@ import { import { resumeSession } from "./src/session/resume.js"; import { logger, flush as flushLogger } from "./src/shared/logger.js"; import { consumeStream } from "./src/stream/consume.js"; +import { + createSegmentLoopDetector, + destroyPool as destroyLoopPool, +} from "./src/stream/loopDetector.js"; // --------------------------------------------------------------------------- // CLI arguments @@ -296,6 +300,17 @@ async function doBootstrap() { * @param {AbortSignal} [signal] - Abort signal */ async function runAgent(input, threadExtras, streamingCallback, signal) { + const loopConfig = config.loopDetection ?? {}; + const loopDetector = + loopConfig.enabled === false + ? undefined + : createSegmentLoopDetector({ + windowSize: loopConfig.windowSize, + wordWindowSize: loopConfig.wordWindowSize, + maxStreams: loopConfig.maxStreams, + threshold: loopConfig.threshold, + ttlMs: loopConfig.ttlMs, + }); const stream = await agent.stream(input, { configurable: { thread_id: sessionState.getSessionId(), ...threadExtras }, maxTokens: providerConfig.maxTokens, @@ -304,11 +319,14 @@ async function runAgent(input, threadExtras, streamingCallback, signal) { streamMode: ["messages", "tools", "values"], subgraphs: true, }); - const { content, reasoning, interrupted, lastEventType } = await consumeStream( + // A detected loop abandons the stream; the caller (the TUI) sends the silent + // nudge, the same way it sends the reasoning-only "Continue". + const { content, reasoning, interrupted, lastEventType, loopDetected } = await consumeStream( stream, streamingCallback, { showToolResults, + loopDetector, }, ); return { @@ -318,6 +336,7 @@ async function runAgent(input, threadExtras, streamingCallback, signal) { tokens: { input: 0, output: 0 }, interrupted, lastEventType, + loopDetected, }; } @@ -410,6 +429,7 @@ async function runShutdown() { await handleShutdown({ onShutdown: async () => { if (gcManager) gcManager.stop(); + await destroyLoopPool(); if (mcpAdapter) await mcpAdapter.close(); if (shutdownFn) await shutdownFn(); }, diff --git a/src/agent/deepAgents.js b/src/agent/deepAgents.js index 6d96ebcc6..577694895 100644 --- a/src/agent/deepAgents.js +++ b/src/agent/deepAgents.js @@ -16,7 +16,6 @@ import { getModelContextLength } from "../provider/modelInfo.js"; import { getActiveProviderConfig, getActiveProviderName } from "../provider/index.js"; import { createTokenBudgetMiddleware } from "../provider/tokenBudgetMiddleware.js"; import { createCopilotRetryMiddleware } from "../provider/copilotRetryMiddleware.js"; -import { createLoopDetectionMiddleware } from "../provider/loopDetectionMiddleware.js"; import { createSummarizationMiddlewareFromConfig } from "../provider/summarizationMiddleware.js"; import { createEvictionAwareFilesystemMiddleware } from "./evictionCeiling.js"; import { createSkillsMiddleware } from "./skillsMiddleware.js"; @@ -535,22 +534,6 @@ export async function createDeepAgentsOrchestrator(checkpointer = null) { ? createCopilotRetryMiddleware({ maxRetries: 3 }) : null; - // Loop-detection middleware. Detects when the model repeats the same message - // or reasoning content and injects a "You're looping" nudge, escalating to the - // agent's compaction path when the loop persists. The `onCompact` callback is - // wired via closure after the agent exists (mirroring - // `compactOnContextWindowExceeded`), so it can call `agent.compactContext`. - // Returns null when disabled, so unset config is a true no-op. - let loopDetectionOnCompact; - const loopDetectionMiddleware = createLoopDetectionMiddleware({ - config: config.loopDetection, - onCompact: async (threadId) => { - if (typeof loopDetectionOnCompact === "function") { - await loopDetectionOnCompact(threadId); - } - }, - }); - // Replacement for deepagents' FilesystemMiddleware with an eviction ceiling // fixed to fit image payloads. The library evicts any ToolMessage whose // stringified content exceeds `toolTokenLimitBeforeEvict * 4` characters @@ -583,7 +566,6 @@ export async function createDeepAgentsOrchestrator(checkpointer = null) { : []), ...(summarizationMiddleware ? [summarizationMiddleware] : []), ...(copilotRetryMiddleware ? [copilotRetryMiddleware] : []), - ...(loopDetectionMiddleware ? [loopDetectionMiddleware] : []), ...(tokenBudgetMiddleware ? [tokenBudgetMiddleware] : []), ], streamTransformers: [() => createTurnTransformer()], @@ -610,14 +592,6 @@ export async function createDeepAgentsOrchestrator(checkpointer = null) { await agent.compactContext(config); }; - // Wire the loop-detection middleware's escalation to the agent's compaction - // path. The middleware is created before the agent, so this closure is set - // once the agent exists. It compacts the current thread's context. - loopDetectionOnCompact = async (threadId) => { - const config = threadId ? { configurable: { thread_id: threadId } } : undefined; - await agent.compactContext(config); - }; - // Expose the MCP adapter (if any) so the caller can close it on shutdown. // `allTools` is an array with the adapter attached as a property when MCP // servers are configured. diff --git a/src/config/schemas/loopDetection.js b/src/config/schemas/loopDetection.js index 40d8c5320..8078a7a63 100644 --- a/src/config/schemas/loopDetection.js +++ b/src/config/schemas/loopDetection.js @@ -1,29 +1,46 @@ import { z } from "zod"; /** - * Configuration for the `LoopDetection` middleware. + * Configuration for streaming loop detection. * - * When `enabled` is false (or the section is absent), madz does not register - * the loop-detection middleware and no loop detection occurs. When enabled, - * the middleware hashes the most recent model message text and reasoning - * content independently per thread, and injects a "You're looping" nudge when - * a hash repeats more than `threshold` times within a rolling `windowSize` - * window. If `escalationCount` nudges occur within `escalationWindowMs`, the - * middleware invokes the agent's compaction path instead of nudging again. + * When `enabled` is false, no loop detection occurs. When enabled, each + * streamed delta is split once, as it arrives, into completed sentences and + * words per kind (reasoning, message), and only the last `windowSize` sentences + * and `wordWindowSize` words are kept. A loop is a sentence repeated more than + * `threshold` times within the sentence window, or a run of words repeated + * back-to-back more than `threshold` times (loops with no punctuation). State + * is kept per stream (a kind plus an id), at most `maxStreams` at once. The + * stream is then abandoned and the TUI sends a silent "You're looping" nudge. + * If `escalationCount` nudges occur within `escalationWindowMs`, the context is + * compacted instead of nudging again. */ export const LoopDetectionSchema = z .object({ enabled: z.boolean().default(true), - // Number of recent content hashes retained per thread for loop detection. - windowSize: z.number().int().positive().default(10), - // A hash older than this many ms is pruned from the window, so past + // Number of recent completed sentences compared for duplicates. + windowSize: z.number().int().positive().default(20), + // Number of recent completed words compared for a back-to-back repeat (loops + // with no sentence terminator, such as "ok ok ok"). Must hold `threshold + 1` + // copies of the longest repeating unit: the two-paragraph capture is 190 + // words, so 3 copies need 570. + wordWindowSize: z.number().int().positive().default(1000), + // Maximum number of streams (a kind plus an id) tracked at once. Two live at + // the same moment is the realistic edge case; 4 leaves headroom. Streams that + // are done are released, so a long turn with hundreds of sequential messages + // never holds more than this. When more are live, the least recently updated + // is evicted; a looping stream is always receiving deltas, so it is never the + // one evicted. A stream with both windows full holds about 30-40 KB (measured). + maxStreams: z.number().int().positive().default(4), + // Accumulated stream text older than this many ms is dropped, so past // content cannot cause false positives as new content streams in. ttlMs: z.number().int().positive().default(60000), - // A hash must appear more than this many times within the window to flag a loop. + // A sentence (or word run) must repeat more than this many times to flag a loop. threshold: z.number().int().positive().default(2), - // Number of loop nudges within the escalation window before compacting. + // Strike, within the escalation window, that compacts the context instead of + // nudging: three strikes and you're out, so strikes 1 and 2 nudge and strike 3 + // compacts. escalationCount: z.number().int().positive().default(3), - // Rolling window (ms) for counting loop nudges before escalation. + // Rolling window (ms) for counting loop strikes before escalation. escalationWindowMs: z.number().int().positive().default(60000), }) .strict() diff --git a/src/provider/loopDetectionMiddleware.js b/src/provider/loopDetectionMiddleware.js deleted file mode 100644 index f50e189bf..000000000 --- a/src/provider/loopDetectionMiddleware.js +++ /dev/null @@ -1,242 +0,0 @@ -import { createHash } from "node:crypto"; -import { createMiddleware } from "langchain"; -import { HumanMessage } from "@langchain/core/messages"; -import { logger } from "../shared/logger.js"; - -/** The literal nudge injected into the model state when a loop is detected. */ -const LOOP_NUDGE = "You're looping"; - -/** - * Extract the message text and reasoning content from a LangChain message. - * - * Mirrors the content-extraction patterns in `src/stream/transformers/turn.js` - * so message text and reasoning content are captured independently. Returns - * `{ text, reasoning }` where each is a string (possibly empty). - * - * @param {Object} msg - A LangChain message - * @returns {{ text: string, reasoning: string }} Extracted content - */ -export function extractMessageContent(msg) { - if (!msg || typeof msg !== "object") return { text: "", reasoning: "" }; - - let text = ""; - let reasoning = ""; - - // String content is plain text. - if (typeof msg.content === "string") { - text = msg.content; - } else if (Array.isArray(msg.content)) { - // Content blocks (Responses API / block format). - for (const block of msg.content) { - if (typeof block === "string") { - text += block; - } else if (block && typeof block === "object") { - if (block.type === "text") text += block.text ?? ""; - else if (block.type === "reasoning") reasoning += block.reasoning ?? ""; - } - } - } - - // Reasoning content in additional_kwargs (Chat Completions API). - if (msg.additional_kwargs && typeof msg.additional_kwargs === "object") { - const rc = msg.additional_kwargs.reasoning_content; - if (typeof rc === "string") reasoning += rc; - } - - return { text, reasoning }; -} - -/** - * Compute a stable sha256 hash of a string. Returns `undefined` for empty or - * non-string input so empty content is not recorded in the hash window. - * @param {string} value - The content to hash - * @returns {string | undefined} The hex hash, or `undefined` when empty - */ -export function hashContent(value) { - if (typeof value !== "string" || value.length === 0) return undefined; - return createHash("sha256").update(value).digest("hex"); -} - -/** - * Create the `LoopDetection` middleware. - * - * Detects when the model repeats the same message text or reasoning content - * across successive calls. On each `beforeModel` hook, the most recent message - * content is hashed independently (text and reasoning) and recorded in a - * per-thread rolling window. When a hash appears more than the configured - * threshold within the window, the middleware injects a `HumanMessage` with - * the text "You're looping" so the model observes the nudge on its next turn. - * - * If the escalation count of nudges is reached within the escalation window, - * the middleware invokes the `onCompact` callback (wired to the agent's - * compaction path) instead of injecting another nudge. - * - * Returns `null` when disabled, so unset config is a true no-op. - * - * @param {Object} options - Middleware options - * @param {Object} [options.config] - The resolved `loopDetection` config section - * @param {Function} [options.onCompact] - Callback invoked on escalation (e.g. `agent.compactContext`) - * @param {Function} [options.now] - Clock function returning ms (injectable for tests) - * @returns {Object|null} The middleware, or `null` when disabled - */ -export function createLoopDetectionMiddleware(options = {}) { - const config = options.config ?? {}; - if (config.enabled === false) return null; - - const windowSize = config.windowSize ?? 10; - const ttlMs = config.ttlMs ?? 60000; - const threshold = config.threshold ?? 2; - const escalationCount = config.escalationCount ?? 3; - const escalationWindowMs = config.escalationWindowMs ?? 60000; - const onCompact = options.onCompact; - const now = options.now ?? (() => Date.now()); - - /** Per-thread state: hash windows (with timestamps) and nudge timestamps. */ - const threadState = new Map(); - - /** - * Get (or create) the state for a thread. - * @param {string} threadKey - The thread_id - * @returns {{ messageHashes: {hash: string, ts: number}[], reasoningHashes: {hash: string, ts: number}[], nudges: number[] }} State - */ - function getThreadState(threadKey) { - let state = threadState.get(threadKey); - if (!state) { - state = { messageHashes: [], reasoningHashes: [], nudges: [] }; - threadState.set(threadKey, state); - } - return state; - } - - /** - * Prune hashes older than the TTL from a window. The window is - * insertion-ordered, so the oldest entries are at the front. - * @param {{hash: string, ts: number}[]} window - The hash window array - * @param {number} nowMs - Current time in ms - * @returns {void} - */ - function pruneHashes(window, nowMs) { - while (window.length > 0 && nowMs - window[0].ts > ttlMs) window.shift(); - } - - /** - * Clear both hash windows for a thread. Called when a nudge is injected (or - * on escalation) so the model gets a fresh slate — otherwise it could repeat - * the same phrase on its next turn and immediately re-trigger detection. - * @param {{messageHashes: {hash: string, ts: number}[], reasoningHashes: {hash: string, ts: number}[]}} thread - The thread state - * @returns {void} - */ - function clearHashes(thread) { - thread.messageHashes = []; - thread.reasoningHashes = []; - } - - /** - * Record a hash with a timestamp in a bounded rolling window, evicting the - * oldest entry when the window exceeds `windowSize` or the TTL. - * @param {{hash: string, ts: number}[]} window - The hash window array - * @param {string} hash - The hash to record - * @param {number} nowMs - Current time in ms - * @returns {void} - */ - function recordHash(window, hash, nowMs) { - pruneHashes(window, nowMs); - window.push({ hash, ts: nowMs }); - while (window.length > windowSize) window.shift(); - } - - /** - * Count occurrences of a hash within a window. - * @param {{hash: string, ts: number}[]} window - The hash window array - * @param {string} hash - The hash to count - * @returns {number} Occurrence count - */ - function countOccurrences(window, hash) { - let count = 0; - for (const entry of window) if (entry.hash === hash) count++; - return count; - } - - /** - * Decide whether a hash exceeds the loop threshold within its window. - * @param {{hash: string, ts: number}[]} window - The hash window array - * @param {string} hash - The hash to check - * @returns {boolean} True when the hash appears more than `threshold` times - */ - function isLooping(window, hash) { - return countOccurrences(window, hash) > threshold; - } - - /** - * Prune nudge timestamps older than the escalation window. - * @param {number[]} nudges - Nudge timestamps - * @param {number} nowMs - Current time in ms - * @returns {void} - */ - function pruneNudges(nudges, nowMs) { - while (nudges.length > 0 && nowMs - nudges[0] > escalationWindowMs) nudges.shift(); - } - - return createMiddleware({ - name: "LoopDetection", - beforeModel: { - canJumpTo: [], - hook: async (state, runtime) => { - const messages = state?.messages ?? []; - if (messages.length === 0) return undefined; - - const threadKey = runtime?.configurable?.thread_id ?? ""; - const thread = getThreadState(threadKey); - const lastMsg = messages[messages.length - 1]; - const { text, reasoning } = extractMessageContent(lastMsg); - - const nowMs = now(); - const textHash = hashContent(text); - const reasoningHash = hashContent(reasoning); - - let looped = false; - if (textHash) { - recordHash(thread.messageHashes, textHash, nowMs); - if (isLooping(thread.messageHashes, textHash)) looped = true; - } - if (reasoningHash) { - recordHash(thread.reasoningHashes, reasoningHash, nowMs); - if (isLooping(thread.reasoningHashes, reasoningHash)) looped = true; - } - - if (!looped) return undefined; - - pruneNudges(thread.nudges, nowMs); - - // Escalate to compaction when the nudge count reaches the threshold. - if (thread.nudges.length >= escalationCount) { - logger.warn( - { thread: threadKey, nudges: thread.nudges.length, escalationCount }, - "[loopDetection] escalating to context compaction", - ); - thread.nudges = []; - clearHashes(thread); - if (typeof onCompact === "function") { - try { - await onCompact(threadKey); - } catch (err) { - logger.warn({ error: err?.message }, "[loopDetection] onCompact callback failed"); - } - } - return undefined; - } - - // Otherwise inject the nudge and record it. Clear the hash windows - // so the model gets a fresh slate — otherwise it could repeat the - // same phrase on its next turn and immediately re-trigger. - thread.nudges.push(nowMs); - clearHashes(thread); - logger.warn( - { thread: threadKey, nudges: thread.nudges.length }, - "[loopDetection] detected loop; injecting nudge", - ); - return { messages: [new HumanMessage(LOOP_NUDGE)] }; - }, - }, - }); -} diff --git a/src/stream/consume.js b/src/stream/consume.js index c76241e65..561dd8b81 100644 --- a/src/stream/consume.js +++ b/src/stream/consume.js @@ -13,14 +13,54 @@ import { isSubagentNamespace } from "./namespace.js"; * @param {Function} [streamingCallback] - Callback for streamed chunks * @param {Object} [options] - Options * @param {boolean} [options.showToolResults=true] - Whether to emit `tool` events - * @returns {Promise<{content: string, reasoning: string, interrupted: boolean, lastEventType: string | null}>} + * @param {{push: Function, settle?: Function, reset: Function}} [options.loopDetector] - Loop + * detector. Each non-subagent message/reasoning delta is pushed to it together with + * the stream block's id (the content-block index for messages, `block.id` for + * Responses API reasoning), so detection is restricted to one block and concurrent + * blocks never mix. Chat Completions `reasoning_content` carries no id. When it + * flags a loop the stream is abandoned (closing the underlying iterator) and + * `loopDetected` is returned true. Checks run in a worker, so a loop can be + * flagged after the stream ends; `settle()` is awaited to collect it. When a + * message turn ends (its last chunk carries a `finish_reason`, a different message + * id arrives, or a tool result follows) `end()` collects the verdict and releases + * the detector's word and sentence lists. The detector is also reset when the + * stream itself ends, however it ends. + * @returns {Promise<{content: string, reasoning: string, interrupted: boolean, lastEventType: string | null, loopDetected: boolean}>} */ export async function consumeStream(stream, streamingCallback, options = {}) { + try { + return await drainStream(stream, streamingCallback, options); + } finally { + // The turn is over (finished, interrupted, abandoned on a loop, or thrown): + // release the detector's lists so nothing from this turn is retained. + options.loopDetector?.reset(); + } +} + +/** + * Drain the agent stream. See `consumeStream`. + * @param {AsyncIterable} stream - The agent stream (subgraphs mode) + * @param {Function} [streamingCallback] - Callback for streamed chunks + * @param {Object} options - Options + * @returns {Promise<{content: string, reasoning: string, interrupted: boolean, lastEventType: string | null, loopDetected: boolean}>} + */ +async function drainStream(stream, streamingCallback, options) { const showToolResults = options.showToolResults; let content = ""; let reasoning = ""; let interrupted = false; let lastEventType = null; + let loopDetected = false; + const loopDetector = options.loopDetector; + /** The id shared by the chunks of the message currently streaming. */ + let currentMsgId = null; + + // A message turn is start, stream, end. When it ends, collect the verdict of + // any worker check still running and release the detector's lists. + const endMessage = async () => { + currentMsgId = null; + if (loopDetector?.end && (await loopDetector.end())) loopDetected = true; + }; for await (const [namespace, mode, payload] of stream) { // Subagent runs are nested under a `tools:` namespace segment. Their @@ -44,6 +84,14 @@ export async function consumeStream(stream, streamingCallback, options = {}) { // never rendered into the assistant's streaming bubble. if (msgType === "human") continue; + if (!isSubagent) { + // A different message id means the previous message's turn has ended. + if (msg?.id != null && currentMsgId != null && msg.id !== currentMsgId) await endMessage(); + if (msg?.id != null) currentMsgId = msg.id; + // A tool result means the AI message that called the tool has ended. + if (msgType === "tool") await endMessage(); + } + if (text) { // ToolMessage text is emitted as a distinct `tool` event, never // folded into the message segment. @@ -62,6 +110,8 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (!isSubagent) { content += text; + const blockIndex = msg?.content?.[0]?.index; + if (loopDetector?.push("message", text, blockIndex)) loopDetected = true; } if (streamingCallback) { @@ -83,6 +133,7 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (reasoningContent) { if (!isSubagent) { reasoning += reasoningContent; + if (loopDetector?.push("reasoning", reasoningContent)) loopDetected = true; } if (streamingCallback) { streamingCallback({ @@ -99,6 +150,7 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (block?.type === "reasoning" && block.reasoning) { if (!isSubagent) { reasoning += block.reasoning; + if (loopDetector?.push("reasoning", block.reasoning, block.id)) loopDetected = true; } if (streamingCallback) { streamingCallback({ @@ -112,6 +164,8 @@ export async function consumeStream(stream, streamingCallback, options = {}) { } } } + // The chunk that carries a finish reason is the last of its message. + if (!isSubagent && msg?.response_metadata?.finish_reason) await endMessage(); } else if (mode === "tools" && streamingCallback) { if (payload?.event === "on_tool_start") { streamingCallback({ @@ -132,7 +186,14 @@ export async function consumeStream(stream, streamingCallback, options = {}) { if (!isSubagent) lastEventType = "on_tool_end"; } } + + // Abandon the stream; leaving the for-await closes the iterator. + if (loopDetected) break; } - return { content, reasoning, interrupted, lastEventType }; + // Worker checks may still be in flight when the stream ends; collect them so + // a loop in the final deltas is not missed. + if (!loopDetected && loopDetector?.settle) loopDetected = await loopDetector.settle(); + + return { content, reasoning, interrupted, lastEventType, loopDetected }; } diff --git a/src/stream/loopCheck.js b/src/stream/loopCheck.js new file mode 100644 index 000000000..80761884f --- /dev/null +++ b/src/stream/loopCheck.js @@ -0,0 +1,97 @@ +/** + * Pure loop checks. They work on already-split sentences and words, never on + * the raw stream, so how the provider cut the stream into deltas does not + * matter, and the cost of a check depends on the window, not on how long the + * stream has been running. Kept free of I/O so the worker thread and the unit + * tests run the exact same code. + */ + +/** + * Take the sentences that have completed in `pending`. A sentence ends at `.`, + * `?` or `!` followed by whitespace, or at a newline. Only the text from `from` + * on is scanned, so a caller that appends a delta and passes the old length + * pays for the delta, not for everything seen so far. + * @param {string} pending - Text not yet consumed (an unfinished sentence plus new text) + * @param {number} [from=0] - Index to start scanning at (the length before the new text was appended) + * @returns {{ sentences: string[], rest: string }} Completed sentences (whitespace collapsed) + * and the unfinished remainder to carry forward + */ +export function takeSentences(pending, from = 0) { + const boundary = /(?<=[.!?])\s+|\n+/g; + boundary.lastIndex = from; + const sentences = []; + let start = 0; + let match; + while ((match = boundary.exec(pending)) !== null) { + const sentence = pending.slice(start, match.index).replace(/\s+/g, " ").trim(); + if (sentence) sentences.push(sentence); + start = match.index + match[0].length; + } + return { sentences, rest: pending.slice(start) }; +} + +/** + * Take the words that have completed once `delta` is appended. A word ends at + * whitespace; a trailing piece with no whitespace after it may continue in the + * next delta, so it is carried forward. + * @param {string} partial - Unfinished word carried from the previous delta + * @param {string} delta - New text + * @returns {{ words: string[], rest: string }} Completed words and the unfinished remainder + */ +export function takeWords(partial, delta) { + const text = partial + delta; + const parts = text.split(/\s+/); + const rest = /\s$/.test(text) ? "" : parts.pop(); + return { words: parts.filter(Boolean), rest }; +} + +/** + * Look for a sentence that occurs more than `threshold` times in `sentences`. + * @param {string[]} sentences - The recent completed sentences + * @param {number} threshold - Occurrences above which a sentence is looping + * @returns {boolean} True when a sentence repeats + */ +export function hasDuplicateSentence(sentences, threshold) { + const counts = new Map(); + for (const sentence of sentences) { + const count = (counts.get(sentence) ?? 0) + 1; + if (count > threshold) return true; + counts.set(sentence, count); + } + return false; +} + +/** + * Look for a run of words repeated back-to-back at the end of `words`. This + * covers loops with no sentence terminator at all, such as "ok ok ok ok". + * @param {string[]} words - The recent completed words + * @param {number} threshold - Back-to-back repeats above which a loop is flagged + * @returns {boolean} True when the tail is one word run repeated more than `threshold` times + */ +export function hasRepeatedWords(words, threshold) { + const needed = threshold + 1; + const n = words.length; + const last = words[n - 1]; + for (let period = 1; period * needed <= n; period++) { + // Cheap reject: the unit's last word must recur exactly one period back. + if (words[n - 1 - period] !== last) continue; + let match = true; + for (let i = 1; i < period * (needed - 1) && match; i++) { + if (words[n - 1 - i] !== words[n - 1 - i - period]) match = false; + } + if (match) return true; + } + return false; +} + +/** + * Check the recent sentences and words for a loop. + * @param {Object} task - The check + * @param {string[]} task.sentences - Recent completed sentences + * @param {string[]} task.words - Recent completed words + * @param {number} task.threshold - Repeats above which a loop is flagged + * @returns {boolean} True when the stream is looping + */ +export function checkLoop({ sentences, words, threshold }) { + return hasDuplicateSentence(sentences, threshold) || hasRepeatedWords(words, threshold); +} diff --git a/src/stream/loopDetector.js b/src/stream/loopDetector.js new file mode 100644 index 000000000..36a3ffd5a --- /dev/null +++ b/src/stream/loopDetector.js @@ -0,0 +1,259 @@ +import { Piscina } from "piscina"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; +import { takeSentences, takeWords } from "./loopCheck.js"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +/** @type {Piscina | null} */ +let pool = null; + +/** + * Piscina worker pool for loop detection. Created lazily on first use and then + * reused, like the indexCode pool, so threads are not spawned per call and + * nothing is spawned at import time. + * @returns {Piscina} The shared pool + */ +export function getPool() { + if (!pool) pool = new Piscina({ filename: join(__dirname, "loopDetectorWorker.js") }); + return pool; +} + +/** + * Destroy the shared pool (shutdown and tests). A later `getPool()` makes a new one. + * @returns {Promise} + */ +export async function destroyPool() { + const current = pool; + pool = null; + await current?.destroy(); +} + +/** + * Create a streaming loop detector. + * + * Each streamed delta is split once, as it arrives, into completed sentences + * and words. State is kept per stream: the key is the kind (reasoning, message) + * plus the stream's id, so concurrent streams never mix and detection is + * restricted to the one stream a delta belongs to. A delta with no id (Chat + * Completions `reasoning_content`) is keyed by kind alone. At most `maxStreams` + * streams are tracked at once; the least recently updated is evicted. A looping + * stream is by definition receiving deltas, so it is never the one evicted; the + * cap only matters when more than `maxStreams` streams are live at the same + * moment. Only the last `windowSize` sentences + * and `wordWindowSize` words are kept, so memory and the cost of a check stay + * bounded however long the stream runs, and the provider's delta boundaries do + * not matter. A loop is a sentence repeated within the sentence window, or a + * run of words repeated back-to-back (for loops with no sentence terminator, + * such as "ok ok ok"). + * + * The check runs in a worker thread so it never blocks the main loop. `push` + * returns immediately with the latest known result. A check starts only when a + * delta completed a new sentence or word, and at most one check per stream is in + * flight. + * + * @param {Object} [options] - Options + * @param {number} [options.windowSize=20] - Recent completed sentences kept and compared + * @param {number} [options.wordWindowSize=1000] - Recent completed words kept and compared + * @param {number} [options.threshold=2] - Repeats above which a loop is flagged + * @param {number} [options.maxStreams=4] - Streams tracked at once; the least recently updated is evicted + * @param {number} [options.ttlMs] - State older than this is dropped (no expiry when unset) + * @param {Function} [options.now] - Clock returning ms (injectable for tests) + * @param {{ run: Function }} [options.runner] - Runs a check; defaults to the shared pool, injectable for tests + * @returns {{ push: (kind: string, delta: string, id?: string | number) => boolean, settle: () => Promise, end: () => Promise, reset: () => void, size: () => number }} Detector + */ +export function createSegmentLoopDetector(options = {}) { + const windowSize = options.windowSize ?? 20; + const wordWindowSize = options.wordWindowSize ?? 1000; + const threshold = options.threshold ?? 2; + const maxStreams = options.maxStreams ?? 4; + const ttlMs = options.ttlMs; + const now = options.now ?? (() => Date.now()); + const runner = options.runner ?? { run: (task) => getPool().run(task) }; + + /** + * @typedef {Object} KindState + * @property {string} pending - Unfinished sentence text + * @property {string} partialWord - Unfinished word + * @property {string[] | null} sentences - Last `windowSize` completed sentences (null once released) + * @property {string[] | null} words - Last `wordWindowSize` completed words (null once released) + * @property {number} ts - Time of the last delta + * @property {boolean} flagged - A check has flagged a loop + * @property {Promise | null} inflight - The running check, if any + * @property {number} version - Bumped when a sentence or word completes + * @property {boolean} dead - The state was replaced, evicted or reset; in-flight answers are stale + */ + /** @type {Map} Insertion order is recency order, oldest first. */ + const kinds = new Map(); + + function freshState() { + return { + pending: "", + partialWord: "", + sentences: [], + words: [], + ts: now(), + flagged: false, + inflight: null, + version: 0, + dead: false, + }; + } + + /** + * Release a stream's state: clear its word and sentence lists and null them so + * the memory is freed even while a closure (an in-flight check) still holds the + * state object. A released state is dead, so any in-flight answer is ignored. + * @param {KindState} state - State to release + * @returns {void} + */ + function release(state) { + state.dead = true; + state.pending = ""; + state.partialWord = ""; + if (state.sentences) state.sentences.length = 0; + if (state.words) state.words.length = 0; + state.sentences = null; + state.words = null; + } + + /** + * Get (or create) the state for a stream and mark it most recently used. The + * least recently used stream is evicted once more than `maxStreams` are tracked. + * @param {string} key - Stream key (kind, or kind plus id) + * @param {KindState} [replacement] - State to install in place of the existing one + * @returns {KindState} The stream's state + */ + function touch(key, replacement) { + const existing = kinds.get(key); + const state = replacement ?? existing ?? freshState(); + if (existing && existing !== state) release(existing); + kinds.delete(key); + kinds.set(key, state); + if (kinds.size > maxStreams) { + const oldest = kinds.keys().next().value; + release(kinds.get(oldest)); + kinds.delete(oldest); + } + return state; + } + + /** + * Keep only the last `size` entries, trimming in place. + * @param {string[]} list - List to trim + * @param {number} size - Entries to keep + * @returns {void} + */ + function trim(list, size) { + if (list.length > size) list.splice(0, list.length - size); + } + + /** + * Start a worker check on a kind's recent sentences and words, unless one is running. + * @param {KindState} state - Kind state + * @returns {void} + */ + function startCheck(state) { + if (state.inflight) return; + const checkedVersion = state.version; + const task = { sentences: state.sentences.slice(), words: state.words.slice(), threshold }; + state.inflight = Promise.resolve(runner.run(task)) + .then((looping) => { + // A state that was replaced, evicted or reset makes its answer stale. + if (!state.dead && looping) state.flagged = true; + }) + .catch(() => { + // A failed check must never break streaming: treat as "not looping". + }) + .finally(() => { + state.inflight = null; + // Sentences or words completed while checking: check again so the tail is covered. + if (!state.dead && !state.flagged && state.version !== checkedVersion) { + startCheck(state); + } + }); + } + + return { + /** + * Record a streamed delta. + * @param {string} kind - Stream kind, e.g. "reasoning" or "message" + * @param {string} delta - The delta exactly as streamed + * @param {string | number} [id] - The stream's id; a delta with no id is keyed by kind alone + * @returns {boolean} True when a completed check has flagged a loop for this stream + */ + push(kind, delta, id) { + if (typeof delta !== "string" || delta.length === 0) return false; + const key = id == null ? kind : `${kind}\0${id}`; + let state = touch(key); + const nowMs = now(); + if (ttlMs !== undefined && nowMs - state.ts > ttlMs) { + // Expired: start this stream over. touch() marks the old state dead, so + // its in-flight answer is dropped without affecting any other stream. + state = touch(key, freshState()); + } + state.ts = nowMs; + + // Split only the new text. The unfinished tail is carried forward. + const joined = state.pending + delta; + const { sentences, rest } = takeSentences(joined, state.pending.length); + state.pending = rest; + const { words, rest: partialWord } = takeWords(state.partialWord, delta); + state.partialWord = partialWord; + + if (sentences.length > 0) { + state.sentences.push(...sentences); + trim(state.sentences, windowSize); + } + if (words.length > 0) { + state.words.push(...words); + trim(state.words, wordWindowSize); + } + if (sentences.length > 0 || words.length > 0) { + state.version++; + if (!state.flagged) startCheck(state); + } + return state.flagged; + }, + + /** + * Wait for in-flight checks and report whether any kind is flagged. + * @returns {Promise} True when a loop has been flagged + */ + async settle() { + let pending = [...kinds.values()].map((s) => s.inflight).filter(Boolean); + while (pending.length > 0) { + await Promise.all(pending); + pending = [...kinds.values()].map((s) => s.inflight).filter(Boolean); + } + return [...kinds.values()].some((s) => s.flagged); + }, + + /** Release every stream's state (called when a message turn ends, and after a nudge). */ + reset() { + for (const state of kinds.values()) release(state); + kinds.clear(); + }, + + /** + * End a message turn: collect the verdict of any check still running (the + * last deltas may have started one), then release every stream's lists. The + * verdict is read before the release, so a loop in the final deltas of the + * message is not lost. + * @returns {Promise} True when a loop was flagged during the message + */ + async end() { + const flagged = await this.settle(); + this.reset(); + return flagged; + }, + + /** + * Number of streams currently tracked. + * @returns {number} Tracked stream count + */ + size() { + return kinds.size; + }, + }; +} diff --git a/src/stream/loopDetectorWorker.js b/src/stream/loopDetectorWorker.js new file mode 100644 index 000000000..a371d030d --- /dev/null +++ b/src/stream/loopDetectorWorker.js @@ -0,0 +1,17 @@ +import { checkLoop } from "./loopCheck.js"; + +/** + * Piscina worker entry point for loop detection. + * + * Runs the check off the main event loop. Stateless: each task carries the + * recent sentences and words, so any worker thread can serve any task. + * + * @param {Object} task - The check to run + * @param {string[]} task.sentences - Recent completed sentences + * @param {string[]} task.words - Recent completed words + * @param {number} task.threshold - Repeats above which a loop is flagged + * @returns {boolean} True when the stream is looping + */ +export default function loopWorker(task) { + return checkLoop(task); +} diff --git a/src/tui/conversationArea.js b/src/tui/conversationArea.js index 6c7dd51a8..8b6cfcea4 100644 --- a/src/tui/conversationArea.js +++ b/src/tui/conversationArea.js @@ -77,6 +77,66 @@ export function shouldAutoContinue({ ); } +/** + * Decide whether to send a silent "Continue" after a loop-triggered compaction. + * + * The model is resumed only when the compaction succeeded (a failed one leaves + * the looping context in place, so continuing would loop again) and the shared + * auto-continue guard is not exhausted, so a model that keeps looping cannot + * cycle through nudge, compact, continue forever. + * + * @param {Object} params - Decision inputs + * @param {boolean} params.compacted - Whether the compaction succeeded + * @param {number} params.autoContinueCount - Current auto-continue count + * @param {number} params.autoContinueLimit - Max auto-continue count + * @returns {boolean} True when a silent "Continue" should be dispatched + */ +export function shouldContinueAfterCompact({ compacted, autoContinueCount, autoContinueLimit }) { + return compacted === true && autoContinueCount < autoContinueLimit; +} + +/** + * Decide what to do after a turn when the loop detector may have abandoned the stream. + * + * Mirrors `shouldAutoContinue`: it acts only when the streaming bubble is the + * last in the message list AND the stream was abandoned because the loop + * detector flagged it. Each detected loop is a strike. Strikes within + * `escalationWindowMs` are counted, this one included, and the `escalationCount`th + * strike compacts the context instead of nudging (three strikes and you're out: + * strikes 1 and 2 nudge, strike 3 compacts). + * + * @param {Object} params - Decision inputs + * @param {boolean | undefined} params.loopDetected - Whether the loop detector flagged the stream + * @param {boolean} [params.interrupted] - True when the turn paused on an interrupt; a paused + * turn is waiting for the user's answer, so it is never nudged or compacted + * @param {string | undefined} params.streamingMsgId - The streaming bubble's message ID + * @param {string | undefined} params.lastMessageId - The last message ID in the list + * @param {number[]} params.nudges - Timestamps (ms) of previous strikes that were nudged + * @param {number} params.now - Current time in ms + * @param {number} params.escalationCount - The strike, within the window, that compacts + * @param {number} params.escalationWindowMs - Rolling window (ms) for counting strikes + * @returns {{ action: "none" | "nudge" | "compact", nudges: number[] }} The action and the + * updated strike timestamps (pruned to the window) + */ +export function decideLoopAction({ + loopDetected, + interrupted = false, + streamingMsgId, + lastMessageId, + nudges, + now, + escalationCount, + escalationWindowMs, +}) { + if (loopDetected !== true || interrupted || streamingMsgId !== lastMessageId) { + return { action: "none", nudges }; + } + const recent = nudges.filter((ts) => now - ts <= escalationWindowMs); + // This loop is itself a strike, so it is counted before deciding. + if (recent.length + 1 >= escalationCount) return { action: "compact", nudges: [] }; + return { action: "nudge", nudges: [...recent, now] }; +} + /** * ConversationArea — owns all conversation and streaming state. * Communicates with InputArea exclusively via stable App-provided callbacks. @@ -134,6 +194,12 @@ const ConversationArea = forwardRef(function ConversationArea( // Counts silent "Continue" nudges dispatched when a turn ends on reasoning. // Bounded by config.agent.autoContinueLimit to prevent an infinite loop. const autoContinueCountRef = useRef(0); + const autoContinueLimit = config?.agent?.autoContinueLimit; + // Timestamps of loop strikes that were nudged. The `escalationCount`th strike + // within `escalationWindowMs` compacts the context instead of nudging. + const loopNudgesRef = useRef([]); + const escalationCount = config?.loopDetection?.escalationCount; + const escalationWindowMs = config?.loopDetection?.escalationWindowMs; const toolMessageTokensRef = useRef(0); const contextUpdateTimerRef = useRef(null); const pendingContextRef = useRef({ content: "" }); @@ -266,11 +332,12 @@ const ConversationArea = forwardRef(function ConversationArea( * Compact the context window on demand. * Invokes the agent's compaction routine, syncs the TUI conversation view, * recomputes the context size. + * @returns {Promise} True when the context was compacted */ const handleCompact = async () => { if (!compactContext) { addMessage({ role: "system", content: "Compaction is not available." }); - return; + return false; } const compactMsgId = addMessage({ role: "system", content: "Compacting context window..." }); try { @@ -288,13 +355,15 @@ const ConversationArea = forwardRef(function ConversationArea( messageListRef.current?.updateMessage(compactMsgId, { content: msg }); // Recompute the context size so the status bar reflects the reduced window. await updateContextSize(sessionState, config); - } else { - const msg = `Compaction failed: ${result?.error || "unknown error"}`; - messageListRef.current?.updateMessage(compactMsgId, { content: msg }); + return true; } + const msg = `Compaction failed: ${result?.error || "unknown error"}`; + messageListRef.current?.updateMessage(compactMsgId, { content: msg }); + return false; } catch (err) { const msg = `Compaction failed: ${err.message}`; messageListRef.current?.updateMessage(compactMsgId, { content: msg }); + return false; } }; @@ -501,13 +570,49 @@ const ConversationArea = forwardRef(function ConversationArea( completedToolCalls, ); + // Loop nudge: when the loop detector abandoned the stream, nudge the model + // with a silent "You're looping", the same way the reasoning-only + // "Continue" below is dispatched. Scoped to the LAST bubble. After + // `escalationCount`th strike within `escalationWindowMs`, compact the + // context instead of nudging. + const loop = decideLoopAction({ + loopDetected: response?.loopDetected, + interrupted: response?.interrupted, + streamingMsgId: streamingMsgIdRef.current, + lastMessageId: messageListRef.current?.getLastMessageId(), + nudges: loopNudgesRef.current, + now: Date.now(), + escalationCount, + escalationWindowMs, + }); + loopNudgesRef.current = loop.nudges; + if (loop.action === "compact") { + const compacted = await handleCompact(); + // After a successful compaction, resume the model with a silent + // "Continue", counted against the same autoContinueLimit guard. + if ( + shouldContinueAfterCompact({ + compacted, + autoContinueCount: autoContinueCountRef.current, + autoContinueLimit, + }) + ) { + autoContinueCountRef.current += 1; + await handleChat("Continue", { silentUser: true }); + } + return; + } + if (loop.action === "nudge") { + await handleChat("You're looping", { silentUser: true }); + return; + } + // Auto-continue: when the last bubble ends on reasoning or a tool // message (the model produced no final message content), nudge it with // a silent "Continue" so it produces an actual response. Skipped for // interrupted turns. Scoped to the LAST bubble only — earlier // incomplete bubbles must not trigger it. Bounded // by config.agent.autoContinueLimit to prevent an infinite loop. - const autoContinueLimit = config?.agent?.autoContinueLimit ?? 1000; if ( shouldAutoContinue({ lastEventType: response?.lastEventType, @@ -584,6 +689,7 @@ const ConversationArea = forwardRef(function ConversationArea( // not per-process. A reasoning loop is a per-thread phenomenon; a fresh // session must not inherit a counter exhausted by a prior thread. autoContinueCountRef.current = 0; + loopNudgesRef.current = []; messageListRef.current?.clear(); // Recompute the context size for the fresh thread rather than hard-setting // 0. The system prompt (SYSTEM_PROMPT.md + AGENTS.md + memory context) is diff --git a/tests/fixtures/loop-reasoning.md b/tests/fixtures/loop-reasoning.md new file mode 100644 index 000000000..0ffb1b43b --- /dev/null +++ b/tests/fixtures/loop-reasoning.md @@ -0,0 +1,94 @@ +For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I'm realizing the core tension: the user wants a module-level cached config like jira uses, but the existing tests call `sqlImpl` without passing a policy, which would + trigger a real `loadConfig()` call and potentially fail or connect to an actual database. I need to figure out how to make the tests work with this pattern—either by having the tests provide a policy, or by + making the config loading safe when no policy is passed. + + The cleanest approach is to have `sqlImpl` read from `options.policy` if provided, otherwise fall back to the cached config, and then update the tests to pass a policy for the connection-related cases. + This keeps the implementation consistent with jira while making the test behavior explicit and predictable. I'm settling on a pragmatic approach: keep the existing schema structure with `postgres`, `mysql`, + `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so + environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the default path uses the module-level cache. This keeps the design consistent with how jira works while allowing the model to + stop supplying credentials directly. + + Writing the sqlImpl function... I need to verify that the config schemas are already set up correctly with the right structure, then focus on wiring the tools to read from the cached config and update the + tests to inject policies where needed. The key is ensuring the env var naming works properly with DROPPED_KEYS so that `PG_HOST` and similar variables map correctly to the nested config paths. I'm settling + on the structure: keep the existing schema with `postgres`, `mysql`, `sqlite` under `sql` and `mongodb`, `redis` under `nosql`, then wire the tools to read from `config.sql[driver]` and + `config.nosql[engine]` respectively. I'll add `sql` and `nosql` to DROPPED_KEYS so environment variables get prefixed with the database name, and update the config.yaml sections to match this structure. + + For the tool implementation, I'm mirroring jira's pattern: the input schema drops the `connection` field entirely, and instead reads from a cached config object that gets populated via `loadConfig()`. + Tests can inject a policy through options to override the cached config, but the + + ⚡ 3 tool calls: file ×3 + + ⏱ 8m 26s + +───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + + [8:23 a.m.] You: + + you're looping + +───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + + [8:23 a.m.] Madz: + + 💭 Thinking… → \ No newline at end of file diff --git a/tests/unit/provider/loopDetectionMiddleware.test.js b/tests/unit/provider/loopDetectionMiddleware.test.js deleted file mode 100644 index d55519b0e..000000000 --- a/tests/unit/provider/loopDetectionMiddleware.test.js +++ /dev/null @@ -1,242 +0,0 @@ -import { describe, it } from "node:test"; -import assert from "node:assert"; -import { AIMessage } from "@langchain/core/messages"; -import { - createLoopDetectionMiddleware, - extractMessageContent, - hashContent, -} from "../../../src/provider/loopDetectionMiddleware.js"; - -const config = (extra = {}) => ({ - enabled: true, - windowSize: 10, - threshold: 2, - escalationCount: 3, - escalationWindowMs: 60000, - ...extra, -}); - -const run = (mw, state, thread = "t") => - mw.beforeModel.hook(state, { configurable: { thread_id: thread } }); - -const ai = (text, reasoning) => { - const msg = new AIMessage(text); - if (reasoning !== undefined) { - msg.additional_kwargs = { reasoning_content: reasoning }; - } - return msg; -}; - -describe("extractMessageContent", () => { - it("extracts string content as text", () => { - assert.deepStrictEqual(extractMessageContent(new AIMessage("hello")), { - text: "hello", - reasoning: "", - }); - }); - - it("extracts reasoning from additional_kwargs", () => { - const msg = new AIMessage("hello"); - msg.additional_kwargs = { reasoning_content: "think" }; - assert.deepStrictEqual(extractMessageContent(msg), { text: "hello", reasoning: "think" }); - }); - - it("extracts reasoning from content blocks", () => { - const msg = new AIMessage([ - { type: "text", text: "hello" }, - { type: "reasoning", reasoning: "think" }, - ]); - assert.deepStrictEqual(extractMessageContent(msg), { text: "hello", reasoning: "think" }); - }); - - it("returns empty strings for null/empty input", () => { - assert.deepStrictEqual(extractMessageContent(null), { text: "", reasoning: "" }); - assert.deepStrictEqual(extractMessageContent({}), { text: "", reasoning: "" }); - }); -}); - -describe("hashContent", () => { - it("hashes identical content identically", () => { - assert.strictEqual(hashContent("abc"), hashContent("abc")); - }); - - it("returns undefined for empty or non-string content", () => { - assert.strictEqual(hashContent(""), undefined); - assert.strictEqual(hashContent(undefined), undefined); - }); -}); - -describe("createLoopDetectionMiddleware", () => { - it("returns null when disabled", () => { - assert.strictEqual(createLoopDetectionMiddleware({ config: { enabled: false } }), null); - }); - - it("is named LoopDetection and uses beforeModel", () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - assert.strictEqual(mw.name, "LoopDetection"); - assert.strictEqual(typeof mw.beforeModel.hook, "function"); - assert.strictEqual(mw.wrapModelCall, undefined); - }); - - it("does nothing for empty state", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - assert.strictEqual(await run(mw, { messages: [] }), undefined); - }); - - it("does not flag a loop below the threshold", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - // Two identical messages: threshold is 2, so 2 occurrences is NOT > 2. - assert.strictEqual(await run(mw, { messages: [ai("same")] }), undefined); - assert.strictEqual(await run(mw, { messages: [ai("same")] }), undefined); - }); - - it("flags a loop above the threshold and injects a nudge", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - await run(mw, { messages: [ai("same")] }); - await run(mw, { messages: [ai("same")] }); - const out = await run(mw, { messages: [ai("same")] }); - assert.ok(out); - assert.strictEqual(out.messages.length, 1); - assert.strictEqual(out.messages[0].content, "You're looping"); - }); - - it("clears hash windows after injecting a nudge, preventing immediate re-trigger", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - // Build up a loop and inject a nudge. - await run(mw, { messages: [ai("same")] }); - await run(mw, { messages: [ai("same")] }); - const out = await run(mw, { messages: [ai("same")] }); - assert.ok(out); - // The model repeats the same phrase on its next turn. Because the hash - // windows were cleared on injection, this must NOT immediately re-trigger. - assert.strictEqual(await run(mw, { messages: [ai("same")] }), undefined); - }); - - it("tracks message and reasoning content independently", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - // Same reasoning, different text: reasoning loop should trigger. - await run(mw, { messages: [ai("a", "think")] }); - await run(mw, { messages: [ai("b", "think")] }); - const out = await run(mw, { messages: [ai("c", "think")] }); - assert.ok(out); - assert.strictEqual(out.messages[0].content, "You're looping"); - }); - - it("does not false-positive on changing content", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - for (const text of ["a", "b", "c", "d", "e"]) { - assert.strictEqual(await run(mw, { messages: [ai(text)] }), undefined); - } - }); - - it("evicts the oldest hash when the window is exceeded", async () => { - const mw = createLoopDetectionMiddleware({ config: config({ windowSize: 3 }) }); - // Fill the window with distinct hashes, then repeat the first one. - // With windowSize 3, only the last 3 hashes are retained. - await run(mw, { messages: [ai("x")] }); - await run(mw, { messages: [ai("y")] }); - await run(mw, { messages: [ai("z")] }); - // "x" was evicted; repeating it should not trigger. - assert.strictEqual(await run(mw, { messages: [ai("x")] }), undefined); - }); - - it("prunes hashes older than the TTL, preventing false positives", async () => { - let clock = 0; - const mw = createLoopDetectionMiddleware({ - config: config({ ttlMs: 100 }), - now: () => clock, - }); - // Build up 3 occurrences of "same" at t=0. - await run(mw, { messages: [ai("same")] }); - await run(mw, { messages: [ai("same")] }); - await run(mw, { messages: [ai("same")] }); - // Advance past the TTL; the old hashes are pruned. - clock = 200; - // "same" again should NOT trigger, because the old occurrences expired. - assert.strictEqual(await run(mw, { messages: [ai("same")] }), undefined); - }); - - // Trigger a single loop detection (3 identical messages, since threshold is 2). - const triggerLoop = async (mw, text = "same") => { - await run(mw, { messages: [ai(text)] }); - await run(mw, { messages: [ai(text)] }); - return run(mw, { messages: [ai(text)] }); - }; - - it("escalates to onCompact after the escalation count within the window", async () => { - let compacted = 0; - const mw = createLoopDetectionMiddleware({ - config: config(), - onCompact: () => { - compacted++; - }, - }); - // 3 nudges injected (each clears the hash window), then the 4th loop - // detection escalates to compaction. - for (let i = 0; i < 3; i++) { - assert.ok(await triggerLoop(mw)); - } - // The 4th loop detection reaches the escalation count and compacts. - assert.strictEqual(await triggerLoop(mw), undefined); - assert.strictEqual(compacted, 1); - }); - - it("resets the escalation counter when the window expires", async () => { - let compacted = 0; - let clock = 0; - const mw = createLoopDetectionMiddleware({ - config: config({ escalationWindowMs: 100 }), - onCompact: () => { - compacted++; - }, - now: () => clock, - }); - // First nudge at t=0. - assert.ok(await triggerLoop(mw)); - // Advance clock past the window; nudges should be pruned. - clock = 200; - // Two more nudges in the new window (below escalationCount 3), so no compaction. - assert.ok(await triggerLoop(mw)); - assert.ok(await triggerLoop(mw)); - assert.strictEqual(compacted, 0); - }); - - it("does not compact below the escalation count", async () => { - let compacted = 0; - const mw = createLoopDetectionMiddleware({ - config: config(), - onCompact: () => { - compacted++; - }, - }); - // Only 2 nudges (below escalationCount 3). - assert.ok(await triggerLoop(mw)); - assert.ok(await triggerLoop(mw)); - assert.strictEqual(compacted, 0); - }); - - it("survives a throwing onCompact callback", async () => { - const mw = createLoopDetectionMiddleware({ - config: config(), - onCompact: () => { - throw new Error("boom"); - }, - }); - // 3 nudges, then the 4th loop detection escalates; the throwing - // onCompact must not propagate. - for (let i = 0; i < 3; i++) { - assert.ok(await triggerLoop(mw)); - } - await triggerLoop(mw); - }); - - it("maintains independent state per thread", async () => { - const mw = createLoopDetectionMiddleware({ config: config() }); - // Thread "a" loops; thread "b" does not. - await run(mw, { messages: [ai("same")] }, "a"); - await run(mw, { messages: [ai("same")] }, "a"); - const outA = await run(mw, { messages: [ai("same")] }, "a"); - assert.ok(outA); - assert.strictEqual(await run(mw, { messages: [ai("same")] }, "b"), undefined); - }); -}); diff --git a/tests/unit/stream/loopCheck.test.js b/tests/unit/stream/loopCheck.test.js new file mode 100644 index 000000000..f4b6d49bc --- /dev/null +++ b/tests/unit/stream/loopCheck.test.js @@ -0,0 +1,170 @@ +import { describe, it } from "node:test"; +import assert from "node:assert"; +import { + checkLoop, + hasDuplicateSentence, + hasRepeatedWords, + takeSentences, + takeWords, +} from "../../../src/stream/loopCheck.js"; + +describe("takeSentences", () => { + it("splits on sentence ends and newlines and returns the unfinished tail", () => { + assert.deepStrictEqual(takeSentences("One. Two? Three!\nFour and"), { + sentences: ["One.", "Two?", "Three!"], + rest: "Four and", + }); + }); + + it("collapses whitespace and skips blanks", () => { + assert.deepStrictEqual(takeSentences(" a b. \n\n c d. tail"), { + sentences: ["a b.", "c d."], + rest: "tail", + }); + }); + + it("returns nothing completed when no sentence has ended", () => { + assert.deepStrictEqual(takeSentences("ok ok ok ok"), { sentences: [], rest: "ok ok ok ok" }); + assert.deepStrictEqual(takeSentences(""), { sentences: [], rest: "" }); + }); + + it("scans only from the given index, so a boundary split across deltas is still found", () => { + const first = takeSentences("Hello there."); + assert.deepStrictEqual(first, { sentences: [], rest: "Hello there." }); + // The next delta starts with the whitespace that completes the sentence. + const joined = first.rest + " Next"; + assert.deepStrictEqual(takeSentences(joined, first.rest.length), { + sentences: ["Hello there."], + rest: "Next", + }); + }); + + it("gives the same sentences however the text is chunked", () => { + const text = "First one. Second one? Third one!\nFourth line\nFifth one. tail"; + const whole = takeSentences(text).sentences; + for (const size of [1, 2, 3, 5, 7, 11]) { + let pending = ""; + const got = []; + for (let i = 0; i < text.length; i += size) { + const delta = text.slice(i, i + size); + const out = takeSentences(pending + delta, pending.length); + pending = out.rest; + got.push(...out.sentences); + } + assert.deepStrictEqual(got, whole, `chunk size ${size}`); + } + }); +}); + +describe("takeWords", () => { + it("returns completed words and carries the unfinished one", () => { + assert.deepStrictEqual(takeWords("", "ok ok o"), { words: ["ok", "ok"], rest: "o" }); + }); + + it("joins a word split across deltas", () => { + const a = takeWords("", "hel"); + const b = takeWords(a.rest, "lo wor"); + assert.deepStrictEqual(a, { words: [], rest: "hel" }); + assert.deepStrictEqual(b, { words: ["hello"], rest: "wor" }); + }); + + it("completes the word when the delta ends in whitespace", () => { + assert.deepStrictEqual(takeWords("", "ok "), { words: ["ok"], rest: "" }); + }); + + it("gives the same words however the text is chunked", () => { + const text = "the quick brown fox jumps over the lazy dog "; + const whole = takeWords("", text).words; + for (const size of [1, 2, 3, 5, 7]) { + let partial = ""; + const got = []; + for (let i = 0; i < text.length; i += size) { + const out = takeWords(partial, text.slice(i, i + size)); + partial = out.rest; + got.push(...out.words); + } + assert.deepStrictEqual(got, whole, `chunk size ${size}`); + } + }); +}); + +describe("hasDuplicateSentence", () => { + it("flags a sentence repeated more than threshold times", () => { + assert.strictEqual( + hasDuplicateSentence(["Same thing.", "Same thing.", "Same thing."], 2), + true, + ); + assert.strictEqual(hasDuplicateSentence(["Same thing.", "Same thing."], 2), false); + }); + + it("does not need the repeats to be back-to-back", () => { + assert.strictEqual( + hasDuplicateSentence(["Alpha.", "Beta.", "Alpha.", "Gamma.", "Alpha."], 2), + true, + ); + }); + + it("does not flag distinct sentences", () => { + const list = Array.from({ length: 30 }, (_, i) => `Sentence number ${i}.`); + assert.strictEqual(hasDuplicateSentence(list, 2), false); + }); + + it("does not lowercase: a differently-cased sentence is a different sentence", () => { + assert.strictEqual(hasDuplicateSentence(["Ok then.", "ok then.", "OK THEN."], 2), false); + }); +}); + +describe("hasRepeatedWords", () => { + const w = (s) => s.split(" "); + + it("flags 'ok' repeated with no punctuation", () => { + assert.strictEqual(hasRepeatedWords(w("ok ok ok"), 2), true); + assert.strictEqual(hasRepeatedWords(w("ok ok"), 2), false); + }); + + it("flags a multi-word run repeated back-to-back", () => { + assert.strictEqual(hasRepeatedWords(w("so then we go so then we go so then we go"), 2), true); + }); + + it("flags a repeat at the tail even after unrelated words", () => { + assert.strictEqual(hasRepeatedWords(w("something else entirely ok ok ok"), 2), true); + }); + + it("does not flag a repeat that is not at the tail", () => { + assert.strictEqual(hasRepeatedWords(w("ok ok ok and then it moved on"), 2), false); + }); + + it("does not flag words with no repeat, or an empty list", () => { + assert.strictEqual( + hasRepeatedWords(w("the quick brown fox jumps over the lazy dog"), 2), + false, + ); + assert.strictEqual(hasRepeatedWords([], 2), false); + }); + + it("honours a higher threshold", () => { + assert.strictEqual(hasRepeatedWords(w("ok ok ok ok"), 4), false); + assert.strictEqual(hasRepeatedWords(w("ok ok ok ok ok"), 4), true); + }); +}); + +describe("checkLoop", () => { + it("is true for either kind of repeat and false otherwise", () => { + assert.strictEqual( + checkLoop({ sentences: [], words: Array(20).fill("ok"), threshold: 2 }), + true, + ); + assert.strictEqual( + checkLoop({ sentences: Array(5).fill("This is a test."), words: [], threshold: 2 }), + true, + ); + assert.strictEqual( + checkLoop({ + sentences: ["A one.", "B two.", "C three."], + words: ["a", "b", "c"], + threshold: 2, + }), + false, + ); + }); +}); diff --git a/tests/unit/stream/loopDetector.sample.test.js b/tests/unit/stream/loopDetector.sample.test.js new file mode 100644 index 000000000..af68537db --- /dev/null +++ b/tests/unit/stream/loopDetector.sample.test.js @@ -0,0 +1,207 @@ +/** + * Replays looping text through the real loop detector (worker pool included) the + * way a provider streams it: as multi-character deltas that split mid-word. Every + * repeat is cut at its OWN random positions, so the repeats do not share delta + * boundaries (a looping model does not re-emit identical chunks). + * + * The fixture is a TUI capture of the model repeating the same paragraph. When + * the loop happens it is one paragraph repeating, sometimes two sentences, + * sometimes one, sometimes more, and sometimes just "ok ok ok" with no + * punctuation. Each shape is replayed here. + */ + +import { describe, it, after } from "node:test"; +import assert from "node:assert"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { consumeStream } from "../../../src/stream/consume.js"; +import { createSegmentLoopDetector, destroyPool } from "../../../src/stream/loopDetector.js"; + +after(async () => { + await destroyPool(); +}); + +// Lines 14-21 of the capture: two paragraphs, hard-wrapped by the TUI. +const PASSAGE = readFileSync( + fileURLToPath(new URL("../../fixtures/loop-reasoning.md", import.meta.url)), + "utf8", +) + .split("\n") + .slice(13, 21) + .map((line) => line.trim()) + .join(" ") + .replace(/\s+/g, " ") + .trim(); + +const sentences = (text) => text.split(/(?<=[.!?])\s+/).filter(Boolean); +const FIRST_PARAGRAPH = PASSAGE.split(". Writing")[0] + "."; + +/** + * Deterministic PRNG (mulberry32). + * @param {number} seed - Seed + * @returns {() => number} Floats in [0, 1) + */ +function rng(seed) { + let a = seed; + return () => { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** + * Cut text into deltas of `min`..`max` characters at random positions. + * @param {string} text - Text to cut + * @param {number} min - Minimum delta length + * @param {number} max - Maximum delta length + * @param {() => number} rand - Random source + * @returns {string[]} Deltas whose concatenation equals `text` + */ +function cut(text, min, max, rand) { + const out = []; + for (let i = 0; i < text.length;) { + const size = min + Math.floor(rand() * (max - min + 1)); + out.push(text.slice(i, i + size)); + i += size; + } + return out; +} + +/** + * Repeat `unit` `repeats` times, cutting each copy independently. + * @param {string} unit - The repeating text + * @param {number} repeats - Number of copies + * @param {string} separator - Text between copies + * @param {number} min - Minimum delta length + * @param {number} max - Maximum delta length + * @param {number} seed - PRNG seed + * @returns {string[]} Deltas + */ +function loopDeltas(unit, repeats, separator, min, max, seed) { + const rand = rng(seed); + const out = []; + for (let r = 0; r < repeats; r++) { + out.push(...cut(r === 0 ? unit : separator + unit, min, max, rand)); + } + return out; +} + +/** + * Wrap deltas as an async stream of the given kind. A short real delay between + * deltas stands in for network latency: a worker check takes a round trip, so + * deltas arriving faster than that would finish the stream before any check + * could return. + */ +const STREAM_KINDS = { + "reasoning_content (Chat Completions)": (s) => [ + [], + "messages", + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: s } }], + ], + "reasoning block (Responses API)": (s) => [ + [], + "messages", + [{ type: "ai", text: "", content: [{ type: "reasoning", reasoning: s, id: "r1" }] }], + ], + "message text": (s) => [[], "messages", [{ type: "ai", text: s, content: [] }]], +}; + +async function run(deltas, wrap) { + const probe = { consumed: 0, closed: false }; + const stream = (async function* () { + try { + for (const d of deltas) { + probe.consumed++; + await new Promise((r) => setTimeout(r, 1)); + yield wrap(d); + } + } finally { + probe.closed = true; + } + })(); + const result = await consumeStream(stream, undefined, { + loopDetector: createSegmentLoopDetector({ windowSize: 20, threshold: 2 }), + }); + return { result, probe, total: deltas.length }; +} + +const SHAPES = [ + ["one paragraph", FIRST_PARAGRAPH, " "], + ["two paragraphs", PASSAGE, " "], + ["two sentences", sentences(FIRST_PARAGRAPH).slice(0, 2).join(" "), " "], + ["one sentence", sentences(FIRST_PARAGRAPH)[0], " "], + ["one sentence (newline separated)", sentences(FIRST_PARAGRAPH)[0], "\n"], + ["'ok ' with no punctuation", "ok", " "], + ["a short phrase with no punctuation", "let me check that", " "], +]; +const SIZES = [ + [1, 3], + [4, 10], +]; + +describe("loop detector on the captured loop, each repeat cut independently", () => { + it("loads the passage", () => { + assert.ok(PASSAGE.length > 600, `passage is ${PASSAGE.length} chars`); + assert.ok(FIRST_PARAGRAPH.length > 200); + }); + + for (const [kindLabel, wrap] of Object.entries(STREAM_KINDS)) { + describe(kindLabel, () => { + for (const [label, unit, separator] of SHAPES) { + for (const [min, max] of SIZES) { + it(`detects ${label} repeating (deltas ${min}-${max} chars)`, async () => { + for (const seed of [1]) { + const deltas = loopDeltas(unit, 5, separator, min, max, seed); + const { result, probe, total } = await run(deltas, wrap); + assert.strictEqual(result.loopDetected, true, `seed ${seed}`); + assert.ok(probe.closed, `seed ${seed}: stream closed`); + // Tiny streams end before a worker round trip can stop them early. + if (total > 60) { + assert.ok(probe.consumed < total, `seed ${seed}: stopped early`); + } + } + }); + } + } + + it("does not flag the passage streamed once", async () => { + const { result } = await run(loopDeltas(PASSAGE, 1, " ", 4, 10, 1), wrap); + assert.strictEqual(result.loopDetected, false); + }); + + it("does not flag a sentence that repeats only twice", async () => { + const { result } = await run( + loopDeltas(sentences(FIRST_PARAGRAPH)[0], 2, " ", 4, 10, 1), + wrap, + ); + assert.strictEqual(result.loopDetected, false); + }); + }); + } + + it("flags 'ok ' repeated 20 times, the case that failed in practice", async () => { + for (const [min, max] of [...SIZES, [5, 20]]) { + for (const seed of [1, 2]) { + const deltas = cut("ok ".repeat(20), min, max, rng(seed)); + const { result } = await run(deltas, STREAM_KINDS["reasoning_content (Chat Completions)"]); + assert.strictEqual(result.loopDetected, true, `deltas ${min}-${max}, seed ${seed}`); + } + } + }); + + it("flags a sentence repeated 5 times, the other case that failed in practice", async () => { + const text = "This is a test of streamed repetition. ".repeat(5); + for (const [min, max] of [...SIZES, [5, 20]]) { + for (const seed of [1, 2]) { + const { result } = await run( + cut(text, min, max, rng(seed)), + STREAM_KINDS["reasoning_content (Chat Completions)"], + ); + assert.strictEqual(result.loopDetected, true, `deltas ${min}-${max}, seed ${seed}`); + } + } + }); +}); diff --git a/tests/unit/stream/loopDetector.test.js b/tests/unit/stream/loopDetector.test.js new file mode 100644 index 000000000..0209ca223 --- /dev/null +++ b/tests/unit/stream/loopDetector.test.js @@ -0,0 +1,323 @@ +import { describe, it, after } from "node:test"; +import assert from "node:assert"; +import { createSegmentLoopDetector, destroyPool } from "../../../src/stream/loopDetector.js"; +import { checkLoop } from "../../../src/stream/loopCheck.js"; +import { consumeStream } from "../../../src/stream/consume.js"; + +after(async () => { + await destroyPool(); +}); + +/** Runner that executes the check inline, so unit tests need no thread. */ +const inline = { run: async (task) => checkLoop(task) }; + +const detector = (extra = {}) => createSegmentLoopDetector({ runner: inline, ...extra }); + +/** Push every delta, letting checks finish between pushes; return the first index flagged. */ +async function firstFlag(d, deltas, kind = "reasoning") { + for (let i = 0; i < deltas.length; i++) { + if (d.push(kind, deltas[i])) return i; + await d.settle(); + if (d.push(kind, "")) return i; // no-op push, keeps shape symmetric + } + return (await d.settle()) ? deltas.length - 1 : -1; +} + +describe("createSegmentLoopDetector (worker-backed)", () => { + it("flags 'ok' repeated with no punctuation", async () => { + const d = detector(); + const deltas = Array.from({ length: 20 }, () => "ok "); + assert.ok((await firstFlag(d, deltas)) >= 0); + }); + + it("flags a repeated sentence", async () => { + const d = detector(); + const deltas = Array.from({ length: 5 }, () => "This is a test of streamed repetition. "); + assert.ok((await firstFlag(d, deltas)) >= 0); + }); + + it("does not flag distinct text", async () => { + const d = detector(); + const deltas = Array.from({ length: 40 }, (_, i) => `Sentence number ${i} is unique. `); + assert.strictEqual(await firstFlag(d, deltas), -1); + }); + + it("ignores empty and non-string deltas", () => { + const d = detector(); + for (let i = 0; i < 10; i++) { + assert.strictEqual(d.push("reasoning", ""), false); + assert.strictEqual(d.push("reasoning", undefined), false); + assert.strictEqual(d.push("reasoning", null), false); + } + }); + + it("tracks reasoning and message text separately", async () => { + const d = detector(); + // Each kind alone has 2 copies; combined text would have 4. + for (const kind of ["reasoning", "message"]) { + d.push(kind, "Same thing. "); + d.push(kind, "Same thing. "); + } + assert.strictEqual(await d.settle(), false); + }); + + it("reports a flag on the first push after the check completes", async () => { + const d = detector(); + d.push("reasoning", "ok ok ok "); + assert.strictEqual(await d.settle(), true); + assert.strictEqual(d.push("reasoning", "more "), true); + }); + + it("keeps at most one check in flight per kind", async () => { + let running = 0; + let peak = 0; + const slow = { + run: async (task) => { + running++; + peak = Math.max(peak, running); + await new Promise((r) => setTimeout(r, 5)); + running--; + return checkLoop(task); + }, + }; + const d = createSegmentLoopDetector({ runner: slow }); + for (let i = 0; i < 30; i++) d.push("reasoning", `word${i} `); + await d.settle(); + assert.strictEqual(peak, 1); + }); + + it("re-checks text that arrived while a check was running", async () => { + const seen = []; + const slow = { + run: async (task) => { + seen.push(task); + await new Promise((r) => setTimeout(r, 5)); + return checkLoop(task); + }, + }; + const d = createSegmentLoopDetector({ runner: slow }); + d.push("reasoning", "ok "); + d.push("reasoning", "ok "); + d.push("reasoning", "ok "); + assert.strictEqual(await d.settle(), true); + assert.ok(seen.length >= 2, "the tail was checked again"); + assert.ok(seen.at(-1).words.length > seen[0].words.length, "the later check saw newer words"); + }); + + it("treats a failing check as not looping and keeps streaming", async () => { + const broken = { + run: async () => { + throw new Error("worker died"); + }, + }; + const d = createSegmentLoopDetector({ runner: broken }); + assert.strictEqual(d.push("reasoning", "ok ok ok ok "), false); + assert.strictEqual(await d.settle(), false); + }); + + it("drops accumulated text older than ttlMs", async () => { + let clock = 0; + const d = detector({ ttlMs: 100, now: () => clock }); + d.push("reasoning", "ok ok "); + await d.settle(); + clock = 500; + d.push("reasoning", "ok "); + assert.strictEqual(await d.settle(), false); + }); + + it("does not expire text when ttlMs is unset", async () => { + let clock = 0; + const d = detector({ now: () => clock }); + d.push("reasoning", "ok ok "); + await d.settle(); + clock = 10_000_000; + d.push("reasoning", "ok "); + assert.strictEqual(await d.settle(), true); + }); + + it("keeps only the last windowSize sentences and wordWindowSize words", async () => { + const seen = []; + const spy = { + run: async (task) => { + seen.push(task); + return false; + }, + }; + const d = createSegmentLoopDetector({ runner: spy, windowSize: 5, wordWindowSize: 12 }); + for (let i = 0; i < 100; i++) { + d.push("reasoning", `Unique sentence number ${i} here. `); + await d.settle(); + } + const last = seen.at(-1); + assert.ok(last.sentences.length <= 5, `sentences ${last.sentences.length}`); + assert.ok(last.words.length <= 12, `words ${last.words.length}`); + assert.strictEqual(last.sentences.at(-1), "Unique sentence number 99 here."); + }); + + it("sends the worker a bounded task however long the stream gets", async () => { + const sizes = []; + const spy = { + run: async (task) => { + sizes.push(task.sentences.length + task.words.length); + return false; + }, + }; + const d = createSegmentLoopDetector({ runner: spy, windowSize: 20, wordWindowSize: 200 }); + for (let i = 0; i < 2000; i++) { + d.push("reasoning", `Distinct sentence ${i} goes here. `); + if (i % 50 === 0) await d.settle(); + } + await d.settle(); + assert.ok(Math.max(...sizes) <= 220, `max task size ${Math.max(...sizes)}`); + }); + + it("does not start a check for a delta that completes no sentence or word", async () => { + let calls = 0; + const spy = { + run: async () => { + calls++; + return false; + }, + }; + const d = createSegmentLoopDetector({ runner: spy }); + d.push("reasoning", "hel"); + d.push("reasoning", "lo"); + d.push("reasoning", " wor"); + await d.settle(); + const afterFirstWord = calls; + d.push("reasoning", "l"); + d.push("reasoning", "d"); + await d.settle(); + assert.strictEqual(calls, afterFirstWord, "no new check until a word completes"); + }); + + it("finds a sentence whose boundary falls across two deltas", async () => { + const d = detector(); + for (const delta of ["Same thing.", " ", "Same thing.", " ", "Same thing.", " ", "x"]) { + d.push("reasoning", delta); + } + assert.strictEqual(await d.settle(), true); + }); + + it("reaches the same verdict however the stream is chunked", async () => { + const text = "This is a test of streamed repetition. ".repeat(5); + for (const size of [1, 2, 3, 5, 8, 13]) { + const d = detector(); + for (let i = 0; i < text.length; i += size) d.push("reasoning", text.slice(i, i + size)); + assert.strictEqual(await d.settle(), true, `chunk size ${size}`); + } + }); + + it("drops state older than ttlMs for that kind only", async () => { + let clock = 0; + const d = detector({ ttlMs: 100, now: () => clock }); + d.push("reasoning", "ok ok "); + await d.settle(); + clock = 50; + d.push("message", "fresh "); + clock = 500; + d.push("reasoning", "ok "); + assert.strictEqual(await d.settle(), false); + }); + + it("reset clears state and discards a stale in-flight result", async () => { + const slow = { + run: async (task) => { + await new Promise((r) => setTimeout(r, 10)); + return checkLoop(task); + }, + }; + const d = createSegmentLoopDetector({ runner: slow }); + d.push("reasoning", "ok ok ok "); + d.reset(); + assert.strictEqual(await d.settle(), false); + assert.strictEqual(d.push("reasoning", "fresh start "), false); + }); + + it("runs the check in a real worker thread by default", async () => { + const d = createSegmentLoopDetector(); + d.push("reasoning", "ok ok ok ok "); + assert.strictEqual(await d.settle(), true); + }); +}); + +describe("consumeStream with loopDetector", () => { + const reasoning = (text) => [ + [], + "messages", + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: text } }], + ]; + const message = (text) => [[], "messages", [{ type: "ai", text, content: [] }]]; + + it("stops consuming, closes the stream and reports loopDetected", async () => { + let closed = false; + let consumed = 0; + const stream = (async function* () { + try { + for (let i = 0; i < 400; i++) { + consumed++; + await new Promise((r) => setImmediate(r)); + yield reasoning("ok "); + } + } finally { + closed = true; + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, true); + assert.ok(consumed < 400, `stopped early (${consumed})`); + assert.ok(closed, "underlying stream is closed"); + }); + + it("collects a loop flagged by a check still running when the stream ends", async () => { + const stream = (async function* () { + yield reasoning("ok "); + yield reasoning("ok "); + yield reasoning("ok "); + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, true); + }); + + it("detects a looping message stream", async () => { + const stream = (async function* () { + for (let i = 0; i < 20; i++) yield message("again "); + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, true); + }); + + it("detects a looping Responses API reasoning block stream", async () => { + const block = (text) => [ + [], + "messages", + [{ type: "ai", text: "", content: [{ type: "reasoning", reasoning: text, id: "r1" }] }], + ]; + const stream = (async function* () { + for (let i = 0; i < 20; i++) yield block("think "); + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, true); + }); + + it("does not feed subagent events to the detector", async () => { + const stream = (async function* () { + for (let i = 0; i < 20; i++) { + yield [ + ["tools:abc"], + "messages", + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: "same " } }], + ]; + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, false); + }); + + it("reports loopDetected false without a detector", async () => { + const stream = (async function* () { + yield reasoning("hello"); + })(); + assert.strictEqual((await consumeStream(stream)).loopDetected, false); + }); +}); diff --git a/tests/unit/stream/loopDetectorMessageEnd.test.js b/tests/unit/stream/loopDetectorMessageEnd.test.js new file mode 100644 index 000000000..3940eab6e --- /dev/null +++ b/tests/unit/stream/loopDetectorMessageEnd.test.js @@ -0,0 +1,196 @@ +/** + * A message turn is: start, stream, end. When it ends the detector's word and + * sentence lists are cleared and nulled, so a long run of hundreds of messages + * never accumulates state. The end is the chunk carrying a `finish_reason`, a + * different message id arriving, or a tool result following the AI message. + */ + +import { describe, it, after } from "node:test"; +import assert from "node:assert"; +import { AIMessageChunk, ToolMessage } from "@langchain/core/messages"; +import { createSegmentLoopDetector, destroyPool } from "../../../src/stream/loopDetector.js"; +import { checkLoop } from "../../../src/stream/loopCheck.js"; +import { consumeStream } from "../../../src/stream/consume.js"; + +after(async () => { + await destroyPool(); +}); + +const inline = { run: async (task) => checkLoop(task) }; +const ev = (msg) => [[], "messages", [msg, {}]]; +const think = (id, text, extra = {}) => + new AIMessageChunk({ id, content: "", additional_kwargs: { reasoning_content: text }, ...extra }); +const say = (id, text, extra = {}) => new AIMessageChunk({ id, content: text, ...extra }); + +/** A detector that records how many streams it held each time something was pushed. */ +function watched() { + const d = createSegmentLoopDetector({ runner: inline }); + const sizes = []; + const push = d.push.bind(d); + d.push = (...args) => { + const out = push(...args); + sizes.push(d.size()); + return out; + }; + const resets = []; + const reset = d.reset.bind(d); + d.reset = () => { + resets.push(d.size()); + return reset(); + }; + return { d, sizes, resets }; +} + +describe("end of a message turn releases the lists", () => { + it("releases when the last chunk carries a finish_reason", async () => { + const { d, resets } = watched(); + const stream = (async function* () { + yield ev(think("A", "plan it. ")); + yield ev(say("A", "done", { response_metadata: { finish_reason: "stop" } })); + yield ev(think("B", "next message. ")); + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + assert.ok( + resets.length >= 2, + `released at the message end and at the stream end (${resets.length})`, + ); + assert.ok(resets[0] > 0, "the first release happened while the message's state existed"); + }); + + it("releases when a different message id arrives", async () => { + const { d, sizes } = watched(); + const stream = (async function* () { + yield ev(think("A", "first thought. ")); + yield ev(think("B", "second thought. ")); // new id, no finish_reason on A + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + // After B's delta the detector holds only B, because A was released at the id change. + assert.strictEqual(sizes.at(-1), 1); + }); + + it("releases when a tool result follows the AI message", async () => { + const { d, resets } = watched(); + const stream = (async function* () { + yield ev(think("A", "calling a tool. ")); + yield ev(new ToolMessage({ content: "result", tool_call_id: "t1" })); + yield ev(say("B", "answer")); + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + assert.ok(resets.length >= 2); + }); + + it("never accumulates across hundreds of sequential messages", async () => { + const { d, sizes } = watched(); + const stream = (async function* () { + for (let i = 0; i < 400; i++) { + yield ev(think(`m${i}`, `reasoning for message ${i}. `)); + yield ev(say(`m${i}`, `answer ${i}`, { response_metadata: { finish_reason: "stop" } })); + } + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + assert.ok(Math.max(...sizes) <= 2, `peak tracked ${Math.max(...sizes)}`); + }); + + it("does not carry one message's words into the next", async () => { + const tasks = []; + const spy = { + run: async (task) => { + tasks.push({ words: [...task.words] }); + return false; + }, + }; + const d = createSegmentLoopDetector({ runner: spy }); + const stream = (async function* () { + yield ev(think("A", "alpha beta gamma delta ")); + yield ev(say("A", "x", { response_metadata: { finish_reason: "stop" } })); + yield ev(think("B", "epsilon zeta ")); + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + const sawBoth = tasks.some((t) => t.words.includes("alpha") && t.words.includes("epsilon")); + assert.strictEqual(sawBoth, false, "no check mixed message A and message B"); + }); +}); + +describe("a loop is still caught across message boundaries", () => { + it("flags a loop inside one message", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + for (let i = 0; i < 200; i++) { + await new Promise((r) => setTimeout(r, 1)); + yield ev(think("A", "ok ")); + } + })(); + assert.strictEqual( + (await consumeStream(stream, undefined, { loopDetector: d })).loopDetected, + true, + ); + }); + + it("keeps the verdict of a loop in the last deltas of a message that ends right after", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + yield ev(think("A", "ok ")); + yield ev(think("A", "ok ")); + yield ev(think("A", "ok ", { response_metadata: { finish_reason: "stop" } })); + })(); + const result = await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(result.loopDetected, true, "end() read the verdict before releasing"); + }); + + it("does not treat the same short answer repeated across separate messages as one loop", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + for (let i = 0; i < 6; i++) { + yield ev(say(`m${i}`, "ok ", { response_metadata: { finish_reason: "stop" } })); + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(result.loopDetected, false); + }); + + it("ignores subagent messages when deciding where a message ends", async () => { + const { d, resets } = watched(); + const stream = (async function* () { + yield ev(think("A", "main thought. ")); + yield [ + ["tools:abc"], + "messages", + [say("S", "sub", { response_metadata: { finish_reason: "stop" } }), {}], + ]; + yield ev(think("A", "still the same message. ")); + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(resets.length, 1, "only the final stream-end release"); + }); + + it("still resets once at the end of the stream", async () => { + const { d, resets } = watched(); + const stream = (async function* () { + yield ev(think("A", "only message. ")); + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(resets.length, 1); + assert.strictEqual(d.size(), 0); + }); +}); + +describe("end()", () => { + it("returns the verdict and releases", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + for (let i = 0; i < 5; i++) d.push("reasoning", "ok ", "a"); + assert.strictEqual(await d.end(), true); + assert.strictEqual(d.size(), 0); + }); + + it("returns false and releases when nothing looped", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + d.push("reasoning", "a fine thought. ", "a"); + assert.strictEqual(await d.end(), false); + assert.strictEqual(d.size(), 0); + }); + + it("can be called on an empty detector", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + assert.strictEqual(await d.end(), false); + }); +}); diff --git a/tests/unit/stream/loopDetectorStreams.test.js b/tests/unit/stream/loopDetectorStreams.test.js new file mode 100644 index 000000000..4d127511c --- /dev/null +++ b/tests/unit/stream/loopDetectorStreams.test.js @@ -0,0 +1,250 @@ +/** + * Detection is restricted to the single stream a delta belongs to. Streams + * can stream concurrently, so deltas from different streams interleave; state + * keyed only by kind would mix them. These tests interleave streams on purpose. + */ + +import { describe, it, after } from "node:test"; +import assert from "node:assert"; +import { createSegmentLoopDetector, destroyPool } from "../../../src/stream/loopDetector.js"; +import { checkLoop } from "../../../src/stream/loopCheck.js"; +import { consumeStream } from "../../../src/stream/consume.js"; + +after(async () => { + await destroyPool(); +}); + +const inline = { run: async (task) => checkLoop(task) }; +const detector = (extra = {}) => createSegmentLoopDetector({ runner: inline, ...extra }); + +describe("per-stream detection", () => { + it("flags a loop inside one stream", async () => { + const d = detector(); + for (let i = 0; i < 5; i++) d.push("reasoning", "ok ", "a"); + assert.strictEqual(await d.settle(), true); + }); + + it("does not flag when each stream repeats only below the threshold", async () => { + const d = detector(); + // 2 copies in stream a and 2 in stream b: 4 combined would trip a shared list. + for (const id of ["a", "b"]) { + d.push("reasoning", "Same thing. ", id); + d.push("reasoning", "Same thing. ", id); + } + assert.strictEqual(await d.settle(), false); + }); + + it("is not hidden by another stream interleaving into the stream", async () => { + const d = detector(); + // Stream a loops on 'ok'; stream b streams unrelated words between every delta. + // A shared word list would see 'ok w1 ok w2 ok w3 ...' and never find a repeat. + for (let i = 0; i < 6; i++) { + d.push("reasoning", "ok ", "a"); + d.push("reasoning", `unrelated${i} `, "b"); + } + assert.strictEqual(await d.settle(), true); + }); + + it("does not invent a loop from two streams that merely alternate", async () => { + const d = detector(); + // a says 'x y' and b says 'x y', interleaved, so a shared list reads 'x x y y x x y y'. + for (let i = 0; i < 4; i++) { + d.push("reasoning", "x ", `a${i}`); + d.push("reasoning", "y ", `b${i}`); + } + assert.strictEqual(await d.settle(), false); + }); + + it("keeps reasoning and message with the same id separate", async () => { + const d = detector(); + for (let i = 0; i < 2; i++) { + d.push("reasoning", "Same thing. ", "0"); + d.push("message", "Same thing. ", "0"); + } + assert.strictEqual(await d.settle(), false); + }); + + it("keys deltas with no id by kind alone", async () => { + const d = detector(); + for (let i = 0; i < 5; i++) d.push("reasoning", "ok "); + assert.strictEqual(await d.settle(), true); + }); + + it("treats a numeric id and its string form as the same stream", async () => { + const d = detector(); + d.push("message", "ok ", 0); + d.push("message", "ok ", "0"); + d.push("message", "ok ", 0); + assert.strictEqual(await d.settle(), true); + }); + + it("treats id 0 as a real stream, not as 'no id'", async () => { + const d = detector(); + for (let i = 0; i < 3; i++) d.push("reasoning", "ok ", 0); + for (let i = 0; i < 3; i++) d.push("reasoning", "different "); + assert.strictEqual(await d.settle(), true, "stream 0 looped"); + }); + + it("a word split across deltas stays inside its own stream", async () => { + const d = detector(); + // Stream a is mid-word when stream b's delta arrives. + d.push("reasoning", "ok", "a"); + d.push("reasoning", "zzz ", "b"); + d.push("reasoning", " ok", "a"); + d.push("reasoning", " ok ", "a"); + assert.strictEqual(await d.settle(), true); + }); + + it("runs one check per stream, so concurrent streams check at the same time", async () => { + let running = 0; + let peak = 0; + const slow = { + run: async (task) => { + running++; + peak = Math.max(peak, running); + await new Promise((r) => setTimeout(r, 10)); + running--; + return checkLoop(task); + }, + }; + const d = createSegmentLoopDetector({ runner: slow }); + d.push("reasoning", "alpha ", "a"); + d.push("reasoning", "beta ", "b"); + d.push("reasoning", "gamma ", "c"); + await d.settle(); + assert.strictEqual(peak, 3); + }); + + it("still allows only one check at a time within a stream", async () => { + let running = 0; + let peak = 0; + const slow = { + run: async (task) => { + running++; + peak = Math.max(peak, running); + await new Promise((r) => setTimeout(r, 5)); + running--; + return checkLoop(task); + }, + }; + const d = createSegmentLoopDetector({ runner: slow }); + for (let i = 0; i < 30; i++) d.push("reasoning", `word${i} `, "a"); + await d.settle(); + assert.strictEqual(peak, 1); + }); +}); + +describe("tracked stream limit", () => { + it("evicts the least recently updated stream past maxStreams", async () => { + const d = detector({ maxStreams: 2 }); + d.push("reasoning", "ok ok ", "old"); + d.push("reasoning", "x ", "b"); + d.push("reasoning", "y ", "c"); // evicts 'old' + d.push("reasoning", "ok ", "old"); // starts over: only one 'ok' + assert.strictEqual(await d.settle(), false); + }); + + it("does not evict a stream that keeps receiving deltas", async () => { + const d = detector({ maxStreams: 2 }); + d.push("reasoning", "ok ", "busy"); + d.push("reasoning", "x ", "b"); + d.push("reasoning", "ok ", "busy"); // refreshes 'busy' + d.push("reasoning", "y ", "c"); // evicts 'b', not 'busy' + d.push("reasoning", "ok ", "busy"); + assert.strictEqual(await d.settle(), true); + }); + + it("bounds memory across many short-lived streams", async () => { + const seen = new Set(); + const spy = { + run: async (task) => { + seen.add(task); + return false; + }, + }; + const d = createSegmentLoopDetector({ runner: spy, maxStreams: 16 }); + for (let i = 0; i < 5000; i++) d.push("message", "hello there ", i); + await d.settle(); + assert.ok(seen.size > 0); + }); + + it("drops a stream's state after ttlMs without touching the others", async () => { + let clock = 0; + const d = detector({ ttlMs: 100, now: () => clock }); + d.push("reasoning", "ok ok ", "a"); + clock = 50; + d.push("reasoning", "ok ok ", "b"); + clock = 120; // a is stale (120 > 100), b is not (70) + d.push("reasoning", "ok ", "a"); + d.push("reasoning", "ok ", "b"); + assert.strictEqual(await d.settle(), true, "b kept its history and reached 3"); + }); +}); + +describe("consumeStream passes each stream's id to the detector", () => { + const reasoningBlock = (id, text) => [ + [], + "messages", + [{ type: "ai", text: "", content: [{ type: "reasoning", reasoning: text, id }] }], + ]; + const message = (index, text) => [[], "messages", [{ type: "ai", text, content: [{ index }] }]]; + const tick = () => new Promise((r) => setTimeout(r, 2)); + + it("flags a loop in one Responses API stream while another stream interleaves", async () => { + const stream = (async function* () { + for (let i = 0; i < 20; i++) { + await tick(); + yield reasoningBlock("r1", "ok "); + yield reasoningBlock("r2", `different${i} `); + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, true); + }); + + it("does not flag two streams whose combined text would repeat", async () => { + const stream = (async function* () { + for (let i = 0; i < 4; i++) { + yield reasoningBlock(`a${i}`, "x "); + yield reasoningBlock(`b${i}`, "y "); + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, false); + }); + + it("keys message text by its content-block index", async () => { + const stream = (async function* () { + for (let i = 0; i < 20; i++) { + await tick(); + yield message(0, "again "); + yield message(1, `other${i} `); + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, true); + }); + + it("does not flag message text split across different block indexes", async () => { + const stream = (async function* () { + for (let i = 0; i < 6; i++) yield message(i, "same "); + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, false); + }); + + it("still detects a Chat Completions reasoning_content loop that has no id", async () => { + const stream = (async function* () { + for (let i = 0; i < 20; i++) { + await tick(); + yield [ + [], + "messages", + [{ type: "ai", text: "", additional_kwargs: { reasoning_content: "ok " } }], + ]; + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: detector() }); + assert.strictEqual(result.loopDetected, true); + }); +}); diff --git a/tests/unit/stream/loopDetectorTurnEnd.test.js b/tests/unit/stream/loopDetectorTurnEnd.test.js new file mode 100644 index 000000000..51f9d4d96 --- /dev/null +++ b/tests/unit/stream/loopDetectorTurnEnd.test.js @@ -0,0 +1,193 @@ +/** + * When a message turn ends, the detector's word and sentence lists are cleared + * and nulled so nothing from the turn is retained, however the turn ends. + */ + +import { describe, it, after } from "node:test"; +import assert from "node:assert"; +import { createSegmentLoopDetector, destroyPool } from "../../../src/stream/loopDetector.js"; +import { checkLoop } from "../../../src/stream/loopCheck.js"; +import { consumeStream } from "../../../src/stream/consume.js"; + +after(async () => { + await destroyPool(); +}); + +const inline = { run: async (task) => checkLoop(task) }; + +/** Runner that captures the task lists it was handed, so tests can see what the detector released. */ +function capturing() { + const tasks = []; + return { + tasks, + run: async (task) => { + tasks.push(task); + return false; + }, + }; +} + +const reasoning = (text, id) => [ + [], + "messages", + [{ type: "ai", text: "", content: [{ type: "reasoning", reasoning: text, id }] }], +]; + +describe("release on reset", () => { + it("empties every stream and reports none tracked", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + d.push("reasoning", "alpha beta gamma. ", "a"); + d.push("message", "delta epsilon. ", 0); + await d.settle(); + assert.strictEqual(d.size(), 2); + d.reset(); + assert.strictEqual(d.size(), 0); + }); + + it("clears and nulls the lists held by an in-flight check's state", async () => { + let release; + const gate = new Promise((r) => { + release = r; + }); + let captured; + const held = { + run: async (task) => { + captured = task; + await gate; + return true; // would flag a loop if the answer were honoured + }, + }; + const d = createSegmentLoopDetector({ runner: held }); + d.push("reasoning", "one two three. ", "a"); + d.reset(); // turn ends while the check is still running + release(); + assert.strictEqual(await d.settle(), false, "a released stream's late answer is ignored"); + assert.ok(captured.words.length > 0, "the task copy the worker held is its own, untouched"); + }); + + it("does not carry state into the next turn", async () => { + const spy = capturing(); + const d = createSegmentLoopDetector({ runner: spy }); + d.push("reasoning", "first turn words here ", "a"); + await d.settle(); + d.reset(); + d.push("reasoning", "second ", "a"); + await d.settle(); + const last = spy.tasks.at(-1); + assert.deepStrictEqual(last.words, ["second"].slice(0, last.words.length)); + assert.ok(!last.words.includes("first"), "no words from the previous turn"); + }); + + it("can be reset more than once and reused", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + d.reset(); + d.reset(); + for (let i = 0; i < 5; i++) d.push("reasoning", "ok ", "a"); + assert.strictEqual(await d.settle(), true); + }); + + it("reset on an empty detector is a no-op", () => { + const d = createSegmentLoopDetector({ runner: inline }); + assert.doesNotThrow(() => d.reset()); + assert.strictEqual(d.size(), 0); + }); +}); + +describe("consumeStream resets the detector when the turn ends", () => { + it("after a normal turn", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + yield reasoning("a fine thought. ", "r1"); + yield reasoning("another one. ", "r2"); + })(); + await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(d.size(), 0); + }); + + it("after the stream is abandoned on a detected loop", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + for (let i = 0; i < 200; i++) { + await new Promise((r) => setTimeout(r, 1)); + yield reasoning("ok ", "r1"); + } + })(); + const result = await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(result.loopDetected, true); + assert.strictEqual(d.size(), 0); + }); + + it("after a clarify interrupt", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + yield reasoning("thinking first. ", "r1"); + yield [ + [], + "values", + { __interrupt__: [{ value: { question: "?", choices: [] }, id: "i1" }] }, + ]; + })(); + const result = await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(result.interrupted, true); + assert.strictEqual(d.size(), 0); + }); + + it("when the stream throws", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + yield reasoning("partial thought. ", "r1"); + throw new Error("network dropped"); + })(); + await assert.rejects( + () => consumeStream(stream, undefined, { loopDetector: d }), + /network dropped/, + ); + assert.strictEqual(d.size(), 0); + }); + + it("still returns the loop verdict even though state is released", async () => { + const d = createSegmentLoopDetector({ runner: inline }); + const stream = (async function* () { + for (let i = 0; i < 5; i++) yield reasoning("ok ", "r1"); + })(); + const result = await consumeStream(stream, undefined, { loopDetector: d }); + assert.strictEqual(result.loopDetected, true); + }); + + it("works when no detector is passed", async () => { + const stream = (async function* () { + yield reasoning("hello. ", "r1"); + })(); + const result = await consumeStream(stream); + assert.strictEqual(result.loopDetected, false); + }); +}); + +describe("maxStreams of 4", () => { + it("holds two concurrent streams, the realistic edge case, without eviction", async () => { + const d = createSegmentLoopDetector({ runner: inline, maxStreams: 4 }); + for (let i = 0; i < 6; i++) { + d.push("reasoning", "ok ", "a"); + d.push("message", `distinct${i} `, 0); + } + assert.strictEqual(await d.settle(), true, "the looping stream was never evicted"); + assert.strictEqual(d.size(), 2); + }); + + it("never tracks more than 4 across hundreds of sequential streams", async () => { + const d = createSegmentLoopDetector({ runner: inline, maxStreams: 4 }); + let peak = 0; + for (let i = 0; i < 500; i++) { + d.push("message", `message number ${i} here. `, i); + peak = Math.max(peak, d.size()); + } + assert.ok(peak <= 4, `peak ${peak}`); + }); + + it("still catches a loop in the stream that is active while old ones age out", async () => { + const d = createSegmentLoopDetector({ runner: inline, maxStreams: 4 }); + for (let i = 0; i < 50; i++) d.push("message", `old message ${i}. `, i); + for (let i = 0; i < 5; i++) d.push("message", "again ", 999); + assert.strictEqual(await d.settle(), true); + }); +}); diff --git a/tests/unit/stream/loopDetectorWindow.test.js b/tests/unit/stream/loopDetectorWindow.test.js new file mode 100644 index 000000000..c4a7223bf --- /dev/null +++ b/tests/unit/stream/loopDetectorWindow.test.js @@ -0,0 +1,112 @@ +import { describe, it, after } from "node:test"; +import assert from "node:assert"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { createSegmentLoopDetector, destroyPool } from "../../../src/stream/loopDetector.js"; +import { LoopDetectionSchema } from "../../../src/config/schemas/loopDetection.js"; + +after(async () => { + await destroyPool(); +}); + +// Lines 14-21 of the capture: two paragraphs, hard-wrapped by the TUI. +const PASSAGE = readFileSync( + fileURLToPath(new URL("../../fixtures/loop-reasoning.md", import.meta.url)), + "utf8", +) + .split("\n") + .slice(13, 21) + .map((line) => line.trim()) + .join(" ") + .replace(/\s+/g, " ") + .trim(); + +const WORDS = PASSAGE.split(" "); + +/** Stream a unit of WORDS-only text (no sentence ends) so only the word check can catch it. */ +async function flaggedWithWordWindow(wordWindowSize, copies) { + const d = createSegmentLoopDetector({ wordWindowSize }); + // Strip sentence terminators and newlines so the sentence check cannot fire. + const unit = WORDS.map((w) => w.replace(/[.!?]/g, "")).join(" ") + " "; + for (let c = 0; c < copies; c++) d.push("reasoning", unit); + return d.settle(); +} + +describe("loopDetection config schema", () => { + it("defaults wordWindowSize to 1000", () => { + assert.strictEqual(LoopDetectionSchema.parse({}).wordWindowSize, 1000); + }); + + it("accepts a configured wordWindowSize", () => { + assert.strictEqual(LoopDetectionSchema.parse({ wordWindowSize: 2500 }).wordWindowSize, 2500); + }); + + it("rejects a zero, negative or fractional wordWindowSize", () => { + for (const bad of [0, -5, 1.5]) { + assert.throws( + () => LoopDetectionSchema.parse({ wordWindowSize: bad }), + `wordWindowSize ${bad}`, + ); + } + }); + + it("still rejects unknown keys (strict)", () => { + assert.throws(() => LoopDetectionSchema.parse({ wordWindow: 5 })); + }); + + it("keeps the other defaults", () => { + const cfg = LoopDetectionSchema.parse({}); + assert.strictEqual(cfg.maxStreams, 4); + assert.strictEqual(cfg.windowSize, 20); + assert.strictEqual(cfg.threshold, 2); + assert.strictEqual(cfg.ttlMs, 60000); + assert.strictEqual(cfg.escalationCount, 3); + assert.strictEqual(cfg.escalationWindowMs, 60000); + }); +}); + +describe("loopDetection maxStreams", () => { + it("defaults to 4", () => { + assert.strictEqual(LoopDetectionSchema.parse({}).maxStreams, 4); + }); + + it("accepts a configured value", () => { + assert.strictEqual(LoopDetectionSchema.parse({ maxStreams: 5000 }).maxStreams, 5000); + }); + + it("rejects zero, negative and fractional values", () => { + for (const bad of [0, -1, 2.5]) { + assert.throws(() => LoopDetectionSchema.parse({ maxStreams: bad }), `maxStreams ${bad}`); + } + }); + + it("is set in config.yaml to the schema default", () => { + const yaml = readFileSync( + fileURLToPath(new URL("../../../config.yaml", import.meta.url)), + "utf8", + ); + assert.match(yaml, /^\s+maxStreams: 4$/m); + }); +}); + +describe("wordWindowSize is big enough for the captured loop", () => { + it("the two-paragraph passage needs more words than the old 200-word window", () => { + assert.ok(WORDS.length * 3 > 200, `3 copies = ${WORDS.length * 3} words`); + }); + + it("the default 1000 holds three copies of the two-paragraph passage", () => { + assert.ok(WORDS.length * 3 <= 1000, `3 copies = ${WORDS.length * 3} words`); + }); + + it("catches the two-paragraph loop with no punctuation at the default size", async () => { + assert.strictEqual(await flaggedWithWordWindow(1000, 4), true); + }); + + it("misses the same loop when the window is too small to hold three copies", async () => { + assert.strictEqual(await flaggedWithWordWindow(200, 4), false); + }); + + it("catches it again once the window just holds three copies", async () => { + assert.strictEqual(await flaggedWithWordWindow(WORDS.length * 3, 4), true); + }); +}); diff --git a/tests/unit/tui/conversationArea.test.js b/tests/unit/tui/conversationArea.test.js index 6df2c84ef..9e00a383c 100644 --- a/tests/unit/tui/conversationArea.test.js +++ b/tests/unit/tui/conversationArea.test.js @@ -41,6 +41,180 @@ describe("countToolMessageTokens", () => { }); }); +describe("shouldContinueAfterCompact", () => { + it("continues after a successful compaction", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual( + mod.shouldContinueAfterCompact({ + compacted: true, + autoContinueCount: 0, + autoContinueLimit: 1000, + }), + true, + ); + }); + + it("does not continue after a failed compaction", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + for (const compacted of [false, undefined, null]) { + assert.strictEqual( + mod.shouldContinueAfterCompact({ + compacted, + autoContinueCount: 0, + autoContinueLimit: 1000, + }), + false, + `compacted ${compacted}`, + ); + } + }); + + it("stops once the shared auto-continue limit is reached", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual( + mod.shouldContinueAfterCompact({ + compacted: true, + autoContinueCount: 999, + autoContinueLimit: 1000, + }), + true, + ); + assert.strictEqual( + mod.shouldContinueAfterCompact({ + compacted: true, + autoContinueCount: 1000, + autoContinueLimit: 1000, + }), + false, + ); + }); +}); + +describe("decideLoopAction", () => { + const base = { + loopDetected: true, + streamingMsgId: "3", + lastMessageId: "3", + nudges: [], + now: 100000, + escalationCount: 3, + escalationWindowMs: 60000, + }; + + it("nudges when the detector flagged the last bubble and there are no recent nudges", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.deepStrictEqual(mod.decideLoopAction(base), { action: "nudge", nudges: [100000] }); + }); + + it("does nothing when no loop was detected", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual(mod.decideLoopAction({ ...base, loopDetected: false }).action, "none"); + assert.strictEqual(mod.decideLoopAction({ ...base, loopDetected: undefined }).action, "none"); + }); + + it("does nothing when the streaming bubble is not the last message", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual(mod.decideLoopAction({ ...base, lastMessageId: "4" }).action, "none"); + }); + + it("does nothing when the turn is paused on an interrupt, even if it looped", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + // A paused turn is waiting for the user's answer: no nudge, no compaction. + const out = mod.decideLoopAction({ ...base, interrupted: true, nudges: [90000, 95000] }); + assert.strictEqual(out.action, "none"); + assert.deepStrictEqual(out.nudges, [90000, 95000], "no strike is recorded"); + }); + + it("treats an undefined interrupted flag as not interrupted", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual(mod.decideLoopAction({ ...base, interrupted: undefined }).action, "nudge"); + }); + + it("strike 1 nudges", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + const out = mod.decideLoopAction({ ...base, nudges: [] }); + assert.strictEqual(out.action, "nudge"); + assert.deepStrictEqual(out.nudges, [100000]); + }); + + it("strike 2 nudges and records both strikes", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + const out = mod.decideLoopAction({ ...base, nudges: [95000] }); + assert.strictEqual(out.action, "nudge"); + assert.deepStrictEqual(out.nudges, [95000, 100000]); + }); + + it("strike 3 compacts instead of nudging: three strikes and you're out", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + const out = mod.decideLoopAction({ ...base, nudges: [90000, 95000] }); + assert.strictEqual(out.action, "compact"); + assert.deepStrictEqual(out.nudges, [], "the counter resets after compaction"); + }); + + it("plays out strikes 1 to 3 in order", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + let nudges = []; + const actions = []; + for (const now of [100000, 110000, 120000]) { + const out = mod.decideLoopAction({ ...base, nudges, now }); + actions.push(out.action); + nudges = out.nudges; + } + assert.deepStrictEqual(actions, ["nudge", "nudge", "compact"]); + }); + + it("starts counting again after a compaction", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + const afterCompact = mod.decideLoopAction({ ...base, nudges: [90000, 95000] }).nudges; + const next = mod.decideLoopAction({ ...base, nudges: afterCompact, now: 130000 }); + assert.strictEqual(next.action, "nudge", "the next loop is strike 1 again"); + }); + + it("drops strikes older than the window so a slow loop does not escalate", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + // Two earlier strikes are outside the 60s window, so this is strike 1 again. + const out = mod.decideLoopAction({ ...base, nudges: [10000, 20000] }); + assert.strictEqual(out.action, "nudge"); + assert.deepStrictEqual(out.nudges, [100000]); + }); + + it("counts only the strikes inside the window", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + // One old strike, one recent: this loop is only strike 2. + const out = mod.decideLoopAction({ ...base, nudges: [10000, 95000] }); + assert.strictEqual(out.action, "nudge"); + assert.deepStrictEqual(out.nudges, [95000, 100000]); + }); + + it("treats a strike exactly at the window edge as still inside it", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + const out = mod.decideLoopAction({ ...base, nudges: [40000, 41000] }); + assert.strictEqual(out.action, "compact"); + }); + + it("honours a different escalationCount", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + assert.strictEqual(mod.decideLoopAction({ ...base, escalationCount: 1 }).action, "compact"); + assert.strictEqual( + mod.decideLoopAction({ ...base, escalationCount: 2, nudges: [95000] }).action, + "compact", + ); + assert.strictEqual( + mod.decideLoopAction({ ...base, escalationCount: 5, nudges: [91000, 92000, 93000] }).action, + "nudge", + ); + }); + + it("leaves the nudge timestamps untouched when it does nothing", async () => { + const mod = await import("../../../src/tui/conversationArea.js"); + const nudges = [90000]; + assert.strictEqual( + mod.decideLoopAction({ ...base, loopDetected: false, nudges }).nudges, + nudges, + ); + }); +}); + describe("shouldAutoContinue", () => { it("returns true when the last bubble ends on reasoning and the guard is not exhausted", async () => { const mod = await import("../../../src/tui/conversationArea.js");