From f15102efb34f1b037ece63587f1482c712cc1e21 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Mon, 14 Sep 2026 22:43:25 +0300 Subject: [PATCH] Report a failed verification model call as a failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit verifyState wrapped its loop in `catch: async (error) => debugLog(error)`. Any error inside an iteration was logged and the loop moved on, so a rate-limited or aborted model call left codeBlocks empty. The result was `totalAttempted === 0`, which verifyState reads as "no assertion could express this claim" — a verdict about the page, returned for an infrastructure failure, with a suggestion telling Tester to reword a claim that was never the problem. The catch also amplified the failure it hid: each iteration retried the model, so one claim cost verifyAttempts x the retries the provider already performs, all against the limit that had just been hit. And it ate the fatal browser errors action.attempt deliberately rethrows. loop() rethrows when no catch handler is given, and handles StopError before reaching one, so dropping the handler lets real failures out without disturbing the stop() calls. Both callers are ready for it: the verify tool reports `Verify tool failed: ` from its own catch, and Pilot guards its call with `.catch(() => null)`. inexpressible now means only what it says — the model answered without usable assertion code. Also make the cached early return match the declared return type; it omitted inexpressible and results, which `tsc --noCheck` never flagged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JCYmFZvUsENZBx71Yqojs6 --- CHANGELOG.md | 11 ++++ src/ai/navigator.ts | 5 +- tests/unit/navigator-verify-failure.test.ts | 58 +++++++++++++++++++++ 3 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 tests/unit/navigator-verify-failure.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f1df0ae..403d9ddb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 2026-09-14 + +### Changes + +- [Navigator] A verification that could not run because the model call itself failed is now reported as + a failure, naming the error. It used to be swallowed and reported as a claim no assertion could + express, so a rate limit or a timed-out request reached Tester as a verdict about the page, and + Tester rewrote a correct assertion and asked again. Each such claim also retried the model up to + three more times on top of the retries the request already does, which made a rate limit worse + rather than passing it on. + ## 2026-09-11 ### Changes diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index 4942ad82..8502bb9a 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -703,7 +703,7 @@ class Navigator implements Agent { const cachedVerification = actionResult.getVerification(message); if (cachedVerification !== null) { tag('operation').log(`Reusing cached verification: ${cachedVerification ? 'PASS' : 'FAIL'}`); - return { verified: cachedVerification, successfulCodes: [], assertionSteps: [], totalAttempted: 0 }; + return { verified: cachedVerification, inexpressible: false, results: [], successfulCodes: [], assertionSteps: [], totalAttempted: 0 }; } const knowledge = this.knowledgeTracker.renderRelevantContext(actionResult); @@ -832,9 +832,6 @@ class Navigator implements Agent { observability: { agent: 'navigator', }, - catch: async (error) => { - debugLog(error); - }, } ); } finally { diff --git a/tests/unit/navigator-verify-failure.test.ts b/tests/unit/navigator-verify-failure.test.ts new file mode 100644 index 00000000..6af1a7a8 --- /dev/null +++ b/tests/unit/navigator-verify-failure.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'bun:test'; +import { Navigator } from '../../src/ai/navigator.ts'; + +function createNavigator(invokeConversation: () => Promise) { + const navigator = Object.create(Navigator.prototype) as any; + navigator.systemPrompt = 'system'; + navigator.knowledgeTracker = { renderRelevantContext: () => '' }; + navigator.experienceTracker = { renderExperienceTocFor: () => '' }; + navigator.stateManager = { updateState: () => {} }; + navigator.config = { playwright: {}, ai: { agents: { navigator: { verifyAttempts: 3, verifyTimeout: 1000 } } } }; + navigator.explorer = { + page: null, + action: () => ({ + assertionSteps: [], + exitIframe: async () => {}, + attempt: async () => true, + }), + }; + navigator.buildExperienceTools = () => ({}); + navigator.provider = { + startConversation: () => ({ addUserText: () => {} }), + invokeConversation, + }; + return navigator as Navigator; +} + +function createActionResult() { + return { + url: '/widgets', + isInsideIframe: false, + verifications: {}, + getVerification: () => null, + addVerification: () => {}, + toAiContext: () => '/widgets', + combinedHtml: async () => '', + } as any; +} + +describe('Navigator.verifyState', () => { + it('reports a failed AI call as a failure instead of an unexpressible claim', async () => { + const navigator = createNavigator(async () => { + throw new Error('Rate limit reached for model on tokens per minute (TPM)'); + }); + + const promise = navigator.verifyState('Widget is visible in the list', createActionResult()); + + expect(promise).rejects.toThrow('Rate limit reached'); + }); + + it('reports an unexpressible claim when the model answers without assertion code', async () => { + const navigator = createNavigator(async () => ({ response: { text: 'I cannot express that as an assertion.' } })); + + const result = await navigator.verifyState('Widget is visible in the list', createActionResult()); + + expect(result.inexpressible).toBe(true); + expect(result.verified).toBe(false); + }); +});