diff --git a/CHANGELOG.md b/CHANGELOG.md index b2e0a573..5a0de747 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,12 @@ repeating the same rejected request. - [Apibot] API test scenarios no longer mutate or delete records discovered as sample data. Chief plans a scenario-owned target for destructive checks, and Curler stops when it cannot create one safely. +- [Navigator] A verification that could not run because the model call itself failed is now reported as + a failure, naming the error. It used to be swallowed and reported as a claim no assertion could + express, so a rate limit or a timed-out request reached Tester as a verdict about the page, and + Tester rewrote a correct assertion and asked again. Each such claim also retried the model up to + three more times on top of the retries the request already does, which made a rate limit worse + rather than passing it on. - [Navigator] A verification is no longer reported as impossible to express when the answer shows a snippet of page markup before its assertions. A code block written in any language other than JavaScript used to shift the reading of every later block, so all the assertions the model had diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index 4942ad82..8502bb9a 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -703,7 +703,7 @@ class Navigator implements Agent { const cachedVerification = actionResult.getVerification(message); if (cachedVerification !== null) { tag('operation').log(`Reusing cached verification: ${cachedVerification ? 'PASS' : 'FAIL'}`); - return { verified: cachedVerification, successfulCodes: [], assertionSteps: [], totalAttempted: 0 }; + return { verified: cachedVerification, inexpressible: false, results: [], successfulCodes: [], assertionSteps: [], totalAttempted: 0 }; } const knowledge = this.knowledgeTracker.renderRelevantContext(actionResult); @@ -832,9 +832,6 @@ class Navigator implements Agent { observability: { agent: 'navigator', }, - catch: async (error) => { - debugLog(error); - }, } ); } finally { diff --git a/tests/unit/navigator-verify-failure.test.ts b/tests/unit/navigator-verify-failure.test.ts new file mode 100644 index 00000000..6af1a7a8 --- /dev/null +++ b/tests/unit/navigator-verify-failure.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'bun:test'; +import { Navigator } from '../../src/ai/navigator.ts'; + +function createNavigator(invokeConversation: () => Promise) { + const navigator = Object.create(Navigator.prototype) as any; + navigator.systemPrompt = 'system'; + navigator.knowledgeTracker = { renderRelevantContext: () => '' }; + navigator.experienceTracker = { renderExperienceTocFor: () => '' }; + navigator.stateManager = { updateState: () => {} }; + navigator.config = { playwright: {}, ai: { agents: { navigator: { verifyAttempts: 3, verifyTimeout: 1000 } } } }; + navigator.explorer = { + page: null, + action: () => ({ + assertionSteps: [], + exitIframe: async () => {}, + attempt: async () => true, + }), + }; + navigator.buildExperienceTools = () => ({}); + navigator.provider = { + startConversation: () => ({ addUserText: () => {} }), + invokeConversation, + }; + return navigator as Navigator; +} + +function createActionResult() { + return { + url: '/widgets', + isInsideIframe: false, + verifications: {}, + getVerification: () => null, + addVerification: () => {}, + toAiContext: () => '/widgets', + combinedHtml: async () => '', + } as any; +} + +describe('Navigator.verifyState', () => { + it('reports a failed AI call as a failure instead of an unexpressible claim', async () => { + const navigator = createNavigator(async () => { + throw new Error('Rate limit reached for model on tokens per minute (TPM)'); + }); + + const promise = navigator.verifyState('Widget is visible in the list', createActionResult()); + + expect(promise).rejects.toThrow('Rate limit reached'); + }); + + it('reports an unexpressible claim when the model answers without assertion code', async () => { + const navigator = createNavigator(async () => ({ response: { text: 'I cannot express that as an assertion.' } })); + + const result = await navigator.verifyState('Widget is visible in the list', createActionResult()); + + expect(result.inexpressible).toBe(true); + expect(result.verified).toBe(false); + }); +});