From 0c0ed1da60c1fb8afed81d0a8de4828685ac1501 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Mon, 14 Sep 2026 22:41:54 +0300 Subject: [PATCH] Keep JS blocks that follow a block in another language extractCodeBlocks matched an opening fence only when the info string was js, javascript or empty, so a ```html block never matched as an opener. The scan then paired that block's closing fence with the next block's closer, shifting every later fence by one and producing text that failed the JavaScript parse guard. One markup snippet in an answer was therefore enough to drop every assertion after it. Navigator read that as zero usable code blocks and reported the claim as inexpressible, which sent Tester off to reword an assertion that had been correct. Match any info string and keep the block only when it names JavaScript, so a foreign block is consumed and skipped instead of desynchronising the scan. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JCYmFZvUsENZBx71Yqojs6 --- CHANGELOG.md | 10 +++++ src/utils/code-extractor.ts | 8 +++- tests/unit/code-extractor.test.ts | 67 +++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 tests/unit/code-extractor.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f1df0ae..f5d31b0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 2026-09-14 + +### Changes + +- [Navigator] A verification is no longer reported as impossible to express when the answer shows a + snippet of page markup before its assertions. A code block written in any language other than + JavaScript used to shift the reading of every later block, so all the assertions the model had + written were dropped and a claim it had answered correctly came back as one no assertion could + express. + ## 2026-09-11 ### Changes diff --git a/src/utils/code-extractor.ts b/src/utils/code-extractor.ts index 845956b9..d7c902c0 100644 --- a/src/utils/code-extractor.ts +++ b/src/utils/code-extractor.ts @@ -2,13 +2,17 @@ import { createDebug } from './logger.js'; const debugLog = createDebug('explorbot:code-extractor'); +const JS_LANGUAGES = new Set(['', 'js', 'javascript']); + export function extractCodeBlocks(aiResponse: string): string[] { - const codeBlockRegex = /```(?:js|javascript)?\s*\n([\s\S]*?)\n```/g; + const codeBlockRegex = /```([^\n`]*)\n([\s\S]*?)\n```/g; const codeBlocks: string[] = []; let match: RegExpExecArray | null = null; while ((match = codeBlockRegex.exec(aiResponse))) { - const code = match[1].trim(); + const language = match[1].trim().toLowerCase(); + if (!JS_LANGUAGES.has(language)) continue; + const code = match[2].trim(); if (!code) continue; try { new Function('I', code); diff --git a/tests/unit/code-extractor.test.ts b/tests/unit/code-extractor.test.ts new file mode 100644 index 00000000..c17e3879 --- /dev/null +++ b/tests/unit/code-extractor.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'bun:test'; +import dedent from 'dedent'; +import { extractCodeBlocks } from '../../src/utils/code-extractor.ts'; + +describe('extractCodeBlocks', () => { + it('extracts a js block', () => { + const response = dedent` + \`\`\`js + I.see('Widget', '.list') + \`\`\` + `; + + expect(extractCodeBlocks(response)).toEqual(["I.see('Widget', '.list')"]); + }); + + it('extracts an unlabelled block', () => { + const response = dedent` + \`\`\` + I.see('Widget', '.list') + \`\`\` + `; + + expect(extractCodeBlocks(response)).toEqual(["I.see('Widget', '.list')"]); + }); + + it('keeps js blocks that follow a block in another language', () => { + const response = dedent` + The element is rendered as: + + \`\`\`html + Widget0 tests + \`\`\` + + ### 1. Verify by visible text + \`\`\`js + I.see('Widget 0 tests', '.list') + \`\`\` + + ### 2. Verify by selector + \`\`\`js + I.seeElement('.list a[href*="widget"]') + \`\`\` + `; + + expect(extractCodeBlocks(response)).toEqual(["I.see('Widget 0 tests', '.list')", 'I.seeElement(\'.list a[href*="widget"]\')']); + }); + + it('skips a block in another language even when its content parses as javascript', () => { + const response = dedent` + \`\`\`text + Saved + \`\`\` + `; + + expect(extractCodeBlocks(response)).toEqual([]); + }); + + it('skips a js block that is not valid javascript', () => { + const response = dedent` + \`\`\`js + I.see('Widget' + \`\`\` + `; + + expect(extractCodeBlocks(response)).toEqual([]); + }); +});