diff --git a/CHANGELOG.md b/CHANGELOG.md index 054ca15e..b2e0a573 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ repeating the same rejected request. - [Apibot] API test scenarios no longer mutate or delete records discovered as sample data. Chief plans a scenario-owned target for destructive checks, and Curler stops when it cannot create one safely. +- [Navigator] A verification is no longer reported as impossible to express when the answer shows a + snippet of page markup before its assertions. A code block written in any language other than + JavaScript used to shift the reading of every later block, so all the assertions the model had + written were dropped and a claim it had answered correctly came back as one no assertion could + express. - A batch of browser commands now reports every command it ran. CodeceptJS hands a passing step its own return value, which was read as an error, so the first command of every successful batch was reported as failed with `[object Promise]` and the commands after it were left out of the report entirely. A diff --git a/src/utils/code-extractor.ts b/src/utils/code-extractor.ts index 845956b9..d7c902c0 100644 --- a/src/utils/code-extractor.ts +++ b/src/utils/code-extractor.ts @@ -2,13 +2,17 @@ import { createDebug } from './logger.js'; const debugLog = createDebug('explorbot:code-extractor'); +const JS_LANGUAGES = new Set(['', 'js', 'javascript']); + export function extractCodeBlocks(aiResponse: string): string[] { - const codeBlockRegex = /```(?:js|javascript)?\s*\n([\s\S]*?)\n```/g; + const codeBlockRegex = /```([^\n`]*)\n([\s\S]*?)\n```/g; const codeBlocks: string[] = []; let match: RegExpExecArray | null = null; while ((match = codeBlockRegex.exec(aiResponse))) { - const code = match[1].trim(); + const language = match[1].trim().toLowerCase(); + if (!JS_LANGUAGES.has(language)) continue; + const code = match[2].trim(); if (!code) continue; try { new Function('I', code); diff --git a/tests/unit/code-extractor.test.ts b/tests/unit/code-extractor.test.ts new file mode 100644 index 00000000..c17e3879 --- /dev/null +++ b/tests/unit/code-extractor.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'bun:test'; +import dedent from 'dedent'; +import { extractCodeBlocks } from '../../src/utils/code-extractor.ts'; + +describe('extractCodeBlocks', () => { + it('extracts a js block', () => { + const response = dedent` + \`\`\`js + I.see('Widget', '.list') + \`\`\` + `; + + expect(extractCodeBlocks(response)).toEqual(["I.see('Widget', '.list')"]); + }); + + it('extracts an unlabelled block', () => { + const response = dedent` + \`\`\` + I.see('Widget', '.list') + \`\`\` + `; + + expect(extractCodeBlocks(response)).toEqual(["I.see('Widget', '.list')"]); + }); + + it('keeps js blocks that follow a block in another language', () => { + const response = dedent` + The element is rendered as: + + \`\`\`html + Widget0 tests + \`\`\` + + ### 1. Verify by visible text + \`\`\`js + I.see('Widget 0 tests', '.list') + \`\`\` + + ### 2. Verify by selector + \`\`\`js + I.seeElement('.list a[href*="widget"]') + \`\`\` + `; + + expect(extractCodeBlocks(response)).toEqual(["I.see('Widget 0 tests', '.list')", 'I.seeElement(\'.list a[href*="widget"]\')']); + }); + + it('skips a block in another language even when its content parses as javascript', () => { + const response = dedent` + \`\`\`text + Saved + \`\`\` + `; + + expect(extractCodeBlocks(response)).toEqual([]); + }); + + it('skips a js block that is not valid javascript', () => { + const response = dedent` + \`\`\`js + I.see('Widget' + \`\`\` + `; + + expect(extractCodeBlocks(response)).toEqual([]); + }); +});