From f9cfbe7fcde3355e9961e5073cf29f7692606c54 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Thu, 10 Sep 2026 14:51:49 +0200 Subject: [PATCH 1/3] Let the Planner see how its tests ended --- CHANGELOG.md | 13 ++++++++++ src/ai/planner.ts | 10 ++++++-- src/ai/planner/session-dedup.ts | 12 ++++++++-- tests/integration/planner.test.ts | 40 ++++++++++++++++++++++++++++++- 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c08b3d26..f73122e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ ### Changes +- [Planner] The session test list now says how each test ended, and a failed or unfinished test carries the + last thing it observed, with a warning not to re-propose the behavior it attempted on another page. The + planner used to see only scenario titles, so a pattern that failed on one page — verifying a state the + interface never shows — was planned again on every page that had a similar control, all night long. Tests + that were started but never finished are named as unfinished rather than left looking unrun. +- [Planner] Expected outcomes must now name what the page displays when they happen. "Data persistency + after page reload" is no longer offered as an outcome in its own right: a persistence check counts + only when the scenario names the on-screen evidence that will show it, and outcomes the page cannot + display at all — a clipboard write, "the operation succeeds" with nothing shown — are rejected at + planning time instead of failing at run time. +- [Planner] A scenario that edits, deletes, reassigns or reconfigures a record must now act on a record + it creates as its own setup. Records the application already had are protected from mutation, so + "edit a visible record" scenarios were refused at run time no matter how visible the record was. - [Pilot] Text an app shows in a tooltip now reaches Pilot along with alerts and status messages. When a page refuses an action and explains why in a hover bubble, that sentence used to stay in the page HTML, which Pilot never sees — so a run could be judged, and reported, on a reason the app had already diff --git a/src/ai/planner.ts b/src/ai/planner.ts index 44fe1ae8..846599c7 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -359,7 +359,10 @@ export class Planner extends PlannerBase implements Agent { You can't test emails, database, SMS, or any external services. Suggest scenarios that can be potentially verified by UI. Focus on error or success messages as outcome. - Focus on URL page change or data persistency after page reload. + Focus on URL page change as outcome. + Every expected outcome must name what the page shows when it happens: a message, a list change, a control state, or a URL change. + Persistency after a reload counts only when the scenario names the on-screen evidence that will show it — a visible marker, a list entry, a stored setting shown in the interface. + An outcome the page cannot display — a clipboard write, an internal state with no visible trace, "the operation succeeds" with nothing shown — is not verifiable and must not be an expected outcome. If there are subpages (pages with same URL path) plan testing of those subpages as well Plan CRUD operations in order: create, read, update, delete. Do not invent specific route names, success messages, validation texts, badge counts, or welcome messages unless they are visible in research, visited pages, or prior observed flows. @@ -374,6 +377,7 @@ export class Planner extends PlannerBase implements Agent { If the list is empty or no concrete item names are visible, do not invent "known" or "existing" items. Prefer empty-state, no-match search, clear-search, or read-only list behavior scenarios. Search, filter, sorting, tab, and list scenarios must start from a stable page where those controls are visible; avoid transient create/edit/new URLs unless the scenario tests that form. For option values and list items, use only visible or previously observed data; do not add create/update/delete setup unless the user explicitly requests that workflow. + A scenario that edits, deletes, reassigns, or reconfigures a record must act on a record the scenario itself creates as its setup — that setup is required, not optional, because records the app already had are protected from mutation. Acting on a pre-existing record makes the scenario unrunnable, however visible the record is. Detail-view scenarios must target visible data entities from list rows, cards, tree nodes, or detail links; do not use filter tabs, counters, status tabs, breadcrumbs, or navigation controls as detail targets. DO NOT propose "verification-only" tests that merely open a UI element (modal, dropdown, panel) and check it exists. Every test must complete a meaningful action that changes application state or produces a business outcome. @@ -591,7 +595,9 @@ export class Planner extends PlannerBase implements Agent { const sessionTests = this.getSessionTestsSummary(); if (sessionTests) { conversation.addUserText(dedent` - Tests already planned in this session across all pages. DO NOT duplicate any of these: + Tests already planned in this session across all pages, with how each one ended. DO NOT duplicate any of these. + A failed test means the app or the harness could not do what it tried: do not re-propose the same behavior on another page unless you can name what makes it work this time. + A failed or unfinished test carries the last thing it observed after the dash — read it before deciding that the behavior is worth trying again. ${sessionTests} diff --git a/src/ai/planner/session-dedup.ts b/src/ai/planner/session-dedup.ts index 85f23905..a97d6dfd 100644 --- a/src/ai/planner/session-dedup.ts +++ b/src/ai/planner/session-dedup.ts @@ -1,4 +1,4 @@ -import type { Plan } from '../../test-plan.ts'; +import { type Plan, TestResult } from '../../test-plan.ts'; import type { Constructor } from '../researcher/mixin.ts'; const previousPlans: Plan[] = []; @@ -18,7 +18,15 @@ export function WithSessionDedup(Base: T) { for (const plan of previousPlans) { if (plan === this.currentPlan) continue; for (const test of plan.tests) { - lines.push(`${plan.url || '/'} | ${test.style || 'default'} | ${test.scenario}`); + const lastNote = Object.values(test.notes) + .filter((note) => note.message) + .pop(); + const outcome = test.result || (lastNote ? 'unfinished' : 'pending'); + let line = `${plan.url || '/'} | ${test.style || 'default'} | ${outcome} | ${test.scenario}`; + if ((outcome === TestResult.FAILED || outcome === 'unfinished') && lastNote) { + line += ` — ${lastNote.message.slice(0, 140)}`; + } + lines.push(line); } } return lines.join('\n'); diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index 774ca812..156fe6d2 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -9,7 +9,7 @@ import { clearStyleCache } from '../../src/ai/planner/styles.ts'; import { clearPlanRegistry, registerPlan } from '../../src/ai/planner/subpages.ts'; import { Provider } from '../../src/ai/provider.ts'; import { ConfigParser } from '../../src/config.ts'; -import { Plan, Test } from '../../src/test-plan.ts'; +import { Plan, Test, TestResult } from '../../src/test-plan.ts'; const UI_MAPS_DIR = join(process.cwd(), 'test-data', 'ui-maps'); @@ -431,4 +431,42 @@ describe('Planner with aimock', () => { expect(extractPromptText(mock.getLastRequest())).toContain('roughly 100%'); }); + + it('tells the planner how session tests ended', async () => { + const finished = new Plan('Pin Testing'); + finished.url = '/tasks/board'; + const failed = new Test('Pin a visible task and verify the pinned state persists', 'normal', ['Pinned state shown'], '/tasks/board', ['Click Pin']); + failed.addNote('Pin action failed, no pinned indicator'); + failed.finish(TestResult.FAILED); + finished.addTest(failed); + const aborted = new Test('Assign an assignee and verify the assignment persists', 'normal', ['Assignee shown'], '/tasks/board', ['Open assignee menu']); + aborted.addNote('Could not find the record in the current list'); + finished.addTest(aborted); + planner.registerPlanInSession(finished); + + await planner.plan(); + + const prompt = extractPromptText(mock.getLastRequest()); + expect(prompt).toContain('with how each one ended'); + expect(prompt).toContain(`failed | Pin a visible task and verify the pinned state persists — Pin action failed, no pinned indicator`); + expect(prompt).toContain(`unfinished | Assign an assignee and verify the assignment persists — Could not find the record in the current list`); + expect(prompt).toContain('do not re-propose the same behavior'); + expect(prompt).toContain('read it before deciding'); + }); + + it('requires outcomes the page can show', async () => { + await planner.plan(); + + const prompt = extractPromptText(mock.getLastRequest()); + expect(prompt).toContain('must name what the page shows when it happens'); + expect(prompt).toContain('not verifiable and must not be an expected outcome'); + }); + + it('forbids mutating records the app already had', async () => { + await planner.plan(); + + const prompt = extractPromptText(mock.getLastRequest()); + expect(prompt).toContain('records the app already had are protected from mutation'); + expect(prompt).toContain('Acting on a pre-existing record makes the scenario unrunnable'); + }); }); From efea1bea91e22c2ef655e5b45ee49e2db288c884 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Thu, 10 Sep 2026 14:57:51 +0200 Subject: [PATCH 2/3] Fix --- tests/integration/planner.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index 156fe6d2..60d4ca7b 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -448,8 +448,8 @@ describe('Planner with aimock', () => { const prompt = extractPromptText(mock.getLastRequest()); expect(prompt).toContain('with how each one ended'); - expect(prompt).toContain(`failed | Pin a visible task and verify the pinned state persists — Pin action failed, no pinned indicator`); - expect(prompt).toContain(`unfinished | Assign an assignee and verify the assignment persists — Could not find the record in the current list`); + expect(prompt).toContain('failed | Pin a visible task and verify the pinned state persists — Pin action failed, no pinned indicator'); + expect(prompt).toContain('unfinished | Assign an assignee and verify the assignment persists — Could not find the record in the current list'); expect(prompt).toContain('do not re-propose the same behavior'); expect(prompt).toContain('read it before deciding'); }); From b3aeb062f64c8ab32600b96ccabd83f7585a439e Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Mon, 14 Sep 2026 10:25:06 +0200 Subject: [PATCH 3/3] Fix after review --- CHANGELOG.md | 16 ++++----- src/ai/planner.ts | 10 +++--- src/ai/planner/session-dedup.ts | 26 +++++++++------ tests/integration/planner.test.ts | 15 +++------ tests/unit/planner-session-dedup.test.ts | 41 ++++++++++++++++++++++++ 5 files changed, 73 insertions(+), 35 deletions(-) create mode 100644 tests/unit/planner-session-dedup.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index f258134f..f2d30de4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 2026-09-14 + +### Changes + +- [Planner] Edit and reconfiguration scenarios no longer include record creation as part of every plan; + Pilot can select suitable data or prepare it when execution starts. Expected outcomes remain verifiable + through the interface without inventing details for pages and subpages that have not been visited yet. + ## 2026-09-11 ### Changes @@ -30,14 +38,6 @@ planner used to see only scenario titles, so a pattern that failed on one page — verifying a state the interface never shows — was planned again on every page that had a similar control, all night long. Tests that were started but never finished are named as unfinished rather than left looking unrun. -- [Planner] Expected outcomes must now name what the page displays when they happen. "Data persistency - after page reload" is no longer offered as an outcome in its own right: a persistence check counts - only when the scenario names the on-screen evidence that will show it, and outcomes the page cannot - display at all — a clipboard write, "the operation succeeds" with nothing shown — are rejected at - planning time instead of failing at run time. -- [Planner] A scenario that edits, deletes, reassigns or reconfigures a record must now act on a record - it creates as its own setup. Records the application already had are protected from mutation, so - "edit a visible record" scenarios were refused at run time no matter how visible the record was. - [Researcher] A list is now checked for how it continues past what is on screen, and the answer is recorded beside that list in the UI map: either it has controls that move between pages, or it grows when scrolled. Lists that scroll inside their own box are covered, not just the page. diff --git a/src/ai/planner.ts b/src/ai/planner.ts index 4e18b7c7..d3d4478c 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -358,11 +358,10 @@ export class Planner extends PlannerBase implements Agent { You can suggest scenarios that can be tested only through web interface. You can't test emails, database, SMS, or any external services. Suggest scenarios that can be potentially verified by UI. - Focus on error or success messages as outcome. - Focus on URL page change as outcome. - Every expected outcome must name what the page shows when it happens: a message, a list change, a control state, or a URL change. - Persistency after a reload counts only when the scenario names the on-screen evidence that will show it — a visible marker, a list entry, a stored setting shown in the interface. - An outcome the page cannot display — a clipboard write, an internal state with no visible trace, "the operation succeeds" with nothing shown — is not verifiable and must not be an expected outcome. + Prefer outcomes grounded in observed interface behavior. + Every expected outcome must be verifiable through the web interface. + If a page or subpage has not been observed, describe the expected visible result generically instead of inventing interface details. + Persistency after a reload counts only when the persisted state can be confirmed through the interface. If there are subpages (pages with same URL path) plan testing of those subpages as well Plan CRUD operations in order: create, read, update, delete. Do not invent specific route names, success messages, validation texts, badge counts, or welcome messages unless they are visible in research, visited pages, or prior observed flows. @@ -377,7 +376,6 @@ export class Planner extends PlannerBase implements Agent { If the list is empty or no concrete item names are visible, do not invent "known" or "existing" items. Prefer empty-state, no-match search, clear-search, or read-only list behavior scenarios. Search, filter, sorting, tab, and list scenarios must start from a stable page where those controls are visible; avoid transient create/edit/new URLs unless the scenario tests that form. For option values and list items, use only visible or previously observed data; do not add create/update/delete setup unless the user explicitly requests that workflow. - A scenario that edits, deletes, reassigns, or reconfigures a record must act on a record the scenario itself creates as its setup — that setup is required, not optional, because records the app already had are protected from mutation. Acting on a pre-existing record makes the scenario unrunnable, however visible the record is. Detail-view scenarios must target visible data entities from list rows, cards, tree nodes, or detail links; do not use filter tabs, counters, status tabs, breadcrumbs, or navigation controls as detail targets. DO NOT propose "verification-only" tests that merely open a UI element (modal, dropdown, panel) and check it exists. Every test must complete a meaningful action that changes application state or produces a business outcome. diff --git a/src/ai/planner/session-dedup.ts b/src/ai/planner/session-dedup.ts index a97d6dfd..fe3bdbb6 100644 --- a/src/ai/planner/session-dedup.ts +++ b/src/ai/planner/session-dedup.ts @@ -1,4 +1,4 @@ -import { type Plan, TestResult } from '../../test-plan.ts'; +import { type Plan, type Test, TestResult } from '../../test-plan.ts'; import type { Constructor } from '../researcher/mixin.ts'; const previousPlans: Plan[] = []; @@ -18,15 +18,7 @@ export function WithSessionDedup(Base: T) { for (const plan of previousPlans) { if (plan === this.currentPlan) continue; for (const test of plan.tests) { - const lastNote = Object.values(test.notes) - .filter((note) => note.message) - .pop(); - const outcome = test.result || (lastNote ? 'unfinished' : 'pending'); - let line = `${plan.url || '/'} | ${test.style || 'default'} | ${outcome} | ${test.scenario}`; - if ((outcome === TestResult.FAILED || outcome === 'unfinished') && lastNote) { - line += ` — ${lastNote.message.slice(0, 140)}`; - } - lines.push(line); + lines.push(formatSessionTest(plan, test)); } } return lines.join('\n'); @@ -42,6 +34,20 @@ export function WithSessionDedup(Base: T) { }; } +export function formatSessionTest(plan: Plan, test: Test): string { + const lastNote = Object.values(test.notes) + .filter((note) => note.message) + .pop(); + let outcome: string | null = test.result; + if (!outcome) outcome = 'pending'; + if (!test.result && lastNote) outcome = 'unfinished'; + + const line = `${plan.url || '/'} | ${test.style || 'default'} | ${outcome} | ${test.scenario}`; + if (!lastNote) return line; + if (outcome !== TestResult.FAILED && outcome !== 'unfinished') return line; + return `${line} — ${lastNote.message.slice(0, 140)}`; +} + export function clearSessionDedup(): void { previousPlans.length = 0; } diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index ad9e310f..0b1ff440 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -470,19 +470,12 @@ describe('Planner with aimock', () => { expect(prompt).toContain('read it before deciding'); }); - it('requires outcomes the page can show', async () => { + it('requires UI-verifiable outcomes without inventing unseen page details', async () => { await planner.plan(); const prompt = extractPromptText(mock.getLastRequest()); - expect(prompt).toContain('must name what the page shows when it happens'); - expect(prompt).toContain('not verifiable and must not be an expected outcome'); - }); - - it('forbids mutating records the app already had', async () => { - await planner.plan(); - - const prompt = extractPromptText(mock.getLastRequest()); - expect(prompt).toContain('records the app already had are protected from mutation'); - expect(prompt).toContain('Acting on a pre-existing record makes the scenario unrunnable'); + expect(prompt).toContain('must be verifiable through the web interface'); + expect(prompt).toContain('page or subpage has not been observed'); + expect(prompt).toContain('describe the expected visible result generically'); }); }); diff --git a/tests/unit/planner-session-dedup.test.ts b/tests/unit/planner-session-dedup.test.ts new file mode 100644 index 00000000..23f92d4a --- /dev/null +++ b/tests/unit/planner-session-dedup.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'bun:test'; +import { formatSessionTest } from '../../src/ai/planner/session-dedup.ts'; +import { Plan, Test, TestResult } from '../../src/test-plan.ts'; + +describe('formatSessionTest', () => { + it('formats a pending test with defaults', () => { + const plan = new Plan('Plan'); + const test = new Test('Search records', 'normal', ['Results shown'], '/', ['Search']); + + expect(formatSessionTest(plan, test)).toBe('/ | default | pending | Search records'); + }); + + it('marks a started test with a note as unfinished', () => { + const plan = new Plan('Plan'); + plan.url = '/records'; + const test = new Test('Edit a record', 'normal', ['Record updated'], '/records', ['Edit']); + test.style = 'stress'; + test.addNote('Editor did not open'); + + expect(formatSessionTest(plan, test)).toBe('/records | stress | unfinished | Edit a record — Editor did not open'); + }); + + it('includes the last note for a failed test', () => { + const plan = new Plan('Plan'); + const test = new Test('Delete a record', 'normal', ['Record removed'], '/', ['Delete']); + test.addNote('Delete control opened'); + test.addNote('Confirmation failed'); + test.finish(TestResult.FAILED); + + expect(formatSessionTest(plan, test)).toBe('/ | default | failed | Delete a record — Confirmation failed'); + }); + + it('does not append notes to successful tests', () => { + const plan = new Plan('Plan'); + const test = new Test('Create a record', 'normal', ['Record created'], '/', ['Create']); + test.addNote('Record appeared'); + test.finish(TestResult.PASSED); + + expect(formatSessionTest(plan, test)).toBe('/ | default | passed | Create a record'); + }); +});