From bee35eb501972ea8aa52d809b53bec07fbd88569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 15:20:19 +0100 Subject: [PATCH 1/8] agent-device 0.21.39: read typed refusals, and press the dismiss control back names 0.21.39 no longer says "in-app back control is not available" or "Unable to dismiss the iOS keyboard", so the fallbacks Explore keyed on those messages would never run. They now read error.details.reason: - back: on iOS, agent-device presses the navigation bar back button, else swipes from the left edge, and refuses with back_swipe_no_change when nothing changed. It lists the screen's dismiss controls (Close, Cancel, Back) in details.dismissControls without pressing them. Explore presses the one it names when its label is safe by Explore's destructive and external rules, requires the destination to change, and otherwise swipes a sheet down as before. - keyboard dismiss refuses with keyboard_dismiss_unavailable; the return-key fallback keys on it. - the remote /health refusal is remote_daemon_unavailable with dispatched "no", which is what Explore inferred from the message before resending. The other controller's bridge gives the same reasons. Co-Authored-By: Claude Opus 5.5 --- docs/quickstart-ci.md | 2 +- package-lock.json | 8 +-- package.json | 2 +- src/binary/run.ts | 16 +++-- src/capture/argent-driver.ts | 12 ++-- src/capture/controller.ts | 42 +++++++++++-- src/capture/route-capture.ts | 18 ++++-- src/shared/agent-device-version.ts | 2 +- src/shared/controller-cache.ts | 2 +- test/node/argent-driver.test.mjs | 7 ++- test/node/ci-map.test.mjs | 65 +++++++++++++++++++-- test/node/public-comment.test.mjs | 6 +- test/node/public-review-validation.test.mjs | 2 +- 13 files changed, 143 insertions(+), 41 deletions(-) diff --git a/docs/quickstart-ci.md b/docs/quickstart-ci.md index fe4ffaf5..0dc2fd7f 100644 --- a/docs/quickstart-ci.md +++ b/docs/quickstart-ci.md @@ -289,7 +289,7 @@ The provider selects the runtime that owns the navigation loop: | `anthropic` | Claude Code through `@anthropic-ai/claude-agent-sdk` 0.3.285 and its platform-native package | `ANTHROPIC_API_KEY` | | `openai` | Codex app-server from `@openai/codex` 0.159.2 and its platform-native package | `OPENAI_API_KEY` | -Both runtimes use Explore's device tools to operate the existing Expo Simulators session through official `agent-device` 0.21.29-nightly.20261018.74. The runtime and native binaries come from the pinned tools installation. CI does not use the development-only `node_modules/.bin/claude` executable or an interactive subscription login. +Both runtimes use Explore's device tools to operate the existing Expo Simulators session through official `agent-device` 0.21.39. The runtime and native binaries come from the pinned tools installation. CI does not use the development-only `node_modules/.bin/claude` executable or an interactive subscription login. Create and commit the nonempty English prose file named by `agent.instructions`. The shared setup skill authors it from source and asks for missing test data or environment inputs. Describe how to reach the app's screens, necessary prerequisites, the representative state to use on both revisions, readiness signals, and actions to avoid. This file guides the runtime agent; it is not a list of mandatory YAML steps or a second screen inventory. diff --git a/package-lock.json b/package-lock.json index 03f648eb..dba9f69e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,7 +29,7 @@ "@types/react-dom": "^19.2.4", "@vitejs/plugin-react": "^4.7.0", "@xyflow/react": "^12.11.3", - "agent-device": "0.21.29-nightly.20261018.74", + "agent-device": "0.21.39", "babel-plugin-react-compiler": "^1.0.0", "eslint": "^10.12.0", "eslint-plugin-react-hooks": "^7.1.1", @@ -4593,9 +4593,9 @@ } }, "node_modules/agent-device": { - "version": "0.21.29-nightly.20261018.74", - "resolved": "https://registry.npmjs.org/agent-device/-/agent-device-0.21.29-nightly.20261018.74.tgz", - "integrity": "sha512-7+ajWLr0J+8iLZ9q11ERbj6U14mC20umBK5BQQdw5aERA79yeXc3Ze6KmtvnILnWjEtoVkYQsXQasL8RL6Uatw==", + "version": "0.21.39", + "resolved": "https://registry.npmjs.org/agent-device/-/agent-device-0.21.39.tgz", + "integrity": "sha512-flgrilleoUU9WNd9BlU8h5KMt8zv0nKWBkqJbzk/bb2NBRA+7H1FGja+RIAuapb7HXdsjK4EPnYFP37N9S8nMA==", "dev": true, "license": "MIT", "bin": { diff --git a/package.json b/package.json index ad3629a9..b6e36d3b 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,7 @@ "@types/react-dom": "^19.2.4", "@vitejs/plugin-react": "^4.7.0", "@xyflow/react": "^12.11.3", - "agent-device": "0.21.29-nightly.20261018.74", + "agent-device": "0.21.39", "babel-plugin-react-compiler": "^1.0.0", "eslint": "^10.12.0", "eslint-plugin-react-hooks": "^7.1.1", diff --git a/src/binary/run.ts b/src/binary/run.ts index c51e38ce..afd15b78 100644 --- a/src/binary/run.ts +++ b/src/binary/run.ts @@ -7,7 +7,7 @@ import { createLocalCiAgentSession } from "../capture/agent/session-local.js"; import type { CiAgentProviderName } from "../capture/agent/provider.js"; import { assertReplayFits, createReplaySession, parseReplayPlan, replaySetup, type ReplayPlan } from "./replay.js"; import { CONTROLLER_SESSION, baseEnvironment, isRecord } from "../capture/common.js"; -import { controllerData, defaultRunner, openedSimulatorDevice } from "../capture/controller.js"; +import { controllerData, controllerFailureReason, defaultRunner, loneBackDismissControl, openedSimulatorDevice } from "../capture/controller.js"; import type { CommandRequest, CommandResult } from "../capture/run.js"; import { executeCaptureSetup } from "../capture/setup-actions.js"; import { SIMULATOR_DOTENV_FILE_NAME, SIMULATOR_DOTENV_HEADER, assertSimulatorSessionIdentity, assertSimulatorStopped, readSimulatorDotenvHandoff, @@ -29,7 +29,7 @@ import { packageBin } from "../shared/package-paths.js"; import { assertDecodedPng, solidColour } from "../shared/png-evidence.js"; import { CONTRACT, type ExploreMap, type Platform } from "../types.js"; import { assertEmulatorCompatible, extractIosBundle, inspectArchive, type ArchiveKind, type InspectedArchive, readArchiveIcon } from "./archive.js"; -import { runBlackBoxExploration, type BlackBoxExplorerResult, type BlackBoxFrame, SYSTEM_DIALOG_PACKAGES } from "./explorer.js"; +import { riskOf, runBlackBoxExploration, type BlackBoxExplorerResult, type BlackBoxFrame, SYSTEM_DIALOG_PACKAGES } from "./explorer.js"; export const DEFAULT_IOS_DEVICE = "iPhone 17 Pro"; export const DEFAULT_BINARY_PROJECT = "expo-explore"; @@ -502,11 +502,15 @@ export async function exploreConnectedSession(ctx: ConnectedSessionExploration): const bounds = frameBounds(frame.nodes); controller(["press", String(Math.round(action.x * bounds.width)), String(Math.round(action.y * bounds.height)), "--level", "full", "--json"], 120_000); } else if (action.action === "back") { - // Onboarding steps, tabs and sheets have no back control; the platform gesture (iOS edge swipe, Android back) still works. + // On iOS agent-device's in-app back already falls back to the edge swipe, and names a modal's one dismiss + // control when that changed nothing. The Argent bridge sends nothing without a back control; Android's + // system back still works there. try { controller(["back", "--in-app", "--level", "full", "--json"], 120_000); } catch (error) { - if (!(error instanceof Error) || !/in-app back control is not available/.test(error.message)) throw error; - controller(["back", "--system", "--level", "full", "--json"], 120_000); + const control = loneBackDismissControl(error); + if (control && riskOf(control.label, allowDestructive ?? []) === "safe") controller(["press", "--level", "full", "--json", "--", control.selector], 120_000); + else if (controllerFailureReason(error) === "back_swipe_not_dispatched") controller(["back", "--system", "--level", "full", "--json"], 120_000); + else throw error; } } else if (action.action === "dismiss-keyboard") await dismissKeyboard(); @@ -516,7 +520,7 @@ export async function exploreConnectedSession(ctx: ConnectedSessionExploration): }; const dismissKeyboard = async () => { try { controller(["keyboard", "dismiss", "--level", "full", "--json"], 120_000); return; } - catch (error) { if (!(error instanceof Error) || !/Unable to dismiss the iOS keyboard/.test(error.message)) throw error; } + catch (error) { if (controllerFailureReason(error) !== "keyboard_dismiss_unavailable") throw error; } // The iOS keyboard has no dismiss key and agent-device never taps the background; its return key (next/done/go) closes it without touching the app. const nodes = (await observe(false)).nodes; // In a multi-line editor the return key inserts a newline instead of ending editing. diff --git a/src/capture/argent-driver.ts b/src/capture/argent-driver.ts index ccc8d48c..e3096e59 100644 --- a/src/capture/argent-driver.ts +++ b/src/capture/argent-driver.ts @@ -76,7 +76,7 @@ const failure = (code: string, message: string, details?: Record failure("COMMAND_FAILED", "Remote daemon is unavailable"); +const remoteUnavailable = () => failure("COMMAND_FAILED", "Remote daemon is unavailable", { reason: "remote_daemon_unavailable", dispatched: "no" }); /** Flags agent-device takes after a command; the bridge needs the platform only from its own context. */ const VALUE_FLAGS = new Set(["--session", "--device", "--platform", "--level", "--udid", "--scope", "--bundle-id", "--serial", "--timeout"]); @@ -613,17 +613,17 @@ export async function runBridgeCommand(argv: readonly string[], context: ArgentB await beforeAction(); if (platform === "android") await call("button", { button: "back" }); else { - // agent-device taps the keyboard's own dismiss control, and fails with this message when there is none, which + // agent-device taps the keyboard's own dismiss control, and refuses with this reason when there is none, which // callers answer with the return key. Only a key in the keyboard's area counts: an app's "Done" above it is not one. const key = flatten(current).find(node => KEYBOARD_DISMISS_LABELS.includes(node.label ?? "") && node.frame && node.frame.y >= KEYBOARD_TOP); - if (!key) return failure("COMMAND_FAILED", "Unable to dismiss the iOS keyboard: the keyboard exposes no dismiss control."); + if (!key) return failure("UNSUPPORTED_OPERATION", "Unable to dismiss the keyboard: it exposes no dismiss key.", { reason: "keyboard_dismiss_unavailable", dispatched: "no" }); const size = current.screen ?? await screen(); await tap(...Object.values(centre({ x: key.frame!.x * size.width, y: key.frame!.y * size.height, width: key.frame!.width * size.width, height: key.frame!.height * size.height }, size)) as [number, number]); } await context.sleep(500); const visible = (await tree()).keyboardVisible === true; - if (visible && platform === "ios") return failure("COMMAND_FAILED", "Unable to dismiss the iOS keyboard: it is still showing after its dismiss key was pressed."); + if (visible && platform === "ios") return failure("COMMAND_FAILED", "Unable to dismiss the keyboard: it is still showing after its dismiss key was pressed.", { reason: "keyboard_dismiss_unavailable" }); return success({ platform, action: "dismiss", wasVisible: true, dismissed: !visible, visible }); } case "scroll": { @@ -666,7 +666,7 @@ export async function runBridgeCommand(argv: readonly string[], context: ArgentB await tap(button.frame.x + button.frame.width / 2, button.frame.y + button.frame.height / 2); return success({ action: "back", mode: "in-app" }); } - if (flags["in-app"] === true || platform === "ios") return failure("COMMAND_FAILED", "in-app back control is not available"); + if (flags["in-app"] === true || platform === "ios") return failure("COMMAND_FAILED", "in-app back control is not available", { reason: "back_swipe_not_dispatched", dispatched: "no" }); await call("button", { button: "back" }); return success({ action: "back", mode: "system" }); } @@ -776,7 +776,7 @@ export async function runBridgeCommand(argv: readonly string[], context: ArgentB if (error instanceof BridgeFailure) return failure(error.code, error.message); if (error instanceof ArgentCallError) { // Never "Remote daemon is unavailable" here: the request may have run, so Explore must not resend it. - return failure(error.sent ? "ARGENT_TOOL_FAILED" : "COMMAND_FAILED", error.sent ? error.message : "Remote daemon is unavailable"); + return error.sent ? failure("ARGENT_TOOL_FAILED", error.message) : remoteUnavailable(); } return failure("ARGENT_BRIDGE_FAILED", error instanceof Error ? error.message : String(error)); } diff --git a/src/capture/controller.ts b/src/capture/controller.ts index 4ed312ed..cca03ccb 100644 --- a/src/capture/controller.ts +++ b/src/capture/controller.ts @@ -106,6 +106,36 @@ export function controllerCommandFailure(stdout: string, command: string): CiErr return failure; } +/** agent-device's typed refusal reason (`error.details.reason`), when the failure carries one. */ +export function controllerFailureReason(error: unknown): string | undefined { + if (!(error instanceof CiError)) return undefined; + const details = nativeControllerFailures.get(error)?.details; + return isRecord(details) && typeof details.reason === "string" ? details.reason : undefined; +} + +/** + * A back that left the screen as it was: agent-device swiped from the left edge and nothing changed + * (`back_swipe_no_change`), or sent nothing (`back_swipe_not_dispatched`, also the Argent bridge's answer + * when the screen has no back control). + */ +export function backLeftScreenUnchanged(error: unknown): boolean { + const reason = controllerFailureReason(error); + return reason === "back_swipe_no_change" || reason === "back_swipe_not_dispatched"; +} + +/** + * The one dismiss control (Close, Cancel, Back) agent-device saw in the screen's top bar when its back changed + * nothing (`details.dismissControls`). agent-device never presses it: on iOS back only pops the navigation stack. + * Undefined when it listed none or several. + */ +export function loneBackDismissControl(error: unknown): { selector: string; label: string } | undefined { + if (controllerFailureReason(error) !== "back_swipe_no_change") return undefined; + const controls = (nativeControllerFailures.get(error as CiError)?.details as Record | undefined)?.dismissControls; + if (!Array.isArray(controls) || controls.length !== 1 || !isRecord(controls[0])) return undefined; + const { selector, label } = controls[0]; + return typeof selector === "string" && selector.trim() && typeof label === "string" ? { selector, label } : undefined; +} + function busyControllerReadFailure(error: unknown): boolean { if (!(error instanceof CiError)) return false; const native = nativeControllerFailures.get(error); @@ -158,14 +188,14 @@ function responsiveAccessibilityRead(args: readonly string[], result: CommandRes } /** - * agent-device checks the remote daemon's /health, with a 3 s timeout, before it sends a command, - * and fails with this error when the check gets no answer or a 5xx. The command itself was never - * sent, so sending it again cannot repeat anything, even a UI action. + * agent-device checks the remote daemon's /health before it sends a command, and refuses with + * `remote_daemon_unavailable` and `dispatched: "no"` when the check gets no answer or a 5xx. The + * command itself was never sent, so sending it again cannot repeat anything, even a UI action. */ export function remoteDaemonUnavailable(error: unknown): boolean { - if (!(error instanceof CiError)) return false; - const native = nativeControllerFailures.get(error); - return !!native && native.code === "COMMAND_FAILED" && native.message === "Remote daemon is unavailable"; + if (controllerFailureReason(error) !== "remote_daemon_unavailable") return false; + const details = nativeControllerFailures.get(error as CiError)?.details; + return isRecord(details) && details.dispatched === "no"; } /** diff --git a/src/capture/route-capture.ts b/src/capture/route-capture.ts index 108600e9..204c9698 100644 --- a/src/capture/route-capture.ts +++ b/src/capture/route-capture.ts @@ -23,7 +23,8 @@ import { CAPTURE_OBSERVER_IDENTIFIER, captureObserverMarkers, parseCaptureDestin import { type CiAgentRuntimeOptions, runCiAgentNavigation, assertAgentCaptureContext } from "./agent/runtime.js"; import { createExplorationInputs } from "../config/exploration-inputs.js"; import { isRecord, captureError, CONTROLLER_SESSION, parseJson } from "./common.js"; -import { MAX_ACCESSIBILITY_EVIDENCE_BYTES, adbDeviceOffline, remoteDaemonUnavailable, controllerData, defaultRunner, type ControllerReadRecovery, type RemoteDaemonRetry, runControllerRead, controllerCommandFailure, runControllerReadAsync, runAgentControllerCommand, wedgedControllerReadFailure, openedSimulatorDevice, alertWatchdogFailure } from "./controller.js"; +import { MAX_ACCESSIBILITY_EVIDENCE_BYTES, adbDeviceOffline, remoteDaemonUnavailable, controllerData, defaultRunner, type ControllerReadRecovery, type RemoteDaemonRetry, runControllerRead, controllerCommandFailure, runControllerReadAsync, runAgentControllerCommand, wedgedControllerReadFailure, openedSimulatorDevice, alertWatchdogFailure, backLeftScreenUnchanged, loneBackDismissControl } from "./controller.js"; +import { riskOf } from "../binary/explorer.js"; import type { ActiveSimulatorRuntime, SimulatorConnection } from "./simulator.js"; import { androidTools } from "./local-simulator.js"; import type { CiMapOptions, CommandResult } from "./run.js"; @@ -771,14 +772,23 @@ export async function capturePlannedRoutes(options: { const toLogical = (space: { width: number; height: number; transposed: boolean }, x: number, y: number): [number, number] => space.transposed ? [y * space.width, (1 - x) * space.height] : [x * space.width, y * space.height]; /** - * agent-device's in-app back taps a navigation control and cannot leave a sheet without one. - * Swipe such a sheet down, then require the observed destination to change. + * agent-device's back presses the navigation bar back button, else swipes from the left edge, which + * cannot leave a modal or a sheet. Press the modal's one dismiss control that agent-device names, or swipe + * a sheet down, then require the observed destination to change. */ const back = async (signal: AbortSignal): Promise<{ via: string } | undefined> => { try { await command(["back", "--level", "full", "--json"], signal); return; } catch (error) { - if (!(error instanceof CiError) || !/in-app back control is not available/.test(error.message)) throw error; + if (!backLeftScreenUnchanged(error)) throw error; const before = (await readSnapshot(signal)).snapshot.nodes; + const control = loneBackDismissControl(error); + if (control && riskOf(control.label, []) === "safe") { + await command(["press", control.selector, "--level", "full", "--json"], signal); + for (let attempt = 0; attempt < 4; attempt++) { + await sleep(500, undefined, { signal: liveSignal(signal) }); + if (observerIdentity((await readSnapshot(signal)).snapshot.nodes) !== observerIdentity(before)) return { via: "dismiss-control" }; + } + } let gesture = sheetDismissGesture(before); if (!gesture) throw error; const gestures: [number, number, number, number][] = []; diff --git a/src/shared/agent-device-version.ts b/src/shared/agent-device-version.ts index aaadb8d6..f95536eb 100644 --- a/src/shared/agent-device-version.ts +++ b/src/shared/agent-device-version.ts @@ -3,4 +3,4 @@ * controller (`--package-version`), the daemon health check, and the screenshot diff tool must * all be this version; each caller checks it by exact equality. */ -export const AGENT_DEVICE_VERSION = "0.21.29-nightly.20261018.74"; +export const AGENT_DEVICE_VERSION = "0.21.39"; diff --git a/src/shared/controller-cache.ts b/src/shared/controller-cache.ts index 2db6e352..0b0abd33 100644 --- a/src/shared/controller-cache.ts +++ b/src/shared/controller-cache.ts @@ -21,7 +21,7 @@ export type ControllerName = "agent-device" | "argent"; /** Each controller's npm release, the file Explore runs from it, and the integrity npm publishes for it. */ export const CONTROLLER_PACKAGES: Record = { "agent-device": { name: "agent-device", version: AGENT_DEVICE_VERSION, bin: "bin/agent-device.mjs", - integrity: "sha512-7+ajWLr0J+8iLZ9q11ERbj6U14mC20umBK5BQQdw5aERA79yeXc3Ze6KmtvnILnWjEtoVkYQsXQasL8RL6Uatw==" }, + integrity: "sha512-flgrilleoUU9WNd9BlU8h5KMt8zv0nKWBkqJbzk/bb2NBRA+7H1FGja+RIAuapb7HXdsjK4EPnYFP37N9S8nMA==" }, argent: { name: "@swmansion/argent", version: ARGENT_VERSION, bin: "dist/cli.js", integrity: "sha512-XXWlWZJlHKqnAvNx3RDhuiVFQOPqh5EDB76+mFAyqC2mutC6irzw03+M/6S/WQ+Dzz2IYOkEfOzQnOOdhrGgJg==" }, }; diff --git a/test/node/argent-driver.test.mjs b/test/node/argent-driver.test.mjs index 290115c7..f0a45f75 100644 --- a/test/node/argent-driver.test.mjs +++ b/test/node/argent-driver.test.mjs @@ -222,11 +222,11 @@ test("keyboard dismiss matches agent-device: Android presses back; iOS taps a di assert.deepEqual(dismissed.json.data, { platform: "android", action: "dismiss", wasVisible: true, dismissed: true, visible: false }); assert.deepEqual(android.calls.find(call => call.tool === "button").args, { udid: "emulator-5554", button: "back" }); - // agent-device fails with this message when the keyboard has no dismiss key, and callers fall back to the return key. + // agent-device refuses with this reason when the keyboard has no dismiss key, and callers fall back to the return key. const noKey = harness("ios", { "list-devices": IOS_DEVICES, "ui-tree": { ...IOS_HOME, keyboardVisible: true } }); const failed = await noKey.run(["keyboard", "dismiss", ...flags("ios")]); assert.equal(failed.exitCode, 1); - assert.match(failed.json.error.message, /^Unable to dismiss the iOS keyboard/); + assert.equal(failed.json.error.details.reason, "keyboard_dismiss_unavailable"); const hidden = harness("ios", { "list-devices": IOS_DEVICES, "ui-tree": IOS_HOME }); assert.equal((await hidden.run(["keyboard", "dismiss", ...flags("ios")])).json.data.wasVisible, false); @@ -237,6 +237,7 @@ test("iOS back with no navigation bar back control fails with agent-device's mes const back = await run(["back", ...flags("ios")]); assert.equal(back.exitCode, 1); assert.equal(back.json.error.message, "in-app back control is not available"); + assert.equal(back.json.error.details.reason, "back_swipe_not_dispatched"); }); test("alerts read and answer in each platform's agent-device shape", async () => { @@ -298,7 +299,7 @@ test("a tool-server that is still starting is waited for before the action is se test("an unreachable tool-server before a UI action is the retryable 'Remote daemon is unavailable'", async () => { const { run, calls } = harness("ios", { "list-devices": IOS_DEVICES, "ui-tree": IOS_HOME, "gesture-tap": {} }, { reachable: false }); const pressed = await run(["press", "10", "10", ...flags("ios")]); - assert.deepEqual(pressed.json.error, { code: "COMMAND_FAILED", message: "Remote daemon is unavailable" }); + assert.deepEqual(pressed.json.error, { code: "COMMAND_FAILED", message: "Remote daemon is unavailable", details: { reason: "remote_daemon_unavailable", dispatched: "no" } }); assert.ok(!calls.some(call => call.tool === "gesture-tap")); }); diff --git a/test/node/ci-map.test.mjs b/test/node/ci-map.test.mjs index 5f43faa4..53ce6fd6 100644 --- a/test/node/ci-map.test.mjs +++ b/test/node/ci-map.test.mjs @@ -58,9 +58,9 @@ const recoveredSnapshot = result => { const value = JSON.parse(result.stdout); return { ...result, stdout: JSON.stringify({ ...value, data: { ...value.data, snapshotQuality: { state: 'recovered', backend: 'tree' } } }) }; }; -// agent-device 0.21.15 raises this before sending a command when the remote daemon's /health check fails. +// agent-device 0.21.39 raises this before sending a command when the remote daemon's /health check fails. const daemonUnavailable = () => json({ success: false, error: { code: 'COMMAND_FAILED', message: 'Remote daemon is unavailable', - details: { daemonBaseUrl: 'https://daemon.example.invalid', hint: 'Verify AGENT_DEVICE_DAEMON_BASE_URL points to a reachable daemon.' } } }); + details: { reason: 'remote_daemon_unavailable', dispatched: 'no', daemonBaseUrl: 'https://daemon.example.invalid', attempts: 1 } } }); const value = (request, name) => request.args[request.args.indexOf(name) + 1]; // agent-device 0.21.15 while its XCTest snapshot tiers are deferred and iOS shows the Save Password // prompt, which com.apple.SafariViewService hosts. @@ -3706,7 +3706,8 @@ function sheetRunner(f, { dismissible = true, sheetOrigin = 72 } = {}) { if (request.command === 'fake-controller') { const [command] = request.args; if (command === 'open' && request.args[2]?.startsWith('sample://')) { path = '/' + request.args[2].slice('sample://'.length); sheet = path === '/settings'; } - if (command === 'back' && sheet) return json({ success: false, error: { code: 'COMMAND_FAILED', message: 'in-app back control is not available' } }); + // agent-device 0.21.39 swipes from the left edge when there is no navigation back button; a sheet stays. + if (command === 'back' && sheet) return json({ success: false, error: { code: 'COMMAND_FAILED', message: 'The back swipe changed nothing.', details: { reason: 'back_swipe_no_change', runnerErrorCode: 'BACK_SWIPE_NO_CHANGE', dispatched: 'unknown' } } }); if (command === 'swipe') { swipes.push(request.args.slice(1, 5)); if (dismissible) { sheet = false; path = '/home'; } return ok({}); } } const response = f.options.runner(request); @@ -3780,10 +3781,66 @@ test('agent back reports a sheet that stays open, and never swipes without sheet const plainTrace = (await readFile(join(plain.options.outputDirectory, 'capture-diagnostics/agent-navigation.jsonl'), 'utf8')).trim().split('\n').map(JSON.parse); const unavailable = plainTrace.find(event => event.action === 'back'); assert.equal(unavailable.result.error.code, 'COMMAND_FAILED'); - assert.match(unavailable.result.error.message, /in-app back control is not available/); + assert.match(unavailable.result.error.message, /The back swipe changed nothing/); assert.equal(screen.swipes.length, 0); }); +/** A full-screen modal whose only exit is a header control agent-device names when its back changes nothing. */ +function modalRunner(f, { label = 'Close' } = {}) { + let path = '/', modal = false; + const presses = []; + const selector = `role=button label="${label}"`; + const runner = request => { + if (request.command === 'fake-controller') { + const [command] = request.args; + if (command === 'open' && request.args[2]?.startsWith('sample://')) { path = '/' + request.args[2].slice('sample://'.length); modal = path === '/settings'; } + if (command === 'back' && modal) return json({ success: false, error: { code: 'COMMAND_FAILED', message: 'The back swipe changed nothing.', + details: { reason: 'back_swipe_no_change', runnerErrorCode: 'BACK_SWIPE_NO_CHANGE', dispatched: 'unknown', dismissControls: [{ selector, label }] } } }); + if (command === 'press' && request.args[1] === selector) { presses.push(request.args[1]); modal = false; path = '/home'; return ok({}); } + } + const response = f.options.runner(request); + if (request.command === 'fake-controller' && request.args[0] === 'snapshot') { + const value = JSON.parse(response.stdout); + for (const node of value.data.nodes) if (node.identifier === CAPTURE_OBSERVER_IDENTIFIER) { + node.label = JSON.stringify({ provider: 'expo-router', pathname: path, segments: path.split('/').filter(Boolean), parameters: {} }); + node.rect = { x: 0, y: 0, width: 1, height: 1 }; + } + value.data.nodes.unshift({ type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 }, visible: true }); + return { ...response, stdout: JSON.stringify(value) }; + } + return response; + }; + return { runner, presses }; +} + +const modalBackAgent = f => agentFixture([ + { action: 'capture', targetId: f.ids.home, description: 'Loaded home.', requiredLabels: ['screen-title'] }, + { action: 'open', url: 'sample://settings' }, + { action: 'capture', targetId: f.ids.settings, description: 'Settings modal.', requiredLabels: ['screen-title'] }, + { action: 'back' }, + { action: 'done' }, +]); + +test('agent back presses the one dismiss control agent-device names when its back changes nothing', async t => { + const f = await fixture(t); + const modal = modalRunner(f); + await runCiMap({ ...f.options, runner: modal.runner, agent: modalBackAgent(f), setup: [{ action: 'open', url: 'sample://home' }] }); + assert.deepEqual(modal.presses, ['role=button label="Close"']); + const trace = (await readFile(join(f.options.outputDirectory, 'capture-diagnostics/agent-navigation.jsonl'), 'utf8')).trim().split('\n').map(JSON.parse); + const back = trace.find(event => event.action === 'back'); + assert.equal(back.result.performed, 'back'); + assert.equal(back.result.via, 'dismiss-control'); +}); + +test('agent back never presses a named dismiss control that reads as destructive', async t => { + const f = await fixture(t); + const modal = modalRunner(f, { label: 'Cancel subscription' }); + await runCiMap({ ...f.options, runner: modal.runner, agent: modalBackAgent(f), setup: [{ action: 'open', url: 'sample://home' }] }); + assert.deepEqual(modal.presses, []); + const trace = (await readFile(join(f.options.outputDirectory, 'capture-diagnostics/agent-navigation.jsonl'), 'utf8')).trim().split('\n').map(JSON.parse); + assert.equal(trace.find(event => event.action === 'back').result.error.code, 'COMMAND_FAILED'); +}); + test('agent back swipes again when the first drag moves a sheet without closing it', async t => { const f = await fixture(t); const sheet = sheetRunner(f, { dismissible: false }); diff --git a/test/node/public-comment.test.mjs b/test/node/public-comment.test.mjs index e7f42319..74ca121d 100644 --- a/test/node/public-comment.test.mjs +++ b/test/node/public-comment.test.mjs @@ -746,10 +746,10 @@ const footnote = 'Added text “Changes apply live while playing. Base resolutio const footnoteText = 'Added text “Changes apply live while playing\\. Base resolution sets the step grid for lanes you add next\\.”'; function euxyReview(target, options = {}) { const own = { sourceImpact: ['own'], sourceScope: 'screen' }; - const matched = { verdict: 'unchanged', reason: 'matched', mismatchPercentage: 0, threshold: 0.1, tool: 'agent-device@0.21.29' }; + const matched = { verdict: 'unchanged', reason: 'matched', mismatchPercentage: 0, threshold: 0.1, tool: 'agent-device@0.21.39' }; const changed = (description, mismatchPercentage) => ({ ...own, review: { status: 'modified', description, facts: [{ category: 'element', templateId: 'element.added', bullet: description }] }, - visual: { verdict: 'modified', reason: 'repeated-cross-build-delta', mismatchPercentage, threshold: 0.1, tool: 'agent-device@0.21.29' } }); + visual: { verdict: 'modified', reason: 'repeated-cross-build-delta', mismatchPercentage, threshold: 0.1, tool: 'agent-device@0.21.39' } }); const inTreeOnly = { ...changed(footnote, 0), visual: matched }; if (options.offScreen !== false) inTreeOnly.review.offScreen = true; if (options.diffFailed) inTreeOnly.visual = { ...matched, reason: 'diff-failed' }; @@ -799,7 +799,7 @@ test('a change one platform has and another lacks keeps "only" and "not on"', as const iosOnly = target => { const reviewed = euxyReview(target); if (target === 'android') for (const entry of reviewed.surfaces) Object.assign(entry, { review: { status: 'unchanged', description: 'No visible change' }, - visual: { verdict: 'unchanged', reason: 'matched', mismatchPercentage: 0, threshold: 0.1, tool: 'agent-device@0.21.29' } }); + visual: { verdict: 'unchanged', reason: 'matched', mismatchPercentage: 0, threshold: 0.1, tool: 'agent-device@0.21.39' } }); return reviewed; }; const { rows } = await euxyRows(iosOnly); diff --git a/test/node/public-review-validation.test.mjs b/test/node/public-review-validation.test.mjs index 932ab537..26774434 100644 --- a/test/node/public-review-validation.test.mjs +++ b/test/node/public-review-validation.test.mjs @@ -325,7 +325,7 @@ test("a review says its change is off-screen only on a modified verdict whose sc const offScreen = (status, verdict) => { const input = review(); input.surfaces[0].review = { status, description: status === "modified" ? "Added text “lanes you add next”." : "No visible change", offScreen: true }; - input.surfaces[0].visual = { verdict, reason: verdict === "unchanged" ? "matched" : "repeated-cross-build-delta", mismatchPercentage: verdict === "unchanged" ? 0 : 3, threshold: 0.1, tool: "agent-device@0.21.29" }; + input.surfaces[0].visual = { verdict, reason: verdict === "unchanged" ? "matched" : "repeated-cross-build-delta", mismatchPercentage: verdict === "unchanged" ? 0 : 3, threshold: 0.1, tool: "agent-device@0.21.39" }; return input; }; assert.doesNotThrow(() => assertPublicReview(offScreen("modified", "unchanged"))); From 745c4aa97f4b6d77637896056658405115ccc7a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 15:40:18 +0100 Subject: [PATCH 2/8] Binary back counts a dismiss-control press only when the screen changed As in source-driven capture, explore:binary compares the screen before and after it presses the dismiss control agent-device named, and fails the back with agent-device's refusal when nothing changed. Co-Authored-By: Claude Opus 5.5 --- src/binary/run.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/binary/run.ts b/src/binary/run.ts index afd15b78..044db8bb 100644 --- a/src/binary/run.ts +++ b/src/binary/run.ts @@ -29,7 +29,7 @@ import { packageBin } from "../shared/package-paths.js"; import { assertDecodedPng, solidColour } from "../shared/png-evidence.js"; import { CONTRACT, type ExploreMap, type Platform } from "../types.js"; import { assertEmulatorCompatible, extractIosBundle, inspectArchive, type ArchiveKind, type InspectedArchive, readArchiveIcon } from "./archive.js"; -import { riskOf, runBlackBoxExploration, type BlackBoxExplorerResult, type BlackBoxFrame, SYSTEM_DIALOG_PACKAGES } from "./explorer.js"; +import { riskOf, runBlackBoxExploration, screenEvidenceKey, type BlackBoxExplorerResult, type BlackBoxFrame, SYSTEM_DIALOG_PACKAGES } from "./explorer.js"; export const DEFAULT_IOS_DEVICE = "iPhone 17 Pro"; export const DEFAULT_BINARY_PROJECT = "expo-explore"; @@ -508,7 +508,13 @@ export async function exploreConnectedSession(ctx: ConnectedSessionExploration): try { controller(["back", "--in-app", "--level", "full", "--json"], 120_000); } catch (error) { const control = loneBackDismissControl(error); - if (control && riskOf(control.label, allowDestructive ?? []) === "safe") controller(["press", "--level", "full", "--json", "--", control.selector], 120_000); + if (control && riskOf(control.label, allowDestructive ?? []) === "safe") { + // As in source-driven capture, the press counts as back only when the screen changed. + const before = screenEvidenceKey((await observe(false)).nodes); + controller(["press", "--level", "full", "--json", "--", control.selector], 120_000); + await settled(); + if (screenEvidenceKey((await observe(false)).nodes) === before) throw error; + } else if (controllerFailureReason(error) === "back_swipe_not_dispatched") controller(["back", "--system", "--level", "full", "--json"], 120_000); else throw error; } From d18339df4f441170c0291e34a3d5ffe80b33712f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 16:23:03 +0100 Subject: [PATCH 3/8] Back accepts a modal that closes without a route change, and never swipes after a press A modal kept in a screen's own state keeps the route when its Close control dismisses it, so the destination check alone read that as a failure and fell through to the sheet swipe-down with the geometry read before the press. A press now counts when the route changes or the pressed control is gone. If neither happens, back fails with AGENT_DISMISS_CONTROL_FAILED instead of swiping. Co-Authored-By: Claude Opus 5.5 --- src/capture/route-capture.ts | 12 +++++++++++- test/node/ci-map.test.mjs | 29 +++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/capture/route-capture.ts b/src/capture/route-capture.ts index 204c9698..0d14d93e 100644 --- a/src/capture/route-capture.ts +++ b/src/capture/route-capture.ts @@ -783,11 +783,21 @@ export async function capturePlannedRoutes(options: { const before = (await readSnapshot(signal)).snapshot.nodes; const control = loneBackDismissControl(error); if (control && riskOf(control.label, []) === "safe") { + // A modal kept in a screen's own state closes without a route change, so the control leaving counts too. + const controlShown = (nodes: readonly Record[]) => nodes.some(node => + node.label === control.label && /button/i.test(String(node.type ?? "")) && node.visible !== false); await command(["press", control.selector, "--level", "full", "--json"], signal); + let after = before; for (let attempt = 0; attempt < 4; attempt++) { await sleep(500, undefined, { signal: liveSignal(signal) }); - if (observerIdentity((await readSnapshot(signal)).snapshot.nodes) !== observerIdentity(before)) return { via: "dismiss-control" }; + after = (await readSnapshot(signal)).snapshot.nodes; + if (observerIdentity(after) !== observerIdentity(before) || !controlShown(after)) return { via: "dismiss-control" }; } + // The press ran, so a sheet swipe from the geometry read before it could land on whatever is there now. + throw new CiError({ code: "AGENT_DISMISS_CONTROL_FAILED", stage: "capture", + message: `Pressed "${control.label}", the control agent-device named for leaving this screen, but the screen did not change.`, + nextAction: "Inspect the returned accessibility: a confirmation may have appeared. Use an observed control rather than repeating back.", + details: { back: serializeCiError(error, { secrets }), control: control.label, destinationBefore: observerIdentity(before), destinationAfter: observerIdentity(after) } }); } let gesture = sheetDismissGesture(before); if (!gesture) throw error; diff --git a/test/node/ci-map.test.mjs b/test/node/ci-map.test.mjs index 53ce6fd6..957de2be 100644 --- a/test/node/ci-map.test.mjs +++ b/test/node/ci-map.test.mjs @@ -3786,9 +3786,9 @@ test('agent back reports a sheet that stays open, and never swipes without sheet }); /** A full-screen modal whose only exit is a header control agent-device names when its back changes nothing. */ -function modalRunner(f, { label = 'Close' } = {}) { +function modalRunner(f, { label = 'Close', sameRoute = false, closes = true } = {}) { let path = '/', modal = false; - const presses = []; + const presses = [], swipes = []; const selector = `role=button label="${label}"`; const runner = request => { if (request.command === 'fake-controller') { @@ -3796,7 +3796,8 @@ function modalRunner(f, { label = 'Close' } = {}) { if (command === 'open' && request.args[2]?.startsWith('sample://')) { path = '/' + request.args[2].slice('sample://'.length); modal = path === '/settings'; } if (command === 'back' && modal) return json({ success: false, error: { code: 'COMMAND_FAILED', message: 'The back swipe changed nothing.', details: { reason: 'back_swipe_no_change', runnerErrorCode: 'BACK_SWIPE_NO_CHANGE', dispatched: 'unknown', dismissControls: [{ selector, label }] } } }); - if (command === 'press' && request.args[1] === selector) { presses.push(request.args[1]); modal = false; path = '/home'; return ok({}); } + if (command === 'press' && request.args[1] === selector) { presses.push(request.args[1]); if (closes) { modal = false; if (!sameRoute) path = '/home'; } return ok({}); } + if (command === 'swipe') swipes.push(request.args.slice(1, 5)); } const response = f.options.runner(request); if (request.command === 'fake-controller' && request.args[0] === 'snapshot') { @@ -3806,11 +3807,12 @@ function modalRunner(f, { label = 'Close' } = {}) { node.rect = { x: 0, y: 0, width: 1, height: 1 }; } value.data.nodes.unshift({ type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 }, visible: true }); + if (modal) value.data.nodes.push({ type: 'Button', label, rect: { x: 340, y: 60, width: 44, height: 44 }, visible: true }); return { ...response, stdout: JSON.stringify(value) }; } return response; }; - return { runner, presses }; + return { runner, presses, swipes }; } const modalBackAgent = f => agentFixture([ @@ -3832,6 +3834,25 @@ test('agent back presses the one dismiss control agent-device names when its bac assert.equal(back.result.via, 'dismiss-control'); }); +test('a dismiss control that closes a modal without changing the route counts once the control is gone', async t => { + const f = await fixture(t); + const modal = modalRunner(f, { sameRoute: true }); + await runCiMap({ ...f.options, runner: modal.runner, agent: modalBackAgent(f), setup: [{ action: 'open', url: 'sample://home' }] }); + assert.deepEqual(modal.presses, ['role=button label="Close"']); + const trace = (await readFile(join(f.options.outputDirectory, 'capture-diagnostics/agent-navigation.jsonl'), 'utf8')).trim().split('\n').map(JSON.parse); + assert.equal(trace.find(event => event.action === 'back').result.via, 'dismiss-control'); +}); + +test('a dismiss control press that changes nothing is reported, and no sheet swipe follows it', async t => { + const f = await fixture(t); + const modal = modalRunner(f, { closes: false }); + await runCiMap({ ...f.options, runner: modal.runner, agent: modalBackAgent(f), setup: [{ action: 'open', url: 'sample://home' }] }); + assert.deepEqual(modal.presses, ['role=button label="Close"']); + assert.deepEqual(modal.swipes, []); + const trace = (await readFile(join(f.options.outputDirectory, 'capture-diagnostics/agent-navigation.jsonl'), 'utf8')).trim().split('\n').map(JSON.parse); + assert.equal(trace.find(event => event.action === 'back').result.error.code, 'AGENT_DISMISS_CONTROL_FAILED'); +}); + test('agent back never presses a named dismiss control that reads as destructive', async t => { const f = await fixture(t); const modal = modalRunner(f, { label: 'Cancel subscription' }); From 01f4877ee094fbe4161c3fd11396cb7a2f845224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 16:28:45 +0100 Subject: [PATCH 4/8] A pressed dismiss control that brings up an alert does not count as leaving the screen The pressed control disappearing counted as back, but a confirmation such as "Discard changes?" also hides it while the screen stays. The control leaving now counts only when no alert took its place; otherwise back fails with AGENT_DISMISS_CONTROL_FAILED. Co-Authored-By: Claude Opus 5.5 --- src/capture/route-capture.ts | 6 ++++-- test/node/ci-map.test.mjs | 18 ++++++++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/capture/route-capture.ts b/src/capture/route-capture.ts index 0d14d93e..723abc45 100644 --- a/src/capture/route-capture.ts +++ b/src/capture/route-capture.ts @@ -783,15 +783,17 @@ export async function capturePlannedRoutes(options: { const before = (await readSnapshot(signal)).snapshot.nodes; const control = loneBackDismissControl(error); if (control && riskOf(control.label, []) === "safe") { - // A modal kept in a screen's own state closes without a route change, so the control leaving counts too. + // A modal kept in a screen's own state closes without a route change, so the control leaving counts too, + // unless an alert took its place (a "Discard changes?" confirmation): that screen is not dismissed. const controlShown = (nodes: readonly Record[]) => nodes.some(node => node.label === control.label && /button/i.test(String(node.type ?? "")) && node.visible !== false); + const alertShown = (nodes: readonly Record[]) => nodes.some(node => node.type === "Alert"); await command(["press", control.selector, "--level", "full", "--json"], signal); let after = before; for (let attempt = 0; attempt < 4; attempt++) { await sleep(500, undefined, { signal: liveSignal(signal) }); after = (await readSnapshot(signal)).snapshot.nodes; - if (observerIdentity(after) !== observerIdentity(before) || !controlShown(after)) return { via: "dismiss-control" }; + if (observerIdentity(after) !== observerIdentity(before) || !controlShown(after) && !alertShown(after)) return { via: "dismiss-control" }; } // The press ran, so a sheet swipe from the geometry read before it could land on whatever is there now. throw new CiError({ code: "AGENT_DISMISS_CONTROL_FAILED", stage: "capture", diff --git a/test/node/ci-map.test.mjs b/test/node/ci-map.test.mjs index 957de2be..836edc7a 100644 --- a/test/node/ci-map.test.mjs +++ b/test/node/ci-map.test.mjs @@ -3786,8 +3786,8 @@ test('agent back reports a sheet that stays open, and never swipes without sheet }); /** A full-screen modal whose only exit is a header control agent-device names when its back changes nothing. */ -function modalRunner(f, { label = 'Close', sameRoute = false, closes = true } = {}) { - let path = '/', modal = false; +function modalRunner(f, { label = 'Close', sameRoute = false, closes = true, confirms = false } = {}) { + let path = '/', modal = false, confirming = false; const presses = [], swipes = []; const selector = `role=button label="${label}"`; const runner = request => { @@ -3796,7 +3796,7 @@ function modalRunner(f, { label = 'Close', sameRoute = false, closes = true } = if (command === 'open' && request.args[2]?.startsWith('sample://')) { path = '/' + request.args[2].slice('sample://'.length); modal = path === '/settings'; } if (command === 'back' && modal) return json({ success: false, error: { code: 'COMMAND_FAILED', message: 'The back swipe changed nothing.', details: { reason: 'back_swipe_no_change', runnerErrorCode: 'BACK_SWIPE_NO_CHANGE', dispatched: 'unknown', dismissControls: [{ selector, label }] } } }); - if (command === 'press' && request.args[1] === selector) { presses.push(request.args[1]); if (closes) { modal = false; if (!sameRoute) path = '/home'; } return ok({}); } + if (command === 'press' && request.args[1] === selector) { presses.push(request.args[1]); if (confirms) confirming = true; else if (closes) { modal = false; if (!sameRoute) path = '/home'; } return ok({}); } if (command === 'swipe') swipes.push(request.args.slice(1, 5)); } const response = f.options.runner(request); @@ -3807,7 +3807,8 @@ function modalRunner(f, { label = 'Close', sameRoute = false, closes = true } = node.rect = { x: 0, y: 0, width: 1, height: 1 }; } value.data.nodes.unshift({ type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 }, visible: true }); - if (modal) value.data.nodes.push({ type: 'Button', label, rect: { x: 340, y: 60, width: 44, height: 44 }, visible: true }); + if (confirming) value.data.nodes.push({ type: 'Alert', label: 'Discard changes?', rect: { x: 50, y: 300, width: 300, height: 200 }, visible: true }); + else if (modal) value.data.nodes.push({ type: 'Button', label, rect: { x: 340, y: 60, width: 44, height: 44 }, visible: true }); return { ...response, stdout: JSON.stringify(value) }; } return response; @@ -3853,6 +3854,15 @@ test('a dismiss control press that changes nothing is reported, and no sheet swi assert.equal(trace.find(event => event.action === 'back').result.error.code, 'AGENT_DISMISS_CONTROL_FAILED'); }); +test('a dismiss control press that brings up a confirmation is not reported as leaving the screen', async t => { + const f = await fixture(t); + const modal = modalRunner(f, { sameRoute: true, confirms: true }); + await runCiMap({ ...f.options, runner: modal.runner, agent: modalBackAgent(f), setup: [{ action: 'open', url: 'sample://home' }] }); + assert.deepEqual(modal.swipes, []); + const trace = (await readFile(join(f.options.outputDirectory, 'capture-diagnostics/agent-navigation.jsonl'), 'utf8')).trim().split('\n').map(JSON.parse); + assert.equal(trace.find(event => event.action === 'back').result.error.code, 'AGENT_DISMISS_CONTROL_FAILED'); +}); + test('agent back never presses a named dismiss control that reads as destructive', async t => { const f = await fixture(t); const modal = modalRunner(f, { label: 'Cancel subscription' }); From add2d04a753d9e92448c867a0dd12cd69f3132f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 17:39:46 +0100 Subject: [PATCH 5/8] A dismiss-control press never counts as back while an alert is showing A "Discard changes?" confirmation after the press changes the screen, and can hide the route observer, so either signal alone read it as having left. Both code paths now require that no alert is showing before a press counts as back. Co-Authored-By: Claude Opus 5.5 --- src/binary/run.ts | 4 +++- src/capture/route-capture.ts | 2 +- test/node/ci-map.test.mjs | 14 +++++++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/binary/run.ts b/src/binary/run.ts index 044db8bb..95686455 100644 --- a/src/binary/run.ts +++ b/src/binary/run.ts @@ -513,7 +513,9 @@ export async function exploreConnectedSession(ctx: ConnectedSessionExploration): const before = screenEvidenceKey((await observe(false)).nodes); controller(["press", "--level", "full", "--json", "--", control.selector], 120_000); await settled(); - if (screenEvidenceKey((await observe(false)).nodes) === before) throw error; + // A "Discard changes?" confirmation changes the screen too, but leaves it in place. + const after = (await observe(false)).nodes; + if (screenEvidenceKey(after) === before || after.some(node => node.type === "Alert")) throw error; } else if (controllerFailureReason(error) === "back_swipe_not_dispatched") controller(["back", "--system", "--level", "full", "--json"], 120_000); else throw error; diff --git a/src/capture/route-capture.ts b/src/capture/route-capture.ts index 723abc45..017e4f8c 100644 --- a/src/capture/route-capture.ts +++ b/src/capture/route-capture.ts @@ -793,7 +793,7 @@ export async function capturePlannedRoutes(options: { for (let attempt = 0; attempt < 4; attempt++) { await sleep(500, undefined, { signal: liveSignal(signal) }); after = (await readSnapshot(signal)).snapshot.nodes; - if (observerIdentity(after) !== observerIdentity(before) || !controlShown(after) && !alertShown(after)) return { via: "dismiss-control" }; + if (!alertShown(after) && (observerIdentity(after) !== observerIdentity(before) || !controlShown(after))) return { via: "dismiss-control" }; } // The press ran, so a sheet swipe from the geometry read before it could land on whatever is there now. throw new CiError({ code: "AGENT_DISMISS_CONTROL_FAILED", stage: "capture", diff --git a/test/node/ci-map.test.mjs b/test/node/ci-map.test.mjs index 836edc7a..1415e310 100644 --- a/test/node/ci-map.test.mjs +++ b/test/node/ci-map.test.mjs @@ -3786,7 +3786,7 @@ test('agent back reports a sheet that stays open, and never swipes without sheet }); /** A full-screen modal whose only exit is a header control agent-device names when its back changes nothing. */ -function modalRunner(f, { label = 'Close', sameRoute = false, closes = true, confirms = false } = {}) { +function modalRunner(f, { label = 'Close', sameRoute = false, closes = true, confirms = false, hidesObserver = false } = {}) { let path = '/', modal = false, confirming = false; const presses = [], swipes = []; const selector = `role=button label="${label}"`; @@ -3807,6 +3807,7 @@ function modalRunner(f, { label = 'Close', sameRoute = false, closes = true, con node.rect = { x: 0, y: 0, width: 1, height: 1 }; } value.data.nodes.unshift({ type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 }, visible: true }); + if (confirming && hidesObserver) value.data.nodes = value.data.nodes.filter(node => node.identifier !== CAPTURE_OBSERVER_IDENTIFIER); if (confirming) value.data.nodes.push({ type: 'Alert', label: 'Discard changes?', rect: { x: 50, y: 300, width: 300, height: 200 }, visible: true }); else if (modal) value.data.nodes.push({ type: 'Button', label, rect: { x: 340, y: 60, width: 44, height: 44 }, visible: true }); return { ...response, stdout: JSON.stringify(value) }; @@ -3863,6 +3864,17 @@ test('a dismiss control press that brings up a confirmation is not reported as l assert.equal(trace.find(event => event.action === 'back').result.error.code, 'AGENT_DISMISS_CONTROL_FAILED'); }); +test('a confirmation that hides the route observer is not reported as leaving the screen', async t => { + const f = await fixture(t); + const modal = modalRunner(f, { sameRoute: true, confirms: true, hidesObserver: true }); + await runCiMap({ ...f.options, runner: modal.runner, agent: modalBackAgent(f), setup: [{ action: 'open', url: 'sample://home' }] }); + assert.deepEqual(modal.swipes, []); + const trace = (await readFile(join(f.options.outputDirectory, 'capture-diagnostics/agent-navigation.jsonl'), 'utf8')).trim().split('\n').map(JSON.parse); + const back = trace.find(event => event.action === 'back'); + assert.notEqual(back.result.via, 'dismiss-control'); + assert.ok(back.result.error, 'back must fail while the confirmation is showing'); +}); + test('agent back never presses a named dismiss control that reads as destructive', async t => { const f = await fixture(t); const modal = modalRunner(f, { label: 'Cancel subscription' }); From 74378c1e00be6a7e10250b724813d06fc2c1d52a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 19 Nov 2026 16:27:49 +0100 Subject: [PATCH 6/8] Match an unlabeled dismiss control by the identifier agent-device names agent-device names a dismiss control by its identifier when it has one, and its label may then be empty. The press check matched only the label, so a button with no label was never seen and the press counted as leaving even when the modal stayed. It now matches the control the way agent-device named it, and the control leaving counts only when it was on screen before the press. Co-Authored-By: Claude Opus 5.5 --- src/capture/controller.ts | 10 ++++++---- src/capture/route-capture.ts | 14 +++++++++----- test/node/ci-map.test.mjs | 27 +++++++++++++++++++++++---- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/src/capture/controller.ts b/src/capture/controller.ts index cca03ccb..e7837f49 100644 --- a/src/capture/controller.ts +++ b/src/capture/controller.ts @@ -126,14 +126,16 @@ export function backLeftScreenUnchanged(error: unknown): boolean { /** * The one dismiss control (Close, Cancel, Back) agent-device saw in the screen's top bar when its back changed * nothing (`details.dismissControls`). agent-device never presses it: on iOS back only pops the navigation stack. - * Undefined when it listed none or several. + * Undefined when it listed none or several. `identifier` is set when agent-device names the control by it; the + * label may then be empty. */ -export function loneBackDismissControl(error: unknown): { selector: string; label: string } | undefined { +export function loneBackDismissControl(error: unknown): { selector: string; label: string; identifier?: string } | undefined { if (controllerFailureReason(error) !== "back_swipe_no_change") return undefined; const controls = (nativeControllerFailures.get(error as CiError)?.details as Record | undefined)?.dismissControls; if (!Array.isArray(controls) || controls.length !== 1 || !isRecord(controls[0])) return undefined; - const { selector, label } = controls[0]; - return typeof selector === "string" && selector.trim() && typeof label === "string" ? { selector, label } : undefined; + const { selector, label, identifier } = controls[0]; + if (typeof selector !== "string" || !selector.trim() || typeof label !== "string") return undefined; + return typeof identifier === "string" && identifier ? { selector, label, identifier } : { selector, label }; } function busyControllerReadFailure(error: unknown): boolean { diff --git a/src/capture/route-capture.ts b/src/capture/route-capture.ts index 017e4f8c..17888c83 100644 --- a/src/capture/route-capture.ts +++ b/src/capture/route-capture.ts @@ -782,24 +782,28 @@ export async function capturePlannedRoutes(options: { if (!backLeftScreenUnchanged(error)) throw error; const before = (await readSnapshot(signal)).snapshot.nodes; const control = loneBackDismissControl(error); - if (control && riskOf(control.label, []) === "safe") { + const controlName = control && (control.label || control.identifier); + if (control && controlName && riskOf(controlName, []) === "safe") { // A modal kept in a screen's own state closes without a route change, so the control leaving counts too, // unless an alert took its place (a "Discard changes?" confirmation): that screen is not dismissed. + // The control is matched the way agent-device named it: by identifier when it has one, else by label. const controlShown = (nodes: readonly Record[]) => nodes.some(node => - node.label === control.label && /button/i.test(String(node.type ?? "")) && node.visible !== false); + (control.identifier ? node.identifier === control.identifier : node.label === control.label) + && /button/i.test(String(node.type ?? "")) && node.visible !== false); + const controlWasShown = controlShown(before); const alertShown = (nodes: readonly Record[]) => nodes.some(node => node.type === "Alert"); await command(["press", control.selector, "--level", "full", "--json"], signal); let after = before; for (let attempt = 0; attempt < 4; attempt++) { await sleep(500, undefined, { signal: liveSignal(signal) }); after = (await readSnapshot(signal)).snapshot.nodes; - if (!alertShown(after) && (observerIdentity(after) !== observerIdentity(before) || !controlShown(after))) return { via: "dismiss-control" }; + if (!alertShown(after) && (observerIdentity(after) !== observerIdentity(before) || (controlWasShown && !controlShown(after)))) return { via: "dismiss-control" }; } // The press ran, so a sheet swipe from the geometry read before it could land on whatever is there now. throw new CiError({ code: "AGENT_DISMISS_CONTROL_FAILED", stage: "capture", - message: `Pressed "${control.label}", the control agent-device named for leaving this screen, but the screen did not change.`, + message: `Pressed "${controlName}", the control agent-device named for leaving this screen, but the screen did not change.`, nextAction: "Inspect the returned accessibility: a confirmation may have appeared. Use an observed control rather than repeating back.", - details: { back: serializeCiError(error, { secrets }), control: control.label, destinationBefore: observerIdentity(before), destinationAfter: observerIdentity(after) } }); + details: { back: serializeCiError(error, { secrets }), control: controlName, destinationBefore: observerIdentity(before), destinationAfter: observerIdentity(after) } }); } let gesture = sheetDismissGesture(before); if (!gesture) throw error; diff --git a/test/node/ci-map.test.mjs b/test/node/ci-map.test.mjs index 1415e310..90679289 100644 --- a/test/node/ci-map.test.mjs +++ b/test/node/ci-map.test.mjs @@ -3786,16 +3786,18 @@ test('agent back reports a sheet that stays open, and never swipes without sheet }); /** A full-screen modal whose only exit is a header control agent-device names when its back changes nothing. */ -function modalRunner(f, { label = 'Close', sameRoute = false, closes = true, confirms = false, hidesObserver = false } = {}) { +function modalRunner(f, { label = 'Close', identifier, sameRoute = false, closes = true, confirms = false, hidesObserver = false } = {}) { let path = '/', modal = false, confirming = false; const presses = [], swipes = []; - const selector = `role=button label="${label}"`; + // agent-device names a control by identifier when it has one no other button shares; its label may then be empty. + const selector = identifier ? `role=button id="${identifier}"` : `role=button label="${label}"`; + const named = identifier ? { selector, label, identifier } : { selector, label }; const runner = request => { if (request.command === 'fake-controller') { const [command] = request.args; if (command === 'open' && request.args[2]?.startsWith('sample://')) { path = '/' + request.args[2].slice('sample://'.length); modal = path === '/settings'; } if (command === 'back' && modal) return json({ success: false, error: { code: 'COMMAND_FAILED', message: 'The back swipe changed nothing.', - details: { reason: 'back_swipe_no_change', runnerErrorCode: 'BACK_SWIPE_NO_CHANGE', dispatched: 'unknown', dismissControls: [{ selector, label }] } } }); + details: { reason: 'back_swipe_no_change', runnerErrorCode: 'BACK_SWIPE_NO_CHANGE', dispatched: 'unknown', dismissControls: [named] } } }); if (command === 'press' && request.args[1] === selector) { presses.push(request.args[1]); if (confirms) confirming = true; else if (closes) { modal = false; if (!sameRoute) path = '/home'; } return ok({}); } if (command === 'swipe') swipes.push(request.args.slice(1, 5)); } @@ -3809,7 +3811,7 @@ function modalRunner(f, { label = 'Close', sameRoute = false, closes = true, con value.data.nodes.unshift({ type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 }, visible: true }); if (confirming && hidesObserver) value.data.nodes = value.data.nodes.filter(node => node.identifier !== CAPTURE_OBSERVER_IDENTIFIER); if (confirming) value.data.nodes.push({ type: 'Alert', label: 'Discard changes?', rect: { x: 50, y: 300, width: 300, height: 200 }, visible: true }); - else if (modal) value.data.nodes.push({ type: 'Button', label, rect: { x: 340, y: 60, width: 44, height: 44 }, visible: true }); + else if (modal) value.data.nodes.push({ type: 'Button', ...(label ? { label } : {}), ...(identifier ? { identifier } : {}), rect: { x: 340, y: 60, width: 44, height: 44 }, visible: true }); return { ...response, stdout: JSON.stringify(value) }; } return response; @@ -3875,6 +3877,23 @@ test('a confirmation that hides the route observer is not reported as leaving th assert.ok(back.result.error, 'back must fail while the confirmation is showing'); }); +test('an unlabeled dismiss control named by identifier counts only once that control is gone', async t => { + const closes = await fixture(t); + const closing = modalRunner(closes, { label: '', identifier: 'modal-close', sameRoute: true }); + await runCiMap({ ...closes.options, runner: closing.runner, agent: modalBackAgent(closes), setup: [{ action: 'open', url: 'sample://home' }] }); + assert.deepEqual(closing.presses, ['role=button id="modal-close"']); + const closed = (await readFile(join(closes.options.outputDirectory, 'capture-diagnostics/agent-navigation.jsonl'), 'utf8')).trim().split('\n').map(JSON.parse); + assert.equal(closed.find(event => event.action === 'back').result.via, 'dismiss-control'); + + const stays = await fixture(t); + const staying = modalRunner(stays, { label: '', identifier: 'modal-close', sameRoute: true, closes: false }); + await runCiMap({ ...stays.options, runner: staying.runner, agent: modalBackAgent(stays), setup: [{ action: 'open', url: 'sample://home' }] }); + assert.deepEqual(staying.presses, ['role=button id="modal-close"']); + assert.deepEqual(staying.swipes, []); + const stayed = (await readFile(join(stays.options.outputDirectory, 'capture-diagnostics/agent-navigation.jsonl'), 'utf8')).trim().split('\n').map(JSON.parse); + assert.equal(stayed.find(event => event.action === 'back').result.error.code, 'AGENT_DISMISS_CONTROL_FAILED'); +}); + test('agent back never presses a named dismiss control that reads as destructive', async t => { const f = await fixture(t); const modal = modalRunner(f, { label: 'Cancel subscription' }); From b36e438f73b188a96c4486a1b297c6345780eac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 19 Nov 2026 16:14:39 +0100 Subject: [PATCH 7/8] Pin agent-device 0.21.40 0.21.40 adds the keyboard fix found while testing this bump: keyboard dismiss no longer presses the return key, so on a field whose return key is Done it refuses with keyboard_dismiss_unavailable instead of submitting the form. The typed reasons Explore reads are unchanged. Co-Authored-By: Claude Opus 5.5 --- docs/quickstart-ci.md | 2 +- package-lock.json | 8 ++++---- package.json | 2 +- src/shared/agent-device-version.ts | 2 +- src/shared/controller-cache.ts | 2 +- test/node/ci-map.test.mjs | 4 ++-- test/node/public-comment.test.mjs | 6 +++--- test/node/public-review-validation.test.mjs | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/quickstart-ci.md b/docs/quickstart-ci.md index 0dc2fd7f..7ce2876e 100644 --- a/docs/quickstart-ci.md +++ b/docs/quickstart-ci.md @@ -289,7 +289,7 @@ The provider selects the runtime that owns the navigation loop: | `anthropic` | Claude Code through `@anthropic-ai/claude-agent-sdk` 0.3.285 and its platform-native package | `ANTHROPIC_API_KEY` | | `openai` | Codex app-server from `@openai/codex` 0.159.2 and its platform-native package | `OPENAI_API_KEY` | -Both runtimes use Explore's device tools to operate the existing Expo Simulators session through official `agent-device` 0.21.39. The runtime and native binaries come from the pinned tools installation. CI does not use the development-only `node_modules/.bin/claude` executable or an interactive subscription login. +Both runtimes use Explore's device tools to operate the existing Expo Simulators session through official `agent-device` 0.21.40. The runtime and native binaries come from the pinned tools installation. CI does not use the development-only `node_modules/.bin/claude` executable or an interactive subscription login. Create and commit the nonempty English prose file named by `agent.instructions`. The shared setup skill authors it from source and asks for missing test data or environment inputs. Describe how to reach the app's screens, necessary prerequisites, the representative state to use on both revisions, readiness signals, and actions to avoid. This file guides the runtime agent; it is not a list of mandatory YAML steps or a second screen inventory. diff --git a/package-lock.json b/package-lock.json index dba9f69e..7b79ae92 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,7 +29,7 @@ "@types/react-dom": "^19.2.4", "@vitejs/plugin-react": "^4.7.0", "@xyflow/react": "^12.11.3", - "agent-device": "0.21.39", + "agent-device": "0.21.40", "babel-plugin-react-compiler": "^1.0.0", "eslint": "^10.12.0", "eslint-plugin-react-hooks": "^7.1.1", @@ -4593,9 +4593,9 @@ } }, "node_modules/agent-device": { - "version": "0.21.39", - "resolved": "https://registry.npmjs.org/agent-device/-/agent-device-0.21.39.tgz", - "integrity": "sha512-flgrilleoUU9WNd9BlU8h5KMt8zv0nKWBkqJbzk/bb2NBRA+7H1FGja+RIAuapb7HXdsjK4EPnYFP37N9S8nMA==", + "version": "0.21.40", + "resolved": "https://registry.npmjs.org/agent-device/-/agent-device-0.21.40.tgz", + "integrity": "sha512-7UlxZ2L6kBAwhmPWdyHP5iM+810YenMMcVQgmsai8GJSF5QgacMflJ+2nXfEwgvRxbbw3H1BiUHzuDDMQxQePg==", "dev": true, "license": "MIT", "bin": { diff --git a/package.json b/package.json index b6e36d3b..19b59663 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,7 @@ "@types/react-dom": "^19.2.4", "@vitejs/plugin-react": "^4.7.0", "@xyflow/react": "^12.11.3", - "agent-device": "0.21.39", + "agent-device": "0.21.40", "babel-plugin-react-compiler": "^1.0.0", "eslint": "^10.12.0", "eslint-plugin-react-hooks": "^7.1.1", diff --git a/src/shared/agent-device-version.ts b/src/shared/agent-device-version.ts index f95536eb..0c08cad9 100644 --- a/src/shared/agent-device-version.ts +++ b/src/shared/agent-device-version.ts @@ -3,4 +3,4 @@ * controller (`--package-version`), the daemon health check, and the screenshot diff tool must * all be this version; each caller checks it by exact equality. */ -export const AGENT_DEVICE_VERSION = "0.21.39"; +export const AGENT_DEVICE_VERSION = "0.21.40"; diff --git a/src/shared/controller-cache.ts b/src/shared/controller-cache.ts index 0b0abd33..f03ddcd5 100644 --- a/src/shared/controller-cache.ts +++ b/src/shared/controller-cache.ts @@ -21,7 +21,7 @@ export type ControllerName = "agent-device" | "argent"; /** Each controller's npm release, the file Explore runs from it, and the integrity npm publishes for it. */ export const CONTROLLER_PACKAGES: Record = { "agent-device": { name: "agent-device", version: AGENT_DEVICE_VERSION, bin: "bin/agent-device.mjs", - integrity: "sha512-flgrilleoUU9WNd9BlU8h5KMt8zv0nKWBkqJbzk/bb2NBRA+7H1FGja+RIAuapb7HXdsjK4EPnYFP37N9S8nMA==" }, + integrity: "sha512-7UlxZ2L6kBAwhmPWdyHP5iM+810YenMMcVQgmsai8GJSF5QgacMflJ+2nXfEwgvRxbbw3H1BiUHzuDDMQxQePg==" }, argent: { name: "@swmansion/argent", version: ARGENT_VERSION, bin: "dist/cli.js", integrity: "sha512-XXWlWZJlHKqnAvNx3RDhuiVFQOPqh5EDB76+mFAyqC2mutC6irzw03+M/6S/WQ+Dzz2IYOkEfOzQnOOdhrGgJg==" }, }; diff --git a/test/node/ci-map.test.mjs b/test/node/ci-map.test.mjs index 90679289..3ef4993d 100644 --- a/test/node/ci-map.test.mjs +++ b/test/node/ci-map.test.mjs @@ -58,7 +58,7 @@ const recoveredSnapshot = result => { const value = JSON.parse(result.stdout); return { ...result, stdout: JSON.stringify({ ...value, data: { ...value.data, snapshotQuality: { state: 'recovered', backend: 'tree' } } }) }; }; -// agent-device 0.21.39 raises this before sending a command when the remote daemon's /health check fails. +// agent-device 0.21.40 raises this before sending a command when the remote daemon's /health check fails. const daemonUnavailable = () => json({ success: false, error: { code: 'COMMAND_FAILED', message: 'Remote daemon is unavailable', details: { reason: 'remote_daemon_unavailable', dispatched: 'no', daemonBaseUrl: 'https://daemon.example.invalid', attempts: 1 } } }); const value = (request, name) => request.args[request.args.indexOf(name) + 1]; @@ -3706,7 +3706,7 @@ function sheetRunner(f, { dismissible = true, sheetOrigin = 72 } = {}) { if (request.command === 'fake-controller') { const [command] = request.args; if (command === 'open' && request.args[2]?.startsWith('sample://')) { path = '/' + request.args[2].slice('sample://'.length); sheet = path === '/settings'; } - // agent-device 0.21.39 swipes from the left edge when there is no navigation back button; a sheet stays. + // agent-device 0.21.40 swipes from the left edge when there is no navigation back button; a sheet stays. if (command === 'back' && sheet) return json({ success: false, error: { code: 'COMMAND_FAILED', message: 'The back swipe changed nothing.', details: { reason: 'back_swipe_no_change', runnerErrorCode: 'BACK_SWIPE_NO_CHANGE', dispatched: 'unknown' } } }); if (command === 'swipe') { swipes.push(request.args.slice(1, 5)); if (dismissible) { sheet = false; path = '/home'; } return ok({}); } } diff --git a/test/node/public-comment.test.mjs b/test/node/public-comment.test.mjs index 74ca121d..7197c493 100644 --- a/test/node/public-comment.test.mjs +++ b/test/node/public-comment.test.mjs @@ -746,10 +746,10 @@ const footnote = 'Added text “Changes apply live while playing. Base resolutio const footnoteText = 'Added text “Changes apply live while playing\\. Base resolution sets the step grid for lanes you add next\\.”'; function euxyReview(target, options = {}) { const own = { sourceImpact: ['own'], sourceScope: 'screen' }; - const matched = { verdict: 'unchanged', reason: 'matched', mismatchPercentage: 0, threshold: 0.1, tool: 'agent-device@0.21.39' }; + const matched = { verdict: 'unchanged', reason: 'matched', mismatchPercentage: 0, threshold: 0.1, tool: 'agent-device@0.21.40' }; const changed = (description, mismatchPercentage) => ({ ...own, review: { status: 'modified', description, facts: [{ category: 'element', templateId: 'element.added', bullet: description }] }, - visual: { verdict: 'modified', reason: 'repeated-cross-build-delta', mismatchPercentage, threshold: 0.1, tool: 'agent-device@0.21.39' } }); + visual: { verdict: 'modified', reason: 'repeated-cross-build-delta', mismatchPercentage, threshold: 0.1, tool: 'agent-device@0.21.40' } }); const inTreeOnly = { ...changed(footnote, 0), visual: matched }; if (options.offScreen !== false) inTreeOnly.review.offScreen = true; if (options.diffFailed) inTreeOnly.visual = { ...matched, reason: 'diff-failed' }; @@ -799,7 +799,7 @@ test('a change one platform has and another lacks keeps "only" and "not on"', as const iosOnly = target => { const reviewed = euxyReview(target); if (target === 'android') for (const entry of reviewed.surfaces) Object.assign(entry, { review: { status: 'unchanged', description: 'No visible change' }, - visual: { verdict: 'unchanged', reason: 'matched', mismatchPercentage: 0, threshold: 0.1, tool: 'agent-device@0.21.39' } }); + visual: { verdict: 'unchanged', reason: 'matched', mismatchPercentage: 0, threshold: 0.1, tool: 'agent-device@0.21.40' } }); return reviewed; }; const { rows } = await euxyRows(iosOnly); diff --git a/test/node/public-review-validation.test.mjs b/test/node/public-review-validation.test.mjs index 26774434..24453366 100644 --- a/test/node/public-review-validation.test.mjs +++ b/test/node/public-review-validation.test.mjs @@ -325,7 +325,7 @@ test("a review says its change is off-screen only on a modified verdict whose sc const offScreen = (status, verdict) => { const input = review(); input.surfaces[0].review = { status, description: status === "modified" ? "Added text “lanes you add next”." : "No visible change", offScreen: true }; - input.surfaces[0].visual = { verdict, reason: verdict === "unchanged" ? "matched" : "repeated-cross-build-delta", mismatchPercentage: verdict === "unchanged" ? 0 : 3, threshold: 0.1, tool: "agent-device@0.21.39" }; + input.surfaces[0].visual = { verdict, reason: verdict === "unchanged" ? "matched" : "repeated-cross-build-delta", mismatchPercentage: verdict === "unchanged" ? 0 : 3, threshold: 0.1, tool: "agent-device@0.21.40" }; return input; }; assert.doesNotThrow(() => assertPublicReview(offScreen("modified", "unchanged"))); From 2adb9baf35df92e27dcf22067540c92bbb412bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 17:25:50 +0100 Subject: [PATCH 8/8] Read the runner's busy, watchdog, and surface refusals from typed fields Explore recognised three iOS controller failures from the prose the controller wrote beside them: the message "still finishing a previous command", the run's derived "runner_main_thread_execution_timeout", and "requires a valid viewport". agent-device 0.21.40 publishes each verdict as typed details, so the prose was only ever a fallback for the releases the pin no longer allows. The busy refusal is read as runnerErrorCode/reason runner_busy through a new runnerBusyRefusal, shared by the read recovery and the binary CLI's retry loop. A snapshot that could not read the system surface over the app is read as the snapshot engine's reason, missing-viewport or invalid-viewport, through a new unreadableSystemSurfaceFailure. The alert watchdog answer is read as runnerErrorCode MAIN_THREAD_TIMEOUT plus its reason. No message text decides any of them now. Co-Authored-By: Claude Opus 5.5 --- src/binary/run.ts | 7 ++-- src/capture/controller.ts | 63 ++++++++++++++++++++++-------- src/capture/route-capture.ts | 22 +++++------ src/capture/run.ts | 2 +- src/capture/setup-actions.ts | 19 ++------- test/node/binary-explorer.test.mjs | 28 ++++++++++--- test/node/capture-setup.test.mjs | 43 ++++++++++++++++---- test/node/ci-map.test.mjs | 47 +++++++++++++++------- 8 files changed, 153 insertions(+), 78 deletions(-) diff --git a/src/binary/run.ts b/src/binary/run.ts index 95686455..066cb5e3 100644 --- a/src/binary/run.ts +++ b/src/binary/run.ts @@ -7,7 +7,7 @@ import { createLocalCiAgentSession } from "../capture/agent/session-local.js"; import type { CiAgentProviderName } from "../capture/agent/provider.js"; import { assertReplayFits, createReplaySession, parseReplayPlan, replaySetup, type ReplayPlan } from "./replay.js"; import { CONTROLLER_SESSION, baseEnvironment, isRecord } from "../capture/common.js"; -import { controllerData, controllerFailureReason, defaultRunner, loneBackDismissControl, openedSimulatorDevice } from "../capture/controller.js"; +import { controllerData, controllerFailureReason, defaultRunner, loneBackDismissControl, openedSimulatorDevice, runnerBusyRefusal } from "../capture/controller.js"; import type { CommandRequest, CommandResult } from "../capture/run.js"; import { executeCaptureSetup } from "../capture/setup-actions.js"; import { SIMULATOR_DOTENV_FILE_NAME, SIMULATOR_DOTENV_HEADER, assertSimulatorSessionIdentity, assertSimulatorStopped, readSimulatorDotenvHandoff, @@ -887,8 +887,6 @@ async function easLoggedIn(runner: typeof defaultRunner, environment: NodeJS.Pro /** Schemes the OS routes to a browser, the store, Settings, or another app: opening one hands the session away from the app under test. */ const SYSTEM_URL_SCHEMES = /^(https?|ftp|file|javascript|data|blob|mailto|tel|telprompt|facetime(-audio)?|sms|mms|imessage|itms(-apps|-services|-beta)?|macappstore|app-settings|prefs|shortcuts|maps|music|videos|photos-redirect|x-apple-[a-z-]+|market|intent|content|android-app|geo|package):$/i; -/** The model may only open app-owned URLs; an https page or a store link would hand the session to another app. */ -const RUNNER_BUSY = /still finishing a previous command/i; const sleepSync = (ms: number): void => { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); }; /** @@ -900,12 +898,13 @@ export function retryWhileBusy(run: () => T, pause: (ms: number) => void, bud for (;;) { try { return run(); } catch (error) { - if (!RUNNER_BUSY.test(error instanceof Error ? error.message : String(error)) || clock() - started >= budgetMs) throw error; + if (!runnerBusyRefusal(error) || clock() - started >= budgetMs) throw error; pause(stepMs); } } } +/** The model may only open app-owned URLs; an https page or a store link would hand the session to another app. */ export function assertAppUrl(url: string, appSchemes: readonly string[] = []): void { let parsed: URL; try { parsed = new URL(url); } catch { throw binaryError("AGENT_URL_INVALID", `Not a URL: ${url}`, "Pass a custom-scheme app URL."); } diff --git a/src/capture/controller.ts b/src/capture/controller.ts index e7837f49..67eceae1 100644 --- a/src/capture/controller.ts +++ b/src/capture/controller.ts @@ -77,9 +77,8 @@ export async function runAgentControllerCommand(request: CommandRequest, signal: // Native payloads are private classification evidence, never serialized into CI artifacts. const nativeControllerFailures = new WeakMap>(); -const RUNNER_BUSY_MESSAGE = "The iOS runner is still finishing a previous command that exceeded its execution watchdog (usually an accessibility capture on a heavy or animating screen)."; -// agent-device 0.21.15 (as 0.20.2) keeps abandoned XCTest work busy for up to -// mainThreadWedgeThreshold (120 s). Work is not cancellable and need not drain in this window. +// The runner keeps abandoned XCTest work busy until it passes its 120 s wedge threshold and reports +// itself wedged. That work is not cancellable and need not drain inside this window. const CONTROLLER_BUSY_WINDOW_MS = 120_000; const CONTROLLER_BUSY_MAX_ATTEMPTS = 32; export type ControllerReadRecovery = { @@ -138,26 +137,55 @@ export function loneBackDismissControl(error: unknown): { selector: string; labe return typeof identifier === "string" && identifier ? { selector, label, identifier } : { selector, label }; } +/** + * The runner refusing a command it never sent, because abandoned main-thread work still drains: + * `runnerErrorCode`/`reason` `runner_busy`, with `dispatched: "no"`. Nothing ran, so a caller may + * send the command again once the work drains. + */ +export function runnerBusyRefusal(error: unknown): boolean { + if (!(error instanceof CiError)) return false; + const native = nativeControllerFailures.get(error); + if (!native || native.code !== "COMMAND_FAILED" && native.code !== "RUNNER_BUSY") return false; + const details = isRecord(native.details) ? native.details : {}; + return details.runnerErrorCode === "RUNNER_BUSY" || details.reason === "runner_busy"; +} + function busyControllerReadFailure(error: unknown): boolean { if (!(error instanceof CiError)) return false; const native = nativeControllerFailures.get(error); if (!native || native.code !== "COMMAND_FAILED" && native.code !== "RUNNER_BUSY") return false; const details = isRecord(native.details) ? native.details : {}; + // A wedged runner needs a restart, not a wait, so it is never a busy read. if (details.runnerErrorCode === "RUNNER_WEDGED" || details.runnerFailureReason === "runner_main_thread_wedged") return false; - if (native.code === "RUNNER_BUSY" || details.runnerErrorCode === "RUNNER_BUSY") return true; - // agent-device 0.21.x also types its watchdog expiry as MAIN_THREAD_TIMEOUT; 0.20.2 is identified by the derived reason. - if (details.runnerErrorCode === "MAIN_THREAD_TIMEOUT" || details.runnerFailureReason === "runner_main_thread_execution_timeout") return true; - // agent-device 0.20.2's lifecycle recovery can drop runnerErrorCode. Accept only its - // exact retained terminal-failure shape and pinned busy/watchdog messages, not arbitrary timeouts. - return details.recovery === "runner_reported_failure" && details.lifecycleState === "failed" - && (native.message === RUNNER_BUSY_MESSAGE || details.transportError === RUNNER_BUSY_MESSAGE - || native.message === "main thread execution timed out"); + if (runnerBusyRefusal(error)) return true; + // The command that tripped the watchdog reports its own expiry as MAIN_THREAD_TIMEOUT, published as + // `runnerErrorCode` plus `reason`, and as the log-derived `runnerFailureReason`. Its work was + // abandoned, so the next read meets the busy refusal: treat both as the same occupied main thread. + return details.runnerErrorCode === "MAIN_THREAD_TIMEOUT" || controllerFailureReason(error) === "runner_main_thread_timeout" + || details.runnerFailureReason === "runner_main_thread_execution_timeout"; +} + +/** + * A system surface the runner could not present: the iOS snapshot engine refused the capture because + * its viewport evidence was missing or invalid, which is what happens while the runner defers its + * XCTest tiers and only com.apple.SafariViewService can read the surface over the app. agent-device + * publishes the refusal as `details.reason` (`toIosSnapshotEngineErrorDetails`), so it is never read + * off the message prose. + */ +export function unreadableSystemSurfaceFailure(error: unknown): boolean { + if (!(error instanceof CiError) || error.code !== "COMMAND_FAILED") return false; + const details = nativeControllerFailures.get(error)?.details; + if (!isRecord(details)) return false; + // A sparse capture of a system surface (Save Password in SafariViewService) is unreadable whatever + // engine reason it carries: agent-device attaches `systemSurface` and a `sparse` quality verdict. + const sparseSystemSurface = isRecord(details.systemSurface) && isRecord(details.snapshotQuality) && details.snapshotQuality.state === "sparse"; + return details.reason === "missing-viewport" || details.reason === "invalid-viewport" || sparseSystemSurface; } /** - * agent-device sends iOS `alert get` to its runner with a fixed 10 s budget and abandons the - * query when that watchdog expires. Another query on the same screen queues more abandoned work, - * so a watchdog answer means "unknown", never "no alert". + * agent-device sends iOS `alert get` to its runner with the caller's read budget (10 s by default) + * and abandons the query when that watchdog expires. Another query on the same screen queues more + * abandoned work, so a watchdog answer means "unknown", never "no alert". */ export function alertWatchdogFailure(error: unknown): boolean { if (!(error instanceof CiError)) return false; @@ -165,7 +193,8 @@ export function alertWatchdogFailure(error: unknown): boolean { if (!native) return false; const details = isRecord(native.details) ? native.details : {}; return native.code === "MAIN_THREAD_TIMEOUT" || details.runnerErrorCode === "MAIN_THREAD_TIMEOUT" - || details.runnerFailureReason === "runner_main_thread_execution_timeout" || native.message === "main thread execution timed out"; + || controllerFailureReason(error) === "runner_main_thread_timeout" + || details.runnerFailureReason === "runner_main_thread_execution_timeout"; } function responsiveAccessibilityRead(args: readonly string[], result: CommandResult): boolean { @@ -307,7 +336,7 @@ export function runControllerRead(options: { const busy = readOnly && busyControllerReadFailure(failure); if (recovery.recycleUsed && wedgedControllerReadFailure(failure)) return terminal(failure); if (accessibilityRead && recovery.busyDeadlineMs !== undefined && !recovery.recycleUsed && wedgedControllerReadFailure(failure)) { - // Pinned runner-session.ts awaits helper-only invalidation before returning WEDGED. + // agent-device awaits the runner's invalidation (runner-exchange.ts) before it returns WEDGED. // The next ordinary read starts a fresh helper; no app open/close or action is replayed. recovery.recycleUsed = true; recovery.needsVerification = true; @@ -424,7 +453,7 @@ export async function runControllerReadAsync(options: { const busy = readOnly && busyControllerReadFailure(failure); if (recovery.recycleUsed && wedgedControllerReadFailure(failure)) return terminal(failure); if (accessibilityRead && recovery.busyDeadlineMs !== undefined && !recovery.recycleUsed && wedgedControllerReadFailure(failure)) { - // Pinned runner-session.ts awaits helper-only invalidation before returning WEDGED. + // agent-device awaits the runner's invalidation (runner-exchange.ts) before it returns WEDGED. // The next ordinary read starts a fresh helper; no app open/close or action is replayed. recovery.recycleUsed = true; recovery.needsVerification = true; diff --git a/src/capture/route-capture.ts b/src/capture/route-capture.ts index 17888c83..e119a61e 100644 --- a/src/capture/route-capture.ts +++ b/src/capture/route-capture.ts @@ -13,7 +13,7 @@ import { ExploreStore, recordObservation } from "../map/store.js"; import { parseControllerOutput } from "./controller-output.js"; import type { SourceRoute } from "../contracts/source-route.js"; import { CiError, serializeCiError, toCiError } from "../shared/ci-errors.js"; -import { type CaptureSetupSnapshot, credentialSavePromptDecline, describeSetupAction, executeCaptureSetup, isUnreadableSystemSurface, setupReadinessHolds } from "./setup-actions.js"; +import { type CaptureSetupSnapshot, credentialSavePromptDecline, describeSetupAction, executeCaptureSetup, setupReadinessHolds } from "./setup-actions.js"; import { silentProgress } from "../shared/progress.js"; import { silentTimings, type Timings } from "../shared/timings.js"; import type { CaptureSetupReady } from "../config/explore-config.js"; @@ -23,7 +23,7 @@ import { CAPTURE_OBSERVER_IDENTIFIER, captureObserverMarkers, parseCaptureDestin import { type CiAgentRuntimeOptions, runCiAgentNavigation, assertAgentCaptureContext } from "./agent/runtime.js"; import { createExplorationInputs } from "../config/exploration-inputs.js"; import { isRecord, captureError, CONTROLLER_SESSION, parseJson } from "./common.js"; -import { MAX_ACCESSIBILITY_EVIDENCE_BYTES, adbDeviceOffline, remoteDaemonUnavailable, controllerData, defaultRunner, type ControllerReadRecovery, type RemoteDaemonRetry, runControllerRead, controllerCommandFailure, runControllerReadAsync, runAgentControllerCommand, wedgedControllerReadFailure, openedSimulatorDevice, alertWatchdogFailure, backLeftScreenUnchanged, loneBackDismissControl } from "./controller.js"; +import { MAX_ACCESSIBILITY_EVIDENCE_BYTES, adbDeviceOffline, remoteDaemonUnavailable, controllerData, defaultRunner, type ControllerReadRecovery, type RemoteDaemonRetry, runControllerRead, controllerCommandFailure, runControllerReadAsync, runAgentControllerCommand, wedgedControllerReadFailure, openedSimulatorDevice, alertWatchdogFailure, backLeftScreenUnchanged, loneBackDismissControl, unreadableSystemSurfaceFailure } from "./controller.js"; import { riskOf } from "../binary/explorer.js"; import type { ActiveSimulatorRuntime, SimulatorConnection } from "./simulator.js"; import { androidTools } from "./local-simulator.js"; @@ -356,8 +356,9 @@ export async function capturePlannedRoutes(options: { } try { // alert accept targets the inspected modal, never an application button named Open. - // The pinned controller recognizes Open and verifies that the modal disappears. - // alert has no --timeout flag in agent-device 0.21.15; bound the process instead. + // The controller recognizes Open and verifies that the modal disappears. It runs accept's own + // retry window (a 2 s window whose last read may still be in flight), so this stays the outer + // bound on the process: a controller that stops answering must not spend the capture budget. controller(["alert", "accept", "--level", "full", "--json"], false, 15_000); } catch (error) { if (error instanceof CiError && error.code === "CAPTURE_BUDGET_EXHAUSTED") throw error; @@ -422,7 +423,7 @@ export async function capturePlannedRoutes(options: { catch (error) { // A system surface the controller cannot read yet (the Save Password prompt while // agent-device defers its XCTest tiers): wait it out, then decline it as usual. - if (isUnreadableSystemSurface(error)) { + if (unreadableSystemSurfaceFailure(error)) { surfaceWaitStarted ??= clock(); const waited = clock() - surfaceWaitStarted; if (waited >= UNREADABLE_SURFACE_WAIT_MS) { @@ -828,10 +829,10 @@ export async function capturePlannedRoutes(options: { } }; /** - * agent-device 0.21.x routes a snapshot to a system surface such as the SafariViewService - * password prompt. While its XCTest tiers are deferred after slow accessibility work, no backend - * can read that surface and the snapshot is refused (observed for about 40 s in r10). Wait for - * the deferral to end rather than hand the agent a frame without accessibility. + * agent-device routes a snapshot to a system surface such as the SafariViewService password + * prompt. While its XCTest tiers are deferred after slow accessibility work, no backend can read + * that surface and the snapshot refuses with a missing viewport (observed for about 40 s in r10). + * Wait for the deferral to end rather than hand the agent a frame without accessibility. */ const readSnapshotThroughDeferral = async (signal: AbortSignal) => { const started = clock(); @@ -840,8 +841,7 @@ export async function capturePlannedRoutes(options: { try { return { ...(await readSnapshot(signal)), retries: attempt, deferredMs: clock() - started }; } catch (error) { const delayMs = delays[attempt]; - if (!(error instanceof CiError) || !/requires a (?:valid|positive finite) viewport/.test(error.message) - || delayMs === undefined || deadline - clock() <= delayMs) throw error; + if (!unreadableSystemSurfaceFailure(error) || delayMs === undefined || deadline - clock() <= delayMs) throw error; await sleep(delayMs, undefined, { signal: liveSignal(signal) }); } } diff --git a/src/capture/run.ts b/src/capture/run.ts index 6b1920c3..901997db 100644 --- a/src/capture/run.ts +++ b/src/capture/run.ts @@ -39,7 +39,7 @@ import { appScheme, hasConcreteRoutePath, unavailableRoute, capturePlannedRoutes export { assertDecodedPng } from "../shared/png-evidence.js"; export { EAS_CLI_VERSION, AGENT_DEVICE_VERSION } from "./common.js"; -export { runAgentControllerCommand, controllerCommandFailure, runControllerRead, runControllerReadAsync } from "./controller.js"; +export { runAgentControllerCommand, controllerCommandFailure, runControllerRead, runControllerReadAsync, unreadableSystemSurfaceFailure, runnerBusyRefusal } from "./controller.js"; export type { ControllerReadRecovery } from "./controller.js"; export { captureDeepLink, capturePlannedRoutes } from "./route-capture.js"; diff --git a/src/capture/setup-actions.ts b/src/capture/setup-actions.ts index e192e227..b1f7957c 100644 --- a/src/capture/setup-actions.ts +++ b/src/capture/setup-actions.ts @@ -75,8 +75,9 @@ export function canTypeAfterFocus(intent: CaptureInputFocus | undefined, snapsho } /** - * Older controllers report one Keyboard element; agent-device 0.21.15 reports only its - * Key elements, which its own occlusion check also treats as the keyboard. + * The controller's keyboard elements: a `Keyboard` container, its `Key` children, or both. A real + * agent-device 0.21.40 capture projects the `Keyboard` plane (role UIKeyboardAutomatic) together + * with its keys, and its own keyboard-chrome check treats either as the keyboard. */ export function isKeyboardNode(node: Record): boolean { const type = typeof node.type === "string" ? node.type.toLowerCase() : ""; @@ -140,20 +141,6 @@ export function credentialSavePromptDecline(nodes: readonly Record ready.enabled === undefined || node.enabled === ready.enabled); diff --git a/test/node/binary-explorer.test.mjs b/test/node/binary-explorer.test.mjs index 9a6a4161..eba37a15 100644 --- a/test/node/binary-explorer.test.mjs +++ b/test/node/binary-explorer.test.mjs @@ -9,7 +9,7 @@ const { runBlackBoxExploration, screenEvidenceKey, elementsOf, fieldName, system const { ExploreStore, validateMap } = await tsImport("../../src/map/store.ts", import.meta.url); const { CONTRACT } = await tsImport("../../src/types.ts", import.meta.url); const tsRun = await tsImport("../../src/binary/run.ts", import.meta.url); -const { assertAppUrl, describeForeground, foreignNode, guardPositionals, leftForeground, retryWhileBusy } = tsRun; +const { assertAppUrl, describeForeground, foreignNode, guardPositionals, leftForeground } = tsRun; const PNG = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", "base64"); const SCREENS = { @@ -358,15 +358,31 @@ test("evidence keys ignore a long, changing label on a role-less element", () => assert.notEqual(screenEvidenceKey(banner("Short label")), screenEvidenceKey(banner("Other label"))); }); -test("a controller command refused while the runner winds down a previous one is retried, within a budget", () => { - const busy = () => new Error("press: The iOS runner is still finishing a previous command that exceeded its execution watchdog."); +test("a controller command refused while the runner winds down a previous one is retried, within a budget", async () => { + // agent-device 0.21.40's classifyRunnerReportedError publishes the typed busy refusal; the CLI's + // own JSON envelope is what Explore reads it from (controllerCommandFailure). Both sides come from + // one build so the failure carries the error class the retry helper looks up. + const { controllerCommandFailure } = await import("../../dist/capture/controller.js"); + const { retryWhileBusy: retry } = await import("../../dist/binary/run.js"); + const busy = () => controllerCommandFailure(JSON.stringify({ success: false, error: { code: "COMMAND_FAILED", + message: "The iOS runner is still finishing a previous command that exceeded its execution watchdog.", + retriable: true, details: { runnerErrorCode: "RUNNER_BUSY", reason: "runner_busy", dispatched: "no" } } }), "press"); let attempts = 0; const pauses = []; - const result = retryWhileBusy(() => { attempts += 1; if (attempts < 3) throw busy(); return "ok"; }, ms => pauses.push(ms)); + const result = retry(() => { attempts += 1; if (attempts < 3) throw busy(); return "ok"; }, ms => pauses.push(ms)); assert.equal(result, "ok"); assert.deepEqual(pauses, [5000, 5000]); let clock = 0; - assert.throws(() => retryWhileBusy(() => { throw busy(); }, ms => { clock += ms; }, 12_000, 5_000, () => clock), /still finishing/); + assert.throws(() => retry(() => { throw busy(); }, ms => { clock += ms; }, 12_000, 5_000, () => clock), + error => error.code === "COMMAND_FAILED"); assert.equal(clock, 15_000, "three pauses, then the budget is spent"); - assert.throws(() => retryWhileBusy(() => { throw new Error("main thread execution timed out"); }, () => assert.fail("no pause")), /timed out/); + // The command that tripped the watchdog already spent its wait: Explore resends nothing on it. + const stalled = () => controllerCommandFailure(JSON.stringify({ success: false, error: { code: "COMMAND_FAILED", + message: "main thread execution timed out", + details: { runnerErrorCode: "MAIN_THREAD_TIMEOUT", reason: "runner_main_thread_timeout", dispatched: "unknown" } } }), "press"); + assert.throws(() => retry(() => { throw stalled(); }, () => assert.fail("no pause")), + error => error.code === "COMMAND_FAILED" && /main thread execution timed out/.test(error.message)); + // A failure with no typed busy refusal is never retried, whatever its prose says. + const notBusy = () => { throw new Error("press: The iOS runner is still finishing a previous command that exceeded its execution watchdog."); }; + assert.throws(() => retry(notBusy, () => assert.fail("no pause")), /still finishing/); }); test("a launch-blocked reason names what appstate found in front, not just \"another app\"", () => { diff --git a/test/node/capture-setup.test.mjs b/test/node/capture-setup.test.mjs index 46fd5075..fc11115b 100644 --- a/test/node/capture-setup.test.mjs +++ b/test/node/capture-setup.test.mjs @@ -1,7 +1,8 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { spawnSync } from 'node:child_process'; -import { credentialSavePromptDecline, executeCaptureSetup, isUnreadableSystemSurface } from '../../dist/capture/setup-actions.js'; +import { credentialSavePromptDecline, executeCaptureSetup } from '../../dist/capture/setup-actions.js'; +import { controllerCommandFailure, unreadableSystemSurfaceFailure } from '../../dist/capture/controller.js'; import { CiError } from '../../dist/shared/ci-errors.js'; import { CAPTURE_OBSERVER_IDENTIFIER } from '../../dist/capture/observer.js'; @@ -488,12 +489,38 @@ test('each emitted command shape passes the installed pinned controller parser w }); test('a snapshot that cannot read the system surface over the app is recognized, and nothing else is', () => { - const failed = (message, nextAction) => new CiError({ code: 'COMMAND_FAILED', stage: 'controller', message, ...(nextAction ? { nextAction } : {}) }); - assert.equal(isUnreadableSystemSurface(failed('snapshot: regular iOS snapshot presentation requires a valid viewport')), true); - assert.equal(isUnreadableSystemSurface(failed('snapshot failed', 'com.apple.SafariViewService hosts the surface presented over the app, and this capture targeted that surface.')), true); - assert.equal(isUnreadableSystemSurface(failed('snapshot failed', 'No snapshot backend could read this screen (XCTest-backed snapshot tiers were deferred after recent slow accessibility work on this screen).')), true); - assert.equal(isUnreadableSystemSurface(failed('Remote daemon is unavailable')), false); - assert.equal(isUnreadableSystemSurface(new CiError({ code: 'SETUP_FAILED', stage: 'setup', message: 'requires a valid viewport' })), false); - assert.equal(isUnreadableSystemSurface(new Error('requires a valid viewport')), false); + // agent-device 0.21.40 publishes the refusal's engine reason as typed details + // (capture-kit's toIosSnapshotEngineErrorDetails, through platform-apple's snapshot-presentation.ts), + // both for a missing viewport and for an invalid one. Explore keys on that field alone. + const refused = error => unreadableSystemSurfaceFailure( + controllerCommandFailure(JSON.stringify({ success: false, error }), 'snapshot')); + assert.equal(refused({ code: 'COMMAND_FAILED', message: 'regular iOS snapshot presentation requires a valid viewport', + details: { reason: 'missing-viewport', field: 'viewport' } }), true); + assert.equal(refused({ code: 'COMMAND_FAILED', message: 'regular iOS snapshot presentation requires a positive finite viewport', + details: { reason: 'invalid-viewport', field: 'viewport' } }), true); + // While the runner defers its XCTest tiers over the Save Password prompt, the same refusal carries the + // deferred verdict and the surface host beside the reason. Explore needs only the reason. + assert.equal(refused({ code: 'COMMAND_FAILED', message: 'regular iOS snapshot presentation requires a valid viewport', + hint: 'com.apple.SafariViewService hosts the surface presented over the app. No snapshot backend could read this screen.', + details: { reason: 'missing-viewport', field: 'viewport', + systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' }, + snapshotQuality: { state: 'sparse', backend: 'private-ax', reasonCode: 'deferred' } } }), true); + // The prose alone no longer marks a surface unreadable: only the typed reason does. + assert.equal(refused({ code: 'COMMAND_FAILED', message: 'regular iOS snapshot presentation requires a valid viewport', + hint: 'com.apple.SafariViewService hosts the surface presented over the app.' }), false); + assert.equal(refused({ code: 'COMMAND_FAILED', message: 'snapshot failed', + details: { snapshotQuality: { state: 'sparse', backend: 'private-ax', reasonCode: 'deferred' } } }), false); + assert.equal(refused({ code: 'COMMAND_FAILED', message: 'snapshot failed', + details: { reason: 'sparse-tree' } }), false); + // A sparse capture of a system surface is unreadable whatever engine reason it carries. + assert.equal(refused({ code: 'COMMAND_FAILED', message: 'iOS runner returned an invalid quality payload', + details: { reason: 'invalid-quality-payload', systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' }, + snapshotQuality: { state: 'sparse', backend: 'private-ax', reasonCode: 'deferred' } } }), true); + assert.equal(refused({ code: 'COMMAND_FAILED', message: 'Remote daemon is unavailable', + details: { reason: 'remote_daemon_unavailable', dispatched: 'no' } }), false); + // A failure Explore derived itself, rather than the controller's own envelope, is not this refusal. + assert.equal(unreadableSystemSurfaceFailure(new CiError({ code: 'SETUP_FAILED', stage: 'setup', message: 'requires a valid viewport', + nextAction: 'x', details: { reason: 'missing-viewport' } })), false); + assert.equal(unreadableSystemSurfaceFailure(new Error('requires a valid viewport')), false); }); diff --git a/test/node/ci-map.test.mjs b/test/node/ci-map.test.mjs index 3ef4993d..452c9d8d 100644 --- a/test/node/ci-map.test.mjs +++ b/test/node/ci-map.test.mjs @@ -46,13 +46,16 @@ const route = (id, path = `/${id}`) => ({ id, kind: 'screen', route: path ?? '/i routeFile: path === null ? `app/item/[id].tsx` : `app/${id}.tsx`, sourceFiles: [`app/${id}.tsx`], variant: 'default', provider: 'expo-router', logicalPath: id }); const ok = data => ({ stdout: JSON.stringify({ success: true, data }), stderr: '' }); const json = value => ({ stdout: JSON.stringify(value), stderr: '' }); -// agent-device 0.21.15 iOS RunnerTests+Alert.swift uses message/items for alerts. When none +// agent-device 0.21.40's iOS RunnerTests+Alert.swift answers a read alert as message/items. When none // exists, the runner reports ALERT_NOT_FOUND and the CLI maps it to this public envelope. const noAlert = () => json({ success: false, error: { code: 'COMMAND_FAILED', message: 'alert not found', details: { runnerErrorCode: 'ALERT_NOT_FOUND' } } }); +// agent-device 0.21.40's refusal of a command it never sent: classifyRunnerReportedError publishes +// runnerErrorCode, reason and dispatched for the runner's RUNNER_BUSY wire code, and normalizeError +// lifts retriable out of details onto the error itself. const busyRead = (changes = {}) => json({ success: false, error: { code: 'COMMAND_FAILED', message: 'The iOS runner is still finishing a previous command that exceeded its execution watchdog (usually an accessibility capture on a heavy or animating screen).', - details: { runnerErrorCode: 'RUNNER_BUSY', retriable: true }, ...changes } }); + retriable: true, details: { runnerErrorCode: 'RUNNER_BUSY', reason: 'runner_busy', dispatched: 'no' }, ...changes } }); // A fresh helper's first XCTest-backed snapshot reports its quality; the fixture's plain snapshots do not. const recoveredSnapshot = result => { const value = JSON.parse(result.stdout); @@ -62,11 +65,15 @@ const recoveredSnapshot = result => { const daemonUnavailable = () => json({ success: false, error: { code: 'COMMAND_FAILED', message: 'Remote daemon is unavailable', details: { reason: 'remote_daemon_unavailable', dispatched: 'no', daemonBaseUrl: 'https://daemon.example.invalid', attempts: 1 } } }); const value = (request, name) => request.args[request.args.indexOf(name) + 1]; -// agent-device 0.21.15 while its XCTest snapshot tiers are deferred and iOS shows the Save Password -// prompt, which com.apple.SafariViewService hosts. +// agent-device 0.21.40 while its XCTest snapshot tiers are deferred and iOS shows the Save Password +// prompt, which com.apple.SafariViewService hosts. The engine reason, the deferred verdict and the +// surface host travel as typed details; the CLI lifts `hint` out of them. const unreadableSurface = () => json({ success: false, error: { code: 'COMMAND_FAILED', message: 'snapshot: regular iOS snapshot presentation requires a valid viewport', - details: { hint: 'com.apple.SafariViewService hosts the surface presented over the app, and this capture targeted that surface. No snapshot backend could read this screen (XCTest-backed snapshot tiers were deferred after recent slow accessibility work on this screen).' } } }); + hint: 'com.apple.SafariViewService hosts the surface presented over the app, and this capture targeted that surface. No snapshot backend could read this screen (XCTest-backed snapshot tiers were deferred after recent slow accessibility work on this screen).', + details: { reason: 'missing-viewport', field: 'viewport', + systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' }, + snapshotQuality: { state: 'sparse', backend: 'private-ax', reasonCode: 'deferred' } } } }); /** @@ -2761,11 +2768,17 @@ test('explicit busy reads back off on the host and retry within the same capture assert.equal(calls, 2); }); -test('watchdog and retained busy read envelopes receive the same bounded observation recovery', () => { +test('a busy refusal and the watchdog expiry receive the same bounded observation recovery', () => { let now = 0, calls = 0; - const lifecycleBusy = busyRead({ details: { recovery: 'runner_reported_failure', lifecycleState: 'failed', - transportError: 'The iOS runner is still finishing a previous command that exceeded its execution watchdog (usually an accessibility capture on a heavy or animating screen).' } }); - const watchdog = busyRead({ message: 'main thread execution timed out', details: { runnerFailureReason: 'runner_main_thread_execution_timeout' } }); + // agent-device 0.21.40's runner-command-recovery.ts re-publishes the journal's code through + // classifyRunnerReportedError, so a busy refusal recovered after a lost response keeps its typed fields. + const lifecycleBusy = busyRead({ details: { runnerErrorCode: 'RUNNER_BUSY', reason: 'runner_busy', dispatched: 'no', + recovery: 'runner_reported_failure', lifecycleState: 'failed' } }); + // The command that tripped the watchdog: runner-contract.ts types its expiry as MAIN_THREAD_TIMEOUT + // with reason runner_main_thread_timeout, and runner-failure-diagnostics.ts can add its log-derived reason. + const watchdog = busyRead({ message: 'main thread execution timed out', retriable: undefined, + details: { runnerErrorCode: 'MAIN_THREAD_TIMEOUT', reason: 'runner_main_thread_timeout', dispatched: 'unknown', + runnerFailureReason: 'runner_main_thread_execution_timeout' } }); const replies = [watchdog, lifecycleBusy, ok({ nodes: [] })]; const response = runControllerRead({ args: ['snapshot'], deadlineMs: 10_000, timeoutMs: 10_000, now: () => now, sleep: milliseconds => { now += milliseconds; }, run: () => replies[calls++] }); @@ -2776,7 +2789,7 @@ test('watchdog and retained busy read envelopes receive the same bounded observa let queries = 0; const alert = runControllerRead({ args: ['alert', 'get'], deadlineMs: 10_000, timeoutMs: 10_000, now: () => now, sleep: () => assert.fail('a timed-out alert query is not replayed'), run: () => { queries++; return watchdog; } }); - assert.equal(JSON.parse(alert.stdout).error.details.runnerFailureReason, 'runner_main_thread_execution_timeout'); + assert.equal(JSON.parse(alert.stdout).error.details.runnerErrorCode, 'MAIN_THREAD_TIMEOUT'); assert.equal(queries, 1); // Busy refusals still wait for the runner to drain, for alert queries as for snapshots. let busyQueries = 0; @@ -2786,9 +2799,11 @@ test('watchdog and retained busy read envelopes receive the same bounded observa assert.equal(busyQueries, 2); }); -test('the typed runner watchdog code retries reads only', () => { - // agent-device 0.21.x types its watchdog expiry with this code; the message is not the contract. - const typed = () => busyRead({ message: 'XCTest stalled on the current screen', details: { runnerErrorCode: 'MAIN_THREAD_TIMEOUT' } }); +test('the watchdog expiry is read from its typed code and reason, not from message text', () => { + // agent-device 0.21.40 types the expiry as runnerErrorCode MAIN_THREAD_TIMEOUT and reason + // runner_main_thread_timeout; the prose the runner writes beside it is not the contract. + const typed = () => busyRead({ message: 'XCTest stalled on the current screen', retriable: undefined, + details: { runnerErrorCode: 'MAIN_THREAD_TIMEOUT', reason: 'runner_main_thread_timeout', dispatched: 'unknown' } }); let now = 0, calls = 0; const response = runControllerRead({ args: ['snapshot'], deadlineMs: 10_000, timeoutMs: 10_000, now: () => now, sleep: milliseconds => { now += milliseconds; }, run: () => ++calls === 1 ? typed() : ok({ nodes: [] }) }); @@ -3983,9 +3998,11 @@ test('agent observation waits out a deferred system-surface snapshot instead of if (request.command === 'fake-controller' && request.args[0] === 'snapshot' && ++snapshots && refusals > 0) { refusals -= 1; now += 3_000; - // agent-device 0.21.15 while the SafariViewService password prompt is presented. + // agent-device 0.21.40 while the SafariViewService password prompt is presented. return json({ success: false, error: { code: 'COMMAND_FAILED', message: 'regular iOS snapshot presentation requires a valid viewport', - details: { hint: 'com.apple.SafariViewService hosts the surface presented over the app. No snapshot backend could read this screen.' } } }); + hint: 'com.apple.SafariViewService hosts the surface presented over the app. No snapshot backend could read this screen.', + details: { reason: 'missing-viewport', field: 'viewport', + systemSurface: { bundleId: 'com.apple.SafariViewService', kind: 'web-auth' } } } }); } return f.options.runner(request); } });