Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/quickstart-ci.md
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ The provider selects the runtime that owns the navigation loop:
| `anthropic` | Claude Code through `@anthropic-ai/claude-agent-sdk` 0.3.285 and its platform-native package | `ANTHROPIC_API_KEY` |
| `openai` | Codex app-server from `@openai/codex` 0.159.2 and its platform-native package | `OPENAI_API_KEY` |

Both runtimes use Explore's device tools to operate the existing Expo Simulators session through official `agent-device` 0.21.29-nightly.20261018.74. The runtime and native binaries come from the pinned tools installation. CI does not use the development-only `node_modules/.bin/claude` executable or an interactive subscription login.
Both runtimes use Explore's device tools to operate the existing Expo Simulators session through official `agent-device` 0.21.40. The runtime and native binaries come from the pinned tools installation. CI does not use the development-only `node_modules/.bin/claude` executable or an interactive subscription login.

Create and commit the nonempty English prose file named by `agent.instructions`. The shared setup skill authors it from source and asks for missing test data or environment inputs. Describe how to reach the app's screens, necessary prerequisites, the representative state to use on both revisions, readiness signals, and actions to avoid. This file guides the runtime agent; it is not a list of mandatory YAML steps or a second screen inventory.

Expand Down
8 changes: 4 additions & 4 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@
"@types/react-dom": "^19.2.4",
"@vitejs/plugin-react": "^4.7.0",
"@xyflow/react": "^12.11.3",
"agent-device": "0.21.29-nightly.20261018.74",
"agent-device": "0.21.40",
"babel-plugin-react-compiler": "^1.0.0",
"eslint": "^10.12.0",
"eslint-plugin-react-hooks": "^7.1.1",
Expand Down
17 changes: 14 additions & 3 deletions src/binary/explorer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -827,17 +827,17 @@ export async function runBlackBoxExploration(options: BlackBoxExplorerOptions):
let error: string | undefined;
try { await options.act(byCleanLabel(disambiguate(typed, matched, fromScreenId)), signal); }
catch (caught) {
error = caught instanceof Error ? caught.message.slice(0, 500) : String(caught).slice(0, 500);
error = actionFailure(caught);
// agent-device names the label it would have accepted (a typographic apostrophe the model typed straight).
const similar = SIMILAR.exec(error)?.[1];
if (similar && action.action === "tap" && "label" in action && similar !== action.label) {
try { await options.act({ action: "tap", label: similar }, signal); error = undefined; }
catch (again) { error = again instanceof Error ? again.message.slice(0, 500) : String(again).slice(0, 500); }
catch (again) { error = actionFailure(again); }
}
const centre = error && COVERED.test(error) ? byCentre(action, matched, named) : undefined;
if (centre) {
try { await options.act(centre, signal); error = undefined; }
catch (again) { error = again instanceof Error ? again.message.slice(0, 500) : String(again).slice(0, 500); }
catch (again) { error = actionFailure(again); }
}
}
let view: Awaited<ReturnType<typeof observeNow>> | undefined;
Expand Down Expand Up @@ -921,3 +921,14 @@ function frameExtent(nodes: readonly Record<string, unknown>[]): { width: number
}
return width > 0 && height > 0 ? { width, height } : undefined;
}

/**
* What the model reads about a failed action: the message, then the controller's own next step. agent-device's hint
* says how to proceed (press a named dismiss control, an app control that ends editing), and the model has no other way
* to learn it.
*/
function actionFailure(caught: unknown): string {
if (!(caught instanceof Error)) return String(caught).slice(0, 500);
const next = caught instanceof CiError && caught.nextAction ? ` Next: ${caught.nextAction.slice(0, 400)}` : "";
return `${caught.message.slice(0, 500)}${next}`;
}
39 changes: 20 additions & 19 deletions src/binary/run.ts
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { createLocalCiAgentSession } from "../capture/agent/session-local.js";
import type { CiAgentProviderName } from "../capture/agent/provider.js";
import { assertReplayFits, createReplaySession, parseReplayPlan, replaySetup, type ReplayPlan } from "./replay.js";
import { CONTROLLER_SESSION, baseEnvironment, isRecord } from "../capture/common.js";
import { controllerData, defaultRunner, openedSimulatorDevice } from "../capture/controller.js";
import { controllerData, controllerFailureReason, defaultRunner, loneBackDismissControl, openedSimulatorDevice } from "../capture/controller.js";
import type { CommandRequest, CommandResult } from "../capture/run.js";
import { executeCaptureSetup } from "../capture/setup-actions.js";
import { SIMULATOR_DOTENV_FILE_NAME, SIMULATOR_DOTENV_HEADER, assertSimulatorSessionIdentity, assertSimulatorStopped, readSimulatorDotenvHandoff,
Expand All @@ -29,7 +29,7 @@ import { packageBin } from "../shared/package-paths.js";
import { assertDecodedPng, solidColour } from "../shared/png-evidence.js";
import { CONTRACT, type ExploreMap, type Platform } from "../types.js";
import { assertEmulatorCompatible, extractIosBundle, inspectArchive, type ArchiveKind, type InspectedArchive, readArchiveIcon } from "./archive.js";
import { runBlackBoxExploration, type BlackBoxExplorerResult, type BlackBoxFrame, SYSTEM_DIALOG_PACKAGES } from "./explorer.js";
import { riskOf, runBlackBoxExploration, screenEvidenceKey, type BlackBoxExplorerResult, type BlackBoxFrame, SYSTEM_DIALOG_PACKAGES } from "./explorer.js";

export const DEFAULT_IOS_DEVICE = "iPhone 17 Pro";
export const DEFAULT_BINARY_PROJECT = "expo-explore";
Expand Down Expand Up @@ -502,30 +502,33 @@ export async function exploreConnectedSession(ctx: ConnectedSessionExploration):
const bounds = frameBounds(frame.nodes);
controller(["press", String(Math.round(action.x * bounds.width)), String(Math.round(action.y * bounds.height)), "--level", "full", "--json"], 120_000);
} else if (action.action === "back") {
// Onboarding steps, tabs and sheets have no back control; the platform gesture (iOS edge swipe, Android back) still works.
// On iOS agent-device's in-app back already falls back to the edge swipe, and names a modal's one dismiss
// control when that changed nothing. The Argent bridge sends nothing without a back control; Android's
// system back still works there.
try { controller(["back", "--in-app", "--level", "full", "--json"], 120_000); }
catch (error) {
if (!(error instanceof Error) || !/in-app back control is not available/.test(error.message)) throw error;
controller(["back", "--system", "--level", "full", "--json"], 120_000);
const control = loneBackDismissControl(error);
if (control && riskOf(control.label, allowDestructive ?? []) === "safe") {
// As in source-driven capture, the press counts as back only when the screen changed.
const before = screenEvidenceKey((await observe(false)).nodes);
controller(["press", "--level", "full", "--json", "--", control.selector], 120_000);
await settled();
// A "Discard changes?" confirmation changes the screen too, but leaves it in place.
const after = (await observe(false)).nodes;
if (screenEvidenceKey(after) === before || after.some(node => node.type === "Alert")) throw error;
}
else if (controllerFailureReason(error) === "back_swipe_not_dispatched") controller(["back", "--system", "--level", "full", "--json"], 120_000);
else throw error;
}
}
else if (action.action === "dismiss-keyboard") await dismissKeyboard();
else if (action.action === "type") controller(["type", "--level", "full", "--json", "--", action.text], 120_000);
else await executeCaptureSetup([action], setupOptions);
if (action.action !== "wait") await settled();
};
const dismissKeyboard = async () => {
try { controller(["keyboard", "dismiss", "--level", "full", "--json"], 120_000); return; }
catch (error) { if (!(error instanceof Error) || !/Unable to dismiss the iOS keyboard/.test(error.message)) throw error; }
// The iOS keyboard has no dismiss key and agent-device never taps the background; its return key (next/done/go) closes it without touching the app.
const nodes = (await observe(false)).nodes;
// In a multi-line editor the return key inserts a newline instead of ending editing.
const multiline = nodes.some(node => /text-?view/i.test(String(node.type ?? "")));
const key = multiline ? undefined : nodes.find(node => /key$/i.test(String(node.type ?? "")) && KEYBOARD_RETURN_KEY.test(String(node.label ?? "")));
const rect = key?.rect as { x: number; y: number; width: number; height: number } | undefined;
if (!rect) throw binaryError("KEYBOARD_DISMISS_UNAVAILABLE", multiline ? "The keyboard cannot be dismissed from a multi-line editor: its return key inserts a newline." : "The keyboard cannot be dismissed: it has neither a dismiss nor a return key.", "Tap one of the app's own controls that is not behind the keyboard (Cancel, Done, Close), or go back.");
controller(["press", String(Math.round(rect.x + rect.width / 2)), String(Math.round(rect.y + rect.height / 2)), "--level", "full", "--json"], 120_000);
};
// A refusal reaches the model with agent-device's own hint. The return key is never a fallback: next moves focus to
// the following field, and done or go submits the form.
const dismissKeyboard = async () => { controller(["keyboard", "dismiss", "--level", "full", "--json"], 120_000); };
try {
try { await perform(); }
catch (error) {
Expand Down Expand Up @@ -921,8 +924,6 @@ export function guardPositionals(args: readonly string[]): string[] {
return [positionals[0]!, ...flags, "--", ...positionals.slice(1)];
}

const KEYBOARD_RETURN_KEY = /^(return|next|done|go|continue|join)$/i;

/**
* How a frame's screenshot is stored. An opened (or half-open) iPhone Duo lights its inner panel, whose native axes are
* portrait, and shows a landscape UI on it: the tree and taps stay on the portrait axes (a 669×951 pt root, the transpose
Expand Down
16 changes: 9 additions & 7 deletions src/capture/argent-driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ const failure = (code: string, message: string, details?: Record<string, unknown
* agent-device's answer when its pre-flight health check fails, which Explore resends because the command was
* provably never sent (controller.ts remoteDaemonUnavailable). The bridge gives it only on the same proof.
*/
const remoteUnavailable = () => failure("COMMAND_FAILED", "Remote daemon is unavailable");
const remoteUnavailable = () => failure("COMMAND_FAILED", "Remote daemon is unavailable", { reason: "remote_daemon_unavailable", dispatched: "no" });

/** Flags agent-device takes after a command; the bridge needs the platform only from its own context. */
const VALUE_FLAGS = new Set(["--session", "--device", "--platform", "--level", "--udid", "--scope", "--bundle-id", "--serial", "--timeout"]);
Expand Down Expand Up @@ -247,6 +247,8 @@ export function iosRuntimeVersion(runtime: unknown): string | undefined {

const DEEP_LINK_PROMPT_CONFIRM = { dxPoints: 74, dyPoints: 37 };
const KEYBOARD_DISMISS_LABELS = ["Hide keyboard", "Dismiss keyboard", "Done"];
/** What the model reads when the keyboard cannot be dismissed: the same next step agent-device's refusal gives. */
const KEYBOARD_DISMISS_HINT = "Press the visible Hide Keyboard key or an app control that ends editing.";
// A keyboard covers the lower part of the screen; its keys start below this share of the height.
const KEYBOARD_TOP = 0.4;
const ALERT_PREFERRED_BUTTONS = ["Open", "OK", "Allow", "Allow While Using App", "Continue"];
Expand Down Expand Up @@ -613,17 +615,17 @@ export async function runBridgeCommand(argv: readonly string[], context: ArgentB
await beforeAction();
if (platform === "android") await call("button", { button: "back" });
else {
// agent-device taps the keyboard's own dismiss control, and fails with this message when there is none, which
// callers answer with the return key. Only a key in the keyboard's area counts: an app's "Done" above it is not one.
// Refuses with agent-device's reason when the keyboard has no dismiss key, and with a hint like agent-device's, which
// the model reads. Only a key in the keyboard's area counts: an app's "Done" above it is not one.
const key = flatten(current).find(node => KEYBOARD_DISMISS_LABELS.includes(node.label ?? "") && node.frame && node.frame.y >= KEYBOARD_TOP);
if (!key) return failure("COMMAND_FAILED", "Unable to dismiss the iOS keyboard: the keyboard exposes no dismiss control.");
if (!key) return failure("UNSUPPORTED_OPERATION", "Unable to dismiss the keyboard: it exposes no dismiss key.", { reason: "keyboard_dismiss_unavailable", dispatched: "no", hint: KEYBOARD_DISMISS_HINT });
const size = current.screen ?? await screen();
await tap(...Object.values(centre({ x: key.frame!.x * size.width, y: key.frame!.y * size.height,
width: key.frame!.width * size.width, height: key.frame!.height * size.height }, size)) as [number, number]);
}
await context.sleep(500);
const visible = (await tree()).keyboardVisible === true;
if (visible && platform === "ios") return failure("COMMAND_FAILED", "Unable to dismiss the iOS keyboard: it is still showing after its dismiss key was pressed.");
if (visible && platform === "ios") return failure("COMMAND_FAILED", "Unable to dismiss the keyboard: it is still showing after its dismiss key was pressed.", { reason: "keyboard_dismiss_unavailable", hint: KEYBOARD_DISMISS_HINT });
return success({ platform, action: "dismiss", wasVisible: true, dismissed: !visible, visible });
}
case "scroll": {
Expand Down Expand Up @@ -666,7 +668,7 @@ export async function runBridgeCommand(argv: readonly string[], context: ArgentB
await tap(button.frame.x + button.frame.width / 2, button.frame.y + button.frame.height / 2);
return success({ action: "back", mode: "in-app" });
}
if (flags["in-app"] === true || platform === "ios") return failure("COMMAND_FAILED", "in-app back control is not available");
if (flags["in-app"] === true || platform === "ios") return failure("COMMAND_FAILED", "in-app back control is not available", { reason: "back_swipe_not_dispatched", dispatched: "no" });
await call("button", { button: "back" });
return success({ action: "back", mode: "system" });
}
Expand Down Expand Up @@ -776,7 +778,7 @@ export async function runBridgeCommand(argv: readonly string[], context: ArgentB
if (error instanceof BridgeFailure) return failure(error.code, error.message);
if (error instanceof ArgentCallError) {
// Never "Remote daemon is unavailable" here: the request may have run, so Explore must not resend it.
return failure(error.sent ? "ARGENT_TOOL_FAILED" : "COMMAND_FAILED", error.sent ? error.message : "Remote daemon is unavailable");
return error.sent ? failure("ARGENT_TOOL_FAILED", error.message) : remoteUnavailable();
}
return failure("ARGENT_BRIDGE_FAILED", error instanceof Error ? error.message : String(error));
}
Expand Down
44 changes: 38 additions & 6 deletions src/capture/controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,38 @@ export function controllerCommandFailure(stdout: string, command: string): CiErr
return failure;
}

/** agent-device's typed refusal reason (`error.details.reason`), when the failure carries one. */
export function controllerFailureReason(error: unknown): string | undefined {
if (!(error instanceof CiError)) return undefined;
const details = nativeControllerFailures.get(error)?.details;
return isRecord(details) && typeof details.reason === "string" ? details.reason : undefined;
}

/**
* A back that left the screen as it was: agent-device swiped from the left edge and nothing changed
* (`back_swipe_no_change`), or sent nothing (`back_swipe_not_dispatched`, also the Argent bridge's answer
* when the screen has no back control).
*/
export function backLeftScreenUnchanged(error: unknown): boolean {
const reason = controllerFailureReason(error);
return reason === "back_swipe_no_change" || reason === "back_swipe_not_dispatched";
}

/**
* The one dismiss control (Close, Cancel, Back) agent-device saw in the screen's top bar when its back changed
* nothing (`details.dismissControls`). agent-device never presses it: on iOS back only pops the navigation stack.
* Undefined when it listed none or several. `identifier` is set when agent-device names the control by it; the
* label may then be empty.
*/
export function loneBackDismissControl(error: unknown): { selector: string; label: string; identifier?: string } | undefined {
if (controllerFailureReason(error) !== "back_swipe_no_change") return undefined;
const controls = (nativeControllerFailures.get(error as CiError)?.details as Record<string, unknown> | undefined)?.dismissControls;
if (!Array.isArray(controls) || controls.length !== 1 || !isRecord(controls[0])) return undefined;
const { selector, label, identifier } = controls[0];
if (typeof selector !== "string" || !selector.trim() || typeof label !== "string") return undefined;
return typeof identifier === "string" && identifier ? { selector, label, identifier } : { selector, label };
}

function busyControllerReadFailure(error: unknown): boolean {
if (!(error instanceof CiError)) return false;
const native = nativeControllerFailures.get(error);
Expand Down Expand Up @@ -158,14 +190,14 @@ function responsiveAccessibilityRead(args: readonly string[], result: CommandRes
}

/**
* agent-device checks the remote daemon's /health, with a 3 s timeout, before it sends a command,
* and fails with this error when the check gets no answer or a 5xx. The command itself was never
* sent, so sending it again cannot repeat anything, even a UI action.
* agent-device checks the remote daemon's /health before it sends a command, and refuses with
* `remote_daemon_unavailable` and `dispatched: "no"` when the check gets no answer or a 5xx. The
* command itself was never sent, so sending it again cannot repeat anything, even a UI action.
*/
export function remoteDaemonUnavailable(error: unknown): boolean {
if (!(error instanceof CiError)) return false;
const native = nativeControllerFailures.get(error);
return !!native && native.code === "COMMAND_FAILED" && native.message === "Remote daemon is unavailable";
if (controllerFailureReason(error) !== "remote_daemon_unavailable") return false;
const details = nativeControllerFailures.get(error as CiError)?.details;
return isRecord(details) && details.dispatched === "no";
}

/**
Expand Down
Loading