diff --git a/.fallowrc.json b/.fallowrc.json index cb747e769c..37d80e8d38 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -182,16 +182,29 @@ }, { "comment": "Daemon route handlers are reached only through the dynamic `import()` table in request-handler-chain.ts, which --production analysis cannot follow to a consumer.", - "file": "src/daemon/handlers/{human-control,lease,session,snapshot,react-native,record-trace,find,interaction}.ts", + "file": "src/daemon/handlers/{human-control,lease,session,snapshot,react-native,record-trace}.ts", "exports": [ "handleHumanControlCommand", "handleLeaseCommands", "handleSessionCommands", "handleSnapshotCommands", "handleReactNativeCommands", - "handleRecordTraceCommands", - "handleFindCommands", - "handleInteractionCommands" + "handleRecordTraceCommands" + ] + }, + { + "comment": "The interaction façade is a named external seam: these helpers are part of the declared public surface, while Fallow cannot infer their consumers from the façade boundary.", + "file": "src/daemon/interaction/index.ts", + "exports": [ + "RefSnapshotFlagGuardResponse", + "readTextForNode", + "assertRecordedFillParameterization", + "publishInteractionAmbiguityCandidates", + "assertRefMutationAdmitted", + "refMutationAdmissionResponse", + "refSnapshotFlagGuardResponse", + "handleInteractionCommands", + "handleFindCommands" ] }, { diff --git a/fallow-baselines/health.json b/fallow-baselines/health.json index 8604759aee..bf5fd067f4 100644 --- a/fallow-baselines/health.json +++ b/fallow-baselines/health.json @@ -138,22 +138,22 @@ "count": 1 } }, - "src/daemon/handlers/__tests__/interaction-touch.test.ts": { + "src/daemon/interaction/internal/__tests__/interaction-touch.test.ts": { "crap_moderate": { "count": 2 } }, - "src/daemon/handlers/__tests__/interaction-touch-press.test.ts": { + "src/daemon/interaction/internal/__tests__/interaction-touch-press.test.ts": { "crap_moderate": { "count": 3 } }, - "src/daemon/handlers/__tests__/interaction-touch-press-admission.test.ts": { + "src/daemon/interaction/internal/__tests__/interaction-touch-press-admission.test.ts": { "crap_moderate": { "count": 2 } }, - "src/daemon/handlers/__tests__/interaction-touch-response.test.ts": { + "src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts": { "crap_moderate": { "count": 1 } @@ -207,7 +207,7 @@ "count": 1 } }, - "src/daemon/handlers/find.ts": { + "src/daemon/interaction/internal/find.ts": { "complexity_moderate": { "count": 1 }, @@ -215,7 +215,7 @@ "count": 1 } }, - "src/daemon/handlers/interaction-touch-reference-frame.ts": { + "src/daemon/interaction/internal/interaction-touch-reference-frame.ts": { "crap_moderate": { "count": 1 } diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index afd90a1806..4c3aa26a68 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -143,7 +143,7 @@ const PARENT_OWNED_TOUCH_POINT_GAP_ISSUE = 'https://github.com/callstack/agent-d // (interaction-response-construction-guard.test.ts) keeps new branches on it. const SHARED_RESPONSE_CONSTRUCTION: GuaranteeEnforcement = { kind: 'runtime', - via: 'src/daemon/handlers/interaction-touch-response.ts#buildInteractionResponseData', + via: 'src/daemon/interaction/internal/interaction-touch-response.ts#buildInteractionResponseData', }; // The two runtime tree paths (selector and ref resolution) run the SAME shared @@ -183,7 +183,7 @@ const RUNTIME_TREE_SHARED_GUARANTEES = { responseConstruction: SHARED_RESPONSE_CONSTRUCTION, responseIdentity: { kind: 'runtime', - via: 'src/daemon/handlers/interaction-touch-targets.ts#interactionResultExtra', + via: 'src/daemon/interaction/internal/interaction-touch-targets.ts#interactionResultExtra', }, verifyEvidence: { kind: 'runtime', @@ -271,7 +271,7 @@ export const INTERACTION_DISPATCH_PATHS: Record> = Object.freeze({ 'packages/platform-apple/src/runner/__tests__/runner-command-retry.test.ts': 1280, 'src/__tests__/cli-client-commands.test.ts': 1304, 'src/__tests__/cli-config.test.ts': 1282, - 'src/daemon/handlers/__tests__/find.test.ts': 1199, + 'src/daemon/interaction/internal/__tests__/find.test.ts': 1198, 'packages/platform-apple/src/core/__tests__/perf.test.ts': 1222, 'src/mcp/__tests__/command-tools.test.ts': 1216, 'src/daemon/replay/internal/__tests__/session-replay-divergence.test.ts': 1100, diff --git a/scripts/depgraph/affected.test.ts b/scripts/depgraph/affected.test.ts index 9b751b2627..862f942906 100644 --- a/scripts/depgraph/affected.test.ts +++ b/scripts/depgraph/affected.test.ts @@ -89,16 +89,16 @@ test('commandsReaching follows the dynamic import a route uses to load its handl const edges = edgesOf({ 'src/daemon/handlers/session.ts': "import { helper } from '../../utils/helper.ts';\nexport const h = helper;", - 'src/daemon/handlers/find.ts': 'export const f = 1;', + 'src/daemon/interaction/index.ts': 'export const f = 1;', 'src/utils/helper.ts': 'export const helper = 1;', 'src/daemon/chain.ts': [ "export const routes = { session: () => import('./handlers/session.ts'),", - " find: () => import('./handlers/find.ts') };", + " find: () => import('./interaction/index.ts') };", ].join('\n'), }); const chains = [ { command: 'open', route: 'session', entry: 'src/daemon/handlers/session.ts' }, - { command: 'find', route: 'find', entry: 'src/daemon/handlers/find.ts' }, + { command: 'find', route: 'find', entry: 'src/daemon/interaction/index.ts' }, ]; assert.deepEqual( @@ -107,7 +107,9 @@ test('commandsReaching follows the dynamic import a route uses to load its handl ); // The entry module itself counts as part of its own chain. assert.deepEqual( - commandsReaching('src/daemon/handlers/find.ts', chains, edges).map((chain) => chain.command), + commandsReaching('src/daemon/interaction/index.ts', chains, edges).map( + (chain) => chain.command, + ), ['find'], ); }); diff --git a/scripts/help-conformance-sample-outputs.mjs b/scripts/help-conformance-sample-outputs.mjs index 869be3f1aa..d48c2426b2 100644 --- a/scripts/help-conformance-sample-outputs.mjs +++ b/scripts/help-conformance-sample-outputs.mjs @@ -110,7 +110,7 @@ export const STALE_REF_SAMPLE = { Hint: Ref @e12 was minted from snapshot s5 but the session's ref frame is now s7 — re-run snapshot -i.`, }; -// AMBIGUOUS_MATCH from buildAmbiguousMatchError (src/daemon/handlers/find.ts) +// AMBIGUOUS_MATCH from buildAmbiguousMatchError (src/daemon/interaction/internal/find.ts) // — the parity test drives that exact producer. The by-design rejection // instead of silent disambiguation: #1597 made the candidate refs (ref, role, // label/identifier — the same compact rendering as snapshot -i) print diff --git a/scripts/layering/architecture-ownership.ts b/scripts/layering/architecture-ownership.ts index 1c0871a2be..40613a5ef8 100644 --- a/scripts/layering/architecture-ownership.ts +++ b/scripts/layering/architecture-ownership.ts @@ -49,6 +49,7 @@ const DAEMON_INTERACTION_FACADE = { exports: [ 'CaptureSnapshotForSession', 'ContextFromFlags', + 'FindRouteInput', 'InteractionRouteInput', 'RefSnapshotFlagGuardResponse', 'assertRecordedFillParameterization', @@ -56,6 +57,8 @@ const DAEMON_INTERACTION_FACADE = { 'captureSnapshotForSession', 'createInteractionRuntime', 'finalizeTouchInteraction', + 'handleFindCommands', + 'handleInteractionCommands', 'publishInteractionAmbiguityCandidates', 'readSettleRequest', 'readTextForNode', @@ -65,6 +68,34 @@ const DAEMON_INTERACTION_FACADE = { ], } as const; +export const INTERACTION_RETIRED_HANDLER_PATHS = [ + 'src/daemon/handlers/find.ts', + 'src/daemon/handlers/find-match-ranking.ts', + 'src/daemon/handlers/find-match-resolution.ts', + 'src/daemon/handlers/find-target-capture.ts', + 'src/daemon/handlers/interaction.ts', + 'src/daemon/handlers/interaction-android-escape.ts', + 'src/daemon/handlers/interaction-gesture.ts', + 'src/daemon/handlers/interaction-gesture-response.ts', + 'src/daemon/handlers/interaction-ios-tap-outcome.ts', + 'src/daemon/handlers/interaction-targeting.ts', + 'src/daemon/handlers/interaction-touch.ts', + 'src/daemon/handlers/interaction-touch-android-freshness.ts', + 'src/daemon/handlers/interaction-touch-android-readiness.ts', + 'src/daemon/handlers/interaction-touch-direct-ios-eligibility.ts', + 'src/daemon/handlers/interaction-touch-direct-ios.ts', + 'src/daemon/handlers/interaction-touch-fill.ts', + 'src/daemon/handlers/interaction-touch-payload.ts', + 'src/daemon/handlers/interaction-touch-policy.ts', + 'src/daemon/handlers/interaction-touch-prepare.ts', + 'src/daemon/handlers/interaction-touch-press-admission.ts', + 'src/daemon/handlers/interaction-touch-press.ts', + 'src/daemon/handlers/interaction-touch-reference-frame.ts', + 'src/daemon/handlers/interaction-touch-response.ts', + 'src/daemon/handlers/interaction-touch-runtime.ts', + 'src/daemon/handlers/interaction-touch-targets.ts', +] as const; + export const SESSION_OBSERVABILITY_RETIRED_HANDLER_PATHS = [ 'src/daemon/handlers/session-observability.ts', 'src/daemon/handlers/session-perf-runtime.ts', diff --git a/scripts/layering/check.ts b/scripts/layering/check.ts index 56e7eb8ad0..3daa709513 100644 --- a/scripts/layering/check.ts +++ b/scripts/layering/check.ts @@ -76,6 +76,7 @@ import { } from './model.ts'; import { checkDaemonModularityRatchets, + checkRetiredInteractionPaths, checkRetiredSessionLifecyclePaths, checkRetiredSessionObservabilityPaths, daemonModularitySummary, @@ -587,6 +588,7 @@ export const LAYERING_RULES: Readonly> = { 'session-state-ownership': (context) => checkSessionStateOwnership(context.sources), 'daemon-modularity-ratchets': (context) => [ ...checkDaemonModularityRatchets(context.edges, context.typeCycleMembers), + ...checkRetiredInteractionPaths(context.sourceFiles), ...checkRetiredSessionLifecyclePaths(context.sourceFiles), ...checkRetiredSessionObservabilityPaths(context.sourceFiles), ], diff --git a/scripts/layering/daemon-modularity.test.ts b/scripts/layering/daemon-modularity.test.ts index d9044b885b..5fc8394a9b 100644 --- a/scripts/layering/daemon-modularity.test.ts +++ b/scripts/layering/daemon-modularity.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import { checkDaemonModularityRatchets, + checkRetiredInteractionPaths, checkRetiredSessionLifecyclePaths, checkRetiredSessionObservabilityPaths, DAEMON_MODULARITY_BASELINE, @@ -317,10 +318,18 @@ test('interaction rejects handler crossings and deep imports around its facade', 'src/daemon/handlers/interaction.ts', "import { refSnapshotFlagGuardResponse } from '../interaction/internal/interaction-flags.ts';\nexport function handleInteractionCommands() {}", ], + [ + 'src/daemon/handlers/find.ts', + "import { preferOnscreenMatches } from '../interaction/internal/find-match-ranking.ts';\nexport function handleFindCommands() {}", + ], [ 'src/daemon/interaction/internal/interaction-runtime.ts', "import { handleInteractionCommands } from '../../handlers/interaction.ts';\nexport function createInteractionRuntime() {}", ], + [ + 'src/daemon/interaction/internal/find.ts', + "import { handleFindCommands } from '../../handlers/find.ts';\nexport function find() {}", + ], [ 'src/daemon/generic-settle.ts', "import { createInteractionRuntime } from './interaction/internal/interaction-runtime.ts';", @@ -337,6 +346,10 @@ test('interaction rejects handler crossings and deep imports around its facade', 'src/daemon/interaction/internal/interaction-flags.ts', 'export function refSnapshotFlagGuardResponse() {}', ], + [ + 'src/daemon/interaction/internal/find-match-ranking.ts', + 'export function preferOnscreenMatches() {}', + ], [ 'src/daemon/interaction/internal/interaction-read.ts', 'export function readTextForNode() {}', @@ -348,7 +361,7 @@ test('interaction rejects handler crossings and deep imports around its facade', [...baselineEdges(), ...edges], baselineTypeCycleMembers(), ); - assert.equal(violations.length, 5); + assert.equal(violations.length, 7); assert.ok( violations.some(({ message }) => message.includes( @@ -367,7 +380,7 @@ test('interaction rejects handler crossings and deep imports around its facade', violations.filter(({ message }) => message.includes("must not import daemon-interaction's internal tree"), ).length, - 4, + 5, ); }); @@ -491,6 +504,26 @@ test('session observability rejects restored handler paths', () => { ); }); +test('interaction rejects restored handler paths', () => { + const restoredPaths = [ + 'src/daemon/handlers/find.ts', + 'src/daemon/handlers/interaction-touch-direct-ios-regressed.ts', + 'src/daemon/handlers/interaction-common-regressed.ts', + 'src/daemon/interaction/internal/find.ts', + ] as const; + const violations = checkRetiredInteractionPaths(restoredPaths); + + assert.deepEqual( + violations.map(({ file, message }) => ({ file, message })), + restoredPaths.slice(0, 3).map((file) => ({ + file, + message: + `retired interaction path was restored: ${file}. ` + + 'Keep the neutral seam at its daemon owner instead of rebuilding a handler grab-bag.', + })), + ); +}); + test('R9 records zone ceilings and keeps engine files outside the largest component', () => { // One commands file and one engine file traded for two provider-webdriver ones, so the // total stays at the baseline and only the per-zone claims are on trial. diff --git a/scripts/layering/daemon-modularity.ts b/scripts/layering/daemon-modularity.ts index 5e007f8c34..37937b4b89 100644 --- a/scripts/layering/daemon-modularity.ts +++ b/scripts/layering/daemon-modularity.ts @@ -2,6 +2,7 @@ import path from 'node:path'; import { LOGICAL_MODULE_POLICIES, matchesDeclaredRoot, + INTERACTION_RETIRED_HANDLER_PATHS, SESSION_LIFECYCLE_RETIRED_HANDLER_PATHS, SESSION_OBSERVABILITY_RETIRED_HANDLER_PATHS, type LogicalModulePolicy, @@ -77,6 +78,15 @@ export function checkRetiredSessionObservabilityPaths( ); } +export function checkRetiredInteractionPaths(sourceFiles: readonly string[]): LayeringViolation[] { + return checkRetiredHandlerPaths( + sourceFiles, + INTERACTION_RETIRED_HANDLER_PATHS, + /^src\/daemon\/handlers\/(?:find(?:-[^/]+)?|interaction(?:-[^/]+)?)\.ts$/, + 'interaction', + ); +} + function checkRetiredHandlerPaths( sourceFiles: readonly string[], retiredPaths: readonly string[], diff --git a/scripts/layering/model.test.ts b/scripts/layering/model.test.ts index 6792c4fd15..888436e0e6 100644 --- a/scripts/layering/model.test.ts +++ b/scripts/layering/model.test.ts @@ -315,7 +315,10 @@ test('session-state writes are found by field, and non-daemon or undeclared name // a local that is not a declared SessionState field ['src/daemon/session-observability/internal/session-audio.ts', 'session.somethingElse = 1;'], // reads and comparisons are not writes - ['src/daemon/handlers/find.ts', "if (session.refFrameState === 'active') return;"], + [ + 'src/daemon/interaction/internal/find.ts', + "if (session.refFrameState === 'active') return;", + ], // a write into a sub-object is not a write to the field itself ['src/daemon/handlers/session-probe.ts', 'session.refFrameState.inner = 1;'], // a different binding that happens to have a matching property diff --git a/src/__tests__/test-utils/boundary-fault-matrix.ts b/src/__tests__/test-utils/boundary-fault-matrix.ts index 65a96b2413..a8b56d2437 100644 --- a/src/__tests__/test-utils/boundary-fault-matrix.ts +++ b/src/__tests__/test-utils/boundary-fault-matrix.ts @@ -193,7 +193,7 @@ export const BOUNDARY_FAULT_MATRIX = { 'optional-optimization-failure': { mutation: { kind: 'covered', - evidence: ['src/daemon/handlers/__tests__/interaction-touch-direct-ios.test.ts'], + evidence: ['src/daemon/interaction/internal/__tests__/interaction-touch-direct-ios.test.ts'], invariants: ['best-effort-degradation'], }, read: { diff --git a/src/commands/__tests__/command-explain.test.ts b/src/commands/__tests__/command-explain.test.ts index b9deac2f68..97146005df 100644 --- a/src/commands/__tests__/command-explain.test.ts +++ b/src/commands/__tests__/command-explain.test.ts @@ -80,7 +80,7 @@ describe('explainCommand', () => { if (!result.found) return; expect(result.explanation.cli?.commandFlags.map((flag) => flag.key)).toContain('settle'); expect(result.explanation.files).toContain('src/commands/interaction/index.ts'); - expect(result.explanation.files).toContain('src/daemon/handlers/interaction.ts'); + expect(result.explanation.files).toContain('src/daemon/interaction/index.ts'); expect(result.explanation.files.every(fileExists)).toBe(true); }); @@ -146,7 +146,7 @@ describe('explainCommand table-driven coverage', () => { }); test.each([ - ['press', ['src/commands/interaction/index.ts', 'src/daemon/handlers/interaction.ts']], + ['press', ['src/commands/interaction/index.ts', 'src/daemon/interaction/index.ts']], ['apps', ['src/commands/management/app.ts']], // R39: screenshot has no dispatch projection left, so its platform work is named directly. ['screenshot', ['src/commands/capture/screenshot.ts', 'src/daemon/screenshot-runtime.ts']], diff --git a/src/daemon/__tests__/android-owner-seam.test.ts b/src/daemon/__tests__/android-owner-seam.test.ts index 531183f42b..f30f3050ac 100644 --- a/src/daemon/__tests__/android-owner-seam.test.ts +++ b/src/daemon/__tests__/android-owner-seam.test.ts @@ -6,8 +6,8 @@ import { ensureAndroidBlockingSystemDialogReady, recoverAndroidBlockingSystemDialog, } from '../android-system-dialog.ts'; -import { detectAndroidEscapeSurface } from '../handlers/interaction-android-escape.ts'; -import { resolveDirectTouchReferenceFrameSafely } from '../handlers/interaction-touch-reference-frame.ts'; +import { detectAndroidEscapeSurface } from '../android-escape-surface.ts'; +import { resolveDirectTouchReferenceFrameSafely } from '../interaction/internal/interaction-touch-reference-frame.ts'; import { SessionStore } from '../session-store.ts'; test('provider-owned Android sessions bypass local observation and recovery', async () => { diff --git a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts b/src/daemon/__tests__/interaction-get-runtime-fixture.ts similarity index 96% rename from src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts rename to src/daemon/__tests__/interaction-get-runtime-fixture.ts index a7efdb341a..6f67304fdb 100644 --- a/src/daemon/handlers/__tests__/interaction-get-runtime-fixture.ts +++ b/src/daemon/__tests__/interaction-get-runtime-fixture.ts @@ -26,13 +26,10 @@ import { HOVER_UNAVAILABLE_HINT, } from '@agent-device/contracts/touch-runtime'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import type { - BindDeviceRuntime, - InspectDeviceRuntimeFacts, -} from '../../request-runtime-binding.ts'; -import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; -import { androidObservationFixture } from '../../__tests__/android-observation-fixture.ts'; -import { createUnavailableRuntimeFactsForTest } from '../../../__tests__/test-utils/runtime-operation-facts.ts'; +import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; +import { captureSnapshotWithInteractor } from '../handlers/snapshot-interactor-capture.ts'; +import { androidObservationFixture } from './android-observation-fixture.ts'; +import { createUnavailableRuntimeFactsForTest } from '../../__tests__/test-utils/runtime-operation-facts.ts'; /** * The request-bound runtime seam `get` consumes, faked at `inspectFacts` / `bindDevice` — never diff --git a/src/daemon/interaction/internal/__tests__/interaction-read.test.ts b/src/daemon/__tests__/interaction-read.test.ts similarity index 100% rename from src/daemon/interaction/internal/__tests__/interaction-read.test.ts rename to src/daemon/__tests__/interaction-read.test.ts diff --git a/src/daemon/__tests__/ref-snapshot-flags.test.ts b/src/daemon/__tests__/ref-snapshot-flags.test.ts new file mode 100644 index 0000000000..8e761bb930 --- /dev/null +++ b/src/daemon/__tests__/ref-snapshot-flags.test.ts @@ -0,0 +1,17 @@ +import { expect, test } from 'vitest'; +import { refSnapshotFlagGuardResponse } from '../ref-snapshot-flags.ts'; + +test('refSnapshotFlagGuardResponse returns unsupported snapshot flags for @ref flows', () => { + const response = refSnapshotFlagGuardResponse('press', { + snapshotDepth: 2, + snapshotScope: 'Login', + snapshotRaw: true, + }); + expect(response).toEqual({ + ok: false, + error: { + code: 'INVALID_ARGS', + message: 'press @ref does not support --depth, --scope, --raw.', + }, + }); +}); diff --git a/src/daemon/__tests__/request-handler-chain.test.ts b/src/daemon/__tests__/request-handler-chain.test.ts index 0ca50b6417..e6c2021df6 100644 --- a/src/daemon/__tests__/request-handler-chain.test.ts +++ b/src/daemon/__tests__/request-handler-chain.test.ts @@ -1,7 +1,9 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; +import { fileURLToPath } from 'node:url'; import { test } from 'vitest'; import { INTERNAL_COMMANDS } from '../../command-catalog.ts'; +import { eagerClosureOf } from '../../__tests__/eager-import-closure.fixtures.ts'; import { LeaseRegistry } from '../lease-registry.ts'; import { runRequestHandlerChain } from '../request-handler-chain.ts'; import { getDaemonRouteOwnerFiles } from '../route-owner-files.ts'; @@ -10,7 +12,7 @@ import { LINUX_DEVICE } from '../../__tests__/test-utils/device-fixtures.ts'; import { makeIosSession, makeSession } from '../../__tests__/test-utils/session-factories.ts'; import { makeSnapshotState } from '../../__tests__/test-utils/snapshot-builders.ts'; import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; -import { dispatchSwipeViaRuntime } from '../handlers/interaction-gesture.ts'; +import { handleInteractionCommands } from '../interaction/index.ts'; import { createPlatformRuntimeGateway } from '../../platform-runtime.ts'; import { createRequestRuntimeBindings } from '../request-runtime-binding.ts'; import { createLocalLinuxToolProvider, withLinuxToolProvider } from '@agent-device/platform-linux'; @@ -95,6 +97,19 @@ test('route owner files match the production module loaders', () => { ); }); +test('interaction route implementations stay outside the request handler eager closure', () => { + const chainFile = fileURLToPath(new URL('../request-handler-chain.ts', import.meta.url)); + const eagerModules = eagerClosureOf(chainFile); + const interactionModules = eagerModules.filter((file) => + file.includes('/src/daemon/interaction/'), + ); + assert.deepEqual( + interactionModules, + [], + 'interaction routes must be loaded only through the request handler lazy closure', + ); +}); + test('request handler chain routes trace commands to the record-trace family', async () => { const response = await runRequestHandlerChain(makeChainParams(makeRequest('trace', ['start']))); @@ -265,7 +280,7 @@ test('duration-less public coordinate swipe retains Linux drag behavior', async const response = await withLinuxToolProvider( provider, async () => - await dispatchSwipeViaRuntime({ + await handleInteractionCommands({ inspectFacts: bindings.inspectFacts, bindDevice: bindings.bindDevice, req: { @@ -284,6 +299,7 @@ test('duration-less public coordinate swipe retains Linux drag behavior', async }), ); + assert.ok(response); assert.equal(response.ok, true); if (!response.ok) return; assert.ok(response.data); diff --git a/src/daemon/handlers/interaction-android-escape.ts b/src/daemon/android-escape-surface.ts similarity index 62% rename from src/daemon/handlers/interaction-android-escape.ts rename to src/daemon/android-escape-surface.ts index 5dca52ec52..9a48c98434 100644 --- a/src/daemon/handlers/interaction-android-escape.ts +++ b/src/daemon/android-escape-surface.ts @@ -1,7 +1,7 @@ import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; import { AppError } from '@agent-device/kernel/errors'; -import { isActiveProviderDevice } from '../../provider-device-runtime.ts'; -import type { SessionState } from '../types.ts'; +import { isActiveProviderDevice } from '../provider-device-runtime.ts'; +import type { SessionState } from './types.ts'; export type AndroidEscapeSurface = { expectedPackage: string; @@ -11,31 +11,6 @@ export type AndroidEscapeSurface = { permissionDialog?: boolean; }; -/** - * Post-press escape guard. Throws when the tap left the app for a genuine - * escape surface (settings/systemui/launcher). A foregrounded permission - * prompt is NOT an escape — the press succeeded and raised a system dialog - * the agent consumes via `alert` — so it returns a response warning instead. - */ -export async function assertAndroidPressStayedInApp( - session: SessionState, - targetLabel: string, - observation?: AndroidObservationAdapter, -): Promise { - const surface = await detectAndroidEscapeSurface(session, observation); - if (!surface) return undefined; - - if (surface.permissionDialog) { - return `press ${targetLabel} opened an Android permission dialog (${surface.foregroundPackage}) over ${surface.expectedPackage}. ${surface.hint}`; - } - - throw new AppError( - 'COMMAND_FAILED', - `press ${targetLabel} left ${session.appBundleId} and foregrounded ${surface.foregroundPackage}. The tap likely escaped the app.`, - surface, - ); -} - export async function detectAndroidEscapeSurface( session: SessionState, observation?: AndroidObservationAdapter, @@ -78,14 +53,6 @@ export function describeAndroidEscapeSurface(surface: AndroidEscapeSurface): str return `${surface.foregroundPackage} is foreground instead of ${surface.expectedPackage}`; } -export function isAndroidEscapeError(error: AppError): boolean { - return ( - error.code === 'COMMAND_FAILED' && - typeof error.details?.expectedPackage === 'string' && - typeof error.details?.foregroundPackage === 'string' - ); -} - function buildAndroidEscapeHint(permissionDialog: boolean): string { if (permissionDialog) { return 'Use "alert get" to inspect it, then "alert accept" or "alert dismiss" to respond.'; diff --git a/src/daemon/handlers/__tests__/react-native.test.ts b/src/daemon/handlers/__tests__/react-native.test.ts index c27be4725c..4a0ea99050 100644 --- a/src/daemon/handlers/__tests__/react-native.test.ts +++ b/src/daemon/handlers/__tests__/react-native.test.ts @@ -9,7 +9,7 @@ import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; vi.mock('../snapshot-capture.ts', () => ({ captureSnapshot: vi.fn(), diff --git a/src/daemon/handlers/__tests__/snapshot-handler.test.ts b/src/daemon/handlers/__tests__/snapshot-handler.test.ts index f851584c04..f3467d374f 100644 --- a/src/daemon/handlers/__tests__/snapshot-handler.test.ts +++ b/src/daemon/handlers/__tests__/snapshot-handler.test.ts @@ -4,7 +4,7 @@ import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../__tests__/interaction-get-runtime-fixture.ts'; import fs from 'node:fs'; import path from 'node:path'; import { handleSnapshotCommands as handleProductionSnapshotCommands } from '../snapshot.ts'; diff --git a/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts b/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts index 8747e66c50..02f93b83c8 100644 --- a/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts +++ b/src/daemon/handlers/__tests__/system-surface-disclosure.test.ts @@ -1,7 +1,7 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { legacyDispatchCapture } from '../../__tests__/legacy-snapshot-capture-fixture.ts'; -import { handleFindCommands } from '../find.ts'; -import { getRuntimeBindings } from './interaction-get-runtime-fixture.ts'; +import { handleFindCommands } from '../../interaction/index.ts'; +import { getRuntimeBindings } from '../../__tests__/interaction-get-runtime-fixture.ts'; import { dispatchFindReadOnlyViaRuntime, dispatchWaitViaRuntime } from '../../selector-runtime.ts'; import type { DaemonRequest, DaemonResponse } from '../../types.ts'; import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '../../../core/android-system-surface-disclosure.ts'; @@ -29,7 +29,7 @@ vi.mock('../../device-ready.ts', () => ({ import { resolveTargetDevice } from '../../../core/dispatch-resolve.ts'; import { ANDROID_EMULATOR } from '../../../__tests__/test-utils/device-fixtures.ts'; -import { withSystemSurfaceDisclosure } from '../system-surface-disclosure.ts'; +import { withSystemSurfaceDisclosure } from '../../system-surface-disclosure.ts'; // The occluding-shade capture every scenario below consumes: no application window content, one // active quick-settings surface. The Android capture route stamps systemSurfaceOnly on both the diff --git a/src/daemon/handlers/react-native.ts b/src/daemon/handlers/react-native.ts index fcb4dd7de2..0f720336e5 100644 --- a/src/daemon/handlers/react-native.ts +++ b/src/daemon/handlers/react-native.ts @@ -22,7 +22,7 @@ import { type InteractionRouteInput, } from '../interaction/index.ts'; import { expireRefFrame } from '../ref-frame.ts'; -import { readSnapshotNodesReferenceFrame } from './interaction-touch-reference-frame.ts'; +import { readSnapshotNodesReferenceFrame } from '../touch-reference-frame.ts'; export async function handleReactNativeCommands( params: InteractionRouteInput, diff --git a/src/daemon/interaction/internal/interaction-read.ts b/src/daemon/interaction-read.ts similarity index 87% rename from src/daemon/interaction/internal/interaction-read.ts rename to src/daemon/interaction-read.ts index 1f9c1bbfd7..ed13feafbb 100644 --- a/src/daemon/interaction/internal/interaction-read.ts +++ b/src/daemon/interaction-read.ts @@ -4,7 +4,7 @@ import type { ElementTextUnreadableReason, } from '@agent-device/contracts/element-text-runtime'; import { isIosFamily } from '@agent-device/kernel/device'; -import { runtimeExecutionFromContext } from '../../snapshot-runtime-capture-input.ts'; +import { runtimeExecutionFromContext } from './snapshot-runtime-capture-input.ts'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; import type { DeviceInfo } from '@agent-device/kernel/device'; @@ -12,10 +12,16 @@ import type { SessionSurface } from '@agent-device/contracts/session'; import { extractReadableText, prefersValueForReadableText, -} from '../../../snapshot/snapshot-presentation/text-surface.ts'; -import type { ContextFromFlags } from './types.ts'; +} from '../snapshot/snapshot-presentation/text-surface.ts'; +import type { DaemonCommandContext } from './context.ts'; import { resolveRectCenter } from '@agent-device/kernel/rect-center'; +type ContextFromFlags = ( + flags: CommandFlags | undefined, + appBundleId?: string, + traceLogPath?: string, +) => DaemonCommandContext; + export type ReadElementTextAtPoint = ElementTextRuntimeOperations['readTextAtPoint']; export async function readTextForNode(params: { diff --git a/src/daemon/interaction/index.ts b/src/daemon/interaction/index.ts index 56939782a2..a743712e35 100644 --- a/src/daemon/interaction/index.ts +++ b/src/daemon/interaction/index.ts @@ -1,68 +1,42 @@ -import type { CommandFlags } from '@agent-device/contracts/command'; -import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; -import type { Rect, SnapshotState } from '@agent-device/kernel/snapshot'; -import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; +import type { Rect } from '@agent-device/kernel/snapshot'; import { buildRuntimeCaptureInput } from '../snapshot-runtime-capture-input.ts'; -import { markDeferredInteractionOutcome } from '../deferred-interaction-outcome.ts'; -import { expireRefFrame } from '../ref-frame.ts'; import { setSessionSnapshot } from '../session-snapshot.ts'; -import { recordTouchVisualizationEvent } from '../recording-gestures.ts'; -import { isSessionRecording } from '../session-script-publication-capability.ts'; -import { createDaemonRuntimeSessionStore } from '../runtime-session.ts'; import { captureSnapshot as captureSnapshotThroughHandler } from '../handlers/snapshot-capture.ts'; -import { NO_ACTIVE_SESSION_MESSAGE } from '../response.ts'; -import { AppError as KernelAppError } from '@agent-device/kernel/errors'; -import { buildAppleRunnerRequestOptions } from '../apple-runner-options.ts'; -import { isLocalIosRunnerSession } from '../direct-ios-selector.ts'; -import { confirmIosOffscreenTargetVisible } from '../offscreen-target-probe.ts'; import type { BoundGestureExecutor } from '../gesture-runtime.ts'; import type { BoundTouchExecutor } from '../touch-runtime.ts'; import { captureInteractionSnapshot } from './internal/interaction-snapshot.ts'; -import { createInteractionRuntime as createInternalInteractionRuntime } from './internal/interaction-runtime.ts'; -import { finalizeTouchInteraction as finalizeInternalInteraction } from './internal/interaction-common.ts'; -import type { ContextFromFlags, InteractionSnapshotOptions } from './internal/types.ts'; -import type { SessionStore } from '../session-store.ts'; -import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts'; - -export type { ContextFromFlags } from './internal/types.ts'; - -export type InteractionRouteInput = { - req: DaemonRequest; - sessionName: string; - logPath?: string; - sessionStore: SessionStore; - captureSnapshotForSession?: CaptureSnapshotForSession; - contextFromFlags: ContextFromFlags; - inspectFacts?: InspectDeviceRuntimeFacts; - bindDevice?: BindDeviceRuntime; - androidObservation?: AndroidObservationAdapter; -}; - -export type CaptureSnapshotForSession = ( - session: SessionState, - flags: CommandFlags | undefined, - sessionStore: SessionStore, - contextFromFlags: ContextFromFlags, - options: InteractionSnapshotOptions, -) => Promise; +import { + createInteractionRuntime as createInternalInteractionRuntime, + finalizeTouchInteraction, +} from './internal/interaction-route-support.ts'; +import type { + CaptureSnapshotForSession, + FindRouteInput, + InteractionRouteInput, +} from './internal/types.ts'; -export type RefSnapshotFlagGuardResponse = typeof refSnapshotFlagGuardResponse; +export type { + CaptureSnapshotForSession, + ContextFromFlags, + FindRouteInput, + InteractionRouteInput, + RefSnapshotFlagGuardResponse, +} from './internal/types.ts'; -export { readTextForNode } from './internal/interaction-read.ts'; +export { readTextForNode } from '../interaction-read.ts'; export { assertRecordedFillParameterization } from './internal/interaction-recorded-input.ts'; export { publishInteractionAmbiguityCandidates } from './internal/interaction-ambiguity-publication.ts'; export { assertRefMutationAdmitted, refMutationAdmissionResponse, } from './internal/interaction-ref-policy.ts'; -import { - readSettleRequest, - refSnapshotFlagGuardResponse, - settleFlagGuardResponse, -} from './internal/interaction-flags.ts'; +import { readSettleRequest, settleFlagGuardResponse } from './internal/interaction-flags.ts'; +import { refSnapshotFlagGuardResponse } from '../ref-snapshot-flags.ts'; export { readSettleRequest, refSnapshotFlagGuardResponse, settleFlagGuardResponse }; +export { finalizeTouchInteraction }; + export const captureSnapshotForSession: CaptureSnapshotForSession = async ( session, flags, @@ -117,73 +91,25 @@ export function createInteractionRuntime( gestures?: BoundGestureExecutor; }, ) { - const session = params.sessionStore.get(params.sessionName); - if (!session) throw new KernelAppError('SESSION_NOT_FOUND', NO_ACTIVE_SESSION_MESSAGE); return createInternalInteractionRuntime({ - requestId: params.req.meta?.requestId, - flags: params.req.flags, - session, - contextFromFlags: params.contextFromFlags, - captureSnapshot: async (flags, options) => - await (params.captureSnapshotForSession ?? captureSnapshotForSession)( - session, - flags, - params.sessionStore, - params.contextFromFlags, - options, - ), - runtimeSessions: createDaemonRuntimeSessionStore({ - sessionName: params.sessionName, - getSession: () => session, - recordOptions: { - includeSnapshot: true, - omitRefFrameSnapshot: params.req.internal?.findResolvedTarget !== undefined, - }, - setRecord: (record) => { - if (!record.snapshot) return; - setSessionSnapshot(session, record.snapshot); - params.sessionStore.set(params.sessionName, session); - }, - }), - expireRefFrame: () => expireRefFrame(session), - confirmOffscreenTargetVisible: isLocalIosRunnerSession(session, { - skipPendingPostGestureStabilization: false, - }) - ? async (node, rootViewport) => - await confirmIosOffscreenTargetVisible({ - session, - node, - rootViewport, - requestOptions: buildAppleRunnerRequestOptions({ - req: params.req, - logPath: params.logPath, - traceLogPath: session.trace?.outPath, - }), - }) - : undefined, - pairedGestureViewport: params.pairedGestureViewport, - touchExecutor: params.touchExecutor, - gestures: params.gestures, + ...params, + captureSnapshotForSession: params.captureSnapshotForSession ?? captureSnapshotForSession, }); } -type FinalizeTouchInteractionInput = Omit< - Parameters[0], - 'operations' -> & { - session: SessionState; - sessionStore: SessionStore; -}; - -export function finalizeTouchInteraction(params: FinalizeTouchInteractionInput): DaemonResponse { - const { session, sessionStore, ...finalization } = params; - return finalizeInternalInteraction({ - ...finalization, - operations: { - recordAction: sessionStore.recordAction.bind(sessionStore, session), - markDeferredOutcome: (mark) => markDeferredInteractionOutcome({ session, ...mark }), - isSessionRecording: isSessionRecording.bind(null, session), - recordGestureVisualization: recordTouchVisualizationEvent.bind(null, session), - }, +export async function handleInteractionCommands( + params: InteractionRouteInput, +): Promise { + const { handleInteractionCommands: handle } = await import('./internal/interaction.ts'); + return await handle({ + ...params, + captureSnapshotForSession: params.captureSnapshotForSession ?? captureSnapshotForSession, }); } + +export async function handleFindCommands( + params: FindRouteInput, +): Promise { + const { handleFindCommands: handle } = await import('./internal/find.ts'); + return await handle(params); +} diff --git a/src/daemon/handlers/__tests__/find-args.test.ts b/src/daemon/interaction/internal/__tests__/find-args.test.ts similarity index 100% rename from src/daemon/handlers/__tests__/find-args.test.ts rename to src/daemon/interaction/internal/__tests__/find-args.test.ts diff --git a/src/daemon/handlers/__tests__/find-handler-fixture.ts b/src/daemon/interaction/internal/__tests__/find-handler-fixture.ts similarity index 75% rename from src/daemon/handlers/__tests__/find-handler-fixture.ts rename to src/daemon/interaction/internal/__tests__/find-handler-fixture.ts index 94ce11baf5..cf873d072b 100644 --- a/src/daemon/handlers/__tests__/find-handler-fixture.ts +++ b/src/daemon/interaction/internal/__tests__/find-handler-fixture.ts @@ -1,7 +1,7 @@ -import type { SessionStore } from '../../session-store.ts'; -import type { DaemonRequest, DaemonResponse } from '../../types.ts'; -import { handleFindCommands } from '../find.ts'; -import { getRuntimeBindings } from './interaction-get-runtime-fixture.ts'; +import type { SessionStore } from '../../../session-store.ts'; +import type { DaemonRequest, DaemonResponse } from '../../../types.ts'; +import { handleFindCommands } from '../../index.ts'; +import { getRuntimeBindings } from '../../../__tests__/interaction-get-runtime-fixture.ts'; /** * One `handleFindCommands` invocation shape. diff --git a/src/daemon/handlers/__tests__/find-match-ranking.test.ts b/src/daemon/interaction/internal/__tests__/find-match-ranking.test.ts similarity index 98% rename from src/daemon/handlers/__tests__/find-match-ranking.test.ts rename to src/daemon/interaction/internal/__tests__/find-match-ranking.test.ts index 72dc4cb55a..a1df59d373 100644 --- a/src/daemon/handlers/__tests__/find-match-ranking.test.ts +++ b/src/daemon/interaction/internal/__tests__/find-match-ranking.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { RawSnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; -import { makeSnapshotState } from '../../../__tests__/test-utils/snapshot-builders.ts'; +import { makeSnapshotState } from '../../../../__tests__/test-utils/snapshot-builders.ts'; import { preferOnscreenMatches } from '../find-match-ranking.ts'; const VIEWPORT = { x: 0, y: 0, width: 390, height: 844 }; diff --git a/src/daemon/handlers/__tests__/find-single-bind.test.ts b/src/daemon/interaction/internal/__tests__/find-single-bind.test.ts similarity index 80% rename from src/daemon/handlers/__tests__/find-single-bind.test.ts rename to src/daemon/interaction/internal/__tests__/find-single-bind.test.ts index 5f06ae4329..0bbc658158 100644 --- a/src/daemon/handlers/__tests__/find-single-bind.test.ts +++ b/src/daemon/interaction/internal/__tests__/find-single-bind.test.ts @@ -1,18 +1,18 @@ import { test, expect, vi, beforeEach } from 'vitest'; -import type { DaemonResponse } from '../../types.ts'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts'; +import type { DaemonResponse } from '../../../types.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { makeIosSession } from '../../../../__tests__/test-utils/session-factories.ts'; import { mockFocusPoint, mockTypeText, resetGetRuntimeFixture, runtimeBindingSpies, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { invokeFindHandler } from './find-handler-fixture.ts'; -import { legacyDispatchCapture } from '../../__tests__/legacy-snapshot-capture-fixture.ts'; +import { legacyDispatchCapture } from '../../../__tests__/legacy-snapshot-capture-fixture.ts'; -vi.mock('../snapshot-interactor-capture.ts', async () => { - const fixture = await import('../../__tests__/legacy-snapshot-capture-fixture.ts'); +vi.mock('../../../handlers/snapshot-interactor-capture.ts', async () => { + const fixture = await import('../../../__tests__/legacy-snapshot-capture-fixture.ts'); return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; }); diff --git a/src/daemon/handlers/__tests__/find-touch-runtime-fixture.ts b/src/daemon/interaction/internal/__tests__/find-touch-runtime-fixture.ts similarity index 83% rename from src/daemon/handlers/__tests__/find-touch-runtime-fixture.ts rename to src/daemon/interaction/internal/__tests__/find-touch-runtime-fixture.ts index c54fec8306..5508bf8f57 100644 --- a/src/daemon/handlers/__tests__/find-touch-runtime-fixture.ts +++ b/src/daemon/interaction/internal/__tests__/find-touch-runtime-fixture.ts @@ -1,12 +1,12 @@ -import { legacyDispatchCapture } from '../../__tests__/legacy-snapshot-capture-fixture.ts'; -import { IOS_SIMULATOR } from '../../../__tests__/test-utils/device-fixtures.ts'; +import { legacyDispatchCapture } from '../../../__tests__/legacy-snapshot-capture-fixture.ts'; +import { IOS_SIMULATOR } from '../../../../__tests__/test-utils/device-fixtures.ts'; import { getRuntimeBindings, mockFillPoint, mockFocusPoint, mockTapPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; export { mockFocusPoint }; export const findTouchRuntimeBindings = getRuntimeBindings; diff --git a/src/daemon/handlers/__tests__/find.test.ts b/src/daemon/interaction/internal/__tests__/find.test.ts similarity index 98% rename from src/daemon/handlers/__tests__/find.test.ts rename to src/daemon/interaction/internal/__tests__/find.test.ts index f87ce3da63..88b044af7c 100644 --- a/src/daemon/handlers/__tests__/find.test.ts +++ b/src/daemon/interaction/internal/__tests__/find.test.ts @@ -1,25 +1,24 @@ import { test, expect, vi, beforeEach } from 'vitest'; -import { handleFindCommands } from '../find.ts'; -import { handleInteractionCommands } from '../interaction.ts'; +import { handleFindCommands, handleInteractionCommands } from '../../index.ts'; import type { CommandFlags } from '@agent-device/contracts/command'; -import type { DaemonRequest, DaemonResponse, SessionState } from '../../types.ts'; -import { buildSnapshotSignatures } from '../../../snapshot/snapshot-freshness/index.ts'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; +import type { DaemonRequest, DaemonResponse, SessionState } from '../../../types.ts'; +import { buildSnapshotSignatures } from '../../../../snapshot/snapshot-freshness/index.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; import { makeIosSession as makeSession, makeAuthoringSession, -} from '../../../__tests__/test-utils/session-factories.ts'; +} from '../../../../__tests__/test-utils/session-factories.ts'; -vi.mock('../../../core/dispatch-resolve.ts', async (importOriginal) => { - const actual = await importOriginal(); +vi.mock('../../../../core/dispatch-resolve.ts', async (importOriginal) => { + const actual = await importOriginal(); return { ...actual, resolveTargetDevice: actual.resolveTargetDevice, }; }); -vi.mock('../snapshot-interactor-capture.ts', async () => { - const fixture = await import('../../__tests__/legacy-snapshot-capture-fixture.ts'); +vi.mock('../../../handlers/snapshot-interactor-capture.ts', async () => { + const fixture = await import('../../../__tests__/legacy-snapshot-capture-fixture.ts'); return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; }); diff --git a/src/daemon/interaction/internal/__tests__/interaction-common.test.ts b/src/daemon/interaction/internal/__tests__/interaction-common.test.ts index 5ebb89eb67..446a27be37 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-common.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-common.test.ts @@ -8,14 +8,13 @@ import { } from '../../../../__tests__/test-utils/session-factories.ts'; import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { handleInteractionCommands } from '../../../handlers/interaction.ts'; -import { finalizeTouchInteraction } from '../../index.ts'; +import { finalizeTouchInteraction, handleInteractionCommands } from '../../index.ts'; import { IOS_SIMULATOR } from '../../../../__tests__/test-utils/device-fixtures.ts'; import { getRuntimeBindings, mockFillPoint, resetGetRuntimeFixture, -} from '../../../handlers/__tests__/interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; const contextFromFlags = (_flags: CommandFlags | undefined) => ({}); diff --git a/src/daemon/interaction/internal/__tests__/interaction-flags.test.ts b/src/daemon/interaction/internal/__tests__/interaction-flags.test.ts deleted file mode 100644 index c88395aa77..0000000000 --- a/src/daemon/interaction/internal/__tests__/interaction-flags.test.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { expect, test } from 'vitest'; -import { unsupportedRefSnapshotFlags } from '../interaction-flags.ts'; - -test('unsupportedRefSnapshotFlags returns unsupported snapshot flags for @ref flows', () => { - const unsupported = unsupportedRefSnapshotFlags({ - snapshotDepth: 2, - snapshotScope: 'Login', - snapshotRaw: true, - }); - expect(unsupported).toEqual(['--depth', '--scope', '--raw']); -}); diff --git a/src/daemon/handlers/__tests__/interaction-gesture-drag.test.ts b/src/daemon/interaction/internal/__tests__/interaction-gesture-drag.test.ts similarity index 86% rename from src/daemon/handlers/__tests__/interaction-gesture-drag.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-gesture-drag.test.ts index 6a70523849..9266d6debd 100644 --- a/src/daemon/handlers/__tests__/interaction-gesture-drag.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-gesture-drag.test.ts @@ -1,24 +1,14 @@ -import { beforeEach, expect, test, vi } from 'vitest'; +import { beforeEach, expect, test } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; import { makeIosSession, authoringPublication, -} from '../../../__tests__/test-utils/session-factories.ts'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { activateCompleteRefFrame, refFrameState } from '../../ref-frame.ts'; +} from '../../../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { activateCompleteRefFrame, refFrameState } from '../../../ref-frame.ts'; -vi.mock('../../interaction/index.ts', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - captureSnapshotForSession: vi.fn( - async (session: import('../../types.ts').SessionState) => session.snapshot!, - ), - }; -}); - -import { handleInteractionCommands } from '../interaction.ts'; -import { gestureRuntimeBindingsFixture } from './gesture-runtime-bindings.fixtures.ts'; +import { handleInteractionCommands } from '../../index.ts'; +import { gestureRuntimeBindingsFixture } from '../../../handlers/__tests__/gesture-runtime-bindings.fixtures.ts'; const contextFromFlags = () => ({}); let gestures = gestureRuntimeBindingsFixture(); @@ -85,6 +75,7 @@ async function runDrag(sessionStore: ReturnType, sessio }, sessionName, sessionStore, + captureSnapshotForSession: async (session) => session.snapshot!, contextFromFlags, inspectFacts: gestures.inspectFacts, bindDevice: gestures.bindDevice, diff --git a/src/daemon/handlers/__tests__/interaction-gesture-response.test.ts b/src/daemon/interaction/internal/__tests__/interaction-gesture-response.test.ts similarity index 100% rename from src/daemon/handlers/__tests__/interaction-gesture-response.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-gesture-response.test.ts diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/interaction/internal/__tests__/interaction-get.test.ts similarity index 58% rename from src/daemon/handlers/__tests__/interaction.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-get.test.ts index c4bbb4ecce..9860e24de2 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-get.test.ts @@ -1,21 +1,15 @@ import { test, expect, vi, beforeEach } from 'vitest'; -import { legacyDispatchCapture } from '../../__tests__/legacy-snapshot-capture-fixture.ts'; +import { legacyDispatchCapture } from '../../../__tests__/legacy-snapshot-capture-fixture.ts'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { WEB_DESKTOP_DEVICE } from '../../../__tests__/test-utils/device-fixtures.ts'; -import { - makeAndroidSession as makeBaseAndroidSession, - makeIosSession, -} from '../../../__tests__/test-utils/session-factories.ts'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { expireRefFrame } from '../../ref-frame.ts'; -import { setSessionSnapshot, STALE_SNAPSHOT_REFS_WARNING } from '../../session-snapshot.ts'; -import { handleInteractionCommands } from '../interaction.ts'; -import { buildSnapshotState } from '../../../core/snapshot-state.ts'; +import { makeIosSession } from '../../../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { expireRefFrame } from '../../../ref-frame.ts'; +import { setSessionSnapshot, STALE_SNAPSHOT_REFS_WARNING } from '../../../session-snapshot.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { contextFromFlags, makeSession, makeStaleRefSession, - makeVisibleButtonSnapshot, runInteraction, } from './interaction-touch-fixtures.ts'; @@ -26,8 +20,8 @@ const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ mockRunAppleRunnerCommand: vi.fn(), })); -vi.mock('../snapshot-interactor-capture.ts', async () => { - const fixture = await import('../../__tests__/legacy-snapshot-capture-fixture.ts'); +vi.mock('../../../handlers/snapshot-interactor-capture.ts', async () => { + const fixture = await import('../../../__tests__/legacy-snapshot-capture-fixture.ts'); return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; }); @@ -55,7 +49,7 @@ import { getRuntimeBindings, mockReadTextAtPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { getAndroidAppState, getAndroidBlockingDialogObservation, @@ -385,320 +379,9 @@ test('get text iOS label selector uses snapshot disambiguation instead of runner } }); -test('is visible preserves CLI snapshot flags during runtime snapshot capture', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'snapshot-flags'; - sessionStore.set(sessionName, makeSession(sessionName)); - - legacyDispatchCapture.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { - nodes: [ - { - index: 0, - depth: 0, - type: 'XCUIElementTypeWindow', - label: 'Login', - rect: { x: 0, y: 0, width: 390, height: 844 }, - }, - { - index: 1, - depth: 1, - parentIndex: 0, - type: 'XCUIElementTypeButton', - label: 'Continue', - identifier: 'auth_continue', - rect: { x: 10, y: 20, width: 100, height: 40 }, - enabled: true, - hittable: true, - visible: true, - }, - ], - backend: 'xctest', - }; - }); - - const response = await handleInteractionCommands({ - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'id=auth_continue'], - flags: { snapshotDepth: 2, snapshotScope: 'Login', snapshotRaw: true }, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(legacyDispatchCapture.mock.calls[0]?.[4]).toMatchObject({ - snapshotDepth: 2, - snapshotScope: 'Login', - snapshotRaw: true, - snapshotInteractiveOnly: false, - snapshotIncludeRects: true, - }); -}); - -test('is visible reuses fresh cached iOS snapshots with rects', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'ios-visible-cached'; - const session = makeSession(sessionName); - session.snapshot = makeVisibleButtonSnapshot('Cached action', 'xctest'); - sessionStore.set(sessionName, session); - legacyDispatchCapture.mockRejectedValue(new Error('unexpected fresh snapshot')); - - const response = await handleInteractionCommands({ - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Cached action"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(legacyDispatchCapture).not.toHaveBeenCalled(); -}); - -test('is visible recaptures web snapshots when cached nodes may lack rects', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'web-visible-refreshes-rects'; - const session = makeSession(sessionName); - session.device = WEB_DESKTOP_DEVICE; - session.snapshot = buildSnapshotState( - { - nodes: [{ index: 0, type: 'button', label: 'Submit order' }], - backend: 'web', - }, - { snapshotInteractiveOnly: false }, - ); - sessionStore.set(sessionName, session); - legacyDispatchCapture.mockResolvedValue(makeVisibleButtonSnapshot('Submit order', 'web')); - - const response = await handleInteractionCommands({ - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Submit order"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response?.ok).toBe(true); - expect(legacyDispatchCapture.mock.calls[0]?.[4]).toMatchObject({ - snapshotIncludeRects: true, - }); -}); - -// PIN CHANGED TWICE (#1739, R37). #557 asserted `ok: true` with `pass: false` and zero snapshots -// here, from the direct-iOS shortcut. That broke `is`'s documented contract — it "exits non-zero -// on failure" (website/docs/docs/commands.md) — and on device printed `Passed: is text` with exit -// 0 for a failed assertion. The reversal made the shortcut answer only when the predicate held; -// the shortcut is now retired outright, so the bound capture answers every predicate and this is -// simply what `is` does. The assertion below is unchanged across both edits because it was always -// about the OUTCOME, not about which path produced it. -test('a failing is predicate is COMMAND_FAILED, never a zero-exit pass', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'is-selected-ios-direct-selector-false'; - sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); - mockRunAppleRunnerCommand.mockResolvedValue({ - found: true, - nodes: [ - { - index: 0, - depth: 0, - type: 'Button', - label: 'Submit', - identifier: 'submit', - selected: false, - rect: { x: 126, y: 555, width: 75, height: 38 }, - }, - ], - }); - - const response = await handleInteractionCommands({ - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['selected', 'id="submit"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - // The session snapshot has no `id=submit`, so the bound capture reports the typed selector - // failure. Nothing can report a failed assertion as a completed command. - expect(response?.ok).toBe(false); - expect(legacyDispatchCapture.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(1); - if (response?.ok === false) { - expect(response.error?.code).toBe('COMMAND_FAILED'); - } -}); - -test('is visible passes for list text that inherits viewport visibility from an ancestor', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'visible-list-item'; - sessionStore.set(sessionName, makeSession(sessionName)); - - legacyDispatchCapture.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { - nodes: [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeCell', - rect: { x: 0, y: 160, width: 390, height: 44 }, - hittable: false, - }, - { - index: 2, - parentIndex: 1, - type: 'XCUIElementTypeStaticText', - label: 'Trip ideas', - hittable: false, - }, - ], - backend: 'xctest', - }; - }); - - const response = await handleInteractionCommands({ - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Trip ideas"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(true); - if (response?.ok) { - expect(response.data?.predicate).toBe('visible'); - expect(response.data?.pass).toBe(true); - expect(response.data?.selector).toBe('label="Trip ideas"'); - } -}); - -test('is visible fails for nodes outside the current viewport', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'visible-offscreen'; - sessionStore.set(sessionName, makeSession(sessionName)); - - legacyDispatchCapture.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { - nodes: [ - { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, - { - index: 1, - parentIndex: 0, - type: 'XCUIElementTypeStaticText', - label: 'Far item', - rect: { x: 20, y: 2600, width: 120, height: 40 }, - hittable: false, - }, - ], - backend: 'xctest', - }; - }); - - const response = await handleInteractionCommands({ - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Far item"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.code).toBe('COMMAND_FAILED'); - expect(response.error.message).toMatch(/actual=\{"visible":false/); - } -}); - -test('is reports Android permission dialog blocker when app content assertion fails', async () => { - const sessionStore = makeSessionStore(); - const sessionName = 'android-permission-blocked'; - sessionStore.set( - sessionName, - makeBaseAndroidSession(sessionName, { appBundleId: 'com.example.demo' }), - ); - - legacyDispatchCapture.mockImplementation(async (_device, command) => { - if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); - return { nodes: [], backend: 'uiautomator' }; - }); - mockGetAndroidAppState.mockResolvedValue({ - package: 'com.google.android.permissioncontroller', - activity: 'com.android.permissioncontroller.permission.ui.GrantPermissionsActivity', - }); - - const response = await handleInteractionCommands({ - req: { - token: 't', - session: sessionName, - command: 'is', - positionals: ['visible', 'label="Metro Ready"'], - flags: {}, - }, - sessionName, - sessionStore, - contextFromFlags, - ...getRuntimeBindings(), - }); - - expect(response).toBeTruthy(); - expect(response?.ok).toBe(false); - if (response && !response.ok) { - expect(response.error.message).toMatch(/permission dialog is blocking/); - expect(response.error.details).toMatchObject({ - blockedBy: 'android_foreground_surface', - expectedPackage: 'com.example.demo', - foregroundPackage: 'com.google.android.permissioncontroller', - }); - } -}); - -test('ADR 0014 evidence #17: get text @ref reads the retained frame tree, not a newer observation', async () => { - const sessionStore = makeSessionStore(); +function seedDivergentReadFrameSession(sessionStore: ReturnType): string { const sessionName = 'read-frame-tree'; - // Frame tree: @e1 = Continue, @e2 = Cancel. const session = makeStaleRefSession(sessionName); - // A read-only capture replaced the OBSERVATION with a divergent tree where the - // same index means a different element. The frame tree is untouched. setSessionSnapshot(session, { nodes: attachRefs([ { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, @@ -717,9 +400,13 @@ test('ADR 0014 evidence #17: get text @ref reads the retained frame tree, not a }); sessionStore.set(sessionName, session); legacyDispatchCapture.mockRejectedValue(new Error('get text @ref must not recapture')); + return sessionName; +} + +test('ADR 0014 evidence #17: get text @ref reads the retained frame tree, not a newer observation', async () => { + const sessionStore = makeSessionStore(); + const sessionName = seedDivergentReadFrameSession(sessionStore); - // Resolves against the frame tree's @e2 (Continue), never the observation's - // positional @e2 (Different) — no fall-through by positional coincidence. const response = await runInteraction(sessionStore, sessionName, 'get', ['text', '@e2']); expect(response?.ok).toBe(true); if (response?.ok) { @@ -728,7 +415,13 @@ test('ADR 0014 evidence #17: get text @ref reads the retained frame tree, not a expect(String(response.data?.text)).not.toContain('Different'); } - // Missing frame evidence FAILS rather than resolving a newer observation. + expect(legacyDispatchCapture).not.toHaveBeenCalled(); +}); + +test('ADR 0014 evidence #17: missing frame evidence fails rather than resolving a newer observation', async () => { + const sessionStore = makeSessionStore(); + const sessionName = seedDivergentReadFrameSession(sessionStore); + const missing = await runInteraction(sessionStore, sessionName, 'get', ['text', '@e9']); expect(missing?.ok).toBe(false); if (missing && !missing.ok) { diff --git a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome-fixtures.ts similarity index 95% rename from src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts rename to src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome-fixtures.ts index ec9d6c739a..1120eef60c 100644 --- a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome-fixtures.ts @@ -1,5 +1,5 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { buildSnapshotState } from '../../../core/snapshot-state.ts'; +import { buildSnapshotState } from '../../../../core/snapshot-state.ts'; export const profileNodes: RawSnapshotNode[] = [ { diff --git a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts similarity index 95% rename from src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts index afcc18fd7b..06fc37975a 100644 --- a/src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-ios-tap-outcome.test.ts @@ -1,38 +1,38 @@ import type { CommandFlags } from '@agent-device/contracts/command'; -import { legacyDispatchCapture } from '../../__tests__/legacy-snapshot-capture-fixture.ts'; +import { legacyDispatchCapture } from '../../../__tests__/legacy-snapshot-capture-fixture.ts'; import { beforeEach, expect, test, vi } from 'vitest'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; import { buildSnapshotPresentationKey } from '@agent-device/kernel/snapshot'; -import { handleInteractionCommands } from '../interaction.ts'; -import { handleSnapshotCommands } from '../snapshot.ts'; +import { handleInteractionCommands } from '../../index.ts'; +import { handleSnapshotCommands } from '../../../handlers/snapshot.ts'; import { makeIosSession, authoringPublication, -} from '../../../__tests__/test-utils/session-factories.ts'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { SessionScriptWriter } from '../../session-script-writer.ts'; -import { runReplayForTest } from '../../replay/__tests__/replay-command-fixture.ts'; -import { replayScriptSourceBundleFor } from '../../../__tests__/test-utils/replay-script-source.ts'; +} from '../../../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { SessionScriptWriter } from '../../../session-script-writer.ts'; +import { runReplayForTest } from '../../../replay/__tests__/replay-command-fixture.ts'; +import { replayScriptSourceBundleFor } from '../../../../__tests__/test-utils/replay-script-source.ts'; import { imageViewerNodes, profileNodes, snapshot, snapshotPayload, } from './interaction-ios-tap-outcome-fixtures.ts'; -import { snapshotRuntimeFixture } from '../../__tests__/snapshot-runtime-fixture.ts'; -import { IOS_SIMULATOR } from '../../../__tests__/test-utils/device-fixtures.ts'; +import { snapshotRuntimeFixture } from '../../../__tests__/snapshot-runtime-fixture.ts'; +import { IOS_SIMULATOR } from '../../../../__tests__/test-utils/device-fixtures.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; -import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; +import { captureSnapshotWithInteractor } from '../../../handlers/snapshot-interactor-capture.ts'; -vi.mock('../snapshot-interactor-capture.ts', async () => { - const fixture = await import('../../__tests__/legacy-snapshot-capture-fixture.ts'); +vi.mock('../../../handlers/snapshot-interactor-capture.ts', async () => { + const fixture = await import('../../../__tests__/legacy-snapshot-capture-fixture.ts'); return { captureSnapshotWithInteractor: vi.fn(fixture.captureSnapshotThroughLegacyDispatchFixture), }; diff --git a/src/daemon/interaction/internal/__tests__/interaction-is.test.ts b/src/daemon/interaction/internal/__tests__/interaction-is.test.ts new file mode 100644 index 0000000000..fb2d3c3de5 --- /dev/null +++ b/src/daemon/interaction/internal/__tests__/interaction-is.test.ts @@ -0,0 +1,374 @@ +import { test, expect, vi, beforeEach } from 'vitest'; +import { legacyDispatchCapture } from '../../../__tests__/legacy-snapshot-capture-fixture.ts'; +import { WEB_DESKTOP_DEVICE } from '../../../../__tests__/test-utils/device-fixtures.ts'; +import { + makeAndroidSession as makeBaseAndroidSession, + makeIosSession, +} from '../../../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { handleInteractionCommands } from '../../index.ts'; +import { buildSnapshotState } from '../../../../core/snapshot-state.ts'; +import { + contextFromFlags, + makeSession, + makeVisibleButtonSnapshot, +} from './interaction-touch-fixtures.ts'; + +// Non-touch interaction routing: the `get` and `is` reads the public handler +// owns. Touch commands live in the interaction-touch* test files. + +const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ + mockRunAppleRunnerCommand: vi.fn(), +})); + +vi.mock('../../../handlers/snapshot-interactor-capture.ts', async () => { + const fixture = await import('../../../__tests__/legacy-snapshot-capture-fixture.ts'); + return { captureSnapshotWithInteractor: fixture.captureSnapshotThroughLegacyDispatchFixture }; +}); + +vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getAndroidAppState: vi.fn(async () => ({})), + getAndroidBlockingDialogObservation: vi.fn(async () => ({ status: 'clear' }) as const), + }; +}); + +vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + runAppleRunnerCommand: mockRunAppleRunnerCommand, + }; +}); + +import { + getRuntimeBindings, + resetGetRuntimeFixture, +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; +import { + getAndroidAppState, + getAndroidBlockingDialogObservation, +} from '@agent-device/platform-android/mechanics'; +const mockGetAndroidAppState = vi.mocked(getAndroidAppState); +const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); +beforeEach(() => { + legacyDispatchCapture.mockReset(); + legacyDispatchCapture.mockResolvedValue({}); + mockGetAndroidAppState.mockReset(); + mockGetAndroidAppState.mockResolvedValue({}); + mockGetAndroidBlockingDialogObservation.mockReset(); + mockGetAndroidBlockingDialogObservation.mockResolvedValue({ status: 'clear' }); + mockRunAppleRunnerCommand.mockReset(); + mockRunAppleRunnerCommand.mockResolvedValue({}); + resetGetRuntimeFixture(); +}); + +test('is visible preserves CLI snapshot flags during runtime snapshot capture', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'snapshot-flags'; + sessionStore.set(sessionName, makeSession(sessionName)); + + legacyDispatchCapture.mockImplementation(async (_device, command) => { + if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); + return { + nodes: [ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeWindow', + label: 'Login', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + identifier: 'auth_continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + visible: true, + }, + ], + backend: 'xctest', + }; + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'id=auth_continue'], + flags: { snapshotDepth: 2, snapshotScope: 'Login', snapshotRaw: true }, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(true); + expect(legacyDispatchCapture.mock.calls[0]?.[4]).toMatchObject({ + snapshotDepth: 2, + snapshotScope: 'Login', + snapshotRaw: true, + snapshotInteractiveOnly: false, + snapshotIncludeRects: true, + }); +}); + +test('is visible reuses fresh cached iOS snapshots with rects', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-visible-cached'; + const session = makeSession(sessionName); + session.snapshot = makeVisibleButtonSnapshot('Cached action', 'xctest'); + sessionStore.set(sessionName, session); + legacyDispatchCapture.mockRejectedValue(new Error('unexpected fresh snapshot')); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'label="Cached action"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(true); + expect(legacyDispatchCapture).not.toHaveBeenCalled(); +}); + +test('is visible recaptures web snapshots when cached nodes may lack rects', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'web-visible-refreshes-rects'; + const session = makeSession(sessionName); + session.device = WEB_DESKTOP_DEVICE; + session.snapshot = buildSnapshotState( + { + nodes: [{ index: 0, type: 'button', label: 'Submit order' }], + backend: 'web', + }, + { snapshotInteractiveOnly: false }, + ); + sessionStore.set(sessionName, session); + legacyDispatchCapture.mockResolvedValue(makeVisibleButtonSnapshot('Submit order', 'web')); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'label="Submit order"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(true); + expect(legacyDispatchCapture.mock.calls[0]?.[4]).toMatchObject({ + snapshotIncludeRects: true, + }); +}); + +// PIN CHANGED TWICE (#1739, R37). #557 asserted `ok: true` with `pass: false` and zero snapshots +// here, from the direct-iOS shortcut. That broke `is`'s documented contract — it "exits non-zero +// on failure" (website/docs/docs/commands.md) — and on device printed `Passed: is text` with exit +// 0 for a failed assertion. The reversal made the shortcut answer only when the predicate held; +// the shortcut is now retired outright, so the bound capture answers every predicate and this is +// simply what `is` does. The assertion below is unchanged across both edits because it was always +// about the OUTCOME, not about which path produced it. +test('a failing is predicate is COMMAND_FAILED, never a zero-exit pass', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'is-selected-ios-direct-selector-false'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + mockRunAppleRunnerCommand.mockResolvedValue({ + found: true, + nodes: [ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Submit', + identifier: 'submit', + selected: false, + rect: { x: 126, y: 555, width: 75, height: 38 }, + }, + ], + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['selected', 'id="submit"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + // The session snapshot has no `id=submit`, so the bound capture reports the typed selector + // failure. Nothing can report a failed assertion as a completed command. + expect(response?.ok).toBe(false); + expect(legacyDispatchCapture.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(1); + if (response?.ok === false) { + expect(response.error?.code).toBe('COMMAND_FAILED'); + } +}); + +test('is visible passes for list text that inherits viewport visibility from an ancestor', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'visible-list-item'; + sessionStore.set(sessionName, makeSession(sessionName)); + + legacyDispatchCapture.mockImplementation(async (_device, command) => { + if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); + return { + nodes: [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeCell', + rect: { x: 0, y: 160, width: 390, height: 44 }, + hittable: false, + }, + { + index: 2, + parentIndex: 1, + type: 'XCUIElementTypeStaticText', + label: 'Trip ideas', + hittable: false, + }, + ], + backend: 'xctest', + }; + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'label="Trip ideas"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response).toBeTruthy(); + expect(response?.ok).toBe(true); + if (response?.ok) { + expect(response.data?.predicate).toBe('visible'); + expect(response.data?.pass).toBe(true); + expect(response.data?.selector).toBe('label="Trip ideas"'); + } +}); + +test('is visible fails for nodes outside the current viewport', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'visible-offscreen'; + sessionStore.set(sessionName, makeSession(sessionName)); + + legacyDispatchCapture.mockImplementation(async (_device, command) => { + if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); + return { + nodes: [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'XCUIElementTypeStaticText', + label: 'Far item', + rect: { x: 20, y: 2600, width: 120, height: 40 }, + hittable: false, + }, + ], + backend: 'xctest', + }; + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'label="Far item"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response).toBeTruthy(); + expect(response?.ok).toBe(false); + if (response && !response.ok) { + expect(response.error.code).toBe('COMMAND_FAILED'); + expect(response.error.message).toMatch(/actual=\{"visible":false/); + } +}); + +test('is reports Android permission dialog blocker when app content assertion fails', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'android-permission-blocked'; + sessionStore.set( + sessionName, + makeBaseAndroidSession(sessionName, { appBundleId: 'com.example.demo' }), + ); + + legacyDispatchCapture.mockImplementation(async (_device, command) => { + if (command !== 'snapshot') throw new Error(`unexpected command: ${command}`); + return { nodes: [], backend: 'uiautomator' }; + }); + mockGetAndroidAppState.mockResolvedValue({ + package: 'com.google.android.permissioncontroller', + activity: 'com.android.permissioncontroller.permission.ui.GrantPermissionsActivity', + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'label="Metro Ready"'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response).toBeTruthy(); + expect(response?.ok).toBe(false); + if (response && !response.ok) { + expect(response.error.message).toMatch(/permission dialog is blocking/); + expect(response.error.details).toMatchObject({ + blockedBy: 'android_foreground_surface', + expectedPackage: 'com.example.demo', + foregroundPackage: 'com.google.android.permissioncontroller', + }); + } +}); diff --git a/src/daemon/interaction/internal/__tests__/interaction-ref-policy.test.ts b/src/daemon/interaction/internal/__tests__/interaction-ref-policy.test.ts index 371d39c31d..c65902c850 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-ref-policy.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-ref-policy.test.ts @@ -1,7 +1,7 @@ import { expect, test } from 'vitest'; import { activatePartialRefFrame, readRefMutationFrame } from '../../../ref-frame.ts'; import { refMutationAdmissionResponse } from '../interaction-ref-policy.ts'; -import { makeStaleRefSession } from '../../../handlers/__tests__/interaction-touch-fixtures.ts'; +import { makeStaleRefSession } from './interaction-touch-fixtures.ts'; test('a plain ref emitted by the current partial frame suggests its exact pinned form', () => { const session = makeStaleRefSession('partial-frame-suggestion'); diff --git a/src/daemon/interaction/internal/__tests__/interaction-response-construction-guard.test.ts b/src/daemon/interaction/internal/__tests__/interaction-response-construction-guard.test.ts index 25e68602df..8cc3edb9a9 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-response-construction-guard.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-response-construction-guard.test.ts @@ -12,7 +12,7 @@ import { test } from 'vitest'; // is assigned anything other than the shared builder's output, so a new // branch cannot regress without tripping CI. -const HANDLERS_DIR = path.resolve(import.meta.dirname, '../../../handlers'); +const HANDLERS_DIR = path.resolve(import.meta.dirname, '..'); const BUILDER_FILE = 'interaction-touch-response.ts'; function touchHandlerSourceFiles(): string[] { diff --git a/src/daemon/handlers/__tests__/interaction-settle-private-ax-route.test.ts b/src/daemon/interaction/internal/__tests__/interaction-settle-private-ax-route.test.ts similarity index 85% rename from src/daemon/handlers/__tests__/interaction-settle-private-ax-route.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-settle-private-ax-route.test.ts index ee81f26a84..18396826f6 100644 --- a/src/daemon/handlers/__tests__/interaction-settle-private-ax-route.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-settle-private-ax-route.test.ts @@ -1,11 +1,11 @@ import { beforeEach, expect, test, vi } from 'vitest'; -import { makeSnapshotState } from '../../../__tests__/test-utils/snapshot-builders.ts'; -import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; +import { makeSnapshotState } from '../../../../__tests__/test-utils/snapshot-builders.ts'; +import { makeIosSession } from '../../../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; import { withAppleRunnerProvider } from '@agent-device/platform-apple/runner'; -import { contextFromFlags as buildDaemonContext } from '../../context.ts'; -import { handleInteractionCommands } from '../interaction.ts'; -import { getRuntimeBindings } from './interaction-get-runtime-fixture.ts'; +import { contextFromFlags as buildDaemonContext } from '../../../context.ts'; +import { handleInteractionCommands } from '../../index.ts'; +import { getRuntimeBindings } from '../../../__tests__/interaction-get-runtime-fixture.ts'; vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) => { const actual = diff --git a/src/daemon/interaction/internal/__tests__/interaction-settle.test.ts b/src/daemon/interaction/internal/__tests__/interaction-settle.test.ts index 82e36ef8c1..8fad3fa360 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-settle.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-settle.test.ts @@ -1,7 +1,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import { legacyDispatchCapture } from '../../../__tests__/legacy-snapshot-capture-fixture.ts'; import { test, expect, vi, beforeEach } from 'vitest'; -import { handleInteractionCommands } from '../../../handlers/interaction.ts'; +import { createInteractionRuntime, handleInteractionCommands } from '../../index.ts'; import type { SessionStore } from '../../../session-store.ts'; import type { SessionState } from '../../../types.ts'; import type { SnapshotBackend } from '@agent-device/kernel/snapshot'; @@ -10,7 +10,6 @@ import { setSessionSnapshot } from '../../../session-snapshot.ts'; import { activateCompleteRefFrame } from '../../../ref-frame.ts'; import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; import { makeIosSession } from '../../../../__tests__/test-utils/session-factories.ts'; -import { createInteractionRuntime } from '../../index.ts'; import { clearRequestAbortRegistration, registerRequestAbort, @@ -21,7 +20,7 @@ import { mockFillPoint, mockTapPoint, resetGetRuntimeFixture, -} from '../../../handlers/__tests__/interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; // #1101 --settle daemon response shape: the settle payload (diff + settled + // refsGeneration) rides the wire response through the shared builder, and a @@ -31,14 +30,6 @@ import { const mockCaptureSnapshotForSession = vi.hoisted(() => vi.fn()); -vi.mock('../../index.ts', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - captureSnapshotForSession: mockCaptureSnapshotForSession, - }; -}); - const BEFORE_NODES = [ { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, { @@ -167,6 +158,13 @@ beforeEach(() => { mockCaptureSnapshotForSession.mockImplementation(emulateCaptureSnapshotForSession); }); +function invokeInteraction(params: Parameters[0]) { + return handleInteractionCommands({ + ...params, + captureSnapshotForSession: mockCaptureSnapshotForSession, + }); +} + const SETTLE_FLAGS = { settle: true, settleQuietMs: 25, timeoutMs: 2_000 }; type SettlePayload = { @@ -209,7 +207,7 @@ test('press --settle responds with the settled diff, refsGeneration, and activat // settled post-action tree. mockCommandDispatch({ snapshots: [BEFORE_NODES, AFTER_NODES, AFTER_NODES, AFTER_NODES] }); - const response = await handleInteractionCommands({ + const response = await invokeInteraction({ req: { token: 't', session: sessionName, @@ -285,7 +283,7 @@ test('press --settle on a removals-only diff attaches the unchanged interactive snapshots: [MODAL_BEFORE_NODES, MODAL_AFTER_NODES, MODAL_AFTER_NODES], }); - const response = await handleInteractionCommands({ + const response = await invokeInteraction({ req: { token: 't', session: sessionName, @@ -321,7 +319,7 @@ test('press --settle rejects an expired-frame ref before dispatch or observation new Error('dispatch should not be called for an expired-frame ref'), ); - const response = await handleInteractionCommands({ + const response = await invokeInteraction({ req: { token: 't', session: sessionName, @@ -360,7 +358,7 @@ test('a settle observation without a diff leaves ref staleness untouched', async return {}; }); - const response = await handleInteractionCommands({ + const response = await invokeInteraction({ req: { token: 't', session: sessionName, @@ -421,7 +419,7 @@ test('a stalled settle capture receives its deadline signal and leaves the inter ); const startedAt = Date.now(); - const response = await handleInteractionCommands({ + const response = await invokeInteraction({ req: { token: 't', session: sessionName, @@ -450,7 +448,7 @@ test('bare timeout without --settle stays compatible', async () => { seedSession(sessionName, sessionStore); mockCommandDispatch({ snapshots: [BEFORE_NODES] }); - const compatible = await handleInteractionCommands({ + const compatible = await invokeInteraction({ req: { token: 't', session: sessionName, @@ -474,7 +472,7 @@ test('settle-specific tuning flags without --settle are rejected', async () => { seedSession(sessionName, sessionStore); mockCommandDispatch({ snapshots: [BEFORE_NODES] }); - const response = await handleInteractionCommands({ + const response = await invokeInteraction({ req: { token: 't', session: sessionName, @@ -498,7 +496,7 @@ test('fill @ref --settle carries the settle payload on the ref wire shape', asyn seedSession(sessionName, sessionStore); mockCommandDispatch({ snapshots: [AFTER_NODES, AFTER_NODES] }); - const response = await handleInteractionCommands({ + const response = await invokeInteraction({ req: { token: 't', session: sessionName, diff --git a/src/daemon/interaction/internal/__tests__/interaction-target-evidence.test.ts b/src/daemon/interaction/internal/__tests__/interaction-target-evidence.test.ts index 3270972fa9..c30693e08f 100644 --- a/src/daemon/interaction/internal/__tests__/interaction-target-evidence.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-target-evidence.test.ts @@ -3,7 +3,7 @@ import { legacyDispatchCapture } from '../../../__tests__/legacy-snapshot-captur import { test, expect, vi, beforeEach } from 'vitest'; import fs from 'node:fs'; import path from 'node:path'; -import { handleInteractionCommands } from '../../../handlers/interaction.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; import { @@ -43,7 +43,7 @@ vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) import { getRuntimeBindings, resetGetRuntimeFixture, -} from '../../../handlers/__tests__/interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; const contextFromFlags = (_flags: CommandFlags | undefined) => ({}); beforeEach(() => { diff --git a/src/daemon/handlers/__tests__/interaction-touch-android-freshness.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-android-freshness.test.ts similarity index 96% rename from src/daemon/handlers/__tests__/interaction-touch-android-freshness.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch-android-freshness.test.ts index 252ce2a669..84c55857f8 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-android-freshness.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-android-freshness.test.ts @@ -1,12 +1,12 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { handleInteractionCommands } from '../interaction.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, makeAndroidSession } from './interaction-touch-fixtures.ts'; // The Android ref-refresh capture a @ref mutation takes before dispatch: when @@ -27,7 +27,7 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('../snapshot-interactor-capture.ts', () => ({ +vi.mock('../../../handlers/snapshot-interactor-capture.ts', () => ({ captureSnapshotWithInteractor: vi.fn(), })); @@ -42,7 +42,7 @@ import { getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; +import { captureSnapshotWithInteractor } from '../../../handlers/snapshot-interactor-capture.ts'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); diff --git a/src/daemon/handlers/__tests__/interaction-touch-android-readiness-spawns.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-android-readiness-spawns.test.ts similarity index 85% rename from src/daemon/handlers/__tests__/interaction-touch-android-readiness-spawns.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch-android-readiness-spawns.test.ts index c80cd7c8ed..613ac7d58c 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-android-readiness-spawns.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-android-readiness-spawns.test.ts @@ -1,9 +1,9 @@ import { expect, test } from 'vitest'; -import { makeAndroidSession } from '../../../__tests__/test-utils/session-factories.ts'; -import { withFakeAdb } from '../../../__tests__/test-utils/fake-adb.ts'; -import { expireRefFrame } from '../../ref-frame.ts'; +import { makeAndroidSession } from '../../../../__tests__/test-utils/session-factories.ts'; +import { withFakeAdb } from '../../../../__tests__/test-utils/fake-adb.ts'; +import { expireRefFrame } from '../../../ref-frame.ts'; import { runWithAndroidDialogReadinessCheck } from '../interaction-touch-android-readiness.ts'; -import { androidObservation } from '../../../platform-runtime.ts'; +import { androidObservation } from '../../../../platform-runtime.ts'; // What blocking-dialog readiness costs a run of taps, end to end. Every one of these `dumpsys` // spawns is a device round trip on the critical path of an interaction, so the budget is pinned diff --git a/src/daemon/handlers/__tests__/interaction-touch-android-readiness.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-android-readiness.test.ts similarity index 93% rename from src/daemon/handlers/__tests__/interaction-touch-android-readiness.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch-android-readiness.test.ts index e843eb8258..2f88031fc0 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-android-readiness.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-android-readiness.test.ts @@ -1,13 +1,13 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { expireRefFrame } from '../../ref-frame.ts'; -import { handleInteractionCommands } from '../interaction.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { expireRefFrame } from '../../../ref-frame.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, makeAndroidSession } from './interaction-touch-fixtures.ts'; // The Android device state a dispatch has to survive: an escape to launcher or @@ -15,7 +15,7 @@ import { contextFromFlags, makeAndroidSession } from './interaction-touch-fixtur // blocking-dialog recovery aborts an admitted @ref rather than retargeting it. type EnsureAndroidBlockingSystemDialogReady = - typeof import('../../android-system-dialog.ts').ensureAndroidBlockingSystemDialogReady; + typeof import('../../../android-system-dialog.ts').ensureAndroidBlockingSystemDialogReady; const { androidDialogReadiness } = vi.hoisted(() => ({ androidDialogReadiness: { @@ -38,7 +38,7 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('../snapshot-interactor-capture.ts', () => ({ +vi.mock('../../../handlers/snapshot-interactor-capture.ts', () => ({ captureSnapshotWithInteractor: vi.fn(), })); @@ -50,8 +50,8 @@ vi.mock('@agent-device/platform-apple/runner/operations', async (importOriginal) // The blocking-dialog readiness check runs for real (its focus probe is mocked // above); the recovery regression below swaps in a recovering implementation. -vi.mock('../../android-system-dialog.ts', async (importOriginal) => { - const actual = await importOriginal(); +vi.mock('../../../android-system-dialog.ts', async (importOriginal) => { + const actual = await importOriginal(); androidDialogReadiness.actual = actual.ensureAndroidBlockingSystemDialogReady; return { ...actual, ensureAndroidBlockingSystemDialogReady: androidDialogReadiness.spy }; }); @@ -61,7 +61,7 @@ import { getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; +import { captureSnapshotWithInteractor } from '../../../handlers/snapshot-interactor-capture.ts'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); diff --git a/src/daemon/handlers/__tests__/interaction-touch-direct-ios-eligibility.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-direct-ios-eligibility.test.ts similarity index 94% rename from src/daemon/handlers/__tests__/interaction-touch-direct-ios-eligibility.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch-direct-ios-eligibility.test.ts index 17dd6d3db5..ee91de51ed 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-direct-ios-eligibility.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-direct-ios-eligibility.test.ts @@ -1,15 +1,15 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { handleInteractionCommands } from '../interaction.ts'; +import { makeIosSession } from '../../../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { getRuntimeBindings, mockFillPoint, mockTapElementSelector, mockTapPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags } from './interaction-touch-fixtures.ts'; // Ordinary selectors stay capture-backed. Only the explicit Maestro @@ -29,7 +29,7 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('../snapshot-interactor-capture.ts', () => ({ +vi.mock('../../../handlers/snapshot-interactor-capture.ts', () => ({ captureSnapshotWithInteractor: vi.fn(), })); @@ -44,7 +44,7 @@ import { getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; +import { captureSnapshotWithInteractor } from '../../../handlers/snapshot-interactor-capture.ts'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); diff --git a/src/daemon/handlers/__tests__/interaction-touch-direct-ios.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-direct-ios.test.ts similarity index 86% rename from src/daemon/handlers/__tests__/interaction-touch-direct-ios.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch-direct-ios.test.ts index 446856ff4d..5b1a51530a 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-direct-ios.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-direct-ios.test.ts @@ -1,13 +1,13 @@ import { AppError } from '@agent-device/kernel/errors'; import { beforeEach, expect, test, vi } from 'vitest'; -import { makeIosSession } from '../../../__tests__/test-utils/session-factories.ts'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { handleInteractionCommands } from '../interaction.ts'; +import { makeIosSession } from '../../../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { getRuntimeBindings, mockTapElementSelector, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, makeStaleRefSession, @@ -24,7 +24,7 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('../snapshot-interactor-capture.ts', () => ({ +vi.mock('../../../handlers/snapshot-interactor-capture.ts', () => ({ captureSnapshotWithInteractor: vi.fn(), })); diff --git a/src/daemon/handlers/__tests__/interaction-touch-fill.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-fill.test.ts similarity index 96% rename from src/daemon/handlers/__tests__/interaction-touch-fill.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch-fill.test.ts index a0318f092a..1c088ee4f9 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-fill.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-fill.test.ts @@ -1,15 +1,15 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; import { withAppleRunnerProvider } from '@agent-device/platform-apple/runner'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { activateCompleteRefFrame, expireRefFrame } from '../../ref-frame.ts'; -import { STALE_SNAPSHOT_REFS_WARNING } from '../../session-snapshot.ts'; -import { handleInteractionCommands } from '../interaction.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { activateCompleteRefFrame, expireRefFrame } from '../../../ref-frame.ts'; +import { STALE_SNAPSHOT_REFS_WARNING } from '../../../session-snapshot.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { getRuntimeBindings, mockFillPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, makeSession, @@ -38,7 +38,7 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('../snapshot-interactor-capture.ts', () => ({ +vi.mock('../../../handlers/snapshot-interactor-capture.ts', () => ({ captureSnapshotWithInteractor: vi.fn(), })); @@ -53,7 +53,7 @@ import { getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; +import { captureSnapshotWithInteractor } from '../../../handlers/snapshot-interactor-capture.ts'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); diff --git a/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-fixtures.ts similarity index 90% rename from src/daemon/handlers/__tests__/interaction-touch-fixtures.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch-fixtures.ts index 89cdb8c0aa..fd63f211a0 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-fixtures.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-fixtures.ts @@ -4,14 +4,14 @@ import { makeAndroidSession as makeBaseAndroidSession, makeIosAppSession, makeMacOsSession as makeBaseMacOsSession, -} from '../../../__tests__/test-utils/session-factories.ts'; -import { makeTestScreenRecordingResource } from '../../../__tests__/test-utils/screen-recording-live-handle.ts'; -import { activateCompleteRefFrame } from '../../ref-frame.ts'; -import type { SessionStore } from '../../session-store.ts'; -import type { SessionState } from '../../types.ts'; -import { handleInteractionCommands } from '../interaction.ts'; -import { getRuntimeBindings } from './interaction-get-runtime-fixture.ts'; -import { buildSnapshotState } from '../../../core/snapshot-state.ts'; +} from '../../../../__tests__/test-utils/session-factories.ts'; +import { makeTestScreenRecordingResource } from '../../../../__tests__/test-utils/screen-recording-live-handle.ts'; +import { activateCompleteRefFrame } from '../../../ref-frame.ts'; +import type { SessionStore } from '../../../session-store.ts'; +import type { SessionState } from '../../../types.ts'; +import { handleInteractionCommands } from '../../index.ts'; +import { getRuntimeBindings } from '../../../__tests__/interaction-get-runtime-fixture.ts'; +import { buildSnapshotState } from '../../../../core/snapshot-state.ts'; /** * Shared factories for the interaction touch handler tests. Named pure diff --git a/src/daemon/handlers/__tests__/interaction-touch-payload.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-payload.test.ts similarity index 97% rename from src/daemon/handlers/__tests__/interaction-touch-payload.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch-payload.test.ts index 148588019e..30e98f0713 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-payload.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-payload.test.ts @@ -1,12 +1,12 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { handleInteractionCommands } from '../interaction.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, installTestScreenRecording, @@ -32,7 +32,7 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('../snapshot-interactor-capture.ts', () => ({ +vi.mock('../../../handlers/snapshot-interactor-capture.ts', () => ({ captureSnapshotWithInteractor: vi.fn(), })); @@ -47,7 +47,7 @@ import { getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; +import { captureSnapshotWithInteractor } from '../../../handlers/snapshot-interactor-capture.ts'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); diff --git a/src/daemon/handlers/__tests__/interaction-touch-press-admission.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-press-admission.test.ts similarity index 97% rename from src/daemon/handlers/__tests__/interaction-touch-press-admission.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch-press-admission.test.ts index 3f4a8a0eb3..9efd6e3659 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-press-admission.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-press-admission.test.ts @@ -1,14 +1,14 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { activateCompleteRefFrame } from '../../ref-frame.ts'; -import { setSessionSnapshot, STALE_SNAPSHOT_REFS_WARNING } from '../../session-snapshot.ts'; -import { handleInteractionCommands } from '../interaction.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { activateCompleteRefFrame } from '../../../ref-frame.ts'; +import { setSessionSnapshot, STALE_SNAPSHOT_REFS_WARNING } from '../../../session-snapshot.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, findResolvedTarget, @@ -39,7 +39,7 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('../snapshot-interactor-capture.ts', () => ({ +vi.mock('../../../handlers/snapshot-interactor-capture.ts', () => ({ captureSnapshotWithInteractor: vi.fn(), })); @@ -54,7 +54,7 @@ import { getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; +import { captureSnapshotWithInteractor } from '../../../handlers/snapshot-interactor-capture.ts'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); diff --git a/src/daemon/handlers/__tests__/interaction-touch-press.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-press.test.ts similarity index 97% rename from src/daemon/handlers/__tests__/interaction-touch-press.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch-press.test.ts index 542a82e52e..9545c6a8dd 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-press.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-press.test.ts @@ -1,12 +1,12 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { handleInteractionCommands } from '../interaction.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, findResolvedTarget, @@ -36,7 +36,7 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('../snapshot-interactor-capture.ts', () => ({ +vi.mock('../../../handlers/snapshot-interactor-capture.ts', () => ({ captureSnapshotWithInteractor: vi.fn(), })); @@ -51,7 +51,7 @@ import { getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; +import { captureSnapshotWithInteractor } from '../../../handlers/snapshot-interactor-capture.ts'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); diff --git a/src/daemon/handlers/__tests__/interaction-touch-response.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts similarity index 97% rename from src/daemon/handlers/__tests__/interaction-touch-response.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts index dd52d8e74e..2d3bcea3ae 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-response.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-response.test.ts @@ -1,14 +1,14 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { handleInteractionCommands } from '../interaction.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { transformTouchResponseData } from '../interaction-touch-response.ts'; import { getRuntimeBindings, mockFillPoint, mockTapPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, installTestScreenRecording, @@ -33,7 +33,7 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('../snapshot-interactor-capture.ts', () => ({ +vi.mock('../../../handlers/snapshot-interactor-capture.ts', () => ({ captureSnapshotWithInteractor: vi.fn(), })); @@ -48,7 +48,7 @@ import { getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; +import { captureSnapshotWithInteractor } from '../../../handlers/snapshot-interactor-capture.ts'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); diff --git a/src/daemon/handlers/__tests__/interaction-touch-runtime.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-runtime.test.ts similarity index 96% rename from src/daemon/handlers/__tests__/interaction-touch-runtime.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch-runtime.test.ts index 74d40981d8..d3b0061c59 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-runtime.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch-runtime.test.ts @@ -1,13 +1,13 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; import { withAppleRunnerProvider } from '@agent-device/platform-apple/runner'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { handleInteractionCommands } from '../interaction.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { getRuntimeBindings, mockTapPoint, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, makeSession } from './interaction-touch-fixtures.ts'; // What the shared runtime dispatch does with the resolved target: refuse @@ -32,7 +32,7 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('../snapshot-interactor-capture.ts', () => ({ +vi.mock('../../../handlers/snapshot-interactor-capture.ts', () => ({ captureSnapshotWithInteractor: vi.fn(), })); @@ -47,7 +47,7 @@ import { getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; +import { captureSnapshotWithInteractor } from '../../../handlers/snapshot-interactor-capture.ts'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); diff --git a/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch-targets.test.ts similarity index 100% rename from src/daemon/handlers/__tests__/interaction-touch-targets.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch-targets.test.ts diff --git a/src/daemon/handlers/__tests__/interaction-touch.test.ts b/src/daemon/interaction/internal/__tests__/interaction-touch.test.ts similarity index 95% rename from src/daemon/handlers/__tests__/interaction-touch.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-touch.test.ts index b24c7ec16e..7e1d1198a0 100644 --- a/src/daemon/handlers/__tests__/interaction-touch.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-touch.test.ts @@ -1,8 +1,8 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { attachRefs } from '@agent-device/kernel/snapshot'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { WEB_DESKTOP_DEVICE } from '../../../__tests__/test-utils/device-fixtures.ts'; -import { handleInteractionCommands } from '../interaction.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { WEB_DESKTOP_DEVICE } from '../../../../__tests__/test-utils/device-fixtures.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { getRuntimeBindings, mockHoverPoint, @@ -11,7 +11,7 @@ import { mockTapPoint, resetGetRuntimeFixture, runtimeBindingSpies, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; import { contextFromFlags, makeMacOsDesktopSession, @@ -36,7 +36,7 @@ vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { }; }); -vi.mock('../snapshot-interactor-capture.ts', () => ({ +vi.mock('../../../handlers/snapshot-interactor-capture.ts', () => ({ captureSnapshotWithInteractor: vi.fn(), })); @@ -51,7 +51,7 @@ import { getAndroidBlockingDialogObservation, getAndroidScreenSize, } from '@agent-device/platform-android/mechanics'; -import { captureSnapshotWithInteractor } from '../snapshot-interactor-capture.ts'; +import { captureSnapshotWithInteractor } from '../../../handlers/snapshot-interactor-capture.ts'; const mockGetAndroidAppState = vi.mocked(getAndroidAppState); const mockGetAndroidBlockingDialogObservation = vi.mocked(getAndroidBlockingDialogObservation); const mockGetAndroidScreenSize = vi.mocked(getAndroidScreenSize); diff --git a/src/daemon/handlers/__tests__/interaction-type-android-readiness.test.ts b/src/daemon/interaction/internal/__tests__/interaction-type-android-readiness.test.ts similarity index 84% rename from src/daemon/handlers/__tests__/interaction-type-android-readiness.test.ts rename to src/daemon/interaction/internal/__tests__/interaction-type-android-readiness.test.ts index c4ff21c0bd..9371210df3 100644 --- a/src/daemon/handlers/__tests__/interaction-type-android-readiness.test.ts +++ b/src/daemon/interaction/internal/__tests__/interaction-type-android-readiness.test.ts @@ -1,15 +1,15 @@ import { test, expect, vi, beforeEach } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; -import { makeAndroidSession } from '../../../__tests__/test-utils/session-factories.ts'; -import { makeTestScreenRecordingResource } from '../../../__tests__/test-utils/screen-recording-live-handle.ts'; -import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; -import { handleInteractionCommands } from '../interaction.ts'; +import { makeAndroidSession } from '../../../../__tests__/test-utils/session-factories.ts'; +import { makeTestScreenRecordingResource } from '../../../../__tests__/test-utils/screen-recording-live-handle.ts'; +import { makeSessionStore } from '../../../../__tests__/test-utils/store-factory.ts'; +import { handleInteractionCommands } from '../../index.ts'; import { contextFromFlags } from './interaction-touch-fixtures.ts'; import { getRuntimeBindings, mockTypeText, resetGetRuntimeFixture, -} from './interaction-get-runtime-fixture.ts'; +} from '../../../__tests__/interaction-get-runtime-fixture.ts'; vi.mock('@agent-device/platform-android/mechanics', async (importOriginal) => { const actual = await importOriginal(); diff --git a/src/daemon/handlers/find-match-ranking.ts b/src/daemon/interaction/internal/find-match-ranking.ts similarity index 98% rename from src/daemon/handlers/find-match-ranking.ts rename to src/daemon/interaction/internal/find-match-ranking.ts index 0c3a87ca9e..3ffe0b279e 100644 --- a/src/daemon/handlers/find-match-ranking.ts +++ b/src/daemon/interaction/internal/find-match-ranking.ts @@ -3,7 +3,7 @@ import { createActionableTouchResolver, isRootInteractionContainer, resolveActionableTouchResolution, -} from '../../core/interaction-targeting.ts'; +} from '../../../core/interaction-targeting.ts'; /** * How `find` orders the candidates its locator matched, before the ambiguity diff --git a/src/daemon/handlers/find-match-resolution.ts b/src/daemon/interaction/internal/find-match-resolution.ts similarity index 92% rename from src/daemon/handlers/find-match-resolution.ts rename to src/daemon/interaction/internal/find-match-resolution.ts index 915f084771..7ee23e8031 100644 --- a/src/daemon/handlers/find-match-resolution.ts +++ b/src/daemon/interaction/internal/find-match-resolution.ts @@ -3,15 +3,15 @@ import { type FindLocator, type SelectorResolutionPolicy, } from '@agent-device/selectors'; -import { listSelectorPipelineMatches } from '../../core/selector-pipeline.ts'; -import { SELECTOR_PIPELINE_POLICIES } from '../../core/selector-pipeline-policy.ts'; +import { listSelectorPipelineMatches } from '../../../core/selector-pipeline.ts'; +import { SELECTOR_PIPELINE_POLICIES } from '../../../core/selector-pipeline-policy.ts'; import type { SnapshotState } from '@agent-device/kernel/snapshot'; -import { isRootInteractionContainer } from '../../core/interaction-targeting.ts'; +import { isRootInteractionContainer } from '../../../core/interaction-targeting.ts'; import { preferOnscreenMatches } from './find-match-ranking.ts'; -import { formatSnapshotLine } from '../../snapshot/snapshot-lines.ts'; +import { formatSnapshotLine } from '../../../snapshot/snapshot-lines.ts'; import type { ElementMatchCandidateDetails } from '@agent-device/kernel/errors'; -import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts'; -import { errorResponse } from '../response.ts'; +import type { DaemonRequest, DaemonResponse, SessionState } from '../../types.ts'; +import { errorResponse } from '../../response.ts'; export type FindMatchResult = | { ok: true; node: SnapshotState['nodes'][number] } diff --git a/src/daemon/handlers/find-target-capture.ts b/src/daemon/interaction/internal/find-target-capture.ts similarity index 90% rename from src/daemon/handlers/find-target-capture.ts rename to src/daemon/interaction/internal/find-target-capture.ts index d7d78408e1..9d33a46c37 100644 --- a/src/daemon/handlers/find-target-capture.ts +++ b/src/daemon/interaction/internal/find-target-capture.ts @@ -1,10 +1,10 @@ import type { FindLocator } from '@agent-device/selectors'; -import type { BoundSelectorCapture } from '../selector-capture-binding.ts'; +import type { BoundSelectorCapture } from '../../selector-capture-binding.ts'; import type { SnapshotQualityVerdict, SnapshotState } from '@agent-device/kernel/snapshot'; -import { createSelectorCaptureRuntime } from '../selector-capture-runtime.ts'; -import { SessionStore } from '../session-store.ts'; -import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts'; -import { errorResponse } from '../response.ts'; +import { createSelectorCaptureRuntime } from '../../selector-capture-runtime.ts'; +import { SessionStore } from '../../session-store.ts'; +import type { DaemonRequest, DaemonResponse, SessionState } from '../../types.ts'; +import { errorResponse } from '../../response.ts'; /** The tree a mutating find resolves its target against, plus what the capture disclosed. */ export type FindTargetTree = { diff --git a/src/daemon/handlers/find.ts b/src/daemon/interaction/internal/find.ts similarity index 91% rename from src/daemon/handlers/find.ts rename to src/daemon/interaction/internal/find.ts index 107431f6df..607b2cdf3e 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/interaction/internal/find.ts @@ -5,29 +5,29 @@ import { parseFindSelectorExpression, type FindLocator, } from '@agent-device/selectors'; -import { runNodePipelineStages } from '../../core/selector-pipeline.ts'; -import { SELECTOR_PIPELINE_POLICIES } from '../../core/selector-pipeline-policy.ts'; +import { runNodePipelineStages } from '../../../core/selector-pipeline.ts'; +import { SELECTOR_PIPELINE_POLICIES } from '../../../core/selector-pipeline-policy.ts'; import { centerOfRect, type SnapshotState } from '@agent-device/kernel/snapshot'; -import { expireRefFrame } from '../ref-frame.ts'; -import type { DaemonInvokeFn, DaemonRequest, DaemonResponse, SessionState } from '../types.ts'; -import { SessionStore } from '../session-store.ts'; -import { contextFromFlags } from '../context.ts'; +import { expireRefFrame } from '../../ref-frame.ts'; +import type { DaemonInvokeFn, DaemonRequest, DaemonResponse, SessionState } from '../../types.ts'; +import type { SessionStore } from '../../session-store.ts'; +import { contextFromFlags } from '../../context.ts'; import { readCommandMessage, successText } from '@agent-device/kernel/success-text'; -import { errorResponse, noActiveSessionError } from '../response.ts'; -import { withSystemSurfaceDisclosure } from './system-surface-disclosure.ts'; -import { recordSessionAction } from '../session-action-recorder.ts'; -import { stripInternalInteractionFlags } from '../interaction-outcome-policy.ts'; +import { errorResponse, noActiveSessionError } from '../../response.ts'; +import { withSystemSurfaceDisclosure } from '../../system-surface-disclosure.ts'; +import { recordSessionAction } from '../../session-action-recorder.ts'; +import { stripInternalInteractionFlags } from '../../interaction-outcome-policy.ts'; import { resolveFindMatch } from './find-match-resolution.ts'; -import { executeFocusPoint } from '../focus-runtime.ts'; -import { executeBoundTypeText } from '../type-text-runtime.ts'; -import { dispatchFindReadOnlyViaRuntime } from '../selector-runtime.ts'; -import { admitAndBindSnapshotCapture } from '../snapshot-runtime-binding.ts'; +import { executeFocusPoint } from '../../focus-runtime.ts'; +import { executeBoundTypeText } from '../../type-text-runtime.ts'; +import { dispatchFindReadOnlyViaRuntime } from '../../selector-runtime.ts'; +import { admitAndBindSnapshotCapture } from '../../snapshot-runtime-binding.ts'; import type { FocusPointInput } from '@agent-device/contracts/focus-runtime'; import { resolveSelectorCaptureRuntimePlan } from '@agent-device/contracts/platform-runtime-operations'; import type { TypeTextRuntimeOperations } from '@agent-device/contracts/type-text-runtime'; -import type { BindDeviceRuntime, InspectDeviceRuntimeFacts } from '../request-runtime-binding.ts'; import { createFindTargetCapture, sparseFindSnapshotResponse } from './find-target-capture.ts'; import { isSparseSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; +import type { FindRouteInput } from './types.ts'; type FindContext = { req: DaemonRequest; @@ -60,15 +60,7 @@ type ResolvedMatch = { occludedNode?: SnapshotState['nodes'][number]; }; -export async function handleFindCommands(params: { - req: DaemonRequest; - sessionName: string; - logPath: string; - sessionStore: SessionStore; - invoke: DaemonInvokeFn; - inspectFacts?: InspectDeviceRuntimeFacts; - bindDevice?: BindDeviceRuntime; -}): Promise { +export async function handleFindCommands(params: FindRouteInput): Promise { const { req, sessionName, logPath, sessionStore, invoke } = params; const command = req.command; if (command !== 'find') return null; diff --git a/src/daemon/interaction/internal/interaction-android-escape.ts b/src/daemon/interaction/internal/interaction-android-escape.ts new file mode 100644 index 0000000000..b2ae095b99 --- /dev/null +++ b/src/daemon/interaction/internal/interaction-android-escape.ts @@ -0,0 +1,37 @@ +import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; +import { AppError } from '@agent-device/kernel/errors'; +import type { SessionState } from '../../types.ts'; +import { detectAndroidEscapeSurface } from '../../android-escape-surface.ts'; + +/** + * Post-press escape guard. Throws when the tap left the app for a genuine + * escape surface (settings/systemui/launcher). A foregrounded permission + * prompt is NOT an escape — the press succeeded and raised a system dialog + * the agent consumes via `alert` — so it returns a response warning instead. + */ +export async function assertAndroidPressStayedInApp( + session: SessionState, + targetLabel: string, + observation?: AndroidObservationAdapter, +): Promise { + const surface = await detectAndroidEscapeSurface(session, observation); + if (!surface) return undefined; + + if (surface.permissionDialog) { + return `press ${targetLabel} opened an Android permission dialog (${surface.foregroundPackage}) over ${surface.expectedPackage}. ${surface.hint}`; + } + + throw new AppError( + 'COMMAND_FAILED', + `press ${targetLabel} left ${session.appBundleId} and foregrounded ${surface.foregroundPackage}. The tap likely escaped the app.`, + surface, + ); +} + +export function isAndroidEscapeError(error: AppError): boolean { + return ( + error.code === 'COMMAND_FAILED' && + typeof error.details?.expectedPackage === 'string' && + typeof error.details?.foregroundPackage === 'string' + ); +} diff --git a/src/daemon/interaction/internal/interaction-flags.ts b/src/daemon/interaction/internal/interaction-flags.ts index a50d86d208..ff94357d86 100644 --- a/src/daemon/interaction/internal/interaction-flags.ts +++ b/src/daemon/interaction/internal/interaction-flags.ts @@ -3,33 +3,6 @@ import type { SettleParams } from '@agent-device/contracts/interaction'; import type { DaemonResponse } from '../../types.ts'; import { interactionErrorResponse } from './interaction-response.ts'; -const REF_UNSUPPORTED_FLAG_MAP: ReadonlyArray<[keyof CommandFlags, string]> = [ - ['snapshotDepth', '--depth'], - ['snapshotScope', '--scope'], - ['snapshotRaw', '--raw'], -]; - -export function refSnapshotFlagGuardResponse( - command: 'press' | 'fill' | 'get' | 'longpress' | 'hover', - flags: CommandFlags | undefined, -): DaemonResponse | null { - const unsupported = unsupportedRefSnapshotFlags(flags); - if (unsupported.length === 0) return null; - return interactionErrorResponse( - 'INVALID_ARGS', - `${command} @ref does not support ${unsupported.join(', ')}.`, - ); -} - -export function unsupportedRefSnapshotFlags(flags: CommandFlags | undefined): string[] { - if (!flags) return []; - const unsupported: string[] = []; - for (const [key, label] of REF_UNSUPPORTED_FLAG_MAP) { - if (flags[key] !== undefined) unsupported.push(label); - } - return unsupported; -} - export function settleFlagGuardResponse( command: string, flags: CommandFlags | undefined, diff --git a/src/daemon/handlers/interaction-gesture-response.ts b/src/daemon/interaction/internal/interaction-gesture-response.ts similarity index 100% rename from src/daemon/handlers/interaction-gesture-response.ts rename to src/daemon/interaction/internal/interaction-gesture-response.ts diff --git a/src/daemon/handlers/interaction-gesture.ts b/src/daemon/interaction/internal/interaction-gesture.ts similarity index 92% rename from src/daemon/handlers/interaction-gesture.ts rename to src/daemon/interaction/internal/interaction-gesture.ts index 1b6b684593..70c3a0c446 100644 --- a/src/daemon/handlers/interaction-gesture.ts +++ b/src/daemon/interaction/internal/interaction-gesture.ts @@ -23,30 +23,27 @@ import { splitRefGenerationSuffix, type Point, } from '@agent-device/kernel/snapshot'; -import { resolveBoundGestureRuntime, type BoundGestureExecutor } from '../gesture-runtime.ts'; -import { isActiveProviderDevice } from '../../provider-device-runtime.ts'; +import { resolveBoundGestureRuntime, type BoundGestureExecutor } from '../../gesture-runtime.ts'; +import { isActiveProviderDevice } from '../../../provider-device-runtime.ts'; import { sleep } from '@agent-device/host-kit/retry'; -import { ensureAndroidBlockingSystemDialogReady } from '../android-system-dialog.ts'; -import { readRefMutationFrame } from '../ref-frame.ts'; -import type { DaemonResponse, SessionState } from '../types.ts'; -import { - assertRefMutationAdmitted, - createInteractionRuntime, - finalizeTouchInteraction, - type InteractionRouteInput, -} from '../interaction/index.ts'; -import { noActiveSessionError } from '../response.ts'; -import type { RecordedTargetCapture } from '../session-target-evidence.ts'; +import { ensureAndroidBlockingSystemDialogReady } from '../../android-system-dialog.ts'; +import { readRefMutationFrame } from '../../ref-frame.ts'; +import type { DaemonResponse, SessionState } from '../../types.ts'; +import { assertRefMutationAdmitted } from './interaction-ref-policy.ts'; +import { createInteractionRuntime, finalizeTouchInteraction } from './interaction-route-support.ts'; +import { noActiveSessionError } from '../../response.ts'; +import type { RecordedTargetCapture } from '../../session-target-evidence.ts'; import { gestureResponseData } from './interaction-gesture-response.ts'; +import type { InteractionRouteExecutionInput } from './types.ts'; -type GestureHandlerParams = InteractionRouteInput; +type GestureHandlerParams = InteractionRouteExecutionInput; type GestureRuntime = ReturnType; type GestureRuntimeResult = Awaited>; type GestureInteractionOutcome = { positionals: string[]; - flags: InteractionRouteInput['req']['flags']; + flags: InteractionRouteExecutionInput['req']['flags']; responseData: Record; recordingResultExtra?: Record; recordedTargets?: { source: RecordedTargetCapture; destination: RecordedTargetCapture }; @@ -101,7 +98,7 @@ async function runPreparedGesture( runtime: GestureRuntime, context: { session: string; requestId: string | undefined }, gesture: GestureCommandInput, - internal: InteractionRouteInput['req']['internal'], + internal: InteractionRouteExecutionInput['req']['internal'], ): Promise { if (gesture.intent !== 'drag') { return await runtime.interactions.gesture({ ...context, gesture }); @@ -120,7 +117,7 @@ function buildGestureOutcome( input: GesturePayload, gesture: GestureCommandInput, result: GestureRuntimeResult, - flags: InteractionRouteInput['req']['flags'], + flags: InteractionRouteExecutionInput['req']['flags'], ): GestureInteractionOutcome { const recording = result.kind === 'drag' ? result.recording : undefined; const sourceTarget = recording?.sourceTarget; @@ -364,7 +361,7 @@ function swipeReplayPositionals(input: SwipePayload): string[] { async function runSwipeRepetitions( runtime: ReturnType, - params: InteractionRouteInput, + params: InteractionRouteExecutionInput, input: SwipePayload, count: number, pauseMs: number, @@ -396,8 +393,8 @@ function swipeMotionAtIndex( function gestureReplayFlags( input: GesturePayload, - flags: InteractionRouteInput['req']['flags'], -): InteractionRouteInput['req']['flags'] { + flags: InteractionRouteExecutionInput['req']['flags'], +): InteractionRouteExecutionInput['req']['flags'] { if (input.kind !== 'pan' || input.pointerCount === undefined) return flags; return { ...flags, pointerCount: input.pointerCount }; } diff --git a/src/daemon/handlers/interaction-ios-tap-outcome.ts b/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts similarity index 97% rename from src/daemon/handlers/interaction-ios-tap-outcome.ts rename to src/daemon/interaction/internal/interaction-ios-tap-outcome.ts index 1650d57284..0e90e501bf 100644 --- a/src/daemon/handlers/interaction-ios-tap-outcome.ts +++ b/src/daemon/interaction/internal/interaction-ios-tap-outcome.ts @@ -6,13 +6,13 @@ import { isSparseSnapshotQualityVerdict, preferredSnapshotBackendForVerdict, } from '@agent-device/capture-kit/snapshot-quality-verdict'; -import { summarizeAxEvidence } from '../../snapshot/snapshot-evidence.ts'; +import { summarizeAxEvidence } from '../../../snapshot/snapshot-evidence.ts'; import { getRequestSignal } from '@agent-device/host-kit/request'; -import { isLocalIosRunnerSession } from '../direct-ios-selector.ts'; +import { isLocalIosRunnerSession } from '../../direct-ios-selector.ts'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; -import type { SessionStore } from '../session-store.ts'; -import type { SessionState } from '../types.ts'; -import type { CaptureSnapshotForSession, ContextFromFlags } from '../interaction/index.ts'; +import type { SessionStore } from '../../session-store.ts'; +import type { SessionState } from '../../types.ts'; +import type { CaptureSnapshotForSession, ContextFromFlags } from './types.ts'; const XCTEST_RECORDED_FAILURE = 'XCTEST_RECORDED_FAILURE'; // A model commonly needs 5-10s to choose a target after receiving a snapshot. diff --git a/src/daemon/interaction/internal/interaction-route-support.ts b/src/daemon/interaction/internal/interaction-route-support.ts new file mode 100644 index 0000000000..5d200b7590 --- /dev/null +++ b/src/daemon/interaction/internal/interaction-route-support.ts @@ -0,0 +1,97 @@ +import { AppError as KernelAppError } from '@agent-device/kernel/errors'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import { buildAppleRunnerRequestOptions } from '../../apple-runner-options.ts'; +import { markDeferredInteractionOutcome } from '../../deferred-interaction-outcome.ts'; +import { isLocalIosRunnerSession } from '../../direct-ios-selector.ts'; +import { expireRefFrame } from '../../ref-frame.ts'; +import { recordTouchVisualizationEvent } from '../../recording-gestures.ts'; +import { createDaemonRuntimeSessionStore } from '../../runtime-session.ts'; +import { isSessionRecording } from '../../session-script-publication-capability.ts'; +import { setSessionSnapshot } from '../../session-snapshot.ts'; +import { confirmIosOffscreenTargetVisible } from '../../offscreen-target-probe.ts'; +import { NO_ACTIVE_SESSION_MESSAGE } from '../../response.ts'; +import type { BoundGestureExecutor } from '../../gesture-runtime.ts'; +import type { BoundTouchExecutor } from '../../touch-runtime.ts'; +import { finalizeTouchInteraction as finalizeInteraction } from './interaction-common.ts'; +import { createInteractionRuntime as createInternalInteractionRuntime } from './interaction-runtime.ts'; +import type { InteractionRouteExecutionInput, InteractionFinalizationOperations } from './types.ts'; +import type { DaemonResponse, SessionState } from '../../types.ts'; +import type { SessionStore } from '../../session-store.ts'; + +export function createInteractionRuntime( + params: InteractionRouteExecutionInput & { + pairedGestureViewport?: Rect; + touchExecutor?: BoundTouchExecutor; + gestures?: BoundGestureExecutor; + }, +) { + const session = params.sessionStore.get(params.sessionName); + if (!session) throw new KernelAppError('SESSION_NOT_FOUND', NO_ACTIVE_SESSION_MESSAGE); + return createInternalInteractionRuntime({ + requestId: params.req.meta?.requestId, + flags: params.req.flags, + session, + contextFromFlags: params.contextFromFlags, + captureSnapshot: async (flags, options) => + await params.captureSnapshotForSession( + session, + flags, + params.sessionStore, + params.contextFromFlags, + options, + ), + runtimeSessions: createDaemonRuntimeSessionStore({ + sessionName: params.sessionName, + getSession: () => session, + recordOptions: { + includeSnapshot: true, + omitRefFrameSnapshot: params.req.internal?.findResolvedTarget !== undefined, + }, + setRecord: (record) => { + if (!record.snapshot) return; + setSessionSnapshot(session, record.snapshot); + params.sessionStore.set(params.sessionName, session); + }, + }), + expireRefFrame: () => expireRefFrame(session), + confirmOffscreenTargetVisible: isLocalIosRunnerSession(session, { + skipPendingPostGestureStabilization: false, + }) + ? async (node, rootViewport) => + await confirmIosOffscreenTargetVisible({ + session, + node, + rootViewport, + requestOptions: buildAppleRunnerRequestOptions({ + req: params.req, + logPath: params.logPath, + traceLogPath: session.trace?.outPath, + }), + }) + : undefined, + pairedGestureViewport: params.pairedGestureViewport, + touchExecutor: params.touchExecutor, + gestures: params.gestures, + }); +} + +type FinalizeTouchInteractionInput = Omit< + Parameters[0], + 'operations' +> & { + session: SessionState; + sessionStore: SessionStore; +}; + +export function finalizeTouchInteraction(params: FinalizeTouchInteractionInput): DaemonResponse { + const { session, sessionStore, ...finalization } = params; + return finalizeInteraction({ + ...finalization, + operations: { + recordAction: sessionStore.recordAction.bind(sessionStore, session), + markDeferredOutcome: (mark) => markDeferredInteractionOutcome({ session, ...mark }), + isSessionRecording: isSessionRecording.bind(null, session), + recordGestureVisualization: recordTouchVisualizationEvent.bind(null, session), + } satisfies InteractionFinalizationOperations, + }); +} diff --git a/src/daemon/handlers/interaction-targeting.ts b/src/daemon/interaction/internal/interaction-targeting.ts similarity index 74% rename from src/daemon/handlers/interaction-targeting.ts rename to src/daemon/interaction/internal/interaction-targeting.ts index 88987e9820..829dfe6db3 100644 --- a/src/daemon/handlers/interaction-targeting.ts +++ b/src/daemon/interaction/internal/interaction-targeting.ts @@ -1,5 +1,3 @@ -import { resolveRectCenter } from '@agent-device/kernel/rect-center'; - export function parseCoordinateTarget(positionals: string[]): { x: number; y: number } | null { if (positionals.length < 2) return null; const x = Number(positionals[0]); @@ -7,5 +5,3 @@ export function parseCoordinateTarget(positionals: string[]): { x: number; y: nu if (!Number.isFinite(x) || !Number.isFinite(y)) return null; return { x, y }; } - -export { resolveRectCenter }; diff --git a/src/daemon/handlers/interaction-touch-android-freshness.ts b/src/daemon/interaction/internal/interaction-touch-android-freshness.ts similarity index 76% rename from src/daemon/handlers/interaction-touch-android-freshness.ts rename to src/daemon/interaction/internal/interaction-touch-android-freshness.ts index 3673c90543..9754e142d5 100644 --- a/src/daemon/handlers/interaction-touch-android-freshness.ts +++ b/src/daemon/interaction/internal/interaction-touch-android-freshness.ts @@ -1,7 +1,7 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; -import { getActiveAndroidSnapshotFreshness } from '../session-snapshot-freshness.ts'; -import type { SessionState } from '../types.ts'; -import type { CaptureSnapshotForSession, InteractionRouteInput } from '../interaction/index.ts'; +import { getActiveAndroidSnapshotFreshness } from '../../session-snapshot-freshness.ts'; +import type { SessionState } from '../../types.ts'; +import type { InteractionRouteExecutionInput } from './types.ts'; /** * The Android ref-refresh capture a `@ref` mutation takes before dispatch, and @@ -10,9 +10,7 @@ import type { CaptureSnapshotForSession, InteractionRouteInput } from '../intera */ export async function refreshAndroidRefSnapshotIfFreshnessActive( - params: InteractionRouteInput & { - captureSnapshotForSession: CaptureSnapshotForSession; - }, + params: InteractionRouteExecutionInput, session: SessionState, ): Promise { if (!getActiveAndroidSnapshotFreshness(session)) return undefined; diff --git a/src/daemon/handlers/interaction-touch-android-readiness.ts b/src/daemon/interaction/internal/interaction-touch-android-readiness.ts similarity index 91% rename from src/daemon/handlers/interaction-touch-android-readiness.ts rename to src/daemon/interaction/internal/interaction-touch-android-readiness.ts index 1ace61be8f..e1bc5f3ff5 100644 --- a/src/daemon/handlers/interaction-touch-android-readiness.ts +++ b/src/daemon/interaction/internal/interaction-touch-android-readiness.ts @@ -1,10 +1,10 @@ import { ensureAndroidBlockingSystemDialogReady, type AndroidBlockingDialogReadinessResult, -} from '../android-system-dialog.ts'; -import type { DaemonResponse, SessionState } from '../types.ts'; -import { readRefMutationFrame } from '../ref-frame.ts'; -import { refMutationAdmissionResponse } from '../interaction/index.ts'; +} from '../../android-system-dialog.ts'; +import type { DaemonResponse, SessionState } from '../../types.ts'; +import { readRefMutationFrame } from '../../ref-frame.ts'; +import { refMutationAdmissionResponse } from './interaction-ref-policy.ts'; import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; /** diff --git a/src/daemon/handlers/interaction-touch-direct-ios-eligibility.ts b/src/daemon/interaction/internal/interaction-touch-direct-ios-eligibility.ts similarity index 91% rename from src/daemon/handlers/interaction-touch-direct-ios-eligibility.ts rename to src/daemon/interaction/internal/interaction-touch-direct-ios-eligibility.ts index 9b4a6a8bc6..b8844f2283 100644 --- a/src/daemon/handlers/interaction-touch-direct-ios-eligibility.ts +++ b/src/daemon/interaction/internal/interaction-touch-direct-ios-eligibility.ts @@ -3,13 +3,13 @@ import type { InteractionTarget } from '@agent-device/contracts/interaction'; import { commandSupportsSettleObservation, commandSupportsVerifyEvidence, -} from '../../core/command-descriptor/registry.ts'; +} from '../../../core/command-descriptor/registry.ts'; import { readSimpleIosSelectorTarget, type DirectIosSelectorTarget, -} from '../direct-ios-selector.ts'; -import { isSessionRecording } from '../session-script-publication-capability.ts'; -import type { SessionState } from '../types.ts'; +} from '../../direct-ios-selector.ts'; +import { isSessionRecording } from '../../session-script-publication-capability.ts'; +import type { SessionState } from '../../types.ts'; /** * Whether a Maestro-compatible click needs the direct iOS selector route. diff --git a/src/daemon/handlers/interaction-touch-direct-ios.ts b/src/daemon/interaction/internal/interaction-touch-direct-ios.ts similarity index 91% rename from src/daemon/handlers/interaction-touch-direct-ios.ts rename to src/daemon/interaction/internal/interaction-touch-direct-ios.ts index 3641e2cc39..5d52458be8 100644 --- a/src/daemon/handlers/interaction-touch-direct-ios.ts +++ b/src/daemon/interaction/internal/interaction-touch-direct-ios.ts @@ -4,16 +4,12 @@ import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { isDirectIosSelectorFallbackError, type DirectIosSelectorTarget, -} from '../direct-ios-selector.ts'; -import { expireRefFrame } from '../ref-frame.ts'; -import type { DaemonResponse, SessionState } from '../types.ts'; -import { - finalizeTouchInteraction, - type CaptureSnapshotForSession, - type InteractionRouteInput, -} from '../interaction/index.ts'; +} from '../../direct-ios-selector.ts'; +import { expireRefFrame } from '../../ref-frame.ts'; +import type { DaemonResponse, SessionState } from '../../types.ts'; +import { finalizeTouchInteraction } from './interaction-route-support.ts'; import { corroborateIosTapFailure } from './interaction-ios-tap-outcome.ts'; -import type { BoundTouchExecutor } from '../touch-runtime.ts'; +import type { BoundTouchExecutor } from '../../touch-runtime.ts'; import { buildCorroboratedTapResponseData, buildInteractionResponseData, @@ -22,6 +18,7 @@ import { readInteractionResponseDataTransformCommand, transformTouchResponseData, } from './interaction-touch-response.ts'; +import type { InteractionRouteExecutionInput } from './types.ts'; /** * How the Maestro-compatible direct iOS selector route dispatches or @@ -31,7 +28,7 @@ import { */ export async function dispatchDirectIosSelectorTap( - params: InteractionRouteInput & { captureSnapshotForSession: CaptureSnapshotForSession }, + params: InteractionRouteExecutionInput, session: SessionState, selector: DirectIosSelectorTarget, tapElementSelector: NonNullable, @@ -119,9 +116,7 @@ export async function dispatchDirectIosSelectorTap( async function buildDirectIosCorroboratedResponse(params: { error: unknown; - handlerParams: InteractionRouteInput & { - captureSnapshotForSession: CaptureSnapshotForSession; - }; + handlerParams: InteractionRouteExecutionInput; session: SessionState; extra: Record; positionals: string[]; diff --git a/src/daemon/handlers/interaction-touch-fill.ts b/src/daemon/interaction/internal/interaction-touch-fill.ts similarity index 88% rename from src/daemon/handlers/interaction-touch-fill.ts rename to src/daemon/interaction/internal/interaction-touch-fill.ts index fb435acc05..38704b6e8c 100644 --- a/src/daemon/handlers/interaction-touch-fill.ts +++ b/src/daemon/interaction/internal/interaction-touch-fill.ts @@ -1,21 +1,15 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import type { FillCommandResult, InteractionTarget } from '@agent-device/contracts/interaction'; -import { issueSettleRefs, resolveRefStalenessWarning } from '../session-snapshot.ts'; -import { readRefMutationFrame } from '../ref-frame.ts'; -import type { DaemonResponse, SessionState } from '../types.ts'; -import { isSessionRecording } from '../session-script-publication-capability.ts'; -import { - assertRecordedFillParameterization, - readSettleRequest, - refMutationAdmissionResponse, - settleFlagGuardResponse, - type CaptureSnapshotForSession, - type InteractionRouteInput, - type RefSnapshotFlagGuardResponse, -} from '../interaction/index.ts'; +import { issueSettleRefs, resolveRefStalenessWarning } from '../../session-snapshot.ts'; +import { readRefMutationFrame } from '../../ref-frame.ts'; +import type { DaemonResponse, SessionState } from '../../types.ts'; +import { isSessionRecording } from '../../session-script-publication-capability.ts'; +import { assertRecordedFillParameterization } from './interaction-recorded-input.ts'; +import { readSettleRequest, settleFlagGuardResponse } from './interaction-flags.ts'; +import { refMutationAdmissionResponse } from './interaction-ref-policy.ts'; import { refreshAndroidRefSnapshotIfFreshnessActive } from './interaction-touch-android-freshness.ts'; import { unsupportedMacOsDesktopSurfaceInteraction } from './interaction-touch-policy.ts'; -import { readSnapshotNodesReferenceFrame } from './interaction-touch-reference-frame.ts'; +import { readSnapshotNodesReferenceFrame } from '../../touch-reference-frame.ts'; import { buildInteractionResponseData, maestroFallbackDisclosure, @@ -24,8 +18,9 @@ import { } from './interaction-touch-response.ts'; import { dispatchRuntimeInteraction } from './interaction-touch-runtime.ts'; import { parseFillTarget } from './interaction-touch-targets.ts'; -import { noActiveSessionError } from '../response.ts'; +import { noActiveSessionError } from '../../response.ts'; import { prepareTouchDispatch } from './interaction-touch-prepare.ts'; +import type { InteractionRouteExecutionInput, RefSnapshotFlagGuardResponse } from './types.ts'; /** * How `fill` is admitted, parameterized, executed, and projected: surface and @@ -34,8 +29,7 @@ import { prepareTouchDispatch } from './interaction-touch-prepare.ts'; */ export async function dispatchFillViaRuntime( - params: InteractionRouteInput & { - captureSnapshotForSession: CaptureSnapshotForSession; + params: InteractionRouteExecutionInput & { refSnapshotFlagGuardResponse: RefSnapshotFlagGuardResponse; }, ): Promise { @@ -111,8 +105,7 @@ export async function dispatchFillViaRuntime( // @ref-incompatible flags, enforce iOS mutation freshness, and run the Android // freshness refresh. async function prepareFillRefTarget( - params: InteractionRouteInput & { - captureSnapshotForSession: CaptureSnapshotForSession; + params: InteractionRouteExecutionInput & { refSnapshotFlagGuardResponse: RefSnapshotFlagGuardResponse; }, session: SessionState, diff --git a/src/daemon/handlers/interaction-touch-payload.ts b/src/daemon/interaction/internal/interaction-touch-payload.ts similarity index 100% rename from src/daemon/handlers/interaction-touch-payload.ts rename to src/daemon/interaction/internal/interaction-touch-payload.ts diff --git a/src/daemon/handlers/interaction-touch-policy.ts b/src/daemon/interaction/internal/interaction-touch-policy.ts similarity index 85% rename from src/daemon/handlers/interaction-touch-policy.ts rename to src/daemon/interaction/internal/interaction-touch-policy.ts index 37184f6c07..a5cccd2641 100644 --- a/src/daemon/handlers/interaction-touch-policy.ts +++ b/src/daemon/interaction/internal/interaction-touch-policy.ts @@ -1,6 +1,6 @@ import { isMacOs } from '@agent-device/kernel/device'; -import type { DaemonResponse, SessionState } from '../types.ts'; -import { errorResponse } from '../response.ts'; +import type { DaemonResponse, SessionState } from '../../types.ts'; +import { errorResponse } from '../../response.ts'; export function unsupportedMacOsDesktopSurfaceInteraction( session: SessionState, diff --git a/src/daemon/handlers/interaction-touch-prepare.ts b/src/daemon/interaction/internal/interaction-touch-prepare.ts similarity index 79% rename from src/daemon/handlers/interaction-touch-prepare.ts rename to src/daemon/interaction/internal/interaction-touch-prepare.ts index ca32bcd68c..cff7d493bd 100644 --- a/src/daemon/handlers/interaction-touch-prepare.ts +++ b/src/daemon/interaction/internal/interaction-touch-prepare.ts @@ -1,12 +1,12 @@ -import type { SessionState } from '../types.ts'; +import type { SessionState } from '../../types.ts'; import { createBoundTouchExecutor, resolveBoundTouchRuntime, type BoundTouchExecutor, type TouchRuntimeCommand, -} from '../touch-runtime.ts'; -import type { DaemonFailureResponse } from '../response.ts'; -import type { InteractionRouteInput } from '../interaction/index.ts'; +} from '../../touch-runtime.ts'; +import type { DaemonFailureResponse } from '../../response.ts'; +import type { InteractionRouteExecutionInput } from './types.ts'; export type PreparedTouchDispatch = | Readonly<{ ok: false; response: DaemonFailureResponse }> @@ -14,7 +14,7 @@ export type PreparedTouchDispatch = /** Exact-owner admission, one bind, and command-context projection shared by every touch route. */ export async function prepareTouchDispatch( - params: InteractionRouteInput, + params: InteractionRouteExecutionInput, session: SessionState, command: TouchRuntimeCommand, requiresCapture: boolean, diff --git a/src/daemon/handlers/interaction-touch-press-admission.ts b/src/daemon/interaction/internal/interaction-touch-press-admission.ts similarity index 91% rename from src/daemon/handlers/interaction-touch-press-admission.ts rename to src/daemon/interaction/internal/interaction-touch-press-admission.ts index 7f48536ad2..f4d0477a40 100644 --- a/src/daemon/handlers/interaction-touch-press-admission.ts +++ b/src/daemon/interaction/internal/interaction-touch-press-admission.ts @@ -6,16 +6,11 @@ import { resolveClickButton, } from '@agent-device/contracts/click-button'; import { publicPlatformString } from '@agent-device/kernel/device'; -import { resolveRefStalenessWarning } from '../session-snapshot.ts'; -import { readRefMutationFrame } from '../ref-frame.ts'; -import type { DaemonResponse, SessionState } from '../types.ts'; -import { - refMutationAdmissionResponse, - settleFlagGuardResponse, - type CaptureSnapshotForSession, - type InteractionRouteInput, - type RefSnapshotFlagGuardResponse, -} from '../interaction/index.ts'; +import { resolveRefStalenessWarning } from '../../session-snapshot.ts'; +import { readRefMutationFrame } from '../../ref-frame.ts'; +import type { DaemonResponse, SessionState } from '../../types.ts'; +import { refMutationAdmissionResponse } from './interaction-ref-policy.ts'; +import { settleFlagGuardResponse } from './interaction-flags.ts'; import type { RefAdmissionContext } from './interaction-touch-android-readiness.ts'; import { unsupportedMacOsDesktopSurfaceInteraction } from './interaction-touch-policy.ts'; import { @@ -24,7 +19,8 @@ import { type ParsedLongPressTarget, type ParsedTouchTarget, } from './interaction-touch-targets.ts'; -import { errorResponse, noActiveSessionError } from '../response.ts'; +import { errorResponse, noActiveSessionError } from '../../response.ts'; +import type { InteractionRouteExecutionInput, RefSnapshotFlagGuardResponse } from './types.ts'; /** * Whether a targeted `press`/`click`/`longpress`/`hover` may act, and on what: macOS @@ -37,8 +33,7 @@ export type TargetedTouchCommand = 'press' | 'click' | 'longpress' | 'hover'; /** The family members that take `--button`; longpress and hover have no button. */ const CLICK_BUTTON_COMMANDS: ReadonlySet = new Set(['press', 'click']); -export type TargetedTouchParams = InteractionRouteInput & { - captureSnapshotForSession: CaptureSnapshotForSession; +export type TargetedTouchParams = InteractionRouteExecutionInput & { refSnapshotFlagGuardResponse: RefSnapshotFlagGuardResponse; }; @@ -149,7 +144,7 @@ function clickButtonValidationResponse( */ function readTargetedTouchStalenessWarning( session: SessionState, - req: InteractionRouteInput['req'], + req: InteractionRouteExecutionInput['req'], parsedTarget: ParsedTargetedTouch, ): string | undefined { if (parsedTarget.target.kind !== 'ref') return undefined; @@ -162,7 +157,7 @@ function readTargetedTouchStalenessWarning( } function readRefAdmissionContext( - req: InteractionRouteInput['req'], + req: InteractionRouteExecutionInput['req'], parsedTarget: ParsedTargetedTouch, staleRefsWarning: string | undefined, ): RefAdmissionContext | undefined { diff --git a/src/daemon/handlers/interaction-touch-press.ts b/src/daemon/interaction/internal/interaction-touch-press.ts similarity index 97% rename from src/daemon/handlers/interaction-touch-press.ts rename to src/daemon/interaction/internal/interaction-touch-press.ts index ca9a00d874..26c5c558d7 100644 --- a/src/daemon/handlers/interaction-touch-press.ts +++ b/src/daemon/interaction/internal/interaction-touch-press.ts @@ -5,9 +5,10 @@ import type { } from '@agent-device/contracts/interaction'; import type { resolveClickButton } from '@agent-device/contracts/click-button'; import type { ReplayTargetGuardDenotation } from '@agent-device/contracts/replay'; -import type { DaemonResponse } from '../types.ts'; +import type { DaemonResponse } from '../../types.ts'; import { assertAndroidPressStayedInApp } from './interaction-android-escape.ts'; -import { createInteractionRuntime, readSettleRequest } from '../interaction/index.ts'; +import { createInteractionRuntime } from './interaction-route-support.ts'; +import { readSettleRequest } from './interaction-flags.ts'; import { dispatchDirectIosSelectorTap } from './interaction-touch-direct-ios.ts'; import { readDirectIosSelectorTapTarget } from './interaction-touch-direct-ios-eligibility.ts'; import { diff --git a/src/daemon/handlers/interaction-touch-reference-frame.ts b/src/daemon/interaction/internal/interaction-touch-reference-frame.ts similarity index 83% rename from src/daemon/handlers/interaction-touch-reference-frame.ts rename to src/daemon/interaction/internal/interaction-touch-reference-frame.ts index 4b6342a4ad..554060192a 100644 --- a/src/daemon/handlers/interaction-touch-reference-frame.ts +++ b/src/daemon/interaction/internal/interaction-touch-reference-frame.ts @@ -1,13 +1,12 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; import type { GestureReferenceFrame } from '@agent-device/contracts/scroll-gesture'; -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; -import type { SessionStore } from '../session-store.ts'; -import { getSnapshotReferenceFrame } from '../touch-reference-frame.ts'; -import type { SessionState } from '../types.ts'; -import type { CaptureSnapshotForSession, ContextFromFlags } from '../interaction/index.ts'; -import { isActiveProviderDevice } from '../../provider-device-runtime.ts'; +import type { SessionStore } from '../../session-store.ts'; +import { getSnapshotReferenceFrame } from '../../touch-reference-frame.ts'; +import type { SessionState } from '../../types.ts'; +import type { CaptureSnapshotForSession, ContextFromFlags } from './types.ts'; +import { isActiveProviderDevice } from '../../../provider-device-runtime.ts'; async function resolveDirectTouchReferenceFrame(params: { session: SessionState; @@ -79,12 +78,3 @@ export async function resolveDirectTouchReferenceFrameSafely(params: { return undefined; } } - -export function readSnapshotNodesReferenceFrame( - nodes: SnapshotNode[], -): GestureReferenceFrame | undefined { - return getSnapshotReferenceFrame({ - nodes, - createdAt: 0, - }); -} diff --git a/src/daemon/handlers/interaction-touch-response.ts b/src/daemon/interaction/internal/interaction-touch-response.ts similarity index 96% rename from src/daemon/handlers/interaction-touch-response.ts rename to src/daemon/interaction/internal/interaction-touch-response.ts index 4bef658d60..85ddaa6acf 100644 --- a/src/daemon/handlers/interaction-touch-response.ts +++ b/src/daemon/interaction/internal/interaction-touch-response.ts @@ -14,17 +14,15 @@ import { stripInternalInteractionDiagnostics, transformInteractionResponseData, type InteractionResponseDataTransformCommand, -} from '../../core/interaction-response-data-transform.ts'; -import { issueSettleRefs } from '../session-snapshot.ts'; -import type { RecordedTargetCapture } from '../session-target-evidence.ts'; -import type { SessionState } from '../types.ts'; -import type { CaptureSnapshotForSession, InteractionRouteInput } from '../interaction/index.ts'; -import { - readSnapshotNodesReferenceFrame, - resolveDirectTouchReferenceFrameSafely, -} from './interaction-touch-reference-frame.ts'; +} from '../../../core/interaction-response-data-transform.ts'; +import { issueSettleRefs } from '../../session-snapshot.ts'; +import type { RecordedTargetCapture } from '../../session-target-evidence.ts'; +import type { SessionState } from '../../types.ts'; +import { resolveDirectTouchReferenceFrameSafely } from './interaction-touch-reference-frame.ts'; +import { readSnapshotNodesReferenceFrame } from '../../touch-reference-frame.ts'; import { buildTouchPayload } from './interaction-touch-payload.ts'; import { interactionResultExtra } from './interaction-touch-targets.ts'; +import type { InteractionRouteExecutionInput } from './types.ts'; /** * The single construction site for interaction response payloads (ADR 0011 @@ -286,9 +284,7 @@ function composeResponseWarning( export type TargetedTouchResult = PressCommandResult | LongPressCommandResult | HoverCommandResult; export async function buildTargetedTouchResponsePayloads(params: { - params: InteractionRouteInput & { - captureSnapshotForSession: CaptureSnapshotForSession; - }; + params: InteractionRouteExecutionInput; session: SessionState; result: TargetedTouchResult; staleRefsWarning: string | undefined; diff --git a/src/daemon/handlers/interaction-touch-runtime.ts b/src/daemon/interaction/internal/interaction-touch-runtime.ts similarity index 90% rename from src/daemon/handlers/interaction-touch-runtime.ts rename to src/daemon/interaction/internal/interaction-touch-runtime.ts index becddcbfd5..0fd86eea80 100644 --- a/src/daemon/handlers/interaction-touch-runtime.ts +++ b/src/daemon/interaction/internal/interaction-touch-runtime.ts @@ -7,32 +7,28 @@ import type { } from '@agent-device/contracts/interaction'; import type { GestureReferenceFrame } from '@agent-device/contracts/scroll-gesture'; import { asAppError, normalizeError } from '@agent-device/kernel/errors'; -import { readResolvedInteractionTarget } from '../../core/interaction-outcome.ts'; -import { markSessionPartialRefsIssued } from '../session-snapshot.ts'; -import { isSessionRecording } from '../session-script-publication-capability.ts'; -import type { DaemonResponse, SessionState } from '../types.ts'; -import { - createInteractionRuntime, - finalizeTouchInteraction, - publishInteractionAmbiguityCandidates, - type CaptureSnapshotForSession, - type InteractionRouteInput, -} from '../interaction/index.ts'; +import { readResolvedInteractionTarget } from '../../../core/interaction-outcome.ts'; +import { markSessionPartialRefsIssued } from '../../session-snapshot.ts'; +import { isSessionRecording } from '../../session-script-publication-capability.ts'; +import type { DaemonResponse, SessionState } from '../../types.ts'; +import { createInteractionRuntime, finalizeTouchInteraction } from './interaction-route-support.ts'; +import { publishInteractionAmbiguityCandidates } from './interaction-ambiguity-publication.ts'; import { isAndroidEscapeError } from './interaction-android-escape.ts'; import { corroborateIosTapFailure, interactionTargetExtra } from './interaction-ios-tap-outcome.ts'; import { runWithAndroidDialogReadinessCheck, type RefAdmissionContext, } from './interaction-touch-android-readiness.ts'; -import { readSnapshotNodesReferenceFrame } from './interaction-touch-reference-frame.ts'; +import { readSnapshotNodesReferenceFrame } from '../../touch-reference-frame.ts'; import { buildCorroboratedTapResponseData, buildInteractionResponseData, pointPositionals, type InteractionResponsePayloads, } from './interaction-touch-response.ts'; -import { noActiveSessionError } from '../response.ts'; -import type { BoundTouchExecutor } from '../touch-runtime.ts'; +import { noActiveSessionError } from '../../response.ts'; +import type { BoundTouchExecutor } from '../../touch-runtime.ts'; +import type { InteractionRouteExecutionInput } from './types.ts'; /** * The lifecycle every tree-resolved touch dispatch shares: Android readiness @@ -43,9 +39,7 @@ import type { BoundTouchExecutor } from '../touch-runtime.ts'; export async function dispatchRuntimeInteraction< TResult extends PressCommandResult | FillCommandResult | LongPressCommandResult, >( - params: InteractionRouteInput & { - captureSnapshotForSession: CaptureSnapshotForSession; - }, + params: InteractionRouteExecutionInput, options: { touchExecutor: BoundTouchExecutor; androidFreshnessBaseline?: SessionState['snapshot']; @@ -141,9 +135,7 @@ export async function dispatchRuntimeInteraction< async function buildRuntimeIosCorroboratedResponse(params: { error: unknown; - handlerParams: InteractionRouteInput & { - captureSnapshotForSession: CaptureSnapshotForSession; - }; + handlerParams: InteractionRouteExecutionInput; session: SessionState; target: InteractionTarget | undefined; extra: Record | undefined; diff --git a/src/daemon/handlers/interaction-touch-targets.ts b/src/daemon/interaction/internal/interaction-touch-targets.ts similarity index 90% rename from src/daemon/handlers/interaction-touch-targets.ts rename to src/daemon/interaction/internal/interaction-touch-targets.ts index 0ce559e6d1..d0448d9c2f 100644 --- a/src/daemon/handlers/interaction-touch-targets.ts +++ b/src/daemon/interaction/internal/interaction-touch-targets.ts @@ -8,11 +8,11 @@ import { readFillTargetFromPositionals, stripAtPrefix, type DecodedFillTarget, -} from '../../core/interaction-positionals.ts'; -import type { DaemonResponse } from '../types.ts'; -import { REF_GRAMMAR_HINT, splitRefGenerationSuffix } from '@agent-device/kernel/snapshot'; +} from '../../../core/interaction-positionals.ts'; +import type { DaemonResponse } from '../../types.ts'; import { parseCoordinateTarget } from './interaction-targeting.ts'; -import { errorResponse } from '../response.ts'; +import { errorResponse } from '../../response.ts'; +import { parseVersionedRefPositional } from '../../ref-positionals.ts'; export type ParsedTouchTarget = | { ok: true; target: InteractionTarget; refGeneration?: number; durationMs?: never } @@ -24,24 +24,6 @@ export type ParsedTouchTarget = * fast paths, recording) sees exactly today's plain `@e12` ref, while the * minted generation is surfaced separately for the staleness warning. */ -type ParsedVersionedRef = - | { ok: true; ref: string; generation?: number } - | { ok: false; response: DaemonResponse }; - -export function parseVersionedRefPositional(refInput: string): ParsedVersionedRef { - const split = splitRefGenerationSuffix(refInput); - if (!split) { - return { - ok: false, - response: errorResponse( - 'INVALID_ARGS', - `Invalid ref "${refInput}" — malformed generation suffix.`, - { hint: REF_GRAMMAR_HINT }, - ), - }; - } - return { ok: true, ref: split.base, generation: split.generation }; -} export function parseTouchTarget(positionals: string[], commandLabel: string): ParsedTouchTarget { const coordinates = parseCoordinateTarget(positionals); diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/interaction/internal/interaction-touch.ts similarity index 76% rename from src/daemon/handlers/interaction-touch.ts rename to src/daemon/interaction/internal/interaction-touch.ts index 6e740d291f..93a07321b7 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/interaction/internal/interaction-touch.ts @@ -1,16 +1,11 @@ -import type { DaemonResponse } from '../types.ts'; -import type { - CaptureSnapshotForSession, - InteractionRouteInput, - RefSnapshotFlagGuardResponse, -} from '../interaction/index.ts'; +import type { DaemonResponse } from '../../types.ts'; +import type { RefSnapshotFlagGuardResponse, InteractionRouteExecutionInput } from './types.ts'; import { dispatchFillViaRuntime } from './interaction-touch-fill.ts'; import { dispatchTargetedTouchViaRuntime } from './interaction-touch-press.ts'; /** Which touch command handler owns this request; every policy lives below. */ export async function handleTouchInteractionCommands( - params: InteractionRouteInput & { - captureSnapshotForSession: CaptureSnapshotForSession; + params: InteractionRouteExecutionInput & { refSnapshotFlagGuardResponse: RefSnapshotFlagGuardResponse; }, ): Promise { diff --git a/src/daemon/handlers/interaction.ts b/src/daemon/interaction/internal/interaction.ts similarity index 85% rename from src/daemon/handlers/interaction.ts rename to src/daemon/interaction/internal/interaction.ts index a6cd9cc2e3..d64c17b1d0 100644 --- a/src/daemon/handlers/interaction.ts +++ b/src/daemon/interaction/internal/interaction.ts @@ -1,29 +1,28 @@ -import type { DaemonResponse, SessionState } from '../types.ts'; -import type { InteractionRouteInput } from '../interaction/index.ts'; +import type { DaemonResponse, SessionState } from '../../types.ts'; +import type { InteractionRouteExecutionInput } from './types.ts'; import { handleTouchInteractionCommands } from './interaction-touch.ts'; -import { - captureSnapshotForSession, - finalizeTouchInteraction, - refSnapshotFlagGuardResponse, -} from '../interaction/index.ts'; -import { dispatchGetViaRuntime, dispatchIsViaRuntime } from '../selector-runtime.ts'; -import { expireRefFrame } from '../ref-frame.ts'; -import { errorResponse, noActiveSessionError } from '../response.ts'; -import { PUBLIC_COMMANDS } from '../../command-catalog.ts'; +import { refSnapshotFlagGuardResponse } from '../../ref-snapshot-flags.ts'; +import { finalizeTouchInteraction } from './interaction-route-support.ts'; +import { dispatchGetViaRuntime, dispatchIsViaRuntime } from '../../selector-runtime.ts'; +import { expireRefFrame } from '../../ref-frame.ts'; +import { errorResponse, noActiveSessionError } from '../../response.ts'; +import { PUBLIC_COMMANDS } from '../../../command-catalog.ts'; import { normalizeError } from '@agent-device/kernel/errors'; import { ensureAndroidBlockingSystemDialogReady, recoverAndroidBlockingSystemDialog, -} from '../android-system-dialog.ts'; +} from '../../android-system-dialog.ts'; import { dispatchGestureViaRuntime, dispatchSwipeViaRuntime } from './interaction-gesture.ts'; -import { resolveBoundTypeTextRuntime, type BoundTypeTextExecutor } from '../type-text-runtime.ts'; +import { + resolveBoundTypeTextRuntime, + type BoundTypeTextExecutor, +} from '../../type-text-runtime.ts'; export async function handleInteractionCommands( - params: InteractionRouteInput, + params: InteractionRouteExecutionInput, ): Promise { const touchResponse = await handleTouchInteractionCommands({ ...params, - captureSnapshotForSession: params.captureSnapshotForSession ?? captureSnapshotForSession, refSnapshotFlagGuardResponse, }); if (touchResponse) { @@ -46,7 +45,9 @@ export async function handleInteractionCommands( } } -async function dispatchTypeViaRuntime(params: InteractionRouteInput): Promise { +async function dispatchTypeViaRuntime( + params: InteractionRouteExecutionInput, +): Promise { const { sessionName, sessionStore } = params; const session = sessionStore.get(sessionName); if (!session) return noActiveSessionError(); @@ -99,7 +100,7 @@ async function recoverAndroidRecordingDialogForType( } async function runTypeTextViaRuntime( - params: InteractionRouteInput, + params: InteractionRouteExecutionInput, session: SessionState, boundTypeText: BoundTypeTextExecutor, recordingRecoveryWarning?: string, diff --git a/src/daemon/interaction/internal/types.ts b/src/daemon/interaction/internal/types.ts index 81869b9f13..33442ac5d2 100644 --- a/src/daemon/interaction/internal/types.ts +++ b/src/daemon/interaction/internal/types.ts @@ -1,11 +1,17 @@ import type { CommandFlags } from '@agent-device/contracts/command'; +import type { AndroidObservationAdapter } from '@agent-device/contracts/android-observation'; import type { Rect, SnapshotPreferredBackend, SnapshotState } from '@agent-device/kernel/snapshot'; import type { DeviceInfo } from '@agent-device/kernel/device'; import type { CommandSessionStore } from '../../../runtime-contract.ts'; +import type { + BindDeviceRuntime, + InspectDeviceRuntimeFacts, +} from '../../request-runtime-binding.ts'; import type { DeferredInteractionOutcomeMark } from '../../deferred-interaction-outcome.ts'; +import type { DaemonInvokeFn, DaemonRequest, SessionState } from '../../types.ts'; import type { RecordActionEntry } from '../../session-action-recorder.ts'; import type { DaemonCommandContext } from '../../context.ts'; -import type { SessionState } from '../../types.ts'; +import type { SessionStore } from '../../session-store.ts'; import type { BoundGestureExecutor } from '../../gesture-runtime.ts'; import type { BoundTouchExecutor } from '../../touch-runtime.ts'; import type { BoundSnapshotCapture } from '../../snapshot-runtime-binding.ts'; @@ -17,6 +23,46 @@ export type ContextFromFlags = ( traceLogPath?: string, ) => DaemonCommandContext; +export type CaptureSnapshotForSession = ( + session: SessionState, + flags: CommandFlags | undefined, + sessionStore: SessionStore, + contextFromFlags: ContextFromFlags, + options: InteractionSnapshotOptions, +) => Promise; + +export type InteractionRouteInput = { + req: DaemonRequest; + sessionName: string; + logPath?: string; + sessionStore: SessionStore; + captureSnapshotForSession?: CaptureSnapshotForSession; + contextFromFlags: ContextFromFlags; + inspectFacts?: InspectDeviceRuntimeFacts; + bindDevice?: BindDeviceRuntime; + androidObservation?: AndroidObservationAdapter; +}; + +export type InteractionRouteExecutionInput = Omit< + InteractionRouteInput, + 'captureSnapshotForSession' +> & { + captureSnapshotForSession: CaptureSnapshotForSession; +}; + +export type FindRouteInput = { + req: DaemonRequest; + sessionName: string; + logPath: string; + sessionStore: SessionStore; + invoke: DaemonInvokeFn; + inspectFacts?: InspectDeviceRuntimeFacts; + bindDevice?: BindDeviceRuntime; +}; + +export type RefSnapshotFlagGuardResponse = + typeof import('../../ref-snapshot-flags.ts').refSnapshotFlagGuardResponse; + export type InteractionSessionView = Readonly<{ device: DeviceInfo; appBundleId?: string; diff --git a/src/daemon/ref-positionals.ts b/src/daemon/ref-positionals.ts new file mode 100644 index 0000000000..9948b8b713 --- /dev/null +++ b/src/daemon/ref-positionals.ts @@ -0,0 +1,22 @@ +import { REF_GRAMMAR_HINT, splitRefGenerationSuffix } from '@agent-device/kernel/snapshot'; +import type { DaemonResponse } from './types.ts'; +import { errorResponse } from './response.ts'; + +export type ParsedVersionedRef = + | { ok: true; ref: string; generation?: number } + | { ok: false; response: DaemonResponse }; + +export function parseVersionedRefPositional(refInput: string): ParsedVersionedRef { + const split = splitRefGenerationSuffix(refInput); + if (!split) { + return { + ok: false, + response: errorResponse( + 'INVALID_ARGS', + `Invalid ref "${refInput}" — malformed generation suffix.`, + { hint: REF_GRAMMAR_HINT }, + ), + }; + } + return { ok: true, ref: split.base, generation: split.generation }; +} diff --git a/src/daemon/ref-snapshot-flags.ts b/src/daemon/ref-snapshot-flags.ts new file mode 100644 index 0000000000..895013b364 --- /dev/null +++ b/src/daemon/ref-snapshot-flags.ts @@ -0,0 +1,30 @@ +import type { CommandFlags } from '@agent-device/contracts/command'; +import type { DaemonResponse } from './types.ts'; +import { errorResponse } from './response.ts'; + +const REF_UNSUPPORTED_FLAG_MAP: ReadonlyArray<[keyof CommandFlags, string]> = [ + ['snapshotDepth', '--depth'], + ['snapshotScope', '--scope'], + ['snapshotRaw', '--raw'], +]; + +export function refSnapshotFlagGuardResponse( + command: 'press' | 'fill' | 'get' | 'longpress' | 'hover', + flags: CommandFlags | undefined, +): DaemonResponse | null { + const unsupported = unsupportedRefSnapshotFlags(flags); + if (unsupported.length === 0) return null; + return errorResponse( + 'INVALID_ARGS', + `${command} @ref does not support ${unsupported.join(', ')}.`, + ); +} + +function unsupportedRefSnapshotFlags(flags: CommandFlags | undefined): string[] { + if (!flags) return []; + const unsupported: string[] = []; + for (const [key, label] of REF_UNSUPPORTED_FLAG_MAP) { + if (flags[key] !== undefined) unsupported.push(label); + } + return unsupported; +} diff --git a/src/daemon/request-handler-chain.ts b/src/daemon/request-handler-chain.ts index e7fa3e781a..9fc78cbac8 100644 --- a/src/daemon/request-handler-chain.ts +++ b/src/daemon/request-handler-chain.ts @@ -92,11 +92,11 @@ const DAEMON_ROUTE_HANDLERS = { run: runRecordTraceHandler, }), find: defineDaemonRoute({ - load: () => import('./handlers/find.ts'), + load: () => import('./interaction/index.ts'), run: runFindHandler, }), interaction: defineDaemonRoute({ - load: () => import('./handlers/interaction.ts'), + load: () => import('./interaction/index.ts'), run: runInteractionHandler, }), generic: defineDaemonRoute({ @@ -255,7 +255,7 @@ async function runRecordTraceHandler( } async function runFindHandler( - { handleFindCommands }: typeof import('./handlers/find.ts'), + { handleFindCommands }: typeof import('./interaction/index.ts'), params: RequestHandlerChainParams, ): Promise { return expectHandlerResponse( @@ -274,7 +274,7 @@ async function runFindHandler( } async function runInteractionHandler( - { handleInteractionCommands }: typeof import('./handlers/interaction.ts'), + { handleInteractionCommands }: typeof import('./interaction/index.ts'), params: RequestHandlerChainParams, ): Promise { return expectHandlerResponse( diff --git a/src/daemon/route-owner-files.ts b/src/daemon/route-owner-files.ts index 15923d522d..de005c0881 100644 --- a/src/daemon/route-owner-files.ts +++ b/src/daemon/route-owner-files.ts @@ -23,8 +23,8 @@ const DAEMON_ROUTE_OWNER_FILES = { snapshot: 'src/daemon/handlers/snapshot.ts', reactNative: 'src/daemon/handlers/react-native.ts', recordTrace: 'src/daemon/handlers/record-trace.ts', - find: 'src/daemon/handlers/find.ts', - interaction: 'src/daemon/handlers/interaction.ts', + find: 'src/daemon/interaction/index.ts', + interaction: 'src/daemon/interaction/index.ts', generic: 'src/daemon/request-generic-dispatch.ts', } as const satisfies Record; diff --git a/src/daemon/selector-runtime-backend.ts b/src/daemon/selector-runtime-backend.ts index 89243dcff0..6d6c18507c 100644 --- a/src/daemon/selector-runtime-backend.ts +++ b/src/daemon/selector-runtime-backend.ts @@ -12,7 +12,7 @@ import { createDaemonRuntimePolicy } from './runtime-policy.ts'; import { createDaemonRuntimeSessionStore } from './runtime-session.ts'; import { contextFromFlags } from './context.ts'; import { ensureDeviceReady } from './device-ready.ts'; -import { readTextForNode } from './interaction/index.ts'; +import { readTextForNode } from './interaction-read.ts'; import { setSessionSnapshot } from './session-snapshot.ts'; import type { ContextFromFlags } from './interaction/index.ts'; import { SessionStore } from './session-store.ts'; diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index 8694a369dd..4138eaca58 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -15,12 +15,12 @@ import { checkFindArgs, isReadOnlyFindAction, } from '@agent-device/selectors'; -import { refSnapshotFlagGuardResponse } from './interaction/index.ts'; -import { parseVersionedRefPositional } from './handlers/interaction-touch-targets.ts'; +import { refSnapshotFlagGuardResponse } from './ref-snapshot-flags.ts'; +import { parseVersionedRefPositional } from './ref-positionals.ts'; import { describeAndroidEscapeSurface, detectAndroidEscapeSurface, -} from './handlers/interaction-android-escape.ts'; +} from './android-escape-surface.ts'; import { buildFindRecordResult, buildGetRecordResult, @@ -34,7 +34,7 @@ import { import type { RecordedTargetCapture } from './session-target-evidence.ts'; import type { TargetAnnotationV1 } from '@agent-device/contracts/replay'; import { maybeWaitTimeoutSurfaceResponse } from './wait-current-surface.ts'; -import { withSystemSurfaceDisclosure } from './handlers/system-surface-disclosure.ts'; +import { withSystemSurfaceDisclosure } from './system-surface-disclosure.ts'; import { createBoundSelectorRuntime, createSelectorRuntimeForDevice, diff --git a/src/daemon/handlers/system-surface-disclosure.ts b/src/daemon/system-surface-disclosure.ts similarity index 89% rename from src/daemon/handlers/system-surface-disclosure.ts rename to src/daemon/system-surface-disclosure.ts index 283b282c0d..e3be4ddc34 100644 --- a/src/daemon/handlers/system-surface-disclosure.ts +++ b/src/daemon/system-surface-disclosure.ts @@ -1,6 +1,6 @@ import type { SnapshotState } from '@agent-device/kernel/snapshot'; -import { systemSurfaceDisclosure } from '../../core/android-system-surface-disclosure.ts'; -import type { DaemonResponse } from '../types.ts'; +import { systemSurfaceDisclosure } from '../core/android-system-surface-disclosure.ts'; +import type { DaemonResponse } from './types.ts'; /** * Append the occluding-system-surface disclosure to a selector-route response whose consumed diff --git a/src/daemon/touch-reference-frame.ts b/src/daemon/touch-reference-frame.ts index 949e5564e2..581973cf83 100644 --- a/src/daemon/touch-reference-frame.ts +++ b/src/daemon/touch-reference-frame.ts @@ -2,7 +2,7 @@ import { type GestureReferenceFrame, inferGestureReferenceFrame, } from '@agent-device/contracts/scroll-gesture'; -import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import type { SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; export type TouchReferenceFrame = GestureReferenceFrame; @@ -21,4 +21,13 @@ export function getSnapshotReferenceFrame( return inferred; } +export function readSnapshotNodesReferenceFrame( + nodes: SnapshotNode[], +): GestureReferenceFrame | undefined { + return getSnapshotReferenceFrame({ + nodes, + createdAt: 0, + }); +} + const inferTouchReferenceFrame = inferGestureReferenceFrame; diff --git a/test/integration/interaction-contract/native-ref.contract.test.ts b/test/integration/interaction-contract/native-ref.contract.test.ts index 37fc42e399..e1fd9207c2 100644 --- a/test/integration/interaction-contract/native-ref.contract.test.ts +++ b/test/integration/interaction-contract/native-ref.contract.test.ts @@ -4,7 +4,7 @@ import type { InteractionGuarantee } from '@agent-device/contracts/interaction-g import type { SnapshotState } from '@agent-device/kernel/snapshot'; import { ref } from '../../../src/commands/interaction/runtime/selector-read-utils.ts'; import { scenarioName } from './coverage-manifest.ts'; -import { buildInteractionResponseData } from '../../../src/daemon/handlers/interaction-touch-response.ts'; +import { buildInteractionResponseData } from '../../../src/daemon/interaction/internal/interaction-touch-response.ts'; import { NATIVE_REF_COVERAGE } from './native-ref.coverage.ts'; import { closedDrawerSnapshot,