diff --git a/CHANGELOG.md b/CHANGELOG.md index 069d1fd4a..f7b75f5c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- Breaking (0.21): iOS Appium/WebDriver snapshots now expose engine-owned acquisition facts and + typed fidelity warnings. The SDK snapshot `truncated` field is optional when Appium cannot report + hierarchy completeness; regular snapshots fail closed without valid viewport evidence, while + `snapshot --raw` remains available for diagnostics (#2195). - Security (daemon, remote/proxy HTTP only): when `AGENT_DEVICE_HTTP_AUTH_HOOK` is configured and a request's hook result does not attest a `tenantId`, the request is now refused (401) outright — the daemon no longer runs it as whichever tenant the client declared (RPC body `meta.tenantId` or diff --git a/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts b/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts index 8afcd9fee..c590b0741 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts @@ -7,7 +7,7 @@ import { createIosSnapshotRequest, deriveIosCaptureHint, } from '@agent-device/capture-kit/ios-snapshot-planning'; -import { IosSnapshotEngineError, presentIosSnapshot } from './index.ts'; +import { IosSnapshotEngineError, presentIosSnapshot, publishIosSnapshot } from './index.ts'; import { runTypeScriptCase, writeDifferentialFailureArtifact } from './conformance-harness.ts'; import { acquisitionForGoldenCase, @@ -86,7 +86,7 @@ test('the independent iOS snapshot goldens match the TypeScript engine', () => { | { outcome: 'success'; nodes: ReturnType; - truncated: boolean; + truncated?: boolean; residue: typeof acquisition.residue; qualityLabels?: readonly (string | null)[]; } @@ -97,16 +97,21 @@ test('the independent iOS snapshot goldens match the TypeScript engine', () => { }; try { - const result = presentIosSnapshot({ stage: 'acquired', acquisition }, request, { + const presentation = presentIosSnapshot({ stage: 'acquired', acquisition }, request, { + foldPolicy: testCase.foldPolicy, + }); + const publication = publishIosSnapshot({ stage: 'acquired', acquisition }, request, { foldPolicy: testCase.foldPolicy, }); actual = { outcome: 'success', - nodes: normalizeGoldenNodes(result.nodes), - truncated: acquisition.truncated, - residue: acquisition.residue, + nodes: normalizeGoldenNodes(publication.payload.nodes), + ...(publication.payload.truncated === undefined + ? {} + : { truncated: publication.payload.truncated }), + residue: publication.residue, ...(testCase.qualityLabels - ? { qualityLabels: result.qualityNodes?.map((node) => node.label ?? null) } + ? { qualityLabels: presentation.qualityNodes?.map((node) => node.label ?? null) } : {}), }; } catch (error) { @@ -125,6 +130,21 @@ test('the independent iOS snapshot goldens match the TypeScript engine', () => { } }); +test('published payload omits unknown truncation instead of defaulting it', () => { + const fixture = readIosSnapshotEngineFixture(); + const testCase = fixture.cases[0]!; + const request = requestForGoldenCase(testCase); + const acquisition = { + ...acquisitionForGoldenCase(fixture, testCase), + truncated: undefined, + }; + + const publication = publishIosSnapshot({ stage: 'acquired', acquisition }, request); + + assert.equal(publication.payload.truncated, undefined); + assert.equal('truncated' in publication.payload, false); +}); + test('the differential TypeScript runner preserves typed failures', () => { const fixture = readIosSnapshotEngineFixture(); const source = fixture.cases.find( diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts index 3e0941987..50de132b2 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts @@ -11,12 +11,14 @@ import { createIosSnapshotRequest, deriveIosCaptureHint, } from '@agent-device/capture-kit/ios-snapshot-planning'; +import { toIosSnapshotEngineErrorDetails } from './types.ts'; import { compactIosInteractiveSnapshot, createIosSnapshotEngine, IosSnapshotEngineError, presentIosSnapshot, publishIosSnapshot, + resolveIosViewportEvidenceFromRoots, } from './index.ts'; import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; @@ -189,6 +191,65 @@ test('regular presentation fails typed when the viewport is missing or the graph ); }); +test('engine error details preserve typed public context', () => { + const error = new IosSnapshotEngineError('invalid-viewport', 'invalid viewport', { + field: 'viewport', + index: 4, + parentIndex: 2, + frame: { x: 0, y: 0, width: 10, height: 10 }, + clip: { x: 1, y: 2, width: 3, height: 4 }, + projection: 'regular', + }); + + assert.deepEqual(toIosSnapshotEngineErrorDetails(error), { + reason: 'invalid-viewport', + field: 'viewport', + index: 4, + parentIndex: 2, + frame: { x: 0, y: 0, width: 10, height: 10 }, + clip: { x: 1, y: 2, width: 3, height: 4 }, + projection: 'regular', + }); +}); + +test('viewport evidence prefers reported application and window roots', () => { + assert.deepEqual( + resolveIosViewportEvidenceFromRoots([ + { type: 'XCUIElementTypeApplication', rectStatus: 'invalid' }, + { + type: 'XCUIElementTypeWindow', + rect: { x: 0, y: 0, width: 390, height: 844 }, + rectStatus: 'reported', + }, + ]), + { kind: 'reported', rect: { x: 0, y: 0, width: 390, height: 844 } }, + ); +}); + +test('viewport evidence can fall back to the largest top-level root', () => { + assert.deepEqual( + resolveIosViewportEvidenceFromRoots( + [ + { type: 'Other', rect: { x: 0, y: 0, width: 100, height: 100 } }, + { type: 'Other', rect: { x: 0, y: 0, width: 200, height: 300 } }, + ], + { fallbackToLargestRoot: true }, + ), + { kind: 'reported', rect: { x: 0, y: 0, width: 200, height: 300 } }, + ); +}); + +test('viewport evidence preserves explicit missing geometry reasons', () => { + assert.deepEqual( + resolveIosViewportEvidenceFromRoots([{ type: 'Application', rectStatus: 'invalid' }]), + { kind: 'missing', reason: 'invalid' }, + ); + assert.deepEqual( + resolveIosViewportEvidenceFromRoots([{ type: 'Application', rectStatus: 'not-provided' }]), + { kind: 'missing', reason: 'not-provided' }, + ); +}); + test('presented runner payloads and optional quality payloads cross the host invariant', () => { const request = createIosSnapshotRequest(); const presentedCapture = publishIosSnapshot(acquiredInput(request, nestedNodes()), request); @@ -234,10 +295,12 @@ test('unavailable hittability never becomes regular actionability', () => { residue: [{ kind: 'unavailable-fact' as const, fact: 'hittability' as const }], } satisfies IosSnapshotAcquisition; const acquired = publishIosSnapshot({ stage: 'acquired', acquisition: unavailable }, request); - assert.equal( - acquired.payload.nodes.find((node) => node.label === 'Partially visible')?.hittable, - false, + const partiallyVisible = acquired.payload.nodes.find( + (node) => node.label === 'Partially visible', ); + assert.ok(partiallyVisible); + assert.equal(partiallyVisible.hittable, undefined); + assert.equal('hittable' in partiallyVisible, false); const available = publishIosSnapshot(acquiredInput(request, nestedNodes()), request); const presented: IosSnapshotInput = { diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.ts index 877c6942b..c4a44d4ce 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.ts @@ -11,8 +11,11 @@ import type { IosSnapshotInput, IosSnapshotPublication, IosSnapshotRequest, + IosViewportEvidence, } from '@agent-device/contracts/ios-snapshot'; -import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { normalizeType } from '@agent-device/contracts/snapshot'; +import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import { attachRefs, type RawSnapshotNode, type Rect } from '@agent-device/kernel/snapshot'; import { buildIosInteractiveSnapshotPresentation } from './semantic-index.ts'; import { validateIosSnapshotGraph } from './graph.ts'; import { foldIosSnapshot } from './geometry.ts'; @@ -28,6 +31,52 @@ import { IosSnapshotEngineError } from './types.ts'; const DEFAULT_FOLD_POLICY: IosSnapshotFoldPolicy = 'cursor-projected'; +export type IosSnapshotViewportRoot = Readonly<{ + type?: string; + rect?: Rect; + rectStatus?: 'reported' | 'invalid' | 'not-provided'; +}>; + +export function resolveIosViewportEvidenceFromRoots( + roots: readonly IosSnapshotViewportRoot[], + options: Readonly<{ fallbackToLargestRoot?: boolean }> = {}, +): IosViewportEvidence | undefined { + const viewportRoots = roots.filter(isViewportRoot); + const candidates = + viewportRoots.length > 0 || options.fallbackToLargestRoot !== true ? viewportRoots : roots; + const root = [...candidates].sort(compareViewportRoots)[0]; + if (!root) return undefined; + if (isPositiveFiniteRect(root.rect)) return { kind: 'reported', rect: root.rect }; + return { + kind: 'missing', + reason: + root.rectStatus === 'invalid' || (root.rectStatus === undefined && root.rect !== undefined) + ? 'invalid' + : 'not-provided', + }; +} + +function isViewportRoot(root: IosSnapshotViewportRoot): boolean { + const type = normalizeType(root.type ?? ''); + return type === 'application' || type === 'window'; +} + +function compareViewportRoots( + left: IosSnapshotViewportRoot, + right: IosSnapshotViewportRoot, +): number { + const status = rootGeometryRank(right.rectStatus) - rootGeometryRank(left.rectStatus); + return status || rectArea(right.rect) - rectArea(left.rect); +} + +function rootGeometryRank(status: IosSnapshotViewportRoot['rectStatus']): number { + return status === 'reported' ? 2 : status === 'invalid' ? 1 : 0; +} + +function rectArea(rect: Rect | undefined): number { + return rect && isPositiveFiniteRect(rect) ? rect.width * rect.height : 0; +} + export function createIosSnapshotEngine(options: IosSnapshotEngineOptions = {}): IosSnapshotEngine { const foldPolicy = options.foldPolicy ?? DEFAULT_FOLD_POLICY; return Object.freeze({ @@ -46,13 +95,12 @@ export function publishIosSnapshot( input.stage === 'presented' ? input.validation.presentationKey : buildIosSnapshotPresentationKey(request); + const truncated = + input.stage === 'acquired' ? input.acquisition.truncated : input.presentation.payload.truncated; return { payload: { nodes: attachRefs(presentation.nodes), - truncated: - input.stage === 'acquired' - ? input.acquisition.truncated - : input.presentation.payload.truncated, + ...(truncated === undefined ? {} : { truncated }), }, presentationKey, comparisonIdentity: buildIosSnapshotComparisonIdentity(input, request), diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts index 6a8365682..3c1406e94 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts @@ -107,17 +107,21 @@ function appendFoldedNode( const index = kept.length; keptDepth += 1; + const { hittable: sourceHittable, ...sourceNode } = node; kept.push({ raw: { - ...node, + ...sourceNode, index, depth: keptDepth, parentIndex: keptIndex, - hittable: - node.parentIndex !== undefined && - options.hittabilityAvailable !== false && - node.hittable === true && - isGeometricallyActionable(node.enabled !== false, decision.effectiveRect, viewport), + ...foldedHittability( + sourceHittable, + node.parentIndex !== undefined, + decision.effectiveRect, + node.enabled !== false, + viewport, + options, + ), }, sourceIndex: node.index, ...(decision.effectiveRect ? { effectiveRect: decision.effectiveRect } : {}), @@ -126,6 +130,25 @@ function appendFoldedNode( return { keptIndex, keptDepth }; } +function foldedHittability( + sourceHittable: RawSnapshotNode['hittable'], + hasParent: boolean, + effectiveRect: Rect | undefined, + enabled: boolean, + viewport: Rect, + options: IosSnapshotFoldOptions, +): Partial> { + if (options.hittabilityAvailable === false) { + return sourceHittable === false ? { hittable: false } : {}; + } + return { + hittable: + hasParent && + sourceHittable === true && + isGeometricallyActionable(enabled, effectiveRect, viewport), + }; +} + function nextScrollAnchor( decision: GeometryDecision, parentAnchor: BranchState['anchor'], diff --git a/packages/capture-kit/src/ios-snapshot-engine/index.ts b/packages/capture-kit/src/ios-snapshot-engine/index.ts index 0c13dc24f..5e182c722 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/index.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/index.ts @@ -3,6 +3,7 @@ export { createIosSnapshotEngine, presentIosSnapshot, publishIosSnapshot, + resolveIosViewportEvidenceFromRoots, } from './engine.ts'; export { presentIosRunnerSnapshot } from './runner-presentation.ts'; export { @@ -11,5 +12,5 @@ export { } from './semantic-index.ts'; export { collectIosStructuralIdentifierSuppression } from './noise-structural.ts'; export { findNearestScrollableContainer, mergeReplacement, updateReplacement } from './tree.ts'; -export { IosSnapshotEngineError } from './types.ts'; +export { IosSnapshotEngineError, toIosSnapshotEngineErrorDetails } from './types.ts'; export type { SnapshotTreeRuleContext } from './tree.ts'; diff --git a/packages/capture-kit/src/ios-snapshot-engine/projection.ts b/packages/capture-kit/src/ios-snapshot-engine/projection.ts index d0a10ba72..1eea69c80 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/projection.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/projection.ts @@ -142,17 +142,19 @@ function createRegularProjectedNode( if (!isEligibleForIosRegularPresentation(node.raw)) return undefined; const depth = parent ? (parent.depth ?? 0) + 1 : 0; if (maximumDepth !== null && depth > maximumDepth) return undefined; + const hittable = isProjectedNodeHittable(node); return { ...node.raw, index, depth, parentIndex: parent?.index, ...(node.effectiveRect ? { rect: node.effectiveRect } : { rect: undefined }), - hittable: isProjectedNodeHittable(node), + ...(hittable === undefined ? {} : { hittable }), }; } -function isProjectedNodeHittable(node: IosSnapshotPresentationNode): boolean { +function isProjectedNodeHittable(node: IosSnapshotPresentationNode): RawSnapshotNode['hittable'] { + if (node.raw.hittable === undefined) return undefined; return Boolean( node.raw.hittable === true && node.effectiveRect && diff --git a/packages/capture-kit/src/ios-snapshot-engine/types.ts b/packages/capture-kit/src/ios-snapshot-engine/types.ts index 7468d0b9f..29e4fb5bd 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/types.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/types.ts @@ -64,3 +64,13 @@ export class IosSnapshotEngineError extends Error { this.details = details; } } + +export type IosSnapshotEnginePublicErrorDetails = Readonly< + IosSnapshotEngineFailureDetails & { reason: IosSnapshotEngineFailureReason } +>; + +export function toIosSnapshotEngineErrorDetails( + error: IosSnapshotEngineError, +): IosSnapshotEnginePublicErrorDetails { + return { reason: error.reason, ...error.details }; +} diff --git a/packages/capture-kit/src/ios-snapshot-planning.test.ts b/packages/capture-kit/src/ios-snapshot-planning.test.ts index 4aadf3d73..8711e60a4 100644 --- a/packages/capture-kit/src/ios-snapshot-planning.test.ts +++ b/packages/capture-kit/src/ios-snapshot-planning.test.ts @@ -16,6 +16,7 @@ import { buildIosSnapshotPresentationKey, createIosSnapshotRequest, deriveIosCaptureHint, + deriveIosSnapshotCapabilityResidue, planIosSnapshot, } from '@agent-device/capture-kit/ios-snapshot-planning'; @@ -147,6 +148,24 @@ test('presented producers cannot claim acquisition narrowing', () => { }); }); +test('Appium source plan carries its viewport evidence capability', () => { + const plan = planIosSnapshot( + createIosSnapshotRequest(), + IOS_SNAPSHOT_PRODUCER_CAPABILITIES['appium-source'], + ); + assert.equal(plan.evidence.viewport, 'available'); +}); + +test('capability residue derives unavailable Appium facts from the registry', () => { + assert.deepEqual( + deriveIosSnapshotCapabilityResidue(IOS_SNAPSHOT_PRODUCER_CAPABILITIES['appium-source']), + [ + { kind: 'unavailable-fact', fact: 'hittability' }, + { kind: 'unavailable-fact', fact: 'acquisition-depth' }, + ], + ); +}); + test('comparison identity rejects every identity axis and residue mismatch', () => { const base = comparisonIdentity(); const mismatches: IosSnapshotComparisonIdentity[] = [ @@ -223,6 +242,7 @@ function acquiredProducer( interactiveQueryCompleteness: 'incomplete', viewportEvidence: 'available', hittabilityEvidence: 'available', + presentationOwner: 'snapshot-state', ...overrides, }; } diff --git a/packages/capture-kit/src/ios-snapshot-planning.ts b/packages/capture-kit/src/ios-snapshot-planning.ts index 37fa1513f..d8f826898 100644 --- a/packages/capture-kit/src/ios-snapshot-planning.ts +++ b/packages/capture-kit/src/ios-snapshot-planning.ts @@ -24,6 +24,7 @@ const IOS_SNAPSHOT_PRODUCER_CAPABILITY_VALUES = { interactiveQueryCompleteness: 'complete', viewportEvidence: 'available', hittabilityEvidence: 'available', + presentationOwner: 'ios-snapshot-engine', }, 'simulator-ax-bridge': { producer: 'simulator-ax-bridge', @@ -36,6 +37,7 @@ const IOS_SNAPSHOT_PRODUCER_CAPABILITY_VALUES = { interactiveQueryCompleteness: 'incomplete', viewportEvidence: 'available', hittabilityEvidence: 'available', + presentationOwner: 'snapshot-state', }, 'appium-source': { producer: 'appium-source', @@ -46,8 +48,9 @@ const IOS_SNAPSHOT_PRODUCER_CAPABILITY_VALUES = { }, scopeCompleteness: 'incomplete', interactiveQueryCompleteness: 'incomplete', - viewportEvidence: 'unavailable', + viewportEvidence: 'available', hittabilityEvidence: 'unavailable', + presentationOwner: 'ios-snapshot-engine', }, 'limrun-ios-tree': { producer: 'limrun-ios-tree', @@ -60,6 +63,7 @@ const IOS_SNAPSHOT_PRODUCER_CAPABILITY_VALUES = { interactiveQueryCompleteness: 'incomplete', viewportEvidence: 'available', hittabilityEvidence: 'unavailable', + presentationOwner: 'snapshot-state', }, } as const satisfies Record; @@ -67,6 +71,23 @@ export const IOS_SNAPSHOT_PRODUCER_CAPABILITIES: Readonly< Record > = Object.freeze(IOS_SNAPSHOT_PRODUCER_CAPABILITY_VALUES); +export function deriveIosSnapshotCapabilityResidue( + producer: IosSnapshotProducerCapabilities, +): readonly IosAcquisitionResidue[] { + const residue: IosAcquisitionResidue[] = []; + if (producer.hittabilityEvidence === 'unavailable') { + residue.push({ kind: 'unavailable-fact', fact: 'hittability' }); + } + if ( + producer.stage === 'acquired' && + (producer.acquisitionDepth.rawTraversal.kind === 'incomplete' || + producer.acquisitionDepth.regularPresented.kind === 'incomplete') + ) { + residue.push({ kind: 'unavailable-fact', fact: 'acquisition-depth' }); + } + return Object.freeze(residue); +} + export function createIosSnapshotRequest(input: IosSnapshotRequestInput = {}): IosSnapshotRequest { return Object.freeze({ projection: input.projection ?? (input.raw === true ? 'raw' : 'regular'), diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index 7d1ea71fd..04c2df491 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -37,7 +37,7 @@ export type CaptureSnapshotOptions = AgentDeviceRequestOverrides & export type CaptureSnapshotResult = { nodes: SnapshotNode[]; - truncated: boolean; + truncated?: boolean; appName?: string; appBundleId?: string; visibility?: SnapshotVisibility; diff --git a/packages/contracts/src/ios-snapshot.ts b/packages/contracts/src/ios-snapshot.ts index 227300931..95b0fead3 100644 --- a/packages/contracts/src/ios-snapshot.ts +++ b/packages/contracts/src/ios-snapshot.ts @@ -11,6 +11,7 @@ export type IosAcquisitionIntent = 'full' | 'surface-observation'; export type IosSnapshotProjection = 'regular' | 'raw'; export type IosSnapshotCompleteness = 'complete' | 'incomplete'; export type IosSnapshotEvidenceAvailability = 'available' | 'unavailable'; +export type IosSnapshotPresentationOwner = 'ios-snapshot-engine' | 'snapshot-state'; export type IosSnapshotGeneration = string; @@ -79,6 +80,7 @@ type IosSnapshotProducerCapabilityFacts = Readonly<{ interactiveQueryCompleteness: IosSnapshotCompleteness; viewportEvidence: IosSnapshotEvidenceAvailability; hittabilityEvidence: IosSnapshotEvidenceAvailability; + presentationOwner: IosSnapshotPresentationOwner; }>; export type IosSnapshotAcquisitionProducerCapabilities = IosSnapshotProducerCapabilityFacts & @@ -148,7 +150,7 @@ type IosSnapshotAcquisitionForIntent = Read intent: Intent; hint: CaptureHint & Readonly<{ acquisitionIntent: Intent }>; nodes: readonly RawSnapshotNode[]; - truncated: boolean; + truncated?: boolean; viewport: IosViewportEvidence; lineage: IosSnapshotLineage; residue: readonly IosAcquisitionResidue[]; @@ -216,7 +218,7 @@ export type IosSnapshotPlan = Readonly<{ export type IosSnapshotPublishedPayload = Readonly<{ nodes: readonly SnapshotNode[]; - truncated: boolean; + truncated?: boolean; }>; export type IosSnapshotComparisonIdentity = Readonly<{ diff --git a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts index d4a242924..6b64b3ba1 100644 --- a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts @@ -275,13 +275,12 @@ test('snapshot reports typed runner presentation failures', async () => { { runCommand: async () => ({ nodes: [{ index: 0, type: 'Application' }] }) }, ); - await assert.rejects( - interactor.snapshot(), - (error: unknown) => - error instanceof AppError && - error.code === 'COMMAND_FAILED' && - error.details?.reason === 'missing-viewport', - ); + await assert.rejects(interactor.snapshot(), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.deepEqual(error.details, { reason: 'missing-viewport', field: 'viewport' }); + return true; + }); }); test('sparse runner payloads with no viewport fail before publishing actionable nodes', async () => { diff --git a/packages/platform-apple/src/runner/snapshot-presentation.ts b/packages/platform-apple/src/runner/snapshot-presentation.ts index 87473d2a3..d1a54f11d 100644 --- a/packages/platform-apple/src/runner/snapshot-presentation.ts +++ b/packages/platform-apple/src/runner/snapshot-presentation.ts @@ -4,17 +4,17 @@ import type { IosViewportEvidence, } from '@agent-device/contracts/ios-snapshot'; import type { SnapshotOptions } from '@agent-device/contracts/interactor-types'; -import { normalizeType } from '@agent-device/contracts/snapshot'; import { IosSnapshotEngineError, presentIosRunnerSnapshot, + resolveIosViewportEvidenceFromRoots, + toIosSnapshotEngineErrorDetails, } from '@agent-device/capture-kit/ios-snapshot-engine'; import { readSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import { createIosSnapshotRequest, buildIosSnapshotPresentationKey, } from '@agent-device/capture-kit/ios-snapshot-planning'; -import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import { AppError } from '@agent-device/kernel/errors'; import type { RawSnapshotNode, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; @@ -119,36 +119,18 @@ function runnerViewportEvidence( qualityNodes: readonly RawSnapshotNode[] | undefined, ): IosViewportEvidence { return ( - readReportedViewport(qualityNodes) ?? - readReportedViewport(nodes) ?? { kind: 'missing', reason: 'not-provided' } + resolveIosViewportEvidenceFromRoots(rootNodes(qualityNodes), { + fallbackToLargestRoot: true, + }) ?? + resolveIosViewportEvidenceFromRoots(rootNodes(nodes), { fallbackToLargestRoot: true }) ?? { + kind: 'missing', + reason: 'not-provided', + } ); } -function readReportedViewport( - nodes: readonly RawSnapshotNode[] | undefined, -): IosViewportEvidence | undefined { - const roots = nodes?.filter((node) => node.parentIndex === undefined) ?? []; - const root = - [...roots] - .filter((node) => isViewportRoot(node)) - .sort(compareRectArea) - .at(0) ?? [...roots].sort(compareRectArea).at(0); - if (!root) return undefined; - if (isPositiveFiniteRect(root.rect)) return { kind: 'reported', rect: root.rect }; - return { kind: 'missing', reason: root.rect ? 'invalid' : 'not-provided' }; -} - -function isViewportRoot(node: RawSnapshotNode): boolean { - const type = normalizeType(node.type ?? ''); - return type === 'application' || type === 'window'; -} - -function compareRectArea(left: RawSnapshotNode, right: RawSnapshotNode): number { - return rectArea(right.rect) - rectArea(left.rect); -} - -function rectArea(rect: RawSnapshotNode['rect']): number { - return rect ? rect.width * rect.height : 0; +function rootNodes(nodes: readonly RawSnapshotNode[] | undefined): readonly RawSnapshotNode[] { + return nodes?.filter((node) => node.parentIndex === undefined) ?? []; } function isRecord(value: unknown): value is Record { @@ -160,7 +142,7 @@ function throwSnapshotEngineError(error: unknown): never { throw new AppError( 'COMMAND_FAILED', error.message, - { reason: error.reason, iosSnapshotEngine: { details: error.details } }, + toIosSnapshotEngineErrorDetails(error), error, ); } diff --git a/packages/provider-webdriver/src/runtime-session.ts b/packages/provider-webdriver/src/runtime-session.ts index 00611a975..cce3f7ca8 100644 --- a/packages/provider-webdriver/src/runtime-session.ts +++ b/packages/provider-webdriver/src/runtime-session.ts @@ -113,6 +113,7 @@ export class WebDriverSessionManager { client, backend: snapshotBackendForPlatform(prepared.platform), capabilities, + targetId: device.id, }), }), ); diff --git a/packages/provider-webdriver/src/webdriver-interactor.test.ts b/packages/provider-webdriver/src/webdriver-interactor.test.ts index e75c821d8..8d29f5393 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.test.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.test.ts @@ -249,6 +249,46 @@ test('fill refuses empty text as an unsupported clear rather than a vacuous succ assert.deepEqual(world.transcript, []); }); +test('iOS WebDriver interactor routes snapshots through the acquisition adapter', async () => { + const source = vi.fn( + async () => + '', + ); + const interactor = createWebDriverInteractor({ + client: { source } as unknown as WebDriverClient, + backend: 'xctest', + capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'ios' }), + targetId: 'ios-1', + }); + + const result = await interactor.snapshot({ raw: true, depth: 1 }); + + assert.equal(result.backend, 'xctest'); + assert.equal(result.producer, 'appium-source'); + assert.equal(source.mock.calls.length, 1); + assert.equal(result.nodes?.[0]?.type, 'XCUIElementTypeApplication'); +}); + +test('Android WebDriver interactor keeps legacy-derived source facts at its call site', async () => { + const source = vi.fn( + async () => + '', + ); + const interactor = createWebDriverInteractor({ + client: { source } as unknown as WebDriverClient, + backend: 'android', + capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'android' }), + }); + + const result = await interactor.snapshot(); + + assert.equal(result.backend, 'android'); + assert.equal(result.nodes?.[0]?.type, 'hierarchy'); + assert.equal(result.nodes?.[1]?.type, 'android.widget.Button'); + assert.equal(result.nodes?.[1]?.hittable, true); + assert.equal(source.mock.calls.length, 1); +}); + async function runFill(world: ReturnType) { vi.useFakeTimers(); try { diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index 0b49fa7b3..e9f22ce90 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -99,25 +99,34 @@ export type WebDriverInteractorOptions = { client: WebDriverClient; backend: Extract; capabilities: CloudWebDriverProviderCapabilities; + targetId?: string; }; export function createWebDriverInteractor(options: WebDriverInteractorOptions): Interactor { - return new WebDriverInteractor(options.client, options.backend, options.capabilities); + return new WebDriverInteractor( + options.client, + options.backend, + options.capabilities, + options.targetId, + ); } class WebDriverInteractor implements Interactor { private readonly client: WebDriverClient; private readonly backend: Extract; private readonly capabilities: CloudWebDriverProviderCapabilities; + private readonly targetId: string | undefined; constructor( client: WebDriverClient, backend: Extract, capabilities: CloudWebDriverProviderCapabilities, + targetId?: string, ) { this.client = client; this.backend = backend; this.capabilities = capabilities; + this.targetId = targetId; } async open( @@ -292,14 +301,16 @@ class WebDriverInteractor implements Interactor { await this.client.screenshot(outPath); } - async snapshot(_options?: SnapshotOptions): Promise { + async snapshot(options?: SnapshotOptions): Promise { this.requireSupport('snapshot'); - // Spelled as a correlated pair per channel so the SnapshotProvenance union accepts it. + if (this.backend === 'xctest') { + const { captureWebDriverIosSnapshot } = await import('./webdriver-ios-snapshot.ts'); + return await captureWebDriverIosSnapshot(this.client, options, this.targetId); + } return { - ...(this.backend === 'xctest' - ? { backend: 'xctest' as const, producer: 'appium-source' as const } - : { backend: 'android' as const, producer: 'appium-source' as const }), - nodes: parseWebDriverSource(await this.client.source()), + backend: 'android' as const, + producer: 'appium-source' as const, + nodes: parseWebDriverSource(await this.client.source(), { mode: 'legacy-derived' }), }; } @@ -496,9 +507,10 @@ class WebDriverInteractor implements Interactor { } private async scrollGestureFrame(): Promise { + const sourceMode = this.backend === 'xctest' ? 'facts' : 'legacy-derived'; const sourceFrame = await this.client .source() - .then((source) => scrollFrameFromWebDriverSource(source)) + .then((source) => scrollFrameFromWebDriverSource(source, { mode: sourceMode })) .catch(() => undefined); if (sourceFrame) return sourceFrame; return await this.client.windowRect(); diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts new file mode 100644 index 000000000..0147f7b7a --- /dev/null +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -0,0 +1,219 @@ +import assert from 'node:assert/strict'; +import { test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { + acquireWebDriverIosSnapshot, + captureWebDriverIosSnapshot, + publishWebDriverIosSnapshot, +} from './webdriver-ios-snapshot.ts'; + +const SOURCE = ` + + + + + +`; + +test('Appium iOS snapshots acquire facts and publish regular output through the engine', async () => { + const source = vi.fn(async () => SOURCE); + const result = await captureWebDriverIosSnapshot({ source }, undefined, 'cloud-ios-1'); + + assert.equal(result.backend, 'xctest'); + assert.equal(result.producer, 'appium-source'); + assert.equal(result.truncated, undefined); + assert.deepEqual( + result.nodes?.map((node) => [node.type, node.label, node.parentIndex]), + [ + ['XCUIElementTypeApplication', 'Example', undefined], + ['XCUIElementTypeOther', 'Content', 0], + ['XCUIElementTypeButton', 'Continue', 1], + ], + ); + assert.equal(result.nodes?.find((node) => node.label === 'Continue')?.hittable, undefined); + assert.equal(source.mock.calls.length, 1); + assert.ok(result.warnings?.some((warning) => warning.includes('hittability evidence'))); + assert.deepEqual(Object.keys(result).sort(), ['backend', 'nodes', 'producer', 'warnings']); + assert.equal( + result.nodes?.every((node) => !('ref' in node)), + true, + ); +}); + +test('Appium iOS options become an engine plan and engine-owned projection', () => { + const acquired = acquireWebDriverIosSnapshot( + SOURCE, + { + raw: true, + interactiveOnly: true, + depth: 1, + scope: 'Continue', + customActions: true, + }, + 'ios-2', + ); + + assert.equal(acquired.input.stage, 'acquired'); + assert.deepEqual(acquired.plan.narrowing, { + depth: null, + scope: null, + interactiveOnly: false, + }); + assert.deepEqual(acquired.input.acquisition.hint, { + projection: 'raw', + rawTraversalDepth: null, + regularPresentedDepth: null, + interactiveOnly: false, + customActions: true, + acquisitionIntent: 'full', + }); + assert.deepEqual(acquired.input.acquisition.lineage, { targetId: 'ios-2' }); + assert.deepEqual(acquired.input.acquisition.viewport, { + kind: 'reported', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }); + + const published = publishWebDriverIosSnapshot(acquired); + assert.deepEqual( + published.result.nodes?.map((node) => [node.type, node.label, node.depth, node.parentIndex]), + [['XCUIElementTypeButton', 'Continue', 0, undefined]], + ); + assert.deepEqual(published.publication.comparisonIdentity.lineage, { targetId: 'ios-2' }); +}); + +test('Appium regular presentation omits unavailable hittability while raw preserves supplied facts', async () => { + const source = + '' + + '' + + ''; + + const regular = await captureWebDriverIosSnapshot({ source: async () => source }); + const regularButton = regular.nodes?.find((node) => node.label === 'Continue'); + assert.ok(regularButton); + assert.equal(regularButton?.hittable, undefined); + assert.equal('hittable' in regularButton, false); + + const raw = await captureWebDriverIosSnapshot({ source: async () => source }, { raw: true }); + assert.equal(raw.nodes?.find((node) => node.label === 'Continue')?.hittable, true); + assert.equal( + raw.warnings?.some((warning) => warning.includes('absent hittable means no evidence')), + true, + ); + assert.ok(raw.warnings?.some((warning) => warning.includes('provider-side depth'))); +}); + +test('Appium iOS interactive requests stay provider-unpruned and use engine presentation', () => { + const acquired = acquireWebDriverIosSnapshot(SOURCE, { interactiveOnly: true }, 'ios-1'); + + assert.equal(acquired.input.acquisition.hint.interactiveOnly, true); + assert.equal(acquired.plan.narrowing.interactiveOnly, false); + const published = publishWebDriverIosSnapshot(acquired); + assert.equal( + published.result.nodes?.some((node) => node.label === 'Continue'), + true, + ); + assert.equal( + published.result.nodes?.find((node) => node.label === 'Continue')?.hittable, + undefined, + ); +}); + +test('Appium iOS regular presentation fails typed when page source has no viewport', () => { + const acquired = acquireWebDriverIosSnapshot( + '', + ); + + assert.deepEqual(acquired.input.acquisition.viewport, { + kind: 'missing', + reason: 'not-provided', + }); + assert.deepEqual(acquired.input.acquisition.residue, [ + { kind: 'unavailable-fact', fact: 'hittability' }, + { kind: 'unavailable-fact', fact: 'acquisition-depth' }, + { kind: 'missing-viewport', reason: 'not-provided' }, + ]); + assert.throws( + () => publishWebDriverIosSnapshot(acquired), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.deepEqual(error.details, { + reason: 'missing-viewport', + field: 'viewport', + hint: 'Use snapshot --raw to inspect the acquired Appium tree; regular presentation requires valid viewport evidence.', + }); + return true; + }, + ); +}); + +test('Appium iOS raw presentation discloses missing viewport without failing', async () => { + const result = await captureWebDriverIosSnapshot( + { + source: async () => + '', + }, + { raw: true }, + ); + + assert.equal( + result.nodes?.some((node) => node.label === 'Continue'), + true, + ); + assert.ok(result.warnings?.some((warning) => warning.includes('cannot be validated'))); +}); + +test('Appium iOS does not promote a non-viewport root rectangle to viewport evidence', () => { + const acquired = acquireWebDriverIosSnapshot( + '', + ); + + assert.deepEqual(acquired.input.acquisition.viewport, { + kind: 'missing', + reason: 'not-provided', + }); +}); + +test('Appium iOS regular presentation fails typed when root geometry is invalid', () => { + const acquired = acquireWebDriverIosSnapshot( + '', + ); + + assert.deepEqual(acquired.input.acquisition.viewport, { + kind: 'missing', + reason: 'invalid', + }); + assert.throws( + () => publishWebDriverIosSnapshot(acquired), + (error: unknown) => error instanceof AppError && error.details?.reason === 'invalid-viewport', + ); +}); + +test('Appium iOS regular presentation fails typed for a zero-size viewport', () => { + const acquired = acquireWebDriverIosSnapshot( + '', + ); + + assert.deepEqual(acquired.input.acquisition.viewport, { + kind: 'missing', + reason: 'invalid', + }); + assert.throws( + () => publishWebDriverIosSnapshot(acquired), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'invalid-viewport'); + assert.match(String(error.details?.hint), /snapshot --raw/); + return true; + }, + ); +}); + +test('Appium iOS hierarchy limits are typed and disclosed at response level', async () => { + const result = await captureWebDriverIosSnapshot({ source: async () => SOURCE }, { raw: true }); + + assert.equal(result.truncated, undefined); + assert.deepEqual(result.warnings, [ + 'Appium page source does not guarantee hittability evidence; absent hittable means no evidence, not false. Regular output omits reported hittable: true without evidence but preserves reported false; raw preserves provider-reported values.', + 'Appium page source does not report hierarchy completeness; provider-side depth or child limits may omit nodes.', + ]); +}); diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts new file mode 100644 index 000000000..1b18aa663 --- /dev/null +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -0,0 +1,168 @@ +import { + createIosSnapshotEngine, + IosSnapshotEngineError, + resolveIosViewportEvidenceFromRoots, + toIosSnapshotEngineErrorDetails, +} from '@agent-device/capture-kit/ios-snapshot-engine'; +import { + createIosSnapshotRequest, + deriveIosSnapshotCapabilityResidue, + IOS_SNAPSHOT_PRODUCER_CAPABILITIES, +} from '@agent-device/capture-kit/ios-snapshot-planning'; +import type { + IosAcquisitionResidue, + IosSnapshotAcquisition, + IosSnapshotInput, + IosSnapshotPlan, + IosSnapshotPublication, + IosSnapshotRequest, + IosViewportEvidence, +} from '@agent-device/contracts/ios-snapshot'; +import type { SnapshotOptions, SnapshotResult } from '@agent-device/contracts/interactor-types'; +import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot'; +import { AppError } from '@agent-device/kernel/errors'; +import type { WebDriverClient } from './webdriver-client.ts'; +import { parseWebDriverSourceFacts } from './webdriver-source.ts'; + +const APPIUM_PRODUCER = IOS_SNAPSHOT_PRODUCER_CAPABILITIES['appium-source']; +const iosSnapshotEngine = createIosSnapshotEngine(); +const RESIDUE_WARNINGS = { + 'missing-viewport': + 'Appium page source does not provide valid viewport evidence; this raw tree cannot be validated for viewport-relative regular presentation. Regular snapshots fail closed.', +} satisfies Record<'missing-viewport', string>; +type WebDriverUnavailableFact = 'acquisition-depth' | 'hittability'; + +const UNAVAILABLE_FACT_WARNINGS = { + 'acquisition-depth': + 'Appium page source does not report hierarchy completeness; provider-side depth or child limits may omit nodes.', + hittability: + 'Appium page source does not guarantee hittability evidence; absent hittable means no evidence, not false. Regular output omits reported hittable: true without evidence but preserves reported false; raw preserves provider-reported values.', +} satisfies Record; + +export type WebDriverIosSnapshotAcquisition = Readonly<{ + request: IosSnapshotRequest; + plan: IosSnapshotPlan; + input: Extract; +}>; + +export type WebDriverIosSnapshotPublication = Readonly<{ + acquisition: WebDriverIosSnapshotAcquisition; + publication: IosSnapshotPublication; + result: SnapshotResult; +}>; + +export async function captureWebDriverIosSnapshot( + client: Pick, + options?: SnapshotOptions, + targetId?: string, +): Promise { + const acquired = acquireWebDriverIosSnapshot(await client.source(), options, targetId); + return publishWebDriverIosSnapshot(acquired).result; +} + +export function acquireWebDriverIosSnapshot( + source: string, + options?: SnapshotOptions, + targetId?: string, +): WebDriverIosSnapshotAcquisition { + const request = createIosSnapshotRequest({ + raw: options?.raw, + interactiveOnly: options?.interactiveOnly, + depth: options?.depth, + scope: options?.scope, + customActions: options?.customActions, + }); + const plan = iosSnapshotEngine.plan(request, APPIUM_PRODUCER); + const sourceFacts = parseWebDriverSourceFacts(source, { mode: 'facts' }); + const viewport = resolveIosViewportEvidenceFromRoots(sourceFacts.roots) ?? { + kind: 'missing' as const, + reason: 'not-provided' as const, + }; + const residue = residueForSource(viewport); + const common = { + producer: 'appium-source' as const, + nodes: sourceFacts.nodes, + viewport, + lineage: targetId ? { targetId } : {}, + residue, + }; + const acquisition: IosSnapshotAcquisition = { + ...common, + intent: 'full', + hint: { ...plan.hint, acquisitionIntent: 'full' }, + }; + return { request, plan, input: { stage: 'acquired', acquisition } }; +} + +export function publishWebDriverIosSnapshot( + acquisition: WebDriverIosSnapshotAcquisition, +): WebDriverIosSnapshotPublication { + let publication: IosSnapshotPublication; + try { + publication = iosSnapshotEngine.publish(acquisition.input, acquisition.request); + } catch (error) { + throwWebDriverIosSnapshotError(error); + } + const result = { + backend: 'xctest', + producer: 'appium-source', + nodes: stripRefs(publication.payload.nodes), + ...warningsForResidue(publication.residue), + } satisfies SnapshotResult; + return { acquisition, publication, result }; +} + +function residueForSource(viewport: IosViewportEvidence): readonly IosAcquisitionResidue[] { + const residue = [...deriveIosSnapshotCapabilityResidue(APPIUM_PRODUCER)]; + if (viewport.kind === 'missing') { + residue.push({ kind: 'missing-viewport', reason: viewport.reason }); + } + return residue; +} + +function warningsForResidue(residue: readonly IosAcquisitionResidue[]): { warnings?: string[] } { + const warnings = new Set(); + for (const entry of residue) { + const warning = warningForResidue(entry); + if (warning) warnings.add(warning); + } + return warnings.size > 0 ? { warnings: [...warnings] } : {}; +} + +function warningForResidue(entry: IosAcquisitionResidue): string | undefined { + switch (entry.kind) { + case 'unavailable-fact': + return entry.fact === 'acquisition-depth' || entry.fact === 'hittability' + ? UNAVAILABLE_FACT_WARNINGS[entry.fact] + : undefined; + case 'missing-viewport': + return RESIDUE_WARNINGS[entry.kind]; + case 'provider-pruned': + case 'truncated': + case 'stale-generation': + case 'fallback-source': + return undefined; + } +} + +function stripRefs(nodes: readonly SnapshotNode[]): RawSnapshotNode[] { + return nodes.map(({ ref: _ref, ...node }) => node); +} + +function throwWebDriverIosSnapshotError(error: unknown): never { + if (!(error instanceof IosSnapshotEngineError)) throw error; + const details = toIosSnapshotEngineErrorDetails(error); + throw new AppError( + 'COMMAND_FAILED', + error.message, + { + ...details, + ...(error.reason === 'missing-viewport' || error.reason === 'invalid-viewport' + ? { + hint: 'Use snapshot --raw to inspect the acquired Appium tree; regular presentation requires valid viewport evidence.', + } + : {}), + }, + error, + ); +} diff --git a/packages/provider-webdriver/src/webdriver-scroll-frame.ts b/packages/provider-webdriver/src/webdriver-scroll-frame.ts index c97728cf1..e7570d82b 100644 --- a/packages/provider-webdriver/src/webdriver-scroll-frame.ts +++ b/packages/provider-webdriver/src/webdriver-scroll-frame.ts @@ -1,9 +1,12 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import type { WebDriverWindowRect } from './webdriver-client.ts'; -import { parseWebDriverSource } from './webdriver-source.ts'; +import { parseWebDriverSource, type WebDriverSourceParseMode } from './webdriver-source.ts'; -export function scrollFrameFromWebDriverSource(source: string): WebDriverWindowRect | undefined { - const rect = parseWebDriverSource(source) +export function scrollFrameFromWebDriverSource( + source: string, + options: Readonly<{ mode: WebDriverSourceParseMode }>, +): WebDriverWindowRect | undefined { + const rect = parseWebDriverSource(source, options) .flatMap((node) => isScrollableSourceNode(node) && isUsableScrollRect(node.rect) ? [node.rect] : [], ) diff --git a/packages/provider-webdriver/src/webdriver-source.test.ts b/packages/provider-webdriver/src/webdriver-source.test.ts index cc624625f..cbb30c105 100644 --- a/packages/provider-webdriver/src/webdriver-source.test.ts +++ b/packages/provider-webdriver/src/webdriver-source.test.ts @@ -1,22 +1,122 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { scrollFrameFromWebDriverSource } from './webdriver-scroll-frame.ts'; -import { parseWebDriverSource } from './webdriver-source.ts'; +import { parseWebDriverSource, parseWebDriverSourceFacts } from './webdriver-source.ts'; test('WebDriver source parsing preserves hardened attributes and geometry', () => { const nodes = parseWebDriverSource( - '', + '', + { mode: 'facts' }, ); assert.equal(nodes[0]?.label, 'A > B'); assert.equal(nodes[0]?.identifier, 'login'); assert.deepEqual(nodes[0]?.rect, { x: 0, y: 0, width: 10, height: 10 }); + assert.equal(nodes[0]?.enabled, true); + assert.equal(nodes[0]?.visibleToUser, true); + assert.equal(nodes[0]?.hittable, undefined); assert.throws( - () => parseWebDriverSource(''), + () => parseWebDriverSource('', { mode: 'facts' }), /Unsupported XML attribute name "__proto__"/, ); }); +test('WebDriver source facts do not fill absent provider attributes', () => { + const node = parseWebDriverSource( + '', + { mode: 'facts' }, + )[0]; + + assert.equal(node?.label, 'Continue'); + assert.equal(node?.identifier, 'Continue'); + assert.equal('enabled' in (node ?? {}), false); + assert.equal('selected' in (node ?? {}), false); + assert.equal('focused' in (node ?? {}), false); + assert.equal('visibleToUser' in (node ?? {}), false); + assert.equal('hittable' in (node ?? {}), false); +}); + +test('WebDriver source facts preserve explicitly reported hittability', () => { + const node = parseWebDriverSource( + '', + { mode: 'facts' }, + )[0]; + + assert.equal(node?.hittable, true); +}); + +test('legacy WebDriver parsing keeps Android-derived hittability explicit at its call site', () => { + const node = parseWebDriverSource( + '', + { mode: 'legacy-derived' }, + )[0]; + + assert.equal(node?.enabled, true); + assert.equal(node?.visibleToUser, true); + assert.equal(node?.hittable, true); +}); + +test('legacy WebDriver parsing keeps Android source booleans and ignores iOS hints', () => { + const node = parseWebDriverSource( + '', + { mode: 'legacy-derived' }, + )[0]; + + assert.equal(node?.enabled, false); + assert.equal(node?.visibleToUser, false); + assert.equal(node?.selected, false); + assert.equal(node?.focused, false); + assert.equal(node?.hittable, false); +}); + +test('WebDriver source facts preserve roots without claiming hierarchy completeness', () => { + const facts = parseWebDriverSourceFacts( + '', + { mode: 'facts' }, + ); + + assert.equal('truncated' in facts, false); + assert.deepEqual(facts.roots, [ + { + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + rectStatus: 'reported', + }, + ]); + assert.deepEqual( + facts.nodes.map((node) => node.type), + ['XCUIElementTypeApplication'], + ); +}); + +test('WebDriver source facts classify invalid root geometry', () => { + const facts = parseWebDriverSourceFacts( + '', + { mode: 'facts' }, + ); + + assert.deepEqual(facts.roots, [ + { + type: 'XCUIElementTypeApplication', + rectStatus: 'invalid', + }, + ]); +}); + +test('WebDriver source facts do not call partial root geometry invalid', () => { + const facts = parseWebDriverSourceFacts( + '', + { mode: 'facts' }, + ); + + assert.deepEqual(facts.roots, [ + { + type: 'XCUIElementTypeApplication', + rectStatus: 'not-provided', + }, + ]); +}); + test('WebDriver scroll frame prefers visible scrollable containers', () => { assert.deepEqual( scrollFrameFromWebDriverSource( @@ -25,6 +125,7 @@ test('WebDriver scroll frame prefers visible scrollable containers', () => { '' + '' + '', + { mode: 'legacy-derived' }, ), { x: 0, y: 393, width: 1080, height: 1103 }, ); diff --git a/packages/provider-webdriver/src/webdriver-source.ts b/packages/provider-webdriver/src/webdriver-source.ts index fa833796b..edaefcda5 100644 --- a/packages/provider-webdriver/src/webdriver-source.ts +++ b/packages/provider-webdriver/src/webdriver-source.ts @@ -3,49 +3,105 @@ import { AppError } from '@agent-device/kernel/errors'; import { parseBounds } from '@agent-device/kernel/bounds'; import { parseXmlDocumentSync, type XmlNode } from '@agent-device/xml'; -export function parseWebDriverSource(source: string): RawSnapshotNode[] { - let roots: XmlNode[]; - try { - roots = parseXmlDocumentSync(source); - } catch (error) { - throw new AppError( - 'COMMAND_FAILED', - `Failed to parse WebDriver page source XML: ${error instanceof Error ? error.message : String(error)}`, - undefined, - error, - ); - } +export type WebDriverSourceParseMode = 'facts' | 'legacy-derived'; + +export type WebDriverSourceFacts = Readonly<{ + nodes: RawSnapshotNode[]; + roots: readonly WebDriverSourceRootFact[]; +}>; + +export type WebDriverSourceRootFact = Readonly<{ + type: string; + rect?: RawSnapshotNode['rect']; + rectStatus: 'reported' | 'invalid' | 'not-provided'; +}>; + +export function parseWebDriverSource( + source: string, + options: Readonly<{ mode: WebDriverSourceParseMode }>, +): RawSnapshotNode[] { + return parseWebDriverSourceFacts(source, options).nodes; +} + +export function parseWebDriverSourceFacts( + source: string, + options: Readonly<{ mode: WebDriverSourceParseMode }>, +): WebDriverSourceFacts { + const roots = parseSourceRoots(source); const nodes: RawSnapshotNode[] = []; + const sourceRoots: WebDriverSourceRootFact[] = []; + const mode = options.mode; for (const root of roots) { - appendSourceNodes(nodes, root); + appendSourceNodes(nodes, root, undefined, 0, mode, sourceRoots); } - return nodes; + return { nodes, roots: sourceRoots }; } function appendSourceNodes( nodes: RawSnapshotNode[], xmlNode: XmlNode, - parentIndex?: number, - depth = 0, + parentIndex: number | undefined, + depth: number, + mode: WebDriverSourceParseMode, + sourceRoots: WebDriverSourceRootFact[], ): void { - const currentIndex = - Object.keys(xmlNode.attributes).length === 0 - ? parentIndex - : appendSourceNode(nodes, xmlNode, parentIndex, depth); + const currentIndex = isSourceContainer(xmlNode, mode) + ? parentIndex + : appendSourceNode(nodes, xmlNode, parentIndex, depth, mode, sourceRoots); const childDepth = currentIndex === parentIndex ? depth : depth + 1; for (const child of xmlNode.children) { - appendSourceNodes(nodes, child, currentIndex, childDepth); + appendSourceNodes(nodes, child, currentIndex, childDepth, mode, sourceRoots); } } +function parseSourceRoots(source: string): XmlNode[] { + try { + return parseXmlDocumentSync(source); + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Failed to parse WebDriver page source XML: ${error instanceof Error ? error.message : String(error)}`, + undefined, + error, + ); + } +} + +function isSourceContainer(xmlNode: XmlNode, mode: WebDriverSourceParseMode): boolean { + if (Object.keys(xmlNode.attributes).length === 0) return true; + if (mode !== 'facts') return false; + const name = xmlNode.name.toLowerCase(); + return name === 'hierarchy' || name === 'appiumaut'; +} + function appendSourceNode( nodes: RawSnapshotNode[], xmlNode: XmlNode, parentIndex: number | undefined, depth: number, + mode: WebDriverSourceParseMode, + sourceRoots: WebDriverSourceRootFact[], ): number { const index = nodes.length; - nodes.push(sourceNodeFromAttributes(index, xmlNode.name, xmlNode.attributes, parentIndex, depth)); + const rect = rectFromAttributes(xmlNode.attributes); + nodes.push( + sourceNodeFromAttributes( + index, + xmlNode.name, + xmlNode.attributes, + parentIndex, + depth, + mode, + rect, + ), + ); + if (parentIndex === undefined) { + sourceRoots.push({ + type: xmlNode.name, + ...(rect ? { rect } : {}), + rectStatus: rectStatus(xmlNode.attributes, rect), + }); + } return index; } @@ -55,10 +111,9 @@ function sourceNodeFromAttributes( attrs: Record, parentIndex: number | undefined, depth: number, + mode: WebDriverSourceParseMode, + rect: RawSnapshotNode['rect'], ): RawSnapshotNode { - const rect = rectFromAttributes(attrs); - const enabled = booleanAttribute(attrs.enabled, true); - const visibleToUser = booleanAttribute(attrs.displayed ?? attrs.visible, true); return { index, type, @@ -67,16 +122,54 @@ function sourceNodeFromAttributes( value: nonEmpty(attrs.value), identifier: firstAttribute(attrs, ['resource-id', 'id', 'accessibility-id', 'name']), rect, - enabled, - selected: booleanAttribute(attrs.selected), - focused: booleanAttribute(attrs.focused), - visibleToUser, - hittable: visibleToUser && enabled && rect !== undefined && rect.width > 0 && rect.height > 0, + ...sourceStateFacts(attrs, rect, mode), depth, parentIndex, }; } +function sourceStateFacts( + attrs: Record, + rect: RawSnapshotNode['rect'], + mode: WebDriverSourceParseMode, +): Partial { + if (mode === 'legacy-derived') { + const enabled = legacyBooleanAttribute(attrs.enabled, true); + const visibleToUser = legacyBooleanAttribute(attrs.displayed ?? attrs.visible, true); + return { + enabled, + selected: legacyBooleanAttribute(attrs.selected), + focused: legacyBooleanAttribute(attrs.focused), + visibleToUser, + hittable: visibleToUser && enabled && isPositiveRect(rect), + }; + } + + const enabled = booleanAttribute(attrs.enabled); + const visibleToUser = booleanAttribute(attrs.displayed ?? attrs.visible); + return { + ...optionalBooleanFact('enabled', enabled), + ...optionalBooleanFact('selected', booleanAttribute(attrs.selected)), + ...optionalBooleanFact('focused', booleanAttribute(attrs.focused)), + ...optionalBooleanFact('visibleToUser', visibleToUser), + ...reportedHittabilityFact(attrs.hittable), + }; +} + +function optionalBooleanFact( + key: 'enabled' | 'selected' | 'focused' | 'visibleToUser', + value: boolean | undefined, +): Partial> { + return value === undefined ? {} : { [key]: value }; +} + +function reportedHittabilityFact( + reported: string | undefined, +): Partial> { + const reportedHittable = booleanAttribute(reported); + return reportedHittable === undefined ? {} : { hittable: reportedHittable }; +} + function rectFromAttributes(attrs: Record): RawSnapshotNode['rect'] | undefined { const bounds = parseBounds(attrs.bounds ?? null); if (bounds) return bounds; @@ -105,11 +198,32 @@ function nonEmpty(value: string | undefined): string | undefined { return value ? value : undefined; } -function booleanAttribute(value: string | undefined, defaultValue = false): boolean { +function booleanAttribute(value: string | undefined): boolean | undefined { + if (value === undefined) return undefined; + if (value === 'true' || value === '1') return true; + if (value === 'false' || value === '0') return false; + return undefined; +} + +function legacyBooleanAttribute(value: string | undefined, defaultValue = false): boolean { if (value === undefined) return defaultValue; return value === 'true' || value === '1'; } +function isPositiveRect(rect: RawSnapshotNode['rect']): boolean { + return Boolean(rect && rect.width > 0 && rect.height > 0); +} + +function rectStatus( + attrs: Record, + rect: RawSnapshotNode['rect'], +): WebDriverSourceRootFact['rectStatus'] { + const hasBoundsAttribute = attrs.bounds !== undefined; + const hasCompleteRect = ['x', 'y', 'width', 'height'].every((name) => attrs[name] !== undefined); + if (!hasBoundsAttribute && !hasCompleteRect) return 'not-provided'; + return isPositiveRect(rect) ? 'reported' : 'invalid'; +} + function numberAttribute(value: string | undefined): number | undefined { if (value === undefined || value === '') return undefined; const parsed = Number(value); diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index ceb362634..28b45cbce 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -511,7 +511,7 @@ function normalizeSnapshotResult( const appBundleId = readOptionalString(data, 'appBundleId'); return { nodes: readSnapshotNodes(data.nodes), - truncated: data.truncated === true, + ...(typeof data.truncated === 'boolean' ? { truncated: data.truncated } : {}), appName: readOptionalString(data, 'appName'), appBundleId, ...optionalSnapshotResponseFields(data), diff --git a/src/commands/capture/runtime/snapshot.test.ts b/src/commands/capture/runtime/snapshot.test.ts index 8b209aa70..5ac0df81d 100644 --- a/src/commands/capture/runtime/snapshot.test.ts +++ b/src/commands/capture/runtime/snapshot.test.ts @@ -42,6 +42,38 @@ test('runtime snapshot captures nodes and updates the session baseline', async ( assert.equal(stored?.snapshot?.nodes[0]?.label, 'Home'); }); +test('runtime snapshot preserves unknown hierarchy completeness for engine-owned iOS acquisition', async () => { + const device = createSnapshotOnlyDevice({ + snapshot: { + nodes: [], + backend: 'xctest', + producer: 'appium-source', + createdAt: 1, + }, + }); + + const result = await device.capture.snapshot({ session: 'default' }); + + assert.equal(result.truncated, undefined); +}); + +test('runtime snapshot uses the Appium sparse-tree disclosure for Appium acquisition', async () => { + const device = createSnapshotOnlyDevice({ + snapshot: { + nodes: [{ ref: 'e1', index: 0, type: 'XCUIElementTypeApplication', depth: 0 }], + backend: 'xctest', + producer: 'appium-source', + createdAt: 1, + }, + }); + + const result = await device.capture.snapshot({ session: 'default', interactiveOnly: true }); + + assert.equal(result.warnings?.length, 1); + assert.match(result.warnings?.[0] ?? '', /^Appium page source exposed only/); + assert.doesNotMatch(result.warnings?.[0] ?? '', /XCTest|simulator/); +}); + test('runtime snapshot forwards interactive capture options', async () => { let observedOptions: BackendSnapshotOptions | undefined; const device = createAgentDevice({ diff --git a/src/commands/capture/runtime/snapshot.ts b/src/commands/capture/runtime/snapshot.ts index b661c4552..7e4833eb3 100644 --- a/src/commands/capture/runtime/snapshot.ts +++ b/src/commands/capture/runtime/snapshot.ts @@ -8,6 +8,7 @@ import { type SnapshotDiagnosticsSummary, } from '@agent-device/contracts/capture'; import { AppError } from '@agent-device/kernel/errors'; +import { normalizeType } from '@agent-device/contracts/snapshot'; import type { SnapshotNode, SnapshotState, @@ -25,6 +26,7 @@ import { buildSnapshotVisibility } from '../../../snapshot/snapshot-visibility.t import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '../../../core/android-system-surface-disclosure.ts'; import { formatReactNativeOverlayWarning } from '../../react-native/overlay.ts'; import { now } from '../../runtime-common.ts'; +import { IOS_SNAPSHOT_PRODUCER_CAPABILITIES } from '@agent-device/capture-kit/ios-snapshot-planning'; import type { DiffSnapshotCommandOptions, RuntimeCommand, @@ -37,7 +39,7 @@ import { export type SnapshotCommandResult = { nodes: SnapshotNode[]; - truncated: boolean; + truncated?: boolean; appName?: string; appBundleId?: string; visibility?: SnapshotVisibility; @@ -68,9 +70,10 @@ export const snapshotCommand: RuntimeCommand< }, }); await runtime.sessions.set(nextSnapshotSession(options.session, capture)); + const truncated = snapshotTruncationForResult(capture.snapshot); return copySnapshotClickabilityEvidence(capture.snapshot, { nodes: capture.snapshot.nodes, - truncated: capture.snapshot.truncated ?? false, + ...(truncated === undefined ? {} : { truncated }), visibility: buildSnapshotVisibility({ nodes: capture.snapshot.nodes, backend: capture.snapshot.backend, @@ -218,6 +221,19 @@ function snapshotAppFields(capture: SnapshotCapture): { }; } +function snapshotTruncationForResult(snapshot: SnapshotState): boolean | undefined { + if (snapshot.truncated !== undefined) return snapshot.truncated; + if (snapshot.backend !== 'xctest' || snapshot.producer === undefined) return false; + const capability = IOS_SNAPSHOT_PRODUCER_CAPABILITIES[snapshot.producer]; + if (!capability) return false; + const acquisitionDepthUnknown = + capability.stage === 'acquired' && + capability.presentationOwner === 'ios-snapshot-engine' && + (capability.acquisitionDepth.rawTraversal.kind === 'incomplete' || + capability.acquisitionDepth.regularPresented.kind === 'incomplete'); + return acquisitionDepthUnknown ? undefined : false; +} + function buildSnapshotWarnings(params: { result: BackendSnapshotResult; annotations: SnapshotCaptureAnnotations; @@ -273,13 +289,27 @@ function buildSparseIosInteractiveWarnings(params: { } const root = params.snapshot.nodes[0]; - if (root?.type !== 'Application') return []; + if (!isApplicationRoot(root)) return []; + + if (params.snapshot.producer === 'appium-source') { + return [ + 'Appium page source exposed only the application root. Descendants may be absent from the acquired hierarchy; use snapshot --raw to inspect the source and verify the app accessibility tree.', + ]; + } + if (params.snapshot.producer !== undefined && params.snapshot.producer !== 'apple-runner') { + return []; + } return [ 'iOS interactive snapshot exposed only the application root. XCTest accessibility queries can fail to enumerate some simulator UI trees even when screenshots and direct gestures still work. Use screenshot as visual truth, try a scoped/full snapshot for diagnostics, and prefer direct selectors when known.', ]; } +function isApplicationRoot(node: SnapshotNode | undefined): boolean { + if (!node) return false; + return normalizeType(node.type ?? '') === 'application'; +} + const MERGED_LEAF_MIN_SEGMENTS = 10; /** diff --git a/src/core/__tests__/snapshot-state.test.ts b/src/core/__tests__/snapshot-state.test.ts index 4d8c6b3fe..ed5c3095b 100644 --- a/src/core/__tests__/snapshot-state.test.ts +++ b/src/core/__tests__/snapshot-state.test.ts @@ -1,5 +1,6 @@ import { expect, test } from 'vitest'; import { buildSnapshotState } from '../snapshot-state.ts'; +import { resolveActionableTouchResolution } from '../interaction-targeting.ts'; import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; import { attachSnapshotOcclusionContextEvidence } from '@agent-device/contracts/capture'; import { @@ -114,7 +115,23 @@ test('buildSnapshotState marks comparisonSafe false for filtered Android snapsho expect(unfiltered.comparisonSafe).toBe(true); }); -test('buildSnapshotState applies iOS interactive presentation for xctest snapshots', () => { +test('buildSnapshotState leaves Apple runner presentation to the engine', () => { + const nodes = [ + { index: 0, depth: 0, type: 'Application', label: 'Settings' }, + { index: 1, depth: 1, parentIndex: 0, type: 'Table', label: 'Settings' }, + { index: 2, depth: 2, parentIndex: 1, type: 'Cell', label: 'General' }, + { index: 3, depth: 3, parentIndex: 2, type: 'Button', label: 'General' }, + ]; + + const state = buildSnapshotState( + { nodes, backend: 'xctest', producer: 'apple-runner' }, + { snapshotInteractiveOnly: true }, + ); + + expect(state.nodes.map((node) => node.type)).toEqual(['Application', 'Table', 'Cell', 'Button']); +}); + +test('buildSnapshotState preserves the legacy acquired iOS presentation path', () => { const rowRect = { x: 16, y: 293, width: 370, height: 52 }; const state = buildSnapshotState( { @@ -125,31 +142,82 @@ test('buildSnapshotState applies iOS interactive presentation for xctest snapsho { index: 3, depth: 3, parentIndex: 2, type: 'Button', label: 'General', rect: rowRect }, ], backend: 'xctest', + producer: 'limrun-ios-tree', }, { snapshotInteractiveOnly: true }, ); - expect(state.nodes.map((node) => [node.type, node.label, node.parentIndex])).toEqual([ - ['Application', 'Settings', undefined], - ['CollectionView', undefined, 0], - ['Cell', 'General', 1], + expect(state.nodes.map((node) => [node.type, node.label])).toEqual([ + ['Application', 'Settings'], + ['CollectionView', undefined], + ['Cell', 'General'], ]); }); -test('buildSnapshotState leaves Apple runner presentation to the engine', () => { - const nodes = [ - { index: 0, depth: 0, type: 'Application', label: 'Settings' }, - { index: 1, depth: 1, parentIndex: 0, type: 'Table', label: 'Settings' }, - { index: 2, depth: 2, parentIndex: 1, type: 'Cell', label: 'General' }, - { index: 3, depth: 3, parentIndex: 2, type: 'Button', label: 'General' }, - ]; +test('buildSnapshotState uses the registered presentation owner for Appium results', () => { + const rowRect = { x: 16, y: 293, width: 370, height: 52 }; + const data = { + nodes: [ + { index: 0, depth: 0, type: 'Application', label: 'Settings' }, + { index: 1, depth: 1, parentIndex: 0, type: 'CollectionView' }, + { index: 2, depth: 2, parentIndex: 1, type: 'Cell', label: 'General', rect: rowRect }, + { index: 3, depth: 3, parentIndex: 2, type: 'Button', label: 'General', rect: rowRect }, + ], + backend: 'xctest' as const, + producer: 'appium-source' as const, + }; + + const state = buildSnapshotState(data, { snapshotInteractiveOnly: true }); + + expect(state.nodes.map((node) => [node.type, node.label])).toEqual([ + ['Application', 'Settings'], + ['CollectionView', undefined], + ['Cell', 'General'], + ['Button', 'General'], + ]); +}); +test('Appium presentation does not infer hittability from an enabled ancestor rectangle', () => { const state = buildSnapshotState( - { nodes, backend: 'xctest', producer: 'apple-runner' }, + { + nodes: [ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Cell', + rect: { x: 16, y: 293, width: 370, height: 52 }, + enabled: true, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'StaticText', + label: 'General', + rect: { x: 24, y: 309, width: 100, height: 20 }, + enabled: true, + }, + ], + backend: 'xctest', + producer: 'appium-source', + }, { snapshotInteractiveOnly: true }, ); + const target = state.nodes.find((node) => node.label === 'General'); - expect(state.nodes.map((node) => node.type)).toEqual(['Application', 'Table', 'Cell', 'Button']); + expect(target).toBeDefined(); + expect(target?.hittable).toBeUndefined(); + expect(resolveActionableTouchResolution(state.nodes, target!)).toMatchObject({ + node: target, + reason: 'original', + }); }); test('buildSnapshotState marks content covered by floating overlays as visible but blocked', () => { diff --git a/src/core/snapshot-state.ts b/src/core/snapshot-state.ts index c5614c42a..7a7dbe0d9 100644 --- a/src/core/snapshot-state.ts +++ b/src/core/snapshot-state.ts @@ -54,7 +54,7 @@ export function buildSnapshotState( const normalizedNodes = normalizeSnapshotTree( snapshotRaw ? backendAnnotatedNodes : pruneGroupNodes(backendAnnotatedNodes), ); - const presentableNodes = shouldPresentIosInteractiveSnapshot(data, flags) + const presentableNodes = shouldPresentLegacyIosInteractiveSnapshot(data, flags) ? presentIosInteractiveSnapshot(normalizedNodes) : normalizedNodes; const scopedNodes = @@ -120,8 +120,8 @@ function backendScopesAfterWire(backend: SnapshotBackend | undefined): boolean { return backend !== 'macos-helper' && backend !== 'android' && backend !== 'xctest'; } -function shouldPresentIosInteractiveSnapshot( - provenance: SnapshotStateProvenance, +function shouldPresentLegacyIosInteractiveSnapshot( + provenance: object & SnapshotStateProvenance, flags: | (Pick & Partial>) @@ -130,6 +130,7 @@ function shouldPresentIosInteractiveSnapshot( return ( provenance.backend === 'xctest' && iosSnapshotPresentationStage(provenance) === 'acquired' && + iosSnapshotPresentationOwner(provenance) !== 'ios-snapshot-engine' && flags?.snapshotInteractiveOnly === true && flags.snapshotRaw !== true ); @@ -140,9 +141,20 @@ function iosSnapshotPresentationStage( ): 'acquired' | 'presented' | undefined { if (provenance.backend !== 'xctest') return undefined; if (provenance.producer === undefined) return 'acquired'; + return iosSnapshotCapabilities(provenance)?.stage; +} + +function iosSnapshotPresentationOwner( + provenance: SnapshotStateProvenance, +): 'ios-snapshot-engine' | 'snapshot-state' | undefined { + return iosSnapshotCapabilities(provenance)?.presentationOwner; +} + +function iosSnapshotCapabilities(provenance: SnapshotStateProvenance) { + if (provenance.backend !== 'xctest' || provenance.producer === undefined) return undefined; return IOS_SNAPSHOT_PRODUCER_CAPABILITIES[ provenance.producer as 'apple-runner' | 'appium-source' | 'limrun-ios-tree' - ].stage; + ]; } function isAndroidComparisonSafeSnapshot( diff --git a/src/daemon/__tests__/selector-capture-runtime.test.ts b/src/daemon/__tests__/selector-capture-runtime.test.ts index a88a6b2a2..af4a9da77 100644 --- a/src/daemon/__tests__/selector-capture-runtime.test.ts +++ b/src/daemon/__tests__/selector-capture-runtime.test.ts @@ -100,6 +100,34 @@ test('legacy iOS sparse recovery retries a full snapshot', async () => { expect(boundCapture.mock.calls[1]?.[0]?.options).toMatchObject({ interactiveOnly: false }); }); +test('legacy iOS sparse recovery recognizes Appium application element types', async () => { + const { runtime } = makeCaptureRuntime('selector-appium-sparse-recovery'); + boundCapture + .mockResolvedValueOnce({ + backend: 'xctest', + producer: 'appium-source', + nodes: [{ index: 0, type: 'XCUIElementTypeApplication' }], + }) + .mockResolvedValueOnce({ + backend: 'xctest', + producer: 'appium-source', + nodes: [{ index: 0, type: 'XCUIElementTypeButton', label: 'Recovered' }], + }); + + const result = await runtime.capture({ + flags: { snapshotInteractiveOnly: true }, + recovery: { + legacyIosSparse: { + query: 'Search', + shouldScope: false, + }, + }, + }); + + expect(result.snapshot.nodes[0]?.label).toBe('Recovered'); + expect(boundCapture).toHaveBeenCalledTimes(2); +}); + test('legacy iOS sparse recovery rethrows full snapshot failure when scoping is disabled', async () => { const { runtime } = makeCaptureRuntime('selector-legacy-sparse-rethrow'); boundCapture diff --git a/src/daemon/result-serialization.ts b/src/daemon/result-serialization.ts index 3e71e3cd4..595257a30 100644 --- a/src/daemon/result-serialization.ts +++ b/src/daemon/result-serialization.ts @@ -67,7 +67,7 @@ export function serializeDevice(device: AgentDeviceDevice): Record { return { nodes: result.nodes, - truncated: result.truncated, + ...(result.truncated === undefined ? {} : { truncated: result.truncated }), ...(result.appName ? { appName: result.appName } : {}), ...(result.appBundleId ? { appBundleId: result.appBundleId } : {}), ...(result.visibility ? { visibility: result.visibility } : {}), diff --git a/src/daemon/selector-capture-runtime.ts b/src/daemon/selector-capture-runtime.ts index f06747679..d111cada2 100644 --- a/src/daemon/selector-capture-runtime.ts +++ b/src/daemon/selector-capture-runtime.ts @@ -1,4 +1,5 @@ import type { CommandFlags } from '@agent-device/contracts/command'; +import { normalizeType } from '@agent-device/contracts/snapshot'; import type { BackendSnapshotResult } from '../backend.ts'; import { buildSnapshotPresentationKey, @@ -318,5 +319,5 @@ function updateSessionSnapshot(params: { function isLegacySparseIosInteractiveSnapshot(snapshot: SnapshotState): boolean { if (snapshot.snapshotQuality) return false; if (snapshot.backend !== 'xctest' || snapshot.nodes.length !== 1) return false; - return snapshot.nodes[0]?.type === 'Application'; + return normalizeType(snapshot.nodes[0]?.type ?? '') === 'application'; } diff --git a/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts b/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts index e3e5efd28..0dd4af972 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts @@ -62,6 +62,71 @@ test('cloud iOS snapshot captures through the provider session after open', asyn }); }, 15_000); +test('cloud iOS engine-presented snapshot survives daemon publication', async () => { + await withProviderScenarioResource(createCloudIosWorld, async ({ daemon, server }) => { + const lease = await openCloudIosSession(daemon); + server.sourceOverride = + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + ''; + + const response = await daemon.callCommand( + 'snapshot', + [], + { ...leaseFlags(lease.leaseId), snapshotInteractiveOnly: true }, + { meta: leaseMeta(lease.leaseId) }, + ); + + const data = assertRpcOk<{ + nodes?: Array<{ + type?: string; + label?: string; + identifier?: string; + enabled?: boolean; + hittable?: boolean; + }>; + truncated?: boolean; + warnings?: string[]; + }>(response); + assert.equal( + data.nodes?.some( + (node) => + node.type === 'Button' && + node.label === 'Team Standup' && + node.identifier === 'DisplayNameTextField' && + node.enabled === true, + ), + true, + ); + assert.equal( + data.nodes?.find((node) => node.identifier === 'DisplayNameTextField')?.hittable, + undefined, + ); + assert.ok(data.warnings?.some((warning) => warning.includes('hittability evidence'))); + assert.equal(data.truncated, undefined); + assert.equal( + data.nodes?.some( + (node) => node.type === 'XCUIElementTypeStaticText' && node.label === 'Team Standup', + ), + false, + ); + assert.equal( + data.nodes?.some((node) => node.identifier === 'RoomDetailsIconImageView'), + false, + ); + }); +}, 15_000); + /** * #1658: `fill` tapped and sent its keys back-to-back. A WebView input takes * first responder asynchronously, so the keys arrived with nothing focused and @@ -254,6 +319,7 @@ class FakeIosWebDriverServer extends CloudWebDriverTestServer { pollsUntilFocus = 1; fieldValues: Record = { email: '', password: '' }; focused: FieldName | undefined; + sourceOverride: string | undefined; private pollsRemaining = Number.POSITIVE_INFINITY; private pendingFocus: FieldName | undefined; @@ -373,6 +439,7 @@ class FakeIosWebDriverServer extends CloudWebDriverTestServer { } private source(): string { + if (this.sourceOverride !== undefined) return this.sourceOverride; const field = (name: FieldName, label: string) => `