From ce31c8aa8942e3919bc45b2ad628562c2af18032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 19:15:37 +0200 Subject: [PATCH 01/15] refactor(ios): route Appium snapshots through engine --- .../provider-webdriver/src/runtime-session.ts | 1 + .../src/webdriver-interactor.test.ts | 20 ++ .../src/webdriver-interactor.ts | 24 ++- .../src/webdriver-ios-snapshot.test.ts | 149 ++++++++++++++ .../src/webdriver-ios-snapshot.ts | 188 ++++++++++++++++++ .../src/webdriver-source.test.ts | 62 +++++- .../src/webdriver-source.ts | 167 +++++++++++++--- src/core/__tests__/snapshot-state.test.ts | 22 -- src/core/snapshot-state.ts | 34 +--- .../snapshot-presentation-transitions.test.ts | 5 +- .../snapshot-publication-membership.test.ts | 6 +- 11 files changed, 584 insertions(+), 94 deletions(-) create mode 100644 packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts create mode 100644 packages/provider-webdriver/src/webdriver-ios-snapshot.ts diff --git a/packages/provider-webdriver/src/runtime-session.ts b/packages/provider-webdriver/src/runtime-session.ts index 00611a975b..cce3f7ca86 100644 --- a/packages/provider-webdriver/src/runtime-session.ts +++ b/packages/provider-webdriver/src/runtime-session.ts @@ -113,6 +113,7 @@ export class WebDriverSessionManager { client, backend: snapshotBackendForPlatform(prepared.platform), capabilities, + targetId: device.id, }), }), ); diff --git a/packages/provider-webdriver/src/webdriver-interactor.test.ts b/packages/provider-webdriver/src/webdriver-interactor.test.ts index e75c821d8a..d150b3afa0 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.test.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.test.ts @@ -249,6 +249,26 @@ test('fill refuses empty text as an unsupported clear rather than a vacuous succ assert.deepEqual(world.transcript, []); }); +test('iOS WebDriver interactor routes snapshots through the acquisition adapter', async () => { + const source = vi.fn( + async () => + '', + ); + const interactor = createWebDriverInteractor({ + client: { source } as unknown as WebDriverClient, + backend: 'xctest', + capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'ios' }), + targetId: 'ios-1', + }); + + const result = await interactor.snapshot({ raw: true, depth: 1 }); + + assert.equal(result.backend, 'xctest'); + assert.equal(result.producer, 'appium-source'); + assert.equal(source.mock.calls.length, 1); + assert.equal(result.nodes?.[0]?.type, 'XCUIElementTypeApplication'); +}); + async function runFill(world: ReturnType) { vi.useFakeTimers(); try { diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index 0b49fa7b3d..d97c12f975 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -25,6 +25,7 @@ import type { W3CPointerAction, WebDriverClient, WebDriverWindowRect } from './w import { touchPointer } from './webdriver-gestures.ts'; import { scrollFrameFromWebDriverSource } from './webdriver-scroll-frame.ts'; import { parseWebDriverSource } from './webdriver-source.ts'; +import { captureWebDriverIosSnapshot } from './webdriver-ios-snapshot.ts'; import { setWebDriverOrientation } from './webdriver-orientation.ts'; /** @@ -99,25 +100,34 @@ export type WebDriverInteractorOptions = { client: WebDriverClient; backend: Extract; capabilities: CloudWebDriverProviderCapabilities; + targetId?: string; }; export function createWebDriverInteractor(options: WebDriverInteractorOptions): Interactor { - return new WebDriverInteractor(options.client, options.backend, options.capabilities); + return new WebDriverInteractor( + options.client, + options.backend, + options.capabilities, + options.targetId, + ); } class WebDriverInteractor implements Interactor { private readonly client: WebDriverClient; private readonly backend: Extract; private readonly capabilities: CloudWebDriverProviderCapabilities; + private readonly targetId: string | undefined; constructor( client: WebDriverClient, backend: Extract, capabilities: CloudWebDriverProviderCapabilities, + targetId?: string, ) { this.client = client; this.backend = backend; this.capabilities = capabilities; + this.targetId = targetId; } async open( @@ -292,14 +302,16 @@ class WebDriverInteractor implements Interactor { await this.client.screenshot(outPath); } - async snapshot(_options?: SnapshotOptions): Promise { + async snapshot(options?: SnapshotOptions): Promise { this.requireSupport('snapshot'); + if (this.backend === 'xctest') { + return await captureWebDriverIosSnapshot(this.client, options, this.targetId); + } // Spelled as a correlated pair per channel so the SnapshotProvenance union accepts it. return { - ...(this.backend === 'xctest' - ? { backend: 'xctest' as const, producer: 'appium-source' as const } - : { backend: 'android' as const, producer: 'appium-source' as const }), - nodes: parseWebDriverSource(await this.client.source()), + backend: 'android' as const, + producer: 'appium-source' as const, + nodes: parseWebDriverSource(await this.client.source(), { mode: 'legacy-derived' }), }; } diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts new file mode 100644 index 0000000000..4ca6472683 --- /dev/null +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -0,0 +1,149 @@ +import assert from 'node:assert/strict'; +import { test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { + acquireWebDriverIosSnapshot, + captureWebDriverIosSnapshot, + publishWebDriverIosSnapshot, +} from './webdriver-ios-snapshot.ts'; + +const SOURCE = ` + + + + + +`; + +test('Appium iOS snapshots acquire facts and publish regular output through the engine', async () => { + const source = vi.fn(async () => SOURCE); + const result = await captureWebDriverIosSnapshot({ source }, undefined, 'cloud-ios-1'); + + assert.equal(result.backend, 'xctest'); + assert.equal(result.producer, 'appium-source'); + assert.equal(result.truncated, false); + assert.deepEqual( + result.nodes?.map((node) => [node.type, node.label, node.parentIndex]), + [ + ['XCUIElementTypeApplication', 'Example', undefined], + ['XCUIElementTypeOther', 'Content', 0], + ['XCUIElementTypeButton', 'Continue', 1], + ], + ); + assert.equal(result.nodes?.find((node) => node.label === 'Continue')?.hittable, false); + assert.equal(source.mock.calls.length, 1); + assert.ok(result.warnings?.some((warning) => warning.includes('hittability evidence'))); + assert.equal(JSON.stringify(result).includes('iosSnapshot'), false); + assert.equal(JSON.stringify(result).includes('apple-runner'), false); +}); + +test('Appium iOS options become an engine plan and engine-owned projection', () => { + const acquired = acquireWebDriverIosSnapshot(SOURCE, { + raw: true, + interactiveOnly: true, + depth: 1, + scope: 'Continue', + customActions: true, + }); + + assert.equal(acquired.input.stage, 'acquired'); + assert.deepEqual(acquired.plan.narrowing, { + depth: null, + scope: null, + interactiveOnly: false, + }); + assert.deepEqual(acquired.input.acquisition.hint, { + projection: 'raw', + rawTraversalDepth: null, + regularPresentedDepth: null, + interactiveOnly: false, + customActions: true, + acquisitionIntent: 'full', + }); + assert.deepEqual(acquired.input.acquisition.lineage, {}); + assert.deepEqual(acquired.input.acquisition.viewport, { + kind: 'reported', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }); + + const published = publishWebDriverIosSnapshot(acquired); + assert.deepEqual( + published.result.nodes?.map((node) => [node.type, node.label, node.depth, node.parentIndex]), + [['XCUIElementTypeButton', 'Continue', 0, undefined]], + ); + assert.deepEqual(published.publication.comparisonIdentity.lineage, {}); +}); + +test('Appium iOS interactive requests stay provider-unpruned and use engine presentation', () => { + const acquired = acquireWebDriverIosSnapshot(SOURCE, { interactiveOnly: true }, 'ios-1'); + + assert.equal(acquired.input.acquisition.hint.interactiveOnly, true); + assert.equal(acquired.plan.narrowing.interactiveOnly, false); + const published = publishWebDriverIosSnapshot(acquired); + assert.equal( + published.result.nodes?.some((node) => node.label === 'Continue'), + true, + ); + assert.equal(published.result.nodes?.find((node) => node.label === 'Continue')?.hittable, false); +}); + +test('Appium iOS regular presentation fails typed when page source has no viewport', () => { + const acquired = acquireWebDriverIosSnapshot( + '', + ); + + assert.deepEqual(acquired.input.acquisition.viewport, { + kind: 'missing', + reason: 'not-provided', + }); + assert.deepEqual(acquired.input.acquisition.residue, [ + { kind: 'unavailable-fact', fact: 'hittability' }, + { kind: 'missing-viewport', reason: 'not-provided' }, + ]); + assert.throws( + () => publishWebDriverIosSnapshot(acquired), + (error: unknown) => + error instanceof AppError && + error.details?.reason === 'missing-viewport' && + !('iosSnapshotEngine' in (error.details ?? {})), + ); +}); + +test('Appium iOS does not promote a non-viewport root rectangle to viewport evidence', () => { + const acquired = acquireWebDriverIosSnapshot( + '', + ); + + assert.deepEqual(acquired.input.acquisition.viewport, { + kind: 'missing', + reason: 'not-provided', + }); +}); + +test('Appium iOS regular presentation fails typed when root geometry is invalid', () => { + const acquired = acquireWebDriverIosSnapshot( + '', + ); + + assert.deepEqual(acquired.input.acquisition.viewport, { + kind: 'missing', + reason: 'invalid', + }); + assert.throws( + () => publishWebDriverIosSnapshot(acquired), + (error: unknown) => error instanceof AppError && error.details?.reason === 'invalid-viewport', + ); +}); + +test('Appium iOS truncation is typed and disclosed at response level', async () => { + const result = await captureWebDriverIosSnapshot( + { source: async () => SOURCE.replace('', '') }, + { raw: true }, + ); + + assert.equal(result.truncated, true); + assert.deepEqual(result.warnings, [ + 'Appium page source does not provide hittability evidence; regular snapshot nodes are not actionable.', + 'Appium page source is truncated; the snapshot hierarchy may be incomplete.', + ]); +}); diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts new file mode 100644 index 0000000000..66a553c335 --- /dev/null +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -0,0 +1,188 @@ +import { + createIosSnapshotEngine, + IosSnapshotEngineError, +} from '@agent-device/capture-kit/ios-snapshot-engine'; +import { + createIosSnapshotRequest, + IOS_SNAPSHOT_PRODUCER_CAPABILITIES, +} from '@agent-device/capture-kit/ios-snapshot-planning'; +import type { + IosAcquisitionResidue, + IosSnapshotAcquisition, + IosSnapshotInput, + IosSnapshotPlan, + IosSnapshotPublication, + IosSnapshotRequest, + IosViewportEvidence, +} from '@agent-device/contracts/ios-snapshot'; +import type { SnapshotOptions, SnapshotResult } from '@agent-device/contracts/interactor-types'; +import { normalizeType } from '@agent-device/contracts/snapshot'; +import type { RawSnapshotNode, Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; +import { AppError } from '@agent-device/kernel/errors'; +import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import type { WebDriverClient } from './webdriver-client.ts'; +import { parseWebDriverSourceFacts, type WebDriverSourceRootFact } from './webdriver-source.ts'; + +const APPIUM_PRODUCER = IOS_SNAPSHOT_PRODUCER_CAPABILITIES['appium-source']; +const iosSnapshotEngine = createIosSnapshotEngine(); +const RESIDUE_WARNINGS: Partial> = { + 'missing-viewport': + 'Appium page source does not provide a valid viewport; regular snapshot presentation is unavailable.', + truncated: 'Appium page source is truncated; the snapshot hierarchy may be incomplete.', + 'provider-pruned': + 'Appium page source is provider-pruned; the snapshot hierarchy may be incomplete.', + 'stale-generation': + 'Appium snapshot generation is stale; the snapshot may not describe the current target.', + 'fallback-source': 'Appium snapshot used a fallback source; snapshot fidelity may be reduced.', +}; + +export type WebDriverIosSnapshotAcquisition = Readonly<{ + request: IosSnapshotRequest; + plan: IosSnapshotPlan; + input: Extract; +}>; + +export type WebDriverIosSnapshotPublication = Readonly<{ + acquisition: WebDriverIosSnapshotAcquisition; + publication: IosSnapshotPublication; + result: SnapshotResult; +}>; + +export async function captureWebDriverIosSnapshot( + client: Pick, + options?: SnapshotOptions, + targetId?: string, +): Promise { + const acquired = acquireWebDriverIosSnapshot(await client.source(), options, targetId); + return publishWebDriverIosSnapshot(acquired).result; +} + +export function acquireWebDriverIosSnapshot( + source: string, + options?: SnapshotOptions, + targetId?: string, +): WebDriverIosSnapshotAcquisition { + const request = createIosSnapshotRequest({ + raw: options?.raw, + interactiveOnly: options?.interactiveOnly, + depth: options?.depth, + scope: options?.scope, + customActions: options?.customActions, + }); + const plan = iosSnapshotEngine.plan(request, APPIUM_PRODUCER); + const sourceFacts = parseWebDriverSourceFacts(source, { mode: 'facts' }); + const viewport = viewportEvidence(sourceFacts.roots); + const residue = residueForSource(sourceFacts.truncated, viewport); + const common = { + producer: 'appium-source' as const, + nodes: sourceFacts.nodes, + truncated: sourceFacts.truncated, + viewport, + lineage: targetId ? { targetId } : {}, + residue, + }; + const acquisition: IosSnapshotAcquisition = + request.acquisitionIntent === 'full' + ? { ...common, intent: 'full', hint: { ...plan.hint, acquisitionIntent: 'full' } } + : { + ...common, + intent: 'surface-observation', + hint: { ...plan.hint, acquisitionIntent: 'surface-observation' }, + }; + return { request, plan, input: { stage: 'acquired', acquisition } }; +} + +export function publishWebDriverIosSnapshot( + acquisition: WebDriverIosSnapshotAcquisition, +): WebDriverIosSnapshotPublication { + let publication: IosSnapshotPublication; + try { + publication = iosSnapshotEngine.publish(acquisition.input, acquisition.request); + } catch (error) { + throwWebDriverIosSnapshotError(error); + } + const result: SnapshotResult = { + backend: 'xctest', + producer: 'appium-source', + nodes: stripRefs(publication.payload.nodes), + truncated: publication.payload.truncated, + ...warningsForResidue(publication.residue), + }; + return { acquisition, publication, result }; +} + +function viewportEvidence(roots: readonly WebDriverSourceRootFact[]): IosViewportEvidence { + const candidates = roots.filter((node) => { + const type = normalizeType(node.type ?? ''); + return type === 'application' || type === 'window'; + }); + const root = [...candidates].sort(compareViewportRoots)[0]; + if (!root) return { kind: 'missing', reason: 'not-provided' }; + if (root.rectStatus === 'reported' && isPositiveFiniteRect(root.rect)) { + return { kind: 'reported', rect: root.rect }; + } + return { kind: 'missing', reason: root.rectStatus === 'invalid' ? 'invalid' : 'not-provided' }; +} + +function rectArea(rect: Rect | undefined): number { + return rect && isPositiveFiniteRect(rect) ? rect.width * rect.height : 0; +} + +function compareViewportRoots( + left: WebDriverSourceRootFact, + right: WebDriverSourceRootFact, +): number { + const status = rootGeometryRank(right.rectStatus) - rootGeometryRank(left.rectStatus); + return status || rectArea(right.rect) - rectArea(left.rect); +} + +function rootGeometryRank(status: WebDriverSourceRootFact['rectStatus']): number { + return status === 'reported' ? 2 : status === 'invalid' ? 1 : 0; +} + +function residueForSource( + truncated: boolean, + viewport: IosViewportEvidence, +): readonly IosAcquisitionResidue[] { + const residue: IosAcquisitionResidue[] = [{ kind: 'unavailable-fact', fact: 'hittability' }]; + if (viewport.kind === 'missing') { + residue.push({ kind: 'missing-viewport', reason: viewport.reason }); + } + if (truncated) residue.push({ kind: 'truncated', dimension: 'nodes' }); + return residue; +} + +function warningsForResidue(residue: readonly IosAcquisitionResidue[]): { warnings?: string[] } { + const warnings = new Set(); + for (const entry of residue) { + const warning = warningForResidue(entry); + if (warning) warnings.add(warning); + } + return warnings.size > 0 ? { warnings: [...warnings] } : {}; +} + +function warningForResidue(entry: IosAcquisitionResidue): string | undefined { + if (entry.kind === 'unavailable-fact') { + return entry.fact === 'hittability' + ? 'Appium page source does not provide hittability evidence; regular snapshot nodes are not actionable.' + : `Appium page source does not provide ${entry.fact} evidence.`; + } + return RESIDUE_WARNINGS[entry.kind]; +} + +function stripRefs(nodes: readonly SnapshotNode[]): RawSnapshotNode[] { + return nodes.map(({ ref: _ref, ...node }) => node); +} + +function throwWebDriverIosSnapshotError(error: unknown): never { + if (!(error instanceof IosSnapshotEngineError)) throw error; + throw new AppError( + 'COMMAND_FAILED', + error.message, + { + reason: error.reason, + ...(error.details.field ? { field: error.details.field } : {}), + }, + error, + ); +} diff --git a/packages/provider-webdriver/src/webdriver-source.test.ts b/packages/provider-webdriver/src/webdriver-source.test.ts index cc624625ff..ddb5bcf18f 100644 --- a/packages/provider-webdriver/src/webdriver-source.test.ts +++ b/packages/provider-webdriver/src/webdriver-source.test.ts @@ -1,22 +1,80 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { scrollFrameFromWebDriverSource } from './webdriver-scroll-frame.ts'; -import { parseWebDriverSource } from './webdriver-source.ts'; +import { parseWebDriverSource, parseWebDriverSourceFacts } from './webdriver-source.ts'; test('WebDriver source parsing preserves hardened attributes and geometry', () => { const nodes = parseWebDriverSource( - '', + '', ); assert.equal(nodes[0]?.label, 'A > B'); assert.equal(nodes[0]?.identifier, 'login'); assert.deepEqual(nodes[0]?.rect, { x: 0, y: 0, width: 10, height: 10 }); + assert.equal(nodes[0]?.enabled, true); + assert.equal(nodes[0]?.visibleToUser, true); + assert.equal(nodes[0]?.hittable, undefined); assert.throws( () => parseWebDriverSource(''), /Unsupported XML attribute name "__proto__"/, ); }); +test('WebDriver source facts do not fill absent provider attributes', () => { + const node = parseWebDriverSource( + '', + )[0]; + + assert.equal(node?.label, 'Continue'); + assert.equal(node?.identifier, 'Continue'); + assert.equal('enabled' in (node ?? {}), false); + assert.equal('visibleToUser' in (node ?? {}), false); + assert.equal('hittable' in (node ?? {}), false); +}); + +test('legacy WebDriver parsing keeps Android-derived hittability explicit at its call site', () => { + const node = parseWebDriverSource( + '', + { mode: 'legacy-derived' }, + )[0]; + + assert.equal(node?.enabled, true); + assert.equal(node?.visibleToUser, true); + assert.equal(node?.hittable, true); +}); + +test('WebDriver source facts expose provider truncation without publishing wrapper nodes', () => { + const facts = parseWebDriverSourceFacts( + '', + ); + + assert.equal(facts.truncated, true); + assert.deepEqual(facts.roots, [ + { + type: 'XCUIElementTypeApplication', + rect: { x: 0, y: 0, width: 390, height: 844 }, + rectStatus: 'reported', + }, + ]); + assert.deepEqual( + facts.nodes.map((node) => node.type), + ['XCUIElementTypeApplication'], + ); +}); + +test('WebDriver source facts classify invalid root geometry', () => { + const facts = parseWebDriverSourceFacts( + '', + ); + + assert.deepEqual(facts.roots, [ + { + type: 'XCUIElementTypeApplication', + rectStatus: 'invalid', + }, + ]); +}); + test('WebDriver scroll frame prefers visible scrollable containers', () => { assert.deepEqual( scrollFrameFromWebDriverSource( diff --git a/packages/provider-webdriver/src/webdriver-source.ts b/packages/provider-webdriver/src/webdriver-source.ts index fa833796b5..87985966fd 100644 --- a/packages/provider-webdriver/src/webdriver-source.ts +++ b/packages/provider-webdriver/src/webdriver-source.ts @@ -3,23 +3,39 @@ import { AppError } from '@agent-device/kernel/errors'; import { parseBounds } from '@agent-device/kernel/bounds'; import { parseXmlDocumentSync, type XmlNode } from '@agent-device/xml'; -export function parseWebDriverSource(source: string): RawSnapshotNode[] { - let roots: XmlNode[]; - try { - roots = parseXmlDocumentSync(source); - } catch (error) { - throw new AppError( - 'COMMAND_FAILED', - `Failed to parse WebDriver page source XML: ${error instanceof Error ? error.message : String(error)}`, - undefined, - error, - ); - } +export type WebDriverSourceParseMode = 'facts' | 'legacy-derived'; + +export type WebDriverSourceFacts = Readonly<{ + nodes: RawSnapshotNode[]; + roots: readonly WebDriverSourceRootFact[]; + truncated: boolean; +}>; + +export type WebDriverSourceRootFact = Readonly<{ + type: string; + rect?: RawSnapshotNode['rect']; + rectStatus: 'reported' | 'invalid' | 'not-provided'; +}>; + +export function parseWebDriverSource( + source: string, + options: Readonly<{ mode?: WebDriverSourceParseMode }> = {}, +): RawSnapshotNode[] { + return parseWebDriverSourceFacts(source, options).nodes; +} + +export function parseWebDriverSourceFacts( + source: string, + options: Readonly<{ mode?: WebDriverSourceParseMode }> = {}, +): WebDriverSourceFacts { + const roots = parseSourceRoots(source); const nodes: RawSnapshotNode[] = []; + const sourceRoots: WebDriverSourceRootFact[] = []; + const mode = options.mode ?? 'facts'; for (const root of roots) { - appendSourceNodes(nodes, root); + appendSourceNodes(nodes, root, undefined, 0, mode, sourceRoots); } - return nodes; + return { nodes, roots: sourceRoots, truncated: hasTruncationMarker(roots) }; } function appendSourceNodes( @@ -27,25 +43,58 @@ function appendSourceNodes( xmlNode: XmlNode, parentIndex?: number, depth = 0, + mode: WebDriverSourceParseMode = 'facts', + sourceRoots: WebDriverSourceRootFact[] = [], ): void { - const currentIndex = - Object.keys(xmlNode.attributes).length === 0 - ? parentIndex - : appendSourceNode(nodes, xmlNode, parentIndex, depth); + const currentIndex = isSourceContainer(xmlNode, mode) + ? parentIndex + : appendSourceNode(nodes, xmlNode, parentIndex, depth, mode, sourceRoots); const childDepth = currentIndex === parentIndex ? depth : depth + 1; for (const child of xmlNode.children) { - appendSourceNodes(nodes, child, currentIndex, childDepth); + appendSourceNodes(nodes, child, currentIndex, childDepth, mode, sourceRoots); + } +} + +function parseSourceRoots(source: string): XmlNode[] { + try { + return parseXmlDocumentSync(source); + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Failed to parse WebDriver page source XML: ${error instanceof Error ? error.message : String(error)}`, + undefined, + error, + ); } } +function isSourceContainer(xmlNode: XmlNode, mode: WebDriverSourceParseMode): boolean { + if (Object.keys(xmlNode.attributes).length === 0) return true; + if (mode !== 'facts') return false; + const name = xmlNode.name.toLowerCase(); + return name === 'hierarchy' || name === 'appiumaut'; +} + function appendSourceNode( nodes: RawSnapshotNode[], xmlNode: XmlNode, parentIndex: number | undefined, depth: number, + mode: WebDriverSourceParseMode, + sourceRoots: WebDriverSourceRootFact[], ): number { const index = nodes.length; - nodes.push(sourceNodeFromAttributes(index, xmlNode.name, xmlNode.attributes, parentIndex, depth)); + const rect = rectFromAttributes(xmlNode.attributes); + nodes.push( + sourceNodeFromAttributes(index, xmlNode.name, xmlNode.attributes, parentIndex, depth, mode), + ); + if (parentIndex === undefined) { + sourceRoots.push({ + type: xmlNode.name, + ...(rect ? { rect } : {}), + rectStatus: rectStatus(xmlNode.attributes, rect), + }); + } return index; } @@ -55,10 +104,9 @@ function sourceNodeFromAttributes( attrs: Record, parentIndex: number | undefined, depth: number, + mode: WebDriverSourceParseMode, ): RawSnapshotNode { const rect = rectFromAttributes(attrs); - const enabled = booleanAttribute(attrs.enabled, true); - const visibleToUser = booleanAttribute(attrs.displayed ?? attrs.visible, true); return { index, type, @@ -67,16 +115,58 @@ function sourceNodeFromAttributes( value: nonEmpty(attrs.value), identifier: firstAttribute(attrs, ['resource-id', 'id', 'accessibility-id', 'name']), rect, - enabled, - selected: booleanAttribute(attrs.selected), - focused: booleanAttribute(attrs.focused), - visibleToUser, - hittable: visibleToUser && enabled && rect !== undefined && rect.width > 0 && rect.height > 0, + ...sourceStateFacts(attrs, rect, mode), depth, parentIndex, }; } +function sourceStateFacts( + attrs: Record, + rect: RawSnapshotNode['rect'], + mode: WebDriverSourceParseMode, +): Partial { + const legacyDerived = mode === 'legacy-derived'; + const enabled = booleanAttribute(attrs.enabled); + const visibleToUser = booleanAttribute(attrs.displayed ?? attrs.visible); + return { + ...optionalBooleanFact('enabled', enabled, legacyDerived), + selected: booleanAttribute(attrs.selected), + focused: booleanAttribute(attrs.focused), + ...optionalBooleanFact('visibleToUser', visibleToUser, legacyDerived), + ...hittabilityFact(attrs.hittable, visibleToUser, enabled, rect, legacyDerived), + }; +} + +function optionalBooleanFact( + key: 'enabled' | 'visibleToUser', + value: boolean | undefined, + defaultWhenAbsent: boolean, +): Partial { + return defaultWhenAbsent || value !== undefined ? { [key]: value ?? true } : {}; +} + +function hittabilityFact( + reported: string | undefined, + visibleToUser: boolean | undefined, + enabled: boolean | undefined, + rect: RawSnapshotNode['rect'], + legacyDerived: boolean, +): Partial> { + const reportedHittable = booleanAttribute(reported); + if (reportedHittable !== undefined) return { hittable: reportedHittable }; + return legacyDerived + ? { hittable: (visibleToUser ?? true) && (enabled ?? true) && isPositiveRect(rect) } + : {}; +} + +function hasTruncationMarker(nodes: readonly XmlNode[]): boolean { + return nodes.some( + (node) => + booleanAttribute(node.attributes.truncated) === true || hasTruncationMarker(node.children), + ); +} + function rectFromAttributes(attrs: Record): RawSnapshotNode['rect'] | undefined { const bounds = parseBounds(attrs.bounds ?? null); if (bounds) return bounds; @@ -105,9 +195,26 @@ function nonEmpty(value: string | undefined): string | undefined { return value ? value : undefined; } -function booleanAttribute(value: string | undefined, defaultValue = false): boolean { - if (value === undefined) return defaultValue; - return value === 'true' || value === '1'; +function booleanAttribute(value: string | undefined): boolean | undefined { + if (value === undefined) return undefined; + if (value === 'true' || value === '1') return true; + if (value === 'false' || value === '0') return false; + return undefined; +} + +function isPositiveRect(rect: RawSnapshotNode['rect']): boolean { + return Boolean(rect && rect.width > 0 && rect.height > 0); +} + +function rectStatus( + attrs: Record, + rect: RawSnapshotNode['rect'], +): WebDriverSourceRootFact['rectStatus'] { + const hasGeometryAttribute = ['bounds', 'x', 'y', 'width', 'height'].some( + (name) => attrs[name] !== undefined, + ); + if (!hasGeometryAttribute) return 'not-provided'; + return isPositiveRect(rect) ? 'reported' : 'invalid'; } function numberAttribute(value: string | undefined): number | undefined { diff --git a/src/core/__tests__/snapshot-state.test.ts b/src/core/__tests__/snapshot-state.test.ts index 4d8c6b3fe3..a66d9fc335 100644 --- a/src/core/__tests__/snapshot-state.test.ts +++ b/src/core/__tests__/snapshot-state.test.ts @@ -114,28 +114,6 @@ test('buildSnapshotState marks comparisonSafe false for filtered Android snapsho expect(unfiltered.comparisonSafe).toBe(true); }); -test('buildSnapshotState applies iOS interactive presentation for xctest snapshots', () => { - const rowRect = { x: 16, y: 293, width: 370, height: 52 }; - const state = buildSnapshotState( - { - nodes: [ - { index: 0, depth: 0, type: 'Application', label: 'Settings' }, - { index: 1, depth: 1, parentIndex: 0, type: 'CollectionView' }, - { index: 2, depth: 2, parentIndex: 1, type: 'Cell', label: 'General', rect: rowRect }, - { index: 3, depth: 3, parentIndex: 2, type: 'Button', label: 'General', rect: rowRect }, - ], - backend: 'xctest', - }, - { snapshotInteractiveOnly: true }, - ); - - expect(state.nodes.map((node) => [node.type, node.label, node.parentIndex])).toEqual([ - ['Application', 'Settings', undefined], - ['CollectionView', undefined, 0], - ['Cell', 'General', 1], - ]); -}); - test('buildSnapshotState leaves Apple runner presentation to the engine', () => { const nodes = [ { index: 0, depth: 0, type: 'Application', label: 'Settings' }, diff --git a/src/core/snapshot-state.ts b/src/core/snapshot-state.ts index c5614c42ab..6c29aeaaf5 100644 --- a/src/core/snapshot-state.ts +++ b/src/core/snapshot-state.ts @@ -21,8 +21,6 @@ import { import { coveredAndroidReplacementNodeIndexes } from '../snapshot/android-replacement-surface-occlusion.ts'; import { scopeSnapshotNodes } from '@agent-device/capture-kit/snapshot-desktop-projection'; import { normalizeSnapshotTree, pruneGroupNodes } from '../core/snapshot-tree-ingestion.ts'; -import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; -import { IOS_SNAPSHOT_PRODUCER_CAPABILITIES } from '@agent-device/capture-kit/ios-snapshot-planning'; /** * The ONE daemon assembly of a captured tree (ADR 0004 / #1797): normalize, group prune, @@ -54,13 +52,10 @@ export function buildSnapshotState( const normalizedNodes = normalizeSnapshotTree( snapshotRaw ? backendAnnotatedNodes : pruneGroupNodes(backendAnnotatedNodes), ); - const presentableNodes = shouldPresentIosInteractiveSnapshot(data, flags) - ? presentIosInteractiveSnapshot(normalizedNodes) - : normalizedNodes; const scopedNodes = flags?.snapshotScope && backendScopesAfterWire(data?.backend) - ? scopeSnapshotNodes(presentableNodes, flags.snapshotScope) - : presentableNodes; + ? scopeSnapshotNodes(normalizedNodes, flags.snapshotScope) + : normalizedNodes; const snapshotQuality = snapshotCaptureAnnotationsFrom(data).quality; const nodes = attachRefs( snapshotRaw @@ -120,31 +115,6 @@ function backendScopesAfterWire(backend: SnapshotBackend | undefined): boolean { return backend !== 'macos-helper' && backend !== 'android' && backend !== 'xctest'; } -function shouldPresentIosInteractiveSnapshot( - provenance: SnapshotStateProvenance, - flags: - | (Pick & - Partial>) - | undefined, -): boolean { - return ( - provenance.backend === 'xctest' && - iosSnapshotPresentationStage(provenance) === 'acquired' && - flags?.snapshotInteractiveOnly === true && - flags.snapshotRaw !== true - ); -} - -function iosSnapshotPresentationStage( - provenance: SnapshotStateProvenance, -): 'acquired' | 'presented' | undefined { - if (provenance.backend !== 'xctest') return undefined; - if (provenance.producer === undefined) return 'acquired'; - return IOS_SNAPSHOT_PRODUCER_CAPABILITIES[ - provenance.producer as 'apple-runner' | 'appium-source' | 'limrun-ios-tree' - ].stage; -} - function isAndroidComparisonSafeSnapshot( backend: SnapshotBackend | undefined, flags: diff --git a/src/daemon/__tests__/snapshot-presentation-transitions.test.ts b/src/daemon/__tests__/snapshot-presentation-transitions.test.ts index f9123598dc..89eb3da1a1 100644 --- a/src/daemon/__tests__/snapshot-presentation-transitions.test.ts +++ b/src/daemon/__tests__/snapshot-presentation-transitions.test.ts @@ -7,7 +7,10 @@ import { navigationTitleWithAppProvidedDetailsAffordanceNodes } from '../../snap test('iOS daemon presentation applies transitions without reapplying runner-owned scope', () => { const snapshot = buildSnapshotState( - { nodes: navigationTitleWithAppProvidedDetailsAffordanceNodes, backend: 'xctest' }, + { + nodes: presentIosInteractiveSnapshot(navigationTitleWithAppProvidedDetailsAffordanceNodes), + backend: 'xctest', + }, { snapshotInteractiveOnly: true, snapshotScope: 'DisplayNameTextField' }, ); diff --git a/src/daemon/__tests__/snapshot-publication-membership.test.ts b/src/daemon/__tests__/snapshot-publication-membership.test.ts index 7cc1232f5d..76b3657fca 100644 --- a/src/daemon/__tests__/snapshot-publication-membership.test.ts +++ b/src/daemon/__tests__/snapshot-publication-membership.test.ts @@ -1,5 +1,6 @@ import { expect, test } from 'vitest'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; import { buildSnapshotState } from '../../core/snapshot-state.ts'; // End-to-end publication-membership contract for the acquire/present design (#1797, external @@ -11,7 +12,10 @@ import { buildSnapshotState } from '../../core/snapshot-state.ts'; // suppression test below fails because promo-banner is published. It passes only when the // production suppression fires. function publish(nodes: RawSnapshotNode[]) { - return buildSnapshotState({ nodes, backend: 'xctest' }, { snapshotInteractiveOnly: true }).nodes; + return buildSnapshotState( + { nodes: presentIosInteractiveSnapshot(nodes), backend: 'xctest' }, + { snapshotInteractiveOnly: true }, + ).nodes; } const screen: RawSnapshotNode[] = [ From 7b20d94325d3b1290673f64e8eb77ed970f3f85e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 19:28:12 +0200 Subject: [PATCH 02/15] perf(ios): keep Appium snapshot adapter lazy --- packages/provider-webdriver/src/webdriver-interactor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index d97c12f975..b92d75c761 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -25,7 +25,6 @@ import type { W3CPointerAction, WebDriverClient, WebDriverWindowRect } from './w import { touchPointer } from './webdriver-gestures.ts'; import { scrollFrameFromWebDriverSource } from './webdriver-scroll-frame.ts'; import { parseWebDriverSource } from './webdriver-source.ts'; -import { captureWebDriverIosSnapshot } from './webdriver-ios-snapshot.ts'; import { setWebDriverOrientation } from './webdriver-orientation.ts'; /** @@ -305,6 +304,7 @@ class WebDriverInteractor implements Interactor { async snapshot(options?: SnapshotOptions): Promise { this.requireSupport('snapshot'); if (this.backend === 'xctest') { + const { captureWebDriverIosSnapshot } = await import('./webdriver-ios-snapshot.ts'); return await captureWebDriverIosSnapshot(this.client, options, this.targetId); } // Spelled as a correlated pair per channel so the SnapshotProvenance union accepts it. From 0a5f01d8d5e8194d24043b4543d0c849838fce0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 19:52:57 +0200 Subject: [PATCH 03/15] fix(ios): preserve legacy snapshot presentation boundary --- packages/contracts/src/facades/capture.ts | 3 ++ .../src/snapshot-private-evidence.ts | 18 +++++++ .../src/webdriver-ios-snapshot.ts | 18 ++++--- src/core/__tests__/snapshot-state.test.ts | 54 ++++++++++++++++++- src/core/snapshot-state.ts | 36 ++++++++++++- 5 files changed, 119 insertions(+), 10 deletions(-) diff --git a/packages/contracts/src/facades/capture.ts b/packages/contracts/src/facades/capture.ts index d11cab7c83..4883e2b0ac 100644 --- a/packages/contracts/src/facades/capture.ts +++ b/packages/contracts/src/facades/capture.ts @@ -53,14 +53,17 @@ export type { ScreenshotResultData, } from '../snapshot-types.ts'; export { + attachSnapshotPresentationEvidence, attachSnapshotClickabilityEvidence, attachSnapshotOcclusionContextEvidence, copySnapshotClickabilityEvidence, readSnapshotClickabilityEvidence, readSnapshotOcclusionContextEvidence, + readSnapshotPresentationEvidence, } from '../snapshot-private-evidence.ts'; export type { AndroidSiblingOrderEvidence, + SnapshotPresentationEvidence, SnapshotClickabilityEvidence, SnapshotOcclusionContextEvidence, } from '../snapshot-private-evidence.ts'; diff --git a/packages/contracts/src/snapshot-private-evidence.ts b/packages/contracts/src/snapshot-private-evidence.ts index 530f0c13ed..e0f41be2cb 100644 --- a/packages/contracts/src/snapshot-private-evidence.ts +++ b/packages/contracts/src/snapshot-private-evidence.ts @@ -28,9 +28,14 @@ export type SnapshotOcclusionContextEvidence = { androidSiblingOrderByNodeIndex?: ReadonlyMap; }; +export type SnapshotPresentationEvidence = { + owner: 'ios-snapshot-engine'; +}; + type SnapshotPrivateEvidence = { clickability?: SnapshotClickabilityEvidence; occlusionContext?: SnapshotOcclusionContextEvidence; + presentation?: SnapshotPresentationEvidence; }; const privateEvidenceBySnapshotObject = new WeakMap(); @@ -59,6 +64,19 @@ export function readSnapshotClickabilityEvidence( return owner ? privateEvidenceBySnapshotObject.get(owner)?.clickability : undefined; } +export function attachSnapshotPresentationEvidence( + owner: T, + evidence: SnapshotPresentationEvidence, +): T { + return attachSnapshotPrivateEvidence(owner, { presentation: evidence }); +} + +export function readSnapshotPresentationEvidence( + owner: object | null | undefined, +): SnapshotPresentationEvidence | undefined { + return owner ? privateEvidenceBySnapshotObject.get(owner)?.presentation : undefined; +} + export function copySnapshotClickabilityEvidence( source: object | null | undefined, target: T, diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index 66a553c335..7a04d3c1d5 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -2,6 +2,7 @@ import { createIosSnapshotEngine, IosSnapshotEngineError, } from '@agent-device/capture-kit/ios-snapshot-engine'; +import { attachSnapshotPresentationEvidence } from '@agent-device/contracts/capture'; import { createIosSnapshotRequest, IOS_SNAPSHOT_PRODUCER_CAPABILITIES, @@ -101,13 +102,16 @@ export function publishWebDriverIosSnapshot( } catch (error) { throwWebDriverIosSnapshotError(error); } - const result: SnapshotResult = { - backend: 'xctest', - producer: 'appium-source', - nodes: stripRefs(publication.payload.nodes), - truncated: publication.payload.truncated, - ...warningsForResidue(publication.residue), - }; + const result = attachSnapshotPresentationEvidence( + { + backend: 'xctest', + producer: 'appium-source', + nodes: stripRefs(publication.payload.nodes), + truncated: publication.payload.truncated, + ...warningsForResidue(publication.residue), + } satisfies SnapshotResult, + { owner: 'ios-snapshot-engine' }, + ); return { acquisition, publication, result }; } diff --git a/src/core/__tests__/snapshot-state.test.ts b/src/core/__tests__/snapshot-state.test.ts index a66d9fc335..cd572928da 100644 --- a/src/core/__tests__/snapshot-state.test.ts +++ b/src/core/__tests__/snapshot-state.test.ts @@ -1,7 +1,10 @@ import { expect, test } from 'vitest'; import { buildSnapshotState } from '../snapshot-state.ts'; import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; -import { attachSnapshotOcclusionContextEvidence } from '@agent-device/contracts/capture'; +import { + attachSnapshotOcclusionContextEvidence, + attachSnapshotPresentationEvidence, +} from '@agent-device/contracts/capture'; import { buildUiHierarchySnapshot, parseUiHierarchyTree, @@ -130,6 +133,55 @@ test('buildSnapshotState leaves Apple runner presentation to the engine', () => expect(state.nodes.map((node) => node.type)).toEqual(['Application', 'Table', 'Cell', 'Button']); }); +test('buildSnapshotState preserves the legacy acquired iOS presentation path', () => { + const rowRect = { x: 16, y: 293, width: 370, height: 52 }; + const state = buildSnapshotState( + { + nodes: [ + { index: 0, depth: 0, type: 'Application', label: 'Settings' }, + { index: 1, depth: 1, parentIndex: 0, type: 'CollectionView' }, + { index: 2, depth: 2, parentIndex: 1, type: 'Cell', label: 'General', rect: rowRect }, + { index: 3, depth: 3, parentIndex: 2, type: 'Button', label: 'General', rect: rowRect }, + ], + backend: 'xctest', + producer: 'limrun-ios-tree', + }, + { snapshotInteractiveOnly: true }, + ); + + expect(state.nodes.map((node) => [node.type, node.label])).toEqual([ + ['Application', 'Settings'], + ['CollectionView', undefined], + ['Cell', 'General'], + ]); +}); + +test('buildSnapshotState skips the legacy path for engine-presented iOS results', () => { + const rowRect = { x: 16, y: 293, width: 370, height: 52 }; + const data = attachSnapshotPresentationEvidence( + { + nodes: [ + { index: 0, depth: 0, type: 'Application', label: 'Settings' }, + { index: 1, depth: 1, parentIndex: 0, type: 'CollectionView' }, + { index: 2, depth: 2, parentIndex: 1, type: 'Cell', label: 'General', rect: rowRect }, + { index: 3, depth: 3, parentIndex: 2, type: 'Button', label: 'General', rect: rowRect }, + ], + backend: 'xctest' as const, + producer: 'appium-source' as const, + }, + { owner: 'ios-snapshot-engine' }, + ); + + const state = buildSnapshotState(data, { snapshotInteractiveOnly: true }); + + expect(state.nodes.map((node) => [node.type, node.label])).toEqual([ + ['Application', 'Settings'], + ['CollectionView', undefined], + ['Cell', 'General'], + ['Button', 'General'], + ]); +}); + test('buildSnapshotState marks content covered by floating overlays as visible but blocked', () => { const state = buildSnapshotState( { diff --git a/src/core/snapshot-state.ts b/src/core/snapshot-state.ts index 6c29aeaaf5..dc4166114e 100644 --- a/src/core/snapshot-state.ts +++ b/src/core/snapshot-state.ts @@ -1,6 +1,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import { readSnapshotOcclusionContextEvidence, + readSnapshotPresentationEvidence, snapshotCaptureAnnotationsFrom, } from '@agent-device/contracts/capture'; import { isAndroidInputMethodNode } from '@agent-device/contracts/android-input-ownership'; @@ -21,6 +22,8 @@ import { import { coveredAndroidReplacementNodeIndexes } from '../snapshot/android-replacement-surface-occlusion.ts'; import { scopeSnapshotNodes } from '@agent-device/capture-kit/snapshot-desktop-projection'; import { normalizeSnapshotTree, pruneGroupNodes } from '../core/snapshot-tree-ingestion.ts'; +import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; +import { IOS_SNAPSHOT_PRODUCER_CAPABILITIES } from '@agent-device/capture-kit/ios-snapshot-planning'; /** * The ONE daemon assembly of a captured tree (ADR 0004 / #1797): normalize, group prune, @@ -52,10 +55,13 @@ export function buildSnapshotState( const normalizedNodes = normalizeSnapshotTree( snapshotRaw ? backendAnnotatedNodes : pruneGroupNodes(backendAnnotatedNodes), ); + const presentableNodes = shouldPresentLegacyIosInteractiveSnapshot(data, flags) + ? presentIosInteractiveSnapshot(normalizedNodes) + : normalizedNodes; const scopedNodes = flags?.snapshotScope && backendScopesAfterWire(data?.backend) - ? scopeSnapshotNodes(normalizedNodes, flags.snapshotScope) - : normalizedNodes; + ? scopeSnapshotNodes(presentableNodes, flags.snapshotScope) + : presentableNodes; const snapshotQuality = snapshotCaptureAnnotationsFrom(data).quality; const nodes = attachRefs( snapshotRaw @@ -115,6 +121,32 @@ function backendScopesAfterWire(backend: SnapshotBackend | undefined): boolean { return backend !== 'macos-helper' && backend !== 'android' && backend !== 'xctest'; } +function shouldPresentLegacyIosInteractiveSnapshot( + provenance: object & SnapshotStateProvenance, + flags: + | (Pick & + Partial>) + | undefined, +): boolean { + return ( + provenance.backend === 'xctest' && + iosSnapshotPresentationStage(provenance) === 'acquired' && + readSnapshotPresentationEvidence(provenance) === undefined && + flags?.snapshotInteractiveOnly === true && + flags.snapshotRaw !== true + ); +} + +function iosSnapshotPresentationStage( + provenance: SnapshotStateProvenance, +): 'acquired' | 'presented' | undefined { + if (provenance.backend !== 'xctest') return undefined; + if (provenance.producer === undefined) return 'acquired'; + return IOS_SNAPSHOT_PRODUCER_CAPABILITIES[ + provenance.producer as 'apple-runner' | 'appium-source' | 'limrun-ios-tree' + ].stage; +} + function isAndroidComparisonSafeSnapshot( backend: SnapshotBackend | undefined, flags: From 5fd9466a8757f522fc88805f0357e64a531c0315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 20:00:31 +0200 Subject: [PATCH 04/15] perf(ios): tighten Appium snapshot facts --- .../src/webdriver-ios-snapshot.ts | 14 ++++++-------- .../provider-webdriver/src/webdriver-source.ts | 12 ++++++++++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index 7a04d3c1d5..bf65f86ef0 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -26,16 +26,11 @@ import { parseWebDriverSourceFacts, type WebDriverSourceRootFact } from './webdr const APPIUM_PRODUCER = IOS_SNAPSHOT_PRODUCER_CAPABILITIES['appium-source']; const iosSnapshotEngine = createIosSnapshotEngine(); -const RESIDUE_WARNINGS: Partial> = { +const RESIDUE_WARNINGS = { 'missing-viewport': 'Appium page source does not provide a valid viewport; regular snapshot presentation is unavailable.', truncated: 'Appium page source is truncated; the snapshot hierarchy may be incomplete.', - 'provider-pruned': - 'Appium page source is provider-pruned; the snapshot hierarchy may be incomplete.', - 'stale-generation': - 'Appium snapshot generation is stale; the snapshot may not describe the current target.', - 'fallback-source': 'Appium snapshot used a fallback source; snapshot fidelity may be reduced.', -}; +} satisfies Pick, 'missing-viewport' | 'truncated'>; export type WebDriverIosSnapshotAcquisition = Readonly<{ request: IosSnapshotRequest; @@ -171,7 +166,10 @@ function warningForResidue(entry: IosAcquisitionResidue): string | undefined { ? 'Appium page source does not provide hittability evidence; regular snapshot nodes are not actionable.' : `Appium page source does not provide ${entry.fact} evidence.`; } - return RESIDUE_WARNINGS[entry.kind]; + if (entry.kind === 'missing-viewport' || entry.kind === 'truncated') { + return RESIDUE_WARNINGS[entry.kind]; + } + return undefined; } function stripRefs(nodes: readonly SnapshotNode[]): RawSnapshotNode[] { diff --git a/packages/provider-webdriver/src/webdriver-source.ts b/packages/provider-webdriver/src/webdriver-source.ts index 87985966fd..0da70345d9 100644 --- a/packages/provider-webdriver/src/webdriver-source.ts +++ b/packages/provider-webdriver/src/webdriver-source.ts @@ -86,7 +86,15 @@ function appendSourceNode( const index = nodes.length; const rect = rectFromAttributes(xmlNode.attributes); nodes.push( - sourceNodeFromAttributes(index, xmlNode.name, xmlNode.attributes, parentIndex, depth, mode), + sourceNodeFromAttributes( + index, + xmlNode.name, + xmlNode.attributes, + parentIndex, + depth, + mode, + rect, + ), ); if (parentIndex === undefined) { sourceRoots.push({ @@ -105,8 +113,8 @@ function sourceNodeFromAttributes( parentIndex: number | undefined, depth: number, mode: WebDriverSourceParseMode, + rect: RawSnapshotNode['rect'], ): RawSnapshotNode { - const rect = rectFromAttributes(attrs); return { index, type, From 234f5a09dcd3ddb30d2faae9bce1da90a3145c09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 20:14:50 +0200 Subject: [PATCH 05/15] test(ios): cover legacy snapshot presentation boundary --- .../src/webdriver-ios-snapshot.ts | 15 ++++++--------- .../snapshot-presentation-transitions.test.ts | 2 +- .../snapshot-publication-membership.test.ts | 6 +----- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index bf65f86ef0..09d26dad05 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -77,14 +77,11 @@ export function acquireWebDriverIosSnapshot( lineage: targetId ? { targetId } : {}, residue, }; - const acquisition: IosSnapshotAcquisition = - request.acquisitionIntent === 'full' - ? { ...common, intent: 'full', hint: { ...plan.hint, acquisitionIntent: 'full' } } - : { - ...common, - intent: 'surface-observation', - hint: { ...plan.hint, acquisitionIntent: 'surface-observation' }, - }; + const acquisition: IosSnapshotAcquisition = { + ...common, + intent: 'full', + hint: { ...plan.hint, acquisitionIntent: 'full' }, + }; return { request, plan, input: { stage: 'acquired', acquisition } }; } @@ -164,7 +161,7 @@ function warningForResidue(entry: IosAcquisitionResidue): string | undefined { if (entry.kind === 'unavailable-fact') { return entry.fact === 'hittability' ? 'Appium page source does not provide hittability evidence; regular snapshot nodes are not actionable.' - : `Appium page source does not provide ${entry.fact} evidence.`; + : undefined; } if (entry.kind === 'missing-viewport' || entry.kind === 'truncated') { return RESIDUE_WARNINGS[entry.kind]; diff --git a/src/daemon/__tests__/snapshot-presentation-transitions.test.ts b/src/daemon/__tests__/snapshot-presentation-transitions.test.ts index 89eb3da1a1..c6545376ab 100644 --- a/src/daemon/__tests__/snapshot-presentation-transitions.test.ts +++ b/src/daemon/__tests__/snapshot-presentation-transitions.test.ts @@ -8,7 +8,7 @@ import { navigationTitleWithAppProvidedDetailsAffordanceNodes } from '../../snap test('iOS daemon presentation applies transitions without reapplying runner-owned scope', () => { const snapshot = buildSnapshotState( { - nodes: presentIosInteractiveSnapshot(navigationTitleWithAppProvidedDetailsAffordanceNodes), + nodes: navigationTitleWithAppProvidedDetailsAffordanceNodes, backend: 'xctest', }, { snapshotInteractiveOnly: true, snapshotScope: 'DisplayNameTextField' }, diff --git a/src/daemon/__tests__/snapshot-publication-membership.test.ts b/src/daemon/__tests__/snapshot-publication-membership.test.ts index 76b3657fca..7cc1232f5d 100644 --- a/src/daemon/__tests__/snapshot-publication-membership.test.ts +++ b/src/daemon/__tests__/snapshot-publication-membership.test.ts @@ -1,6 +1,5 @@ import { expect, test } from 'vitest'; import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; import { buildSnapshotState } from '../../core/snapshot-state.ts'; // End-to-end publication-membership contract for the acquire/present design (#1797, external @@ -12,10 +11,7 @@ import { buildSnapshotState } from '../../core/snapshot-state.ts'; // suppression test below fails because promo-banner is published. It passes only when the // production suppression fires. function publish(nodes: RawSnapshotNode[]) { - return buildSnapshotState( - { nodes: presentIosInteractiveSnapshot(nodes), backend: 'xctest' }, - { snapshotInteractiveOnly: true }, - ).nodes; + return buildSnapshotState({ nodes, backend: 'xctest' }, { snapshotInteractiveOnly: true }).nodes; } const screen: RawSnapshotNode[] = [ From e4af7a1e5b47f80b7fedfcd057ce52d925feb52b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 20:40:38 +0200 Subject: [PATCH 06/15] test(ios): cover WebDriver snapshot seams --- .../src/webdriver-interactor.test.ts | 20 ++++++++ .../src/webdriver-ios-snapshot.test.ts | 22 +++++--- .../cloud-webdriver-ios-text-entry.test.ts | 51 +++++++++++++++++++ 3 files changed, 87 insertions(+), 6 deletions(-) diff --git a/packages/provider-webdriver/src/webdriver-interactor.test.ts b/packages/provider-webdriver/src/webdriver-interactor.test.ts index d150b3afa0..8d29f5393d 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.test.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.test.ts @@ -269,6 +269,26 @@ test('iOS WebDriver interactor routes snapshots through the acquisition adapter' assert.equal(result.nodes?.[0]?.type, 'XCUIElementTypeApplication'); }); +test('Android WebDriver interactor keeps legacy-derived source facts at its call site', async () => { + const source = vi.fn( + async () => + '', + ); + const interactor = createWebDriverInteractor({ + client: { source } as unknown as WebDriverClient, + backend: 'android', + capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'android' }), + }); + + const result = await interactor.snapshot(); + + assert.equal(result.backend, 'android'); + assert.equal(result.nodes?.[0]?.type, 'hierarchy'); + assert.equal(result.nodes?.[1]?.type, 'android.widget.Button'); + assert.equal(result.nodes?.[1]?.hittable, true); + assert.equal(source.mock.calls.length, 1); +}); + async function runFill(world: ReturnType) { vi.useFakeTimers(); try { diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index 4ca6472683..ccce37ac94 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -33,8 +33,17 @@ test('Appium iOS snapshots acquire facts and publish regular output through the assert.equal(result.nodes?.find((node) => node.label === 'Continue')?.hittable, false); assert.equal(source.mock.calls.length, 1); assert.ok(result.warnings?.some((warning) => warning.includes('hittability evidence'))); - assert.equal(JSON.stringify(result).includes('iosSnapshot'), false); - assert.equal(JSON.stringify(result).includes('apple-runner'), false); + assert.deepEqual(Object.keys(result).sort(), [ + 'backend', + 'nodes', + 'producer', + 'truncated', + 'warnings', + ]); + assert.equal( + result.nodes?.every((node) => !('ref' in node)), + true, + ); }); test('Appium iOS options become an engine plan and engine-owned projection', () => { @@ -102,10 +111,11 @@ test('Appium iOS regular presentation fails typed when page source has no viewpo ]); assert.throws( () => publishWebDriverIosSnapshot(acquired), - (error: unknown) => - error instanceof AppError && - error.details?.reason === 'missing-viewport' && - !('iosSnapshotEngine' in (error.details ?? {})), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.deepEqual(error.details, { reason: 'missing-viewport', field: 'viewport' }); + return true; + }, ); }); diff --git a/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts b/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts index e3e5efd283..57c2f8b807 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts @@ -62,6 +62,55 @@ test('cloud iOS snapshot captures through the provider session after open', asyn }); }, 15_000); +test('cloud iOS engine-presented snapshot survives daemon publication', async () => { + await withProviderScenarioResource(createCloudIosWorld, async ({ daemon, server }) => { + const lease = await openCloudIosSession(daemon); + server.sourceOverride = + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + ''; + + const response = await daemon.callCommand( + 'snapshot', + [], + { ...leaseFlags(lease.leaseId), snapshotInteractiveOnly: true }, + { meta: leaseMeta(lease.leaseId) }, + ); + + const data = assertRpcOk<{ + nodes?: Array<{ type?: string; label?: string; identifier?: string; enabled?: boolean }>; + }>(response); + assert.equal( + data.nodes?.some( + (node) => + node.type === 'Button' && + node.label === 'Team Standup' && + node.identifier === 'DisplayNameTextField' && + node.enabled === true, + ), + true, + ); + assert.equal( + data.nodes?.some((node) => node.type === 'StaticText' && node.label === 'Team Standup'), + false, + ); + assert.equal( + data.nodes?.some((node) => node.identifier === 'RoomDetailsIconImageView'), + false, + ); + }); +}, 15_000); + /** * #1658: `fill` tapped and sent its keys back-to-back. A WebView input takes * first responder asynchronously, so the keys arrived with nothing focused and @@ -254,6 +303,7 @@ class FakeIosWebDriverServer extends CloudWebDriverTestServer { pollsUntilFocus = 1; fieldValues: Record = { email: '', password: '' }; focused: FieldName | undefined; + sourceOverride: string | undefined; private pollsRemaining = Number.POSITIVE_INFINITY; private pendingFocus: FieldName | undefined; @@ -373,6 +423,7 @@ class FakeIosWebDriverServer extends CloudWebDriverTestServer { } private source(): string { + if (this.sourceOverride !== undefined) return this.sourceOverride; const field = (name: FieldName, label: string) => ` Date: Tue, 1 Sep 2026 21:13:21 +0200 Subject: [PATCH 07/15] fix(ios): centralize WebDriver snapshot evidence --- .../src/ios-snapshot-engine/errors.test.ts | 17 +++++++ .../src/ios-snapshot-engine/errors.ts | 15 ++++++ .../src/ios-snapshot-engine/index.ts | 2 + .../src/ios-snapshot-engine/viewport.test.ts | 41 +++++++++++++++ .../src/ios-snapshot-engine/viewport.ts | 50 +++++++++++++++++++ .../src/ios-snapshot-planning.test.ts | 4 ++ .../capture-kit/src/ios-snapshot-planning.ts | 2 +- .../src/snapshot-private-evidence.ts | 1 + .../interactor-runner-provider.test.ts | 13 +++-- .../src/runner/snapshot-presentation.ts | 42 +++++----------- .../src/webdriver-interactor.ts | 1 - .../src/webdriver-ios-snapshot.test.ts | 4 +- .../src/webdriver-ios-snapshot.ts | 49 ++++-------------- .../src/webdriver-scroll-frame.ts | 2 +- .../src/webdriver-source.test.ts | 21 ++++++++ .../src/webdriver-source.ts | 43 ++++++++++------ src/core/snapshot-state.ts | 3 +- .../snapshot-presentation-transitions.test.ts | 5 +- 18 files changed, 214 insertions(+), 101 deletions(-) create mode 100644 packages/capture-kit/src/ios-snapshot-engine/errors.test.ts create mode 100644 packages/capture-kit/src/ios-snapshot-engine/errors.ts create mode 100644 packages/capture-kit/src/ios-snapshot-engine/viewport.test.ts create mode 100644 packages/capture-kit/src/ios-snapshot-engine/viewport.ts diff --git a/packages/capture-kit/src/ios-snapshot-engine/errors.test.ts b/packages/capture-kit/src/ios-snapshot-engine/errors.test.ts new file mode 100644 index 0000000000..1d21f52a0a --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/errors.test.ts @@ -0,0 +1,17 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { toIosSnapshotEngineErrorDetails } from './errors.ts'; +import { IosSnapshotEngineError } from './types.ts'; + +test('engine error details expose only the public reason and field', () => { + const error = new IosSnapshotEngineError('invalid-viewport', 'invalid viewport', { + field: 'viewport', + index: 4, + frame: { x: 0, y: 0, width: 10, height: 10 }, + }); + + assert.deepEqual(toIosSnapshotEngineErrorDetails(error), { + reason: 'invalid-viewport', + field: 'viewport', + }); +}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/errors.ts b/packages/capture-kit/src/ios-snapshot-engine/errors.ts new file mode 100644 index 0000000000..9480a22698 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/errors.ts @@ -0,0 +1,15 @@ +import type { IosSnapshotEngineError, IosSnapshotEngineFailureReason } from './types.ts'; + +export type IosSnapshotEnginePublicErrorDetails = Readonly<{ + reason: IosSnapshotEngineFailureReason; + field?: string; +}>; + +export function toIosSnapshotEngineErrorDetails( + error: IosSnapshotEngineError, +): IosSnapshotEnginePublicErrorDetails { + return { + reason: error.reason, + ...(error.details.field ? { field: error.details.field } : {}), + }; +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/index.ts b/packages/capture-kit/src/ios-snapshot-engine/index.ts index 0c13dc24f8..11f9de9734 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/index.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/index.ts @@ -12,4 +12,6 @@ export { export { collectIosStructuralIdentifierSuppression } from './noise-structural.ts'; export { findNearestScrollableContainer, mergeReplacement, updateReplacement } from './tree.ts'; export { IosSnapshotEngineError } from './types.ts'; +export { toIosSnapshotEngineErrorDetails } from './errors.ts'; +export { resolveIosViewportEvidenceFromRoots } from './viewport.ts'; export type { SnapshotTreeRuleContext } from './tree.ts'; diff --git a/packages/capture-kit/src/ios-snapshot-engine/viewport.test.ts b/packages/capture-kit/src/ios-snapshot-engine/viewport.test.ts new file mode 100644 index 0000000000..009cda3bb6 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/viewport.test.ts @@ -0,0 +1,41 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { resolveIosViewportEvidenceFromRoots } from './viewport.ts'; + +test('viewport evidence prefers reported application and window roots', () => { + assert.deepEqual( + resolveIosViewportEvidenceFromRoots([ + { type: 'XCUIElementTypeApplication', rectStatus: 'invalid' }, + { + type: 'XCUIElementTypeWindow', + rect: { x: 0, y: 0, width: 390, height: 844 }, + rectStatus: 'reported', + }, + ]), + { kind: 'reported', rect: { x: 0, y: 0, width: 390, height: 844 } }, + ); +}); + +test('viewport evidence can fall back to the largest top-level root', () => { + assert.deepEqual( + resolveIosViewportEvidenceFromRoots( + [ + { type: 'Other', rect: { x: 0, y: 0, width: 100, height: 100 } }, + { type: 'Other', rect: { x: 0, y: 0, width: 200, height: 300 } }, + ], + { fallbackToLargestRoot: true }, + ), + { kind: 'reported', rect: { x: 0, y: 0, width: 200, height: 300 } }, + ); +}); + +test('viewport evidence preserves explicit missing geometry reasons', () => { + assert.deepEqual( + resolveIosViewportEvidenceFromRoots([{ type: 'Application', rectStatus: 'invalid' }]), + { kind: 'missing', reason: 'invalid' }, + ); + assert.deepEqual( + resolveIosViewportEvidenceFromRoots([{ type: 'Application', rectStatus: 'not-provided' }]), + { kind: 'missing', reason: 'not-provided' }, + ); +}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/viewport.ts b/packages/capture-kit/src/ios-snapshot-engine/viewport.ts new file mode 100644 index 0000000000..ef0f8db9c8 --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/viewport.ts @@ -0,0 +1,50 @@ +import type { IosViewportEvidence } from '@agent-device/contracts/ios-snapshot'; +import { normalizeType } from '@agent-device/contracts/snapshot'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; + +export type IosSnapshotViewportRoot = Readonly<{ + type?: string; + rect?: Rect; + rectStatus?: 'reported' | 'invalid' | 'not-provided'; +}>; + +export function resolveIosViewportEvidenceFromRoots( + roots: readonly IosSnapshotViewportRoot[], + options: Readonly<{ fallbackToLargestRoot?: boolean }> = {}, +): IosViewportEvidence | undefined { + const viewportRoots = roots.filter(isViewportRoot); + const candidates = + viewportRoots.length > 0 || options.fallbackToLargestRoot !== true ? viewportRoots : roots; + const root = [...candidates].sort(compareViewportRoots)[0]; + if (!root) return undefined; + if (isPositiveFiniteRect(root.rect)) return { kind: 'reported', rect: root.rect }; + return { + kind: 'missing', + reason: + root.rectStatus === 'invalid' || (root.rectStatus === undefined && root.rect !== undefined) + ? 'invalid' + : 'not-provided', + }; +} + +function isViewportRoot(root: IosSnapshotViewportRoot): boolean { + const type = normalizeType(root.type ?? ''); + return type === 'application' || type === 'window'; +} + +function compareViewportRoots( + left: IosSnapshotViewportRoot, + right: IosSnapshotViewportRoot, +): number { + const status = rootGeometryRank(right.rectStatus) - rootGeometryRank(left.rectStatus); + return status || rectArea(right.rect) - rectArea(left.rect); +} + +function rootGeometryRank(status: IosSnapshotViewportRoot['rectStatus']): number { + return status === 'reported' ? 2 : status === 'invalid' ? 1 : 0; +} + +function rectArea(rect: Rect | undefined): number { + return rect && isPositiveFiniteRect(rect) ? rect.width * rect.height : 0; +} diff --git a/packages/capture-kit/src/ios-snapshot-planning.test.ts b/packages/capture-kit/src/ios-snapshot-planning.test.ts index 4aadf3d737..1b5dbbbb0e 100644 --- a/packages/capture-kit/src/ios-snapshot-planning.test.ts +++ b/packages/capture-kit/src/ios-snapshot-planning.test.ts @@ -147,6 +147,10 @@ test('presented producers cannot claim acquisition narrowing', () => { }); }); +test('Appium source advertises viewport evidence when its root reports geometry', () => { + assert.equal(IOS_SNAPSHOT_PRODUCER_CAPABILITIES['appium-source'].viewportEvidence, 'available'); +}); + test('comparison identity rejects every identity axis and residue mismatch', () => { const base = comparisonIdentity(); const mismatches: IosSnapshotComparisonIdentity[] = [ diff --git a/packages/capture-kit/src/ios-snapshot-planning.ts b/packages/capture-kit/src/ios-snapshot-planning.ts index 37fa1513fb..575335f8a3 100644 --- a/packages/capture-kit/src/ios-snapshot-planning.ts +++ b/packages/capture-kit/src/ios-snapshot-planning.ts @@ -46,7 +46,7 @@ const IOS_SNAPSHOT_PRODUCER_CAPABILITY_VALUES = { }, scopeCompleteness: 'incomplete', interactiveQueryCompleteness: 'incomplete', - viewportEvidence: 'unavailable', + viewportEvidence: 'available', hittabilityEvidence: 'unavailable', }, 'limrun-ios-tree': { diff --git a/packages/contracts/src/snapshot-private-evidence.ts b/packages/contracts/src/snapshot-private-evidence.ts index e0f41be2cb..43e524f022 100644 --- a/packages/contracts/src/snapshot-private-evidence.ts +++ b/packages/contracts/src/snapshot-private-evidence.ts @@ -28,6 +28,7 @@ export type SnapshotOcclusionContextEvidence = { androidSiblingOrderByNodeIndex?: ReadonlyMap; }; +/** Identifies the engine that owns interactive iOS snapshot presentation. */ export type SnapshotPresentationEvidence = { owner: 'ios-snapshot-engine'; }; diff --git a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts index d4a2429247..6b64b3ba1d 100644 --- a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts @@ -275,13 +275,12 @@ test('snapshot reports typed runner presentation failures', async () => { { runCommand: async () => ({ nodes: [{ index: 0, type: 'Application' }] }) }, ); - await assert.rejects( - interactor.snapshot(), - (error: unknown) => - error instanceof AppError && - error.code === 'COMMAND_FAILED' && - error.details?.reason === 'missing-viewport', - ); + await assert.rejects(interactor.snapshot(), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.deepEqual(error.details, { reason: 'missing-viewport', field: 'viewport' }); + return true; + }); }); test('sparse runner payloads with no viewport fail before publishing actionable nodes', async () => { diff --git a/packages/platform-apple/src/runner/snapshot-presentation.ts b/packages/platform-apple/src/runner/snapshot-presentation.ts index 87473d2a35..d1a54f11d6 100644 --- a/packages/platform-apple/src/runner/snapshot-presentation.ts +++ b/packages/platform-apple/src/runner/snapshot-presentation.ts @@ -4,17 +4,17 @@ import type { IosViewportEvidence, } from '@agent-device/contracts/ios-snapshot'; import type { SnapshotOptions } from '@agent-device/contracts/interactor-types'; -import { normalizeType } from '@agent-device/contracts/snapshot'; import { IosSnapshotEngineError, presentIosRunnerSnapshot, + resolveIosViewportEvidenceFromRoots, + toIosSnapshotEngineErrorDetails, } from '@agent-device/capture-kit/ios-snapshot-engine'; import { readSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import { createIosSnapshotRequest, buildIosSnapshotPresentationKey, } from '@agent-device/capture-kit/ios-snapshot-planning'; -import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import { AppError } from '@agent-device/kernel/errors'; import type { RawSnapshotNode, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; @@ -119,36 +119,18 @@ function runnerViewportEvidence( qualityNodes: readonly RawSnapshotNode[] | undefined, ): IosViewportEvidence { return ( - readReportedViewport(qualityNodes) ?? - readReportedViewport(nodes) ?? { kind: 'missing', reason: 'not-provided' } + resolveIosViewportEvidenceFromRoots(rootNodes(qualityNodes), { + fallbackToLargestRoot: true, + }) ?? + resolveIosViewportEvidenceFromRoots(rootNodes(nodes), { fallbackToLargestRoot: true }) ?? { + kind: 'missing', + reason: 'not-provided', + } ); } -function readReportedViewport( - nodes: readonly RawSnapshotNode[] | undefined, -): IosViewportEvidence | undefined { - const roots = nodes?.filter((node) => node.parentIndex === undefined) ?? []; - const root = - [...roots] - .filter((node) => isViewportRoot(node)) - .sort(compareRectArea) - .at(0) ?? [...roots].sort(compareRectArea).at(0); - if (!root) return undefined; - if (isPositiveFiniteRect(root.rect)) return { kind: 'reported', rect: root.rect }; - return { kind: 'missing', reason: root.rect ? 'invalid' : 'not-provided' }; -} - -function isViewportRoot(node: RawSnapshotNode): boolean { - const type = normalizeType(node.type ?? ''); - return type === 'application' || type === 'window'; -} - -function compareRectArea(left: RawSnapshotNode, right: RawSnapshotNode): number { - return rectArea(right.rect) - rectArea(left.rect); -} - -function rectArea(rect: RawSnapshotNode['rect']): number { - return rect ? rect.width * rect.height : 0; +function rootNodes(nodes: readonly RawSnapshotNode[] | undefined): readonly RawSnapshotNode[] { + return nodes?.filter((node) => node.parentIndex === undefined) ?? []; } function isRecord(value: unknown): value is Record { @@ -160,7 +142,7 @@ function throwSnapshotEngineError(error: unknown): never { throw new AppError( 'COMMAND_FAILED', error.message, - { reason: error.reason, iosSnapshotEngine: { details: error.details } }, + toIosSnapshotEngineErrorDetails(error), error, ); } diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index b92d75c761..cff36669a0 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -307,7 +307,6 @@ class WebDriverInteractor implements Interactor { const { captureWebDriverIosSnapshot } = await import('./webdriver-ios-snapshot.ts'); return await captureWebDriverIosSnapshot(this.client, options, this.targetId); } - // Spelled as a correlated pair per channel so the SnapshotProvenance union accepts it. return { backend: 'android' as const, producer: 'appium-source' as const, diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index ccce37ac94..1996b40b1d 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import { test, vi } from 'vitest'; +import { readSnapshotPresentationEvidence } from '@agent-device/contracts/capture'; import { AppError } from '@agent-device/kernel/errors'; import { acquireWebDriverIosSnapshot, @@ -44,6 +45,7 @@ test('Appium iOS snapshots acquire facts and publish regular output through the result.nodes?.every((node) => !('ref' in node)), true, ); + assert.deepEqual(readSnapshotPresentationEvidence(result), { owner: 'ios-snapshot-engine' }); }); test('Appium iOS options become an engine plan and engine-owned projection', () => { @@ -153,7 +155,7 @@ test('Appium iOS truncation is typed and disclosed at response level', async () assert.equal(result.truncated, true); assert.deepEqual(result.warnings, [ - 'Appium page source does not provide hittability evidence; regular snapshot nodes are not actionable.', + 'Appium page source does not provide hittability evidence; the capture carries no hittability fact.', 'Appium page source is truncated; the snapshot hierarchy may be incomplete.', ]); }); diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index 09d26dad05..ec3c40f861 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -1,6 +1,8 @@ import { createIosSnapshotEngine, IosSnapshotEngineError, + resolveIosViewportEvidenceFromRoots, + toIosSnapshotEngineErrorDetails, } from '@agent-device/capture-kit/ios-snapshot-engine'; import { attachSnapshotPresentationEvidence } from '@agent-device/contracts/capture'; import { @@ -17,12 +19,10 @@ import type { IosViewportEvidence, } from '@agent-device/contracts/ios-snapshot'; import type { SnapshotOptions, SnapshotResult } from '@agent-device/contracts/interactor-types'; -import { normalizeType } from '@agent-device/contracts/snapshot'; -import type { RawSnapshotNode, Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { RawSnapshotNode, SnapshotNode } from '@agent-device/kernel/snapshot'; import { AppError } from '@agent-device/kernel/errors'; -import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; import type { WebDriverClient } from './webdriver-client.ts'; -import { parseWebDriverSourceFacts, type WebDriverSourceRootFact } from './webdriver-source.ts'; +import { parseWebDriverSourceFacts } from './webdriver-source.ts'; const APPIUM_PRODUCER = IOS_SNAPSHOT_PRODUCER_CAPABILITIES['appium-source']; const iosSnapshotEngine = createIosSnapshotEngine(); @@ -67,7 +67,10 @@ export function acquireWebDriverIosSnapshot( }); const plan = iosSnapshotEngine.plan(request, APPIUM_PRODUCER); const sourceFacts = parseWebDriverSourceFacts(source, { mode: 'facts' }); - const viewport = viewportEvidence(sourceFacts.roots); + const viewport = resolveIosViewportEvidenceFromRoots(sourceFacts.roots) ?? { + kind: 'missing' as const, + reason: 'not-provided' as const, + }; const residue = residueForSource(sourceFacts.truncated, viewport); const common = { producer: 'appium-source' as const, @@ -107,35 +110,6 @@ export function publishWebDriverIosSnapshot( return { acquisition, publication, result }; } -function viewportEvidence(roots: readonly WebDriverSourceRootFact[]): IosViewportEvidence { - const candidates = roots.filter((node) => { - const type = normalizeType(node.type ?? ''); - return type === 'application' || type === 'window'; - }); - const root = [...candidates].sort(compareViewportRoots)[0]; - if (!root) return { kind: 'missing', reason: 'not-provided' }; - if (root.rectStatus === 'reported' && isPositiveFiniteRect(root.rect)) { - return { kind: 'reported', rect: root.rect }; - } - return { kind: 'missing', reason: root.rectStatus === 'invalid' ? 'invalid' : 'not-provided' }; -} - -function rectArea(rect: Rect | undefined): number { - return rect && isPositiveFiniteRect(rect) ? rect.width * rect.height : 0; -} - -function compareViewportRoots( - left: WebDriverSourceRootFact, - right: WebDriverSourceRootFact, -): number { - const status = rootGeometryRank(right.rectStatus) - rootGeometryRank(left.rectStatus); - return status || rectArea(right.rect) - rectArea(left.rect); -} - -function rootGeometryRank(status: WebDriverSourceRootFact['rectStatus']): number { - return status === 'reported' ? 2 : status === 'invalid' ? 1 : 0; -} - function residueForSource( truncated: boolean, viewport: IosViewportEvidence, @@ -160,7 +134,7 @@ function warningsForResidue(residue: readonly IosAcquisitionResidue[]): { warnin function warningForResidue(entry: IosAcquisitionResidue): string | undefined { if (entry.kind === 'unavailable-fact') { return entry.fact === 'hittability' - ? 'Appium page source does not provide hittability evidence; regular snapshot nodes are not actionable.' + ? 'Appium page source does not provide hittability evidence; the capture carries no hittability fact.' : undefined; } if (entry.kind === 'missing-viewport' || entry.kind === 'truncated') { @@ -178,10 +152,7 @@ function throwWebDriverIosSnapshotError(error: unknown): never { throw new AppError( 'COMMAND_FAILED', error.message, - { - reason: error.reason, - ...(error.details.field ? { field: error.details.field } : {}), - }, + toIosSnapshotEngineErrorDetails(error), error, ); } diff --git a/packages/provider-webdriver/src/webdriver-scroll-frame.ts b/packages/provider-webdriver/src/webdriver-scroll-frame.ts index c97728cf18..5b3a265c20 100644 --- a/packages/provider-webdriver/src/webdriver-scroll-frame.ts +++ b/packages/provider-webdriver/src/webdriver-scroll-frame.ts @@ -3,7 +3,7 @@ import type { WebDriverWindowRect } from './webdriver-client.ts'; import { parseWebDriverSource } from './webdriver-source.ts'; export function scrollFrameFromWebDriverSource(source: string): WebDriverWindowRect | undefined { - const rect = parseWebDriverSource(source) + const rect = parseWebDriverSource(source, { mode: 'facts' }) .flatMap((node) => isScrollableSourceNode(node) && isUsableScrollRect(node.rect) ? [node.rect] : [], ) diff --git a/packages/provider-webdriver/src/webdriver-source.test.ts b/packages/provider-webdriver/src/webdriver-source.test.ts index ddb5bcf18f..ecc26555f6 100644 --- a/packages/provider-webdriver/src/webdriver-source.test.ts +++ b/packages/provider-webdriver/src/webdriver-source.test.ts @@ -32,6 +32,14 @@ test('WebDriver source facts do not fill absent provider attributes', () => { assert.equal('hittable' in (node ?? {}), false); }); +test('WebDriver source facts preserve explicitly reported hittability', () => { + const node = parseWebDriverSource( + '', + )[0]; + + assert.equal(node?.hittable, true); +}); + test('legacy WebDriver parsing keeps Android-derived hittability explicit at its call site', () => { const node = parseWebDriverSource( '', @@ -43,6 +51,19 @@ test('legacy WebDriver parsing keeps Android-derived hittability explicit at its assert.equal(node?.hittable, true); }); +test('legacy WebDriver parsing keeps Android source booleans and ignores iOS hints', () => { + const node = parseWebDriverSource( + '', + { mode: 'legacy-derived' }, + )[0]; + + assert.equal(node?.enabled, false); + assert.equal(node?.visibleToUser, false); + assert.equal(node?.selected, false); + assert.equal(node?.focused, false); + assert.equal(node?.hittable, false); +}); + test('WebDriver source facts expose provider truncation without publishing wrapper nodes', () => { const facts = parseWebDriverSourceFacts( '', diff --git a/packages/provider-webdriver/src/webdriver-source.ts b/packages/provider-webdriver/src/webdriver-source.ts index 0da70345d9..0d8869eb54 100644 --- a/packages/provider-webdriver/src/webdriver-source.ts +++ b/packages/provider-webdriver/src/webdriver-source.ts @@ -8,6 +8,7 @@ export type WebDriverSourceParseMode = 'facts' | 'legacy-derived'; export type WebDriverSourceFacts = Readonly<{ nodes: RawSnapshotNode[]; roots: readonly WebDriverSourceRootFact[]; + /** True only when the source explicitly marks the hierarchy as truncated. */ truncated: boolean; }>; @@ -35,7 +36,7 @@ export function parseWebDriverSourceFacts( for (const root of roots) { appendSourceNodes(nodes, root, undefined, 0, mode, sourceRoots); } - return { nodes, roots: sourceRoots, truncated: hasTruncationMarker(roots) }; + return { nodes, roots: sourceRoots, truncated: hasExplicitTruncationMarker(roots) }; } function appendSourceNodes( @@ -134,15 +135,26 @@ function sourceStateFacts( rect: RawSnapshotNode['rect'], mode: WebDriverSourceParseMode, ): Partial { - const legacyDerived = mode === 'legacy-derived'; + if (mode === 'legacy-derived') { + const enabled = legacyBooleanAttribute(attrs.enabled, true); + const visibleToUser = legacyBooleanAttribute(attrs.displayed ?? attrs.visible, true); + return { + enabled, + selected: legacyBooleanAttribute(attrs.selected), + focused: legacyBooleanAttribute(attrs.focused), + visibleToUser, + hittable: visibleToUser && enabled && isPositiveRect(rect), + }; + } + const enabled = booleanAttribute(attrs.enabled); const visibleToUser = booleanAttribute(attrs.displayed ?? attrs.visible); return { - ...optionalBooleanFact('enabled', enabled, legacyDerived), + ...optionalBooleanFact('enabled', enabled, false), selected: booleanAttribute(attrs.selected), focused: booleanAttribute(attrs.focused), - ...optionalBooleanFact('visibleToUser', visibleToUser, legacyDerived), - ...hittabilityFact(attrs.hittable, visibleToUser, enabled, rect, legacyDerived), + ...optionalBooleanFact('visibleToUser', visibleToUser, false), + ...reportedHittabilityFact(attrs.hittable), }; } @@ -154,24 +166,18 @@ function optionalBooleanFact( return defaultWhenAbsent || value !== undefined ? { [key]: value ?? true } : {}; } -function hittabilityFact( +function reportedHittabilityFact( reported: string | undefined, - visibleToUser: boolean | undefined, - enabled: boolean | undefined, - rect: RawSnapshotNode['rect'], - legacyDerived: boolean, ): Partial> { const reportedHittable = booleanAttribute(reported); - if (reportedHittable !== undefined) return { hittable: reportedHittable }; - return legacyDerived - ? { hittable: (visibleToUser ?? true) && (enabled ?? true) && isPositiveRect(rect) } - : {}; + return reportedHittable === undefined ? {} : { hittable: reportedHittable }; } -function hasTruncationMarker(nodes: readonly XmlNode[]): boolean { +function hasExplicitTruncationMarker(nodes: readonly XmlNode[]): boolean { return nodes.some( (node) => - booleanAttribute(node.attributes.truncated) === true || hasTruncationMarker(node.children), + booleanAttribute(node.attributes.truncated) === true || + hasExplicitTruncationMarker(node.children), ); } @@ -210,6 +216,11 @@ function booleanAttribute(value: string | undefined): boolean | undefined { return undefined; } +function legacyBooleanAttribute(value: string | undefined, defaultValue = false): boolean { + if (value === undefined) return defaultValue; + return value === 'true' || value === '1'; +} + function isPositiveRect(rect: RawSnapshotNode['rect']): boolean { return Boolean(rect && rect.width > 0 && rect.height > 0); } diff --git a/src/core/snapshot-state.ts b/src/core/snapshot-state.ts index dc4166114e..ad46d4407a 100644 --- a/src/core/snapshot-state.ts +++ b/src/core/snapshot-state.ts @@ -128,10 +128,11 @@ function shouldPresentLegacyIosInteractiveSnapshot( Partial>) | undefined, ): boolean { + const presentationEvidence = readSnapshotPresentationEvidence(provenance); return ( provenance.backend === 'xctest' && iosSnapshotPresentationStage(provenance) === 'acquired' && - readSnapshotPresentationEvidence(provenance) === undefined && + presentationEvidence?.owner !== 'ios-snapshot-engine' && flags?.snapshotInteractiveOnly === true && flags.snapshotRaw !== true ); diff --git a/src/daemon/__tests__/snapshot-presentation-transitions.test.ts b/src/daemon/__tests__/snapshot-presentation-transitions.test.ts index c6545376ab..f9123598dc 100644 --- a/src/daemon/__tests__/snapshot-presentation-transitions.test.ts +++ b/src/daemon/__tests__/snapshot-presentation-transitions.test.ts @@ -7,10 +7,7 @@ import { navigationTitleWithAppProvidedDetailsAffordanceNodes } from '../../snap test('iOS daemon presentation applies transitions without reapplying runner-owned scope', () => { const snapshot = buildSnapshotState( - { - nodes: navigationTitleWithAppProvidedDetailsAffordanceNodes, - backend: 'xctest', - }, + { nodes: navigationTitleWithAppProvidedDetailsAffordanceNodes, backend: 'xctest' }, { snapshotInteractiveOnly: true, snapshotScope: 'DisplayNameTextField' }, ); From 0fd991f42ee4cf5470071fe0fecb24158a87697e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 21:39:32 +0200 Subject: [PATCH 08/15] fix(ios): preserve unavailable snapshot facts --- .../src/ios-snapshot-engine/engine.test.ts | 4 ++ .../src/ios-snapshot-engine/geometry.ts | 35 +++++++++++++++--- .../src/ios-snapshot-engine/projection.ts | 6 ++- .../src/ios-snapshot-planning.test.ts | 8 +++- .../src/webdriver-ios-snapshot.test.ts | 37 ++++++++++++++----- .../src/webdriver-ios-snapshot.ts | 31 +++++++++------- .../src/webdriver-source.test.ts | 6 ++- .../src/webdriver-source.ts | 27 ++++---------- 8 files changed, 99 insertions(+), 55 deletions(-) diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts index 3e09419879..0c7f499da7 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts @@ -236,6 +236,10 @@ test('unavailable hittability never becomes regular actionability', () => { const acquired = publishIosSnapshot({ stage: 'acquired', acquisition: unavailable }, request); assert.equal( acquired.payload.nodes.find((node) => node.label === 'Partially visible')?.hittable, + undefined, + ); + assert.equal( + 'hittable' in (acquired.payload.nodes.find((node) => node.label === 'Partially visible') ?? {}), false, ); diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts index 6a83656827..3c1406e946 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts @@ -107,17 +107,21 @@ function appendFoldedNode( const index = kept.length; keptDepth += 1; + const { hittable: sourceHittable, ...sourceNode } = node; kept.push({ raw: { - ...node, + ...sourceNode, index, depth: keptDepth, parentIndex: keptIndex, - hittable: - node.parentIndex !== undefined && - options.hittabilityAvailable !== false && - node.hittable === true && - isGeometricallyActionable(node.enabled !== false, decision.effectiveRect, viewport), + ...foldedHittability( + sourceHittable, + node.parentIndex !== undefined, + decision.effectiveRect, + node.enabled !== false, + viewport, + options, + ), }, sourceIndex: node.index, ...(decision.effectiveRect ? { effectiveRect: decision.effectiveRect } : {}), @@ -126,6 +130,25 @@ function appendFoldedNode( return { keptIndex, keptDepth }; } +function foldedHittability( + sourceHittable: RawSnapshotNode['hittable'], + hasParent: boolean, + effectiveRect: Rect | undefined, + enabled: boolean, + viewport: Rect, + options: IosSnapshotFoldOptions, +): Partial> { + if (options.hittabilityAvailable === false) { + return sourceHittable === false ? { hittable: false } : {}; + } + return { + hittable: + hasParent && + sourceHittable === true && + isGeometricallyActionable(enabled, effectiveRect, viewport), + }; +} + function nextScrollAnchor( decision: GeometryDecision, parentAnchor: BranchState['anchor'], diff --git a/packages/capture-kit/src/ios-snapshot-engine/projection.ts b/packages/capture-kit/src/ios-snapshot-engine/projection.ts index d0a10ba727..1eea69c80b 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/projection.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/projection.ts @@ -142,17 +142,19 @@ function createRegularProjectedNode( if (!isEligibleForIosRegularPresentation(node.raw)) return undefined; const depth = parent ? (parent.depth ?? 0) + 1 : 0; if (maximumDepth !== null && depth > maximumDepth) return undefined; + const hittable = isProjectedNodeHittable(node); return { ...node.raw, index, depth, parentIndex: parent?.index, ...(node.effectiveRect ? { rect: node.effectiveRect } : { rect: undefined }), - hittable: isProjectedNodeHittable(node), + ...(hittable === undefined ? {} : { hittable }), }; } -function isProjectedNodeHittable(node: IosSnapshotPresentationNode): boolean { +function isProjectedNodeHittable(node: IosSnapshotPresentationNode): RawSnapshotNode['hittable'] { + if (node.raw.hittable === undefined) return undefined; return Boolean( node.raw.hittable === true && node.effectiveRect && diff --git a/packages/capture-kit/src/ios-snapshot-planning.test.ts b/packages/capture-kit/src/ios-snapshot-planning.test.ts index 1b5dbbbb0e..025ae2f245 100644 --- a/packages/capture-kit/src/ios-snapshot-planning.test.ts +++ b/packages/capture-kit/src/ios-snapshot-planning.test.ts @@ -147,8 +147,12 @@ test('presented producers cannot claim acquisition narrowing', () => { }); }); -test('Appium source advertises viewport evidence when its root reports geometry', () => { - assert.equal(IOS_SNAPSHOT_PRODUCER_CAPABILITIES['appium-source'].viewportEvidence, 'available'); +test('Appium source plan carries its viewport evidence capability', () => { + const plan = planIosSnapshot( + createIosSnapshotRequest(), + IOS_SNAPSHOT_PRODUCER_CAPABILITIES['appium-source'], + ); + assert.equal(plan.evidence.viewport, 'available'); }); test('comparison identity rejects every identity axis and residue mismatch', () => { diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index 1996b40b1d..af22720ff1 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -31,7 +31,7 @@ test('Appium iOS snapshots acquire facts and publish regular output through the ['XCUIElementTypeButton', 'Continue', 1], ], ); - assert.equal(result.nodes?.find((node) => node.label === 'Continue')?.hittable, false); + assert.equal(result.nodes?.find((node) => node.label === 'Continue')?.hittable, undefined); assert.equal(source.mock.calls.length, 1); assert.ok(result.warnings?.some((warning) => warning.includes('hittability evidence'))); assert.deepEqual(Object.keys(result).sort(), [ @@ -85,6 +85,22 @@ test('Appium iOS options become an engine plan and engine-owned projection', () assert.deepEqual(published.publication.comparisonIdentity.lineage, {}); }); +test('Appium regular presentation omits unavailable hittability while raw preserves supplied facts', async () => { + const source = + '' + + '' + + ''; + + const regular = await captureWebDriverIosSnapshot({ source: async () => source }); + const regularButton = regular.nodes?.find((node) => node.label === 'Continue'); + assert.equal(regularButton?.hittable, undefined); + assert.equal('hittable' in (regularButton ?? {}), false); + + const raw = await captureWebDriverIosSnapshot({ source: async () => source }, { raw: true }); + assert.equal(raw.nodes?.find((node) => node.label === 'Continue')?.hittable, true); + assert.ok(raw.warnings?.some((warning) => warning.includes('does not guarantee'))); +}); + test('Appium iOS interactive requests stay provider-unpruned and use engine presentation', () => { const acquired = acquireWebDriverIosSnapshot(SOURCE, { interactiveOnly: true }, 'ios-1'); @@ -95,7 +111,10 @@ test('Appium iOS interactive requests stay provider-unpruned and use engine pres published.result.nodes?.some((node) => node.label === 'Continue'), true, ); - assert.equal(published.result.nodes?.find((node) => node.label === 'Continue')?.hittable, false); + assert.equal( + published.result.nodes?.find((node) => node.label === 'Continue')?.hittable, + undefined, + ); }); test('Appium iOS regular presentation fails typed when page source has no viewport', () => { @@ -109,6 +128,7 @@ test('Appium iOS regular presentation fails typed when page source has no viewpo }); assert.deepEqual(acquired.input.acquisition.residue, [ { kind: 'unavailable-fact', fact: 'hittability' }, + { kind: 'unavailable-fact', fact: 'acquisition-depth' }, { kind: 'missing-viewport', reason: 'not-provided' }, ]); assert.throws( @@ -147,15 +167,12 @@ test('Appium iOS regular presentation fails typed when root geometry is invalid' ); }); -test('Appium iOS truncation is typed and disclosed at response level', async () => { - const result = await captureWebDriverIosSnapshot( - { source: async () => SOURCE.replace('', '') }, - { raw: true }, - ); +test('Appium iOS hierarchy limits are typed and disclosed at response level', async () => { + const result = await captureWebDriverIosSnapshot({ source: async () => SOURCE }, { raw: true }); - assert.equal(result.truncated, true); + assert.equal(result.truncated, false); assert.deepEqual(result.warnings, [ - 'Appium page source does not provide hittability evidence; the capture carries no hittability fact.', - 'Appium page source is truncated; the snapshot hierarchy may be incomplete.', + 'Appium page source does not guarantee hittability evidence; regular presentation treats it as unavailable.', + 'Appium page source does not report hierarchy completeness; depth- or child-limited nodes may be absent.', ]); }); diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index ec3c40f861..2d1eee9c79 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -16,6 +16,7 @@ import type { IosSnapshotPlan, IosSnapshotPublication, IosSnapshotRequest, + IosSnapshotFact, IosViewportEvidence, } from '@agent-device/contracts/ios-snapshot'; import type { SnapshotOptions, SnapshotResult } from '@agent-device/contracts/interactor-types'; @@ -29,8 +30,13 @@ const iosSnapshotEngine = createIosSnapshotEngine(); const RESIDUE_WARNINGS = { 'missing-viewport': 'Appium page source does not provide a valid viewport; regular snapshot presentation is unavailable.', - truncated: 'Appium page source is truncated; the snapshot hierarchy may be incomplete.', -} satisfies Pick, 'missing-viewport' | 'truncated'>; +} satisfies Pick, 'missing-viewport'>; +const UNAVAILABLE_FACT_WARNINGS: Partial> = { + 'acquisition-depth': + 'Appium page source does not report hierarchy completeness; depth- or child-limited nodes may be absent.', + hittability: + 'Appium page source does not guarantee hittability evidence; regular presentation treats it as unavailable.', +}; export type WebDriverIosSnapshotAcquisition = Readonly<{ request: IosSnapshotRequest; @@ -71,11 +77,11 @@ export function acquireWebDriverIosSnapshot( kind: 'missing' as const, reason: 'not-provided' as const, }; - const residue = residueForSource(sourceFacts.truncated, viewport); + const residue = residueForSource(viewport); const common = { producer: 'appium-source' as const, nodes: sourceFacts.nodes, - truncated: sourceFacts.truncated, + truncated: false, viewport, lineage: targetId ? { targetId } : {}, residue, @@ -110,15 +116,14 @@ export function publishWebDriverIosSnapshot( return { acquisition, publication, result }; } -function residueForSource( - truncated: boolean, - viewport: IosViewportEvidence, -): readonly IosAcquisitionResidue[] { - const residue: IosAcquisitionResidue[] = [{ kind: 'unavailable-fact', fact: 'hittability' }]; +function residueForSource(viewport: IosViewportEvidence): readonly IosAcquisitionResidue[] { + const residue: IosAcquisitionResidue[] = [ + { kind: 'unavailable-fact', fact: 'hittability' }, + { kind: 'unavailable-fact', fact: 'acquisition-depth' }, + ]; if (viewport.kind === 'missing') { residue.push({ kind: 'missing-viewport', reason: viewport.reason }); } - if (truncated) residue.push({ kind: 'truncated', dimension: 'nodes' }); return residue; } @@ -133,11 +138,9 @@ function warningsForResidue(residue: readonly IosAcquisitionResidue[]): { warnin function warningForResidue(entry: IosAcquisitionResidue): string | undefined { if (entry.kind === 'unavailable-fact') { - return entry.fact === 'hittability' - ? 'Appium page source does not provide hittability evidence; the capture carries no hittability fact.' - : undefined; + return UNAVAILABLE_FACT_WARNINGS[entry.fact]; } - if (entry.kind === 'missing-viewport' || entry.kind === 'truncated') { + if (entry.kind === 'missing-viewport') { return RESIDUE_WARNINGS[entry.kind]; } return undefined; diff --git a/packages/provider-webdriver/src/webdriver-source.test.ts b/packages/provider-webdriver/src/webdriver-source.test.ts index ecc26555f6..1c628b6d5c 100644 --- a/packages/provider-webdriver/src/webdriver-source.test.ts +++ b/packages/provider-webdriver/src/webdriver-source.test.ts @@ -28,6 +28,8 @@ test('WebDriver source facts do not fill absent provider attributes', () => { assert.equal(node?.label, 'Continue'); assert.equal(node?.identifier, 'Continue'); assert.equal('enabled' in (node ?? {}), false); + assert.equal('selected' in (node ?? {}), false); + assert.equal('focused' in (node ?? {}), false); assert.equal('visibleToUser' in (node ?? {}), false); assert.equal('hittable' in (node ?? {}), false); }); @@ -64,12 +66,12 @@ test('legacy WebDriver parsing keeps Android source booleans and ignores iOS hin assert.equal(node?.hittable, false); }); -test('WebDriver source facts expose provider truncation without publishing wrapper nodes', () => { +test('WebDriver source facts preserve roots without claiming hierarchy completeness', () => { const facts = parseWebDriverSourceFacts( '', ); - assert.equal(facts.truncated, true); + assert.equal('truncated' in facts, false); assert.deepEqual(facts.roots, [ { type: 'XCUIElementTypeApplication', diff --git a/packages/provider-webdriver/src/webdriver-source.ts b/packages/provider-webdriver/src/webdriver-source.ts index 0d8869eb54..3d14611467 100644 --- a/packages/provider-webdriver/src/webdriver-source.ts +++ b/packages/provider-webdriver/src/webdriver-source.ts @@ -8,8 +8,6 @@ export type WebDriverSourceParseMode = 'facts' | 'legacy-derived'; export type WebDriverSourceFacts = Readonly<{ nodes: RawSnapshotNode[]; roots: readonly WebDriverSourceRootFact[]; - /** True only when the source explicitly marks the hierarchy as truncated. */ - truncated: boolean; }>; export type WebDriverSourceRootFact = Readonly<{ @@ -36,7 +34,7 @@ export function parseWebDriverSourceFacts( for (const root of roots) { appendSourceNodes(nodes, root, undefined, 0, mode, sourceRoots); } - return { nodes, roots: sourceRoots, truncated: hasExplicitTruncationMarker(roots) }; + return { nodes, roots: sourceRoots }; } function appendSourceNodes( @@ -150,20 +148,19 @@ function sourceStateFacts( const enabled = booleanAttribute(attrs.enabled); const visibleToUser = booleanAttribute(attrs.displayed ?? attrs.visible); return { - ...optionalBooleanFact('enabled', enabled, false), - selected: booleanAttribute(attrs.selected), - focused: booleanAttribute(attrs.focused), - ...optionalBooleanFact('visibleToUser', visibleToUser, false), + ...optionalBooleanFact('enabled', enabled), + ...optionalBooleanFact('selected', booleanAttribute(attrs.selected)), + ...optionalBooleanFact('focused', booleanAttribute(attrs.focused)), + ...optionalBooleanFact('visibleToUser', visibleToUser), ...reportedHittabilityFact(attrs.hittable), }; } function optionalBooleanFact( - key: 'enabled' | 'visibleToUser', + key: 'enabled' | 'selected' | 'focused' | 'visibleToUser', value: boolean | undefined, - defaultWhenAbsent: boolean, -): Partial { - return defaultWhenAbsent || value !== undefined ? { [key]: value ?? true } : {}; +): Partial> { + return value === undefined ? {} : { [key]: value }; } function reportedHittabilityFact( @@ -173,14 +170,6 @@ function reportedHittabilityFact( return reportedHittable === undefined ? {} : { hittable: reportedHittable }; } -function hasExplicitTruncationMarker(nodes: readonly XmlNode[]): boolean { - return nodes.some( - (node) => - booleanAttribute(node.attributes.truncated) === true || - hasExplicitTruncationMarker(node.children), - ); -} - function rectFromAttributes(attrs: Record): RawSnapshotNode['rect'] | undefined { const bounds = parseBounds(attrs.bounds ?? null); if (bounds) return bounds; From 85dcf8090eab0caefdef4fa5797a3468662b7901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 21:59:11 +0200 Subject: [PATCH 09/15] fix(ios): preserve snapshot error context --- .../src/ios-snapshot-engine/errors.test.ts | 10 +++++++- .../src/ios-snapshot-engine/errors.ts | 20 ++++++++++----- .../src/webdriver-ios-snapshot.test.ts | 25 +++++++++++-------- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/packages/capture-kit/src/ios-snapshot-engine/errors.test.ts b/packages/capture-kit/src/ios-snapshot-engine/errors.test.ts index 1d21f52a0a..74d9133a58 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/errors.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/errors.test.ts @@ -3,15 +3,23 @@ import { test } from 'vitest'; import { toIosSnapshotEngineErrorDetails } from './errors.ts'; import { IosSnapshotEngineError } from './types.ts'; -test('engine error details expose only the public reason and field', () => { +test('engine error details preserve typed public context', () => { const error = new IosSnapshotEngineError('invalid-viewport', 'invalid viewport', { field: 'viewport', index: 4, + parentIndex: 2, frame: { x: 0, y: 0, width: 10, height: 10 }, + clip: { x: 1, y: 2, width: 3, height: 4 }, + projection: 'regular', }); assert.deepEqual(toIosSnapshotEngineErrorDetails(error), { reason: 'invalid-viewport', field: 'viewport', + index: 4, + parentIndex: 2, + frame: { x: 0, y: 0, width: 10, height: 10 }, + clip: { x: 1, y: 2, width: 3, height: 4 }, + projection: 'regular', }); }); diff --git a/packages/capture-kit/src/ios-snapshot-engine/errors.ts b/packages/capture-kit/src/ios-snapshot-engine/errors.ts index 9480a22698..f034097399 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/errors.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/errors.ts @@ -1,15 +1,23 @@ -import type { IosSnapshotEngineError, IosSnapshotEngineFailureReason } from './types.ts'; +import type { + IosSnapshotEngineError, + IosSnapshotEngineFailureDetails, + IosSnapshotEngineFailureReason, +} from './types.ts'; -export type IosSnapshotEnginePublicErrorDetails = Readonly<{ - reason: IosSnapshotEngineFailureReason; - field?: string; -}>; +export type IosSnapshotEnginePublicErrorDetails = Readonly< + IosSnapshotEngineFailureDetails & { reason: IosSnapshotEngineFailureReason } +>; export function toIosSnapshotEngineErrorDetails( error: IosSnapshotEngineError, ): IosSnapshotEnginePublicErrorDetails { return { reason: error.reason, - ...(error.details.field ? { field: error.details.field } : {}), + ...(error.details.index !== undefined ? { index: error.details.index } : {}), + ...(error.details.parentIndex !== undefined ? { parentIndex: error.details.parentIndex } : {}), + ...(error.details.frame !== undefined ? { frame: error.details.frame } : {}), + ...(error.details.clip !== undefined ? { clip: error.details.clip } : {}), + ...(error.details.projection !== undefined ? { projection: error.details.projection } : {}), + ...(error.details.field !== undefined ? { field: error.details.field } : {}), }; } diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index af22720ff1..cc5f31ca7e 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -49,13 +49,17 @@ test('Appium iOS snapshots acquire facts and publish regular output through the }); test('Appium iOS options become an engine plan and engine-owned projection', () => { - const acquired = acquireWebDriverIosSnapshot(SOURCE, { - raw: true, - interactiveOnly: true, - depth: 1, - scope: 'Continue', - customActions: true, - }); + const acquired = acquireWebDriverIosSnapshot( + SOURCE, + { + raw: true, + interactiveOnly: true, + depth: 1, + scope: 'Continue', + customActions: true, + }, + 'ios-2', + ); assert.equal(acquired.input.stage, 'acquired'); assert.deepEqual(acquired.plan.narrowing, { @@ -71,7 +75,7 @@ test('Appium iOS options become an engine plan and engine-owned projection', () customActions: true, acquisitionIntent: 'full', }); - assert.deepEqual(acquired.input.acquisition.lineage, {}); + assert.deepEqual(acquired.input.acquisition.lineage, { targetId: 'ios-2' }); assert.deepEqual(acquired.input.acquisition.viewport, { kind: 'reported', rect: { x: 0, y: 0, width: 390, height: 844 }, @@ -82,7 +86,7 @@ test('Appium iOS options become an engine plan and engine-owned projection', () published.result.nodes?.map((node) => [node.type, node.label, node.depth, node.parentIndex]), [['XCUIElementTypeButton', 'Continue', 0, undefined]], ); - assert.deepEqual(published.publication.comparisonIdentity.lineage, {}); + assert.deepEqual(published.publication.comparisonIdentity.lineage, { targetId: 'ios-2' }); }); test('Appium regular presentation omits unavailable hittability while raw preserves supplied facts', async () => { @@ -93,8 +97,9 @@ test('Appium regular presentation omits unavailable hittability while raw preser const regular = await captureWebDriverIosSnapshot({ source: async () => source }); const regularButton = regular.nodes?.find((node) => node.label === 'Continue'); + assert.ok(regularButton); assert.equal(regularButton?.hittable, undefined); - assert.equal('hittable' in (regularButton ?? {}), false); + assert.equal('hittable' in regularButton, false); const raw = await captureWebDriverIosSnapshot({ source: async () => source }, { raw: true }); assert.equal(raw.nodes?.find((node) => node.label === 'Continue')?.hittable, true); From 0117644fb3c0efb11485d23f37dfbdee3597343a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 22:26:35 +0200 Subject: [PATCH 10/15] fix(ios): centralize snapshot presentation ownership --- .../src/ios-snapshot-engine/engine.test.ts | 61 +++++++++++++++ .../src/ios-snapshot-engine/engine.ts | 51 ++++++++++++- .../src/ios-snapshot-engine/errors.test.ts | 25 ------- .../src/ios-snapshot-engine/errors.ts | 23 ------ .../src/ios-snapshot-engine/index.ts | 5 +- .../src/ios-snapshot-engine/types.ts | 10 +++ .../src/ios-snapshot-engine/viewport.test.ts | 41 ---------- .../src/ios-snapshot-engine/viewport.ts | 50 ------------- .../src/ios-snapshot-planning.test.ts | 1 + .../capture-kit/src/ios-snapshot-planning.ts | 4 + packages/contracts/src/facades/capture.ts | 3 - packages/contracts/src/ios-snapshot.ts | 2 + .../src/snapshot-private-evidence.ts | 19 ----- .../src/webdriver-ios-snapshot.test.ts | 24 +++++- .../src/webdriver-ios-snapshot.ts | 30 ++++---- .../src/webdriver-source.test.ts | 7 +- .../src/webdriver-source.ts | 14 ++-- src/core/__tests__/snapshot-state.test.ts | 74 ++++++++++++++----- src/core/snapshot-state.ts | 13 +++- 19 files changed, 247 insertions(+), 210 deletions(-) delete mode 100644 packages/capture-kit/src/ios-snapshot-engine/errors.test.ts delete mode 100644 packages/capture-kit/src/ios-snapshot-engine/errors.ts delete mode 100644 packages/capture-kit/src/ios-snapshot-engine/viewport.test.ts delete mode 100644 packages/capture-kit/src/ios-snapshot-engine/viewport.ts diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts index 0c7f499da7..948048d95b 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts @@ -11,12 +11,14 @@ import { createIosSnapshotRequest, deriveIosCaptureHint, } from '@agent-device/capture-kit/ios-snapshot-planning'; +import { toIosSnapshotEngineErrorDetails } from './types.ts'; import { compactIosInteractiveSnapshot, createIosSnapshotEngine, IosSnapshotEngineError, presentIosSnapshot, publishIosSnapshot, + resolveIosViewportEvidenceFromRoots, } from './index.ts'; import type { Rect, RawSnapshotNode } from '@agent-device/kernel/snapshot'; @@ -189,6 +191,65 @@ test('regular presentation fails typed when the viewport is missing or the graph ); }); +test('engine error details preserve typed public context', () => { + const error = new IosSnapshotEngineError('invalid-viewport', 'invalid viewport', { + field: 'viewport', + index: 4, + parentIndex: 2, + frame: { x: 0, y: 0, width: 10, height: 10 }, + clip: { x: 1, y: 2, width: 3, height: 4 }, + projection: 'regular', + }); + + assert.deepEqual(toIosSnapshotEngineErrorDetails(error), { + reason: 'invalid-viewport', + field: 'viewport', + index: 4, + parentIndex: 2, + frame: { x: 0, y: 0, width: 10, height: 10 }, + clip: { x: 1, y: 2, width: 3, height: 4 }, + projection: 'regular', + }); +}); + +test('viewport evidence prefers reported application and window roots', () => { + assert.deepEqual( + resolveIosViewportEvidenceFromRoots([ + { type: 'XCUIElementTypeApplication', rectStatus: 'invalid' }, + { + type: 'XCUIElementTypeWindow', + rect: { x: 0, y: 0, width: 390, height: 844 }, + rectStatus: 'reported', + }, + ]), + { kind: 'reported', rect: { x: 0, y: 0, width: 390, height: 844 } }, + ); +}); + +test('viewport evidence can fall back to the largest top-level root', () => { + assert.deepEqual( + resolveIosViewportEvidenceFromRoots( + [ + { type: 'Other', rect: { x: 0, y: 0, width: 100, height: 100 } }, + { type: 'Other', rect: { x: 0, y: 0, width: 200, height: 300 } }, + ], + { fallbackToLargestRoot: true }, + ), + { kind: 'reported', rect: { x: 0, y: 0, width: 200, height: 300 } }, + ); +}); + +test('viewport evidence preserves explicit missing geometry reasons', () => { + assert.deepEqual( + resolveIosViewportEvidenceFromRoots([{ type: 'Application', rectStatus: 'invalid' }]), + { kind: 'missing', reason: 'invalid' }, + ); + assert.deepEqual( + resolveIosViewportEvidenceFromRoots([{ type: 'Application', rectStatus: 'not-provided' }]), + { kind: 'missing', reason: 'not-provided' }, + ); +}); + test('presented runner payloads and optional quality payloads cross the host invariant', () => { const request = createIosSnapshotRequest(); const presentedCapture = publishIosSnapshot(acquiredInput(request, nestedNodes()), request); diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.ts index 877c6942bd..cda241415f 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.ts @@ -11,8 +11,11 @@ import type { IosSnapshotInput, IosSnapshotPublication, IosSnapshotRequest, + IosViewportEvidence, } from '@agent-device/contracts/ios-snapshot'; -import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { normalizeType } from '@agent-device/contracts/snapshot'; +import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import { attachRefs, type RawSnapshotNode, type Rect } from '@agent-device/kernel/snapshot'; import { buildIosInteractiveSnapshotPresentation } from './semantic-index.ts'; import { validateIosSnapshotGraph } from './graph.ts'; import { foldIosSnapshot } from './geometry.ts'; @@ -28,6 +31,52 @@ import { IosSnapshotEngineError } from './types.ts'; const DEFAULT_FOLD_POLICY: IosSnapshotFoldPolicy = 'cursor-projected'; +export type IosSnapshotViewportRoot = Readonly<{ + type?: string; + rect?: Rect; + rectStatus?: 'reported' | 'invalid' | 'not-provided'; +}>; + +export function resolveIosViewportEvidenceFromRoots( + roots: readonly IosSnapshotViewportRoot[], + options: Readonly<{ fallbackToLargestRoot?: boolean }> = {}, +): IosViewportEvidence | undefined { + const viewportRoots = roots.filter(isViewportRoot); + const candidates = + viewportRoots.length > 0 || options.fallbackToLargestRoot !== true ? viewportRoots : roots; + const root = [...candidates].sort(compareViewportRoots)[0]; + if (!root) return undefined; + if (isPositiveFiniteRect(root.rect)) return { kind: 'reported', rect: root.rect }; + return { + kind: 'missing', + reason: + root.rectStatus === 'invalid' || (root.rectStatus === undefined && root.rect !== undefined) + ? 'invalid' + : 'not-provided', + }; +} + +function isViewportRoot(root: IosSnapshotViewportRoot): boolean { + const type = normalizeType(root.type ?? ''); + return type === 'application' || type === 'window'; +} + +function compareViewportRoots( + left: IosSnapshotViewportRoot, + right: IosSnapshotViewportRoot, +): number { + const status = rootGeometryRank(right.rectStatus) - rootGeometryRank(left.rectStatus); + return status || rectArea(right.rect) - rectArea(left.rect); +} + +function rootGeometryRank(status: IosSnapshotViewportRoot['rectStatus']): number { + return status === 'reported' ? 2 : status === 'invalid' ? 1 : 0; +} + +function rectArea(rect: Rect | undefined): number { + return rect && isPositiveFiniteRect(rect) ? rect.width * rect.height : 0; +} + export function createIosSnapshotEngine(options: IosSnapshotEngineOptions = {}): IosSnapshotEngine { const foldPolicy = options.foldPolicy ?? DEFAULT_FOLD_POLICY; return Object.freeze({ diff --git a/packages/capture-kit/src/ios-snapshot-engine/errors.test.ts b/packages/capture-kit/src/ios-snapshot-engine/errors.test.ts deleted file mode 100644 index 74d9133a58..0000000000 --- a/packages/capture-kit/src/ios-snapshot-engine/errors.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { toIosSnapshotEngineErrorDetails } from './errors.ts'; -import { IosSnapshotEngineError } from './types.ts'; - -test('engine error details preserve typed public context', () => { - const error = new IosSnapshotEngineError('invalid-viewport', 'invalid viewport', { - field: 'viewport', - index: 4, - parentIndex: 2, - frame: { x: 0, y: 0, width: 10, height: 10 }, - clip: { x: 1, y: 2, width: 3, height: 4 }, - projection: 'regular', - }); - - assert.deepEqual(toIosSnapshotEngineErrorDetails(error), { - reason: 'invalid-viewport', - field: 'viewport', - index: 4, - parentIndex: 2, - frame: { x: 0, y: 0, width: 10, height: 10 }, - clip: { x: 1, y: 2, width: 3, height: 4 }, - projection: 'regular', - }); -}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/errors.ts b/packages/capture-kit/src/ios-snapshot-engine/errors.ts deleted file mode 100644 index f034097399..0000000000 --- a/packages/capture-kit/src/ios-snapshot-engine/errors.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { - IosSnapshotEngineError, - IosSnapshotEngineFailureDetails, - IosSnapshotEngineFailureReason, -} from './types.ts'; - -export type IosSnapshotEnginePublicErrorDetails = Readonly< - IosSnapshotEngineFailureDetails & { reason: IosSnapshotEngineFailureReason } ->; - -export function toIosSnapshotEngineErrorDetails( - error: IosSnapshotEngineError, -): IosSnapshotEnginePublicErrorDetails { - return { - reason: error.reason, - ...(error.details.index !== undefined ? { index: error.details.index } : {}), - ...(error.details.parentIndex !== undefined ? { parentIndex: error.details.parentIndex } : {}), - ...(error.details.frame !== undefined ? { frame: error.details.frame } : {}), - ...(error.details.clip !== undefined ? { clip: error.details.clip } : {}), - ...(error.details.projection !== undefined ? { projection: error.details.projection } : {}), - ...(error.details.field !== undefined ? { field: error.details.field } : {}), - }; -} diff --git a/packages/capture-kit/src/ios-snapshot-engine/index.ts b/packages/capture-kit/src/ios-snapshot-engine/index.ts index 11f9de9734..bf1ccb93c7 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/index.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/index.ts @@ -11,7 +11,6 @@ export { } from './semantic-index.ts'; export { collectIosStructuralIdentifierSuppression } from './noise-structural.ts'; export { findNearestScrollableContainer, mergeReplacement, updateReplacement } from './tree.ts'; -export { IosSnapshotEngineError } from './types.ts'; -export { toIosSnapshotEngineErrorDetails } from './errors.ts'; -export { resolveIosViewportEvidenceFromRoots } from './viewport.ts'; +export { IosSnapshotEngineError, toIosSnapshotEngineErrorDetails } from './types.ts'; +export { resolveIosViewportEvidenceFromRoots } from './engine.ts'; export type { SnapshotTreeRuleContext } from './tree.ts'; diff --git a/packages/capture-kit/src/ios-snapshot-engine/types.ts b/packages/capture-kit/src/ios-snapshot-engine/types.ts index 7468d0b9f4..29e4fb5bdc 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/types.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/types.ts @@ -64,3 +64,13 @@ export class IosSnapshotEngineError extends Error { this.details = details; } } + +export type IosSnapshotEnginePublicErrorDetails = Readonly< + IosSnapshotEngineFailureDetails & { reason: IosSnapshotEngineFailureReason } +>; + +export function toIosSnapshotEngineErrorDetails( + error: IosSnapshotEngineError, +): IosSnapshotEnginePublicErrorDetails { + return { reason: error.reason, ...error.details }; +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/viewport.test.ts b/packages/capture-kit/src/ios-snapshot-engine/viewport.test.ts deleted file mode 100644 index 009cda3bb6..0000000000 --- a/packages/capture-kit/src/ios-snapshot-engine/viewport.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { resolveIosViewportEvidenceFromRoots } from './viewport.ts'; - -test('viewport evidence prefers reported application and window roots', () => { - assert.deepEqual( - resolveIosViewportEvidenceFromRoots([ - { type: 'XCUIElementTypeApplication', rectStatus: 'invalid' }, - { - type: 'XCUIElementTypeWindow', - rect: { x: 0, y: 0, width: 390, height: 844 }, - rectStatus: 'reported', - }, - ]), - { kind: 'reported', rect: { x: 0, y: 0, width: 390, height: 844 } }, - ); -}); - -test('viewport evidence can fall back to the largest top-level root', () => { - assert.deepEqual( - resolveIosViewportEvidenceFromRoots( - [ - { type: 'Other', rect: { x: 0, y: 0, width: 100, height: 100 } }, - { type: 'Other', rect: { x: 0, y: 0, width: 200, height: 300 } }, - ], - { fallbackToLargestRoot: true }, - ), - { kind: 'reported', rect: { x: 0, y: 0, width: 200, height: 300 } }, - ); -}); - -test('viewport evidence preserves explicit missing geometry reasons', () => { - assert.deepEqual( - resolveIosViewportEvidenceFromRoots([{ type: 'Application', rectStatus: 'invalid' }]), - { kind: 'missing', reason: 'invalid' }, - ); - assert.deepEqual( - resolveIosViewportEvidenceFromRoots([{ type: 'Application', rectStatus: 'not-provided' }]), - { kind: 'missing', reason: 'not-provided' }, - ); -}); diff --git a/packages/capture-kit/src/ios-snapshot-engine/viewport.ts b/packages/capture-kit/src/ios-snapshot-engine/viewport.ts deleted file mode 100644 index ef0f8db9c8..0000000000 --- a/packages/capture-kit/src/ios-snapshot-engine/viewport.ts +++ /dev/null @@ -1,50 +0,0 @@ -import type { IosViewportEvidence } from '@agent-device/contracts/ios-snapshot'; -import { normalizeType } from '@agent-device/contracts/snapshot'; -import type { Rect } from '@agent-device/kernel/snapshot'; -import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; - -export type IosSnapshotViewportRoot = Readonly<{ - type?: string; - rect?: Rect; - rectStatus?: 'reported' | 'invalid' | 'not-provided'; -}>; - -export function resolveIosViewportEvidenceFromRoots( - roots: readonly IosSnapshotViewportRoot[], - options: Readonly<{ fallbackToLargestRoot?: boolean }> = {}, -): IosViewportEvidence | undefined { - const viewportRoots = roots.filter(isViewportRoot); - const candidates = - viewportRoots.length > 0 || options.fallbackToLargestRoot !== true ? viewportRoots : roots; - const root = [...candidates].sort(compareViewportRoots)[0]; - if (!root) return undefined; - if (isPositiveFiniteRect(root.rect)) return { kind: 'reported', rect: root.rect }; - return { - kind: 'missing', - reason: - root.rectStatus === 'invalid' || (root.rectStatus === undefined && root.rect !== undefined) - ? 'invalid' - : 'not-provided', - }; -} - -function isViewportRoot(root: IosSnapshotViewportRoot): boolean { - const type = normalizeType(root.type ?? ''); - return type === 'application' || type === 'window'; -} - -function compareViewportRoots( - left: IosSnapshotViewportRoot, - right: IosSnapshotViewportRoot, -): number { - const status = rootGeometryRank(right.rectStatus) - rootGeometryRank(left.rectStatus); - return status || rectArea(right.rect) - rectArea(left.rect); -} - -function rootGeometryRank(status: IosSnapshotViewportRoot['rectStatus']): number { - return status === 'reported' ? 2 : status === 'invalid' ? 1 : 0; -} - -function rectArea(rect: Rect | undefined): number { - return rect && isPositiveFiniteRect(rect) ? rect.width * rect.height : 0; -} diff --git a/packages/capture-kit/src/ios-snapshot-planning.test.ts b/packages/capture-kit/src/ios-snapshot-planning.test.ts index 025ae2f245..99bb499ee5 100644 --- a/packages/capture-kit/src/ios-snapshot-planning.test.ts +++ b/packages/capture-kit/src/ios-snapshot-planning.test.ts @@ -231,6 +231,7 @@ function acquiredProducer( interactiveQueryCompleteness: 'incomplete', viewportEvidence: 'available', hittabilityEvidence: 'available', + presentationOwner: 'snapshot-state', ...overrides, }; } diff --git a/packages/capture-kit/src/ios-snapshot-planning.ts b/packages/capture-kit/src/ios-snapshot-planning.ts index 575335f8a3..6605666f5e 100644 --- a/packages/capture-kit/src/ios-snapshot-planning.ts +++ b/packages/capture-kit/src/ios-snapshot-planning.ts @@ -24,6 +24,7 @@ const IOS_SNAPSHOT_PRODUCER_CAPABILITY_VALUES = { interactiveQueryCompleteness: 'complete', viewportEvidence: 'available', hittabilityEvidence: 'available', + presentationOwner: 'ios-snapshot-engine', }, 'simulator-ax-bridge': { producer: 'simulator-ax-bridge', @@ -36,6 +37,7 @@ const IOS_SNAPSHOT_PRODUCER_CAPABILITY_VALUES = { interactiveQueryCompleteness: 'incomplete', viewportEvidence: 'available', hittabilityEvidence: 'available', + presentationOwner: 'snapshot-state', }, 'appium-source': { producer: 'appium-source', @@ -48,6 +50,7 @@ const IOS_SNAPSHOT_PRODUCER_CAPABILITY_VALUES = { interactiveQueryCompleteness: 'incomplete', viewportEvidence: 'available', hittabilityEvidence: 'unavailable', + presentationOwner: 'ios-snapshot-engine', }, 'limrun-ios-tree': { producer: 'limrun-ios-tree', @@ -60,6 +63,7 @@ const IOS_SNAPSHOT_PRODUCER_CAPABILITY_VALUES = { interactiveQueryCompleteness: 'incomplete', viewportEvidence: 'available', hittabilityEvidence: 'unavailable', + presentationOwner: 'snapshot-state', }, } as const satisfies Record; diff --git a/packages/contracts/src/facades/capture.ts b/packages/contracts/src/facades/capture.ts index 4883e2b0ac..d11cab7c83 100644 --- a/packages/contracts/src/facades/capture.ts +++ b/packages/contracts/src/facades/capture.ts @@ -53,17 +53,14 @@ export type { ScreenshotResultData, } from '../snapshot-types.ts'; export { - attachSnapshotPresentationEvidence, attachSnapshotClickabilityEvidence, attachSnapshotOcclusionContextEvidence, copySnapshotClickabilityEvidence, readSnapshotClickabilityEvidence, readSnapshotOcclusionContextEvidence, - readSnapshotPresentationEvidence, } from '../snapshot-private-evidence.ts'; export type { AndroidSiblingOrderEvidence, - SnapshotPresentationEvidence, SnapshotClickabilityEvidence, SnapshotOcclusionContextEvidence, } from '../snapshot-private-evidence.ts'; diff --git a/packages/contracts/src/ios-snapshot.ts b/packages/contracts/src/ios-snapshot.ts index 227300931c..cbc6d9d08d 100644 --- a/packages/contracts/src/ios-snapshot.ts +++ b/packages/contracts/src/ios-snapshot.ts @@ -11,6 +11,7 @@ export type IosAcquisitionIntent = 'full' | 'surface-observation'; export type IosSnapshotProjection = 'regular' | 'raw'; export type IosSnapshotCompleteness = 'complete' | 'incomplete'; export type IosSnapshotEvidenceAvailability = 'available' | 'unavailable'; +export type IosSnapshotPresentationOwner = 'ios-snapshot-engine' | 'snapshot-state'; export type IosSnapshotGeneration = string; @@ -79,6 +80,7 @@ type IosSnapshotProducerCapabilityFacts = Readonly<{ interactiveQueryCompleteness: IosSnapshotCompleteness; viewportEvidence: IosSnapshotEvidenceAvailability; hittabilityEvidence: IosSnapshotEvidenceAvailability; + presentationOwner: IosSnapshotPresentationOwner; }>; export type IosSnapshotAcquisitionProducerCapabilities = IosSnapshotProducerCapabilityFacts & diff --git a/packages/contracts/src/snapshot-private-evidence.ts b/packages/contracts/src/snapshot-private-evidence.ts index 43e524f022..530f0c13ed 100644 --- a/packages/contracts/src/snapshot-private-evidence.ts +++ b/packages/contracts/src/snapshot-private-evidence.ts @@ -28,15 +28,9 @@ export type SnapshotOcclusionContextEvidence = { androidSiblingOrderByNodeIndex?: ReadonlyMap; }; -/** Identifies the engine that owns interactive iOS snapshot presentation. */ -export type SnapshotPresentationEvidence = { - owner: 'ios-snapshot-engine'; -}; - type SnapshotPrivateEvidence = { clickability?: SnapshotClickabilityEvidence; occlusionContext?: SnapshotOcclusionContextEvidence; - presentation?: SnapshotPresentationEvidence; }; const privateEvidenceBySnapshotObject = new WeakMap(); @@ -65,19 +59,6 @@ export function readSnapshotClickabilityEvidence( return owner ? privateEvidenceBySnapshotObject.get(owner)?.clickability : undefined; } -export function attachSnapshotPresentationEvidence( - owner: T, - evidence: SnapshotPresentationEvidence, -): T { - return attachSnapshotPrivateEvidence(owner, { presentation: evidence }); -} - -export function readSnapshotPresentationEvidence( - owner: object | null | undefined, -): SnapshotPresentationEvidence | undefined { - return owner ? privateEvidenceBySnapshotObject.get(owner)?.presentation : undefined; -} - export function copySnapshotClickabilityEvidence( source: object | null | undefined, target: T, diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index cc5f31ca7e..44177fe00d 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -1,6 +1,5 @@ import assert from 'node:assert/strict'; import { test, vi } from 'vitest'; -import { readSnapshotPresentationEvidence } from '@agent-device/contracts/capture'; import { AppError } from '@agent-device/kernel/errors'; import { acquireWebDriverIosSnapshot, @@ -45,7 +44,6 @@ test('Appium iOS snapshots acquire facts and publish regular output through the result.nodes?.every((node) => !('ref' in node)), true, ); - assert.deepEqual(readSnapshotPresentationEvidence(result), { owner: 'ios-snapshot-engine' }); }); test('Appium iOS options become an engine plan and engine-owned projection', () => { @@ -140,12 +138,32 @@ test('Appium iOS regular presentation fails typed when page source has no viewpo () => publishWebDriverIosSnapshot(acquired), (error: unknown) => { assert.ok(error instanceof AppError); - assert.deepEqual(error.details, { reason: 'missing-viewport', field: 'viewport' }); + assert.deepEqual(error.details, { + reason: 'missing-viewport', + field: 'viewport', + hint: 'Use snapshot --raw to inspect the acquired Appium tree; regular presentation requires valid viewport evidence.', + }); return true; }, ); }); +test('Appium iOS raw presentation discloses missing viewport without failing', async () => { + const result = await captureWebDriverIosSnapshot( + { + source: async () => + '', + }, + { raw: true }, + ); + + assert.equal( + result.nodes?.some((node) => node.label === 'Continue'), + true, + ); + assert.ok(result.warnings?.some((warning) => warning.includes('snapshot --raw to inspect'))); +}); + test('Appium iOS does not promote a non-viewport root rectangle to viewport evidence', () => { const acquired = acquireWebDriverIosSnapshot( '', diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index 2d1eee9c79..7916bbc5f4 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -4,7 +4,6 @@ import { resolveIosViewportEvidenceFromRoots, toIosSnapshotEngineErrorDetails, } from '@agent-device/capture-kit/ios-snapshot-engine'; -import { attachSnapshotPresentationEvidence } from '@agent-device/contracts/capture'; import { createIosSnapshotRequest, IOS_SNAPSHOT_PRODUCER_CAPABILITIES, @@ -29,7 +28,7 @@ const APPIUM_PRODUCER = IOS_SNAPSHOT_PRODUCER_CAPABILITIES['appium-source']; const iosSnapshotEngine = createIosSnapshotEngine(); const RESIDUE_WARNINGS = { 'missing-viewport': - 'Appium page source does not provide a valid viewport; regular snapshot presentation is unavailable.', + 'Appium page source does not provide valid viewport evidence; regular snapshots fail closed. Use snapshot --raw to inspect the acquired tree.', } satisfies Pick, 'missing-viewport'>; const UNAVAILABLE_FACT_WARNINGS: Partial> = { 'acquisition-depth': @@ -103,16 +102,13 @@ export function publishWebDriverIosSnapshot( } catch (error) { throwWebDriverIosSnapshotError(error); } - const result = attachSnapshotPresentationEvidence( - { - backend: 'xctest', - producer: 'appium-source', - nodes: stripRefs(publication.payload.nodes), - truncated: publication.payload.truncated, - ...warningsForResidue(publication.residue), - } satisfies SnapshotResult, - { owner: 'ios-snapshot-engine' }, - ); + const result = { + backend: 'xctest', + producer: 'appium-source', + nodes: stripRefs(publication.payload.nodes), + truncated: publication.payload.truncated, + ...warningsForResidue(publication.residue), + } satisfies SnapshotResult; return { acquisition, publication, result }; } @@ -152,10 +148,18 @@ function stripRefs(nodes: readonly SnapshotNode[]): RawSnapshotNode[] { function throwWebDriverIosSnapshotError(error: unknown): never { if (!(error instanceof IosSnapshotEngineError)) throw error; + const details = toIosSnapshotEngineErrorDetails(error); throw new AppError( 'COMMAND_FAILED', error.message, - toIosSnapshotEngineErrorDetails(error), + { + ...details, + ...(error.reason === 'missing-viewport' || error.reason === 'invalid-viewport' + ? { + hint: 'Use snapshot --raw to inspect the acquired Appium tree; regular presentation requires valid viewport evidence.', + } + : {}), + }, error, ); } diff --git a/packages/provider-webdriver/src/webdriver-source.test.ts b/packages/provider-webdriver/src/webdriver-source.test.ts index 1c628b6d5c..caec811d7e 100644 --- a/packages/provider-webdriver/src/webdriver-source.test.ts +++ b/packages/provider-webdriver/src/webdriver-source.test.ts @@ -6,6 +6,7 @@ import { parseWebDriverSource, parseWebDriverSourceFacts } from './webdriver-sou test('WebDriver source parsing preserves hardened attributes and geometry', () => { const nodes = parseWebDriverSource( '', + { mode: 'facts' }, ); assert.equal(nodes[0]?.label, 'A > B'); @@ -15,7 +16,7 @@ test('WebDriver source parsing preserves hardened attributes and geometry', () = assert.equal(nodes[0]?.visibleToUser, true); assert.equal(nodes[0]?.hittable, undefined); assert.throws( - () => parseWebDriverSource(''), + () => parseWebDriverSource('', { mode: 'facts' }), /Unsupported XML attribute name "__proto__"/, ); }); @@ -23,6 +24,7 @@ test('WebDriver source parsing preserves hardened attributes and geometry', () = test('WebDriver source facts do not fill absent provider attributes', () => { const node = parseWebDriverSource( '', + { mode: 'facts' }, )[0]; assert.equal(node?.label, 'Continue'); @@ -37,6 +39,7 @@ test('WebDriver source facts do not fill absent provider attributes', () => { test('WebDriver source facts preserve explicitly reported hittability', () => { const node = parseWebDriverSource( '', + { mode: 'facts' }, )[0]; assert.equal(node?.hittable, true); @@ -69,6 +72,7 @@ test('legacy WebDriver parsing keeps Android source booleans and ignores iOS hin test('WebDriver source facts preserve roots without claiming hierarchy completeness', () => { const facts = parseWebDriverSourceFacts( '', + { mode: 'facts' }, ); assert.equal('truncated' in facts, false); @@ -88,6 +92,7 @@ test('WebDriver source facts preserve roots without claiming hierarchy completen test('WebDriver source facts classify invalid root geometry', () => { const facts = parseWebDriverSourceFacts( '', + { mode: 'facts' }, ); assert.deepEqual(facts.roots, [ diff --git a/packages/provider-webdriver/src/webdriver-source.ts b/packages/provider-webdriver/src/webdriver-source.ts index 3d14611467..207115cfa0 100644 --- a/packages/provider-webdriver/src/webdriver-source.ts +++ b/packages/provider-webdriver/src/webdriver-source.ts @@ -18,19 +18,19 @@ export type WebDriverSourceRootFact = Readonly<{ export function parseWebDriverSource( source: string, - options: Readonly<{ mode?: WebDriverSourceParseMode }> = {}, + options: Readonly<{ mode: WebDriverSourceParseMode }>, ): RawSnapshotNode[] { return parseWebDriverSourceFacts(source, options).nodes; } export function parseWebDriverSourceFacts( source: string, - options: Readonly<{ mode?: WebDriverSourceParseMode }> = {}, + options: Readonly<{ mode: WebDriverSourceParseMode }>, ): WebDriverSourceFacts { const roots = parseSourceRoots(source); const nodes: RawSnapshotNode[] = []; const sourceRoots: WebDriverSourceRootFact[] = []; - const mode = options.mode ?? 'facts'; + const mode = options.mode; for (const root of roots) { appendSourceNodes(nodes, root, undefined, 0, mode, sourceRoots); } @@ -40,10 +40,10 @@ export function parseWebDriverSourceFacts( function appendSourceNodes( nodes: RawSnapshotNode[], xmlNode: XmlNode, - parentIndex?: number, - depth = 0, - mode: WebDriverSourceParseMode = 'facts', - sourceRoots: WebDriverSourceRootFact[] = [], + parentIndex: number | undefined, + depth: number, + mode: WebDriverSourceParseMode, + sourceRoots: WebDriverSourceRootFact[], ): void { const currentIndex = isSourceContainer(xmlNode, mode) ? parentIndex diff --git a/src/core/__tests__/snapshot-state.test.ts b/src/core/__tests__/snapshot-state.test.ts index cd572928da..ed5c3095bd 100644 --- a/src/core/__tests__/snapshot-state.test.ts +++ b/src/core/__tests__/snapshot-state.test.ts @@ -1,10 +1,8 @@ import { expect, test } from 'vitest'; import { buildSnapshotState } from '../snapshot-state.ts'; +import { resolveActionableTouchResolution } from '../interaction-targeting.ts'; import { createSnapshotVisibility } from '@agent-device/contracts/snapshot'; -import { - attachSnapshotOcclusionContextEvidence, - attachSnapshotPresentationEvidence, -} from '@agent-device/contracts/capture'; +import { attachSnapshotOcclusionContextEvidence } from '@agent-device/contracts/capture'; import { buildUiHierarchySnapshot, parseUiHierarchyTree, @@ -156,21 +154,18 @@ test('buildSnapshotState preserves the legacy acquired iOS presentation path', ( ]); }); -test('buildSnapshotState skips the legacy path for engine-presented iOS results', () => { +test('buildSnapshotState uses the registered presentation owner for Appium results', () => { const rowRect = { x: 16, y: 293, width: 370, height: 52 }; - const data = attachSnapshotPresentationEvidence( - { - nodes: [ - { index: 0, depth: 0, type: 'Application', label: 'Settings' }, - { index: 1, depth: 1, parentIndex: 0, type: 'CollectionView' }, - { index: 2, depth: 2, parentIndex: 1, type: 'Cell', label: 'General', rect: rowRect }, - { index: 3, depth: 3, parentIndex: 2, type: 'Button', label: 'General', rect: rowRect }, - ], - backend: 'xctest' as const, - producer: 'appium-source' as const, - }, - { owner: 'ios-snapshot-engine' }, - ); + const data = { + nodes: [ + { index: 0, depth: 0, type: 'Application', label: 'Settings' }, + { index: 1, depth: 1, parentIndex: 0, type: 'CollectionView' }, + { index: 2, depth: 2, parentIndex: 1, type: 'Cell', label: 'General', rect: rowRect }, + { index: 3, depth: 3, parentIndex: 2, type: 'Button', label: 'General', rect: rowRect }, + ], + backend: 'xctest' as const, + producer: 'appium-source' as const, + }; const state = buildSnapshotState(data, { snapshotInteractiveOnly: true }); @@ -182,6 +177,49 @@ test('buildSnapshotState skips the legacy path for engine-presented iOS results' ]); }); +test('Appium presentation does not infer hittability from an enabled ancestor rectangle', () => { + const state = buildSnapshotState( + { + nodes: [ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Cell', + rect: { x: 16, y: 293, width: 370, height: 52 }, + enabled: true, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'StaticText', + label: 'General', + rect: { x: 24, y: 309, width: 100, height: 20 }, + enabled: true, + }, + ], + backend: 'xctest', + producer: 'appium-source', + }, + { snapshotInteractiveOnly: true }, + ); + const target = state.nodes.find((node) => node.label === 'General'); + + expect(target).toBeDefined(); + expect(target?.hittable).toBeUndefined(); + expect(resolveActionableTouchResolution(state.nodes, target!)).toMatchObject({ + node: target, + reason: 'original', + }); +}); + test('buildSnapshotState marks content covered by floating overlays as visible but blocked', () => { const state = buildSnapshotState( { diff --git a/src/core/snapshot-state.ts b/src/core/snapshot-state.ts index ad46d4407a..b6910f6d28 100644 --- a/src/core/snapshot-state.ts +++ b/src/core/snapshot-state.ts @@ -1,7 +1,6 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import { readSnapshotOcclusionContextEvidence, - readSnapshotPresentationEvidence, snapshotCaptureAnnotationsFrom, } from '@agent-device/contracts/capture'; import { isAndroidInputMethodNode } from '@agent-device/contracts/android-input-ownership'; @@ -128,11 +127,10 @@ function shouldPresentLegacyIosInteractiveSnapshot( Partial>) | undefined, ): boolean { - const presentationEvidence = readSnapshotPresentationEvidence(provenance); return ( provenance.backend === 'xctest' && iosSnapshotPresentationStage(provenance) === 'acquired' && - presentationEvidence?.owner !== 'ios-snapshot-engine' && + iosSnapshotPresentationOwner(provenance) !== 'ios-snapshot-engine' && flags?.snapshotInteractiveOnly === true && flags.snapshotRaw !== true ); @@ -148,6 +146,15 @@ function iosSnapshotPresentationStage( ].stage; } +function iosSnapshotPresentationOwner( + provenance: SnapshotStateProvenance, +): 'ios-snapshot-engine' | 'snapshot-state' | undefined { + if (provenance.backend !== 'xctest' || provenance.producer === undefined) return undefined; + return IOS_SNAPSHOT_PRODUCER_CAPABILITIES[ + provenance.producer as 'apple-runner' | 'appium-source' | 'limrun-ios-tree' + ].presentationOwner; +} + function isAndroidComparisonSafeSnapshot( backend: SnapshotBackend | undefined, flags: From 5b44c166c0306eed48887673e244efa7f77cc8b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 22:42:36 +0200 Subject: [PATCH 11/15] fix(ios): disclose Appium snapshot limits accurately --- .../src/ios-snapshot-engine/index.ts | 2 +- .../src/webdriver-ios-snapshot.test.ts | 41 +++++++++++++------ .../src/webdriver-ios-snapshot.ts | 18 +++++--- .../src/webdriver-scroll-frame.ts | 2 +- 4 files changed, 43 insertions(+), 20 deletions(-) diff --git a/packages/capture-kit/src/ios-snapshot-engine/index.ts b/packages/capture-kit/src/ios-snapshot-engine/index.ts index bf1ccb93c7..5e182c7225 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/index.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/index.ts @@ -3,6 +3,7 @@ export { createIosSnapshotEngine, presentIosSnapshot, publishIosSnapshot, + resolveIosViewportEvidenceFromRoots, } from './engine.ts'; export { presentIosRunnerSnapshot } from './runner-presentation.ts'; export { @@ -12,5 +13,4 @@ export { export { collectIosStructuralIdentifierSuppression } from './noise-structural.ts'; export { findNearestScrollableContainer, mergeReplacement, updateReplacement } from './tree.ts'; export { IosSnapshotEngineError, toIosSnapshotEngineErrorDetails } from './types.ts'; -export { resolveIosViewportEvidenceFromRoots } from './engine.ts'; export type { SnapshotTreeRuleContext } from './tree.ts'; diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index 44177fe00d..f4abf01242 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -21,7 +21,7 @@ test('Appium iOS snapshots acquire facts and publish regular output through the assert.equal(result.backend, 'xctest'); assert.equal(result.producer, 'appium-source'); - assert.equal(result.truncated, false); + assert.equal(result.truncated, undefined); assert.deepEqual( result.nodes?.map((node) => [node.type, node.label, node.parentIndex]), [ @@ -33,13 +33,7 @@ test('Appium iOS snapshots acquire facts and publish regular output through the assert.equal(result.nodes?.find((node) => node.label === 'Continue')?.hittable, undefined); assert.equal(source.mock.calls.length, 1); assert.ok(result.warnings?.some((warning) => warning.includes('hittability evidence'))); - assert.deepEqual(Object.keys(result).sort(), [ - 'backend', - 'nodes', - 'producer', - 'truncated', - 'warnings', - ]); + assert.deepEqual(Object.keys(result).sort(), ['backend', 'nodes', 'producer', 'warnings']); assert.equal( result.nodes?.every((node) => !('ref' in node)), true, @@ -101,7 +95,11 @@ test('Appium regular presentation omits unavailable hittability while raw preser const raw = await captureWebDriverIosSnapshot({ source: async () => source }, { raw: true }); assert.equal(raw.nodes?.find((node) => node.label === 'Continue')?.hittable, true); - assert.ok(raw.warnings?.some((warning) => warning.includes('does not guarantee'))); + assert.equal( + raw.warnings?.some((warning) => warning.includes('hittability')), + false, + ); + assert.ok(raw.warnings?.some((warning) => warning.includes('provider-side depth'))); }); test('Appium iOS interactive requests stay provider-unpruned and use engine presentation', () => { @@ -190,12 +188,31 @@ test('Appium iOS regular presentation fails typed when root geometry is invalid' ); }); +test('Appium iOS regular presentation fails typed for a zero-size viewport', () => { + const acquired = acquireWebDriverIosSnapshot( + '', + ); + + assert.deepEqual(acquired.input.acquisition.viewport, { + kind: 'missing', + reason: 'invalid', + }); + assert.throws( + () => publishWebDriverIosSnapshot(acquired), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.details?.reason, 'invalid-viewport'); + assert.match(String(error.details?.hint), /snapshot --raw/); + return true; + }, + ); +}); + test('Appium iOS hierarchy limits are typed and disclosed at response level', async () => { const result = await captureWebDriverIosSnapshot({ source: async () => SOURCE }, { raw: true }); - assert.equal(result.truncated, false); + assert.equal(result.truncated, undefined); assert.deepEqual(result.warnings, [ - 'Appium page source does not guarantee hittability evidence; regular presentation treats it as unavailable.', - 'Appium page source does not report hierarchy completeness; depth- or child-limited nodes may be absent.', + 'Appium page source does not report hierarchy completeness; provider-side depth or child limits may omit nodes.', ]); }); diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index 7916bbc5f4..dca7924c11 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -32,7 +32,7 @@ const RESIDUE_WARNINGS = { } satisfies Pick, 'missing-viewport'>; const UNAVAILABLE_FACT_WARNINGS: Partial> = { 'acquisition-depth': - 'Appium page source does not report hierarchy completeness; depth- or child-limited nodes may be absent.', + 'Appium page source does not report hierarchy completeness; provider-side depth or child limits may omit nodes.', hittability: 'Appium page source does not guarantee hittability evidence; regular presentation treats it as unavailable.', }; @@ -106,8 +106,7 @@ export function publishWebDriverIosSnapshot( backend: 'xctest', producer: 'appium-source', nodes: stripRefs(publication.payload.nodes), - truncated: publication.payload.truncated, - ...warningsForResidue(publication.residue), + ...warningsForResidue(publication.residue, acquisition.request), } satisfies SnapshotResult; return { acquisition, publication, result }; } @@ -123,17 +122,24 @@ function residueForSource(viewport: IosViewportEvidence): readonly IosAcquisitio return residue; } -function warningsForResidue(residue: readonly IosAcquisitionResidue[]): { warnings?: string[] } { +function warningsForResidue( + residue: readonly IosAcquisitionResidue[], + request: IosSnapshotRequest, +): { warnings?: string[] } { const warnings = new Set(); for (const entry of residue) { - const warning = warningForResidue(entry); + const warning = warningForResidue(entry, request); if (warning) warnings.add(warning); } return warnings.size > 0 ? { warnings: [...warnings] } : {}; } -function warningForResidue(entry: IosAcquisitionResidue): string | undefined { +function warningForResidue( + entry: IosAcquisitionResidue, + request: IosSnapshotRequest, +): string | undefined { if (entry.kind === 'unavailable-fact') { + if (entry.fact === 'hittability' && request.projection === 'raw') return undefined; return UNAVAILABLE_FACT_WARNINGS[entry.fact]; } if (entry.kind === 'missing-viewport') { diff --git a/packages/provider-webdriver/src/webdriver-scroll-frame.ts b/packages/provider-webdriver/src/webdriver-scroll-frame.ts index 5b3a265c20..c98f4e8362 100644 --- a/packages/provider-webdriver/src/webdriver-scroll-frame.ts +++ b/packages/provider-webdriver/src/webdriver-scroll-frame.ts @@ -3,7 +3,7 @@ import type { WebDriverWindowRect } from './webdriver-client.ts'; import { parseWebDriverSource } from './webdriver-source.ts'; export function scrollFrameFromWebDriverSource(source: string): WebDriverWindowRect | undefined { - const rect = parseWebDriverSource(source, { mode: 'facts' }) + const rect = parseWebDriverSource(source, { mode: 'legacy-derived' }) .flatMap((node) => isScrollableSourceNode(node) && isUsableScrollRect(node.rect) ? [node.rect] : [], ) From e81da98c1edf665a0b8f0a139679e494a420504e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 23:16:22 +0200 Subject: [PATCH 12/15] fix(ios): disclose Appium snapshot evidence limits --- .../ios-snapshot-engine/conformance.test.ts | 2 +- .../src/ios-snapshot-engine/engine.ts | 7 ++-- .../src/ios-snapshot-planning.test.ts | 11 ++++++ .../capture-kit/src/ios-snapshot-planning.ts | 17 +++++++++ packages/contracts/src/client-capture.ts | 2 +- packages/contracts/src/ios-snapshot.ts | 4 +-- .../src/webdriver-interactor.ts | 3 +- .../src/webdriver-ios-snapshot.test.ts | 5 +-- .../src/webdriver-ios-snapshot.ts | 35 ++++++++----------- .../src/webdriver-scroll-frame.ts | 9 +++-- .../src/webdriver-source.test.ts | 1 + src/agent-device-client.ts | 2 +- src/commands/capture/runtime/snapshot.ts | 10 ++++-- src/daemon/result-serialization.ts | 2 +- .../cloud-webdriver-ios-text-entry.test.ts | 16 ++++++++- 15 files changed, 86 insertions(+), 40 deletions(-) diff --git a/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts b/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts index 8afcd9fee4..d1128fdbaa 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts @@ -103,7 +103,7 @@ test('the independent iOS snapshot goldens match the TypeScript engine', () => { actual = { outcome: 'success', nodes: normalizeGoldenNodes(result.nodes), - truncated: acquisition.truncated, + truncated: acquisition.truncated ?? false, residue: acquisition.residue, ...(testCase.qualityLabels ? { qualityLabels: result.qualityNodes?.map((node) => node.label ?? null) } diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.ts index cda241415f..c4a44d4ce5 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.ts @@ -95,13 +95,12 @@ export function publishIosSnapshot( input.stage === 'presented' ? input.validation.presentationKey : buildIosSnapshotPresentationKey(request); + const truncated = + input.stage === 'acquired' ? input.acquisition.truncated : input.presentation.payload.truncated; return { payload: { nodes: attachRefs(presentation.nodes), - truncated: - input.stage === 'acquired' - ? input.acquisition.truncated - : input.presentation.payload.truncated, + ...(truncated === undefined ? {} : { truncated }), }, presentationKey, comparisonIdentity: buildIosSnapshotComparisonIdentity(input, request), diff --git a/packages/capture-kit/src/ios-snapshot-planning.test.ts b/packages/capture-kit/src/ios-snapshot-planning.test.ts index 99bb499ee5..8711e60a42 100644 --- a/packages/capture-kit/src/ios-snapshot-planning.test.ts +++ b/packages/capture-kit/src/ios-snapshot-planning.test.ts @@ -16,6 +16,7 @@ import { buildIosSnapshotPresentationKey, createIosSnapshotRequest, deriveIosCaptureHint, + deriveIosSnapshotCapabilityResidue, planIosSnapshot, } from '@agent-device/capture-kit/ios-snapshot-planning'; @@ -155,6 +156,16 @@ test('Appium source plan carries its viewport evidence capability', () => { assert.equal(plan.evidence.viewport, 'available'); }); +test('capability residue derives unavailable Appium facts from the registry', () => { + assert.deepEqual( + deriveIosSnapshotCapabilityResidue(IOS_SNAPSHOT_PRODUCER_CAPABILITIES['appium-source']), + [ + { kind: 'unavailable-fact', fact: 'hittability' }, + { kind: 'unavailable-fact', fact: 'acquisition-depth' }, + ], + ); +}); + test('comparison identity rejects every identity axis and residue mismatch', () => { const base = comparisonIdentity(); const mismatches: IosSnapshotComparisonIdentity[] = [ diff --git a/packages/capture-kit/src/ios-snapshot-planning.ts b/packages/capture-kit/src/ios-snapshot-planning.ts index 6605666f5e..d8f8268983 100644 --- a/packages/capture-kit/src/ios-snapshot-planning.ts +++ b/packages/capture-kit/src/ios-snapshot-planning.ts @@ -71,6 +71,23 @@ export const IOS_SNAPSHOT_PRODUCER_CAPABILITIES: Readonly< Record > = Object.freeze(IOS_SNAPSHOT_PRODUCER_CAPABILITY_VALUES); +export function deriveIosSnapshotCapabilityResidue( + producer: IosSnapshotProducerCapabilities, +): readonly IosAcquisitionResidue[] { + const residue: IosAcquisitionResidue[] = []; + if (producer.hittabilityEvidence === 'unavailable') { + residue.push({ kind: 'unavailable-fact', fact: 'hittability' }); + } + if ( + producer.stage === 'acquired' && + (producer.acquisitionDepth.rawTraversal.kind === 'incomplete' || + producer.acquisitionDepth.regularPresented.kind === 'incomplete') + ) { + residue.push({ kind: 'unavailable-fact', fact: 'acquisition-depth' }); + } + return Object.freeze(residue); +} + export function createIosSnapshotRequest(input: IosSnapshotRequestInput = {}): IosSnapshotRequest { return Object.freeze({ projection: input.projection ?? (input.raw === true ? 'raw' : 'regular'), diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index 7d1ea71fdb..04c2df4913 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -37,7 +37,7 @@ export type CaptureSnapshotOptions = AgentDeviceRequestOverrides & export type CaptureSnapshotResult = { nodes: SnapshotNode[]; - truncated: boolean; + truncated?: boolean; appName?: string; appBundleId?: string; visibility?: SnapshotVisibility; diff --git a/packages/contracts/src/ios-snapshot.ts b/packages/contracts/src/ios-snapshot.ts index cbc6d9d08d..95b0fead3a 100644 --- a/packages/contracts/src/ios-snapshot.ts +++ b/packages/contracts/src/ios-snapshot.ts @@ -150,7 +150,7 @@ type IosSnapshotAcquisitionForIntent = Read intent: Intent; hint: CaptureHint & Readonly<{ acquisitionIntent: Intent }>; nodes: readonly RawSnapshotNode[]; - truncated: boolean; + truncated?: boolean; viewport: IosViewportEvidence; lineage: IosSnapshotLineage; residue: readonly IosAcquisitionResidue[]; @@ -218,7 +218,7 @@ export type IosSnapshotPlan = Readonly<{ export type IosSnapshotPublishedPayload = Readonly<{ nodes: readonly SnapshotNode[]; - truncated: boolean; + truncated?: boolean; }>; export type IosSnapshotComparisonIdentity = Readonly<{ diff --git a/packages/provider-webdriver/src/webdriver-interactor.ts b/packages/provider-webdriver/src/webdriver-interactor.ts index cff36669a0..e9f22ce909 100644 --- a/packages/provider-webdriver/src/webdriver-interactor.ts +++ b/packages/provider-webdriver/src/webdriver-interactor.ts @@ -507,9 +507,10 @@ class WebDriverInteractor implements Interactor { } private async scrollGestureFrame(): Promise { + const sourceMode = this.backend === 'xctest' ? 'facts' : 'legacy-derived'; const sourceFrame = await this.client .source() - .then((source) => scrollFrameFromWebDriverSource(source)) + .then((source) => scrollFrameFromWebDriverSource(source, { mode: sourceMode })) .catch(() => undefined); if (sourceFrame) return sourceFrame; return await this.client.windowRect(); diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index f4abf01242..a59f6178e4 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -96,8 +96,8 @@ test('Appium regular presentation omits unavailable hittability while raw preser const raw = await captureWebDriverIosSnapshot({ source: async () => source }, { raw: true }); assert.equal(raw.nodes?.find((node) => node.label === 'Continue')?.hittable, true); assert.equal( - raw.warnings?.some((warning) => warning.includes('hittability')), - false, + raw.warnings?.some((warning) => warning.includes('absent hittable means no evidence')), + true, ); assert.ok(raw.warnings?.some((warning) => warning.includes('provider-side depth'))); }); @@ -213,6 +213,7 @@ test('Appium iOS hierarchy limits are typed and disclosed at response level', as assert.equal(result.truncated, undefined); assert.deepEqual(result.warnings, [ + 'Appium page source does not guarantee hittability evidence; absent hittable means no evidence, not false. Raw output preserves any provider-reported value.', 'Appium page source does not report hierarchy completeness; provider-side depth or child limits may omit nodes.', ]); }); diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index dca7924c11..42aee93f7b 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -6,6 +6,7 @@ import { } from '@agent-device/capture-kit/ios-snapshot-engine'; import { createIosSnapshotRequest, + deriveIosSnapshotCapabilityResidue, IOS_SNAPSHOT_PRODUCER_CAPABILITIES, } from '@agent-device/capture-kit/ios-snapshot-planning'; import type { @@ -15,7 +16,6 @@ import type { IosSnapshotPlan, IosSnapshotPublication, IosSnapshotRequest, - IosSnapshotFact, IosViewportEvidence, } from '@agent-device/contracts/ios-snapshot'; import type { SnapshotOptions, SnapshotResult } from '@agent-device/contracts/interactor-types'; @@ -30,12 +30,14 @@ const RESIDUE_WARNINGS = { 'missing-viewport': 'Appium page source does not provide valid viewport evidence; regular snapshots fail closed. Use snapshot --raw to inspect the acquired tree.', } satisfies Pick, 'missing-viewport'>; -const UNAVAILABLE_FACT_WARNINGS: Partial> = { +type WebDriverUnavailableFact = 'acquisition-depth' | 'hittability'; + +const UNAVAILABLE_FACT_WARNINGS = { 'acquisition-depth': 'Appium page source does not report hierarchy completeness; provider-side depth or child limits may omit nodes.', hittability: - 'Appium page source does not guarantee hittability evidence; regular presentation treats it as unavailable.', -}; + 'Appium page source does not guarantee hittability evidence; absent hittable means no evidence, not false. Raw output preserves any provider-reported value.', +} satisfies Record; export type WebDriverIosSnapshotAcquisition = Readonly<{ request: IosSnapshotRequest; @@ -80,7 +82,6 @@ export function acquireWebDriverIosSnapshot( const common = { producer: 'appium-source' as const, nodes: sourceFacts.nodes, - truncated: false, viewport, lineage: targetId ? { targetId } : {}, residue, @@ -106,41 +107,33 @@ export function publishWebDriverIosSnapshot( backend: 'xctest', producer: 'appium-source', nodes: stripRefs(publication.payload.nodes), - ...warningsForResidue(publication.residue, acquisition.request), + ...warningsForResidue(publication.residue), } satisfies SnapshotResult; return { acquisition, publication, result }; } function residueForSource(viewport: IosViewportEvidence): readonly IosAcquisitionResidue[] { - const residue: IosAcquisitionResidue[] = [ - { kind: 'unavailable-fact', fact: 'hittability' }, - { kind: 'unavailable-fact', fact: 'acquisition-depth' }, - ]; + const residue = [...deriveIosSnapshotCapabilityResidue(APPIUM_PRODUCER)]; if (viewport.kind === 'missing') { residue.push({ kind: 'missing-viewport', reason: viewport.reason }); } return residue; } -function warningsForResidue( - residue: readonly IosAcquisitionResidue[], - request: IosSnapshotRequest, -): { warnings?: string[] } { +function warningsForResidue(residue: readonly IosAcquisitionResidue[]): { warnings?: string[] } { const warnings = new Set(); for (const entry of residue) { - const warning = warningForResidue(entry, request); + const warning = warningForResidue(entry); if (warning) warnings.add(warning); } return warnings.size > 0 ? { warnings: [...warnings] } : {}; } -function warningForResidue( - entry: IosAcquisitionResidue, - request: IosSnapshotRequest, -): string | undefined { +function warningForResidue(entry: IosAcquisitionResidue): string | undefined { if (entry.kind === 'unavailable-fact') { - if (entry.fact === 'hittability' && request.projection === 'raw') return undefined; - return UNAVAILABLE_FACT_WARNINGS[entry.fact]; + return entry.fact === 'acquisition-depth' || entry.fact === 'hittability' + ? UNAVAILABLE_FACT_WARNINGS[entry.fact] + : undefined; } if (entry.kind === 'missing-viewport') { return RESIDUE_WARNINGS[entry.kind]; diff --git a/packages/provider-webdriver/src/webdriver-scroll-frame.ts b/packages/provider-webdriver/src/webdriver-scroll-frame.ts index c98f4e8362..e7570d82b3 100644 --- a/packages/provider-webdriver/src/webdriver-scroll-frame.ts +++ b/packages/provider-webdriver/src/webdriver-scroll-frame.ts @@ -1,9 +1,12 @@ import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; import type { WebDriverWindowRect } from './webdriver-client.ts'; -import { parseWebDriverSource } from './webdriver-source.ts'; +import { parseWebDriverSource, type WebDriverSourceParseMode } from './webdriver-source.ts'; -export function scrollFrameFromWebDriverSource(source: string): WebDriverWindowRect | undefined { - const rect = parseWebDriverSource(source, { mode: 'legacy-derived' }) +export function scrollFrameFromWebDriverSource( + source: string, + options: Readonly<{ mode: WebDriverSourceParseMode }>, +): WebDriverWindowRect | undefined { + const rect = parseWebDriverSource(source, options) .flatMap((node) => isScrollableSourceNode(node) && isUsableScrollRect(node.rect) ? [node.rect] : [], ) diff --git a/packages/provider-webdriver/src/webdriver-source.test.ts b/packages/provider-webdriver/src/webdriver-source.test.ts index caec811d7e..5ec9df80ca 100644 --- a/packages/provider-webdriver/src/webdriver-source.test.ts +++ b/packages/provider-webdriver/src/webdriver-source.test.ts @@ -111,6 +111,7 @@ test('WebDriver scroll frame prefers visible scrollable containers', () => { '' + '' + '', + { mode: 'legacy-derived' }, ), { x: 0, y: 393, width: 1080, height: 1103 }, ); diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index ceb3626342..28b45cbce3 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -511,7 +511,7 @@ function normalizeSnapshotResult( const appBundleId = readOptionalString(data, 'appBundleId'); return { nodes: readSnapshotNodes(data.nodes), - truncated: data.truncated === true, + ...(typeof data.truncated === 'boolean' ? { truncated: data.truncated } : {}), appName: readOptionalString(data, 'appName'), appBundleId, ...optionalSnapshotResponseFields(data), diff --git a/src/commands/capture/runtime/snapshot.ts b/src/commands/capture/runtime/snapshot.ts index b661c4552a..418a308487 100644 --- a/src/commands/capture/runtime/snapshot.ts +++ b/src/commands/capture/runtime/snapshot.ts @@ -37,7 +37,7 @@ import { export type SnapshotCommandResult = { nodes: SnapshotNode[]; - truncated: boolean; + truncated?: boolean; appName?: string; appBundleId?: string; visibility?: SnapshotVisibility; @@ -68,9 +68,10 @@ export const snapshotCommand: RuntimeCommand< }, }); await runtime.sessions.set(nextSnapshotSession(options.session, capture)); + const truncated = snapshotTruncationForResult(capture.snapshot); return copySnapshotClickabilityEvidence(capture.snapshot, { nodes: capture.snapshot.nodes, - truncated: capture.snapshot.truncated ?? false, + ...(truncated === undefined ? {} : { truncated }), visibility: buildSnapshotVisibility({ nodes: capture.snapshot.nodes, backend: capture.snapshot.backend, @@ -218,6 +219,11 @@ function snapshotAppFields(capture: SnapshotCapture): { }; } +function snapshotTruncationForResult(snapshot: SnapshotState): boolean | undefined { + if (snapshot.truncated !== undefined) return snapshot.truncated; + return snapshot.backend === 'xctest' && snapshot.producer === 'appium-source' ? undefined : false; +} + function buildSnapshotWarnings(params: { result: BackendSnapshotResult; annotations: SnapshotCaptureAnnotations; diff --git a/src/daemon/result-serialization.ts b/src/daemon/result-serialization.ts index 3e71e3cd49..595257a303 100644 --- a/src/daemon/result-serialization.ts +++ b/src/daemon/result-serialization.ts @@ -67,7 +67,7 @@ export function serializeDevice(device: AgentDeviceDevice): Record { return { nodes: result.nodes, - truncated: result.truncated, + ...(result.truncated === undefined ? {} : { truncated: result.truncated }), ...(result.appName ? { appName: result.appName } : {}), ...(result.appBundleId ? { appBundleId: result.appBundleId } : {}), ...(result.visibility ? { visibility: result.visibility } : {}), diff --git a/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts b/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts index 57c2f8b807..ea6dcd1ef8 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts @@ -88,7 +88,15 @@ test('cloud iOS engine-presented snapshot survives daemon publication', async () ); const data = assertRpcOk<{ - nodes?: Array<{ type?: string; label?: string; identifier?: string; enabled?: boolean }>; + nodes?: Array<{ + type?: string; + label?: string; + identifier?: string; + enabled?: boolean; + hittable?: boolean; + }>; + truncated?: boolean; + warnings?: string[]; }>(response); assert.equal( data.nodes?.some( @@ -100,6 +108,12 @@ test('cloud iOS engine-presented snapshot survives daemon publication', async () ), true, ); + assert.equal( + data.nodes?.find((node) => node.identifier === 'DisplayNameTextField')?.hittable, + undefined, + ); + assert.ok(data.warnings?.some((warning) => warning.includes('hittability evidence'))); + assert.equal(data.truncated, undefined); assert.equal( data.nodes?.some((node) => node.type === 'StaticText' && node.label === 'Team Standup'), false, From 01710938ed31fcdad7f873888123170a2c2db09f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 1 Sep 2026 23:46:08 +0200 Subject: [PATCH 13/15] fix(ios): harden Appium evidence disclosure --- .../src/webdriver-ios-snapshot.test.ts | 4 +-- .../src/webdriver-ios-snapshot.ts | 26 +++++++++++-------- src/commands/capture/runtime/snapshot.test.ts | 15 +++++++++++ src/commands/capture/runtime/snapshot.ts | 10 ++++++- 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index a59f6178e4..c095f5115a 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -159,7 +159,7 @@ test('Appium iOS raw presentation discloses missing viewport without failing', a result.nodes?.some((node) => node.label === 'Continue'), true, ); - assert.ok(result.warnings?.some((warning) => warning.includes('snapshot --raw to inspect'))); + assert.ok(result.warnings?.some((warning) => warning.includes('cannot be validated'))); }); test('Appium iOS does not promote a non-viewport root rectangle to viewport evidence', () => { @@ -213,7 +213,7 @@ test('Appium iOS hierarchy limits are typed and disclosed at response level', as assert.equal(result.truncated, undefined); assert.deepEqual(result.warnings, [ - 'Appium page source does not guarantee hittability evidence; absent hittable means no evidence, not false. Raw output preserves any provider-reported value.', + 'Appium page source does not guarantee hittability evidence; absent hittable means no evidence, not false. Regular output omits reported hittable: true without evidence; raw preserves provider-reported values.', 'Appium page source does not report hierarchy completeness; provider-side depth or child limits may omit nodes.', ]); }); diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index 42aee93f7b..7f087b5653 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -28,15 +28,15 @@ const APPIUM_PRODUCER = IOS_SNAPSHOT_PRODUCER_CAPABILITIES['appium-source']; const iosSnapshotEngine = createIosSnapshotEngine(); const RESIDUE_WARNINGS = { 'missing-viewport': - 'Appium page source does not provide valid viewport evidence; regular snapshots fail closed. Use snapshot --raw to inspect the acquired tree.', -} satisfies Pick, 'missing-viewport'>; + 'Appium page source does not provide valid viewport evidence; this raw tree cannot be validated for viewport-relative regular presentation. Regular snapshots fail closed.', +} satisfies Record<'missing-viewport', string>; type WebDriverUnavailableFact = 'acquisition-depth' | 'hittability'; const UNAVAILABLE_FACT_WARNINGS = { 'acquisition-depth': 'Appium page source does not report hierarchy completeness; provider-side depth or child limits may omit nodes.', hittability: - 'Appium page source does not guarantee hittability evidence; absent hittable means no evidence, not false. Raw output preserves any provider-reported value.', + 'Appium page source does not guarantee hittability evidence; absent hittable means no evidence, not false. Regular output omits reported hittable: true without evidence; raw preserves provider-reported values.', } satisfies Record; export type WebDriverIosSnapshotAcquisition = Readonly<{ @@ -130,15 +130,19 @@ function warningsForResidue(residue: readonly IosAcquisitionResidue[]): { warnin } function warningForResidue(entry: IosAcquisitionResidue): string | undefined { - if (entry.kind === 'unavailable-fact') { - return entry.fact === 'acquisition-depth' || entry.fact === 'hittability' - ? UNAVAILABLE_FACT_WARNINGS[entry.fact] - : undefined; + switch (entry.kind) { + case 'unavailable-fact': + return entry.fact === 'acquisition-depth' || entry.fact === 'hittability' + ? UNAVAILABLE_FACT_WARNINGS[entry.fact] + : undefined; + case 'missing-viewport': + return RESIDUE_WARNINGS[entry.kind]; + case 'provider-pruned': + case 'truncated': + case 'stale-generation': + case 'fallback-source': + return undefined; } - if (entry.kind === 'missing-viewport') { - return RESIDUE_WARNINGS[entry.kind]; - } - return undefined; } function stripRefs(nodes: readonly SnapshotNode[]): RawSnapshotNode[] { diff --git a/src/commands/capture/runtime/snapshot.test.ts b/src/commands/capture/runtime/snapshot.test.ts index 8b209aa70a..7e9747563f 100644 --- a/src/commands/capture/runtime/snapshot.test.ts +++ b/src/commands/capture/runtime/snapshot.test.ts @@ -42,6 +42,21 @@ test('runtime snapshot captures nodes and updates the session baseline', async ( assert.equal(stored?.snapshot?.nodes[0]?.label, 'Home'); }); +test('runtime snapshot preserves unknown hierarchy completeness for engine-owned iOS acquisition', async () => { + const device = createSnapshotOnlyDevice({ + snapshot: { + nodes: [], + backend: 'xctest', + producer: 'appium-source', + createdAt: 1, + }, + }); + + const result = await device.capture.snapshot({ session: 'default' }); + + assert.equal(result.truncated, undefined); +}); + test('runtime snapshot forwards interactive capture options', async () => { let observedOptions: BackendSnapshotOptions | undefined; const device = createAgentDevice({ diff --git a/src/commands/capture/runtime/snapshot.ts b/src/commands/capture/runtime/snapshot.ts index 418a308487..02f983f6ca 100644 --- a/src/commands/capture/runtime/snapshot.ts +++ b/src/commands/capture/runtime/snapshot.ts @@ -25,6 +25,7 @@ import { buildSnapshotVisibility } from '../../../snapshot/snapshot-visibility.t import { ANDROID_SYSTEM_SURFACE_DISCLOSURE } from '../../../core/android-system-surface-disclosure.ts'; import { formatReactNativeOverlayWarning } from '../../react-native/overlay.ts'; import { now } from '../../runtime-common.ts'; +import { IOS_SNAPSHOT_PRODUCER_CAPABILITIES } from '@agent-device/capture-kit/ios-snapshot-planning'; import type { DiffSnapshotCommandOptions, RuntimeCommand, @@ -221,7 +222,14 @@ function snapshotAppFields(capture: SnapshotCapture): { function snapshotTruncationForResult(snapshot: SnapshotState): boolean | undefined { if (snapshot.truncated !== undefined) return snapshot.truncated; - return snapshot.backend === 'xctest' && snapshot.producer === 'appium-source' ? undefined : false; + if (snapshot.backend !== 'xctest' || snapshot.producer === undefined) return false; + const capability = IOS_SNAPSHOT_PRODUCER_CAPABILITIES[snapshot.producer]; + const acquisitionDepthUnknown = + capability.stage === 'acquired' && + capability.presentationOwner === 'ios-snapshot-engine' && + (capability.acquisitionDepth.rawTraversal.kind === 'incomplete' || + capability.acquisitionDepth.regularPresented.kind === 'incomplete'); + return acquisitionDepthUnknown ? undefined : false; } function buildSnapshotWarnings(params: { From a8f81edc042f8391b22171cd5cc7b2dc545db466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 2 Sep 2026 00:06:48 +0200 Subject: [PATCH 14/15] fix(ios): tighten Appium snapshot disclosures --- .../src/ios-snapshot-engine/engine.test.ts | 12 +++++------- .../src/webdriver-ios-snapshot.test.ts | 2 +- .../src/webdriver-ios-snapshot.ts | 16 +++++++++++++++- .../src/webdriver-source.test.ts | 14 ++++++++++++++ .../provider-webdriver/src/webdriver-source.ts | 7 +++---- src/commands/capture/runtime/snapshot.test.ts | 17 +++++++++++++++++ src/commands/capture/runtime/snapshot.ts | 10 ++++++++++ src/core/snapshot-state.ts | 10 ++++++---- 8 files changed, 71 insertions(+), 17 deletions(-) diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts index 948048d95b..50de132b25 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.test.ts @@ -295,14 +295,12 @@ test('unavailable hittability never becomes regular actionability', () => { residue: [{ kind: 'unavailable-fact' as const, fact: 'hittability' as const }], } satisfies IosSnapshotAcquisition; const acquired = publishIosSnapshot({ stage: 'acquired', acquisition: unavailable }, request); - assert.equal( - acquired.payload.nodes.find((node) => node.label === 'Partially visible')?.hittable, - undefined, - ); - assert.equal( - 'hittable' in (acquired.payload.nodes.find((node) => node.label === 'Partially visible') ?? {}), - false, + const partiallyVisible = acquired.payload.nodes.find( + (node) => node.label === 'Partially visible', ); + assert.ok(partiallyVisible); + assert.equal(partiallyVisible.hittable, undefined); + assert.equal('hittable' in partiallyVisible, false); const available = publishIosSnapshot(acquiredInput(request, nestedNodes()), request); const presented: IosSnapshotInput = { diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts index c095f5115a..0147f7b7a2 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.test.ts @@ -213,7 +213,7 @@ test('Appium iOS hierarchy limits are typed and disclosed at response level', as assert.equal(result.truncated, undefined); assert.deepEqual(result.warnings, [ - 'Appium page source does not guarantee hittability evidence; absent hittable means no evidence, not false. Regular output omits reported hittable: true without evidence; raw preserves provider-reported values.', + 'Appium page source does not guarantee hittability evidence; absent hittable means no evidence, not false. Regular output omits reported hittable: true without evidence but preserves reported false; raw preserves provider-reported values.', 'Appium page source does not report hierarchy completeness; provider-side depth or child limits may omit nodes.', ]); }); diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index 7f087b5653..802bb76843 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -36,7 +36,7 @@ const UNAVAILABLE_FACT_WARNINGS = { 'acquisition-depth': 'Appium page source does not report hierarchy completeness; provider-side depth or child limits may omit nodes.', hittability: - 'Appium page source does not guarantee hittability evidence; absent hittable means no evidence, not false. Regular output omits reported hittable: true without evidence; raw preserves provider-reported values.', + 'Appium page source does not guarantee hittability evidence; absent hittable means no evidence, not false. Regular output omits reported hittable: true without evidence but preserves reported false; raw preserves provider-reported values.', } satisfies Record; export type WebDriverIosSnapshotAcquisition = Readonly<{ @@ -73,6 +73,7 @@ export function acquireWebDriverIosSnapshot( customActions: options?.customActions, }); const plan = iosSnapshotEngine.plan(request, APPIUM_PRODUCER); + assertAppiumAcquisitionPlan(plan); const sourceFacts = parseWebDriverSourceFacts(source, { mode: 'facts' }); const viewport = resolveIosViewportEvidenceFromRoots(sourceFacts.roots) ?? { kind: 'missing' as const, @@ -120,6 +121,19 @@ function residueForSource(viewport: IosViewportEvidence): readonly IosAcquisitio return residue; } +function assertAppiumAcquisitionPlan(plan: IosSnapshotPlan): void { + if ( + plan.narrowing.depth !== null || + plan.narrowing.scope !== null || + plan.narrowing.interactiveOnly + ) { + throw new AppError( + 'COMMAND_FAILED', + 'Appium page source cannot narrow acquisition; requested options must remain engine-owned', + ); + } +} + function warningsForResidue(residue: readonly IosAcquisitionResidue[]): { warnings?: string[] } { const warnings = new Set(); for (const entry of residue) { diff --git a/packages/provider-webdriver/src/webdriver-source.test.ts b/packages/provider-webdriver/src/webdriver-source.test.ts index 5ec9df80ca..cbb30c105f 100644 --- a/packages/provider-webdriver/src/webdriver-source.test.ts +++ b/packages/provider-webdriver/src/webdriver-source.test.ts @@ -103,6 +103,20 @@ test('WebDriver source facts classify invalid root geometry', () => { ]); }); +test('WebDriver source facts do not call partial root geometry invalid', () => { + const facts = parseWebDriverSourceFacts( + '', + { mode: 'facts' }, + ); + + assert.deepEqual(facts.roots, [ + { + type: 'XCUIElementTypeApplication', + rectStatus: 'not-provided', + }, + ]); +}); + test('WebDriver scroll frame prefers visible scrollable containers', () => { assert.deepEqual( scrollFrameFromWebDriverSource( diff --git a/packages/provider-webdriver/src/webdriver-source.ts b/packages/provider-webdriver/src/webdriver-source.ts index 207115cfa0..edaefcda53 100644 --- a/packages/provider-webdriver/src/webdriver-source.ts +++ b/packages/provider-webdriver/src/webdriver-source.ts @@ -218,10 +218,9 @@ function rectStatus( attrs: Record, rect: RawSnapshotNode['rect'], ): WebDriverSourceRootFact['rectStatus'] { - const hasGeometryAttribute = ['bounds', 'x', 'y', 'width', 'height'].some( - (name) => attrs[name] !== undefined, - ); - if (!hasGeometryAttribute) return 'not-provided'; + const hasBoundsAttribute = attrs.bounds !== undefined; + const hasCompleteRect = ['x', 'y', 'width', 'height'].every((name) => attrs[name] !== undefined); + if (!hasBoundsAttribute && !hasCompleteRect) return 'not-provided'; return isPositiveRect(rect) ? 'reported' : 'invalid'; } diff --git a/src/commands/capture/runtime/snapshot.test.ts b/src/commands/capture/runtime/snapshot.test.ts index 7e9747563f..4bec1151ed 100644 --- a/src/commands/capture/runtime/snapshot.test.ts +++ b/src/commands/capture/runtime/snapshot.test.ts @@ -57,6 +57,23 @@ test('runtime snapshot preserves unknown hierarchy completeness for engine-owned assert.equal(result.truncated, undefined); }); +test('runtime snapshot uses the Appium sparse-tree disclosure for Appium acquisition', async () => { + const device = createSnapshotOnlyDevice({ + snapshot: { + nodes: [{ ref: 'e1', index: 0, type: 'Application', depth: 0 }], + backend: 'xctest', + producer: 'appium-source', + createdAt: 1, + }, + }); + + const result = await device.capture.snapshot({ session: 'default', interactiveOnly: true }); + + assert.equal(result.warnings?.length, 1); + assert.match(result.warnings?.[0] ?? '', /^Appium page source exposed only/); + assert.doesNotMatch(result.warnings?.[0] ?? '', /XCTest|simulator/); +}); + test('runtime snapshot forwards interactive capture options', async () => { let observedOptions: BackendSnapshotOptions | undefined; const device = createAgentDevice({ diff --git a/src/commands/capture/runtime/snapshot.ts b/src/commands/capture/runtime/snapshot.ts index 02f983f6ca..04808df503 100644 --- a/src/commands/capture/runtime/snapshot.ts +++ b/src/commands/capture/runtime/snapshot.ts @@ -224,6 +224,7 @@ function snapshotTruncationForResult(snapshot: SnapshotState): boolean | undefin if (snapshot.truncated !== undefined) return snapshot.truncated; if (snapshot.backend !== 'xctest' || snapshot.producer === undefined) return false; const capability = IOS_SNAPSHOT_PRODUCER_CAPABILITIES[snapshot.producer]; + if (!capability) return false; const acquisitionDepthUnknown = capability.stage === 'acquired' && capability.presentationOwner === 'ios-snapshot-engine' && @@ -289,6 +290,15 @@ function buildSparseIosInteractiveWarnings(params: { const root = params.snapshot.nodes[0]; if (root?.type !== 'Application') return []; + if (params.snapshot.producer === 'appium-source') { + return [ + 'Appium page source exposed only the application root. Descendants may be absent from the acquired hierarchy; use snapshot --raw to inspect the source and verify the app accessibility tree.', + ]; + } + if (params.snapshot.producer !== undefined && params.snapshot.producer !== 'apple-runner') { + return []; + } + return [ 'iOS interactive snapshot exposed only the application root. XCTest accessibility queries can fail to enumerate some simulator UI trees even when screenshots and direct gestures still work. Use screenshot as visual truth, try a scoped/full snapshot for diagnostics, and prefer direct selectors when known.', ]; diff --git a/src/core/snapshot-state.ts b/src/core/snapshot-state.ts index b6910f6d28..7a7dbe0d9b 100644 --- a/src/core/snapshot-state.ts +++ b/src/core/snapshot-state.ts @@ -141,18 +141,20 @@ function iosSnapshotPresentationStage( ): 'acquired' | 'presented' | undefined { if (provenance.backend !== 'xctest') return undefined; if (provenance.producer === undefined) return 'acquired'; - return IOS_SNAPSHOT_PRODUCER_CAPABILITIES[ - provenance.producer as 'apple-runner' | 'appium-source' | 'limrun-ios-tree' - ].stage; + return iosSnapshotCapabilities(provenance)?.stage; } function iosSnapshotPresentationOwner( provenance: SnapshotStateProvenance, ): 'ios-snapshot-engine' | 'snapshot-state' | undefined { + return iosSnapshotCapabilities(provenance)?.presentationOwner; +} + +function iosSnapshotCapabilities(provenance: SnapshotStateProvenance) { if (provenance.backend !== 'xctest' || provenance.producer === undefined) return undefined; return IOS_SNAPSHOT_PRODUCER_CAPABILITIES[ provenance.producer as 'apple-runner' | 'appium-source' | 'limrun-ios-tree' - ].presentationOwner; + ]; } function isAndroidComparisonSafeSnapshot( From 60e6494dcf6b39174569c6299fb6eb4b02d4dac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 2 Sep 2026 00:28:12 +0200 Subject: [PATCH 15/15] fix(ios): close Appium audit gaps --- CHANGELOG.md | 4 +++ .../ios-snapshot-engine/conformance.test.ts | 34 +++++++++++++++---- .../src/webdriver-ios-snapshot.ts | 14 -------- src/commands/capture/runtime/snapshot.test.ts | 2 +- src/commands/capture/runtime/snapshot.ts | 8 ++++- .../selector-capture-runtime.test.ts | 28 +++++++++++++++ src/daemon/selector-capture-runtime.ts | 3 +- .../cloud-webdriver-ios-text-entry.test.ts | 4 ++- 8 files changed, 72 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 069d1fd4a0..f7b75f5c05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- Breaking (0.21): iOS Appium/WebDriver snapshots now expose engine-owned acquisition facts and + typed fidelity warnings. The SDK snapshot `truncated` field is optional when Appium cannot report + hierarchy completeness; regular snapshots fail closed without valid viewport evidence, while + `snapshot --raw` remains available for diagnostics (#2195). - Security (daemon, remote/proxy HTTP only): when `AGENT_DEVICE_HTTP_AUTH_HOOK` is configured and a request's hook result does not attest a `tenantId`, the request is now refused (401) outright — the daemon no longer runs it as whichever tenant the client declared (RPC body `meta.tenantId` or diff --git a/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts b/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts index d1128fdbaa..c590b07415 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/conformance.test.ts @@ -7,7 +7,7 @@ import { createIosSnapshotRequest, deriveIosCaptureHint, } from '@agent-device/capture-kit/ios-snapshot-planning'; -import { IosSnapshotEngineError, presentIosSnapshot } from './index.ts'; +import { IosSnapshotEngineError, presentIosSnapshot, publishIosSnapshot } from './index.ts'; import { runTypeScriptCase, writeDifferentialFailureArtifact } from './conformance-harness.ts'; import { acquisitionForGoldenCase, @@ -86,7 +86,7 @@ test('the independent iOS snapshot goldens match the TypeScript engine', () => { | { outcome: 'success'; nodes: ReturnType; - truncated: boolean; + truncated?: boolean; residue: typeof acquisition.residue; qualityLabels?: readonly (string | null)[]; } @@ -97,16 +97,21 @@ test('the independent iOS snapshot goldens match the TypeScript engine', () => { }; try { - const result = presentIosSnapshot({ stage: 'acquired', acquisition }, request, { + const presentation = presentIosSnapshot({ stage: 'acquired', acquisition }, request, { + foldPolicy: testCase.foldPolicy, + }); + const publication = publishIosSnapshot({ stage: 'acquired', acquisition }, request, { foldPolicy: testCase.foldPolicy, }); actual = { outcome: 'success', - nodes: normalizeGoldenNodes(result.nodes), - truncated: acquisition.truncated ?? false, - residue: acquisition.residue, + nodes: normalizeGoldenNodes(publication.payload.nodes), + ...(publication.payload.truncated === undefined + ? {} + : { truncated: publication.payload.truncated }), + residue: publication.residue, ...(testCase.qualityLabels - ? { qualityLabels: result.qualityNodes?.map((node) => node.label ?? null) } + ? { qualityLabels: presentation.qualityNodes?.map((node) => node.label ?? null) } : {}), }; } catch (error) { @@ -125,6 +130,21 @@ test('the independent iOS snapshot goldens match the TypeScript engine', () => { } }); +test('published payload omits unknown truncation instead of defaulting it', () => { + const fixture = readIosSnapshotEngineFixture(); + const testCase = fixture.cases[0]!; + const request = requestForGoldenCase(testCase); + const acquisition = { + ...acquisitionForGoldenCase(fixture, testCase), + truncated: undefined, + }; + + const publication = publishIosSnapshot({ stage: 'acquired', acquisition }, request); + + assert.equal(publication.payload.truncated, undefined); + assert.equal('truncated' in publication.payload, false); +}); + test('the differential TypeScript runner preserves typed failures', () => { const fixture = readIosSnapshotEngineFixture(); const source = fixture.cases.find( diff --git a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts index 802bb76843..1b18aa663f 100644 --- a/packages/provider-webdriver/src/webdriver-ios-snapshot.ts +++ b/packages/provider-webdriver/src/webdriver-ios-snapshot.ts @@ -73,7 +73,6 @@ export function acquireWebDriverIosSnapshot( customActions: options?.customActions, }); const plan = iosSnapshotEngine.plan(request, APPIUM_PRODUCER); - assertAppiumAcquisitionPlan(plan); const sourceFacts = parseWebDriverSourceFacts(source, { mode: 'facts' }); const viewport = resolveIosViewportEvidenceFromRoots(sourceFacts.roots) ?? { kind: 'missing' as const, @@ -121,19 +120,6 @@ function residueForSource(viewport: IosViewportEvidence): readonly IosAcquisitio return residue; } -function assertAppiumAcquisitionPlan(plan: IosSnapshotPlan): void { - if ( - plan.narrowing.depth !== null || - plan.narrowing.scope !== null || - plan.narrowing.interactiveOnly - ) { - throw new AppError( - 'COMMAND_FAILED', - 'Appium page source cannot narrow acquisition; requested options must remain engine-owned', - ); - } -} - function warningsForResidue(residue: readonly IosAcquisitionResidue[]): { warnings?: string[] } { const warnings = new Set(); for (const entry of residue) { diff --git a/src/commands/capture/runtime/snapshot.test.ts b/src/commands/capture/runtime/snapshot.test.ts index 4bec1151ed..5ac0df81d8 100644 --- a/src/commands/capture/runtime/snapshot.test.ts +++ b/src/commands/capture/runtime/snapshot.test.ts @@ -60,7 +60,7 @@ test('runtime snapshot preserves unknown hierarchy completeness for engine-owned test('runtime snapshot uses the Appium sparse-tree disclosure for Appium acquisition', async () => { const device = createSnapshotOnlyDevice({ snapshot: { - nodes: [{ ref: 'e1', index: 0, type: 'Application', depth: 0 }], + nodes: [{ ref: 'e1', index: 0, type: 'XCUIElementTypeApplication', depth: 0 }], backend: 'xctest', producer: 'appium-source', createdAt: 1, diff --git a/src/commands/capture/runtime/snapshot.ts b/src/commands/capture/runtime/snapshot.ts index 04808df503..7e4833eb37 100644 --- a/src/commands/capture/runtime/snapshot.ts +++ b/src/commands/capture/runtime/snapshot.ts @@ -8,6 +8,7 @@ import { type SnapshotDiagnosticsSummary, } from '@agent-device/contracts/capture'; import { AppError } from '@agent-device/kernel/errors'; +import { normalizeType } from '@agent-device/contracts/snapshot'; import type { SnapshotNode, SnapshotState, @@ -288,7 +289,7 @@ function buildSparseIosInteractiveWarnings(params: { } const root = params.snapshot.nodes[0]; - if (root?.type !== 'Application') return []; + if (!isApplicationRoot(root)) return []; if (params.snapshot.producer === 'appium-source') { return [ @@ -304,6 +305,11 @@ function buildSparseIosInteractiveWarnings(params: { ]; } +function isApplicationRoot(node: SnapshotNode | undefined): boolean { + if (!node) return false; + return normalizeType(node.type ?? '') === 'application'; +} + const MERGED_LEAF_MIN_SEGMENTS = 10; /** diff --git a/src/daemon/__tests__/selector-capture-runtime.test.ts b/src/daemon/__tests__/selector-capture-runtime.test.ts index a88a6b2a2a..af4a9da770 100644 --- a/src/daemon/__tests__/selector-capture-runtime.test.ts +++ b/src/daemon/__tests__/selector-capture-runtime.test.ts @@ -100,6 +100,34 @@ test('legacy iOS sparse recovery retries a full snapshot', async () => { expect(boundCapture.mock.calls[1]?.[0]?.options).toMatchObject({ interactiveOnly: false }); }); +test('legacy iOS sparse recovery recognizes Appium application element types', async () => { + const { runtime } = makeCaptureRuntime('selector-appium-sparse-recovery'); + boundCapture + .mockResolvedValueOnce({ + backend: 'xctest', + producer: 'appium-source', + nodes: [{ index: 0, type: 'XCUIElementTypeApplication' }], + }) + .mockResolvedValueOnce({ + backend: 'xctest', + producer: 'appium-source', + nodes: [{ index: 0, type: 'XCUIElementTypeButton', label: 'Recovered' }], + }); + + const result = await runtime.capture({ + flags: { snapshotInteractiveOnly: true }, + recovery: { + legacyIosSparse: { + query: 'Search', + shouldScope: false, + }, + }, + }); + + expect(result.snapshot.nodes[0]?.label).toBe('Recovered'); + expect(boundCapture).toHaveBeenCalledTimes(2); +}); + test('legacy iOS sparse recovery rethrows full snapshot failure when scoping is disabled', async () => { const { runtime } = makeCaptureRuntime('selector-legacy-sparse-rethrow'); boundCapture diff --git a/src/daemon/selector-capture-runtime.ts b/src/daemon/selector-capture-runtime.ts index f06747679d..d111cada23 100644 --- a/src/daemon/selector-capture-runtime.ts +++ b/src/daemon/selector-capture-runtime.ts @@ -1,4 +1,5 @@ import type { CommandFlags } from '@agent-device/contracts/command'; +import { normalizeType } from '@agent-device/contracts/snapshot'; import type { BackendSnapshotResult } from '../backend.ts'; import { buildSnapshotPresentationKey, @@ -318,5 +319,5 @@ function updateSessionSnapshot(params: { function isLegacySparseIosInteractiveSnapshot(snapshot: SnapshotState): boolean { if (snapshot.snapshotQuality) return false; if (snapshot.backend !== 'xctest' || snapshot.nodes.length !== 1) return false; - return snapshot.nodes[0]?.type === 'Application'; + return normalizeType(snapshot.nodes[0]?.type ?? '') === 'application'; } diff --git a/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts b/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts index ea6dcd1ef8..0dd4af972f 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-ios-text-entry.test.ts @@ -115,7 +115,9 @@ test('cloud iOS engine-presented snapshot survives daemon publication', async () assert.ok(data.warnings?.some((warning) => warning.includes('hittability evidence'))); assert.equal(data.truncated, undefined); assert.equal( - data.nodes?.some((node) => node.type === 'StaticText' && node.label === 'Team Standup'), + data.nodes?.some( + (node) => node.type === 'XCUIElementTypeStaticText' && node.label === 'Team Standup', + ), false, ); assert.equal(