Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions packages/capture-kit/src/ios-snapshot-engine/errors.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import assert from 'node:assert/strict';
import { test } from 'vitest';
import { toIosSnapshotEngineErrorDetails } from './errors.ts';
import { IosSnapshotEngineError } from './types.ts';

test('engine error details expose only the public reason and field', () => {
const error = new IosSnapshotEngineError('invalid-viewport', 'invalid viewport', {
field: 'viewport',
index: 4,
frame: { x: 0, y: 0, width: 10, height: 10 },
});

assert.deepEqual(toIosSnapshotEngineErrorDetails(error), {
reason: 'invalid-viewport',
field: 'viewport',
});
});
15 changes: 15 additions & 0 deletions packages/capture-kit/src/ios-snapshot-engine/errors.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import type { IosSnapshotEngineError, IosSnapshotEngineFailureReason } from './types.ts';

export type IosSnapshotEnginePublicErrorDetails = Readonly<{
reason: IosSnapshotEngineFailureReason;
field?: string;
}>;

export function toIosSnapshotEngineErrorDetails(
error: IosSnapshotEngineError,
): IosSnapshotEnginePublicErrorDetails {
return {
reason: error.reason,
...(error.details.field ? { field: error.details.field } : {}),
};
}
2 changes: 2 additions & 0 deletions packages/capture-kit/src/ios-snapshot-engine/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,6 @@ export {
export { collectIosStructuralIdentifierSuppression } from './noise-structural.ts';
export { findNearestScrollableContainer, mergeReplacement, updateReplacement } from './tree.ts';
export { IosSnapshotEngineError } from './types.ts';
export { toIosSnapshotEngineErrorDetails } from './errors.ts';
export { resolveIosViewportEvidenceFromRoots } from './viewport.ts';
export type { SnapshotTreeRuleContext } from './tree.ts';
41 changes: 41 additions & 0 deletions packages/capture-kit/src/ios-snapshot-engine/viewport.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import assert from 'node:assert/strict';
import { test } from 'vitest';
import { resolveIosViewportEvidenceFromRoots } from './viewport.ts';

test('viewport evidence prefers reported application and window roots', () => {
assert.deepEqual(
resolveIosViewportEvidenceFromRoots([
{ type: 'XCUIElementTypeApplication', rectStatus: 'invalid' },
{
type: 'XCUIElementTypeWindow',
rect: { x: 0, y: 0, width: 390, height: 844 },
rectStatus: 'reported',
},
]),
{ kind: 'reported', rect: { x: 0, y: 0, width: 390, height: 844 } },
);
});

test('viewport evidence can fall back to the largest top-level root', () => {
assert.deepEqual(
resolveIosViewportEvidenceFromRoots(
[
{ type: 'Other', rect: { x: 0, y: 0, width: 100, height: 100 } },
{ type: 'Other', rect: { x: 0, y: 0, width: 200, height: 300 } },
],
{ fallbackToLargestRoot: true },
),
{ kind: 'reported', rect: { x: 0, y: 0, width: 200, height: 300 } },
);
});

test('viewport evidence preserves explicit missing geometry reasons', () => {
assert.deepEqual(
resolveIosViewportEvidenceFromRoots([{ type: 'Application', rectStatus: 'invalid' }]),
{ kind: 'missing', reason: 'invalid' },
);
assert.deepEqual(
resolveIosViewportEvidenceFromRoots([{ type: 'Application', rectStatus: 'not-provided' }]),
{ kind: 'missing', reason: 'not-provided' },
);
});
50 changes: 50 additions & 0 deletions packages/capture-kit/src/ios-snapshot-engine/viewport.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import type { IosViewportEvidence } from '@agent-device/contracts/ios-snapshot';
import { normalizeType } from '@agent-device/contracts/snapshot';
import type { Rect } from '@agent-device/kernel/snapshot';
import { isPositiveFiniteRect } from '@agent-device/kernel/rect';

export type IosSnapshotViewportRoot = Readonly<{
type?: string;
rect?: Rect;
rectStatus?: 'reported' | 'invalid' | 'not-provided';
}>;

export function resolveIosViewportEvidenceFromRoots(
roots: readonly IosSnapshotViewportRoot[],
options: Readonly<{ fallbackToLargestRoot?: boolean }> = {},
): IosViewportEvidence | undefined {
const viewportRoots = roots.filter(isViewportRoot);
const candidates =
viewportRoots.length > 0 || options.fallbackToLargestRoot !== true ? viewportRoots : roots;
const root = [...candidates].sort(compareViewportRoots)[0];
if (!root) return undefined;
if (isPositiveFiniteRect(root.rect)) return { kind: 'reported', rect: root.rect };
return {
kind: 'missing',
reason:
root.rectStatus === 'invalid' || (root.rectStatus === undefined && root.rect !== undefined)
? 'invalid'
: 'not-provided',
};
}

function isViewportRoot(root: IosSnapshotViewportRoot): boolean {
const type = normalizeType(root.type ?? '');
return type === 'application' || type === 'window';
}

function compareViewportRoots(
left: IosSnapshotViewportRoot,
right: IosSnapshotViewportRoot,
): number {
const status = rootGeometryRank(right.rectStatus) - rootGeometryRank(left.rectStatus);
return status || rectArea(right.rect) - rectArea(left.rect);
}

function rootGeometryRank(status: IosSnapshotViewportRoot['rectStatus']): number {
return status === 'reported' ? 2 : status === 'invalid' ? 1 : 0;
}

function rectArea(rect: Rect | undefined): number {
return rect && isPositiveFiniteRect(rect) ? rect.width * rect.height : 0;
}
4 changes: 4 additions & 0 deletions packages/capture-kit/src/ios-snapshot-planning.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,10 @@ test('presented producers cannot claim acquisition narrowing', () => {
});
});

test('Appium source advertises viewport evidence when its root reports geometry', () => {
assert.equal(IOS_SNAPSHOT_PRODUCER_CAPABILITIES['appium-source'].viewportEvidence, 'available');
});

test('comparison identity rejects every identity axis and residue mismatch', () => {
const base = comparisonIdentity();
const mismatches: IosSnapshotComparisonIdentity[] = [
Expand Down
2 changes: 1 addition & 1 deletion packages/capture-kit/src/ios-snapshot-planning.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ const IOS_SNAPSHOT_PRODUCER_CAPABILITY_VALUES = {
},
scopeCompleteness: 'incomplete',
interactiveQueryCompleteness: 'incomplete',
viewportEvidence: 'unavailable',
viewportEvidence: 'available',
hittabilityEvidence: 'unavailable',
},
'limrun-ios-tree': {
Expand Down
3 changes: 3 additions & 0 deletions packages/contracts/src/facades/capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,17 @@ export type {
ScreenshotResultData,
} from '../snapshot-types.ts';
export {
attachSnapshotPresentationEvidence,
attachSnapshotClickabilityEvidence,
attachSnapshotOcclusionContextEvidence,
copySnapshotClickabilityEvidence,
readSnapshotClickabilityEvidence,
readSnapshotOcclusionContextEvidence,
readSnapshotPresentationEvidence,
} from '../snapshot-private-evidence.ts';
export type {
AndroidSiblingOrderEvidence,
SnapshotPresentationEvidence,
SnapshotClickabilityEvidence,
SnapshotOcclusionContextEvidence,
} from '../snapshot-private-evidence.ts';
Expand Down
19 changes: 19 additions & 0 deletions packages/contracts/src/snapshot-private-evidence.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,15 @@ export type SnapshotOcclusionContextEvidence = {
androidSiblingOrderByNodeIndex?: ReadonlyMap<number, AndroidSiblingOrderEvidence>;
};

/** Identifies the engine that owns interactive iOS snapshot presentation. */
export type SnapshotPresentationEvidence = {
owner: 'ios-snapshot-engine';
};

type SnapshotPrivateEvidence = {
clickability?: SnapshotClickabilityEvidence;
occlusionContext?: SnapshotOcclusionContextEvidence;
presentation?: SnapshotPresentationEvidence;
};

const privateEvidenceBySnapshotObject = new WeakMap<object, SnapshotPrivateEvidence>();
Expand Down Expand Up @@ -59,6 +65,19 @@ export function readSnapshotClickabilityEvidence(
return owner ? privateEvidenceBySnapshotObject.get(owner)?.clickability : undefined;
}

export function attachSnapshotPresentationEvidence<T extends object>(
owner: T,
evidence: SnapshotPresentationEvidence,
): T {
return attachSnapshotPrivateEvidence(owner, { presentation: evidence });
}

export function readSnapshotPresentationEvidence(
owner: object | null | undefined,
): SnapshotPresentationEvidence | undefined {
return owner ? privateEvidenceBySnapshotObject.get(owner)?.presentation : undefined;
}

export function copySnapshotClickabilityEvidence<T extends object>(
source: object | null | undefined,
target: T,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -275,13 +275,12 @@ test('snapshot reports typed runner presentation failures', async () => {
{ runCommand: async () => ({ nodes: [{ index: 0, type: 'Application' }] }) },
);

await assert.rejects(
interactor.snapshot(),
(error: unknown) =>
error instanceof AppError &&
error.code === 'COMMAND_FAILED' &&
error.details?.reason === 'missing-viewport',
);
await assert.rejects(interactor.snapshot(), (error: unknown) => {
assert.ok(error instanceof AppError);
assert.equal(error.code, 'COMMAND_FAILED');
assert.deepEqual(error.details, { reason: 'missing-viewport', field: 'viewport' });
return true;
});
});

test('sparse runner payloads with no viewport fail before publishing actionable nodes', async () => {
Expand Down
42 changes: 12 additions & 30 deletions packages/platform-apple/src/runner/snapshot-presentation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,17 @@ import type {
IosViewportEvidence,
} from '@agent-device/contracts/ios-snapshot';
import type { SnapshotOptions } from '@agent-device/contracts/interactor-types';
import { normalizeType } from '@agent-device/contracts/snapshot';
import {
IosSnapshotEngineError,
presentIosRunnerSnapshot,
resolveIosViewportEvidenceFromRoots,
toIosSnapshotEngineErrorDetails,
} from '@agent-device/capture-kit/ios-snapshot-engine';
import { readSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict';
import {
createIosSnapshotRequest,
buildIosSnapshotPresentationKey,
} from '@agent-device/capture-kit/ios-snapshot-planning';
import { isPositiveFiniteRect } from '@agent-device/kernel/rect';
import { AppError } from '@agent-device/kernel/errors';
import type { RawSnapshotNode, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot';

Expand Down Expand Up @@ -119,36 +119,18 @@ function runnerViewportEvidence(
qualityNodes: readonly RawSnapshotNode[] | undefined,
): IosViewportEvidence {
return (
readReportedViewport(qualityNodes) ??
readReportedViewport(nodes) ?? { kind: 'missing', reason: 'not-provided' }
resolveIosViewportEvidenceFromRoots(rootNodes(qualityNodes), {
fallbackToLargestRoot: true,
}) ??
resolveIosViewportEvidenceFromRoots(rootNodes(nodes), { fallbackToLargestRoot: true }) ?? {
kind: 'missing',
reason: 'not-provided',
}
);
}

function readReportedViewport(
nodes: readonly RawSnapshotNode[] | undefined,
): IosViewportEvidence | undefined {
const roots = nodes?.filter((node) => node.parentIndex === undefined) ?? [];
const root =
[...roots]
.filter((node) => isViewportRoot(node))
.sort(compareRectArea)
.at(0) ?? [...roots].sort(compareRectArea).at(0);
if (!root) return undefined;
if (isPositiveFiniteRect(root.rect)) return { kind: 'reported', rect: root.rect };
return { kind: 'missing', reason: root.rect ? 'invalid' : 'not-provided' };
}

function isViewportRoot(node: RawSnapshotNode): boolean {
const type = normalizeType(node.type ?? '');
return type === 'application' || type === 'window';
}

function compareRectArea(left: RawSnapshotNode, right: RawSnapshotNode): number {
return rectArea(right.rect) - rectArea(left.rect);
}

function rectArea(rect: RawSnapshotNode['rect']): number {
return rect ? rect.width * rect.height : 0;
function rootNodes(nodes: readonly RawSnapshotNode[] | undefined): readonly RawSnapshotNode[] {
return nodes?.filter((node) => node.parentIndex === undefined) ?? [];
}

function isRecord(value: unknown): value is Record<string, unknown> {
Expand All @@ -160,7 +142,7 @@ function throwSnapshotEngineError(error: unknown): never {
throw new AppError(
'COMMAND_FAILED',
error.message,
{ reason: error.reason, iosSnapshotEngine: { details: error.details } },
toIosSnapshotEngineErrorDetails(error),
error,
);
}
1 change: 1 addition & 0 deletions packages/provider-webdriver/src/runtime-session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ export class WebDriverSessionManager {
client,
backend: snapshotBackendForPlatform(prepared.platform),
capabilities,
targetId: device.id,
}),
}),
);
Expand Down
40 changes: 40 additions & 0 deletions packages/provider-webdriver/src/webdriver-interactor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,46 @@ test('fill refuses empty text as an unsupported clear rather than a vacuous succ
assert.deepEqual(world.transcript, []);
});

test('iOS WebDriver interactor routes snapshots through the acquisition adapter', async () => {
const source = vi.fn(
async () =>
'<AppiumAUT><XCUIElementTypeApplication x="0" y="0" width="390" height="844" /></AppiumAUT>',
);
const interactor = createWebDriverInteractor({
client: { source } as unknown as WebDriverClient,
backend: 'xctest',
capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'ios' }),
targetId: 'ios-1',
});

const result = await interactor.snapshot({ raw: true, depth: 1 });

assert.equal(result.backend, 'xctest');
assert.equal(result.producer, 'appium-source');
assert.equal(source.mock.calls.length, 1);
assert.equal(result.nodes?.[0]?.type, 'XCUIElementTypeApplication');
});

test('Android WebDriver interactor keeps legacy-derived source facts at its call site', async () => {
const source = vi.fn(
async () =>
'<hierarchy rotation="0"><android.widget.Button bounds="[0,0][100,40]" displayed="true" enabled="true" /></hierarchy>',
);
const interactor = createWebDriverInteractor({
client: { source } as unknown as WebDriverClient,
backend: 'android',
capabilities: createCloudWebDriverCapabilities({ provider: 'test', platform: 'android' }),
});

const result = await interactor.snapshot();

assert.equal(result.backend, 'android');
assert.equal(result.nodes?.[0]?.type, 'hierarchy');
assert.equal(result.nodes?.[1]?.type, 'android.widget.Button');
assert.equal(result.nodes?.[1]?.hittable, true);
assert.equal(source.mock.calls.length, 1);
});

async function runFill(world: ReturnType<typeof createTextEntryWorld>) {
vi.useFakeTimers();
try {
Expand Down
Loading
Loading