diff --git a/.github/actions/setup-apple-runner-build/action.yml b/.github/actions/setup-apple-runner-build/action.yml index 4fda0f79ee..6f72548419 100644 --- a/.github/actions/setup-apple-runner-build/action.yml +++ b/.github/actions/setup-apple-runner-build/action.yml @@ -44,7 +44,7 @@ runs: id: source-hash run: | set -euo pipefail - echo "value=${{ hashFiles('apple/runner/**', 'scripts/build-xcuitest-apple.sh', 'scripts/patch-xcuitest-runner-icon.ts', 'scripts/write-xcuitest-cache-metadata.mjs', 'packages/platform-apple/src/runner/apple-runner-platform.ts', 'packages/platform-apple/src/runner/runner-cache-metadata.ts', 'packages/platform-apple/src/runner/runner-icon.ts', 'packages/platform-apple/src/runner/runner-xctestrun.ts', 'packages/platform-apple/src/runner/runner-xctestrun-products.ts', '.github/actions/setup-apple-runner-build/action.yml', 'package.json', 'pnpm-lock.yaml') }}" >> "$GITHUB_OUTPUT" + echo "value=${{ hashFiles('apple/runner/**', 'apple/snapshot-presentation/**', 'scripts/build-xcuitest-apple.sh', 'scripts/patch-xcuitest-runner-icon.ts', 'scripts/write-xcuitest-cache-metadata.mjs', 'packages/platform-apple/src/runner/apple-runner-platform.ts', 'packages/platform-apple/src/runner/runner-cache-metadata.ts', 'packages/platform-apple/src/runner/runner-icon.ts', 'packages/platform-apple/src/runner/runner-xctestrun.ts', 'packages/platform-apple/src/runner/runner-xctestrun-products.ts', '.github/actions/setup-apple-runner-build/action.yml', 'package.json', 'pnpm-lock.yaml') }}" >> "$GITHUB_OUTPUT" shell: bash - name: Resolve Apple runner build variant diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 32795feddc..5cc88cf668 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -239,6 +239,7 @@ struct DataPayload: Codable { var items: [String]? var nodes: [PresentedNode]? var truncated: Bool? + var qualityPayload: SnapshotQualityPayload? = nil var snapshotQuality: SnapshotQuality? var gestureStartUptimeMs: Double? var gestureEndUptimeMs: Double? @@ -274,6 +275,31 @@ struct DataPayload: Codable { var sequenceResults: [SequenceStepResult]? } +struct SnapshotQualityPayload: Codable { + let nodes: [PresentedNode] + let truncated: Bool + let scope: String? + + init(nodes: [PresentedNode], truncated: Bool) { + self.nodes = nodes + self.truncated = truncated + self.scope = nil + } + + private enum CodingKeys: String, CodingKey { + case nodes + case truncated + case scope + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(nodes, forKey: .nodes) + try container.encode(truncated, forKey: .truncated) + try container.encodeNil(forKey: .scope) + } +} + struct ErrorPayload: Codable { var code: String? let message: String diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 0d5d6c07ef..05a9a3c2cf 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -490,6 +490,11 @@ extension RunnerTests { do { presented = try timer.measure(.presentation) { guard let result = try SnapshotPresentation.present(acquisition, options: options) else { + NSLog( + "AGENT_DEVICE_RUNNER_SNAPSHOT_PROJECTION_MISMATCH requested=%@ acquired=%@", + hint.projection.rawValue, + acquisition.hint.projection.rawValue + ) throw Self.snapshotProjectionMismatchFailure( kind, requested: hint.projection, @@ -641,6 +646,10 @@ extension RunnerTests { message: Self.legacyQualityMessage(quality) ?? payload.message, nodes: payload.nodes, truncated: payload.truncated == true || state != "healthy" || capture.effectiveDepth != nil, + qualityPayload: capture.qualityPayload.flatMap { quality in + guard let nodes = quality.nodes else { return nil } + return SnapshotQualityPayload(nodes: nodes, truncated: quality.truncated == true) + }, snapshotQuality: quality, runnerFatal: payload.runnerFatal, runnerFatalReason: payload.runnerFatalReason @@ -877,6 +886,29 @@ extension RunnerTests { XCTAssertEqual(payload.nodes?.count, 1) } + func testSnapshotQualityCarriesUnscopedQualityPayload() { + let quality = DataPayload( + nodes: [planTestNode(index: 0, type: "Application", label: "App")], + truncated: false + ) + let capture = SnapshotBackendCapture( + payload: quality, + effectiveDepth: nil, + qualityPayload: quality + ) + + let payload = stampedSnapshotPayload( + capture, + backend: .recursiveTree, + state: "healthy", + reason: nil + ) + + XCTAssertEqual(payload.qualityPayload?.nodes.count, 1) + XCTAssertEqual(payload.qualityPayload?.truncated, false) + XCTAssertNil(payload.qualityPayload?.scope) + } + func testDirectPresentationDoesNotClaimPlanTiming() { let options = PresentationOptions( interactiveOnly: false, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift index cfa4df03e1..65012e982d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SnapshotPresentationInvariantTests.swift @@ -307,7 +307,7 @@ extension RunnerTests { } XCTAssertThrowsError( - try SnapshotPresentationInvariant.validateRegularWithStats( + try SnapshotPresentationInvariant.validateRegular( folded, viewport: viewport, policy: .cursorProjected @@ -323,30 +323,6 @@ extension RunnerTests { } } - func testRegularInvariantUsesOneParentClipLookupPerNode() throws { - let nodeCount = 5_000 - let viewport = CGRect(x: 0, y: 0, width: 100, height: 100) - let nodes = (0..= 0 && index < states.count ? states[index] : nil - } + let parentState = node.parentIndex.flatMap { states[$0] } let parentTraversal = parentState?.traversal ?? .root let parentAnchor = policy == .cursorProjected ? parentState?.anchor : nil let rect = node.rect.cgRect diff --git a/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/InvariantTests.swift b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/InvariantTests.swift new file mode 100644 index 0000000000..6b06f8c5a4 --- /dev/null +++ b/apple/snapshot-presentation/Tests/AgentDeviceSnapshotPresentationTests/InvariantTests.swift @@ -0,0 +1,37 @@ +import CoreGraphics +import XCTest +@testable import AgentDeviceSnapshotPresentation + +final class InvariantTests: XCTestCase { + func testRegularInvariantUsesOneParentClipLookupPerNode() throws { + let nodeCount = 5_000 + let viewport = CGRect(x: 0, y: 0, width: 100, height: 100) + let nodes = (0.. node.label === 'Escaped child')?.hittable, true); }); +test('presentation mapping retains acquisition lineage through projection and scoped reindexing', () => { + const nodes: RawSnapshotNode[] = [ + node(10, 'Application', 'App', viewport), + node(20, 'Other', undefined, viewport, 10, 1), + node(40, 'Button', 'Target', { x: 20, y: 20, width: 80, height: 40 }, 20, 2), + ]; + const regularRequest = createIosSnapshotRequest(); + const regular = presentIosSnapshot(acquiredInput(regularRequest, nodes), regularRequest); + + assert.deepEqual( + [...regular.presentedIndexesBySourceIndex], + [ + [10, [0]], + [20, []], + [40, [1]], + ], + ); + + const rawRequest = createIosSnapshotRequest({ raw: true, scope: 'Target' }); + const rawNodes = [ + node(10, 'Application', 'App', viewport), + node(20, 'Other', 'Target', viewport, 10, 1), + node(40, 'Button', 'Child', { x: 20, y: 20, width: 80, height: 40 }, 20, 2), + ]; + const raw = presentIosSnapshot(acquiredInput(rawRequest, rawNodes), rawRequest); + + assert.deepEqual( + [...raw.presentedIndexesBySourceIndex], + [ + [10, []], + [20, [0]], + [40, [1]], + ], + ); +}); + +test('raw scoped depth derives missing source depths from parent order', () => { + const request = createIosSnapshotRequest({ raw: true, scope: 'Target', depth: 1 }); + const nodes: RawSnapshotNode[] = [ + node(10, 'Application', 'App', viewport), + { ...node(20, 'Other', 'Target', viewport, 10), depth: undefined }, + { + ...node(40, 'Button', 'Child', { x: 20, y: 20, width: 80, height: 40 }, 20), + depth: undefined, + }, + { + ...node(50, 'Button', 'Grandchild', { x: 24, y: 24, width: 60, height: 32 }, 40), + depth: undefined, + }, + ]; + const result = presentIosSnapshot(acquiredInput(request, nodes), request); + + assert.deepEqual( + result.nodes.map((entry) => entry.label), + ['Target', 'Child'], + ); +}); + test('cursor projection keeps geometryless nodes neutral while plain viewport keeps child visibility independent', () => { const request = createIosSnapshotRequest(); const nodes = [ @@ -285,7 +343,7 @@ function scopedNodes(): RawSnapshotNode[] { function node( index: number, type: string, - label: string, + label: string | undefined, rect: Rect, parentIndex?: number, depth?: number, diff --git a/packages/capture-kit/src/ios-snapshot-engine/engine.ts b/packages/capture-kit/src/ios-snapshot-engine/engine.ts index 590a392872..877c6942bd 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/engine.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/engine.ts @@ -87,8 +87,9 @@ function presentAcquiredSnapshot( if (request.projection === 'raw') { validateIosSnapshotGraph(acquisition.nodes); + const sourceNodes = acquisition.nodes.map((raw) => ({ raw, sourceIndex: raw.index })); const projected = projectIosSnapshot({ - nodes: acquisition.nodes.map((raw) => ({ raw })), + nodes: sourceNodes, projection: 'raw', scope: request.scope, depth: request.depth, @@ -98,7 +99,7 @@ function presentAcquiredSnapshot( request.scope === null ? undefined : projectIosQualitySnapshot({ - nodes: acquisition.nodes.map((raw) => ({ raw })), + nodes: sourceNodes, projection: 'raw', depth: null, foldPolicy, @@ -106,7 +107,12 @@ function presentAcquiredSnapshot( return { nodes: projected.nodes, ...(qualityNodes ? { qualityNodes } : {}), - presentedIndexesBySourceIndex: identityMapping(projected.nodes), + presentedIndexesBySourceIndex: remapPresentedIndexes( + acquisition.nodes, + projected.nodes, + projected.sourceIndexes, + identityMapping(projected.nodes), + ), stats: { presentedNodeCount: projected.nodes.length, sourceNodeCount: acquisition.nodes.length, @@ -155,7 +161,12 @@ function presentAcquiredSnapshot( return { nodes: compacted.nodes, ...(qualityNodes ? { qualityNodes } : {}), - presentedIndexesBySourceIndex: compacted.presentedIndexesBySourceIndex, + presentedIndexesBySourceIndex: remapPresentedIndexes( + acquisition.nodes, + projected.nodes, + projected.sourceIndexes, + compacted.presentedIndexesBySourceIndex, + ), stats: { presentedNodeCount: compacted.nodes.length, sourceNodeCount: acquisition.nodes.length, @@ -194,3 +205,24 @@ function identityMapping( ): ReadonlyMap { return new Map(nodes.map((node) => [node.index, [node.index]])); } + +function remapPresentedIndexes( + sourceNodes: readonly RawSnapshotNode[], + projectedNodes: readonly RawSnapshotNode[], + sourceIndexes: readonly number[], + presentedIndexesByProjectedIndex: ReadonlyMap, +): ReadonlyMap { + const sourceIndexByProjectedIndex = new Map( + projectedNodes.map((node, position) => [node.index, sourceIndexes[position]!]), + ); + const remapped = new Map( + sourceNodes.map((node) => [node.index, []] as const), + ); + for (const [projectedIndex, presentedIndexes] of presentedIndexesByProjectedIndex) { + const sourceIndex = sourceIndexByProjectedIndex.get(projectedIndex); + if (sourceIndex !== undefined) { + remapped.set(sourceIndex, presentedIndexes); + } + } + return remapped; +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts index 313abe853a..6a83656827 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/geometry.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/geometry.ts @@ -119,6 +119,7 @@ function appendFoldedNode( node.hittable === true && isGeometricallyActionable(node.enabled !== false, decision.effectiveRect, viewport), }, + sourceIndex: node.index, ...(decision.effectiveRect ? { effectiveRect: decision.effectiveRect } : {}), }); keptIndex = index; diff --git a/packages/capture-kit/src/ios-snapshot-engine/index.ts b/packages/capture-kit/src/ios-snapshot-engine/index.ts index 7a55916eb5..0c13dc24f8 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/index.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/index.ts @@ -4,6 +4,7 @@ export { presentIosSnapshot, publishIosSnapshot, } from './engine.ts'; +export { presentIosRunnerSnapshot } from './runner-presentation.ts'; export { buildIosInteractiveSnapshotPresentation, presentIosInteractiveSnapshot, diff --git a/src/snapshot/snapshot-presentation/ios/mapping.test.ts b/packages/capture-kit/src/ios-snapshot-engine/mapping.test.ts similarity index 100% rename from src/snapshot/snapshot-presentation/ios/mapping.test.ts rename to packages/capture-kit/src/ios-snapshot-engine/mapping.test.ts diff --git a/src/snapshot/snapshot-presentation/ios/noise.test.ts b/packages/capture-kit/src/ios-snapshot-engine/noise.test.ts similarity index 100% rename from src/snapshot/snapshot-presentation/ios/noise.test.ts rename to packages/capture-kit/src/ios-snapshot-engine/noise.test.ts diff --git a/src/snapshot/snapshot-presentation/ios/presentation.test.ts b/packages/capture-kit/src/ios-snapshot-engine/presentation.test.ts similarity index 100% rename from src/snapshot/snapshot-presentation/ios/presentation.test.ts rename to packages/capture-kit/src/ios-snapshot-engine/presentation.test.ts diff --git a/packages/capture-kit/src/ios-snapshot-engine/projection.ts b/packages/capture-kit/src/ios-snapshot-engine/projection.ts index ecb33c8c3e..d0a10ba727 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/projection.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/projection.ts @@ -38,17 +38,19 @@ export type IosSnapshotProjectionResult = Readonly<{ }>; export function projectIosSnapshot(input: ProjectionInput): IosSnapshotProjectionResult { - const scoped = scopeIosSnapshotNodes(input); + const depths = resolveRawDepths(input.nodes); + const scoped = scopeIosSnapshotNodes(input, depths); return input.projection === 'raw' - ? projectRawNodes(scoped, input.depth) + ? projectRawNodes(scoped, input.depth, depths) : projectRegularNodes(scoped, input.depth); } export function projectIosQualitySnapshot( input: Omit, ): IosSnapshotProjectionResult { + const depths = resolveRawDepths(input.nodes); return input.projection === 'raw' - ? projectRawNodes(input.nodes, input.depth) + ? projectRawNodes(input.nodes, input.depth, depths) : projectRegularNodes(input.nodes, input.depth); } @@ -57,14 +59,17 @@ function isEligibleForIosRegularPresentation(node: RawSnapshotNode): boolean { return REGULAR_ELIGIBLE_TYPES.has(normalizeType(node.type ?? '')) || hasSemanticContent(node); } -function scopeIosSnapshotNodes(input: ProjectionInput): IosSnapshotPresentationNode[] { +function scopeIosSnapshotNodes( + input: ProjectionInput, + depths: ReadonlyMap, +): IosSnapshotPresentationNode[] { const query = input.scope?.trim().toLowerCase(); if (!query) return [...input.nodes]; for (let start = 0; start < input.nodes.length; start += 1) { const candidate = input.nodes[start]; if (!candidate || !matchesScope(candidate.raw, query)) continue; - const range = subtreeRange(input.nodes, start); + const range = subtreeRange(input.nodes, start, depths); const contributes = input.projection === 'raw' || range.some((position) => isEligibleForIosRegularPresentation(input.nodes[position]!.raw)); @@ -73,9 +78,11 @@ function scopeIosSnapshotNodes(input: ProjectionInput): IosSnapshotPresentationN const scoped = range.map((position) => input.nodes[position]!); const limited = input.projection === 'raw' && input.depth !== null - ? scoped.filter((node) => rawDepth(node) - rawDepth(candidate) <= input.depth!) + ? scoped.filter( + (node) => rawDepth(node, depths) - rawDepth(candidate, depths) <= input.depth!, + ) : scoped; - return reindexScopedNodes(limited, rawDepth(candidate)); + return reindexScopedNodes(limited, rawDepth(candidate, depths), depths); } return []; } @@ -83,12 +90,15 @@ function scopeIosSnapshotNodes(input: ProjectionInput): IosSnapshotPresentationN function projectRawNodes( nodes: readonly IosSnapshotPresentationNode[], maximumDepth: number | null, + depths: ReadonlyMap, ): IosSnapshotProjectionResult { const selected = - maximumDepth === null ? [...nodes] : nodes.filter((node) => rawDepth(node) <= maximumDepth); + maximumDepth === null + ? [...nodes] + : nodes.filter((node) => rawDepth(node, depths) <= maximumDepth); return { nodes: selected.map((node) => ({ ...node.raw, rect: node.raw.rect })), - sourceIndexes: selected.map((node) => node.raw.index), + sourceIndexes: selected.map((node) => node.sourceIndex), }; } @@ -108,7 +118,7 @@ function projectRegularNodes( continue; } presented.push(projected); - sourceIndexes.push(node.raw.index); + sourceIndexes.push(node.sourceIndex); nearestPresented.set(node.raw.index, projected); } return { nodes: presented, sourceIndexes }; @@ -162,6 +172,7 @@ function rememberNearestPresented( function reindexScopedNodes( nodes: readonly IosSnapshotPresentationNode[], depthOffset: number, + depths: ReadonlyMap, ): IosSnapshotPresentationNode[] { const indexMap = new Map(nodes.map((node, index) => [node.raw.index, index])); return nodes.map((node, index) => ({ @@ -169,7 +180,7 @@ function reindexScopedNodes( raw: { ...node.raw, index, - depth: Math.max(0, rawDepth(node) - depthOffset), + depth: Math.max(0, rawDepth(node, depths) - depthOffset), parentIndex: node.raw.parentIndex === undefined ? undefined : indexMap.get(node.raw.parentIndex), }, @@ -182,18 +193,38 @@ function matchesScope(node: RawSnapshotNode, query: string): boolean { ); } -function subtreeRange(nodes: readonly IosSnapshotPresentationNode[], start: number): number[] { - const rootDepth = rawDepth(nodes[start]!); +function subtreeRange( + nodes: readonly IosSnapshotPresentationNode[], + start: number, + depths: ReadonlyMap, +): number[] { + const rootDepth = rawDepth(nodes[start]!, depths); const positions: number[] = []; for (let position = start; position < nodes.length; position += 1) { - if (position > start && rawDepth(nodes[position]!) <= rootDepth) break; + if (position > start && rawDepth(nodes[position]!, depths) <= rootDepth) break; positions.push(position); } return positions; } -function rawDepth(node: IosSnapshotPresentationNode): number { - return Math.max(0, node.raw.depth ?? 0); +function rawDepth(node: IosSnapshotPresentationNode, depths?: ReadonlyMap): number { + return Math.max(0, node.raw.depth ?? depths?.get(node.raw.index) ?? 0); +} + +function resolveRawDepths( + nodes: readonly IosSnapshotPresentationNode[], +): ReadonlyMap { + const depths = new Map(); + for (const node of nodes) { + const parentDepth = + node.raw.parentIndex === undefined ? -1 : (depths.get(node.raw.parentIndex) ?? -1); + const structuralDepth = parentDepth + 1; + depths.set( + node.raw.index, + node.raw.depth === undefined ? structuralDepth : Math.max(0, node.raw.depth), + ); + } + return depths; } function hasSemanticContent(node: RawSnapshotNode): boolean { diff --git a/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.test.ts b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.test.ts new file mode 100644 index 0000000000..58f1c0b1aa --- /dev/null +++ b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.test.ts @@ -0,0 +1,135 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { + IosSnapshotInput, + IosSnapshotRequest, + IosSnapshotValidationFacts, +} from '@agent-device/contracts/ios-snapshot'; +import { + buildIosSnapshotPresentationKey, + createIosSnapshotRequest, +} from '@agent-device/capture-kit/ios-snapshot-planning'; +import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; +import { presentIosSnapshot } from './index.ts'; + +const viewport: Rect = { x: 0, y: 0, width: 402, height: 874 }; + +test('runner presentation clips rows to a scroll viewport derived from its indicator', () => { + const request = createIosSnapshotRequest({ interactiveOnly: true }); + const nodes = runnerNodes(); + const result = presentIosSnapshot(runnerInput(request, nodes), request); + const screenTime = result.nodes.find((node) => node.label === 'Screen Time'); + + assert.deepEqual(screenTime?.rect, { + x: 16, + y: 796.3333333333334, + width: 370, + height: 15.666666666666629, + }); + assert.equal(screenTime?.hittable, true); + assert.equal( + result.nodes.some((node) => node.label === 'Offscreen'), + false, + ); +}); + +function runnerInput(request: IosSnapshotRequest, nodes: RawSnapshotNode[]): IosSnapshotInput { + return { + stage: 'presented', + presentation: { + producer: 'apple-runner', + intent: 'full', + payload: { nodes, truncated: false }, + }, + validation: validationFacts(request), + }; +} + +function validationFacts(request: IosSnapshotRequest): IosSnapshotValidationFacts { + return { + presentationKey: buildIosSnapshotPresentationKey(request), + viewport: { kind: 'reported', rect: viewport }, + hittability: { kind: 'available' }, + lineage: { targetId: 'runner-target', generation: 'runner-generation' }, + residue: [], + }; +} + +function runnerNodes(): RawSnapshotNode[] { + return [ + runnerNode(0, 'Application', 'Settings', viewport), + runnerNode(1, 'Other', undefined, viewport, 0, 1), + runnerNode(2, 'CollectionView', 'Settings', viewport, 1, 2), + runnerNode( + 3, + 'Cell', + 'Screen Time', + { x: 16, y: 796.3333333333334, width: 370, height: 52 }, + 2, + 3, + ), + runnerNode( + 4, + 'Other', + 'Screen Time', + { x: 16, y: 796.3333333333334, width: 370, height: 52 }, + 3, + 4, + ), + runnerNode( + 5, + 'Button', + 'Screen Time', + { x: 16, y: 796.3333333333334, width: 370, height: 52 }, + 4, + 5, + ), + runnerNode( + 6, + 'StaticText', + 'Screen Time', + { x: 30, y: 808.3333, width: 137.3333, height: 28 }, + 5, + 6, + ), + runnerNode(7, 'Image', undefined, { x: 30, y: 808.3333333333334, width: 28, height: 28 }, 5, 6), + runnerNode(8, 'Cell', 'Offscreen', { x: 16, y: 820, width: 370, height: 52 }, 2, 3), + runnerNode(9, 'Button', 'Offscreen', { x: 16, y: 820, width: 370, height: 52 }, 8, 4), + { + ...runnerNode( + 10, + 'Other', + 'Vertical scroll bar, 2 pages', + { + x: 369, + y: 116, + width: 30, + height: 696, + }, + 2, + 3, + ), + value: '0%', + }, + ]; +} + +function runnerNode( + index: number, + type: string, + label: string | undefined, + rect: Rect, + parentIndex?: number, + depth = parentIndex === undefined ? 0 : 1, +): RawSnapshotNode { + return { + index, + type, + ...(label ? { label } : {}), + rect, + enabled: true, + hittable: true, + depth, + ...(parentIndex === undefined ? {} : { parentIndex }), + }; +} diff --git a/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts index 361f7bbcdc..4a48b11284 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/runner-presentation.ts @@ -9,7 +9,7 @@ import { IosSnapshotEngineError } from './types.ts'; export function presentIosRunnerSnapshot( input: Extract, request: IosSnapshotRequest, - foldPolicy: IosSnapshotFoldPolicy, + foldPolicy: IosSnapshotFoldPolicy = 'cursor-projected', ): IosSnapshotEnginePresentation { validateRunnerRequest(input, request); const projection = input.validation.presentationKey.projection; diff --git a/packages/capture-kit/src/ios-snapshot-engine/scroll.ts b/packages/capture-kit/src/ios-snapshot-engine/scroll.ts index 537a2d5942..7278d6f682 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/scroll.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/scroll.ts @@ -1,4 +1,5 @@ -import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import type { RawSnapshotNode, Rect } from '@agent-device/kernel/snapshot'; import { inferVerticalScrollIndicatorDirections, isSystemScrollIndicatorLabel, @@ -6,6 +7,7 @@ import { import { findNearestScrollableContainer, isScrollableSnapshotType, + mergeReplacement, updateReplacement, type SnapshotTreeRuleContext, } from './tree.ts'; @@ -14,13 +16,20 @@ export function collectIosScrollIndicatorPresentation( nodes: RawSnapshotNode[], context: SnapshotTreeRuleContext, ): void { + const derivedScrollContainerIndexes = new Set(); for (const node of nodes) { const presentedNode = context.replacements.get(node.index) ?? node; if (!isIosScrollIndicatorNode(presentedNode)) { continue; } - collectIosScrollIndicatorNodePresentation(node, context.sourceNodesByIndex, context); + collectIosScrollIndicatorNodePresentation( + node, + context.sourceNodesByIndex, + context, + derivedScrollContainerIndexes, + ); } + clipDescendantsToDerivedScrollViewports(nodes, context, derivedScrollContainerIndexes); } function isIosScrollIndicatorNode(node: RawSnapshotNode): boolean { @@ -32,6 +41,7 @@ function collectIosScrollIndicatorNodePresentation( node: RawSnapshotNode, byIndex: ReadonlyMap, context: SnapshotTreeRuleContext, + derivedScrollContainerIndexes: Set, ): void { const suppressed = !isScrollableSnapshotType(node.type) || context.isSuppressed(node); const directions = inferVerticalScrollIndicatorDirections(node.label?.trim() ?? '', node.value); @@ -39,8 +49,143 @@ function collectIosScrollIndicatorNodePresentation( ? findNearestScrollableContainer(node, byIndex, { includeSelf: true }) : undefined; if (suppressed) context.suppressNode(node, container ? [container] : []); - if (container && directions) - applyScrollIndicatorReplacement(context, container, node, directions); + if ( + container && + directions && + applyScrollIndicatorReplacement(context, container, node, directions) + ) { + derivedScrollContainerIndexes.add(container.index); + } +} + +function clipDescendantsToDerivedScrollViewports( + nodes: RawSnapshotNode[], + context: SnapshotTreeRuleContext, + derivedScrollContainerIndexes: ReadonlySet, +): void { + const states = new Map(); + for (const node of nodes) { + const parentState = states.get(node.parentIndex ?? -1); + const ancestorClip = parentState?.clip; + const current = context.replacements.get(node.index) ?? node; + const effectiveRect = intersectRect(current.rect, ancestorClip); + const fullyClipped = isFullyClipped(current, ancestorClip, effectiveRect); + const projectedOut = projectNodeOut(parentState, fullyClipped, current); + applyViewportClip( + context, + node, + current, + ancestorClip, + effectiveRect, + fullyClipped, + projectedOut, + ); + states.set( + node.index, + buildDerivedViewportState( + node, + current, + ancestorClip, + effectiveRect, + projectedOut, + derivedScrollContainerIndexes, + ), + ); + } +} + +type DerivedViewportState = Readonly<{ clip?: Rect; projectedOut: boolean }>; + +function isFullyClipped( + node: RawSnapshotNode, + ancestorClip: Rect | undefined, + effectiveRect: Rect | undefined, +): boolean { + return Boolean( + ancestorClip && isPositiveFiniteRect(node.rect) && !isPositiveFiniteRect(effectiveRect), + ); +} + +function projectNodeOut( + parentState: DerivedViewportState | undefined, + fullyClipped: boolean, + node: RawSnapshotNode, +): boolean { + return Boolean(parentState?.projectedOut || (fullyClipped && ownsDescendants(node))); +} + +function applyViewportClip( + context: SnapshotTreeRuleContext, + node: RawSnapshotNode, + current: RawSnapshotNode, + ancestorClip: Rect | undefined, + effectiveRect: Rect | undefined, + fullyClipped: boolean, + projectedOut: boolean, +): void { + if (projectedOut || fullyClipped) { + context.suppressNode(node, []); + } else if (ancestorClip && effectiveRect && !rectsEqual(current.rect, effectiveRect)) { + mergeReplacement(context.replacements, node, { rect: effectiveRect }); + } +} + +function buildDerivedViewportState( + node: RawSnapshotNode, + current: RawSnapshotNode, + ancestorClip: Rect | undefined, + effectiveRect: Rect | undefined, + projectedOut: boolean, + derivedScrollContainerIndexes: ReadonlySet, +): DerivedViewportState { + const establishesClip = canEstablishDerivedClip( + node, + current, + ancestorClip, + effectiveRect, + derivedScrollContainerIndexes, + ); + return { + projectedOut, + ...(establishesClip ? { clip: effectiveRect } : ancestorClip ? { clip: ancestorClip } : {}), + }; +} + +function canEstablishDerivedClip( + node: RawSnapshotNode, + current: RawSnapshotNode, + ancestorClip: Rect | undefined, + effectiveRect: Rect | undefined, + derivedScrollContainerIndexes: ReadonlySet, +): boolean { + return Boolean( + isPositiveFiniteRect(effectiveRect) && + (derivedScrollContainerIndexes.has(node.index) || + (ancestorClip !== undefined && isScrollableSnapshotType(current.type))), + ); +} + +function ownsDescendants(node: RawSnapshotNode): boolean { + return node.type?.trim().toLowerCase() === 'cell' || isScrollableSnapshotType(node.type); +} + +function intersectRect(rect: RawSnapshotNode['rect'], clip: Rect | undefined): Rect | undefined { + if (!rect || !clip) return rect; + const x = Math.max(rect.x, clip.x); + const y = Math.max(rect.y, clip.y); + const right = Math.min(rect.x + rect.width, clip.x + clip.width); + const bottom = Math.min(rect.y + rect.height, clip.y + clip.height); + return { x, y, width: Math.max(0, right - x), height: Math.max(0, bottom - y) }; +} + +function rectsEqual(left: RawSnapshotNode['rect'], right: Rect): boolean { + return Boolean( + left && + left.x === right.x && + left.y === right.y && + left.width === right.width && + left.height === right.height, + ); } function applyScrollIndicatorReplacement( @@ -48,12 +193,17 @@ function applyScrollIndicatorReplacement( container: RawSnapshotNode, indicator: RawSnapshotNode, directions: { above: boolean; below: boolean }, -): void { +): boolean { + const derivedRect = deriveScrollableViewportRect( + (context.replacements.get(container.index) ?? container).rect, + indicator.rect, + ); updateReplacement(context.replacements, container, (current) => ({ - rect: deriveScrollableViewportRect(current.rect, indicator.rect) ?? current.rect, + rect: derivedRect ?? current.rect, hiddenContentAbove: mergeHiddenContentFlag(current.hiddenContentAbove, directions.above), hiddenContentBelow: mergeHiddenContentFlag(current.hiddenContentBelow, directions.below), })); + return Boolean(derivedRect); } function mergeHiddenContentFlag( diff --git a/src/snapshot/snapshot-presentation/tree.test.ts b/packages/capture-kit/src/ios-snapshot-engine/tree.test.ts similarity index 100% rename from src/snapshot/snapshot-presentation/tree.test.ts rename to packages/capture-kit/src/ios-snapshot-engine/tree.test.ts diff --git a/packages/capture-kit/src/ios-snapshot-engine/types.ts b/packages/capture-kit/src/ios-snapshot-engine/types.ts index 88b4970788..7468d0b9f4 100644 --- a/packages/capture-kit/src/ios-snapshot-engine/types.ts +++ b/packages/capture-kit/src/ios-snapshot-engine/types.ts @@ -12,6 +12,7 @@ export type IosSnapshotFoldOptions = Readonly<{ export type IosSnapshotPresentationNode = Readonly<{ raw: RawSnapshotNode; + sourceIndex: number; effectiveRect?: Rect; }>; diff --git a/src/snapshot/snapshot-presentation/ios/web.test.ts b/packages/capture-kit/src/ios-snapshot-engine/web.test.ts similarity index 100% rename from src/snapshot/snapshot-presentation/ios/web.test.ts rename to packages/capture-kit/src/ios-snapshot-engine/web.test.ts diff --git a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts index 7bde3797a8..d4a2429247 100644 --- a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts @@ -3,7 +3,7 @@ import type { Interactor, RunnerContext } from '@agent-device/contracts/interact import { AppError } from '@agent-device/kernel/errors'; import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { IOS_SIMULATOR } from './device-fixtures.ts'; +import { IOS_SIMULATOR, MACOS_DEVICE } from './device-fixtures.ts'; import type { AppleRunnerCommandOptions, AppleRunnerProvider, @@ -180,6 +180,168 @@ test('snapshot over the injected transport keeps the shared xctest result shape' assert.equal(result.nodes?.length, 2); }); +test('snapshot publishes runner presentation through the engine and drops its quality view', async () => { + const interactor = createAppleInteractor( + IOS_SIMULATOR, + {}, + { + runCommand: async () => ({ + nodes: [ + { + index: 0, + depth: 0, + type: 'Application', + label: 'App', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Table', + label: 'Settings', + rect: { x: 0, y: 40, width: 390, height: 804 }, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Cell', + label: 'General', + rect: { x: 16, y: 80, width: 358, height: 52 }, + }, + { + index: 3, + depth: 3, + parentIndex: 2, + type: 'Button', + label: 'General', + rect: { x: 16, y: 80, width: 358, height: 52 }, + hittable: true, + }, + { + index: 4, + depth: 4, + parentIndex: 3, + type: 'StaticText', + label: 'General', + rect: { x: 16, y: 80, width: 358, height: 52 }, + }, + ], + truncated: false, + snapshotQuality: { state: 'healthy', backend: 'tree' }, + qualityPayload: { + nodes: [ + { + index: 0, + type: 'Application', + label: 'App', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + ], + truncated: false, + scope: null, + }, + }), + }, + ); + + const result = await interactor.snapshot({ interactiveOnly: true }); + + assert.deepEqual( + result.nodes?.map((node) => node.type), + ['Application', 'Table', 'Cell'], + ); + assert.equal('qualityPayload' in result, false); +}); + +test('macOS app snapshots preserve runner nodes outside the iOS presentation engine', async () => { + const nodes = [{ index: 0, type: 'Application', label: 'System Settings' }]; + const interactor = createAppleInteractor( + MACOS_DEVICE, + {}, + { runCommand: async () => ({ nodes }) }, + ); + + const result = await interactor.snapshot({ interactiveOnly: true }); + + assert.deepEqual(result.nodes, nodes); +}); + +test('snapshot reports typed runner presentation failures', async () => { + const interactor = createAppleInteractor( + IOS_SIMULATOR, + {}, + { runCommand: async () => ({ nodes: [{ index: 0, type: 'Application' }] }) }, + ); + + await assert.rejects( + interactor.snapshot(), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + error.details?.reason === 'missing-viewport', + ); +}); + +test('sparse runner payloads with no viewport fail before publishing actionable nodes', async () => { + const interactor = createAppleInteractor( + IOS_SIMULATOR, + {}, + { + runCommand: async () => ({ + nodes: [ + { index: 0, type: 'Application', label: 'App' }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Escaped action', + rect: { x: 10, y: 10, width: 80, height: 40 }, + hittable: true, + }, + ], + truncated: true, + snapshotQuality: { + state: 'sparse', + backend: 'tree', + reason: 'no usable snapshot backend', + reasonCode: 'sparse-tree', + }, + }), + }, + ); + + await assert.rejects( + interactor.snapshot(), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + error.details?.reason === 'missing-viewport', + ); +}); + +test('snapshot rejects a scoped quality payload at the runner boundary', async () => { + const interactor = createAppleInteractor( + IOS_SIMULATOR, + {}, + { + runCommand: async () => ({ + nodes: [{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }], + qualityPayload: { nodes: [], truncated: false, scope: 'Settings' }, + }), + }, + ); + + await assert.rejects( + interactor.snapshot(), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + error.details?.reason === 'invalid-quality-payload', + ); +}); + test('snapshot accepts only structured healthy empty scope results', async () => { const healthyEmptyProvider: AppleRunnerProvider = { runCommand: async () => ({ diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index f860de748d..1806e382fa 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -21,7 +21,7 @@ import { withDiagnosticTimer } from '@agent-device/host-kit/diagnostics'; import { isMacOs, isTvOsDevice, type DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; import { withMethodScope } from '@agent-device/kernel/scoped-provider'; -import type { Point, RawSnapshotNode, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; +import type { Point, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; import type { Interactor, RunnerCallOptions, @@ -29,8 +29,12 @@ import type { ScreenshotOptions, SnapshotOptions, } from '@agent-device/contracts/interactor-types'; -import { readSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import { captureMacOsSurfaceSnapshot } from './os/macos/surface-snapshot.ts'; +import { + presentAppleRunnerSnapshot, + readAppleSnapshotResult, +} from './runner/snapshot-presentation.ts'; +import type { AppleRunnerSnapshotResult } from './runner/snapshot-presentation.ts'; export function createAppleInteractor( device: DeviceInfo, @@ -242,7 +246,7 @@ async function captureAppleRunnerSnapshot( throw new AppError('COMMAND_FAILED', 'XCTest snapshot returned 0 nodes on iOS simulator.'); } return { - nodes, + nodes: presentRunnerSnapshotForDevice(device, options, result), truncated: result.truncated ?? false, backend: 'xctest' as const, producer: 'apple-runner' as const, @@ -252,6 +256,15 @@ async function captureAppleRunnerSnapshot( }; } +function presentRunnerSnapshotForDevice( + device: DeviceInfo, + options: SnapshotOptions | undefined, + result: AppleRunnerSnapshotResult, +) { + if (isMacOs(device)) return result.nodes ?? []; + return presentAppleRunnerSnapshot(device.id, options, result); +} + function acceptsEmptyScopedSnapshot( options: SnapshotOptions | undefined, quality: SnapshotQualityVerdict | undefined, @@ -377,24 +390,6 @@ function usesMacOsSurfaceScreenshot( return isMacOs(device) && surface !== undefined && surface !== 'app'; } -function readAppleSnapshotResult(result: Record): { - nodes?: RawSnapshotNode[]; - truncated?: boolean; - message?: string; - quality?: SnapshotQualityVerdict; -} { - return { - nodes: Array.isArray(result.nodes) ? (result.nodes as RawSnapshotNode[]) : undefined, - truncated: typeof result.truncated === 'boolean' ? result.truncated : undefined, - quality: readSnapshotQualityVerdict(result.snapshotQuality), - // Legacy runner context for builds that predate the structured verdict. - message: - typeof result.message === 'string' && result.message.trim().length > 0 - ? result.message - : undefined, - }; -} - /** Only non-app macOS surfaces are helper-read; an app session is runner-read like any leaf. */ function usesMacOsHelperSurface(device: DeviceInfo, surface: SessionSurface | undefined): boolean { return isMacOs(device) && surface !== undefined && surface !== 'app'; diff --git a/packages/platform-apple/src/runner/__tests__/runner-cache-metadata.test.ts b/packages/platform-apple/src/runner/__tests__/runner-cache-metadata.test.ts index 1ef6da9d39..c34536a306 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-cache-metadata.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-cache-metadata.test.ts @@ -1,4 +1,6 @@ -import { test } from 'vitest'; +import fs from 'node:fs'; +import path from 'node:path'; +import { onTestFinished, test } from 'vitest'; import assert from 'node:assert/strict'; import { IOS_DEVICE, IOS_SIMULATOR, MACOS_DEVICE } from './device-fixtures.ts'; import { @@ -7,7 +9,9 @@ import { resolveRunnerSigningBuildSettings, resolveRunnerPerformanceBuildSettings, resolveRunnerSandboxBuildArgs, + resolveExpectedRunnerCacheMetadata, } from '../runner-cache-metadata.ts'; +import { mkdtempForTestSync } from './tmp-dir.ts'; test('resolveRunnerMaxConcurrentDestinationsFlag uses simulator flag for simulators', () => { assert.equal( @@ -148,3 +152,84 @@ test('resolveRunnerBundleBuildSettings uses AGENT_DEVICE_IOS_BUNDLE_ID when prov ], ); }); + +test('runner cache metadata fingerprints shared snapshot presentation sources', () => { + const root = mkdtempForTestSync('agent-device-runner-cache-fingerprint-'); + onTestFinished(() => fs.rmSync(root, { recursive: true, force: true })); + fs.writeFileSync(path.join(root, 'package.json'), JSON.stringify({ version: '0.0.0' })); + fs.mkdirSync(path.join(root, 'apple', 'runner', 'AgentDeviceRunner'), { recursive: true }); + fs.mkdirSync(path.join(root, 'apple', 'snapshot-presentation', 'Sources'), { recursive: true }); + fs.writeFileSync( + path.join(root, 'apple', 'runner', 'AgentDeviceRunner', 'Runner.swift'), + 'runner\n', + ); + const sharedSource = path.join( + root, + 'apple', + 'snapshot-presentation', + 'Sources', + 'Presentation.swift', + ); + fs.writeFileSync(sharedSource, 'shared-one\n'); + + const before = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR, root).runnerSourceFingerprint; + fs.writeFileSync(sharedSource, 'shared-two\n'); + const after = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR, root).runnerSourceFingerprint; + + assert.notEqual(after, before); +}); + +test('runner cache metadata ignores development-only SwiftPM trees but keeps runner unit tests', () => { + const root = mkdtempForTestSync('agent-device-runner-cache-source-roots-'); + onTestFinished(() => fs.rmSync(root, { recursive: true, force: true })); + fs.writeFileSync(path.join(root, 'package.json'), JSON.stringify({ version: '0.0.0' })); + + const runnerRoot = path.join(root, 'apple', 'runner', 'AgentDeviceRunner'); + const runnerUnitTest = path.join( + runnerRoot, + 'AgentDeviceRunnerUITests', + 'UnitTests', + 'Invariant.swift', + ); + const sharedRoot = path.join(root, 'apple', 'snapshot-presentation'); + fs.mkdirSync(path.dirname(runnerUnitTest), { recursive: true }); + fs.mkdirSync(path.join(sharedRoot, 'Sources'), { recursive: true }); + fs.writeFileSync(path.join(runnerRoot, 'Runner.swift'), 'runner\n'); + fs.writeFileSync(runnerUnitTest, 'unit-one\n'); + fs.writeFileSync(path.join(sharedRoot, 'Sources', 'Presentation.swift'), 'shared\n'); + + for (const directory of [ + 'Tests', + 'SnapshotPresentationConformance', + '.build', + '.swiftpm', + 'xcuserdata', + ]) { + const file = path.join(sharedRoot, directory, 'Ignored.swift'); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, 'ignored-one\n'); + } + + const before = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR, root).runnerSourceFingerprint; + for (const directory of [ + 'Tests', + 'SnapshotPresentationConformance', + '.build', + '.swiftpm', + 'xcuserdata', + ]) { + fs.writeFileSync(path.join(sharedRoot, directory, 'Ignored.swift'), 'ignored-two\n'); + } + const afterIgnoredChanges = resolveExpectedRunnerCacheMetadata( + IOS_SIMULATOR, + root, + ).runnerSourceFingerprint; + assert.equal(afterIgnoredChanges, before); + + fs.writeFileSync(runnerUnitTest, 'unit-two\n'); + const afterRunnerTestChange = resolveExpectedRunnerCacheMetadata( + IOS_SIMULATOR, + root, + ).runnerSourceFingerprint; + assert.notEqual(afterRunnerTestChange, afterIgnoredChanges); +}); diff --git a/packages/platform-apple/src/runner/__tests__/runner-source.test.ts b/packages/platform-apple/src/runner/__tests__/runner-source.test.ts index 57393daa31..94baf98c37 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-source.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-source.test.ts @@ -2,7 +2,11 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { onTestFinished, test } from 'vitest'; -import { resolveAppleRunnerProjectPath, resolveAppleRunnerSourceRoot } from '../runner-source.ts'; +import { + resolveAppleRunnerProjectPath, + resolveAppleRunnerSourceRoot, + resolveAppleSnapshotPresentationSourceRoot, +} from '../runner-source.ts'; import { mkdtempForTestSync } from './tmp-dir.ts'; test('resolveAppleRunnerSourceRoot prefers checkout source over packaged source', () => { @@ -31,6 +35,24 @@ test('resolveAppleRunnerSourceRoot falls back to packaged source', () => { ); }); +test('resolveAppleSnapshotPresentationSourceRoot prefers checkout source over packaged source', () => { + const root = makeTempRoot(); + const checkoutSource = path.join(root, 'apple', 'snapshot-presentation'); + const packagedSource = path.join(root, 'dist', 'apple', 'snapshot-presentation'); + fs.mkdirSync(checkoutSource, { recursive: true }); + fs.mkdirSync(packagedSource, { recursive: true }); + + assert.equal(resolveAppleSnapshotPresentationSourceRoot(root), checkoutSource); +}); + +test('resolveAppleSnapshotPresentationSourceRoot falls back to packaged source', () => { + const root = makeTempRoot(); + const packagedSource = path.join(root, 'dist', 'apple', 'snapshot-presentation'); + fs.mkdirSync(packagedSource, { recursive: true }); + + assert.equal(resolveAppleSnapshotPresentationSourceRoot(root), packagedSource); +}); + function makeTempRoot(): string { const root = mkdtempForTestSync('agent-device-runner-source-'); onTestFinished(() => fs.rmSync(root, { recursive: true, force: true })); diff --git a/packages/platform-apple/src/runner/__tests__/runner-xctestrun.test.ts b/packages/platform-apple/src/runner/__tests__/runner-xctestrun.test.ts index 4a8ed79b51..1d5b5d67f2 100644 --- a/packages/platform-apple/src/runner/__tests__/runner-xctestrun.test.ts +++ b/packages/platform-apple/src/runner/__tests__/runner-xctestrun.test.ts @@ -256,10 +256,33 @@ test('setup metadata script matches expected iOS simulator cache metadata', asyn path.join(projectRoot, 'apple', 'runner', 'AgentDeviceRunner', 'Runner.swift'), 'final class Runner {}\n', ); + const runnerUnitTest = path.join( + projectRoot, + 'apple', + 'runner', + 'AgentDeviceRunner', + 'AgentDeviceRunnerUITests', + 'UnitTests', + 'Invariant.swift', + ); + fs.mkdirSync(path.dirname(runnerUnitTest), { recursive: true }); + fs.writeFileSync(runnerUnitTest, 'unit-one\n'); + const ignoredSharedSource = path.join( + projectRoot, + 'apple', + 'snapshot-presentation', + 'Tests', + 'Ignored.swift', + ); + fs.mkdirSync(path.dirname(ignoredSharedSource), { recursive: true }); + fs.writeFileSync(ignoredSharedSource, 'ignored-one\n'); const { writeXcuitestCacheMetadata } = await import( `${pathToFileURL(scriptPath).href}?case=${Date.now()}` ); - writeXcuitestCacheMetadata(['ios', derivedRoot, 'generic/platform=iOS Simulator'], projectRoot); + const firstMetadata = writeXcuitestCacheMetadata( + ['ios', derivedRoot, 'generic/platform=iOS Simulator'], + projectRoot, + ); const actual = JSON.parse( fs.readFileSync(path.join(derivedRoot, '.agent-device-runner-cache.json'), 'utf8'), @@ -269,6 +292,20 @@ test('setup metadata script matches expected iOS simulator cache metadata', asyn resolveExpectedRunnerCacheMetadata(iosSimulator, projectRoot); assert.deepEqual(actualComparable, expectedComparable); + + fs.writeFileSync(ignoredSharedSource, 'ignored-two\n'); + const secondMetadata = writeXcuitestCacheMetadata( + ['ios', derivedRoot, 'generic/platform=iOS Simulator'], + projectRoot, + ); + assert.equal(secondMetadata.runnerSourceFingerprint, firstMetadata.runnerSourceFingerprint); + + fs.writeFileSync(runnerUnitTest, 'unit-two\n'); + const thirdMetadata = writeXcuitestCacheMetadata( + ['ios', derivedRoot, 'generic/platform=iOS Simulator'], + projectRoot, + ); + assert.notEqual(thirdMetadata.runnerSourceFingerprint, secondMetadata.runnerSourceFingerprint); }); }, 15_000); diff --git a/packages/platform-apple/src/runner/runner-cache-metadata.ts b/packages/platform-apple/src/runner/runner-cache-metadata.ts index 01a8961b53..9c39fa50ea 100644 --- a/packages/platform-apple/src/runner/runner-cache-metadata.ts +++ b/packages/platform-apple/src/runner/runner-cache-metadata.ts @@ -17,12 +17,23 @@ import { resolveRunnerPlatformName, resolveRunnerSdkName, } from './apple-runner-platform.ts'; -import { resolveAppleRunnerSourceRoot } from './runner-source.ts'; +import { + resolveAppleRunnerSourceRoot, + resolveAppleSnapshotPresentationSourceRoot, +} from './runner-source.ts'; const DEFAULT_IOS_RUNNER_APP_BUNDLE_ID = 'com.callstack.agentdevice.runner'; const RUNNER_DERIVED_ROOT = path.join(os.homedir(), '.agent-device', 'apple-runner'); export const RUNNER_CACHE_METADATA_FILE = '.agent-device-runner-cache.json'; const RUNNER_CACHE_SCHEMA_VERSION = 2; +const RUNNER_SOURCE_IGNORED_DIR_NAMES = new Set(['.build', '.swiftpm', 'xcuserdata']); +const SNAPSHOT_PRESENTATION_SOURCE_IGNORED_DIR_NAMES = new Set([ + '.build', + '.swiftpm', + 'SnapshotPresentationConformance', + 'Tests', + 'xcuserdata', +]); const RUNNER_SANDBOX_BUILD_ARGS = [ '-IDEPackageSupportDisableManifestSandbox=1', '-IDEPackageSupportDisablePluginExecutionSandbox=1', @@ -243,33 +254,43 @@ type RunnerSourceFingerprintCacheEntry = { const runnerSourceFingerprintCache = new Map(); function computeRunnerSourceFingerprint(projectRoot: string): string { - const runnerRoot = resolveAppleRunnerSourceRoot(projectRoot); - const files = collectRunnerSourceFiles(runnerRoot); - const fileStatsFingerprint = computeRunnerSourceFileStatsFingerprint(runnerRoot, files); - const cached = runnerSourceFingerprintCache.get(runnerRoot); + const sourceRoots = [ + { + path: resolveAppleRunnerSourceRoot(projectRoot), + ignoredDirectoryNames: RUNNER_SOURCE_IGNORED_DIR_NAMES, + }, + { + path: resolveAppleSnapshotPresentationSourceRoot(projectRoot), + ignoredDirectoryNames: SNAPSHOT_PRESENTATION_SOURCE_IGNORED_DIR_NAMES, + }, + ]; + const files = collectRunnerSourceFiles(sourceRoots); + const fileStatsFingerprint = computeRunnerSourceFileStatsFingerprint(projectRoot, files); + const cacheKey = JSON.stringify(sourceRoots.map(({ path: sourcePath }) => sourcePath)); + const cached = runnerSourceFingerprintCache.get(cacheKey); if (cached?.fileStatsFingerprint === fileStatsFingerprint) { return cached.sourceFingerprint; } const hash = crypto.createHash('sha256'); for (const file of files) { - const relativePath = path.relative(runnerRoot, file); + const relativePath = path.relative(projectRoot, file); hash.update(relativePath); hash.update('\0'); hash.update(fs.readFileSync(file)); hash.update('\0'); } const sourceFingerprint = hash.digest('hex'); - runnerSourceFingerprintCache.set(runnerRoot, { fileStatsFingerprint, sourceFingerprint }); + runnerSourceFingerprintCache.set(cacheKey, { fileStatsFingerprint, sourceFingerprint }); return sourceFingerprint; } function computeRunnerSourceFileStatsFingerprint( - runnerRoot: string, + projectRoot: string, files: readonly string[], ): string { const hash = crypto.createHash('sha256'); for (const file of files) { - const relativePath = path.relative(runnerRoot, file); + const relativePath = path.relative(projectRoot, file); const stat = fs.statSync(file); hash.update(relativePath); hash.update('\0'); @@ -281,27 +302,46 @@ function computeRunnerSourceFileStatsFingerprint( return hash.digest('hex'); } -function collectRunnerSourceFiles(root: string): string[] { - if (!fs.existsSync(root)) { - return []; - } +type RunnerSourceRoot = Readonly<{ + path: string; + ignoredDirectoryNames: ReadonlySet; +}>; + +function collectRunnerSourceFiles(roots: readonly RunnerSourceRoot[]): string[] { + return [ + ...new Set( + roots.flatMap(({ path: sourcePath, ignoredDirectoryNames }) => + collectRunnerSourceFilesUnderRoot(sourcePath, ignoredDirectoryNames), + ), + ), + ].sort((a, b) => a.localeCompare(b)); +} + +function collectRunnerSourceFilesUnderRoot( + root: string, + ignoredDirectoryNames: ReadonlySet, +): string[] { + return fs.existsSync(root) + ? collectRunnerSourceFilesInDirectory(root, ignoredDirectoryNames) + : []; +} + +function collectRunnerSourceFilesInDirectory( + directory: string, + ignoredDirectoryNames: ReadonlySet, +): string[] { const files: string[] = []; - const stack = [root]; - while (stack.length > 0) { - const current = stack.pop() as string; - for (const entry of fs.readdirSync(current, { withFileTypes: true })) { - const fullPath = path.join(current, entry.name); - if (entry.isDirectory()) { - if (entry.name === 'xcuserdata') continue; - stack.push(fullPath); - continue; - } - if (entry.isFile() && isRunnerSourceFile(entry.name, fullPath)) { - files.push(fullPath); + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const fullPath = path.join(directory, entry.name); + if (entry.isDirectory()) { + if (!ignoredDirectoryNames.has(entry.name)) { + files.push(...collectRunnerSourceFilesInDirectory(fullPath, ignoredDirectoryNames)); } + } else if (entry.isFile() && isRunnerSourceFile(entry.name, fullPath)) { + files.push(fullPath); } } - return files.sort((a, b) => a.localeCompare(b)); + return files; } function isRunnerSourceFile(fileName: string, filePath: string): boolean { diff --git a/packages/platform-apple/src/runner/runner-source.ts b/packages/platform-apple/src/runner/runner-source.ts index 567f32390c..a0e4406beb 100644 --- a/packages/platform-apple/src/runner/runner-source.ts +++ b/packages/platform-apple/src/runner/runner-source.ts @@ -3,6 +3,12 @@ import path from 'node:path'; const APPLE_RUNNER_SOURCE_ROOT = path.join('apple', 'runner', 'AgentDeviceRunner'); const PACKAGED_APPLE_RUNNER_SOURCE_ROOT = path.join('dist', 'apple', 'runner', 'AgentDeviceRunner'); +const APPLE_SNAPSHOT_PRESENTATION_SOURCE_ROOT = path.join('apple', 'snapshot-presentation'); +const PACKAGED_APPLE_SNAPSHOT_PRESENTATION_SOURCE_ROOT = path.join( + 'dist', + 'apple', + 'snapshot-presentation', +); export function resolveAppleRunnerSourceRoot(projectRoot: string): string { const checkoutSourceRoot = path.join(projectRoot, APPLE_RUNNER_SOURCE_ROOT); @@ -15,3 +21,11 @@ export function resolveAppleRunnerSourceRoot(projectRoot: string): string { export function resolveAppleRunnerProjectPath(projectRoot: string): string { return path.join(resolveAppleRunnerSourceRoot(projectRoot), 'AgentDeviceRunner.xcodeproj'); } + +export function resolveAppleSnapshotPresentationSourceRoot(projectRoot: string): string { + const checkoutSourceRoot = path.join(projectRoot, APPLE_SNAPSHOT_PRESENTATION_SOURCE_ROOT); + if (fs.existsSync(checkoutSourceRoot)) { + return checkoutSourceRoot; + } + return path.join(projectRoot, PACKAGED_APPLE_SNAPSHOT_PRESENTATION_SOURCE_ROOT); +} diff --git a/packages/platform-apple/src/runner/snapshot-presentation.ts b/packages/platform-apple/src/runner/snapshot-presentation.ts new file mode 100644 index 0000000000..87473d2a35 --- /dev/null +++ b/packages/platform-apple/src/runner/snapshot-presentation.ts @@ -0,0 +1,166 @@ +import type { + IosRunnerQualityPayloadFacts, + IosSnapshotInput, + IosViewportEvidence, +} from '@agent-device/contracts/ios-snapshot'; +import type { SnapshotOptions } from '@agent-device/contracts/interactor-types'; +import { normalizeType } from '@agent-device/contracts/snapshot'; +import { + IosSnapshotEngineError, + presentIosRunnerSnapshot, +} from '@agent-device/capture-kit/ios-snapshot-engine'; +import { readSnapshotQualityVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; +import { + createIosSnapshotRequest, + buildIosSnapshotPresentationKey, +} from '@agent-device/capture-kit/ios-snapshot-planning'; +import { isPositiveFiniteRect } from '@agent-device/kernel/rect'; +import { AppError } from '@agent-device/kernel/errors'; +import type { RawSnapshotNode, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; + +export type AppleRunnerSnapshotResult = Readonly<{ + nodes?: RawSnapshotNode[]; + truncated?: boolean; + message?: string; + quality?: SnapshotQualityVerdict; + qualityPayload?: IosRunnerQualityPayloadFacts; + runnerFatal?: boolean; +}>; + +export function readAppleSnapshotResult( + result: Record, +): AppleRunnerSnapshotResult { + return { + nodes: Array.isArray(result.nodes) ? (result.nodes as RawSnapshotNode[]) : undefined, + truncated: typeof result.truncated === 'boolean' ? result.truncated : undefined, + quality: readSnapshotQualityVerdict(result.snapshotQuality), + qualityPayload: readQualityPayload(result.qualityPayload), + runnerFatal: result.runnerFatal === true, + message: + typeof result.message === 'string' && result.message.trim().length > 0 + ? result.message + : undefined, + }; +} + +export function presentAppleRunnerSnapshot( + deviceId: string, + options: SnapshotOptions | undefined, + result: AppleRunnerSnapshotResult, +): RawSnapshotNode[] { + const nodes = result.nodes ?? []; + if (result.runnerFatal === true || (nodes.length === 0 && result.qualityPayload === undefined)) { + return nodes; + } + + const request = createIosSnapshotRequest({ + raw: options?.raw, + interactiveOnly: options?.interactiveOnly, + depth: options?.depth, + scope: options?.scope, + customActions: options?.customActions, + }); + const viewport = runnerViewportEvidence(nodes, result.qualityPayload?.nodes); + + const input: IosSnapshotInput = { + stage: 'presented', + presentation: { + producer: 'apple-runner', + intent: 'full', + payload: { + nodes, + truncated: result.truncated ?? false, + ...(result.quality?.effectiveDepth !== undefined + ? { effectiveDepth: result.quality.effectiveDepth } + : {}), + }, + ...(result.qualityPayload ? { qualityPayload: result.qualityPayload } : {}), + }, + validation: { + presentationKey: buildIosSnapshotPresentationKey(request), + viewport, + hittability: { kind: 'available' }, + lineage: { targetId: deviceId }, + residue: [], + }, + }; + + try { + return presentIosRunnerSnapshot(input, request).nodes; + } catch (error) { + throwSnapshotEngineError(error); + } +} + +function readQualityPayload(value: unknown): IosRunnerQualityPayloadFacts | undefined { + if (value === undefined) return undefined; + if (!isRecord(value) || !Array.isArray(value.nodes) || typeof value.truncated !== 'boolean') { + throwSnapshotEngineError( + new IosSnapshotEngineError( + 'invalid-quality-payload', + 'iOS runner returned an invalid quality payload', + ), + ); + } + if (value.scope !== undefined && value.scope !== null) { + throwSnapshotEngineError( + new IosSnapshotEngineError( + 'invalid-quality-payload', + 'iOS runner quality payload must be unscoped', + { field: 'scope' }, + ), + ); + } + return { nodes: value.nodes as RawSnapshotNode[], truncated: value.truncated, scope: null }; +} + +function runnerViewportEvidence( + nodes: readonly RawSnapshotNode[], + qualityNodes: readonly RawSnapshotNode[] | undefined, +): IosViewportEvidence { + return ( + readReportedViewport(qualityNodes) ?? + readReportedViewport(nodes) ?? { kind: 'missing', reason: 'not-provided' } + ); +} + +function readReportedViewport( + nodes: readonly RawSnapshotNode[] | undefined, +): IosViewportEvidence | undefined { + const roots = nodes?.filter((node) => node.parentIndex === undefined) ?? []; + const root = + [...roots] + .filter((node) => isViewportRoot(node)) + .sort(compareRectArea) + .at(0) ?? [...roots].sort(compareRectArea).at(0); + if (!root) return undefined; + if (isPositiveFiniteRect(root.rect)) return { kind: 'reported', rect: root.rect }; + return { kind: 'missing', reason: root.rect ? 'invalid' : 'not-provided' }; +} + +function isViewportRoot(node: RawSnapshotNode): boolean { + const type = normalizeType(node.type ?? ''); + return type === 'application' || type === 'window'; +} + +function compareRectArea(left: RawSnapshotNode, right: RawSnapshotNode): number { + return rectArea(right.rect) - rectArea(left.rect); +} + +function rectArea(rect: RawSnapshotNode['rect']): number { + return rect ? rect.width * rect.height : 0; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function throwSnapshotEngineError(error: unknown): never { + if (!(error instanceof IosSnapshotEngineError)) throw error; + throw new AppError( + 'COMMAND_FAILED', + error.message, + { reason: error.reason, iosSnapshotEngine: { details: error.details } }, + error, + ); +} diff --git a/scripts/__tests__/fixtures/size-report-npm-pack.json b/scripts/__tests__/fixtures/size-report-npm-pack.json index b365a32c1e..1a94d8903c 100644 --- a/scripts/__tests__/fixtures/size-report-npm-pack.json +++ b/scripts/__tests__/fixtures/size-report-npm-pack.json @@ -1,9 +1,10 @@ { - "unpackedSize": 1701, + "unpackedSize": 1814, "files": [ { "path": "dist/src/index.js", "size": 401 }, { "path": "dist/src/index.d.ts", "size": 102 }, { "path": "dist/apple/runner/RunnerTests.swift", "size": 503 }, + { "path": "dist/apple/snapshot-presentation/Package.swift", "size": 113 }, { "path": "apple/macos-helper/Sources/main.swift", "size": 211 }, { "path": "android/snapshot-helper/dist/helper.apk", "size": 307 }, { "path": "package.json", "size": 99 }, diff --git a/scripts/__tests__/size-report-package.test.ts b/scripts/__tests__/size-report-package.test.ts index b437aabeed..e149254f51 100644 --- a/scripts/__tests__/size-report-package.test.ts +++ b/scripts/__tests__/size-report-package.test.ts @@ -18,6 +18,7 @@ test('classifies every shipped entry into one named component', () => { ['dist/src/index.js', 'js'], ['dist/src/index.d.ts', 'js'], ['dist/apple/runner/RunnerTests.swift', 'apple-runner'], + ['dist/apple/snapshot-presentation/Package.swift', 'apple-snapshot-presentation'], ['apple/macos-helper/Sources/main.swift', 'macos-helper'], ['android/snapshot-helper/dist/helper.apk', 'android-helpers'], ['package.json', 'other'], @@ -45,13 +46,14 @@ test('component bytes sum exactly to npm pack unpackedSize', () => { { js: 503, 'apple-runner': 503, + 'apple-snapshot-presentation': 113, 'macos-helper': 211, 'android-helpers': 307, other: 177, }, ); assert.throws( - () => summarizeNpmPackComponents({ ...fixturePack, unpackedSize: 1700 }), + () => summarizeNpmPackComponents({ ...fixturePack, unpackedSize: 1800 }), /does not match npm pack unpackedSize/, ); }); @@ -61,7 +63,7 @@ test('Markdown reports component diffs and changed packed files', () => { js: { rawBytes: 10, gzipBytes: 8 }, npmPack: { tarballBytes: 100, - unpackedBytes: 1701, + unpackedBytes: 1814, components: summarizeNpmPackComponents(fixturePack), entries: fixturePack.files, }, @@ -74,10 +76,10 @@ test('Markdown reports component diffs and changed packed files', () => { js: { rawBytes: 10, gzipBytes: 8 }, npmPack: { tarballBytes: 100, - unpackedBytes: 1600, + unpackedBytes: 1713, components: summarizeNpmPackComponents({ ...fixturePack, - unpackedSize: 1600, + unpackedSize: 1713, files: baseEntries, }), entries: baseEntries, diff --git a/scripts/__tests__/xctest-selection.test.ts b/scripts/__tests__/xctest-selection.test.ts index 2da1ff810a..177b5726e0 100644 --- a/scripts/__tests__/xctest-selection.test.ts +++ b/scripts/__tests__/xctest-selection.test.ts @@ -142,6 +142,9 @@ describe('the real tree', () => { test('the package-source boundary rejects an unguarded runner unit test', () => { const root = mkdtempForTestSync('agent-device-runner-package-selection-'); onTestFinished(() => fs.rmSync(root, { recursive: true, force: true })); + const packageManifestPath = path.join(root, 'apple/snapshot-presentation/Package.runner.swift'); + fs.mkdirSync(path.dirname(packageManifestPath), { recursive: true }); + fs.writeFileSync(packageManifestPath, '// fixture package manifest\n'); const sourcePath = path.join( root, 'apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Fixture.swift', diff --git a/scripts/package-apple-runner-source.mjs b/scripts/package-apple-runner-source.mjs index 9591098331..5df0c73188 100644 --- a/scripts/package-apple-runner-source.mjs +++ b/scripts/package-apple-runner-source.mjs @@ -62,10 +62,9 @@ function packageAppleRunnerSource(options = {}) { function packageSnapshotPresentationSource(root, options, summary) { const sourceRoot = path.join(root, SNAPSHOT_PRESENTATION_SOURCE_DIR); if (!fs.existsSync(sourceRoot)) { - return; + throw new Error(`Apple snapshot presentation source not found at ${sourceRoot}`); } const outputRoot = path.join(root, SNAPSHOT_PRESENTATION_OUTPUT_DIR); - prepareSnapshotPresentationOutput(outputRoot, options.checkOnly); const manifestSource = requireSnapshotPresentationManifest(sourceRoot); processDirectory(sourceRoot, options.checkOnly ? undefined : outputRoot, '', summary, { validateSwift: false, @@ -75,11 +74,6 @@ function packageSnapshotPresentationSource(root, options, summary) { copySnapshotPresentationManifest(manifestSource, outputRoot, summary, options.checkOnly); } -function prepareSnapshotPresentationOutput(outputRoot, checkOnly) { - if (checkOnly) return; - fs.rmSync(outputRoot, { recursive: true, force: true }); -} - function requireSnapshotPresentationManifest(sourceRoot) { const manifestSource = path.join(sourceRoot, SNAPSHOT_PRESENTATION_RUNNER_MANIFEST); if (fs.existsSync(manifestSource)) return manifestSource; diff --git a/scripts/size-report-package.mjs b/scripts/size-report-package.mjs index ad5f560f0d..673158888a 100644 --- a/scripts/size-report-package.mjs +++ b/scripts/size-report-package.mjs @@ -20,6 +20,13 @@ const PACKAGE_COMPONENTS = [ matches: (entryPath) => entryPath === 'dist/apple/runner' || entryPath.startsWith('dist/apple/runner/'), }, + { + id: 'apple-snapshot-presentation', + label: 'Apple snapshot presentation source', + matches: (entryPath) => + entryPath === 'dist/apple/snapshot-presentation' || + entryPath.startsWith('dist/apple/snapshot-presentation/'), + }, { id: 'macos-helper', label: 'macOS helper source', diff --git a/scripts/write-xcuitest-cache-metadata.mjs b/scripts/write-xcuitest-cache-metadata.mjs index dd3d250ee1..8520802af6 100644 --- a/scripts/write-xcuitest-cache-metadata.mjs +++ b/scripts/write-xcuitest-cache-metadata.mjs @@ -15,6 +15,14 @@ const USAGE = 'Usage: write-xcuitest-cache-metadata.mjs '; const DEFAULT_IOS_RUNNER_APP_BUNDLE_ID = 'com.callstack.agentdevice.runner'; +const RUNNER_SOURCE_IGNORED_DIR_NAMES = new Set(['.build', '.swiftpm', 'xcuserdata']); +const SNAPSHOT_PRESENTATION_SOURCE_IGNORED_DIR_NAMES = new Set([ + '.build', + '.swiftpm', + 'SnapshotPresentationConformance', + 'Tests', + 'xcuserdata', +]); function isTruthy(value) { return ['1', 'true', 'TRUE', 'yes', 'YES', 'on', 'ON'].includes(String(value ?? '')); @@ -49,11 +57,20 @@ function resolveRunnerTestBundleId() { } function computeRunnerSourceFingerprint() { - const runnerRoot = path.join(projectRoot, 'apple', 'runner', 'AgentDeviceRunner'); - const files = collectRunnerSourceFiles(runnerRoot); + const sourceRoots = [ + { + path: path.join(projectRoot, 'apple', 'runner', 'AgentDeviceRunner'), + ignoredDirectoryNames: RUNNER_SOURCE_IGNORED_DIR_NAMES, + }, + { + path: path.join(projectRoot, 'apple', 'snapshot-presentation'), + ignoredDirectoryNames: SNAPSHOT_PRESENTATION_SOURCE_IGNORED_DIR_NAMES, + }, + ]; + const files = collectRunnerSourceFiles(sourceRoots); const hash = crypto.createHash('sha256'); for (const file of files) { - hash.update(path.relative(runnerRoot, file)); + hash.update(path.relative(projectRoot, file)); hash.update('\0'); hash.update(fs.readFileSync(file)); hash.update('\0'); @@ -61,27 +78,29 @@ function computeRunnerSourceFingerprint() { return hash.digest('hex'); } -function collectRunnerSourceFiles(root) { - if (!fs.existsSync(root)) { - return []; - } +function collectRunnerSourceFiles(roots) { const files = []; - const stack = [root]; - while (stack.length > 0) { - const current = stack.pop(); - for (const entry of fs.readdirSync(current, { withFileTypes: true })) { - const fullPath = path.join(current, entry.name); - if (entry.isDirectory()) { - if (entry.name === 'xcuserdata') continue; - stack.push(fullPath); - continue; - } - if (entry.isFile() && isRunnerSourceFile(entry.name, fullPath)) { - files.push(fullPath); + for (const { path: root, ignoredDirectoryNames } of roots) { + if (!fs.existsSync(root)) { + continue; + } + const stack = [root]; + while (stack.length > 0) { + const current = stack.pop(); + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + const fullPath = path.join(current, entry.name); + if (entry.isDirectory()) { + if (ignoredDirectoryNames.has(entry.name)) continue; + stack.push(fullPath); + continue; + } + if (entry.isFile() && isRunnerSourceFile(entry.name, fullPath)) { + files.push(fullPath); + } } } } - return files.sort((a, b) => a.localeCompare(b)); + return [...new Set(files)].sort((a, b) => a.localeCompare(b)); } function isRunnerSourceFile(fileName, filePath) { diff --git a/src/__tests__/apple-runner-package-source.test.ts b/src/__tests__/apple-runner-package-source.test.ts index 880352e107..f2477b3b60 100644 --- a/src/__tests__/apple-runner-package-source.test.ts +++ b/src/__tests__/apple-runner-package-source.test.ts @@ -41,6 +41,17 @@ test('package apple runner source strips unit-test blocks without mutating check path.join(root, 'dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj'), ), ); + const packagedRunnerRoot = path.join(root, 'dist/apple/runner/AgentDeviceRunner'); + const packagedProject = fs.readFileSync( + path.join(packagedRunnerRoot, 'AgentDeviceRunner.xcodeproj/project.pbxproj'), + 'utf8', + ); + const sharedPackageRelativePath = + packagedProject.match(/relativePath = ([^;]+);/)?.[1]?.trim() ?? ''; + assert.equal(sharedPackageRelativePath, '../../snapshot-presentation'); + assert.ok( + fs.existsSync(path.resolve(packagedRunnerRoot, sharedPackageRelativePath, 'Package.swift')), + ); assert.ok(fs.existsSync(path.join(root, 'dist/apple/snapshot-presentation/Package.swift'))); assert.equal( fs.readFileSync(path.join(root, 'dist/apple/snapshot-presentation/Package.swift'), 'utf8'), @@ -77,6 +88,7 @@ test('package apple runner source strips unit-test blocks without mutating check test('package apple runner source skips the explicit unit-test directory', async () => { const root = mkdtempForTestSync('agent-device-runner-package-unit-tests-'); onTestFinished(() => fs.rmSync(root, { recursive: true, force: true })); + writeFixtureFile(root, 'apple/snapshot-presentation/Package.runner.swift', 'runner package\n'); const uitestsDir = 'apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests'; writeFixtureFile( root, @@ -144,6 +156,7 @@ test('package apple runner source check rejects unit tests without writing dist' test('package apple runner source allows only the runner entrypoint test method', async () => { const root = mkdtempForTestSync('agent-device-runner-package-entry-'); onTestFinished(() => fs.rmSync(root, { recursive: true, force: true })); + writeFixtureFile(root, 'apple/snapshot-presentation/Package.runner.swift', 'runner package\n'); writeFixtureFile( root, @@ -183,6 +196,7 @@ test('package apple runner source removes legacy dist/apple-runner output before 'apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj', '', ); + writeFixtureFile(root, 'apple/snapshot-presentation/Package.runner.swift', 'runner package\n'); // Stale packaged trees left by builds/checkouts predating the apple-runner -> apple/runner // move. `dist` ships wholesale, so these must not survive packaging or they double-ship. writeFixtureFile( @@ -203,6 +217,40 @@ test('package apple runner source removes legacy dist/apple-runner output before assert.ok(fs.existsSync(path.join(root, 'dist/apple/runner/AgentDeviceRunner'))); }); +test('package apple runner source requires the shared snapshot presentation source', async () => { + const root = mkdtempForTestSync('agent-device-runner-package-missing-presentation-'); + onTestFinished(() => fs.rmSync(root, { recursive: true, force: true })); + writeFixtureFile( + root, + 'apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj', + '', + ); + + const result = await runCmd(process.execPath, [packageScript, '--root', root, '--quiet'], { + allowFailure: true, + }); + + assert.notEqual(result.exitCode, 0); + assert.match(result.stderr, /snapshot presentation source not found/); +}); + +test('snapshot presentation manifests keep their supported platform declarations in parity', () => { + const manifest = fs.readFileSync( + path.join(repoRoot, 'apple/snapshot-presentation/Package.swift'), + 'utf8', + ); + const runnerManifest = fs.readFileSync( + path.join(repoRoot, 'apple/snapshot-presentation/Package.runner.swift'), + 'utf8', + ); + + for (const declaration of ['.iOS(.v15)', '.macOS(.v13)', '.tvOS(.v15)', '.visionOS(.v1)']) { + const escaped = declaration.replaceAll(/[.()]/g, String.raw`\$&`); + assert.match(manifest, new RegExp(escaped)); + assert.match(runnerManifest, new RegExp(escaped)); + } +}); + test('apple runner tree snapshot capture stays on the main queue', () => { const source = fs.readFileSync(runnerSnapshotSwiftPath, 'utf8'); const boundedCapture = extractSwiftFunction(source, 'captureSnapshotRootBounded'); @@ -244,7 +292,7 @@ function writeStripFixtureTree(root: string): void { writeFixtureFile( root, 'apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj', - '', + 'relativePath = ../../snapshot-presentation;\n', ); writeFixtureFile( root, diff --git a/src/core/__tests__/snapshot-state.test.ts b/src/core/__tests__/snapshot-state.test.ts index a3fbdc923a..4d8c6b3fe3 100644 --- a/src/core/__tests__/snapshot-state.test.ts +++ b/src/core/__tests__/snapshot-state.test.ts @@ -136,6 +136,22 @@ test('buildSnapshotState applies iOS interactive presentation for xctest snapsho ]); }); +test('buildSnapshotState leaves Apple runner presentation to the engine', () => { + const nodes = [ + { index: 0, depth: 0, type: 'Application', label: 'Settings' }, + { index: 1, depth: 1, parentIndex: 0, type: 'Table', label: 'Settings' }, + { index: 2, depth: 2, parentIndex: 1, type: 'Cell', label: 'General' }, + { index: 3, depth: 3, parentIndex: 2, type: 'Button', label: 'General' }, + ]; + + const state = buildSnapshotState( + { nodes, backend: 'xctest', producer: 'apple-runner' }, + { snapshotInteractiveOnly: true }, + ); + + expect(state.nodes.map((node) => node.type)).toEqual(['Application', 'Table', 'Cell', 'Button']); +}); + test('buildSnapshotState marks content covered by floating overlays as visible but blocked', () => { const state = buildSnapshotState( { diff --git a/src/core/snapshot-state.ts b/src/core/snapshot-state.ts index b914c80c7a..c5614c42ab 100644 --- a/src/core/snapshot-state.ts +++ b/src/core/snapshot-state.ts @@ -22,11 +22,12 @@ import { coveredAndroidReplacementNodeIndexes } from '../snapshot/android-replac import { scopeSnapshotNodes } from '@agent-device/capture-kit/snapshot-desktop-projection'; import { normalizeSnapshotTree, pruneGroupNodes } from '../core/snapshot-tree-ingestion.ts'; import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; +import { IOS_SNAPSHOT_PRODUCER_CAPABILITIES } from '@agent-device/capture-kit/ios-snapshot-planning'; /** - * The ONE daemon presentation of a captured tree (ADR 0004 / #1797 "compaction layer"): normalize, - * group prune, iOS interactive presentation, post-wire scope for backends that do not scope in - * their own projection, occlusion annotation, refs. Every consumer of a captured tree — the + * The ONE daemon assembly of a captured tree (ADR 0004 / #1797): normalize, group prune, + * post-wire scope for backends that do not scope in their own projection, occlusion annotation, + * refs. Every consumer of a captured tree — the * snapshot command, selector captures, settle observation, Android blocking-dialog recovery — * goes through here, so no two call sites can disagree about what a snapshot contains. * @@ -53,7 +54,7 @@ export function buildSnapshotState( const normalizedNodes = normalizeSnapshotTree( snapshotRaw ? backendAnnotatedNodes : pruneGroupNodes(backendAnnotatedNodes), ); - const presentableNodes = shouldPresentIosInteractiveSnapshot(data?.backend, flags) + const presentableNodes = shouldPresentIosInteractiveSnapshot(data, flags) ? presentIosInteractiveSnapshot(normalizedNodes) : normalizedNodes; const scopedNodes = @@ -120,17 +121,30 @@ function backendScopesAfterWire(backend: SnapshotBackend | undefined): boolean { } function shouldPresentIosInteractiveSnapshot( - backend: SnapshotBackend | undefined, + provenance: SnapshotStateProvenance, flags: | (Pick & Partial>) | undefined, ): boolean { return ( - backend === 'xctest' && flags?.snapshotInteractiveOnly === true && flags.snapshotRaw !== true + provenance.backend === 'xctest' && + iosSnapshotPresentationStage(provenance) === 'acquired' && + flags?.snapshotInteractiveOnly === true && + flags.snapshotRaw !== true ); } +function iosSnapshotPresentationStage( + provenance: SnapshotStateProvenance, +): 'acquired' | 'presented' | undefined { + if (provenance.backend !== 'xctest') return undefined; + if (provenance.producer === undefined) return 'acquired'; + return IOS_SNAPSHOT_PRODUCER_CAPABILITIES[ + provenance.producer as 'apple-runner' | 'appium-source' | 'limrun-ios-tree' + ].stage; +} + function isAndroidComparisonSafeSnapshot( backend: SnapshotBackend | undefined, flags: diff --git a/src/daemon/__tests__/request-router-screenshot.test.ts b/src/daemon/__tests__/request-router-screenshot.test.ts index 55a318cdef..4caf921917 100644 --- a/src/daemon/__tests__/request-router-screenshot.test.ts +++ b/src/daemon/__tests__/request-router-screenshot.test.ts @@ -447,7 +447,7 @@ test('screenshot --overlay-refs captures a fresh snapshot when the session has n expect(runtime.binds).toHaveLength(1); }); -test('screenshot --overlay-refs uses interactive iOS presentation for row-like other nodes', async () => { +test('screenshot --overlay-refs uses presented iOS runner rows for overlay refs', async () => { const screenshotPath = path.join(os.tmpdir(), `agent-device-overlay-ios-${Date.now()}.png`); const { handler, sessionStore, runtime } = screenshotRouter(makeIosSession('default'), { onCapture: (input) => writeSolidPng(input.outPath, 402, 874), @@ -490,7 +490,7 @@ test('screenshot --overlay-refs uses interactive iOS presentation for row-like o index: 4, depth: 2, parentIndex: 2, - type: 'Other', + type: 'Cell', label: 'Receipt missing details, Receipt scanning failed. Enter details manually.', rect: { x: 8, y: 367, width: 386, height: 64 }, }, @@ -522,7 +522,13 @@ test('screenshot --overlay-refs uses interactive iOS presentation for row-like o expect(runtime.captureSnapshot.mock.calls[0]?.[0].options).toMatchObject({ interactiveOnly: true, }); - expect(sessionStore.get('default')?.snapshot?.nodes[4]?.type).toBe('Cell'); + expect(sessionStore.get('default')?.snapshot?.producer).toBe('apple-runner'); + expect( + sessionStore.get('default')?.snapshot?.nodes.find((node) => node.ref === 'e5'), + ).toMatchObject({ + type: 'Cell', + label: 'Receipt missing details, Receipt scanning failed. Enter details manually.', + }); }); test('screenshot --overlay-refs uses a fresh snapshot instead of stale session snapshot', async () => { diff --git a/src/snapshot/snapshot-presentation/ios/presentation-visibility.test.ts b/src/snapshot/__tests__/snapshot-visibility-ios.test.ts similarity index 98% rename from src/snapshot/snapshot-presentation/ios/presentation-visibility.test.ts rename to src/snapshot/__tests__/snapshot-visibility-ios.test.ts index abcda11a20..79e9e16bb4 100644 --- a/src/snapshot/snapshot-presentation/ios/presentation-visibility.test.ts +++ b/src/snapshot/__tests__/snapshot-visibility-ios.test.ts @@ -1,6 +1,6 @@ import { expect, test } from 'vitest'; import { attachRefs, type RawSnapshotNode } from '@agent-device/kernel/snapshot'; -import { buildSnapshotVisibility } from '../../snapshot-visibility.ts'; +import { buildSnapshotVisibility } from '../snapshot-visibility.ts'; import { presentIosInteractiveSnapshot } from '@agent-device/capture-kit/ios-snapshot-engine'; function buildSnapshotState(data: { nodes?: RawSnapshotNode[]; backend?: 'xctest' }) { diff --git a/test/integration/interaction-contract/fixtures.ts b/test/integration/interaction-contract/fixtures.ts index 4c5e54ebb0..11eb3046b1 100644 --- a/test/integration/interaction-contract/fixtures.ts +++ b/test/integration/interaction-contract/fixtures.ts @@ -1,4 +1,4 @@ -import type { SnapshotState } from '@agent-device/kernel/snapshot'; +import type { RawSnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot'; import { makeSnapshotState } from '../../../src/__tests__/test-utils/snapshot-builders.ts'; /** @@ -335,6 +335,12 @@ export function dragEndpointsSnapshot(): SnapshotState { ]); } +export function runnerPresentedDragEndpointsNodes(): RawSnapshotNode[] { + return dragEndpointsSnapshot() + .nodes.filter((node) => node.index !== 3) + .map(({ ref: _ref, ...node }) => node); +} + /** * Runner-side node payloads (the shape `ios.runner.snapshot` returns) for the * provider-transcript scenarios. diff --git a/test/integration/interaction-contract/target-drag.contract.test.ts b/test/integration/interaction-contract/target-drag.contract.test.ts index 770ba773e6..bee5241362 100644 --- a/test/integration/interaction-contract/target-drag.contract.test.ts +++ b/test/integration/interaction-contract/target-drag.contract.test.ts @@ -11,6 +11,7 @@ import { coveredButtonSnapshot, dragEndpointsSnapshot, fullyTiledParentSnapshot, + runnerPresentedDragEndpointsNodes, } from './fixtures.ts'; import { createContractDevice } from './runtime-harness.ts'; import { @@ -146,7 +147,7 @@ test(scenario('errorTaxonomy'), async () => { test( scenario('responseConstruction'), async () => { - const nodes = dragEndpointsSnapshot().nodes; + const nodes = runnerPresentedDragEndpointsNodes(); await withIosContractDaemon( [ runnerGestureViewportEntry(), diff --git a/test/integration/provider-scenarios/settle-observation.test.ts b/test/integration/provider-scenarios/settle-observation.test.ts index d2a52aa2fe..9024c041b5 100644 --- a/test/integration/provider-scenarios/settle-observation.test.ts +++ b/test/integration/provider-scenarios/settle-observation.test.ts @@ -456,6 +456,9 @@ test('Provider-backed integration modal-dismiss press --settle attaches the unch // July 2026, org.reactnavigation.playground rne://stack-prevent-remove Input // screen; `snapshot -i --json` before and after `fill @e6 "hello" --settle`, // with the 31-key block reduced to 2 representative keys and rects rounded). +// The retained TextField frames are normalized to the visible content frame after trimming, and +// the left candidate-bar height is rounded from 69 to 68; those are intentional fixture +// adjustments, not raw capture claims. // The load-bearing real-world facts they preserve: // - The keyboard renders in its OWN window; the "Next keyboard" and "Dictate" // candidate-bar buttons are SIBLINGS of the [Keyboard] container's wrapper, @@ -511,7 +514,7 @@ const FILL_BEFORE_NODES = [ depth: 4, parentIndex: 4, type: 'TextField', - rect: { x: 12, y: 129, width: 377, height: 41 }, + rect: { x: 12, y: 145, width: 377, height: 25 }, }, { index: 6, @@ -616,7 +619,7 @@ const FILL_SETTLED_NODES = [ label: 'Next keyboard', value: 'Polski', hittable: true, - rect: { x: 8, y: 806, width: 68, height: 69 }, + rect: { x: 8, y: 806, width: 68, height: 68 }, }, { index: 10, @@ -666,7 +669,7 @@ const FILL_SETTLED_NODES = [ type: 'TextField', label: 'hello', value: 'hello', - rect: { x: 12, y: 129, width: 377, height: 41 }, + rect: { x: 12, y: 145, width: 377, height: 25 }, }, { index: 16,