Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/actions/setup-apple-runner-build/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ runs:
id: source-hash
run: |
set -euo pipefail
echo "value=${{ hashFiles('apple/runner/**', 'scripts/build-xcuitest-apple.sh', 'scripts/patch-xcuitest-runner-icon.ts', 'scripts/write-xcuitest-cache-metadata.mjs', 'packages/platform-apple/src/runner/apple-runner-platform.ts', 'packages/platform-apple/src/runner/runner-cache-metadata.ts', 'packages/platform-apple/src/runner/runner-icon.ts', 'packages/platform-apple/src/runner/runner-xctestrun.ts', 'packages/platform-apple/src/runner/runner-xctestrun-products.ts', '.github/actions/setup-apple-runner-build/action.yml', 'package.json', 'pnpm-lock.yaml') }}" >> "$GITHUB_OUTPUT"
echo "value=${{ hashFiles('apple/runner/**', 'apple/snapshot-presentation/**', 'scripts/build-xcuitest-apple.sh', 'scripts/patch-xcuitest-runner-icon.ts', 'scripts/write-xcuitest-cache-metadata.mjs', 'packages/platform-apple/src/runner/apple-runner-platform.ts', 'packages/platform-apple/src/runner/runner-cache-metadata.ts', 'packages/platform-apple/src/runner/runner-icon.ts', 'packages/platform-apple/src/runner/runner-xctestrun.ts', 'packages/platform-apple/src/runner/runner-xctestrun-products.ts', '.github/actions/setup-apple-runner-build/action.yml', 'package.json', 'pnpm-lock.yaml') }}" >> "$GITHUB_OUTPUT"
shell: bash

- name: Resolve Apple runner build variant
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ struct DataPayload: Codable {
var items: [String]?
var nodes: [PresentedNode]?
var truncated: Bool?
var qualityPayload: SnapshotQualityPayload? = nil
var snapshotQuality: SnapshotQuality?
var gestureStartUptimeMs: Double?
var gestureEndUptimeMs: Double?
Expand Down Expand Up @@ -274,6 +275,31 @@ struct DataPayload: Codable {
var sequenceResults: [SequenceStepResult]?
}

struct SnapshotQualityPayload: Codable {
let nodes: [PresentedNode]
let truncated: Bool
let scope: String?

init(nodes: [PresentedNode], truncated: Bool) {
self.nodes = nodes
self.truncated = truncated
self.scope = nil
}

private enum CodingKeys: String, CodingKey {
case nodes
case truncated
case scope
}

func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(nodes, forKey: .nodes)
try container.encode(truncated, forKey: .truncated)
try container.encodeNil(forKey: .scope)
}
}

struct ErrorPayload: Codable {
var code: String?
let message: String
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,11 @@ extension RunnerTests {
do {
presented = try timer.measure(.presentation) {
guard let result = try SnapshotPresentation.present(acquisition, options: options) else {
NSLog(
"AGENT_DEVICE_RUNNER_SNAPSHOT_PROJECTION_MISMATCH requested=%@ acquired=%@",
hint.projection.rawValue,
acquisition.hint.projection.rawValue
)
throw Self.snapshotProjectionMismatchFailure(
kind,
requested: hint.projection,
Expand Down Expand Up @@ -641,6 +646,10 @@ extension RunnerTests {
message: Self.legacyQualityMessage(quality) ?? payload.message,
nodes: payload.nodes,
truncated: payload.truncated == true || state != "healthy" || capture.effectiveDepth != nil,
qualityPayload: capture.qualityPayload.flatMap { quality in
guard let nodes = quality.nodes else { return nil }
return SnapshotQualityPayload(nodes: nodes, truncated: quality.truncated == true)
},
snapshotQuality: quality,
runnerFatal: payload.runnerFatal,
runnerFatalReason: payload.runnerFatalReason
Expand Down Expand Up @@ -877,6 +886,29 @@ extension RunnerTests {
XCTAssertEqual(payload.nodes?.count, 1)
}

func testSnapshotQualityCarriesUnscopedQualityPayload() {
let quality = DataPayload(
nodes: [planTestNode(index: 0, type: "Application", label: "App")],
truncated: false
)
let capture = SnapshotBackendCapture(
payload: quality,
effectiveDepth: nil,
qualityPayload: quality
)

let payload = stampedSnapshotPayload(
capture,
backend: .recursiveTree,
state: "healthy",
reason: nil
)

XCTAssertEqual(payload.qualityPayload?.nodes.count, 1)
XCTAssertEqual(payload.qualityPayload?.truncated, false)
XCTAssertNil(payload.qualityPayload?.scope)
}

func testDirectPresentationDoesNotClaimPlanTiming() {
let options = PresentationOptions(
interactiveOnly: false,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ extension RunnerTests {
}

XCTAssertThrowsError(
try SnapshotPresentationInvariant.validateRegularWithStats(
try SnapshotPresentationInvariant.validateRegular(
folded,
viewport: viewport,
policy: .cursorProjected
Expand All @@ -323,30 +323,6 @@ extension RunnerTests {
}
}

func testRegularInvariantUsesOneParentClipLookupPerNode() throws {
let nodeCount = 5_000
let viewport = CGRect(x: 0, y: 0, width: 100, height: 100)
let nodes = (0..<nodeCount).map { index in
let raw = Self.invariantNode(
index,
type: index == 0 ? "Application" : "ScrollView",
rect: SnapshotRect(x: 0, y: 0, width: 100, height: 100),
parentIndex: index == 0 ? nil : index - 1
)
return SnapshotPresentationNode(raw: raw, effectiveRect: raw.rect)
}

let stats = try SnapshotPresentationInvariant.validateRegularWithStats(
nodes,
viewport: viewport,
policy: .cursorProjected
)

// The former per-node ancestor walk performs 12,497,500 lookups for this chain. Counting the
// cached parent resolutions makes the linear guarantee deterministic without timing the test.
XCTAssertEqual(stats.parentClipLookups, nodeCount - 1)
}

func testPresentationFailureKeepsItsNamedSnapshotQualityReason() {
// Non-vacuity: dropping the typed reason preference in the plan mapper changes the assertion
// below to `capture-failed`, losing the distinction this contract is meant to preserve.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ extension RunnerTests {
func testSnapshotScopePolicyMatchesGoldenParityTable() throws {
// Non-vacuity: label-only semantic values fail the identifier-only and value-only fixtures.
let fixtureURL = URL(fileURLWithPath: #filePath)
.deletingLastPathComponent() // UnitTests
.deletingLastPathComponent() // AgentDeviceRunnerUITests
.deletingLastPathComponent() // AgentDeviceRunner
.deletingLastPathComponent() // runner
Expand Down
2 changes: 2 additions & 0 deletions apple/snapshot-presentation/Package.runner.swift
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ let package = Package(
platforms: [
.iOS(.v15),
.macOS(.v13),
.tvOS(.v15),
.visionOS(.v1),
],
products: [
.library(
Expand Down
2 changes: 2 additions & 0 deletions apple/snapshot-presentation/Package.swift
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ let package = Package(
platforms: [
.iOS(.v15),
.macOS(.v13),
.tvOS(.v15),
.visionOS(.v1),
],
products: [
.library(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@ import Foundation
import CoreGraphics

public enum SnapshotPresentation {
public typealias PresentedNode = AgentDeviceSnapshotPresentation.PresentedNode

public static func present(
_ acquisition: SnapshotAcquisition,
options: PresentationOptions
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@ import Foundation
import CoreGraphics

public enum SnapshotPresentationInvariant {
public struct ValidationStats: Equatable {
public let parentClipLookups: Int
struct ValidationStats: Equatable {
let parentClipLookups: Int

public init(parentClipLookups: Int) {
init(parentClipLookups: Int) {
self.parentClipLookups = parentClipLookups
}
}
Expand All @@ -18,7 +18,7 @@ public enum SnapshotPresentationInvariant {
_ = try validateRegularWithStats(nodes, viewport: viewport, policy: policy)
}

public static func validateRegularWithStats(
static func validateRegularWithStats(
_ nodes: [SnapshotPresentationNode],
viewport: CGRect,
policy: SnapshotVisibilityFold.Policy
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -169,9 +169,7 @@ public enum SnapshotVisibilityFold {
var hints: [Int: (above: Bool, below: Bool)] = [:]

for (offset, node) in nodes.enumerated() {
let parentState = node.parentIndex.flatMap { index in
index >= 0 && index < states.count ? states[index] : nil
}
let parentState = node.parentIndex.flatMap { states[$0] }
let parentTraversal = parentState?.traversal ?? .root
let parentAnchor = policy == .cursorProjected ? parentState?.anchor : nil
let rect = node.rect.cgRect
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import CoreGraphics
import XCTest
@testable import AgentDeviceSnapshotPresentation

final class InvariantTests: XCTestCase {
func testRegularInvariantUsesOneParentClipLookupPerNode() throws {
let nodeCount = 5_000
let viewport = CGRect(x: 0, y: 0, width: 100, height: 100)
let nodes = (0..<nodeCount).map { index in
let raw = RawAXNode(
index: index,
type: index == 0 ? "Application" : "ScrollView",
label: nil,
identifier: nil,
value: nil,
rect: SnapshotRect(x: 0, y: 0, width: 100, height: 100),
enabled: true,
focused: nil,
selected: nil,
hittable: false,
depth: index,
parentIndex: index == 0 ? nil : index - 1,
hiddenContentAbove: nil,
hiddenContentBelow: nil
)
return SnapshotPresentationNode(raw: raw, effectiveRect: raw.rect)
}

let stats = try SnapshotPresentationInvariant.validateRegularWithStats(
nodes,
viewport: viewport,
policy: .cursorProjected
)

XCTAssertEqual(stats.parentClipLookups, nodeCount - 1)
}
}
32 changes: 21 additions & 11 deletions docs/adr/0004-ios-snapshot-backend-strategy.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
Accepted. Amended after iOS snapshot capture was simplified to two public modes:
regular interactive snapshots and raw diagnostic snapshots.

The current implementation is owned by `RunnerTests+SnapshotCapturePlan.swift`. Capture plans
declare their XCTest backend chain, and structured snapshot quality verdicts make degraded or
recovered output observable end to end.
The runner owns capture-plan acquisition and backend fallback. Host-side iOS validation, semantic
presentation, and publication are owned by `@agent-device/capture-kit`; structured snapshot quality
verdicts make degraded or recovered output observable end to end.

## Context

Expand Down Expand Up @@ -66,17 +66,19 @@ agents know the snapshot is degraded output rather than proof that the screen ha
## Host-side ownership boundary

The shared TypeScript side has one snapshot-presentation facet. The neutral acquisition-to-presented
carrier and clip-fold geometry contract live in `@agent-device/contracts/snapshot-presentation`; the
host-side iOS post-wire policies and shared tree helpers live under `src/snapshot/snapshot-presentation/`.
Platform-specific presentation adapters retain only the policy mechanics that cannot yet cross their
runtime boundary. Daemon assembly owns only the ordering of capture, compaction, occlusion, and ref
publication. It does not own the presentation vocabulary or a second geometry carrier.
carrier and clip-fold geometry contract live in `@agent-device/contracts/snapshot-presentation`, while
`@agent-device/capture-kit` owns host-side iOS planning, folding, projection, eligibility, semantic
compaction, validation, and publication. Platform-specific presentation adapters retain only the
policy mechanics that cannot yet cross their runtime boundary. Daemon assembly owns only the ordering
of capture, compaction, occlusion, and ref publication. It does not own the presentation vocabulary
or a second geometry carrier.

Android acquisition remains in its platform module and adapts its raw hierarchy to the shared
carrier. Swift keeps its runner-side `SnapshotPresentation` implementation because it consumes the
capture-plan tier before the process boundary. The contract fixture under
`contracts/fixtures/snapshot-presentation-conformance.json` is the shared proof between those
runtimes; it does not imply that Swift and TypeScript share an implementation.
capture-plan tier before the process boundary. The iOS engine fixture is the shared proof between
those runtimes; it does not imply that Swift and TypeScript share an implementation.
The macOS XCTest runner is the desktop-surface exception: its already-presented nodes bypass the iOS
presentation engine and continue through neutral snapshot assembly.

The same split now holds for the three remaining Wave 4 policies tracked by #1983, so
`src/snapshot/` is the host-side owner of snapshot policy generally rather than of presentation
Expand Down Expand Up @@ -134,6 +136,14 @@ session binding.

New consumers must use the facet rather than add another daemon presentation path.

The acquisition/presentation boundary has two explicit vocabularies. An acquired input is raw node
evidence accompanied by its capture hint, viewport, lineage, and residue; the host engine folds and
projects that evidence. A presented input is the runner's primary payload plus validation facts; the
host engine validates it and performs semantic compaction once. Regular eligibility decides which
nodes belong in the regular presentation, while publication adds refs and emits only the primary
payload. An optional unscoped quality payload is validated for classification evidence and is never
published.

## Regression Notes

PR #639 made XCTest AX serialization failures explicit instead of swallowing them as empty
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@
"maestro:conformance": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/maestro-conformance/format-generated-json.test.mjs packages/maestro/test/conformance/verify.test.ts packages/maestro/test/conformance/differential/engine-process.test.ts packages/maestro/test/conformance/differential/report-output.test.ts packages/maestro/test/conformance/differential/run.test.ts packages/maestro/test/conformance/differential/invariants.test.ts",
"maestro:conformance:regenerate": "node --experimental-strip-types scripts/maestro-conformance/regenerate.mjs",
"maestro:conformance:differential": "node --experimental-strip-types packages/maestro/test/conformance/differential/run.ts",
"test:ios-snapshot-differential": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/ios-snapshot-differential.test.ts",
"test:ios-snapshot-differential": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift test --package-path apple/snapshot-presentation && node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/ios-snapshot-differential.test.ts",
"size": "node scripts/size-report.mjs",
"perf": "node --experimental-strip-types scripts/perf/run.ts",
"mutation:run": "node --experimental-strip-types scripts/mutation/run.ts",
Expand Down
60 changes: 59 additions & 1 deletion packages/capture-kit/src/ios-snapshot-engine/engine.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,64 @@ test('raw projection preserves reported geometry while regular projection clips
assert.equal(raw.payload.nodes.find((node) => node.label === 'Escaped child')?.hittable, true);
});

test('presentation mapping retains acquisition lineage through projection and scoped reindexing', () => {
const nodes: RawSnapshotNode[] = [
node(10, 'Application', 'App', viewport),
node(20, 'Other', undefined, viewport, 10, 1),
node(40, 'Button', 'Target', { x: 20, y: 20, width: 80, height: 40 }, 20, 2),
];
const regularRequest = createIosSnapshotRequest();
const regular = presentIosSnapshot(acquiredInput(regularRequest, nodes), regularRequest);

assert.deepEqual(
[...regular.presentedIndexesBySourceIndex],
[
[10, [0]],
[20, []],
[40, [1]],
],
);

const rawRequest = createIosSnapshotRequest({ raw: true, scope: 'Target' });
const rawNodes = [
node(10, 'Application', 'App', viewport),
node(20, 'Other', 'Target', viewport, 10, 1),
node(40, 'Button', 'Child', { x: 20, y: 20, width: 80, height: 40 }, 20, 2),
];
const raw = presentIosSnapshot(acquiredInput(rawRequest, rawNodes), rawRequest);

assert.deepEqual(
[...raw.presentedIndexesBySourceIndex],
[
[10, []],
[20, [0]],
[40, [1]],
],
);
});

test('raw scoped depth derives missing source depths from parent order', () => {
const request = createIosSnapshotRequest({ raw: true, scope: 'Target', depth: 1 });
const nodes: RawSnapshotNode[] = [
node(10, 'Application', 'App', viewport),
{ ...node(20, 'Other', 'Target', viewport, 10), depth: undefined },
{
...node(40, 'Button', 'Child', { x: 20, y: 20, width: 80, height: 40 }, 20),
depth: undefined,
},
{
...node(50, 'Button', 'Grandchild', { x: 24, y: 24, width: 60, height: 32 }, 40),
depth: undefined,
},
];
const result = presentIosSnapshot(acquiredInput(request, nodes), request);

assert.deepEqual(
result.nodes.map((entry) => entry.label),
['Target', 'Child'],
);
});

test('cursor projection keeps geometryless nodes neutral while plain viewport keeps child visibility independent', () => {
const request = createIosSnapshotRequest();
const nodes = [
Expand Down Expand Up @@ -285,7 +343,7 @@ function scopedNodes(): RawSnapshotNode[] {
function node(
index: number,
type: string,
label: string,
label: string | undefined,
rect: Rect,
parentIndex?: number,
depth?: number,
Expand Down
Loading
Loading