diff --git a/docs/evidence/ios-simulator-ax-bridge-2026-09-01.json.gz b/docs/evidence/ios-simulator-ax-bridge-2026-09-01.json.gz new file mode 100644 index 000000000..cb88f34a3 Binary files /dev/null and b/docs/evidence/ios-simulator-ax-bridge-2026-09-01.json.gz differ diff --git a/docs/evidence/ios-simulator-ax-bridge-2026-09-01.md b/docs/evidence/ios-simulator-ax-bridge-2026-09-01.md new file mode 100644 index 000000000..7976254d0 --- /dev/null +++ b/docs/evidence/ios-simulator-ax-bridge-2026-09-01.md @@ -0,0 +1,132 @@ +# iOS Simulator AX bridge spike + +- Decision: **INCONCLUSIVE — public AX NO-GO; guest AX bridge requires the full corpus** +- Status: **reopened after candidate-coverage correction** +- Revision: 5ad244596c73f0884a48eb6445da3262d7eecb87 (takeover/2209-valid-evidence) +- Target: AgentDevice-2209-Takeover (F578F08D-BEA1-4A56-8A4B-C92B040FBA94, com.apple.CoreSimulator.SimRuntime.iOS-26-2) +- Generated: 2026-09-01T14:27:51.339Z +- Corpus: states=warm, screens=quiet, list, samples=20 +- Corpus coverage: **decisive-early-stop** + +The generated run originally reported an overall NO-GO. That verdict was too broad: the +`private-coresimulator-ax` adapter returned `private-tool-unavailable` solely because no +`--private-tool` path was supplied. It did not discover, build, or execute a private candidate. +The gzipped JSON remains the immutable original run; the correction and independent live probe +below are intentionally recorded separately. + +## Environment and limits + +- Node: v26.1.0 +- pnpm: 11.21.0 +- Xcode: Xcode 26.2; Build version 17C52 +- simctl: @(#)PROGRAM:simctl PROJECT:CoreSimulator-1155.4 +- Swift: Apple Swift version 6.2.3 (swiftlang-6.2.3.3.21 clang-1700.6.3.2) +Target: arm64-apple-macosx26.0 +- OS: darwin 25.5.0; arch=arm64 +- Bounds: request=65536 B, response=4194304 B, nodes=1500, traversal=12, CPU=2000 ms, memory=268435456 B, duration=5000 ms + +## Candidate fidelity and limitation matrix + +| Candidate | Mechanism | App surface | System surface | Lifecycle | Main limitation | +|---|---|---|---|---|---| +| public-macos-ax | public macOS ApplicationServices AX | observed in successful cells | not exercised | framed protocol | list evidence is flatter and has different identifier coverage (depth 1 vs 4; identifiers 42 vs 11) | +| private-coresimulator-ax | external/private CoreSimulator AX tool | not exercised in the generated corpus | not exercised | framed protocol contract only | the run supplied no tool, so it cannot support a viability verdict | +| xctest-control | #2189 XCTest runner control | observed in successful cells | not exercised | existing runner lifecycle | control, not a host-side AX bridge | + +## Raw acquisition and prototype presentation results + +| Candidate | State | Screen | Readable/attempted | Wall p50/p95 ms | Gated duration p50/p95 ms | First look p95 ms | Presentation p50/p95 ms | Nodes | Failures | +|---|---|---|---:|---:|---:|---:|---:|---:|---:| +| public-macos-ax | warm | quiet | 20/20 | 135.4/276.3 | 121.6/231.9 | 135.4/276.3 | 0.0/0.2 | 4.0 | 0 | +| public-macos-ax | warm | list | 20/20 | 1330.6/2554.9 | 1314.5/2527.0 | 1330.6/2554.9 | 1.6/2.5 | 114.0 | 0 | +| xctest-control | warm | quiet | 20/20 | 249.7/706.0 | 249.5/705.7 | 249.7/706.0 | 0.1/0.2 | 4.0 | 0 | +| xctest-control | warm | list | 20/20 | 471.8/579.0 | 471.4/578.8 | 471.8/579.0 | 0.2/0.5 | 33.0 | 0 | + +Raw exemplar fidelity (public AX vs XCTest control): +- quiet: nodes 4/4; depth 1/2; identifiers 0/1. +- list: nodes 114/33; depth 1/4; identifiers 42/11. + +Every acquisition sample retains timing, resource, readiness, and failure evidence; the first successful sample in each cell also retains one raw node-tree exemplar with viewport, target generation, truncation, and residue. Presentation samples measure only construction of the #2190 acquired carrier; they do not apply visibility, hittability, scope, depth, or semantic compaction. + +## Direct protocol probes + +- public-macos-ax/protocol-probe:public-macos-ax: ok=true, failure=none, code=none, nodes=1, duration=2234.7 ms, CPU=5.1 ms, memory=11468800 B, response=707 B +- private-coresimulator-ax/protocol-probe:private-coresimulator-ax: ok=false, failure=unsupported-mechanism, code=private-tool-unavailable, nodes=0, duration=0.0 ms, CPU=– ms, memory=– B, response=0 B +- stderr public-macos-ax/protocol-probe:public-macos-ax: [ios-ax-spike] capture id=protocol-probe:public-macos-ax candidate=public-macos-ax screen=unprepared-surface +- stderr private-coresimulator-ax/protocol-probe:private-coresimulator-ax: empty + +`private-tool-unavailable` is setup evidence, not mechanism evidence. In the adapter at this +revision, omitting `--private-tool` or naming a path that does not exist constructs this response +without starting a process. + +## Post-run candidate audit + +An independent audit on 2026-09-01 used the official idb v1.5.2 arm64 release +(`idb-companion.macos-arm64.tar.gz`, SHA-256 +`f17b718a513931705542a7fbfa9cfc11895ee191562c9ffd2343cf7f8254bc08`), including its bundled +`Resources/SimulatorFrameworkBridge`, on macOS 27.0 (26A5421a), Xcode 27.0 (27A5252f), and a booted +iOS 27.0 iPhone 17 Pro Simulator. Settings was foregrounded; the read used +`ui describe-all --api axbridge-persistent --format complete --profile` with label, identifier, +frame, and type. No `.xctest` bundle, test runner, or agent-device runner was started. + +- Backend reported: `axbridge-exclusive` (the persistent guest bridge's held session). +- First successful read: 167 elements, not truncated, one Mach round trip, 49,313 response bytes, + 956.0 ms total including bridge startup, and 56.9 ms guest read time. +- Six subsequent reads returned the same 167-element, non-truncated tree with one Mach round trip. + Total profile time ranged from 39.9 to 53.7 ms (p50 43.6 ms); guest read time ranged from 30.9 to + 39.4 ms. +- A prior attempt against a black/unready Simulator failed with an explicit frontmost-application + resolution error. Foregrounding Settings on a healthy Simulator separated target readiness from + bridge availability. +- The companion was stopped after the audit and removed its temporary directory. + +This audit proves that a compatible, no-XCTest guest AX mechanism exists, runs on the current host, +returns a detailed batched tree, and has enough warm latency headroom to justify the full #2192 +corpus. It does not prove the remaining system-surface, lifecycle, cancellation, stale-generation, +relaunch, or multi-screen acceptance cells, so it is not a production GO by itself. + +## Independent positive-control evidence + +- Invalid shallow rule: exit=1; command=pnpm bench:ios-snapshot:deep-button -- --rule invalid-shallow; assertion=AssertionError: changed descendant was omitted by shallow observation; no-effect claim is invalid. +- Safe full rule: exit=0; command=pnpm bench:ios-snapshot:deep-button -- --rule safe-full; assertion=full observation changed and includes the changed descendant. + +## Preference experiment + +- Applied: **true** +- Restored: **true** +- Fixture launch compatible: **true** +- Simulator state before experiment: Shutdown +- Private/preboot preference keys are experimental only; they were applied to this shutdown disposable Simulator and the original plist bytes were restored. +- /Users/thymikee/Library/Developer/CoreSimulator/Devices/F578F08D-BEA1-4A56-8A4B-C92B040FBA94/data/Library/Preferences/com.apple.Accessibility.plist: existed=true, beforeSha256=0c85a9ace2ad2c1be37a09a2ceea94fb786d4e163b4c181307bc991038a3b514, afterSha256=23695eb18c42e49869332f9149ad7005efd9d8e143c951e1396b8e67f2b4c179 + - Changes: AccessibilityEnabled: false -> true; ApplicationAccessibilityEnabled: 0 -> true; AutomationEnabled: 0 -> true; IgnoreAXServerEntitlements: undefined -> true +- /Users/thymikee/Library/Developer/CoreSimulator/Devices/F578F08D-BEA1-4A56-8A4B-C92B040FBA94/data/Library/Preferences/com.apple.UIAutomation.plist: existed=true, beforeSha256=db8995177327a963486dd0607260f0fad74ad10d9dec6c2f5abdbaf0dbd00b2c, afterSha256=db8995177327a963486dd0607260f0fad74ad10d9dec6c2f5abdbaf0dbd00b2c + - Changes: none + +## Lifecycle, cancellation, and recovery + +- Source: framed-protocol-fixture +- Process crash: process-crash; recovered=true +- Timeout: timeout; recovered=true +- Cancellation: cancelled; recovered=true +- Stale generation: stale-generation; recovered=true + +## Decision rationale + +- public-macos-ax warm/quiet acquisition missed the 75/150 ms target. +- public-macos-ax warm/list acquisition missed the 75/150 ms target. +- The generated private candidate result only proved that no tool path was supplied. +- The post-run idb audit disproved the claim that a compatible private mechanism was unavailable + and passed the warm latency threshold on one detailed Settings screen. +- The overall verdict is therefore inconclusive. Public AX remains NO-GO; the idb-style persistent + guest bridge is GO for full-corpus evaluation, not yet GO for production. + +## Next interface boundary + +- Adapt the idb-style persistent guest reader behind the #2190 acquisition boundary and run every + remaining #2192 state and screen cell while preserving raw facts. + +## Production boundary + +- No production backend selection, fallback, runner-demand, open/relaunch, proxy, XCTest interaction, or public CLI changes were made. +- Production routing remains blocked until the guest bridge passes the full correctness, lifecycle, + and latency corpus. The original zero-cell private result must not be used to close #2192. diff --git a/package.json b/package.json index 03faecdba..c609cd6f8 100644 --- a/package.json +++ b/package.json @@ -106,6 +106,7 @@ "package:android-ime-helper": "AGENT_DEVICE_ANDROID_HELPER=ime sh ./scripts/package-android-helper.sh $(node -p \"require('./package.json').version\") .tmp/android-ime-helper", "package:android-ime-helper:npm": "rm -rf android/ime-helper/dist && AGENT_DEVICE_ANDROID_HELPER=ime sh ./scripts/package-android-helper.sh $(node -p \"require('./package.json').version\") android/ime-helper/dist", "build:macos-helper": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift build -c release --package-path apple/macos-helper", + "build:ios-ax-bridge-spike": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift build -c release --package-path scripts/ios-ax-bridge-spike/swift --product agent-device-ios-ax-bridge-spike", "build:macos-helper:clean": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift package --package-path apple/macos-helper clean && pnpm build:macos-helper", "build:package": "pnpm build && pnpm build:xcuitest:ios && pnpm build:xcuitest:macos && pnpm build:xcuitest:tvos && pnpm build:xcuitest:visionos && pnpm build:macos-helper:clean && pnpm package:apple-runner:npm && pnpm build:android", "package:npm": "pnpm build:package && pnpm check:package", @@ -120,6 +121,7 @@ "size": "node scripts/size-report.mjs", "perf": "node --experimental-strip-types scripts/perf/run.ts", "bench:ios-snapshot": "node --experimental-strip-types scripts/ios-snapshot-benchmark/run.ts", + "bench:ios-ax-bridge": "node --experimental-strip-types scripts/ios-ax-bridge-spike/run.ts", "bench:ios-snapshot:deep-button": "node --experimental-strip-types scripts/ios-snapshot-benchmark/deep-button.ts", "mutation:run": "node --experimental-strip-types scripts/mutation/run.ts", "mutation:check": "node --experimental-strip-types scripts/mutation/run.ts --no-run", diff --git a/scripts/ios-ax-bridge-spike/README.md b/scripts/ios-ax-bridge-spike/README.md new file mode 100644 index 000000000..a5cd89bd9 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/README.md @@ -0,0 +1,34 @@ +# iOS Simulator AX bridge spike + +This bounded harness supplies the decision evidence for [#2192](https://github.com/callstack/agent-device/issues/2192). It compares host-side public macOS AX, an explicitly configured private CoreSimulator AX tool, and the #2189 XCTest control baseline behind one framed acquisition adapter. It does not select a production backend or change daemon, runner, open, relaunch, proxy, interaction, or public CLI behavior. + +Build the repository and the repository-only spike helper first: + +```sh +pnpm install --frozen-lockfile +pnpm build +pnpm build:ios-ax-bridge-spike +``` + +The rejected helper remains under this spike tooling package so it is reproducible without entering +the distributed `apple/macos-helper` package or npm artifact. + +Use a newly created, task-owned iOS Simulator. Xcode 27 hosts should target the DeviceHub process explicitly when using the public candidate: + +```sh +pnpm bench:ios-ax-bridge -- \ + --udid SIMULATOR_UDID \ + --target-process-id DEVICEHUB_PID \ + --candidate public-macos-ax,private-coresimulator-ax,xctest-control \ + --state cold-cold,cold,warm,relaunch \ + --screen quiet,list,nested-scroll,alert,system-surface,xctest-stress \ + --samples 20 \ + --apply-preferences \ + --out .tmp/ios-ax-bridge-spike.v1.json.gz +``` + +The default candidate set, state set, screen set, and sample minimums come from the #2189 benchmark definitions. Each request carries an optional expected target generation and fixed request, response, node-count, traversal-depth, CPU, memory, and duration bounds; each successful response reports the observed generation. The native helper sends one newline-delimited response per request and writes diagnostics to stderr only. Every sample keeps resource metrics and target status; each cell keeps one raw-tree exemplar with viewport, lineage, truncation, residue, and bounded diagnostics, plus separate prototype presentation measurements. + +`--apply-preferences` is the only way the experiment edits Simulator preference plists. The Simulator must be shutdown; the harness records exact plist hashes and targeted key changes, then restores the original bytes before reporting. The keys are not production defaults. Omit `--private-tool` unless a disposable, compatible private mechanism is being tested; the harness never invents a private fallback. + +The harness fails closed. It reports `NO-GO` when a candidate is unsupported, unavailable, unreadable, stale, over a bound, below the sample minimum, or when crash/timeout/cancellation recovery is not typed and recovered. It stops before reporting timings if the fixture app cannot be prepared deterministically. The adjacent gzipped JSON and readable Markdown report are the decision artifact; no production route should be implemented from a `NO-GO` run. diff --git a/scripts/ios-ax-bridge-spike/adapter.test.ts b/scripts/ios-ax-bridge-spike/adapter.test.ts new file mode 100644 index 000000000..c5fc1e27c --- /dev/null +++ b/scripts/ios-ax-bridge-spike/adapter.test.ts @@ -0,0 +1,45 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { test } from 'vitest'; +import { defaultPublicMacOsAxHelperPath, readControlSnapshot } from './adapter.ts'; + +test('public AX adapter resolves the SwiftPM release product', () => { + assert.equal( + defaultPublicMacOsAxHelperPath('/repo'), + '/repo/scripts/ios-ax-bridge-spike/swift/.build/release/agent-device-ios-ax-bridge-spike', + ); +}); + +test('spike executable stays outside the distributed macOS helper package', () => { + const manifest = fs.readFileSync( + new URL('../../apple/macos-helper/Package.swift', import.meta.url), + 'utf8', + ); + + assert.doesNotMatch(manifest, /AgentDeviceIosAxBridgeSpike|agent-device-ios-ax-bridge-spike/); +}); + +test('control mapping preserves the producer raw node type', () => { + const result = readControlSnapshot({ + data: { + results: [ + { + data: { + snapshot: { + nodes: [ + { + index: 7, + type: 'XCUIElementTypeButton', + role: 'AXButton', + }, + ], + }, + }, + }, + ], + }, + }); + + assert.equal(result?.nodes[0]?.type, 'XCUIElementTypeButton'); + assert.equal(result?.nodes[0]?.role, 'AXButton'); +}); diff --git a/scripts/ios-ax-bridge-spike/adapter.ts b/scripts/ios-ax-bridge-spike/adapter.ts new file mode 100644 index 000000000..d475ef1a0 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/adapter.ts @@ -0,0 +1,329 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { + classifyFailure, + snapshotFixture, + type CliContext, + type CliResult, +} from '../ios-snapshot-benchmark/command.ts'; +import { runFramedBatch, type FramedProcessSpec } from './framed-process.ts'; +import { DEFAULT_SPIKE_LIMITS, validateRawAcquisition } from './limits.ts'; +import { failureResponse } from './protocol.ts'; +import type { + CandidateId, + RawAcquiredNode, + ResourceLimits, + SpikeRequest, + SpikeResponse, +} from './types.ts'; + +export type AcquisitionAdapter = Readonly<{ + candidate: CandidateId; + acquireBatch( + requests: readonly SpikeRequest[], + options?: Readonly<{ signal?: AbortSignal }>, + ): Promise; +}>; + +export type AcquisitionBatchResult = Readonly<{ + responses: readonly SpikeResponse[]; + stderr: string; +}>; + +export type AdapterOptions = Readonly<{ + repoRoot: string; + privateTool?: string; + helperPath?: string; + limits?: ResourceLimits; +}>; + +export function createPublicMacOsAxAdapter(options: AdapterOptions): AcquisitionAdapter { + const limits = options.limits ?? DEFAULT_SPIKE_LIMITS; + const helperPath = options.helperPath ?? defaultPublicMacOsAxHelperPath(options.repoRoot); + return framedAdapter('public-macos-ax', helperPath, limits); +} + +export function defaultPublicMacOsAxHelperPath(repoRoot: string): string { + return path.join( + repoRoot, + 'scripts', + 'ios-ax-bridge-spike', + 'swift', + '.build', + 'release', + 'agent-device-ios-ax-bridge-spike', + ); +} + +export function createPrivateCoreSimulatorAxAdapter(options: AdapterOptions): AcquisitionAdapter { + const limits = options.limits ?? DEFAULT_SPIKE_LIMITS; + if (!options.privateTool || !fs.existsSync(options.privateTool)) { + return unsupportedAdapter('private-coresimulator-ax', 'private-tool-unavailable', limits); + } + return framedAdapter('private-coresimulator-ax', options.privateTool, limits); +} + +export function createXCTestControlAdapter( + contextFor: (request: SpikeRequest) => CliContext, +): AcquisitionAdapter { + return { + candidate: 'xctest-control', + async acquireBatch(requests) { + return { + responses: requests.map((request) => { + const started = performance.now(); + return controlResponse( + request, + snapshotFixture(contextFor(request)), + performance.now() - started, + ); + }), + stderr: '', + }; + }, + }; +} + +function framedAdapter( + candidate: Exclude, + file: string, + limits: ResourceLimits, +): AcquisitionAdapter { + const spec: FramedProcessSpec = { file }; + return { + candidate, + async acquireBatch(requests, options = {}) { + if (!fs.existsSync(file)) { + return { + responses: requests.map((request) => + failureResponse(request, { + kind: 'unsupported-mechanism', + code: 'adapter-binary-unavailable', + }), + ), + stderr: '', + }; + } + const result = await runFramedBatch(spec, requests, { ...options, limits }); + return result; + }, + }; +} + +function unsupportedAdapter( + candidate: Exclude, + code: string, + limits: ResourceLimits, +): AcquisitionAdapter { + return { + candidate, + async acquireBatch(requests) { + return { + responses: requests.map((request) => + failureResponse( + request, + { kind: 'unsupported-mechanism', code }, + { + requestBytes: Buffer.byteLength(JSON.stringify({ ...request, limits })) + 1, + }, + ), + ), + stderr: '', + }; + }, + }; +} + +function controlResponse( + request: SpikeRequest, + result: CliResult, + durationMs: number, +): SpikeResponse { + const snapshot = readControlSnapshot(result.payload); + return result.ok && snapshot + ? successfulControlResponse(request, snapshot, result, durationMs) + : failedControlResponse(request, result); +} + +function successfulControlResponse( + request: SpikeRequest, + snapshot: ControlSnapshot, + result: CliResult, + durationMs: number, +): SpikeResponse { + const acquisition = { + targetId: `simulator:${request.simulatorUdid}`, + targetGeneration: snapshot.targetGeneration, + nodes: snapshot.nodes, + viewport: snapshot.viewport, + truncated: snapshot.truncated, + residue: [{ kind: 'missing-viewport', reason: 'not-provided' }], + } as const; + const validated = validateRawAcquisition(acquisition, request.limits); + if (!validated.ok) { + return failureResponse(request, { kind: 'malformed-tree', code: validated.code }); + } + return { + version: 1, + id: request.id, + candidate: request.candidate, + ok: true, + acquisition, + metrics: { + requestBytes: 0, + responseBytes: Buffer.byteLength(result.stdout), + nodeCount: acquisition.nodes.length, + maxTraversalDepth: validated.maxTraversalDepth, + cpuMs: null, + memoryBytes: null, + durationMs, + }, + }; +} + +function failedControlResponse(request: SpikeRequest, result: CliResult): SpikeResponse { + const failure = classifyFailure(result.payload, result); + return failureResponse( + request, + { + kind: controlFailureKind(failure.category), + ...(failure.code ? { code: failure.code } : {}), + }, + { + responseBytes: Buffer.byteLength(result.stdout), + durationMs: result.wallClockMs, + }, + ); +} + +function controlFailureKind( + category: string, +): 'timeout' | 'stale-generation' | 'transport-failure' { + if (category === 'timeout') return 'timeout'; + if (category === 'stale-generation') return 'stale-generation'; + return 'transport-failure'; +} + +export type ControlSnapshot = Readonly<{ + nodes: RawAcquiredNode[]; + targetGeneration: string | null; + truncated: boolean; + viewport: { kind: 'missing'; reason: 'not-provided' }; +}>; + +export function readControlSnapshot(value: unknown): ControlSnapshot | undefined { + const snapshot = findSnapshotRecord(value); + if (!snapshot || !Array.isArray(snapshot.nodes)) return undefined; + const nodes = snapshot.nodes.flatMap((rawNode) => toRawNode(rawNode)); + return { + nodes, + targetGeneration: readGeneration(snapshot), + truncated: snapshot.truncated === true, + viewport: { kind: 'missing', reason: 'not-provided' }, + }; +} + +function findSnapshotRecord(value: unknown): Record | undefined { + const root = record(value); + return root ? snapshotFromRoot(root) : undefined; +} + +function snapshotFromRoot(root: Record): Record | undefined { + const stepData = firstBatchStep(record(root.data)); + return snapshotFromStep(stepData); +} + +function snapshotFromStep( + stepData: Record | undefined, +): Record | undefined { + if (!stepData) return undefined; + return record(stepData.snapshot) ?? stepData; +} + +function firstBatchStep( + data: Record | undefined, +): Record | undefined { + if (!data) return undefined; + const results = data.results; + return Array.isArray(results) ? firstResultData(results, data) : data; +} + +function firstResultData( + results: readonly unknown[], + fallback: Record, +): Record { + const first = record(results[0]); + return record(first?.data) ?? fallback; +} + +function toRawNode(value: unknown): RawAcquiredNode[] { + const node = record(value); + if (!node || typeof node.index !== 'number') return []; + return [ + { + id: String(node.index), + ...rawNodeFacts(node), + }, + ]; +} + +function rawNodeFacts(node: Record): Partial { + return { + ...optionalParent(node.parentIndex), + ...optionalString(node, 'type'), + ...optionalString(node, 'role'), + ...optionalString(node, 'subrole'), + ...optionalString(node, 'label'), + ...optionalString(node, 'value'), + ...optionalString(node, 'identifier'), + ...optionalRect(node.rect), + ...optionalBoolean(node, 'enabled'), + ...optionalBoolean(node, 'selected'), + ...optionalBoolean(node, 'focused'), + }; +} + +function optionalParent(value: unknown): Partial { + return typeof value === 'number' ? { parentId: String(value) } : {}; +} + +function readGeneration(snapshot: Record): string | null { + const value = snapshot.refsGeneration ?? snapshot.targetGeneration; + return typeof value === 'string' || typeof value === 'number' ? String(value) : null; +} + +function optionalString( + recordValue: Record, + key: 'type' | 'role' | 'subrole' | 'label' | 'value' | 'identifier', +): Partial { + return typeof recordValue[key] === 'string' ? { [key]: recordValue[key] } : {}; +} + +function optionalBoolean( + recordValue: Record, + key: 'enabled' | 'selected' | 'focused', +): Partial { + return typeof recordValue[key] === 'boolean' ? { [key]: recordValue[key] } : {}; +} + +function optionalRect(value: unknown): Partial { + const rect = record(value); + if (!rect || !['x', 'y', 'width', 'height'].every((key) => typeof rect[key] === 'number')) { + return {}; + } + return { + frame: { + x: rect.x as number, + y: rect.y as number, + width: rect.width as number, + height: rect.height as number, + }, + }; +} + +function record(value: unknown): Record | undefined { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined; +} diff --git a/scripts/ios-ax-bridge-spike/config.ts b/scripts/ios-ax-bridge-spike/config.ts new file mode 100644 index 000000000..a71f569ba --- /dev/null +++ b/scripts/ios-ax-bridge-spike/config.ts @@ -0,0 +1,235 @@ +import path from 'node:path'; +import { + parseLocalStates, + parseSampleCount, + parseScreenIds, +} from '../ios-snapshot-benchmark/definitions.ts'; +import { resolveRepoRoot } from '../ios-snapshot-benchmark/host.ts'; +import { + assertBenchmarkOwner, + assertOwnedDerivedPath, + createBenchmarkStateRoot, +} from '../ios-snapshot-benchmark/state-ownership.ts'; +import type { CandidateId, ResourceLimits } from './types.ts'; +import { DEFAULT_SPIKE_LIMITS } from './limits.ts'; + +class SpikeConfigurationError extends Error { + constructor(message: string) { + super(message); + this.name = 'SpikeConfigurationError'; + } +} + +export type SpikeConfig = Readonly<{ + repoRoot: string; + udid: string; + appBundleId: string; + targetWindowName?: string; + targetProcessId?: number; + privateTool?: string; + helperPath?: string; + stateDir: string; + derivedPath: string; + outputPath: string; + screens: ReturnType; + states: ReturnType; + samples: number; + candidates: CandidateId[]; + limits: ResourceLimits; + applyPreferences: boolean; + keepDevice: boolean; +}>; + +const CANDIDATES: readonly CandidateId[] = [ + 'public-macos-ax', + 'private-coresimulator-ax', + 'xctest-control', +]; +const BOOLEAN_FLAGS = new Set(['--apply-preferences', '--keep-device']); +const VALUE_FLAGS = new Set([ + '--udid', + '--app-bundle-id', + '--target-process-id', + '--private-tool', + '--helper-path', + '--state-dir', + '--derived-path', + '--out', + '--screen', + '--state', + '--samples', + '--candidate', +]); + +export function parseConfig(argv: readonly string[]): SpikeConfig { + const parsed = parseArguments(argv[0] === '--' ? argv.slice(1) : argv); + const states = parseStates(parsed.values.get('--state')); + const screens = parseScreens(parsed.values.get('--screen')); + const candidates = parseCandidates(parsed.values.get('--candidate')); + const samples = parseSamples(parsed.values.get('--samples'), states); + const { stateDir, derivedPath } = resolveOwnedStatePaths(parsed.values); + return { + repoRoot: resolveRepoRoot(), + udid: required(parsed.values, '--udid'), + appBundleId: parsed.values.get('--app-bundle-id') ?? 'com.apple.dt.Devices', + ...optionalNumber(parsed.values.get('--target-process-id')), + ...optionalValue(parsed.values.get('--private-tool'), 'privateTool'), + ...optionalValue(parsed.values.get('--helper-path'), 'helperPath'), + stateDir, + derivedPath, + outputPath: resolvePath( + parsed.values.get('--out'), + path.join(stateDir, 'ios-simulator-ax-bridge-spike.v1.json.gz'), + ), + screens, + states, + samples, + candidates, + limits: DEFAULT_SPIKE_LIMITS, + applyPreferences: parsed.booleans.has('--apply-preferences'), + keepDevice: parsed.booleans.has('--keep-device'), + }; +} + +function resolveOwnedStatePaths(values: ReadonlyMap): { + stateDir: string; + derivedPath: string; +} { + const stateDir = values.has('--state-dir') + ? resolvePath(values.get('--state-dir'), '') + : createBenchmarkStateRoot(); + const derivedPath = resolvePath( + values.get('--derived-path'), + path.join(stateDir, 'derived-data'), + ); + try { + assertBenchmarkOwner(stateDir); + assertOwnedDerivedPath(derivedPath, stateDir); + } catch (error) { + throw new SpikeConfigurationError(error instanceof Error ? error.message : String(error)); + } + return { stateDir, derivedPath }; +} + +function parseArguments(argv: readonly string[]): { + values: Map; + booleans: Set; +} { + exitAfterHelp(argv); + const values = new Map(); + const booleans = new Set(); + for (let index = 0; index < argv.length; index += 1) { + const argument = parseArgument(argv, index); + recordArgument(argument, values, booleans); + index = argument.nextIndex; + } + return { values, booleans }; +} + +function exitAfterHelp(argv: readonly string[]): void { + if (!argv.includes('--help') && !argv.includes('-h')) return; + printHelp(); + process.exit(0); +} + +function recordArgument( + argument: { flag: string; value?: string }, + values: Map, + booleans: Set, +): void { + if (argument.value === undefined) booleans.add(argument.flag); + else values.set(argument.flag, argument.value); +} + +function parseArgument( + argv: readonly string[], + index: number, +): { flag: string; value?: string; nextIndex: number } { + const flag = argv[index]; + if (flag === undefined) throw new SpikeConfigurationError('Missing option.'); + if (BOOLEAN_FLAGS.has(flag)) return { flag, nextIndex: index }; + return parseValueArgument(flag, argv[index + 1], index); +} + +function parseValueArgument( + flag: string, + value: string | undefined, + index: number, +): { flag: string; value: string; nextIndex: number } { + if (!VALUE_FLAGS.has(flag)) throw new SpikeConfigurationError(`Unknown option: ${flag}`); + if (!value || value.startsWith('--')) { + throw new SpikeConfigurationError(`${flag} requires a value.`); + } + return { flag, value, nextIndex: index + 1 }; +} + +function parseStates(value: string | undefined): SpikeConfig['states'] { + try { + return parseLocalStates(value); + } catch (error) { + throw new SpikeConfigurationError(error instanceof Error ? error.message : String(error)); + } +} + +function parseScreens(value: string | undefined): SpikeConfig['screens'] { + try { + return parseScreenIds(value); + } catch (error) { + throw new SpikeConfigurationError(error instanceof Error ? error.message : String(error)); + } +} + +function parseSamples(value: string | undefined, states: SpikeConfig['states']): number { + try { + return parseSampleCount(value, states); + } catch (error) { + throw new SpikeConfigurationError(error instanceof Error ? error.message : String(error)); + } +} + +function parseCandidates(value: string | undefined): CandidateId[] { + const candidates = (value ?? CANDIDATES.join(',')) + .split(',') + .map((candidate) => candidate.trim()) + .filter(Boolean); + const unknown = candidates.filter((candidate) => !CANDIDATES.includes(candidate as CandidateId)); + if (unknown.length > 0) + throw new SpikeConfigurationError(`Unknown --candidate value: ${unknown.join(', ')}`); + if (candidates.length === 0) + throw new SpikeConfigurationError('--candidate requires at least one value.'); + return [...new Set(candidates)] as CandidateId[]; +} + +function required(values: Map, flag: string): string { + const value = values.get(flag); + if (!value) throw new SpikeConfigurationError(`${flag} is required for a reproducible run.`); + return value; +} + +function optionalValue( + value: string | undefined, + key: 'privateTool' | 'helperPath', +): { privateTool: string } | { helperPath: string } | Record { + return value === undefined + ? {} + : ({ [key]: path.resolve(value) } as { privateTool: string } | { helperPath: string }); +} + +function optionalNumber(value: string | undefined): { targetProcessId?: number } { + if (value === undefined) return {}; + const number = Number(value); + if (!Number.isInteger(number) || number < 1) { + throw new SpikeConfigurationError('--target-process-id must be a positive integer.'); + } + return { targetProcessId: number }; +} + +function resolvePath(value: string | undefined, fallback: string): string { + return path.resolve(value ?? fallback); +} + +function printHelp(): void { + process.stdout.write( + `Usage: pnpm bench:ios-ax-bridge -- [options]\n\nRequired:\n --udid \n\nOptions:\n --candidate public-macos-ax, private-coresimulator-ax, xctest-control\n --state #2189 state names\n --screen #2189 fixture names\n --samples #2189 minimums: cold 10, warm/relaunch 20\n --apply-preferences apply task-owned preboot AX preference experiment\n --private-tool optional private-tool framed adapter\n --target-process-id DeviceHub/Simulator host process to inspect\n --out raw JSON report path\n --keep-device leave the dedicated Simulator shutdown/boot state unchanged\n`, + ); +} diff --git a/scripts/ios-ax-bridge-spike/corpus-coverage.ts b/scripts/ios-ax-bridge-spike/corpus-coverage.ts new file mode 100644 index 000000000..b53ab3a02 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/corpus-coverage.ts @@ -0,0 +1,32 @@ +import { LOCAL_STATES, SCREEN_FIXTURES } from '../ios-snapshot-benchmark/definitions.ts'; +import type { LocalState, ScreenId } from '../ios-snapshot-benchmark/types.ts'; +import type { CandidateId, SpikeReport } from './types.ts'; + +const FULL_STATES = LOCAL_STATES; +const FULL_SCREENS = SCREEN_FIXTURES.map((fixture) => fixture.id); + +export function corpusCoverage( + states: readonly LocalState[], + screens: readonly ScreenId[], + cells: SpikeReport['cells'], + candidates: readonly CandidateId[], +): SpikeReport['corpusCoverage'] { + const fullRequested = + FULL_STATES.every((state) => states.includes(state)) && + FULL_SCREENS.every((screen) => screens.includes(screen)); + if (!fullRequested || cells.length === 0) return 'decisive-early-stop'; + const fullProduced = candidates.every((candidate) => + FULL_STATES.every((state) => + FULL_SCREENS.every((screen) => + cells.some( + (cell) => + cell.candidate === candidate && + cell.state === state && + cell.screen === screen && + cell.acquisitionSamples.length >= cell.sampleMinimum, + ), + ), + ), + ); + return fullProduced ? 'full' : 'decisive-early-stop'; +} diff --git a/scripts/ios-ax-bridge-spike/decision.test.ts b/scripts/ios-ax-bridge-spike/decision.test.ts new file mode 100644 index 000000000..e9a1c6bcf --- /dev/null +++ b/scripts/ios-ax-bridge-spike/decision.test.ts @@ -0,0 +1,132 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { decideSpike } from './decision.ts'; +import { DEFAULT_SPIKE_LIMITS } from './limits.ts'; +import type { CandidateId, LifecycleEvidence, PreferenceEvidence, SpikeCell } from './types.ts'; +import type { LocalState, ScreenId } from '../ios-snapshot-benchmark/types.ts'; + +const lifecycle: LifecycleEvidence = { + source: 'framed-protocol-fixture', + crash: { failure: 'process-crash', recovered: true }, + timeout: { failure: 'timeout', recovered: true }, + cancellation: { failure: 'cancelled', recovered: true }, + staleGeneration: { failure: 'stale-generation', recovered: true }, +}; + +const preferences: PreferenceEvidence = { + applied: true, + restored: true, + fixtureLaunchCompatible: true, + simulatorStateBefore: 'Shutdown', + diffs: [], +}; + +test('fails closed when a bridge has no readable corpus cells', () => { + const result = decideSpike([], lifecycle, preferences, DEFAULT_SPIKE_LIMITS, 'completed', [ + { + candidate: 'public-macos-ax', + failure: { kind: 'unsupported-mechanism', code: 'permission' }, + }, + ]); + assert.equal(result.decision, 'NO-GO'); + assert.ok(result.reasons.some((reason) => reason.includes('unsupported-mechanism/permission'))); +}); + +test('does not let failed samples contribute fabricated zero latency', () => { + const cell = readableCell('public-macos-ax', 'warm', 'list'); + const failed = { + ...cell.acquisitionSamples[0]!, + ok: false, + firstTree: 'not-observed' as const, + firstLookMs: 0, + metrics: { ...cell.acquisitionSamples[0]!.metrics!, durationMs: 0 }, + failure: { kind: 'timeout' as const, code: 'batch-duration-limit' }, + }; + const result = decideSpike( + [{ ...cell, acquisitionSamples: [failed, ...cell.acquisitionSamples.slice(1)] }], + lifecycle, + preferences, + DEFAULT_SPIKE_LIMITS, + 'completed', + [{ candidate: 'public-macos-ax' }], + ); + assert.equal(result.decision, 'NO-GO'); + assert.ok(result.reasons.some((reason) => reason.includes('duration bound'))); + assert.ok(result.reasons.some((reason) => reason.includes('not produce 20 readable samples'))); +}); + +test('reports a decisive partial corpus failure instead of replacing it with completeness', () => { + const cell = readableCell('public-macos-ax', 'warm', 'list'); + const slow = { + ...cell, + acquisitionSamples: cell.acquisitionSamples.map((sample) => ({ + ...sample, + metrics: { ...sample.metrics!, durationMs: 1_000 }, + })), + }; + const result = decideSpike([slow], lifecycle, preferences, DEFAULT_SPIKE_LIMITS, 'completed', [ + { candidate: 'public-macos-ax' }, + ]); + assert.equal(result.decision, 'NO-GO'); + assert.ok(result.reasons.some((reason) => reason.includes('warm/list acquisition'))); + assert.equal( + result.reasons.some((reason) => reason.includes('required corpus')), + false, + ); +}); + +test('selects one complete viable bridge without requiring every candidate to pass', () => { + const states: LocalState[] = ['cold-cold', 'cold', 'warm', 'relaunch']; + const screens: ScreenId[] = [ + 'quiet', + 'list', + 'nested-scroll', + 'alert', + 'system-surface', + 'xctest-stress', + ]; + const cells = states.flatMap((state) => + screens.map((screen) => readableCell('public-macos-ax', state, screen)), + ); + const result = decideSpike(cells, lifecycle, preferences, DEFAULT_SPIKE_LIMITS, 'completed', [ + { candidate: 'public-macos-ax' }, + { + candidate: 'private-coresimulator-ax', + failure: { kind: 'unsupported-mechanism', code: 'private-tool-unavailable' }, + }, + ]); + assert.deepEqual(result, { decision: 'GO', reasons: [] }); +}); + +function readableCell(candidate: CandidateId, state: LocalState, screen: ScreenId): SpikeCell { + const sampleMinimum = state === 'cold' || state === 'cold-cold' ? 10 : 20; + return { + candidate, + state, + screen, + sampleMinimum, + acquisitionSamples: Array.from({ length: sampleMinimum }, (_, index) => ({ + index: index + 1, + candidate, + state, + screen, + startedAt: '2026-09-01T00:00:00.000Z', + finishedAt: '2026-09-01T00:00:00.010Z', + operation: 'acquisition' as const, + wallClockMs: 10, + firstLookMs: 100, + firstTree: 'readable' as const, + ok: true, + metrics: { + requestBytes: 1, + responseBytes: 1, + nodeCount: 1, + maxTraversalDepth: 0, + cpuMs: 1, + memoryBytes: 1, + durationMs: 10, + }, + })), + presentationSamples: [], + }; +} diff --git a/scripts/ios-ax-bridge-spike/decision.ts b/scripts/ios-ax-bridge-spike/decision.ts new file mode 100644 index 000000000..0a0af7423 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/decision.ts @@ -0,0 +1,273 @@ +import type { + CandidateId, + LifecycleEvidence, + PreferenceEvidence, + ResourceLimits, + SpikeCell, +} from './types.ts'; +import { LOCAL_STATES, SCREEN_FIXTURES } from '../ios-snapshot-benchmark/definitions.ts'; + +export function decideSpike( + cells: readonly SpikeCell[], + lifecycle: LifecycleEvidence, + preferences: PreferenceEvidence, + limits: ResourceLimits, + status: 'completed' | 'stopped' = 'completed', + protocolProbes: readonly { + candidate: CandidateId; + failure?: { kind: string; code?: string }; + }[] = [], +): { decision: 'GO' | 'NO-GO'; reasons: string[] } { + const reasons = [ + ...statusReasons(status), + ...preferenceReasons(preferences), + ...bridgeRouteReasons(cells, protocolProbes, limits), + ...lifecycleReasons(lifecycle), + ]; + const uniqueReasons = [...new Set(reasons)]; + return { + decision: uniqueReasons.length === 0 ? 'GO' : 'NO-GO', + reasons: uniqueReasons, + }; +} + +function statusReasons(status: 'completed' | 'stopped'): string[] { + return status === 'stopped' ? ['The live run stopped before the full corpus completed.'] : []; +} + +function preferenceReasons(preferences: PreferenceEvidence): string[] { + if (!preferences.applied) + return ['The required task-owned Simulator preference experiment was not run.']; + if (!preferences.restored) + return ['The task-owned Simulator preference experiment was not restored.']; + if (preferences.fixtureLaunchCompatible === false) + return ['The task-owned Simulator preference experiment prevented the fixture from launching.']; + return []; +} + +function bridgeRouteReasons( + cells: readonly SpikeCell[], + probes: readonly { candidate: CandidateId; failure?: { kind: string; code?: string } }[], + limits: ResourceLimits, +): string[] { + const evaluated = (['public-macos-ax', 'private-coresimulator-ax'] as const).flatMap( + (candidate) => { + const candidateCells = cells.filter((cell) => cell.candidate === candidate); + const candidateProbes = probes.filter((probe) => probe.candidate === candidate); + if (candidateCells.length === 0 && candidateProbes.length === 0) return []; + const coreReasons = [ + ...probeReasons(candidateProbes), + ...candidateDecisionReasons(candidateCells, limits), + ]; + return [ + { + candidate, + reasons: + coreReasons.length > 0 + ? coreReasons + : candidateCompletenessReasons(candidate, candidateCells), + }, + ]; + }, + ); + if (evaluated.some((candidate) => candidate.reasons.length === 0)) return []; + if (evaluated.length === 0) return ['No bridge candidate produced evidence.']; + return evaluated.flatMap((candidate) => candidate.reasons); +} + +function probeReasons( + probes: readonly { candidate: CandidateId; failure?: { kind: string; code?: string } }[], +): string[] { + return probes.flatMap((probe) => + probe.failure && !isReadinessProbeFailure(probe.failure.code) + ? [ + `${probe.candidate} protocol probe returned ${probe.failure.kind}/${probe.failure.code ?? 'no-code'}.`, + ] + : [], + ); +} + +function isReadinessProbeFailure(code: string | undefined): boolean { + return [ + 'target-application-unavailable', + 'target-has-no-accessibility-windows', + 'target-simulator-window-unavailable', + 'target-simulator-content-unavailable', + ].includes(code ?? ''); +} + +const REQUIRED_STATES = LOCAL_STATES; +const REQUIRED_SCREENS = SCREEN_FIXTURES.map((fixture) => fixture.id); + +function candidateCompletenessReasons( + candidate: Exclude, + cells: readonly SpikeCell[], +): string[] { + if (cells.length === 0) return [`${candidate} produced no cells.`]; + const observed = new Set(cells.map((cell) => `${cell.state}/${cell.screen}`)); + const missing = REQUIRED_STATES.flatMap((state) => + REQUIRED_SCREENS.flatMap((screen) => + observed.has(`${state}/${screen}`) ? [] : [`${state}/${screen}`], + ), + ); + if (missing.length === 0) return []; + return [`${candidate} did not complete the required corpus (${missing.length} cells missing).`]; +} + +function lifecycleReasons(lifecycle: LifecycleEvidence): string[] { + const checks = [ + ['process crash', lifecycle.crash, 'process-crash'], + ['timeout', lifecycle.timeout, 'timeout'], + ['cancellation', lifecycle.cancellation, 'cancelled'], + ['stale target-generation', lifecycle.staleGeneration, 'stale-generation'], + ] as const; + const reasons: string[] = []; + for (const [label, result, expected] of checks) { + if (result.failure === expected && result.recovered) continue; + reasons.push( + `Framed ${label} recovery did not produce the required typed result and recovery.`, + ); + } + return reasons; +} + +function candidateDecisionReasons(cells: readonly SpikeCell[], limits: ResourceLimits): string[] { + const reasons: string[] = []; + for (const cell of cells) { + reasons.push(...sampleShapeReasons(cell)); + reasons.push(...resourceReasons(cell, limits)); + reasons.push(...latencyReasons(cell)); + } + reasons.push(...candidateAvailabilityReasons(cells)); + return [...new Set(reasons)]; +} + +function sampleShapeReasons(cell: SpikeCell): string[] { + const samples = cell.acquisitionSamples; + const successful = samples.filter((sample) => sample.ok && sample.firstTree === 'readable'); + const reasons: string[] = []; + if (successful.length < cell.sampleMinimum) { + reasons.push( + `${cell.candidate} ${cell.state}/${cell.screen} did not produce ${cell.sampleMinimum} readable samples.`, + ); + } + if ( + samples.some((sample) => ['unreadable', 'empty', 'not-observed'].includes(sample.firstTree)) + ) { + reasons.push( + `${cell.candidate} ${cell.state}/${cell.screen} has unreadable or empty first-tree evidence.`, + ); + } + if (samples.some((sample) => sample.failure?.kind === 'stale-generation')) { + reasons.push( + `${cell.candidate} ${cell.state}/${cell.screen} has stale-generation acquisition evidence.`, + ); + } + return reasons; +} + +function resourceReasons(cell: SpikeCell, limits: ResourceLimits): string[] { + const checks = [ + [ + 'duration', + limits.maxDurationMs, + (sample: SpikeCell['acquisitionSamples'][number]) => sample.metrics?.durationMs, + ], + [ + 'CPU', + limits.maxCpuMs, + (sample: SpikeCell['acquisitionSamples'][number]) => sample.metrics?.cpuMs, + ], + [ + 'memory', + limits.maxMemoryBytes, + (sample: SpikeCell['acquisitionSamples'][number]) => sample.metrics?.memoryBytes, + ], + ] as const; + const reasons: string[] = []; + if ( + cell.acquisitionSamples.some( + (sample) => + sample.failure?.kind === 'timeout' || sample.failure?.code === 'batch-duration-limit', + ) + ) { + reasons.push(`${cell.candidate} ${cell.state}/${cell.screen} exceeded the duration bound.`); + } + for (const [label, limit, readValue] of checks) { + if (cell.acquisitionSamples.some((sample) => exceedsLimit(readValue(sample), limit))) { + reasons.push(`${cell.candidate} ${cell.state}/${cell.screen} exceeded the ${label} bound.`); + } + } + return reasons; +} + +function latencyReasons(cell: SpikeCell): string[] { + const successful = cell.acquisitionSamples.filter( + (sample) => sample.ok && sample.firstTree === 'readable', + ); + const firstLook = finite(successful.map((sample) => sample.firstLookMs)); + const firstLookTarget = { + 'cold-cold': { limit: 5_000, label: 'cold-cold first look missed the 5 second target.' }, + cold: { limit: 1_500, label: 'cold prepared first look missed the 1.5 second target.' }, + relaunch: { limit: 250, label: 'relaunch first look missed the 250 ms target.' }, + warm: undefined, + }[cell.state]; + const reasons: string[] = []; + if (firstLookTarget && percentile(firstLook, 95) >= firstLookTarget.limit) { + reasons.push(`${cell.candidate} ${firstLookTarget.label}`); + } + const acquisition = finite(successful.map((sample) => sample.metrics?.durationMs)); + if ( + cell.state === 'warm' && + (percentile(acquisition, 50) >= 75 || percentile(acquisition, 95) >= 150) + ) { + reasons.push( + `${cell.candidate} ${cell.state}/${cell.screen} acquisition missed the 75/150 ms target.`, + ); + } + return reasons; +} + +function candidateAvailabilityReasons(cells: readonly SpikeCell[]): string[] { + const reasons: string[] = []; + if (cells.some(hasUnavailablePrivateTool)) { + reasons.push('The private CoreSimulator AX mechanism has no configured tool on this host.'); + } + if (cells.some(hasUnsupportedPublicAx)) { + reasons.push( + 'The public macOS AX mechanism was unsupported or unreadable on the host Simulator surface.', + ); + } + return reasons; +} + +function hasUnavailablePrivateTool(cell: SpikeCell): boolean { + return ( + cell.candidate === 'private-coresimulator-ax' && + cell.acquisitionSamples.some((sample) => sample.failure?.code === 'private-tool-unavailable') + ); +} + +function hasUnsupportedPublicAx(cell: SpikeCell): boolean { + return ( + cell.candidate === 'public-macos-ax' && + cell.acquisitionSamples.some((sample) => sample.failure?.kind === 'unsupported-mechanism') + ); +} + +function exceedsLimit(value: number | null | undefined, limit: number): boolean { + return value !== null && value !== undefined && value > limit; +} + +function finite(values: readonly (number | undefined)[]): number[] { + return values.filter( + (value): value is number => typeof value === 'number' && Number.isFinite(value), + ); +} + +function percentile(values: readonly number[], percentage: number): number { + if (values.length === 0) return Number.POSITIVE_INFINITY; + const sorted = [...values].sort((left, right) => left - right); + const rank = Math.ceil((percentage / 100) * sorted.length); + return sorted[Math.min(sorted.length - 1, Math.max(0, rank - 1))]!; +} diff --git a/scripts/ios-ax-bridge-spike/framed-process.test.ts b/scripts/ios-ax-bridge-spike/framed-process.test.ts new file mode 100644 index 000000000..52dcf77f1 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/framed-process.test.ts @@ -0,0 +1,118 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { runFramedBatch } from './framed-process.ts'; +import { DEFAULT_SPIKE_LIMITS } from './limits.ts'; +import type { SpikeRequest } from './types.ts'; + +function request(id: string): SpikeRequest { + return { + version: 1, + id, + candidate: 'public-macos-ax', + simulatorUdid: '00000000-0000-0000-0000-000000000000', + state: 'warm', + screen: 'quiet', + appBundleId: 'com.apple.dt.Devices', + limits: DEFAULT_SPIKE_LIMITS, + }; +} + +function childScript(mode: 'healthy' | 'delayed' | 'malformed' | 'crash' | 'hang'): { + file: string; + args: string[]; +} { + const payload = JSON.stringify({ + version: 1, + ok: true, + acquisition: { + targetId: 'simulator:test', + targetGeneration: 'generation-1', + nodes: [{ id: 'n0', role: 'AXApplication' }], + viewport: { kind: 'missing', reason: 'not-provided' }, + truncated: false, + residue: [], + }, + metrics: { + requestBytes: 10, + responseBytes: 10, + nodeCount: 1, + maxTraversalDepth: 0, + cpuMs: 1, + memoryBytes: 1, + durationMs: 1, + }, + }); + const malformed = payload.replace('"id":"n0"', '"id":"n0","hittable":true'); + const response = mode === 'malformed' ? malformed : payload; + const script = ` + import process from 'node:process'; + if (${JSON.stringify(mode)} === 'crash') process.exit(17); + if (${JSON.stringify(mode)} === 'hang') { + setInterval(() => {}, 1000); + } else { + let input = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (chunk) => { input += chunk; }); + process.stdin.on('end', async () => { + for (const line of input.split('\\n').filter(Boolean)) { + const request = JSON.parse(line); + if (${JSON.stringify(mode)} === 'delayed') await new Promise((resolve) => setTimeout(resolve, 100)); + process.stdout.write(JSON.stringify({ ...${response}, id: request.id, candidate: request.candidate }) + '\\n'); + } + }); + } + `; + return { file: process.execPath, args: ['--input-type=module', '-e', script] }; +} + +test('uses one framed response per request and keeps diagnostics on stderr', async () => { + const result = await runFramedBatch(childScript('healthy'), [request('one'), request('two')]); + assert.deepEqual( + result.responses.map((response) => response.id), + ['one', 'two'], + ); + assert.equal( + result.responses.every((response) => response.ok), + true, + ); +}); + +test('budgets a framed batch per request rather than timing the whole batch as one request', async () => { + const result = await runFramedBatch( + childScript('delayed'), + [request('one'), request('two'), request('three')], + { limits: { ...DEFAULT_SPIKE_LIMITS, maxDurationMs: 200 } }, + ); + assert.equal( + result.responses.every((response) => response.ok), + true, + ); +}); + +test('classifies malformed trees, crashes, timeouts, and cancellation', async () => { + const malformed = await runFramedBatch(childScript('malformed'), [request('malformed')]); + assert.equal(malformed.responses[0]?.failure?.kind, 'malformed-tree'); + + const crashed = await runFramedBatch(childScript('crash'), [request('crash')]); + assert.equal(crashed.responses[0]?.failure?.kind, 'process-crash'); + + const timeout = await runFramedBatch(childScript('hang'), [request('timeout')], { + limits: { ...DEFAULT_SPIKE_LIMITS, maxDurationMs: 40 }, + }); + assert.equal(timeout.responses[0]?.failure?.kind, 'timeout'); + + const controller = new AbortController(); + const cancellation = runFramedBatch(childScript('hang'), [request('cancel')], { + signal: controller.signal, + limits: { ...DEFAULT_SPIKE_LIMITS, maxDurationMs: 200 }, + }); + setTimeout(() => controller.abort(), 10); + const cancelled = await cancellation; + assert.equal(cancelled.responses[0]?.failure?.kind, 'cancelled'); + + const transport = await runFramedBatch( + { file: '/private/tmp/agent-device-ios-ax-spike-missing-helper' }, + [request('transport')], + ); + assert.equal(transport.responses[0]?.failure?.kind, 'transport-failure'); +}); diff --git a/scripts/ios-ax-bridge-spike/framed-process.ts b/scripts/ios-ax-bridge-spike/framed-process.ts new file mode 100644 index 000000000..935dadf02 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/framed-process.ts @@ -0,0 +1,214 @@ +import { spawn } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { encodeFrame, DEFAULT_SPIKE_LIMITS } from './limits.ts'; +import { failureResponse, parseSpikeResponse } from './protocol.ts'; +import type { ResourceLimits, SpikeFailureKind, SpikeRequest, SpikeResponse } from './types.ts'; + +export type FramedProcessSpec = Readonly<{ + file: string; + args?: readonly string[]; + cwd?: string; + env?: NodeJS.ProcessEnv; +}>; + +export type FramedBatchResult = Readonly<{ + responses: readonly SpikeResponse[]; + stderr: string; +}>; + +export async function runFramedBatch( + spec: FramedProcessSpec, + requests: readonly SpikeRequest[], + options: Readonly<{ signal?: AbortSignal; limits?: ResourceLimits }> = {}, +): Promise { + const limits = options.limits ?? DEFAULT_SPIKE_LIMITS; + const encodedRequests = requests.map((request) => ({ + request, + ...encodeFrame(request), + })); + const oversized = encodedRequests.find(({ bytes }) => bytes > limits.maxRequestBytes); + if (oversized) { + return { + responses: requests.map((request) => + failureResponse( + request, + { kind: 'transport-failure', code: 'request-limit-exceeded' }, + { + requestBytes: + encodedRequests.find((item) => item.request.id === request.id)?.bytes ?? 0, + }, + ), + ), + stderr: '', + }; + } + if (requests.length === 0) return { responses: [], stderr: '' }; + if (options.signal?.aborted) return cancelledBatch(requests, encodedRequests); + + return new Promise((resolve) => { + const child = spawn(spec.file, [...(spec.args ?? [])], { + cwd: spec.cwd, + env: { ...process.env, ...spec.env }, + stdio: ['pipe', 'pipe', 'pipe'], + }); + const responses = new Map(); + const requestById = new Map(requests.map((request) => [request.id, request])); + let stdoutBuffer = Buffer.alloc(0); + let stdoutBytes = 0; + let stderr = ''; + let settled = false; + const batchDurationMs = limits.maxDurationMs * requests.length; + const timer = setTimeout(() => finish('timeout', 'batch-duration-limit'), batchDurationMs); + + const finish = (kind?: SpikeFailureKind, code?: string): void => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + options.signal?.removeEventListener('abort', onAbort); + if (kind) { + for (const request of requests) { + if (!responses.has(request.id)) { + responses.set( + request.id, + failureResponse( + request, + { kind, ...(code ? { code } : {}) }, + { + requestBytes: requestBytesFor(request, encodedRequests), + responseBytes: stdoutBytes, + }, + ), + ); + } + } + } + if (kind === 'timeout' || kind === 'cancelled') terminate(child); + resolve({ + responses: requests.map( + (request) => + responses.get(request.id) ?? + failureResponse( + request, + { kind: 'transport-failure', code: 'missing-response' }, + { + requestBytes: requestBytesFor(request, encodedRequests), + responseBytes: stdoutBytes, + }, + ), + ), + stderr, + }); + }; + const onAbort = (): void => finish('cancelled', 'abort-signal'); + options.signal?.addEventListener('abort', onAbort, { once: true }); + child.stdout.on('data', (chunk: Buffer | string) => { + if (settled) return; + const data = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + stdoutBytes += data.length; + if (stdoutBytes > limits.maxResponseBytes) { + finish('malformed-tree', 'response-limit-exceeded'); + return; + } + stdoutBuffer = Buffer.concat([stdoutBuffer, data]); + consumeLines(); + }); + child.stderr.on('data', (chunk: Buffer | string) => { + if (stderr.length >= 64 * 1024) return; + stderr += (Buffer.isBuffer(chunk) ? chunk.toString('utf8') : chunk).slice( + 0, + 64 * 1024 - stderr.length, + ); + }); + child.on('error', (error: NodeJS.ErrnoException) => { + finish('transport-failure', error.code ?? 'spawn-error'); + }); + child.on('close', (code) => { + if (settled) return; + consumeLines(true); + if (settled) return; + if (code !== 0) { + finish('process-crash', `exit-${code ?? 'unknown'}`); + return; + } + if (responses.size !== requests.length) { + finish('transport-failure', 'missing-response'); + return; + } + finish(); + }); + child.stdin.on('error', () => finish('transport-failure', 'stdin-error')); + child.stdin.end(encodedRequests.map(({ line }) => line).join('')); + + function consumeLines(final = false): void { + let newline = stdoutBuffer.indexOf(0x0a); + while (newline >= 0) { + const line = stdoutBuffer.subarray(0, newline); + stdoutBuffer = stdoutBuffer.subarray(newline + 1); + consumeLine(line); + if (settled) return; + newline = stdoutBuffer.indexOf(0x0a); + } + if (final && stdoutBuffer.length > 0) consumeLine(stdoutBuffer); + } + + function consumeLine(line: Buffer): void { + const text = line.toString('utf8').trim(); + if (text.length === 0) return; + if (line.length > limits.maxResponseBytes) { + finish('malformed-tree', 'frame-limit-exceeded'); + return; + } + let value: unknown; + try { + value = JSON.parse(text); + } catch { + finish('malformed-tree', 'invalid-json'); + return; + } + const id = readId(value); + const request = id ? requestById.get(id) : undefined; + if (!request || responses.has(request.id)) { + finish('malformed-tree', 'response-id-invalid'); + return; + } + responses.set(request.id, parseSpikeResponse(value, request, line.length + 1)); + } + }); +} + +function cancelledBatch( + requests: readonly SpikeRequest[], + encodedRequests: readonly Readonly<{ request: SpikeRequest; bytes: number; line: string }>[], +): FramedBatchResult { + return { + responses: requests.map((request) => + failureResponse( + request, + { kind: 'cancelled', code: 'abort-signal' }, + { + requestBytes: requestBytesFor(request, encodedRequests), + }, + ), + ), + stderr: '', + }; +} + +function requestBytesFor( + request: SpikeRequest, + encodedRequests: readonly Readonly<{ request: SpikeRequest; bytes: number; line: string }>[], +): number { + return encodedRequests.find((item) => item.request.id === request.id)?.bytes ?? 0; +} + +function readId(value: unknown): string | undefined { + if (value && typeof value === 'object' && !Array.isArray(value)) { + const id = (value as Record).id; + return typeof id === 'string' ? id : undefined; + } + return undefined; +} + +function terminate(child: ChildProcess): void { + if (!child.killed) child.kill('SIGTERM'); +} diff --git a/scripts/ios-ax-bridge-spike/lifecycle.test.ts b/scripts/ios-ax-bridge-spike/lifecycle.test.ts new file mode 100644 index 000000000..9254bd99a --- /dev/null +++ b/scripts/ios-ax-bridge-spike/lifecycle.test.ts @@ -0,0 +1,21 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { runLifecycleProbes } from './lifecycle.ts'; + +test('records typed lifecycle failures and recovery probes', async () => { + const evidence = await runLifecycleProbes(); + assert.deepEqual( + { + crash: evidence.crash, + timeout: evidence.timeout, + cancellation: evidence.cancellation, + staleGeneration: evidence.staleGeneration, + }, + { + crash: { failure: 'process-crash', recovered: true }, + timeout: { failure: 'timeout', recovered: true }, + cancellation: { failure: 'cancelled', recovered: true }, + staleGeneration: { failure: 'stale-generation', recovered: true }, + }, + ); +}); diff --git a/scripts/ios-ax-bridge-spike/lifecycle.ts b/scripts/ios-ax-bridge-spike/lifecycle.ts new file mode 100644 index 000000000..ec388e9bf --- /dev/null +++ b/scripts/ios-ax-bridge-spike/lifecycle.ts @@ -0,0 +1,127 @@ +import { runFramedBatch } from './framed-process.ts'; +import { DEFAULT_SPIKE_LIMITS } from './limits.ts'; +import { failureResponse } from './protocol.ts'; +import type { LifecycleEvidence, SpikeRequest, SpikeResponse } from './types.ts'; + +export async function runLifecycleProbes(): Promise { + const request = probeRequest('lifecycle'); + const limits = { ...DEFAULT_SPIKE_LIMITS, maxDurationMs: 1_000 }; + const crash = await runFailureProbe('crash', request, limits); + const timeout = await runFailureProbe('hang', request, limits); + const cancellation = await runCancellationProbe(request, limits); + const staleResponse = failureResponse(request, { + kind: 'stale-generation', + code: 'target-generation-mismatch', + expectedTargetGeneration: 'expected', + observedTargetGeneration: 'observed', + }); + const stale = await runFramedBatch(nodeScript('stale-generation', staleResponse), [request], { + limits, + }); + return { + source: 'framed-protocol-fixture', + crash: { failure: crash.failure, recovered: crash.recovered }, + timeout: { failure: timeout.failure, recovered: timeout.recovered }, + cancellation: { failure: cancellation.failure, recovered: cancellation.recovered }, + staleGeneration: { + failure: stale.responses[0]?.failure?.kind ?? 'transport-failure', + recovered: await runHealthyProbe(request, limits), + }, + }; +} + +function probeRequest(id: string): SpikeRequest { + return { + version: 1, + id, + candidate: 'public-macos-ax', + simulatorUdid: '00000000-0000-0000-0000-000000000000', + state: 'warm', + screen: 'quiet', + appBundleId: 'com.apple.dt.Devices', + limits: DEFAULT_SPIKE_LIMITS, + }; +} + +async function runFailureProbe( + script: string, + request: SpikeRequest, + limits: typeof DEFAULT_SPIKE_LIMITS, +): Promise<{ failure: NonNullable['kind']; recovered: boolean }> { + const failed = await runFramedBatch(nodeScript(script), [request], { limits }); + const failure = failed.responses[0]?.failure?.kind ?? 'transport-failure'; + const recovered = await runHealthyProbe(request, limits); + return { failure, recovered }; +} + +async function runCancellationProbe( + request: SpikeRequest, + limits: typeof DEFAULT_SPIKE_LIMITS, +): Promise<{ failure: NonNullable['kind']; recovered: boolean }> { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 15); + const result = await runFramedBatch(nodeScript('hang'), [request], { + signal: controller.signal, + limits, + }); + clearTimeout(timer); + return { + failure: result.responses[0]?.failure?.kind ?? 'transport-failure', + recovered: (await runHealthyProbe(request, limits)) === true, + }; +} + +async function runHealthyProbe( + request: SpikeRequest, + limits: typeof DEFAULT_SPIKE_LIMITS, +): Promise { + const result = await runFramedBatch(nodeScript('healthy'), [request], { limits }); + return result.responses[0]?.ok === true; +} + +function nodeScript( + mode: string, + overrideResponse?: SpikeResponse, +): { file: string; args: string[] } { + const response = JSON.stringify( + overrideResponse ?? { + version: 1, + ok: true, + acquisition: { + targetId: 'simulator:probe', + targetGeneration: 'generation', + nodes: [{ id: 'n0', role: 'AXApplication' }], + viewport: { kind: 'missing', reason: 'not-provided' }, + truncated: false, + residue: [], + }, + metrics: { + requestBytes: 1, + responseBytes: 1, + nodeCount: 1, + maxTraversalDepth: 0, + cpuMs: 0, + memoryBytes: 1, + durationMs: 1, + }, + }, + ); + const modeStatement = + mode === 'crash' ? 'process.exit(17);' : mode === 'hang' ? 'setInterval(() => {}, 1000);' : ''; + const script = ` + import process from 'node:process'; + ${modeStatement} + if (${JSON.stringify(mode)} !== 'crash' && ${JSON.stringify(mode)} !== 'hang') { + let input = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (chunk) => { input += chunk; }); + process.stdin.on('end', () => { + for (const line of input.split('\\n').filter(Boolean)) { + const request = JSON.parse(line); + process.stdout.write(JSON.stringify({ ...${response}, id: request.id, candidate: request.candidate }) + '\\n'); + } + }); + } + `; + return { file: process.execPath, args: ['--input-type=module', '-e', script] }; +} diff --git a/scripts/ios-ax-bridge-spike/limits.test.ts b/scripts/ios-ax-bridge-spike/limits.test.ts new file mode 100644 index 000000000..85b1dc5b5 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/limits.test.ts @@ -0,0 +1,70 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { DEFAULT_SPIKE_LIMITS, encodeFrame, validateRawAcquisition } from './limits.ts'; + +const acquisition = { + targetId: 'simulator:test', + targetGeneration: 'generation-1', + nodes: [ + { id: 'n0', type: 'XCUIElementTypeApplication', role: 'AXApplication' }, + { id: 'n1', parentId: 'n0', role: 'AXWindow', frame: { x: 0, y: 0, width: 100, height: 200 } }, + ], + viewport: { kind: 'reported', rect: { x: 0, y: 0, width: 100, height: 200 } }, + truncated: false, + residue: [], +} as const; + +test('accepts a raw tree and reports structural depth without publishing it on nodes', () => { + const result = validateRawAcquisition(acquisition, DEFAULT_SPIKE_LIMITS); + assert.equal(result.ok, true); + if (result.ok) { + assert.equal(result.maxTraversalDepth, 1); + assert.equal(result.acquisition.nodes[0]?.type, 'XCUIElementTypeApplication'); + } + assert.equal('depth' in acquisition.nodes[0], false); + assert.equal('hittable' in acquisition.nodes[0], false); +}); + +test('rejects presentation facts in the acquisition reader', () => { + const result = validateRawAcquisition( + { + ...acquisition, + nodes: [{ id: 'n0', visibleToUser: true }], + }, + DEFAULT_SPIKE_LIMITS, + ); + assert.deepEqual(result, { ok: false, code: 'node-contains-presentation-fact' }); +}); + +test('rejects cycles, missing parents, and resource-limit violations', () => { + assert.deepEqual( + validateRawAcquisition( + { ...acquisition, nodes: [{ id: 'n0', parentId: 'missing' }] }, + DEFAULT_SPIKE_LIMITS, + ), + { ok: false, code: 'parent-node-missing' }, + ); + assert.deepEqual( + validateRawAcquisition( + { ...acquisition, nodes: [{ id: 'n0', parentId: 'n0' }] }, + DEFAULT_SPIKE_LIMITS, + ), + { ok: false, code: 'traversal-depth-exceeded' }, + ); + assert.deepEqual( + validateRawAcquisition( + { ...acquisition, nodes: Array.from({ length: 3 }, (_, index) => ({ id: `n${index}` })) }, + { + ...DEFAULT_SPIKE_LIMITS, + maxNodes: 2, + }, + ), + { ok: false, code: 'node-limit-exceeded' }, + ); +}); + +test('frames are newline-delimited and byte bounded', () => { + const frame = encodeFrame({ id: 'one', text: 'ok' }); + assert.equal(frame.line.endsWith('\n'), true); + assert.equal(frame.bytes, Buffer.byteLength(frame.line)); +}); diff --git a/scripts/ios-ax-bridge-spike/limits.ts b/scripts/ios-ax-bridge-spike/limits.ts new file mode 100644 index 000000000..540f1f4dc --- /dev/null +++ b/scripts/ios-ax-bridge-spike/limits.ts @@ -0,0 +1,230 @@ +import type { RawAcquiredNode, RawAcquisition, ResourceLimits, SpikeRect } from './types.ts'; + +export const DEFAULT_SPIKE_LIMITS: ResourceLimits = Object.freeze({ + maxRequestBytes: 64 * 1024, + maxResponseBytes: 4 * 1024 * 1024, + maxNodes: 1500, + maxTraversalDepth: 12, + maxCpuMs: 2_000, + maxMemoryBytes: 256 * 1024 * 1024, + maxDurationMs: 5_000, +}); + +const NODE_KEYS = new Set([ + 'id', + 'type', + 'parentId', + 'role', + 'subrole', + 'label', + 'value', + 'identifier', + 'frame', + 'enabled', + 'selected', + 'focused', +]); + +export type TreeValidation = + | Readonly<{ ok: true; acquisition: RawAcquisition; maxTraversalDepth: number }> + | Readonly<{ ok: false; code: string }>; + +export function encodeFrame(value: unknown): { bytes: number; line: string } { + const line = `${JSON.stringify(value)}\n`; + return { bytes: Buffer.byteLength(line), line }; +} + +export function validateRawAcquisition(value: unknown, limits: ResourceLimits): TreeValidation { + const record = readAcquisitionRecord(value); + if (!record.ok) return record; + const envelope = validateAcquisitionEnvelope(record.value); + if (!envelope.ok) return envelope; + if (record.value.nodes.length > limits.maxNodes) + return { ok: false, code: 'node-limit-exceeded' }; + const nodes = validateNodes(record.value.nodes); + if (!nodes.ok) return nodes; + const maxTraversalDepth = treeDepth(nodes.nodes); + if (maxTraversalDepth > limits.maxTraversalDepth) { + return { ok: false, code: 'traversal-depth-exceeded' }; + } + return { + ok: true, + acquisition: record.value as unknown as RawAcquisition, + maxTraversalDepth, + }; +} + +function readAcquisitionRecord( + value: unknown, +): + | { ok: true; value: Record & { nodes: unknown[] } } + | { ok: false; code: string } { + if (!isRecord(value)) return { ok: false, code: 'acquisition-not-object' }; + if (typeof value.targetId !== 'string' || value.targetId.length === 0) { + return { ok: false, code: 'target-id-missing' }; + } + if (value.targetGeneration !== null && typeof value.targetGeneration !== 'string') { + return { ok: false, code: 'target-generation-invalid' }; + } + if (!Array.isArray(value.nodes)) return { ok: false, code: 'nodes-not-array' }; + return { ok: true, value: value as Record & { nodes: unknown[] } }; +} + +function validateAcquisitionEnvelope( + value: Record, +): { ok: true } | { ok: false; code: string } { + if (typeof value.truncated !== 'boolean') return { ok: false, code: 'truncated-invalid' }; + if (!validateViewport(value.viewport)) return { ok: false, code: 'viewport-invalid' }; + if (!validateResidue(value.residue)) return { ok: false, code: 'residue-invalid' }; + return { ok: true }; +} + +function validateNodes( + rawNodes: readonly unknown[], +): { ok: true; nodes: RawAcquiredNode[] } | { ok: false; code: string } { + const nodes: RawAcquiredNode[] = []; + const ids = new Set(); + for (const rawNode of rawNodes) { + const node = validateNode(rawNode); + if (!node.ok) return node; + if (ids.has(node.node.id)) return { ok: false, code: 'duplicate-node-id' }; + ids.add(node.node.id); + nodes.push(node.node); + } + return validateParents(nodes, ids); +} + +function validateParents( + nodes: readonly RawAcquiredNode[], + ids: ReadonlySet, +): { ok: true; nodes: RawAcquiredNode[] } | { ok: false; code: string } { + for (const node of nodes) { + if (node.parentId !== undefined && !ids.has(node.parentId)) { + return { ok: false, code: 'parent-node-missing' }; + } + } + return { ok: true, nodes: [...nodes] }; +} + +function validateNode( + value: unknown, +): { ok: true; node: RawAcquiredNode } | { ok: false; code: string } { + const node = asNodeRecord(value); + if (!node) return { ok: false, code: 'node-not-object' }; + const code = nodeValidationCode(node); + return code === undefined ? { ok: true, node: node as RawAcquiredNode } : { ok: false, code }; +} + +function asNodeRecord(value: unknown): Record | undefined { + return isRecord(value) ? value : undefined; +} + +function nodeValidationCode(value: Record): string | undefined { + const identityCode = nodeIdentityCode(value); + if (identityCode) return identityCode; + return nodeFactCode(value); +} + +function nodeIdentityCode(value: Record): string | undefined { + if (Object.keys(value).some((key) => !NODE_KEYS.has(key))) { + return 'node-contains-presentation-fact'; + } + if (typeof value.id !== 'string' || value.id.length === 0) return 'node-id-missing'; + if (value.parentId !== undefined && typeof value.parentId !== 'string') { + return 'parent-id-invalid'; + } + return undefined; +} + +function nodeFactCode(value: Record): string | undefined { + const textCode = optionalStringCode(value); + if (textCode) return textCode; + const booleanCode = optionalBooleanCode(value); + if (booleanCode) return booleanCode; + return value.frame !== undefined && !validateRect(value.frame) ? 'frame-invalid' : undefined; +} + +function optionalStringCode(value: Record): string | undefined { + for (const key of ['type', 'role', 'subrole', 'label', 'value', 'identifier'] as const) { + if (value[key] !== undefined && typeof value[key] !== 'string') return `${key}-invalid`; + } + return undefined; +} + +function optionalBooleanCode(value: Record): string | undefined { + for (const key of ['enabled', 'selected', 'focused'] as const) { + if (value[key] !== undefined && typeof value[key] !== 'boolean') return `${key}-invalid`; + } + return undefined; +} + +function validateViewport(value: unknown): boolean { + if (!isRecord(value) || typeof value.kind !== 'string') return false; + if (value.kind === 'missing') { + return ( + value.reason === 'not-provided' || + value.reason === 'not-supported' || + value.reason === 'invalid' + ); + } + return (value.kind === 'reported' || value.kind === 'derived') && validateRect(value.rect); +} + +function validateRect(value: unknown): value is SpikeRect { + if (!isRecord(value)) return false; + for (const key of ['x', 'y', 'width', 'height']) { + if (typeof value[key] !== 'number' || !Number.isFinite(value[key])) return false; + } + const width = value.width; + const height = value.height; + return typeof width === 'number' && typeof height === 'number' && width >= 0 && height >= 0; +} + +function validateResidue(value: unknown): boolean { + if (!Array.isArray(value)) return false; + for (const residue of value) { + if (!validateResidueItem(residue)) return false; + } + return true; +} + +const RESIDUE_VALIDATORS: Readonly) => boolean>> = { + 'provider-pruned': (value) => + Array.isArray(value.fields) && value.fields.every((field) => typeof field === 'string'), + 'missing-viewport': (value) => + ['not-provided', 'not-supported', 'invalid'].includes(String(value.reason)), + truncated: (value) => ['nodes', 'depth', 'payload'].includes(String(value.dimension)), + 'stale-generation': (value) => value.expected === undefined || typeof value.expected === 'string', + 'unavailable-fact': (value) => typeof value.fact === 'string', + 'fallback-source': (value) => typeof value.producer === 'string', +}; + +function validateResidueItem(value: unknown): boolean { + if (!isRecord(value) || typeof value.kind !== 'string') return false; + return RESIDUE_VALIDATORS[value.kind]?.(value) ?? false; +} + +function treeDepth(nodes: readonly RawAcquiredNode[]): number { + const byId = new Map(nodes.map((node) => [node.id, node])); + const memo = new Map(); + const visiting = new Set(); + let maximum = 0; + for (const node of nodes) maximum = Math.max(maximum, depth(node.id)); + return maximum; + + function depth(id: string): number { + const cached = memo.get(id); + if (cached !== undefined) return cached; + if (visiting.has(id)) return Number.POSITIVE_INFINITY; + visiting.add(id); + const parentId = byId.get(id)?.parentId; + const result = parentId === undefined ? 0 : depth(parentId) + 1; + visiting.delete(id); + memo.set(id, result); + return result; + } +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} diff --git a/scripts/ios-ax-bridge-spike/preference-experiment.ts b/scripts/ios-ax-bridge-spike/preference-experiment.ts new file mode 100644 index 000000000..6a43ee01c --- /dev/null +++ b/scripts/ios-ax-bridge-spike/preference-experiment.ts @@ -0,0 +1,86 @@ +import { execFileSync } from 'node:child_process'; +import { screenFixture } from '../ios-snapshot-benchmark/definitions.ts'; +import { bootSimulator, shutdownSimulator } from '../ios-snapshot-benchmark/lifecycle.ts'; +import type { SpikeConfig } from './config.ts'; +import { + applyPrebootPreferences, + type PlistSnapshot, + readSimulatorState, + restorePrebootPreferences, +} from './preferences.ts'; +import type { PreferenceEvidence } from './types.ts'; + +export function runPreferenceExperiment(config: SpikeConfig): PreferenceEvidence { + if (!config.applyPreferences) return initialPreferenceEvidence(config.udid); + shutdownSimulator(config.udid); + const applied = applyPrebootPreferences(config.udid); + return exerciseAppliedPreferences(config, applied.evidence, applied.snapshots); +} + +function exerciseAppliedPreferences( + config: SpikeConfig, + evidence: PreferenceEvidence, + snapshots: readonly PlistSnapshot[], +): PreferenceEvidence { + let fixtureLaunchCompatible: boolean; + try { + bootSimulator(config.udid); + fixtureLaunchCompatible = tryPrimeFixtureApp( + config.udid, + screenFixture(config.screens[0]!).app, + ); + } catch { + fixtureLaunchCompatible = false; + } + return { + ...evidence, + fixtureLaunchCompatible, + restored: restorePreferences(config.udid, snapshots), + }; +} + +function restorePreferences(udid: string, snapshots: readonly PlistSnapshot[]): boolean { + try { + if (readSimulatorState(udid) !== 'Shutdown') shutdownSimulator(udid); + return restorePrebootPreferences(udid, snapshots); + } catch { + return false; + } +} + +export function primeFixtureApps(config: SpikeConfig): void { + const apps = new Set(config.screens.map((screen) => screenFixture(screen).app)); + for (const app of apps) { + if (!tryPrimeFixtureApp(config.udid, app)) + throw new Error(`Failed to prime ${app} after booting the restored disposable Simulator.`); + } +} + +function tryPrimeFixtureApp(udid: string, app: string): boolean { + try { + execFileSync('xcrun', ['simctl', 'launch', udid, app], { + encoding: 'utf8', + timeout: 60_000, + stdio: ['ignore', 'pipe', 'pipe'], + }); + return true; + } catch { + return false; + } +} + +export function initialPreferenceEvidence(udid: string): PreferenceEvidence { + let simulatorStateBefore = 'unknown'; + try { + simulatorStateBefore = readSimulatorState(udid); + } catch { + simulatorStateBefore = 'unavailable'; + } + return { + applied: false, + restored: false, + fixtureLaunchCompatible: null, + simulatorStateBefore, + diffs: [], + }; +} diff --git a/scripts/ios-ax-bridge-spike/preferences.test.ts b/scripts/ios-ax-bridge-spike/preferences.test.ts new file mode 100644 index 000000000..65091dcc3 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/preferences.test.ts @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { diffPlistValues, simulatorPreferencePaths } from './preferences.ts'; + +test('records exact targeted preference changes and ignores unrelated keys', () => { + assert.deepEqual( + diffPlistValues( + { AutomationEnabled: false, Unrelated: 'preserve' }, + { AutomationEnabled: true, Unrelated: 'preserve' }, + ['AutomationEnabled', 'IgnoreAXServerEntitlements'], + ), + [ + { key: 'AutomationEnabled', before: false, after: true }, + { key: 'IgnoreAXServerEntitlements' }, + ], + ); +}); + +test('builds preference paths only from a validated Simulator UDID', () => { + assert.equal(simulatorPreferencePaths('793B72F6-02C9-4BCD-BEC9-1B3EB42A7ED4').length, 2); + assert.throws(() => simulatorPreferencePaths('../other-device'), /Invalid Simulator UDID/); +}); diff --git a/scripts/ios-ax-bridge-spike/preferences.ts b/scripts/ios-ax-bridge-spike/preferences.ts new file mode 100644 index 000000000..ab241ed22 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/preferences.ts @@ -0,0 +1,254 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import type { PlistDiff, PlistKeyChange, PreferenceEvidence } from './types.ts'; + +const PREFERENCE_VALUES = { + 'com.apple.Accessibility.plist': { + AccessibilityEnabled: true, + ApplicationAccessibilityEnabled: true, + AutomationEnabled: true, + IgnoreAXServerEntitlements: true, + }, + 'com.apple.UIAutomation.plist': { + UIAutomationEnabled: true, + }, +} as const; + +const UUID_PATTERN = /^[0-9A-Fa-f-]{36}$/u; +const EMPTY_PLIST = `\n\n\n`; + +class PreferenceSafetyError extends Error { + readonly code: 'invalid-udid' | 'simulator-not-shutdown' | 'simulator-state-unknown'; + + constructor(code: PreferenceSafetyError['code'], message: string) { + super(message); + this.name = 'PreferenceSafetyError'; + this.code = code; + } +} + +export type PlistSnapshot = Readonly<{ + path: string; + existedBefore: boolean; + beforeBytes: Buffer | null; + beforeValues: Record; +}>; + +export function simulatorPreferencePaths(udid: string): string[] { + validateUdid(udid); + const directory = path.join( + os.homedir(), + 'Library', + 'Developer', + 'CoreSimulator', + 'Devices', + udid, + 'data', + 'Library', + 'Preferences', + ); + return Object.keys(PREFERENCE_VALUES).map((name) => path.join(directory, name)); +} + +export function diffPlistValues( + before: Record, + after: Record, + keys: readonly string[], +): PlistKeyChange[] { + return keys.flatMap((key) => { + const hasBefore = Object.prototype.hasOwnProperty.call(before, key); + const hasAfter = Object.prototype.hasOwnProperty.call(after, key); + if (hasBefore && hasAfter && Object.is(before[key], after[key])) return []; + return [ + { + key, + ...(hasBefore ? { before: before[key] } : {}), + ...(hasAfter ? { after: after[key] } : {}), + }, + ]; + }); +} + +export function readSimulatorState(udid: string): string { + validateUdid(udid); + const result = spawnSync('xcrun', ['simctl', 'list', 'devices', '-j'], { + encoding: 'utf8', + timeout: 30_000, + }); + if (result.status !== 0 || typeof result.stdout !== 'string') { + throw new PreferenceSafetyError('simulator-state-unknown', `Unable to read state for ${udid}.`); + } + let payload: unknown; + try { + payload = JSON.parse(result.stdout); + } catch { + throw new PreferenceSafetyError( + 'simulator-state-unknown', + `Simulator state was not JSON for ${udid}.`, + ); + } + const state = findDeviceState(payload, udid); + if (!state) { + throw new PreferenceSafetyError('simulator-state-unknown', `Simulator ${udid} was not found.`); + } + return state; +} + +export function applyPrebootPreferences(udid: string): { + evidence: PreferenceEvidence; + snapshots: readonly PlistSnapshot[]; +} { + const simulatorStateBefore = readSimulatorState(udid); + if (simulatorStateBefore !== 'Shutdown') { + throw new PreferenceSafetyError( + 'simulator-not-shutdown', + `Preference experiment requires a shutdown Simulator; ${udid} is ${simulatorStateBefore}.`, + ); + } + const snapshots = simulatorPreferencePaths(udid).map(snapshotPlist); + try { + for (const snapshot of snapshots) applyPlistValues(snapshot.path); + } catch (error) { + restoreSnapshotBytes(snapshots); + throw error; + } + const diffs = snapshots.map((snapshot) => { + const afterBytes = readBytes(snapshot.path); + const afterValues = readPlist(snapshot.path); + return { + path: snapshot.path, + existedBefore: snapshot.existedBefore, + beforeSha256: hashBytes(snapshot.beforeBytes), + afterSha256: hashBytes(afterBytes), + changes: diffPlistValues(snapshot.beforeValues, afterValues, changedKeys(snapshot.path)), + } satisfies PlistDiff; + }); + return { + snapshots, + evidence: { + applied: true, + restored: false, + simulatorStateBefore, + diffs, + }, + }; +} + +export function restorePrebootPreferences( + udid: string, + snapshots: readonly PlistSnapshot[], +): PreferenceEvidence['restored'] { + const state = readSimulatorState(udid); + if (state !== 'Shutdown') { + throw new PreferenceSafetyError( + 'simulator-not-shutdown', + `Preference restore requires a shutdown Simulator; ${udid} is ${state}.`, + ); + } + restoreSnapshotBytes(snapshots); + return true; +} + +function restoreSnapshotBytes(snapshots: readonly PlistSnapshot[]): void { + for (const snapshot of snapshots) { + if (snapshot.beforeBytes === null) { + fs.rmSync(snapshot.path, { force: true }); + continue; + } + fs.writeFileSync(snapshot.path, snapshot.beforeBytes); + } +} + +function snapshotPlist(filePath: string): PlistSnapshot { + const beforeBytes = readBytes(filePath); + return { + path: filePath, + existedBefore: beforeBytes !== null, + beforeBytes, + beforeValues: beforeBytes === null ? {} : readPlist(filePath), + }; +} + +function applyPlistValues(filePath: string): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + if (!fs.existsSync(filePath)) fs.writeFileSync(filePath, EMPTY_PLIST); + for (const key of changedKeys(filePath)) { + const value = true; + const replaced = spawnSync( + '/usr/bin/plutil', + ['-replace', key, '-bool', String(value), filePath], + { + encoding: 'utf8', + timeout: 10_000, + }, + ); + if (replaced.status === 0) continue; + const inserted = spawnSync( + '/usr/bin/plutil', + ['-insert', key, '-bool', String(value), filePath], + { + encoding: 'utf8', + timeout: 10_000, + }, + ); + if (inserted.status !== 0) { + throw new PreferenceSafetyError('simulator-state-unknown', `Unable to update ${filePath}.`); + } + } +} + +function readPlist(filePath: string): Record { + const result = spawnSync('/usr/bin/plutil', ['-convert', 'json', '-o', '-', filePath], { + encoding: 'utf8', + timeout: 10_000, + }); + if (result.status !== 0 || typeof result.stdout !== 'string') return {}; + try { + const value: unknown = JSON.parse(result.stdout); + return isRecord(value) ? value : {}; + } catch { + return {}; + } +} + +function readBytes(filePath: string): Buffer | null { + try { + return fs.readFileSync(filePath); + } catch { + return null; + } +} + +function hashBytes(value: Buffer | null): string | null { + return value === null ? null : crypto.createHash('sha256').update(value).digest('hex'); +} + +function changedKeys(filePath: string): string[] { + const name = path.basename(filePath) as keyof typeof PREFERENCE_VALUES; + return name in PREFERENCE_VALUES ? Object.keys(PREFERENCE_VALUES[name]) : []; +} + +function findDeviceState(payload: unknown, udid: string): string | undefined { + if (!isRecord(payload) || !isRecord(payload.devices)) return undefined; + for (const runtimeDevices of Object.values(payload.devices)) { + if (!Array.isArray(runtimeDevices)) continue; + const device = runtimeDevices.find( + (candidate) => isRecord(candidate) && candidate.udid === udid, + ); + if (isRecord(device) && typeof device.state === 'string') return device.state; + } + return undefined; +} + +function validateUdid(udid: string): void { + if (!UUID_PATTERN.test(udid)) { + throw new PreferenceSafetyError('invalid-udid', `Invalid Simulator UDID: ${udid}.`); + } +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} diff --git a/scripts/ios-ax-bridge-spike/presentation.test.ts b/scripts/ios-ax-bridge-spike/presentation.test.ts new file mode 100644 index 000000000..6abd9a807 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/presentation.test.ts @@ -0,0 +1,30 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { presentAcquisitionForMeasurement } from './presentation.ts'; +import type { RawAcquisition } from './types.ts'; + +test('prototype presentation creates the #2190 acquired carrier without semantic fields', () => { + const raw: RawAcquisition = { + targetId: 'simulator:test', + targetGeneration: 'generation-1', + nodes: [ + { + id: 'root', + type: 'XCUIElementTypeApplication', + role: 'AXApplication', + enabled: true, + }, + { id: 'child', parentId: 'root', role: 'AXWindow' }, + ], + viewport: { kind: 'missing', reason: 'not-provided' }, + truncated: false, + residue: [], + }; + const result = presentAcquisitionForMeasurement(raw); + assert.equal(result.acquisition.producer, 'simulator-ax-bridge'); + assert.equal(result.acquisition.nodes.length, 2); + assert.equal(result.acquisition.nodes[0]?.type, 'XCUIElementTypeApplication'); + assert.equal(result.acquisition.nodes[1]?.parentIndex, 0); + assert.equal('hittable' in result.acquisition.nodes[0]!, false); + assert.equal(result.measurement.nodeCount, 2); +}); diff --git a/scripts/ios-ax-bridge-spike/presentation.ts b/scripts/ios-ax-bridge-spike/presentation.ts new file mode 100644 index 000000000..cfd99aeb1 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/presentation.ts @@ -0,0 +1,100 @@ +import { performance } from 'node:perf_hooks'; +import { + createIosSnapshotRequest, + deriveIosCaptureHint, +} from '@agent-device/capture-kit/ios-snapshot-planning'; +import type { IosSnapshotAcquisition } from '@agent-device/contracts/ios-snapshot'; +import type { RawSnapshotNode } from '@agent-device/kernel/snapshot'; +import type { PresentationMeasurement, RawAcquisition } from './types.ts'; + +export function presentAcquisitionForMeasurement(raw: RawAcquisition): { + acquisition: IosSnapshotAcquisition; + measurement: PresentationMeasurement; +} { + const started = performance.now(); + const usage = process.resourceUsage(); + const acquisitionIntent = 'full' as const; + const request = createIosSnapshotRequest({ raw: true, acquisitionIntent }); + const acquisition: IosSnapshotAcquisition = { + producer: 'simulator-ax-bridge', + intent: acquisitionIntent, + hint: { ...deriveIosCaptureHint(request), acquisitionIntent }, + nodes: raw.nodes.map((node, index) => toRawSnapshotNode(node, index, nodeIndexes(raw.nodes))), + truncated: raw.truncated, + viewport: raw.viewport, + lineage: { + targetId: raw.targetId, + ...(raw.targetGeneration === null ? {} : { generation: raw.targetGeneration }), + }, + residue: raw.residue, + }; + const payloadBytes = Buffer.byteLength(JSON.stringify(acquisition)); + const nextUsage = process.resourceUsage(); + return { + acquisition, + measurement: { + ok: true, + payloadBytes, + nodeCount: raw.nodes.length, + durationMs: performance.now() - started, + cpuMs: cpuMilliseconds(nextUsage) - cpuMilliseconds(usage), + memoryBytes: process.memoryUsage().rss, + }, + }; +} + +function toRawSnapshotNode( + node: RawAcquisition['nodes'][number], + index: number, + nodeIndexes: ReadonlyMap, +): RawSnapshotNode { + return { + index, + ...optionalParent(node.parentId, nodeIndexes), + ...optionalString(node, 'type'), + ...optionalString(node, 'role'), + ...optionalString(node, 'subrole'), + ...optionalString(node, 'label'), + ...optionalString(node, 'value'), + ...optionalString(node, 'identifier'), + ...optionalFrame(node.frame), + ...optionalBoolean(node, 'enabled'), + ...optionalBoolean(node, 'selected'), + ...optionalBoolean(node, 'focused'), + }; +} + +function nodeIndexes(nodes: RawAcquisition['nodes']): ReadonlyMap { + return new Map(nodes.map((node, index) => [node.id, index])); +} + +function optionalParent( + parentId: string | undefined, + indexes: ReadonlyMap, +): Partial { + if (parentId === undefined) return {}; + const parentIndex = indexes.get(parentId); + return parentIndex === undefined ? {} : { parentIndex }; +} + +function optionalString( + node: RawAcquisition['nodes'][number], + key: 'type' | 'role' | 'subrole' | 'label' | 'value' | 'identifier', +): Partial { + return node[key] === undefined ? {} : { [key]: node[key] }; +} + +function optionalBoolean( + node: RawAcquisition['nodes'][number], + key: 'enabled' | 'selected' | 'focused', +): Partial { + return node[key] === undefined ? {} : { [key]: node[key] }; +} + +function optionalFrame(frame: RawAcquisition['nodes'][number]['frame']): Partial { + return frame === undefined ? {} : { rect: frame }; +} + +function cpuMilliseconds(usage: NodeJS.ResourceUsage): number { + return (usage.userCPUTime + usage.systemCPUTime) / 1_000; +} diff --git a/scripts/ios-ax-bridge-spike/protocol.ts b/scripts/ios-ax-bridge-spike/protocol.ts new file mode 100644 index 000000000..872cef243 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/protocol.ts @@ -0,0 +1,187 @@ +import { validateRawAcquisition } from './limits.ts'; +import type { + ResourceMetrics, + SpikeFailure, + SpikeFailureKind, + SpikeRequest, + SpikeResponse, +} from './types.ts'; +import type { FirstTreeStatus } from '../ios-snapshot-benchmark/types.ts'; + +const FAILURE_KINDS: ReadonlySet = new Set([ + 'unsupported-mechanism', + 'malformed-tree', + 'stale-generation', + 'timeout', + 'cancelled', + 'process-crash', + 'transport-failure', +]); + +export function parseSpikeResponse( + value: unknown, + request: SpikeRequest, + responseBytes: number, +): SpikeResponse { + if (!isRecord(value)) return malformedResponse(request, 'response-not-object', responseBytes); + if (value.version !== 1) return malformedResponse(request, 'protocol-version', responseBytes); + if (value.id !== request.id) + return malformedResponse(request, 'response-id-mismatch', responseBytes); + if (value.candidate !== request.candidate) { + return malformedResponse(request, 'response-candidate-mismatch', responseBytes); + } + const metrics = parseMetrics(value.metrics, responseBytes); + if (!metrics) return malformedResponse(request, 'metrics-invalid', responseBytes); + + if (value.ok === true) { + const tree = validateRawAcquisition(value.acquisition, request.limits); + if (!tree.ok) return malformedResponse(request, tree.code, responseBytes, metrics); + return { + version: 1, + id: request.id, + candidate: request.candidate, + ok: true, + acquisition: tree.acquisition, + metrics: { + ...metrics, + nodeCount: tree.acquisition.nodes.length, + maxTraversalDepth: tree.maxTraversalDepth, + }, + }; + } + + const failure = parseFailure(value.failure); + if (!failure) return malformedResponse(request, 'failure-invalid', responseBytes, metrics); + return { + version: 1, + id: request.id, + candidate: request.candidate, + ok: false, + failure, + metrics, + }; +} + +export function failureResponse( + request: SpikeRequest, + failure: SpikeFailure, + metrics: Partial = {}, +): SpikeResponse { + return { + version: 1, + id: request.id, + candidate: request.candidate, + ok: false, + failure, + metrics: { + requestBytes: metrics.requestBytes ?? 0, + responseBytes: metrics.responseBytes ?? 0, + nodeCount: metrics.nodeCount ?? 0, + maxTraversalDepth: metrics.maxTraversalDepth ?? 0, + cpuMs: metrics.cpuMs ?? null, + memoryBytes: metrics.memoryBytes ?? null, + durationMs: metrics.durationMs ?? 0, + }, + }; +} + +export function firstTreeStatus(response: SpikeResponse): FirstTreeStatus { + if (response.ok) return response.acquisition?.nodes.length ? 'readable' : 'empty'; + if (response.failure?.kind === 'unsupported-mechanism') return 'unreadable'; + return 'not-observed'; +} + +function malformedResponse( + request: SpikeRequest, + code: string, + responseBytes: number, + metrics?: ResourceMetrics, +): SpikeResponse { + return failureResponse( + request, + { kind: 'malformed-tree', code }, + { + ...(metrics ?? {}), + responseBytes, + }, + ); +} + +function parseMetrics(value: unknown, responseBytes: number): ResourceMetrics | undefined { + if (!isRecord(value)) return undefined; + const requestBytes = finiteNonNegative(value.requestBytes); + const nodeCount = finiteNonNegative(value.nodeCount); + const maxTraversalDepth = finiteNonNegative(value.maxTraversalDepth); + const durationMs = finiteNonNegative(value.durationMs); + if ( + requestBytes === undefined || + nodeCount === undefined || + maxTraversalDepth === undefined || + durationMs === undefined + ) { + return undefined; + } + const cpuMs = nullableFiniteNonNegative(value.cpuMs); + const memoryBytes = nullableFiniteNonNegative(value.memoryBytes); + if (cpuMs === 'invalid' || memoryBytes === 'invalid') return undefined; + return { + requestBytes, + responseBytes, + nodeCount, + maxTraversalDepth, + cpuMs, + memoryBytes, + durationMs, + }; +} + +function parseFailure(value: unknown): SpikeFailure | undefined { + const record = asFailureRecord(value); + if (!record) return undefined; + const kind = readFailureKind(record.kind); + if (!kind) return undefined; + const code = optionalFailureString(record, 'code'); + const expected = optionalFailureString(record, 'expectedTargetGeneration'); + const observed = optionalFailureString(record, 'observedTargetGeneration'); + if (!code.valid || !expected.valid || !observed.valid) return undefined; + return { + kind, + ...(code.value === undefined ? {} : { code: code.value }), + ...(expected.value === undefined ? {} : { expectedTargetGeneration: expected.value }), + ...(observed.value === undefined ? {} : { observedTargetGeneration: observed.value }), + }; +} + +function asFailureRecord(value: unknown): Record | undefined { + return isRecord(value) && typeof value.kind === 'string' ? value : undefined; +} + +function readFailureKind(value: unknown): SpikeFailureKind | undefined { + return typeof value === 'string' && FAILURE_KINDS.has(value as SpikeFailureKind) + ? (value as SpikeFailureKind) + : undefined; +} + +function optionalFailureString( + value: Record, + key: 'code' | 'expectedTargetGeneration' | 'observedTargetGeneration', +): { valid: boolean; value?: string } { + if (value[key] === undefined) return { valid: true }; + return typeof value[key] === 'string' + ? { valid: true, value: value[key] as string } + : { valid: false }; +} + +function finiteNonNegative(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined; +} + +function nullableFiniteNonNegative(value: unknown): number | null | 'invalid' { + if (value === null) return null; + const number = finiteNonNegative(value); + return number === undefined ? 'invalid' : number; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} diff --git a/scripts/ios-ax-bridge-spike/report.test.ts b/scripts/ios-ax-bridge-spike/report.test.ts new file mode 100644 index 000000000..c298b5003 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/report.test.ts @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { gunzipSync } from 'node:zlib'; +import { test } from 'vitest'; +import { corpusCoverage } from './corpus-coverage.ts'; +import { markdownPath } from './report.ts'; + +test('keeps markdown separate from every supported raw artifact path', () => { + assert.equal(markdownPath('/tmp/evidence.json.gz'), '/tmp/evidence.md'); + assert.equal(markdownPath('/tmp/evidence.json'), '/tmp/evidence.md'); + assert.equal(markdownPath('/tmp/evidence'), '/tmp/evidence.md'); +}); + +test('does not call an unproduced requested corpus full', () => { + assert.equal( + corpusCoverage( + ['cold-cold', 'cold', 'warm', 'relaunch'], + ['quiet', 'list', 'nested-scroll', 'alert', 'system-surface', 'xctest-stress'], + [], + ['public-macos-ax'], + ), + 'decisive-early-stop', + ); +}); + +test('ships a readable completed decision inside the checked gzip artifact', () => { + const compressed = fs.readFileSync('docs/evidence/ios-simulator-ax-bridge-2026-09-01.json.gz'); + const report = JSON.parse(gunzipSync(compressed).toString('utf8')) as { + schemaVersion?: string; + status?: string; + decision?: string; + corpusCoverage?: string; + }; + assert.deepEqual( + { + schemaVersion: report.schemaVersion, + status: report.status, + decision: report.decision, + corpusCoverage: report.corpusCoverage, + }, + { + schemaVersion: 'ios-simulator-ax-bridge-spike.v1', + status: 'completed', + decision: 'NO-GO', + corpusCoverage: 'decisive-early-stop', + }, + ); +}); diff --git a/scripts/ios-ax-bridge-spike/report.ts b/scripts/ios-ax-bridge-spike/report.ts new file mode 100644 index 000000000..2107915af --- /dev/null +++ b/scripts/ios-ax-bridge-spike/report.ts @@ -0,0 +1,277 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { gzipSync } from 'node:zlib'; +import type { SpikeCell, SpikeReport, SpikeSample } from './types.ts'; + +export function writeSpikeReport(outputPath: string, report: SpikeReport): void { + const compact = compactReportEvidence(report); + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(outputPath, gzipSync(`${JSON.stringify(compact)}\n`, { level: 9 })); + fs.writeFileSync(markdownPath(outputPath), renderSpikeMarkdown(compact)); +} + +export function markdownPath(outputPath: string): string { + const replaced = outputPath.replace(/\.json(?:\.gz)?$/u, '.md'); + return replaced === outputPath ? `${outputPath}.md` : replaced; +} + +function renderSpikeMarkdown(report: SpikeReport): string { + const lines = [ + '# iOS Simulator AX bridge spike', + '', + `- Decision: **${report.decision}**`, + `- Status: **${report.status}**`, + `- Revision: ${report.revision.commit} (${report.revision.branch})`, + `- Target: ${report.target.name} (${report.target.udid}, ${report.target.runtime})`, + `- Generated: ${report.generatedAt}`, + `- Corpus: states=${report.config.states.join(', ')}, screens=${report.config.screens.join(', ')}, samples=${report.config.requestedSamples}`, + `- Corpus coverage: **${report.corpusCoverage}**`, + '', + '## Environment and limits', + '', + ...environmentLines(report), + '', + '## Candidate fidelity and limitation matrix', + '', + '| Candidate | Mechanism | App surface | System surface | Lifecycle | Main limitation |', + '|---|---|---|---|---|---|', + `| public-macos-ax | public macOS ApplicationServices AX | ${surfaceStatus(report, 'public-macos-ax', 'app')} | ${surfaceStatus(report, 'public-macos-ax', 'system')} | framed protocol | ${publicAxLimitation(report)} |`, + `| private-coresimulator-ax | external/private CoreSimulator AX tool | ${surfaceStatus(report, 'private-coresimulator-ax', 'app')} | ${surfaceStatus(report, 'private-coresimulator-ax', 'system')} | framed protocol contract only | private interface/tool compatibility |`, + `| xctest-control | #2189 XCTest runner control | ${surfaceStatus(report, 'xctest-control', 'app')} | ${surfaceStatus(report, 'xctest-control', 'system')} | existing runner lifecycle | control, not a host-side AX bridge |`, + '', + '## Raw acquisition and prototype presentation results', + '', + '| Candidate | State | Screen | Readable/attempted | Wall p50/p95 ms | Gated duration p50/p95 ms | First look p95 ms | Presentation p50/p95 ms | Nodes | Failures |', + '|---|---|---|---:|---:|---:|---:|---:|---:|---:|', + ...report.cells.map(renderCellRow), + '', + ...fidelityLines(report), + '', + 'Every acquisition sample retains timing, resource, readiness, and failure evidence; the first successful sample in each cell also retains one raw node-tree exemplar with viewport, target generation, truncation, and residue. Presentation samples measure only construction of the #2190 acquired carrier; they do not apply visibility, hittability, scope, depth, or semantic compaction.', + '', + '## Direct protocol probes', + '', + ...probeLines(report), + '', + '## Independent positive-control evidence', + '', + `- Invalid shallow rule: exit=${report.positiveControl.invalidShallowRule.exitCode}; command=${report.positiveControl.invalidShallowRule.command}; assertion=${report.positiveControl.invalidShallowRule.assertion}`, + `- Safe full rule: exit=${report.positiveControl.safeFullRule.exitCode}; command=${report.positiveControl.safeFullRule.command}; assertion=${report.positiveControl.safeFullRule.assertion}`, + '', + '## Preference experiment', + '', + `- Applied: **${report.preferenceEvidence.applied}**`, + `- Restored: **${report.preferenceEvidence.applied ? report.preferenceEvidence.restored : 'not required'}**`, + `- Fixture launch compatible: **${report.preferenceEvidence.fixtureLaunchCompatible ?? 'not exercised'}**`, + `- Simulator state before experiment: ${report.preferenceEvidence.simulatorStateBefore}`, + preferenceExperimentLine(report), + ...preferenceLines(report), + '', + '## Lifecycle, cancellation, and recovery', + '', + `- Source: ${report.lifecycle.source}`, + `- Process crash: ${report.lifecycle.crash.failure}; recovered=${report.lifecycle.crash.recovered}`, + `- Timeout: ${report.lifecycle.timeout.failure}; recovered=${report.lifecycle.timeout.recovered}`, + `- Cancellation: ${report.lifecycle.cancellation.failure}; recovered=${report.lifecycle.cancellation.recovered}`, + `- Stale generation: ${report.lifecycle.staleGeneration.failure}; recovered=${report.lifecycle.staleGeneration.recovered}`, + '', + '## Decision rationale', + '', + ...report.decisionReasons.map((reason) => `- ${reason}`), + '', + '## Next interface boundary', + '', + `- ${report.nextInterface}`, + '', + '## Production boundary', + '', + '- No production backend selection, fallback, runner-demand, open/relaunch, proxy, XCTest interaction, or public CLI changes were made.', + '- A production bridge should not start until this report has a GO result; this run is the #2192 boundary.', + ]; + if (report.stop) { + lines.push('', '## Stop condition', '', `- ${report.stop.category}: ${report.stop.message}`); + if (report.stop.command) lines.push(`- Command: ${report.stop.command}`); + } + return `${lines.join('\n')}\n`; +} + +function environmentLines(report: SpikeReport): string[] { + return [ + `- Node: ${report.toolchain.node}`, + `- pnpm: ${report.toolchain.pnpm}`, + `- Xcode: ${report.toolchain.xcode.replaceAll('\n', '; ')}`, + `- simctl: ${report.toolchain.simctl}`, + `- Swift: ${report.toolchain.swift}`, + `- OS: ${report.toolchain.os}; arch=${report.toolchain.arch}`, + `- Bounds: request=${report.limits.maxRequestBytes} B, response=${report.limits.maxResponseBytes} B, nodes=${report.limits.maxNodes}, traversal=${report.limits.maxTraversalDepth}, CPU=${report.limits.maxCpuMs} ms, memory=${report.limits.maxMemoryBytes} B, duration=${report.limits.maxDurationMs} ms`, + ]; +} + +function probeLines(report: SpikeReport): string[] { + const lines: string[] = []; + for (const probe of report.protocolProbes) { + lines.push(renderProbeLine(probe)); + } + for (const log of report.protocolProbeLogs) { + lines.push(renderProbeLog(log)); + } + return lines; +} + +function renderProbeLine(probe: SpikeReport['protocolProbes'][number]): string { + return `- ${probe.candidate}/${probe.id}: ok=${probe.ok}, failure=${failureValue(probe.failure, 'kind')}, code=${failureValue(probe.failure, 'code')}, nodes=${probe.metrics.nodeCount}, duration=${probe.metrics.durationMs.toFixed(1)} ms, CPU=${metricValue(probe.metrics.cpuMs)} ms, memory=${probe.metrics.memoryBytes ?? '–'} B, response=${probe.metrics.responseBytes} B`; +} + +function renderProbeLog(log: SpikeReport['protocolProbeLogs'][number]): string { + return `- stderr ${log.candidate}/${log.id}: ${log.stderr.trim().replaceAll('\n', ' ⏎ ') || 'empty'}`; +} + +function failureValue( + failure: SpikeReport['protocolProbes'][number]['failure'], + key: 'kind' | 'code', +): string { + return failure?.[key] ?? 'none'; +} + +function metricValue(value: number | null): string { + return value === null ? '–' : value.toFixed(1); +} + +function renderCellRow(cell: SpikeCell): string { + const acquisition = cell.acquisitionSamples; + const presentation = cell.presentationSamples; + const readable = acquisition.filter((sample) => sample.ok && sample.firstTree === 'readable'); + const failures = acquisition.length - readable.length; + const nodeCounts = readable.flatMap((sample) => + typeof sample.metrics?.nodeCount === 'number' ? [sample.metrics.nodeCount] : [], + ); + return `| ${cell.candidate} | ${cell.state} | ${cell.screen} | ${readable.length}/${acquisition.length} | ${summary(readable, 'wallClockMs')} | ${metricSummary(readable)} | ${summary(readable, 'firstLookMs')} | ${summary( + presentation.filter((sample) => sample.ok), + 'wallClockMs', + )} | ${formatNumber(median(nodeCounts))} | ${failures} |`; +} + +function metricSummary(samples: readonly SpikeSample[]): string { + const values = samples.flatMap((sample) => + sample.metrics && Number.isFinite(sample.metrics.durationMs) ? [sample.metrics.durationMs] : [], + ); + return values.length === 0 + ? '–' + : `${formatNumber(median(values))}/${formatNumber(percentile(values, 95))}`; +} + +function surfaceStatus( + report: SpikeReport, + candidate: SpikeCell['candidate'], + surface: 'app' | 'system', +): string { + const cells = report.cells.filter( + (cell) => + cell.candidate === candidate && + (surface === 'system' ? cell.screen === 'system-surface' : cell.screen !== 'system-surface'), + ); + if (cells.some((cell) => cell.acquisitionSamples.some((sample) => sample.ok))) { + return 'observed in successful cells'; + } + if (cells.length > 0) return 'failed in cells'; + const probe = report.protocolProbes.find((item) => item.candidate === candidate); + if (probe?.failure?.kind === 'unsupported-mechanism') return 'unsupported before corpus'; + if (report.candidates.includes(candidate)) return 'not exercised'; + return 'not selected'; +} + +function preferenceLines(report: SpikeReport): string[] { + return report.preferenceEvidence.diffs.flatMap((diff) => [ + `- ${diff.path}: existed=${diff.existedBefore}, beforeSha256=${diff.beforeSha256 ?? 'missing'}, afterSha256=${diff.afterSha256 ?? 'missing'}`, + ` - Changes: ${diff.changes.length === 0 ? 'none' : diff.changes.map((change) => `${change.key}: ${JSON.stringify(change.before)} -> ${JSON.stringify(change.after)}`).join('; ')}`, + ]); +} + +function preferenceExperimentLine(report: SpikeReport): string { + return report.preferenceEvidence.applied && report.preferenceEvidence.restored + ? '- Private/preboot preference keys are experimental only; they were applied to this shutdown disposable Simulator and the original plist bytes were restored.' + : report.preferenceEvidence.applied + ? '- Private/preboot preference keys were applied, but restoration was not proven.' + : '- No private/preboot preference keys were applied in this run.'; +} + +function publicAxLimitation(report: SpikeReport): string { + const publicList = exemplarSample(report, 'public-macos-ax', 'list'); + const controlList = exemplarSample(report, 'xctest-control', 'list'); + if (!publicList || !controlList) return 'fidelity and latency remain unproven'; + const publicDepth = publicList.metrics?.maxTraversalDepth ?? 0; + const controlDepth = controlList.metrics?.maxTraversalDepth ?? 0; + const publicIdentifiers = publicList.acquisition!.nodes.filter((node) => node.identifier).length; + const controlIdentifiers = controlList.acquisition!.nodes.filter( + (node) => node.identifier, + ).length; + const shape = publicDepth < controlDepth ? 'flatter' : 'structurally different'; + return `list evidence is ${shape} and has different identifier coverage (depth ${publicDepth} vs ${controlDepth}; identifiers ${publicIdentifiers} vs ${controlIdentifiers})`; +} + +function compactReportEvidence(report: SpikeReport): SpikeReport { + return { + ...report, + cells: report.cells.map((cell) => ({ + ...cell, + acquisitionSamples: cell.acquisitionSamples.map((sample, index) => + index === 0 || !sample.ok || sample.stderr === undefined ? sample : withoutStderr(sample), + ), + presentationSamples: cell.presentationSamples.map((sample) => withoutStderr(sample)), + })), + }; +} + +function withoutStderr(sample: SpikeSample): SpikeSample { + const { stderr: _stderr, ...rest } = sample; + return rest; +} + +function fidelityLines(report: SpikeReport): string[] { + const lines = ['Raw exemplar fidelity (public AX vs XCTest control):']; + for (const screen of report.config.screens) { + const publicSample = exemplarSample(report, 'public-macos-ax', screen); + const controlSample = exemplarSample(report, 'xctest-control', screen); + if (!publicSample || !controlSample) continue; + const publicNodes = publicSample.acquisition!.nodes; + const controlNodes = controlSample.acquisition!.nodes; + lines.push( + `- ${screen}: nodes ${publicNodes.length}/${controlNodes.length}; depth ${publicSample.metrics?.maxTraversalDepth ?? '–'}/${controlSample.metrics?.maxTraversalDepth ?? '–'}; identifiers ${publicNodes.filter((node) => node.identifier).length}/${controlNodes.filter((node) => node.identifier).length}.`, + ); + } + return lines.length === 1 ? ['Raw exemplar fidelity comparison was not available.'] : lines; +} + +function exemplarSample( + report: SpikeReport, + candidate: SpikeCell['candidate'], + screen: SpikeCell['screen'], +): SpikeSample | undefined { + return report.cells + .find((cell) => cell.candidate === candidate && cell.screen === screen) + ?.acquisitionSamples.find((sample) => sample.acquisition); +} + +function summary(samples: readonly SpikeSample[], key: 'wallClockMs' | 'firstLookMs'): string { + const values = samples.flatMap((sample) => { + const value = sample[key]; + return typeof value === 'number' && Number.isFinite(value) ? [value] : []; + }); + if (values.length === 0) return '–'; + return `${formatNumber(median(values))}/${formatNumber(percentile(values, 95))}`; +} + +function median(values: readonly number[]): number { + return percentile(values, 50); +} + +function percentile(values: readonly number[], percentage: number): number { + if (values.length === 0) return Number.NaN; + const sorted = [...values].sort((left, right) => left - right); + const rank = Math.ceil((percentage / 100) * sorted.length); + return sorted[Math.min(sorted.length - 1, Math.max(0, rank - 1))]!; +} + +function formatNumber(value: number): string { + return Number.isFinite(value) ? value.toFixed(1) : '–'; +} diff --git a/scripts/ios-ax-bridge-spike/run.test.ts b/scripts/ios-ax-bridge-spike/run.test.ts new file mode 100644 index 000000000..2874cb984 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/run.test.ts @@ -0,0 +1,13 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { test } from 'vitest'; + +test('loads the executable spike entrypoint', () => { + const output = execFileSync( + process.execPath, + ['--experimental-strip-types', 'scripts/ios-ax-bridge-spike/run.ts', '--help'], + { encoding: 'utf8' }, + ); + + assert.match(output, /Usage: pnpm bench:ios-ax-bridge/); +}); diff --git a/scripts/ios-ax-bridge-spike/run.ts b/scripts/ios-ax-bridge-spike/run.ts new file mode 100644 index 000000000..cc1d890a4 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/run.ts @@ -0,0 +1,318 @@ +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { + createPrivateCoreSimulatorAxAdapter, + createPublicMacOsAxAdapter, + createXCTestControlAdapter, +} from './adapter.ts'; +import { parseConfig, type SpikeConfig } from './config.ts'; +import { decideSpike } from './decision.ts'; +import { runLifecycleProbes } from './lifecycle.ts'; +import { markdownPath, writeSpikeReport } from './report.ts'; +import { corpusCoverage } from './corpus-coverage.ts'; +import { + initialPreferenceEvidence, + primeFixtureApps, + runPreferenceExperiment, +} from './preference-experiment.ts'; +import { createAdapterOptions, runSpikeCells } from './runner.ts'; +import { readGitRevision, readTarget, readToolchain } from '../ios-snapshot-benchmark/host.ts'; +import { runDeepButtonControls } from '../ios-snapshot-benchmark/deep-control.ts'; +import { deepButtonFixtureEvidence } from '../ios-snapshot-benchmark/deep-button.ts'; +import { bootSimulator, shutdownSimulator } from '../ios-snapshot-benchmark/lifecycle.ts'; +import type { AcquisitionAdapter } from './adapter.ts'; +import { SPIKE_ISSUE, SPIKE_PARENT, SPIKE_PREREQUISITES, SPIKE_SCHEMA_VERSION } from './types.ts'; +import type { + PreferenceEvidence, + ProtocolProbeLog, + SpikeReport, + SpikeRequest, + SpikeResponse, + Toolchain, +} from './types.ts'; + +if (isMainModule()) void main(process.argv.slice(2)).catch(reportFailure); + +function reportFailure(error: unknown): void { + process.stderr.write( + `${error instanceof Error ? (error.stack ?? error.message) : String(error)}\n`, + ); + process.exitCode = 1; +} + +async function main(argv: readonly string[]): Promise { + const config = parseConfig(argv); + const metadata = readMetadata(config); + const runConfig = { ...config, targetWindowName: metadata.target.name }; + const lifecycle = await runLifecycleProbes(); + const evidence = await executeSpikeRun(runConfig); + const decision = decideSpike( + evidence.cells, + lifecycle, + evidence.preferenceEvidence, + config.limits, + evidence.status, + evidence.protocolProbes, + ); + const report = createReport(runConfig, metadata, lifecycle, evidence, decision); + writeSpikeReport(config.outputPath, report); + process.stdout.write( + `Decision: ${report.decision}\nRaw: ${config.outputPath}\nMarkdown: ${markdownPath(config.outputPath)}\n`, + ); + if (evidence.status === 'stopped') process.exitCode = 2; +} + +type SpikeRunEvidence = Readonly<{ + status: SpikeReport['status']; + stop?: SpikeReport['stop']; + cells: Awaited>; + protocolProbes: SpikeResponse[]; + protocolProbeLogs: SpikeReport['protocolProbeLogs']; + preferenceEvidence: PreferenceEvidence; + positiveControl: SpikeReport['positiveControl']; +}>; + +async function executeSpikeRun(config: SpikeConfig): Promise { + return collectSpikeEvidence(config); +} + +type CollectedSpikeEvidence = SpikeRunEvidence; + +async function collectSpikeEvidence(config: SpikeConfig): Promise { + let preferenceEvidence = initialPreferenceEvidence(config.udid); + let cells: Awaited> = []; + let protocolProbes: SpikeResponse[] = []; + let protocolProbeLogs: SpikeReport['protocolProbeLogs'] = []; + let positiveControl = deepButtonFixtureEvidence(); + try { + positiveControl = runDeepButtonControls(config.repoRoot); + preferenceEvidence = runPreferenceExperiment(config); + assertPreferenceRestored(config, preferenceEvidence); + const adapters = createAdapters(config); + bootSimulator(config.udid); + primeFixtureApps(config); + const probes = await runProtocolProbes(config, adapters); + protocolProbes = probes.responses; + protocolProbeLogs = probes.logs; + cells = await runSpikeCells(config, supportedAdapters(adapters, protocolProbes)); + return collectedEvidence( + 'completed', + cells, + protocolProbes, + protocolProbeLogs, + preferenceEvidence, + positiveControl, + ); + } catch (error) { + return { + ...collectedEvidence( + 'stopped', + cells, + protocolProbes, + protocolProbeLogs, + preferenceEvidence, + positiveControl, + ), + stop: stopForError(error), + }; + } finally { + cleanupDevice(config); + } +} + +function collectedEvidence( + status: SpikeReport['status'], + cells: SpikeReport['cells'], + protocolProbes: SpikeResponse[], + protocolProbeLogs: SpikeReport['protocolProbeLogs'], + preferenceEvidence: PreferenceEvidence, + positiveControl: SpikeReport['positiveControl'], +): CollectedSpikeEvidence { + return { + status, + cells, + protocolProbes, + protocolProbeLogs, + preferenceEvidence, + positiveControl, + }; +} + +function assertPreferenceRestored(config: SpikeConfig, evidence: PreferenceEvidence): void { + if (config.applyPreferences && !evidence.restored) { + throw new Error('The task-owned Simulator preference experiment was not restored.'); + } +} + +function cleanupDevice(config: SpikeConfig): void { + if (!config.keepDevice) shutdownSimulator(config.udid); +} + +function stopForError(error: unknown): SpikeReport['stop'] { + return { + category: isConfigurationError(error) ? 'configuration' : 'infrastructure', + message: error instanceof Error ? error.message : String(error), + ...(errorCommand(error) ? { command: errorCommand(error) } : {}), + }; +} + +function createReport( + config: SpikeConfig, + metadata: { target: SpikeReport['target']; toolchain: Toolchain }, + lifecycle: SpikeReport['lifecycle'], + evidence: SpikeRunEvidence, + decision: { decision: SpikeReport['decision']; reasons: string[] }, +): SpikeReport { + return { + schemaVersion: SPIKE_SCHEMA_VERSION, + issue: SPIKE_ISSUE, + parent: SPIKE_PARENT, + prerequisites: SPIKE_PREREQUISITES, + generatedAt: new Date().toISOString(), + revision: readGitRevision(config.repoRoot), + toolchain: metadata.toolchain, + target: metadata.target, + limits: config.limits, + candidates: config.candidates, + config: { + states: config.states, + screens: config.screens, + requestedSamples: config.samples, + }, + protocolProbes: evidence.protocolProbes, + protocolProbeLogs: evidence.protocolProbeLogs, + preferenceEvidence: evidence.preferenceEvidence, + lifecycle, + positiveControl: evidence.positiveControl, + status: evidence.status, + corpusCoverage: corpusCoverage( + config.states, + config.screens, + evidence.cells, + config.candidates, + ), + cells: evidence.cells, + decision: decision.decision, + decisionReasons: decision.reasons, + nextInterface: + 'Keep any future bridge behind the #2190 acquisition adapter and preserve raw facts until a separate GO evidence run proves fidelity, lifecycle, and latency.', + ...(evidence.stop ? { stop: evidence.stop } : {}), + }; +} + +function createAdapters(config: SpikeConfig) { + const options = createAdapterOptions(config); + const adapters = config.candidates.flatMap((candidate) => { + if (candidate === 'public-macos-ax') return [createPublicMacOsAxAdapter(options)]; + if (candidate === 'private-coresimulator-ax') + return [createPrivateCoreSimulatorAxAdapter(options)]; + return [ + createXCTestControlAdapter((request) => ({ + repoRoot: config.repoRoot, + stateDir: config.stateDir, + session: `ax-spike-xctest-control-${request.state ?? 'state'}-${request.screen}`, + udid: request.simulatorUdid, + derivedPath: path.join(config.derivedPath, 'xctest-control', request.screen), + })), + ]; + }); + return adapters; +} + +async function runProtocolProbes( + config: SpikeConfig, + adapters: readonly AcquisitionAdapter[], +): Promise<{ responses: SpikeResponse[]; logs: SpikeReport['protocolProbeLogs'] }> { + const responses: SpikeResponse[] = []; + const logs: ProtocolProbeLog[] = []; + for (const adapter of adapters.filter(isBridgeAdapter)) { + const request = protocolProbeRequest(config, adapter.candidate); + const result = await adapter.acquireBatch([request]); + responses.push(...result.responses.slice(0, 1)); + logs.push({ candidate: adapter.candidate, id: request.id, stderr: result.stderr }); + } + return { responses, logs }; +} + +function isBridgeAdapter(adapter: AcquisitionAdapter): adapter is AcquisitionAdapter & { + candidate: Exclude; +} { + return adapter.candidate !== 'xctest-control'; +} + +function protocolProbeRequest( + config: SpikeConfig, + candidate: Exclude, +): SpikeRequest { + return { + version: 1, + id: `protocol-probe:${candidate}`, + candidate, + simulatorUdid: config.udid, + state: 'warm', + screen: 'unprepared-surface', + appBundleId: config.appBundleId, + ...(config.targetWindowName === undefined ? {} : { targetWindowName: config.targetWindowName }), + ...(config.targetProcessId === undefined ? {} : { targetProcessId: config.targetProcessId }), + limits: config.limits, + }; +} + +function readMetadata(config: SpikeConfig): { + target: SpikeReport['target']; + toolchain: Toolchain; +} { + const target = readTarget(config.udid, 'com.callstack.agentdevicelab'); + const base = readToolchain(); + return { + target: { udid: target.udid, name: target.name, runtime: target.runtime }, + toolchain: { ...base, swift: commandText('swift', ['--version']) }, + }; +} + +function supportedAdapters( + adapters: readonly AcquisitionAdapter[], + probes: readonly SpikeResponse[], +): readonly AcquisitionAdapter[] { + return adapters.filter((adapter) => { + if (adapter.candidate === 'xctest-control') return true; + const probe = probes.find((candidate) => candidate.candidate === adapter.candidate); + return ![ + 'private-tool-unavailable', + 'host-accessibility-permission', + 'candidate-not-supported', + ].includes(probe?.failure?.code ?? ''); + }); +} + +function commandText(command: string, args: readonly string[]): string { + try { + return execFileSync(command, [...args], { + encoding: 'utf8', + timeout: 30_000, + stdio: ['ignore', 'pipe', 'ignore'], + }).trim(); + } catch { + return 'unavailable'; + } +} + +function isConfigurationError(error: unknown): boolean { + return error instanceof Error && error.name === 'SpikeConfigurationError'; +} + +function errorCommand(error: unknown): string | undefined { + if (error instanceof Error && 'command' in error) { + const command = error.command; + return typeof command === 'string' ? command : undefined; + } + return undefined; +} + +function isMainModule(): boolean { + return Boolean( + process.argv[1] && + path.resolve(process.argv[1]) === path.resolve(fileURLToPath(import.meta.url)), + ); +} diff --git a/scripts/ios-ax-bridge-spike/runner.ts b/scripts/ios-ax-bridge-spike/runner.ts new file mode 100644 index 000000000..a784aa53d --- /dev/null +++ b/scripts/ios-ax-bridge-spike/runner.ts @@ -0,0 +1,293 @@ +import { spawnSync } from 'node:child_process'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { + classifyFailure, + openFixture, + type CliContext, +} from '../ios-snapshot-benchmark/command.ts'; +import { + admitReadyCell, + admitStableWarmSample, + admitSuccessfulSample, + prepareCellState, + prepareSampleState, + type CellAdmissionOptions, +} from '../ios-snapshot-benchmark/cell-admission.ts'; +import { screenFixture, sampleMinimumForState } from '../ios-snapshot-benchmark/definitions.ts'; +import { BenchmarkInfrastructureError, stopDaemon } from '../ios-snapshot-benchmark/lifecycle.ts'; +import { appendSamples, makeRequest, type CapturedResponse } from './sample-evidence.ts'; +import type { AcquisitionAdapter, AdapterOptions } from './adapter.ts'; +import type { SpikeConfig } from './config.ts'; +import type { CandidateId, SpikeCell, SpikeRequest, SpikeSample } from './types.ts'; + +export async function runSpikeCells( + config: SpikeConfig, + adapters: readonly AcquisitionAdapter[], +): Promise { + const cells: SpikeCell[] = []; + for (const adapter of adapters) { + for (const state of config.states) { + for (const screen of config.screens) { + cells.push(await runCell(config, adapter, state, screen)); + } + } + } + return cells; +} + +export function createAdapterOptions(config: SpikeConfig): AdapterOptions { + return { + repoRoot: config.repoRoot, + ...(config.privateTool ? { privateTool: config.privateTool } : {}), + ...(config.helperPath ? { helperPath: config.helperPath } : {}), + limits: config.limits, + }; +} + +async function runCell( + config: SpikeConfig, + adapter: AcquisitionAdapter, + state: SpikeConfig['states'][number], + screen: SpikeConfig['screens'][number], +): Promise { + const fixture = screenFixture(screen); + const context = contextFor(config, adapter.candidate, state, screen); + const admission: CellAdmissionOptions = { + repoRoot: config.repoRoot, + stateDir: config.stateDir, + derivedPath: context.derivedPath, + udid: config.udid, + samples: config.samples, + state, + fixture, + }; + const acquisitionSamples: SpikeSample[] = []; + const presentationSamples: SpikeSample[] = []; + try { + prepareCellState(admission); + if (state === 'warm') { + await collectWarmSamples( + config, + adapter, + context, + admission, + acquisitionSamples, + presentationSamples, + ); + } else { + await collectNonWarmSamples( + config, + adapter, + context, + admission, + acquisitionSamples, + presentationSamples, + ); + } + return { + candidate: adapter.candidate, + state, + screen, + sampleMinimum: sampleMinimumForState(state), + acquisitionSamples, + presentationSamples, + }; + } finally { + closeSession(context); + stopDaemon(config.repoRoot, config.stateDir); + } +} + +async function collectWarmSamples( + config: SpikeConfig, + adapter: AcquisitionAdapter, + context: CliContext, + admission: CellAdmissionOptions, + acquisitionSamples: SpikeSample[], + presentationSamples: SpikeSample[], +): Promise { + let appPid = admitReadyCell(context, admission); + if (appPid === undefined) + throw new BenchmarkInfrastructureError('Warm admission returned no app PID.'); + for (let index = 0; index < config.samples; index += 1) { + if (index > 0) appPid = admitStableWarmSample(admission, appPid); + const request = makeRequest( + config, + adapter.candidate, + admission.state, + admission.fixture.id, + index, + ); + const captured = await capture(adapter, request); + appendSamples( + adapter.candidate, + admission.state, + admission.fixture.id, + index, + captured, + 0, + acquisitionSamples, + presentationSamples, + ); + } +} + +async function collectNonWarmSamples( + config: SpikeConfig, + adapter: AcquisitionAdapter, + context: CliContext, + admission: CellAdmissionOptions, + acquisitionSamples: SpikeSample[], + presentationSamples: SpikeSample[], +): Promise { + let appPid: number | undefined; + for (let index = 0; index < config.samples; index += 1) { + if (index > 0) prepareSampleState(admission); + const launchStarted = performance.now(); + const opened = openFixture(context, admission.fixture, { relaunch: true }); + if (!opened.ok) { + throw preparationError( + opened, + `open ${admission.fixture.id}`, + context, + openArguments(admission.fixture), + ); + } + const preparationMs = performance.now() - launchStarted; + appPid = admitSuccessfulSample(context, admission, opened, appPid); + const request = makeRequest( + config, + adapter.candidate, + admission.state, + admission.fixture.id, + index, + ); + const captured = await capture(adapter, request); + appendSamples( + adapter.candidate, + admission.state, + admission.fixture.id, + index, + captured, + preparationMs, + acquisitionSamples, + presentationSamples, + ); + } +} + +function openArguments(fixture: ReturnType): string[] { + return ['open', fixture.app, '--relaunch', ...launchUrlArguments(fixture), '--foreground']; +} + +function launchUrlArguments(fixture: ReturnType): string[] { + return fixture.launchUrl ? ['--launch-url', fixture.launchUrl] : []; +} + +async function capture( + adapter: AcquisitionAdapter, + request: SpikeRequest, +): Promise { + const startedAt = new Date().toISOString(); + const started = performance.now(); + const batch = await adapter.acquireBatch([request]); + const response = batch.responses[0]; + if (!response) { + throw new BenchmarkInfrastructureError(`Adapter ${adapter.candidate} returned no response.`); + } + return { + response, + stderr: batch.stderr, + startedAt, + wallClockMs: performance.now() - started, + }; +} + +function contextFor( + config: SpikeConfig, + candidate: CandidateId, + state: SpikeConfig['states'][number], + screen: SpikeConfig['screens'][number], +): CliContext { + return { + repoRoot: config.repoRoot, + stateDir: config.stateDir, + session: `ax-spike-${candidate}-${state}-${screen}`, + udid: config.udid, + derivedPath: path.join(config.derivedPath, candidate, state, screen), + }; +} + +function closeSession(context: CliContext): void { + spawnSync( + process.execPath, + [ + 'bin/agent-device.mjs', + 'close', + '--state-dir', + context.stateDir, + '--session', + context.session, + '--platform', + 'ios', + '--udid', + context.udid, + '--json', + ], + { + cwd: context.repoRoot, + env: { ...process.env, AGENT_DEVICE_NO_UPDATE_NOTIFIER: '1' }, + stdio: 'ignore', + timeout: 60_000, + }, + ); +} + +function preparationError( + result: { payload: unknown; stderr: string; exitCode: number }, + operation: string, + context: CliContext, + args: readonly string[], +): Error { + const failure = classifyFailure(result.payload, result); + return new BenchmarkInfrastructureError( + `${operation} failed during fixture preparation (exit ${result.exitCode}; ${failureDetails(failure, result)})`, + commandFor(context, args), + ); +} + +function failureDetails( + failure: ReturnType, + result: { stderr: string }, +): string { + return `code=${fallbackText(failure.code)}; reason=${fallbackText(failure.reason)}; diagnostic=${diagnosticText(failure.message, result.stderr)}`; +} + +function fallbackText(value: string | undefined): string { + return value ?? 'none'; +} + +function diagnosticText(message: string | undefined, stderr: string): string { + const text = message ?? stderr.trim(); + return (text || 'no diagnostic').slice(0, 800); +} + +function commandFor(context: CliContext, args: readonly string[]): string { + return [ + process.execPath, + 'bin/agent-device.mjs', + ...args, + '--state-dir', + context.stateDir, + '--session', + context.session, + '--platform', + 'ios', + '--udid', + context.udid, + '--ios-xctest-derived-data-path', + context.derivedPath, + '--json', + ].join(' '); +} diff --git a/scripts/ios-ax-bridge-spike/sample-evidence.test.ts b/scripts/ios-ax-bridge-spike/sample-evidence.test.ts new file mode 100644 index 000000000..5e88cb256 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/sample-evidence.test.ts @@ -0,0 +1,102 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { appendSamples, type CapturedResponse } from './sample-evidence.ts'; +import type { SpikeSample } from './types.ts'; + +test('keeps one raw acquisition exemplar while retaining every sample measurement', () => { + const acquisitionSamples: SpikeSample[] = []; + const presentationSamples: SpikeSample[] = []; + const captured: CapturedResponse = { + startedAt: '2026-09-01T00:00:00.000Z', + wallClockMs: 12, + stderr: '', + response: { + version: 1, + id: 'sample', + candidate: 'public-macos-ax', + ok: true, + acquisition: { + targetId: 'simulator:test', + nodes: [{ id: 'root', role: 'AXGroup', label: 'Inert surface' }], + viewport: { kind: 'missing', reason: 'not-provided' }, + truncated: false, + residue: [], + }, + metrics: { + requestBytes: 1, + responseBytes: 2, + nodeCount: 1, + maxTraversalDepth: 0, + cpuMs: 1, + memoryBytes: 1, + durationMs: 12, + }, + }, + }; + + for (const index of [0, 1]) { + appendSamples( + 'public-macos-ax', + 'warm', + 'quiet', + index, + captured, + 3, + acquisitionSamples, + presentationSamples, + ); + } + + assert.equal(acquisitionSamples.length, 2); + assert.ok(acquisitionSamples[0]?.acquisition); + assert.equal(acquisitionSamples[1]?.acquisition, undefined); + assert.equal(presentationSamples.length, 2); + assert.equal( + presentationSamples.every((sample) => sample.acquisition === undefined), + true, + ); +}); + +test('rejects a non-empty acquisition that is not bound to the prepared fixture', () => { + const acquisitionSamples: SpikeSample[] = []; + const presentationSamples: SpikeSample[] = []; + const captured: CapturedResponse = { + startedAt: '2026-09-01T00:00:00.000Z', + wallClockMs: 12, + stderr: '', + response: { + version: 1, + id: 'wrong-tree', + candidate: 'public-macos-ax', + ok: true, + acquisition: { + targetId: 'simulator:test', + targetGeneration: 'one', + nodes: [{ id: 'root', label: 'Another screen' }], + viewport: { kind: 'missing', reason: 'not-provided' }, + truncated: false, + residue: [], + }, + metrics: { + requestBytes: 1, + responseBytes: 2, + nodeCount: 1, + maxTraversalDepth: 0, + cpuMs: 1, + memoryBytes: 1, + durationMs: 12, + }, + }, + }; + appendSamples( + 'public-macos-ax', + 'warm', + 'quiet', + 0, + captured, + 0, + acquisitionSamples, + presentationSamples, + ); + assert.equal(acquisitionSamples[0]?.firstTree, 'unreadable'); +}); diff --git a/scripts/ios-ax-bridge-spike/sample-evidence.ts b/scripts/ios-ax-bridge-spike/sample-evidence.ts new file mode 100644 index 000000000..7334854e3 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/sample-evidence.ts @@ -0,0 +1,156 @@ +import { firstTreeStatus } from './protocol.ts'; +import { presentAcquisitionForMeasurement } from './presentation.ts'; +import { screenFixture } from '../ios-snapshot-benchmark/definitions.ts'; +import type { SpikeConfig } from './config.ts'; +import type { SpikeCell, SpikeRequest, SpikeResponse, SpikeSample } from './types.ts'; + +export type CapturedResponse = Readonly<{ + response: SpikeResponse; + stderr: string; + startedAt: string; + wallClockMs: number; +}>; + +export function appendSamples( + candidate: SpikeCell['candidate'], + state: SpikeCell['state'], + screen: SpikeCell['screen'], + index: number, + captured: CapturedResponse, + preparationMs: number, + acquisitionSamples: SpikeSample[], + presentationSamples: SpikeSample[], +): void { + const acquiredAt = new Date().toISOString(); + const status = fixtureBoundStatus(captured.response, screen); + const keepRawExemplar = + captured.response.acquisition !== undefined && + acquisitionSamples.every((sample) => sample.acquisition === undefined); + acquisitionSamples.push({ + index: index + 1, + candidate, + state, + screen, + startedAt: captured.startedAt, + finishedAt: acquiredAt, + operation: 'acquisition', + wallClockMs: captured.wallClockMs, + preparationMs, + firstLookMs: preparationMs + captured.wallClockMs, + firstTree: status, + ok: captured.response.ok, + ...(keepRawExemplar ? { acquisition: captured.response.acquisition } : {}), + metrics: captured.response.metrics, + ...(captured.stderr ? { stderr: captured.stderr } : {}), + ...(captured.response.failure ? { failure: captured.response.failure } : {}), + }); + presentationSamples.push(presentationSample(candidate, state, screen, index, captured, status)); +} + +function fixtureBoundStatus( + response: SpikeResponse, + screen: SpikeCell['screen'], +): SpikeSample['firstTree'] { + const status = firstTreeStatus(response); + if (status !== 'readable' || !response.acquisition) return status; + const fixture = screenFixture(screen); + const anchor = fixture.setupAnchorText ?? fixture.anchorText; + return response.acquisition.nodes.some((node) => + [node.label, node.value, node.identifier].some((value) => value?.includes(anchor)), + ) + ? 'readable' + : 'unreadable'; +} + +export function makeRequest( + config: SpikeConfig, + candidate: SpikeCell['candidate'], + state: SpikeCell['state'], + screen: SpikeCell['screen'], + index: number, +): SpikeRequest { + return { + version: 1, + id: `${candidate}:${state}:${screen}:${index + 1}`, + candidate, + simulatorUdid: config.udid, + state, + screen, + appBundleId: config.appBundleId, + ...(config.targetWindowName === undefined ? {} : { targetWindowName: config.targetWindowName }), + ...(config.targetProcessId === undefined ? {} : { targetProcessId: config.targetProcessId }), + limits: config.limits, + }; +} + +function presentationSample( + candidate: SpikeCell['candidate'], + state: SpikeCell['state'], + screen: SpikeCell['screen'], + index: number, + captured: CapturedResponse, + status: SpikeSample['firstTree'], +): SpikeSample { + if (!captured.response.acquisition) + return failedPresentationSample(candidate, state, screen, index, captured, status); + return successfulPresentationSample(candidate, state, screen, index, captured, status); +} + +function failedPresentationSample( + candidate: SpikeCell['candidate'], + state: SpikeCell['state'], + screen: SpikeCell['screen'], + index: number, + captured: CapturedResponse, + status: SpikeSample['firstTree'], +): SpikeSample { + const startedAt = new Date().toISOString(); + return { + index: index + 1, + candidate, + state, + screen, + startedAt, + finishedAt: new Date().toISOString(), + operation: 'presentation', + wallClockMs: 0, + firstTree: status, + ok: false, + ...(captured.stderr ? { stderr: captured.stderr } : {}), + presentation: { + ok: false, + payloadBytes: 0, + nodeCount: 0, + durationMs: 0, + cpuMs: null, + memoryBytes: process.memoryUsage().rss, + }, + ...(captured.response.failure ? { failure: captured.response.failure } : {}), + }; +} + +function successfulPresentationSample( + candidate: SpikeCell['candidate'], + state: SpikeCell['state'], + screen: SpikeCell['screen'], + index: number, + captured: CapturedResponse, + status: SpikeSample['firstTree'], +): SpikeSample { + const startedAt = new Date().toISOString(); + const presented = presentAcquisitionForMeasurement(captured.response.acquisition!); + return { + index: index + 1, + candidate, + state, + screen, + startedAt, + finishedAt: new Date().toISOString(), + operation: 'presentation', + wallClockMs: presented.measurement.durationMs, + firstTree: status, + ok: true, + ...(captured.stderr ? { stderr: captured.stderr } : {}), + presentation: presented.measurement, + }; +} diff --git a/scripts/ios-ax-bridge-spike/swift/Package.swift b/scripts/ios-ax-bridge-spike/swift/Package.swift new file mode 100644 index 000000000..7796b51ff --- /dev/null +++ b/scripts/ios-ax-bridge-spike/swift/Package.swift @@ -0,0 +1,18 @@ +// swift-tools-version: 5.9 +import PackageDescription + +let package = Package( + name: "agent-device-ios-ax-bridge-spike", + platforms: [.macOS(.v13)], + products: [ + .executable( + name: "agent-device-ios-ax-bridge-spike", + targets: ["AgentDeviceIosAxBridgeSpike"] + ), + ], + targets: [ + .executableTarget( + name: "AgentDeviceIosAxBridgeSpike" + ), + ] +) diff --git a/scripts/ios-ax-bridge-spike/swift/Sources/AgentDeviceIosAxBridgeSpike/ProcessMetrics.swift b/scripts/ios-ax-bridge-spike/swift/Sources/AgentDeviceIosAxBridgeSpike/ProcessMetrics.swift new file mode 100644 index 000000000..4ebb2f0e1 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/swift/Sources/AgentDeviceIosAxBridgeSpike/ProcessMetrics.swift @@ -0,0 +1,48 @@ +import Darwin +import Foundation + +struct SpikeProcessMetrics { + let cpuMs: Double + let memoryBytes: Int64? +} + +func spikeProcessMetrics(since start: rusage) -> SpikeProcessMetrics { + var current = rusage() + getrusage(RUSAGE_SELF, ¤t) + let startCpu = cpuMilliseconds(start) + let currentCpu = cpuMilliseconds(current) + return SpikeProcessMetrics( + cpuMs: max(0, currentCpu - startCpu), + memoryBytes: residentMemoryBytes() + ) +} + +func currentResourceUsage() -> rusage { + var usage = rusage() + getrusage(RUSAGE_SELF, &usage) + return usage +} + +private func cpuMilliseconds(_ usage: rusage) -> Double { + let user = Double(usage.ru_utime.tv_sec) * 1_000 + Double(usage.ru_utime.tv_usec) / 1_000 + let system = Double(usage.ru_stime.tv_sec) * 1_000 + Double(usage.ru_stime.tv_usec) / 1_000 + return user + system +} + +private func residentMemoryBytes() -> Int64? { + var info = mach_task_basic_info() + var count = mach_msg_type_number_t( + MemoryLayout.size / MemoryLayout.size + ) + let status = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { rebound in + task_info( + mach_task_self_, + task_flavor_t(MACH_TASK_BASIC_INFO), + rebound, + &count + ) + } + } + return status == KERN_SUCCESS ? Int64(info.resident_size) : nil +} diff --git a/scripts/ios-ax-bridge-spike/swift/Sources/AgentDeviceIosAxBridgeSpike/Protocol.swift b/scripts/ios-ax-bridge-spike/swift/Sources/AgentDeviceIosAxBridgeSpike/Protocol.swift new file mode 100644 index 000000000..72bfd9780 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/swift/Sources/AgentDeviceIosAxBridgeSpike/Protocol.swift @@ -0,0 +1,139 @@ +import Foundation + +struct SpikeRequest: Decodable { + let version: Int + let id: String + let candidate: String + let simulatorUdid: String + let state: String + let screen: String + let appBundleId: String + let targetWindowName: String? + let targetProcessId: Int32? + let expectedTargetGeneration: String? + let limits: SpikeLimits +} + +struct SpikeLimits: Decodable { + let maxRequestBytes: Int + let maxResponseBytes: Int + let maxNodes: Int + let maxTraversalDepth: Int + let maxCpuMs: Int + let maxMemoryBytes: Int + let maxDurationMs: Int +} + +struct SpikeRect: Encodable { + let x: Double + let y: Double + let width: Double + let height: Double +} + +struct SpikeNode: Encodable { + let id: String + let type: String? + let parentId: String? + let role: String? + let subrole: String? + let label: String? + let value: String? + let identifier: String? + let frame: SpikeRect? + let enabled: Bool? + let selected: Bool? + let focused: Bool? +} + +struct SpikeViewport: Encodable { + let kind: String + let rect: SpikeRect? + let reason: String? + let coordinateSpace: String? + let source: String? +} + +struct SpikeResidue: Encodable { + let kind: String + let fields: [String]? + let dimension: String? + let limit: Int? + let fact: String? + let expected: String? + let observed: String? +} + +struct SpikeAcquisition: Encodable { + let targetId: String + let targetGeneration: String? + let nodes: [SpikeNode] + let viewport: SpikeViewport + let truncated: Bool + let residue: [SpikeResidue] +} + +struct SpikeFailure: Encodable { + let kind: String + let code: String? + let expectedTargetGeneration: String? + let observedTargetGeneration: String? +} + +struct SpikeMetrics: Encodable { + let requestBytes: Int + let responseBytes: Int + let nodeCount: Int + let maxTraversalDepth: Int + let cpuMs: Double? + let memoryBytes: Int64? + let durationMs: Double +} + +struct SpikeResponse: Encodable { + let version: Int + let id: String + let candidate: String + let ok: Bool + let acquisition: SpikeAcquisition? + let failure: SpikeFailure? + let metrics: SpikeMetrics +} + +struct SpikeCapture { + let acquisition: SpikeAcquisition? + let failure: SpikeFailure? + let maxTraversalDepth: Int +} + +func unsupportedCapture( + code: String, + observedTargetGeneration: String? = nil +) -> SpikeCapture { + SpikeCapture( + acquisition: nil, + failure: SpikeFailure( + kind: "unsupported-mechanism", + code: code, + expectedTargetGeneration: nil, + observedTargetGeneration: observedTargetGeneration + ), + maxTraversalDepth: 0 + ) +} + +func failureResponse( + request: SpikeRequest, + failure: SpikeFailure, + metrics: SpikeMetrics +) -> SpikeResponse { + SpikeResponse( + version: 1, + id: request.id, + candidate: request.candidate, + ok: false, + acquisition: nil, + failure: failure, + metrics: metrics + ) +} diff --git a/scripts/ios-ax-bridge-spike/swift/Sources/AgentDeviceIosAxBridgeSpike/RawAccessibility.swift b/scripts/ios-ax-bridge-spike/swift/Sources/AgentDeviceIosAxBridgeSpike/RawAccessibility.swift new file mode 100644 index 000000000..650fcff71 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/swift/Sources/AgentDeviceIosAxBridgeSpike/RawAccessibility.swift @@ -0,0 +1,329 @@ +import AppKit +import ApplicationServices +import Foundation + +private struct RawTraversalState { + var nodes: [SpikeNode] = [] + var visited: [AXUIElement] = [] + var maxDepth = 0 + var hitNodeLimit = false + var hitDepthLimit = false +} + +private struct RootSearchResult { + let element: AXUIElement + let depth: Int + let visitedCount: Int +} + +func capturePublicAccessibility(request: SpikeRequest) -> SpikeCapture { + guard request.version == 1, request.candidate == "public-macos-ax" else { + return unsupportedCapture(code: "candidate-not-supported") + } + guard AXIsProcessTrusted() else { + return unsupportedCapture(code: "host-accessibility-permission") + } + guard let application = targetApplication(request: request) else { + return unsupportedCapture(code: "target-application-unavailable") + } + + let generation = targetGeneration(application) + if let expected = request.expectedTargetGeneration, expected != generation { + return SpikeCapture( + acquisition: nil, + failure: SpikeFailure( + kind: "stale-generation", + code: "target-generation-mismatch", + expectedTargetGeneration: expected, + observedTargetGeneration: generation + ), + maxTraversalDepth: 0 + ) + } + + let applicationElement = AXUIElementCreateApplication(application.processIdentifier) + let windows = axWindows(applicationElement) + guard !windows.isEmpty else { + return unsupportedCapture( + code: "target-has-no-accessibility-windows", + observedTargetGeneration: generation + ) + } + guard let window = targetWindow(windows, name: request.targetWindowName) else { + return unsupportedCapture( + code: "target-simulator-window-unavailable", + observedTargetGeneration: generation + ) + } + guard let rootSearch = firstDescendant( + window, + subrole: "iOSContentGroup", + maxDepth: request.limits.maxTraversalDepth, + maxNodes: request.limits.maxNodes + ) else { + return unsupportedCapture( + code: "target-simulator-content-unavailable", + observedTargetGeneration: generation + ) + } + let traversalRoot = rootSearch.element + let traversalLimits = remainingLimits(request.limits, after: rootSearch) + + var state = RawTraversalState() + _ = appendRawNode( + traversalRoot, + parentId: nil, + depth: 0, + limits: traversalLimits, + state: &state + ) + let viewport = axRect(traversalRoot).map { + SpikeViewport( + kind: "reported", + rect: $0, + reason: nil, + coordinateSpace: "host-screen", + source: "AXPosition+AXSize" + ) + } ?? SpikeViewport( + kind: "missing", + rect: nil, + reason: "not-provided", + coordinateSpace: nil, + source: nil + ) + var residue: [SpikeResidue] = [] + if state.hitNodeLimit { + residue.append( + SpikeResidue( + kind: "truncated", + fields: nil, + dimension: "nodes", + limit: request.limits.maxNodes, + fact: nil, + expected: nil, + observed: nil + ) + ) + } + if state.hitDepthLimit { + residue.append( + SpikeResidue( + kind: "truncated", + fields: nil, + dimension: "depth", + limit: request.limits.maxTraversalDepth, + fact: nil, + expected: nil, + observed: nil + ) + ) + } + if viewport.kind == "missing" { + residue.append( + SpikeResidue( + kind: "missing-viewport", + fields: nil, + dimension: nil, + limit: nil, + fact: nil, + expected: nil, + observed: nil + ) + ) + } + return SpikeCapture( + acquisition: SpikeAcquisition( + targetId: "simulator:\(request.simulatorUdid)", + targetGeneration: generation, + nodes: state.nodes, + viewport: viewport, + truncated: state.hitNodeLimit || state.hitDepthLimit, + residue: residue + ), + failure: nil, + maxTraversalDepth: state.maxDepth + ) +} + +private func targetWindow(_ windows: [AXUIElement], name: String?) -> AXUIElement? { + guard let name else { return windows.first } + return windows.first { window in + guard let title = axString(window, kAXTitleAttribute as String) else { return false } + return title == name || title.hasPrefix("\(name) ") + } +} + +private func firstDescendant( + _ element: AXUIElement, + subrole: String, + maxDepth: Int, + maxNodes: Int +) -> RootSearchResult? { + var queue: [(AXUIElement, Int)] = [(element, 0)] + var visited = Set() + var retained: [AXUIElement] = [] + while !queue.isEmpty && visited.count < maxNodes { + let (candidate, depth) = queue.removeFirst() + if !visited.insert(ObjectIdentifier(candidate)).inserted { continue } + retained.append(candidate) + if axString(candidate, kAXSubroleAttribute as String) == subrole { + return RootSearchResult(element: candidate, depth: depth, visitedCount: visited.count) + } + if depth < maxDepth { + queue.append(contentsOf: axChildren(candidate).map { ($0, depth + 1) }) + } + } + return nil +} + +private func remainingLimits(_ limits: SpikeLimits, after search: RootSearchResult) -> SpikeLimits { + SpikeLimits( + maxRequestBytes: limits.maxRequestBytes, + maxResponseBytes: limits.maxResponseBytes, + maxNodes: max(1, limits.maxNodes - search.visitedCount + 1), + maxTraversalDepth: max(0, limits.maxTraversalDepth - search.depth), + maxCpuMs: limits.maxCpuMs, + maxMemoryBytes: limits.maxMemoryBytes, + maxDurationMs: limits.maxDurationMs + ) +} + +private func targetApplication(request: SpikeRequest) -> NSRunningApplication? { + if let processId = request.targetProcessId, + let application = NSRunningApplication(processIdentifier: processId), + !application.isTerminated + { + return application + } + return NSRunningApplication.runningApplications( + withBundleIdentifier: request.appBundleId + ).first(where: { !$0.isTerminated }) +} + +private func targetGeneration(_ application: NSRunningApplication) -> String { + let launch = application.launchDate?.timeIntervalSince1970.description ?? "unknown" + return "pid:\(application.processIdentifier):launch:\(launch)" +} + +@discardableResult +private func appendRawNode( + _ element: AXUIElement, + parentId: String?, + depth: Int, + limits: SpikeLimits, + state: inout RawTraversalState +) -> String? { + if state.visited.contains(where: { CFEqual($0, element) }) { return nil } + guard state.nodes.count < limits.maxNodes else { + state.hitNodeLimit = true + return nil + } + state.visited.append(element) + state.maxDepth = max(state.maxDepth, depth) + let id = "n\(state.nodes.count)" + state.nodes.append( + SpikeNode( + id: id, + type: nil, + parentId: parentId, + role: axString(element, kAXRoleAttribute as String), + subrole: axString(element, kAXSubroleAttribute as String), + label: axString(element, kAXTitleAttribute as String) + ?? axString(element, kAXDescriptionAttribute as String), + value: axString(element, kAXValueAttribute as String), + identifier: axString(element, "AXIdentifier"), + frame: axRect(element), + enabled: axBool(element, kAXEnabledAttribute as String), + selected: axBool(element, kAXSelectedAttribute as String), + focused: axBool(element, kAXFocusedAttribute as String) + ) + ) + guard depth < limits.maxTraversalDepth else { + if !axChildren(element).isEmpty { state.hitDepthLimit = true } + return id + } + for child in axChildren(element) { + _ = appendRawNode( + child, + parentId: id, + depth: depth + 1, + limits: limits, + state: &state + ) + } + return id +} + +private func axWindows(_ element: AXUIElement) -> [AXUIElement] { + axElements(element, attribute: kAXWindowsAttribute as String) +} + +private func axChildren(_ element: AXUIElement) -> [AXUIElement] { + axElements(element, attribute: kAXChildrenAttribute as String) +} + +private func axElements(_ element: AXUIElement, attribute: String) -> [AXUIElement] { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, + let elements = value as? [AXUIElement] + else { + return [] + } + return elements +} + +private func axString(_ element: AXUIElement, _ attribute: String) -> String? { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, + let text = value as? String + else { + return nil + } + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? nil : trimmed +} + +private func axBool(_ element: AXUIElement, _ attribute: String) -> Bool? { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, + let number = value as? NSNumber + else { + return nil + } + return number.boolValue +} + +private func axRect(_ element: AXUIElement) -> SpikeRect? { + var positionValue: CFTypeRef? + var sizeValue: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue) == .success, + AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) == .success, + let position = axPoint(positionValue), + let size = axSize(sizeValue) + else { + return nil + } + return SpikeRect( + x: Double(position.x), + y: Double(position.y), + width: Double(size.width), + height: Double(size.height) + ) +} + +private func axPoint(_ value: CFTypeRef?) -> CGPoint? { + guard let value, CFGetTypeID(value) == AXValueGetTypeID() else { return nil } + let axValue = value as! AXValue + guard AXValueGetType(axValue) == .cgPoint else { return nil } + var point = CGPoint.zero + return AXValueGetValue(axValue, .cgPoint, &point) ? point : nil +} + +private func axSize(_ value: CFTypeRef?) -> CGSize? { + guard let value, CFGetTypeID(value) == AXValueGetTypeID() else { return nil } + let axValue = value as! AXValue + guard AXValueGetType(axValue) == .cgSize else { return nil } + var size = CGSize.zero + return AXValueGetValue(axValue, .cgSize, &size) ? size : nil +} diff --git a/scripts/ios-ax-bridge-spike/swift/Sources/AgentDeviceIosAxBridgeSpike/main.swift b/scripts/ios-ax-bridge-spike/swift/Sources/AgentDeviceIosAxBridgeSpike/main.swift new file mode 100644 index 000000000..40f0e5421 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/swift/Sources/AgentDeviceIosAxBridgeSpike/main.swift @@ -0,0 +1,73 @@ +import Foundation + +let encoder = JSONEncoder() +encoder.outputFormatting = [.sortedKeys] +let decoder = JSONDecoder() +var inputBuffer = Data() + +while true { + let chunk = FileHandle.standardInput.readData(ofLength: 4 * 1024) + if chunk.isEmpty { break } + inputBuffer.append(chunk) + while let newline = inputBuffer.firstIndex(of: 0x0A) { + let line = inputBuffer.subdata(in: inputBuffer.startIndex.. 64 * 1024 { + writeLog("request frame exceeded 65536 bytes") + inputBuffer.removeAll(keepingCapacity: true) + } +} + +if !inputBuffer.isEmpty { handleRequest(inputBuffer) } + +func handleRequest(_ line: Data) { + guard line.count <= 64 * 1024 else { + writeLog("discarded oversized request frame") + return + } + let start = currentResourceUsage() + let started = DispatchTime.now().uptimeNanoseconds + do { + let request = try decoder.decode(SpikeRequest.self, from: line) + writeLog("capture id=\(request.id) candidate=\(request.candidate) screen=\(request.screen)") + let capture = capturePublicAccessibility(request: request) + let elapsedMs = Double(DispatchTime.now().uptimeNanoseconds - started) / 1_000_000 + let processMetrics = spikeProcessMetrics(since: start) + let baseMetrics = SpikeMetrics( + requestBytes: line.count + 1, + responseBytes: 0, + nodeCount: capture.acquisition?.nodes.count ?? 0, + maxTraversalDepth: capture.maxTraversalDepth, + cpuMs: processMetrics.cpuMs, + memoryBytes: processMetrics.memoryBytes, + durationMs: elapsedMs + ) + let response = capture.failure.map { + failureResponse(request: request, failure: $0, metrics: baseMetrics) + } ?? SpikeResponse( + version: 1, + id: request.id, + candidate: request.candidate, + ok: true, + acquisition: capture.acquisition, + failure: nil, + metrics: baseMetrics + ) + writeResponse(response) + } catch { + writeLog("malformed request frame") + } +} + +func writeResponse(_ response: SpikeResponse) { + guard let data = try? encoder.encode(response) else { return } + FileHandle.standardOutput.write(data) + FileHandle.standardOutput.write(Data([0x0A])) +} + +func writeLog(_ message: String) { + let data = Data("[ios-ax-spike] \(message)\n".utf8) + FileHandle.standardError.write(data) +} diff --git a/scripts/ios-ax-bridge-spike/types.ts b/scripts/ios-ax-bridge-spike/types.ts new file mode 100644 index 000000000..fe98e3ba6 --- /dev/null +++ b/scripts/ios-ax-bridge-spike/types.ts @@ -0,0 +1,227 @@ +import type { + IosAcquisitionResidue, + IosViewportEvidence, +} from '@agent-device/contracts/ios-snapshot'; +import type { DeepButtonEvidence, LocalState, ScreenId } from '../ios-snapshot-benchmark/types.ts'; + +export const SPIKE_SCHEMA_VERSION = 'ios-simulator-ax-bridge-spike.v1' as const; +export const SPIKE_ISSUE = '#2192' as const; +export const SPIKE_PARENT = '#2188' as const; +export const SPIKE_PREREQUISITES = ['#2189', '#2190'] as const; + +export type CandidateId = 'public-macos-ax' | 'private-coresimulator-ax' | 'xctest-control'; + +export type SpikeFailureKind = + | 'unsupported-mechanism' + | 'malformed-tree' + | 'stale-generation' + | 'timeout' + | 'cancelled' + | 'process-crash' + | 'transport-failure'; + +export type SpikeFailure = Readonly<{ + kind: SpikeFailureKind; + code?: string; + expectedTargetGeneration?: string; + observedTargetGeneration?: string; +}>; + +export type SpikeRect = Readonly<{ + x: number; + y: number; + width: number; + height: number; +}>; + +export type RawAcquiredNode = Readonly<{ + id: string; + type?: string; + parentId?: string; + role?: string; + subrole?: string; + label?: string; + value?: string; + identifier?: string; + frame?: SpikeRect; + enabled?: boolean; + selected?: boolean; + focused?: boolean; +}>; + +export type RawAcquisition = Readonly<{ + targetId: string; + targetGeneration: string | null; + nodes: readonly RawAcquiredNode[]; + viewport: IosViewportEvidence; + truncated: boolean; + residue: readonly IosAcquisitionResidue[]; +}>; + +export type ResourceLimits = Readonly<{ + maxRequestBytes: number; + maxResponseBytes: number; + maxNodes: number; + maxTraversalDepth: number; + maxCpuMs: number; + maxMemoryBytes: number; + maxDurationMs: number; +}>; + +export type ResourceMetrics = Readonly<{ + requestBytes: number; + responseBytes: number; + nodeCount: number; + maxTraversalDepth: number; + cpuMs: number | null; + memoryBytes: number | null; + durationMs: number; +}>; + +export type SpikeRequest = Readonly<{ + version: 1; + id: string; + candidate: CandidateId; + simulatorUdid: string; + state: LocalState; + screen: ScreenId | 'unprepared-surface'; + appBundleId: string; + targetWindowName?: string; + targetProcessId?: number; + expectedTargetGeneration?: string; + limits: ResourceLimits; +}>; + +export type SpikeResponse = Readonly<{ + version: 1; + id: string; + candidate: CandidateId; + ok: boolean; + acquisition?: RawAcquisition; + failure?: SpikeFailure; + metrics: ResourceMetrics; +}>; + +export type PresentationMeasurement = Readonly<{ + ok: boolean; + payloadBytes: number; + nodeCount: number; + durationMs: number; + cpuMs: number | null; + memoryBytes: number | null; +}>; + +export type SpikeSample = Readonly<{ + index: number; + candidate: CandidateId; + state: LocalState; + screen: ScreenId; + startedAt: string; + finishedAt: string; + operation: 'acquisition' | 'presentation'; + wallClockMs: number; + preparationMs?: number; + firstLookMs?: number; + firstTree: 'readable' | 'empty' | 'unreadable' | 'not-observed'; + ok: boolean; + stderr?: string; + acquisition?: RawAcquisition; + metrics?: ResourceMetrics; + presentation?: PresentationMeasurement; + failure?: SpikeFailure; +}>; + +export type ProtocolProbeLog = Readonly<{ + candidate: Exclude; + id: string; + stderr: string; +}>; + +export type SpikeCell = Readonly<{ + candidate: CandidateId; + state: LocalState; + screen: ScreenId; + sampleMinimum: number; + acquisitionSamples: readonly SpikeSample[]; + presentationSamples: readonly SpikeSample[]; +}>; + +export type Toolchain = Readonly<{ + node: string; + pnpm: string; + xcode: string; + simctl: string; + os: string; + arch: string; + swift: string; +}>; + +export type Target = Readonly<{ + udid: string; + name: string; + runtime: string; +}>; + +export type PlistKeyChange = Readonly<{ + key: string; + before?: unknown; + after?: unknown; +}>; + +export type PlistDiff = Readonly<{ + path: string; + existedBefore: boolean; + beforeSha256: string | null; + afterSha256: string | null; + changes: readonly PlistKeyChange[]; +}>; + +export type PreferenceEvidence = Readonly<{ + applied: boolean; + restored: boolean; + fixtureLaunchCompatible: boolean | null; + simulatorStateBefore: string; + diffs: readonly PlistDiff[]; +}>; + +export type LifecycleEvidence = Readonly<{ + source: 'framed-protocol-fixture'; + crash: Readonly<{ failure: SpikeFailureKind; recovered: boolean }>; + timeout: Readonly<{ failure: SpikeFailureKind; recovered: boolean }>; + cancellation: Readonly<{ failure: SpikeFailureKind; recovered: boolean }>; + staleGeneration: Readonly<{ failure: SpikeFailureKind; recovered: boolean }>; +}>; + +export type SpikeReport = Readonly<{ + schemaVersion: typeof SPIKE_SCHEMA_VERSION; + issue: typeof SPIKE_ISSUE; + parent: typeof SPIKE_PARENT; + prerequisites: readonly string[]; + generatedAt: string; + revision: Readonly<{ commit: string; branch: string; dirty: boolean }>; + toolchain: Toolchain; + target: Target; + limits: ResourceLimits; + status: 'completed' | 'stopped'; + corpusCoverage: 'full' | 'decisive-early-stop'; + candidates: readonly CandidateId[]; + config: Readonly<{ + states: readonly LocalState[]; + screens: readonly ScreenId[]; + requestedSamples: number; + }>; + protocolProbes: readonly SpikeResponse[]; + protocolProbeLogs: readonly ProtocolProbeLog[]; + preferenceEvidence: PreferenceEvidence; + lifecycle: LifecycleEvidence; + positiveControl: DeepButtonEvidence; + cells: readonly SpikeCell[]; + decision: 'GO' | 'NO-GO'; + decisionReasons: readonly string[]; + nextInterface: string; + stop?: Readonly<{ + category: 'infrastructure' | 'configuration'; + message: string; + command?: string; + }>; +}>; diff --git a/scripts/ios-snapshot-benchmark/cell-admission.ts b/scripts/ios-snapshot-benchmark/cell-admission.ts index 6b822e3d2..bf079cb51 100644 --- a/scripts/ios-snapshot-benchmark/cell-admission.ts +++ b/scripts/ios-snapshot-benchmark/cell-admission.ts @@ -7,6 +7,7 @@ import { formatCliFailure, openFixture, pressFixtureTarget, + scrollFixtureToBottom, snapshotFixture, snapshotHasAnchor, type CliContext, @@ -88,6 +89,22 @@ export function admitSuccessfulSample( return admitNonWarmSample(context, options, previousAppPid); } +export function admitStableWarmSample( + options: CellAdmissionOptions, + previousAppPid: number, +): number { + assertReadyState(options); + const appPid = assertAppRunning(options.udid, options.fixture.app); + if (appPid !== previousAppPid) { + throw new BenchmarkCellAdmissionError( + 'cell-state', + `Warm cell ${options.fixture.id} changed app PID from ${String(previousAppPid)} to ${String(appPid)}.`, + 'agent-device batch --steps snapshot', + ); + } + return appPid; +} + function admitNonWarmSample( context: CliContext, options: CellAdmissionOptions, @@ -146,6 +163,8 @@ function admitOpenedFixture(context: CliContext, options: CellAdmissionOptions): ); requireAnchor(observed, options.fixture); if (options.fixture.setupAction === 'open-alert') { + const scrolled = scrollFixtureToBottom(context); + requireCommandSuccess(scrolled, `${options.fixture.id} setup scroll`, 'cell-state'); const setup = pressFixtureTarget(context, 'id="automation-open-alert"'); requireCommandSuccess(setup, `${options.fixture.id} setup action`, 'cell-state'); const prepared = snapshotFixture(context); @@ -154,7 +173,11 @@ function admitOpenedFixture(context: CliContext, options: CellAdmissionOptions): `${options.fixture.id} post-setup semantic anchor observation`, 'fixture-anchor', ); - requireAnchor(prepared, options.fixture); + requireAnchorText( + prepared, + options.fixture.setupAnchorText ?? options.fixture.anchorText, + options.fixture.id, + ); } const appPid = assertAppRunning(options.udid, options.fixture.app); return appPid; @@ -223,10 +246,14 @@ function assertAppRunning(udid: string, appId: string): number { } function requireAnchor(result: CliResult, fixture: ScreenFixture): void { - if (!snapshotHasAnchor(result.payload, fixture.anchorText)) { + requireAnchorText(result, fixture.anchorText, fixture.id); +} + +function requireAnchorText(result: CliResult, anchorText: string, fixtureId: string): void { + if (!snapshotHasAnchor(result.payload, anchorText)) { throw new BenchmarkCellAdmissionError( 'fixture-anchor', - `Fixture ${fixture.id} did not expose the exact anchor ${JSON.stringify(fixture.anchorText)}.`, + `Fixture ${fixtureId} did not expose the exact anchor ${JSON.stringify(anchorText)}.`, 'agent-device snapshot', ); } diff --git a/scripts/ios-snapshot-benchmark/command.test.ts b/scripts/ios-snapshot-benchmark/command.test.ts index 323d3d50e..510785c91 100644 --- a/scripts/ios-snapshot-benchmark/command.test.ts +++ b/scripts/ios-snapshot-benchmark/command.test.ts @@ -65,20 +65,19 @@ test('admits only an exact semantic anchor from snapshot node fields', () => { }); test('recognizes the first-install deep-link confirmation as a setup prompt', () => { + for (const node of [ + { role: 'alert', label: 'Open in “Agent Device Tester”?' }, + { type: 'Alert', label: 'Open in “Agent Device Tester”?' }, + ]) { + assert.equal( + hasDeepLinkConfirmation({ + data: { results: [{ data: { snapshot: { nodes: [node] } } }] }, + }), + true, + ); + } assert.equal( - hasDeepLinkConfirmation({ - data: { - results: [ - { - data: { - snapshot: { - nodes: [{ role: 'alert', label: 'Open in “Agent Device Tester”?' }], - }, - }, - }, - ], - }, - }), + hasDeepLinkConfirmation({ data: { message: 'Open in “Agent Device Tester”?' } }), true, ); assert.equal( diff --git a/scripts/ios-snapshot-benchmark/command.ts b/scripts/ios-snapshot-benchmark/command.ts index ca6696ab8..9c6ac9f24 100644 --- a/scripts/ios-snapshot-benchmark/command.ts +++ b/scripts/ios-snapshot-benchmark/command.ts @@ -52,7 +52,7 @@ export function openFixture( '--foreground', ]); if (!fixture.launchUrl || !hasDeepLinkConfirmation(opened.payload)) return opened; - const accepted = pressFixtureTarget(context, 'label="Open"'); + const accepted = runCli(context, ['alert', 'accept']); if (accepted.ok) return opened; return { ...opened, @@ -75,7 +75,7 @@ export async function openFixtureAsync( '--foreground', ]); if (!fixture.launchUrl || !hasDeepLinkConfirmation(opened.payload)) return opened; - const accepted = await pressFixtureTargetAsync(context, 'label="Open"'); + const accepted = await runCliAsync(context, ['alert', 'accept']); if (accepted.ok) return opened; return { ...opened, @@ -105,6 +105,14 @@ export function pressFixtureTarget(context: CliContext, selector: string): CliRe return runCli(context, ['click', selector]); } +export function scrollFixtureToBottom(context: CliContext): CliResult { + return runCli(context, ['scroll', 'bottom', '--settle']); +} + +export async function scrollFixtureToBottomAsync(context: CliContext): Promise { + return await runCliAsync(context, ['scroll', 'bottom', '--settle']); +} + export async function pressFixtureTargetAsync( context: CliContext, selector: string, @@ -123,8 +131,11 @@ export function snapshotHasAnchor(payload: unknown, anchorText: string): boolean } export function hasDeepLinkConfirmation(payload: unknown): boolean { + const root = asRecord(payload); + const data = asRecord(root?.data); + if (readString(data?.message)?.startsWith('Open in ') === true) return true; return snapshotNodes(payload).some((record) => { - const role = readString(record.role); + const role = readString(record.role) ?? readString(record.type)?.toLowerCase(); const label = readString(record.label); return role === 'alert' && label?.startsWith('Open in ') === true; }); diff --git a/scripts/ios-snapshot-benchmark/definitions.ts b/scripts/ios-snapshot-benchmark/definitions.ts index 6d86dc42a..c1857246a 100644 --- a/scripts/ios-snapshot-benchmark/definitions.ts +++ b/scripts/ios-snapshot-benchmark/definitions.ts @@ -15,7 +15,7 @@ const FIXTURE_APP_ID = 'com.callstack.agentdevicelab'; const FIXTURE_SCHEME = 'agent-device-test-app://'; const IOS_SETTINGS_APP_ID = 'com.apple.Preferences'; -const SCREEN_FIXTURES: readonly ScreenFixture[] = [ +export const SCREEN_FIXTURES: readonly ScreenFixture[] = [ { id: 'quiet', label: 'Quiet inert surface', @@ -44,6 +44,7 @@ const SCREEN_FIXTURES: readonly ScreenFixture[] = [ launchUrl: `${FIXTURE_SCHEME}/automation`, anchorText: 'Automation lab', setupAction: 'open-alert', + setupAnchorText: 'Automation confirmation', }, { id: 'system-surface', @@ -60,6 +61,8 @@ const SCREEN_FIXTURES: readonly ScreenFixture[] = [ }, ]; +export const LOCAL_STATES: readonly LocalState[] = ['cold-cold', 'cold', 'warm', 'relaunch']; + export function screenFixture(id: ScreenId): ScreenFixture { const fixture = SCREEN_FIXTURES.find((candidate) => candidate.id === id); if (!fixture) throw new Error(`Unknown screen fixture: ${id}`); @@ -83,7 +86,7 @@ export function parseLocalStates(value: string | undefined): LocalState[] { .split(',') .map((item) => item.trim()) .filter(Boolean) as LocalState[]; - const valid = new Set(['cold-cold', 'cold', 'warm', 'relaunch']); + const valid = new Set(LOCAL_STATES); const unknown = states.filter((state) => !valid.has(state)); if (unknown.length > 0) throw new Error(`Unknown --state value: ${unknown.join(', ')}`); if (states.length === 0) throw new Error('--state requires at least one cell.'); diff --git a/scripts/ios-snapshot-benchmark/proxy-client-support.ts b/scripts/ios-snapshot-benchmark/proxy-client-support.ts index 0b9608a3d..736182174 100644 --- a/scripts/ios-snapshot-benchmark/proxy-client-support.ts +++ b/scripts/ios-snapshot-benchmark/proxy-client-support.ts @@ -18,7 +18,10 @@ import type { Failure, ProxyNetwork, RawSample, ScreenFixture } from './types.ts export type AgentClient = { apps: { open(options: Record): Promise }; - interactions: { click(options: Record): Promise }; + interactions: { + click(options: Record): Promise; + scroll(options: Record): Promise; + }; batch: { run(options: Record): Promise> }; sessions: { close(): Promise }; leases: { @@ -106,6 +109,12 @@ export async function openClientFixture( foreground: true, }); if (fixture.setupAction === 'open-alert') { + await client.interactions.scroll({ + direction: 'bottom', + settle: true, + platform: 'ios', + udid, + }); await client.interactions.click({ target: { kind: 'selector', selector: 'id="automation-open-alert"' }, platform: 'ios', diff --git a/scripts/ios-snapshot-benchmark/proxy-client.ts b/scripts/ios-snapshot-benchmark/proxy-client.ts index 4bedfbbd9..234a97c94 100644 --- a/scripts/ios-snapshot-benchmark/proxy-client.ts +++ b/scripts/ios-snapshot-benchmark/proxy-client.ts @@ -2,6 +2,7 @@ import path from 'node:path'; import { openFixtureAsync, pressFixtureTargetAsync, + scrollFixtureToBottomAsync, sampleFromCli, snapshotFixtureAsync, } from './command.ts'; @@ -140,6 +141,8 @@ async function prepareFreshCliFixture( fixture: ScreenFixture, ): Promise { if (fixture.setupAction !== 'open-alert') return; + const scrolled = await scrollFixtureToBottomAsync(context); + if (!scrolled.ok) throw setupFailure('fresh CLI alert setup scroll', scrolled); const pressed = await pressFixtureTargetAsync(context, 'id="automation-open-alert"'); if (!pressed.ok) throw setupFailure('fresh CLI alert setup', pressed); } diff --git a/scripts/ios-snapshot-benchmark/types.ts b/scripts/ios-snapshot-benchmark/types.ts index 134effa3f..a9398a18e 100644 --- a/scripts/ios-snapshot-benchmark/types.ts +++ b/scripts/ios-snapshot-benchmark/types.ts @@ -38,6 +38,7 @@ export type ScreenFixture = { launchUrl?: string; anchorText: string; setupAction?: 'open-alert'; + setupAnchorText?: string; }; export type Failure = { diff --git a/vitest.config.ts b/vitest.config.ts index f34ea31ff..fd59b8044 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -147,10 +147,11 @@ export default defineConfig({ // The Bundle Size lane's PR-comment path: spawns the real script against a // stubbed fetch, so it needs no network; pins retry/reconcile/fatal outcomes. 'scripts/__tests__/size-report-post-comment.test.ts', - // Package attribution models npm-pack output. Keep it in the fast lane so every new - // package path remains accounted for. + // Package attribution is a pure npm-pack manifest model. Keep it in the fast lane so + // every new package path remains accounted for without building an archive. 'scripts/__tests__/size-report-package.test.ts', 'scripts/ios-snapshot-benchmark/*.test.ts', + 'scripts/ios-ax-bridge-spike/*.test.ts', // Parses CI configuration only, so this action guard needs no device or subprocess lane. 'test/ci/upload-agent-device-artifacts.test.ts', 'test/ci/upload-artifact-hidden-paths.test.ts',