Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions docs/CLI.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,10 +120,17 @@ without speculative work stages; reclaiming work is reported separately:
```json
{"event":"queued","queue_position":1}
{"event":"provisioning","eta_seconds":90}
{"event":"booting","eta_seconds":30}
{"event":"reclaiming","eta_seconds":15}
{"event":"booting","eta_seconds":60}
{"event":"reclaiming","eta_seconds":34}
```

`reclaiming` follows `queued` when the device the request is waiting on is
being purged for its previous holder: the position alone would not say that
the wait is an iOS erase rather than a moment. Every `eta_seconds` comes from
the driver's own estimate for the work it selected, which for a reclaim means
the strategy that clean level uses -- an erase runs tens of seconds, a
snapshot restore a few.

Once granted, held mode also relays the health monitor's findings about the
leased device for as long as the connection holds it, on the same stderr
stream:
Expand Down
44 changes: 44 additions & 0 deletions e2e/contention.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,52 @@ describe("contention & queueing", () => {
waiterE.kill("SIGTERM");
await waiterE.waitForExit(15_000);
});

it("tells a waiter queued behind a reclaim how long that reclaim runs", async () => {
const env = await withDaemon({
configOverrides: { limits: { maxRunning: 1, ios: { maxDevices: 1, maxRunning: 1 } } },
});
await env.driverScript.set({
ios: {
knownModels: ["iPhone 16"],
availableOsVersions: ["18.4"],
estimateMs: { reclaim: 34_000 },
latencyMs: { reclaim: 4_000 },
},
});

const holder = await env.cli([...LEASE_ARGS, "--agent-id", "agent-holder", "--detach"]);
expect(holder.code).toBe(0);
const holderGrant = holder.json as { lease: string };

// Release hands the caller back before the purge (#58), so the device is `reclaiming` with
// its full latency still ahead when the next requester arrives -- the case the stage exists
// for, and the one nothing exercised before this covered the whole path out to the CLI.
const release = await env.cli(["release", holderGrant.lease]);
expect(release.code).toBe(0);

const waiter = env.cliBackground([...LEASE_ARGS, "--agent-id", "agent-waiter"]);
await waitFor(() => waiter.progressEvents().some(isReclaimingWithEta(34)), {
label: "agent-waiter told the reclaim's ETA",
});
expect(waiter.progressEvents().some((event) => isQueuedAt(event, 1))).toBe(true);

const granted = JSON.parse(await waiter.firstStdoutLine(15_000)) as { device: string };
expect(granted.device).toBe("iPhone 16");

waiter.kill("SIGTERM");
await waiter.waitForExit(15_000);
});
});

function isReclaimingWithEta(seconds: number): (event: unknown) => boolean {
return (event) =>
typeof event === "object" &&
event !== null &&
(event as Record<string, unknown>).event === "reclaiming" &&
(event as Record<string, unknown>).eta_seconds === seconds;
}

function isQueuedAt(event: unknown, position: number): boolean {
return (
typeof event === "object" &&
Expand Down
5 changes: 3 additions & 2 deletions e2e/fake-driver/fake-driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
type Driver,
type DriverCatalogEntry,
type DriverDevice,
type DriverEstimate,
type DriverReality,
type ObservedMark,
type ObservedRunState,
Expand Down Expand Up @@ -187,10 +188,10 @@ export class OutOfProcessFakeDriver implements Driver {
};
}

estimate(operation: "provision" | "boot" | "reclaim", _spec: DeviceSpec): number {
estimate(estimate: DriverEstimate, _spec: DeviceSpec): number {
// estimate() is synchronous in the Driver interface, so it reads the script
// synchronously best-effort; a stale/missing read just falls back to 0.
return this.#lastKnownEstimateMs?.[operation] ?? 0;
return this.#lastKnownEstimateMs?.[estimate.operation] ?? 0;
}

async #beforeCall(
Expand Down
16 changes: 7 additions & 9 deletions src/core/acquisition-planner.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
import type { CapacityReservation, CapacityCoordinator } from "./capacity-coordinator.js";
import type { DeviceOperationClaim, DeviceOperationClaims } from "./device-operation-claims.js";
import type { DeviceRecord, DeviceSpec, LeaseRecord, Platform } from "./domain.js";
import {
type DeviceRecord,
type DeviceSpec,
type LeaseRecord,
type Platform,
sameSpec,
} from "./domain.js";
import { selectManagedVictim, selectWarmVictim, type WarmVictimScope } from "./warm-pool.js";

export interface AcquisitionPlannerSnapshot {
Expand Down Expand Up @@ -154,11 +160,3 @@ function capacityDevices(snapshot: AcquisitionPlannerSnapshot) {
state: device.state,
}));
}

function sameSpec(left: DeviceSpec, right: DeviceSpec): boolean {
return (
left.platform === right.platform &&
left.model === right.model &&
left.osVersion === right.osVersion
);
}
10 changes: 8 additions & 2 deletions src/core/device-provisioner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,10 @@ export class DeviceProvisioner {
const startedAt = this.options.clock.now();
let driverDevice: DriverDevice;
try {
options.onProgress?.({ stage: "provisioning", etaMs: driver.estimate("provision", spec) });
options.onProgress?.({
stage: "provisioning",
etaMs: driver.estimate({ operation: "provision" }, spec),
});
driverDevice = await driver.provision(spec);
} catch (error: unknown) {
options.reservation.release();
Expand All @@ -66,7 +69,10 @@ export class DeviceProvisioner {
}

try {
options.onProgress?.({ stage: "booting", etaMs: driver.estimate("boot", spec) });
options.onProgress?.({
stage: "booting",
etaMs: driver.estimate({ operation: "boot" }, spec),
});
const ready = await this.options.lifecycle.readyProvisionedForLease(device);
if (ready === undefined)
throw new Error(`Registered device could not be made ready: ${device.id}`);
Expand Down
52 changes: 52 additions & 0 deletions src/core/doctor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -846,6 +846,58 @@ describe("Doctor", () => {
);
});

it("holds a reclaiming device to the slower clean level's estimate", async () => {
const clock = new FakeClock(10_000);
const eventBus = new EventBus(clock);
const registry = await Registry.load({
clock,
eventBus,
filesystem: new MemoryFilesystem(),
idGenerator: sequence(),
statePath: "/state.json",
});
// A record in `reclaiming` does not say which clean level started it, so the threshold
// has to clear the slower of the two: pricing it at `standard` would call a healthy
// `full` reclaim a stall.
const driver = new FakeDriver({
clock,
estimateMs: { reclaim: 2_000 },
fullCleanReclaimEstimateMs: 20_000,
platform: "ios",
});
const device = await readyDevice(registry, "simlock-slow-clean", "ios");
const lease = await registry.createLease({
deviceId: device.id,
mode: "held",
requesterId: "agent",
ttlDeadline: 999_999,
});
await registry.beginRelease(lease.id);
const doctor = new Doctor({
clock,
config: config(),
drivers: [driver],
eventBus,
registry,
});

// Past the `standard` threshold (2_000 * 3) but inside the `full` one (20_000 * 3).
clock.advance(6_001);
const early = await doctor.reconcile();
expect(early.findings.filter((finding) => finding.kind === "stalled-transition")).toEqual([]);

clock.advance(60_000);
const late = await doctor.reconcile();
expect(late.findings).toContainEqual(
expect.objectContaining({
deviceId: device.id,
kind: "stalled-transition",
state: "reclaiming",
thresholdMs: 60_000,
}),
);
});

it("emits device.stalled-transition-detected for a stalled device", async () => {
const clock = new FakeClock(10_000);
const eventBus = new EventBus(clock);
Expand Down
32 changes: 25 additions & 7 deletions src/core/doctor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type { Clock } from "../ports/index.js";
import type { Config } from "./config.js";
import {
type DeviceRecord,
type DeviceSpec,
type DeviceState,
type Platform,
transitionEnteredAt,
Expand Down Expand Up @@ -326,6 +327,21 @@ function registryDriftFindings(
return findings;
}

/**
* The reclaim estimate for the slowest strategy the driver would pick, whatever the clean
* level. A `reclaiming` record does not say which level started it -- every production path
* asks for `standard`, but that is the caller's choice, not an invariant this finding can
* depend on -- and the two errors are not symmetric: an estimate that is too tight turns a
* healthy reclaim into a false stall finding, while one that is too loose only delays a real
* one. So this takes the slower branch rather than assuming.
*/
function slowestReclaimEstimateMs(driver: Driver, spec: DeviceSpec): number {
return Math.max(
driver.estimate({ clean: "standard", operation: "reclaim" }, spec),
driver.estimate({ clean: "full", operation: "reclaim" }, spec),
);
}

/**
* A `provisioning` / `reclaiming` device is normally in-flight work Simlock itself is
* driving (see `expectedRunState`), not drift -- but only up to a point. Past a
Expand All @@ -348,11 +364,12 @@ function registryDriftFindings(
* runs. This is the same live-versus-orphaned test `StartupConverger
* #recoverInterruptedReclaims` already makes, and it is load-bearing rather than
* belt-and-braces -- every release now backgrounds its reclaim, holding the device in
* `reclaiming` for a full erase, measured at ~34s against a threshold that floors at
* 60s for both drivers, and several such erases run at once and contend for the same
* disk. Tuning the threshold against that would be guessing at disk speed; the claim
* answers it exactly. A reclaim orphaned by a crash has no claim in the new process,
* so the case this finding exists to catch is untouched.
* `reclaiming` for a full erase, measured at ~34s, and several such erases run at once
* and contend for the same disk. The estimate the threshold is built from now reflects
* that erase rather than the 1s it used to claim (#56), but tuning a threshold against
* contended disk speed would still be guessing; the claim answers it exactly. A reclaim
* orphaned by a crash has no claim in the new process, so the case this finding exists to
* catch is untouched.
*/
function stalledTransitionFinding(
device: DeviceRecord,
Expand All @@ -375,8 +392,9 @@ function stalledTransitionFinding(

const estimateMs =
device.state === "provisioning"
? driver.estimate("provision", device.spec) + driver.estimate("boot", device.spec)
: driver.estimate("reclaim", device.spec);
? driver.estimate({ operation: "provision" }, device.spec) +
driver.estimate({ operation: "boot" }, device.spec)
: slowestReclaimEstimateMs(driver, device.spec);
const thresholdMs = Math.max(estimateMs * config.thresholdMultiplier, config.minimumThresholdMs);
const ageMs = now - enteredAt;
if (ageMs <= thresholdMs) {
Expand Down
9 changes: 9 additions & 0 deletions src/core/domain.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,15 @@ export interface DeviceSpec {
readonly osVersion: string;
}

/** Spec identity as every selection path means it: same platform, model, and OS version. */
export function sameSpec(left: DeviceSpec, right: DeviceSpec): boolean {
return (
left.platform === right.platform &&
left.model === right.model &&
left.osVersion === right.osVersion
);
}

export type DeviceState =
| "provisioning"
| "ready"
Expand Down
15 changes: 14 additions & 1 deletion src/core/driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,19 @@ export interface ReclaimResult {

export type ReclaimStrategy = ReclaimResult["strategy"];

/**
* What `estimate` is being asked to price. `reclaim` carries the clean level because it is
* the input `reclaimStrategy` already selects on, and the strategies it picks between differ
* by an order of magnitude -- an iOS `erase` runs tens of seconds while an Android `snapshot`
* restore runs in a few. A single blended reclaim number cannot be right for both, and the
* callers that consume it (a requester's ETA, `Doctor`'s stalled-transition threshold) are
* both misled by one that is wrong in the optimistic direction.
*/
export type DriverEstimate =
| { readonly operation: "provision" }
| { readonly operation: "boot" }
| { readonly operation: "reclaim"; readonly clean: "standard" | "full" };

/**
* What a driver can resolve right now, read from the platform SDK without
* side effects: resolvable device models plus installed runtimes / system
Expand Down Expand Up @@ -107,7 +120,7 @@ export interface Driver {
listManaged(): Promise<DriverReality>;
/** Read-only: must never trigger a runtime / system-image download. */
listCatalog(): Promise<DriverCatalogEntry>;
estimate(operation: "provision" | "boot" | "reclaim", spec: DeviceSpec): number;
estimate(estimate: DriverEstimate, spec: DeviceSpec): number;
}

export class RuntimeMissingError extends Error {
Expand Down
6 changes: 3 additions & 3 deletions src/core/fake-driver.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,9 @@ describe("FakeDriver", () => {
});
const spec = { model: "iPhone 16", osVersion: "26.5", platform: "ios" } as const;

expect(driver.estimate("provision", spec)).toBe(10);
expect(driver.estimate("boot", spec)).toBe(20);
expect(driver.estimate("reclaim", spec)).toBe(30);
expect(driver.estimate({ operation: "provision" }, spec)).toBe(10);
expect(driver.estimate({ operation: "boot" }, spec)).toBe(20);
expect(driver.estimate({ clean: "standard", operation: "reclaim" }, spec)).toBe(30);
});

it("implements the platform-agnostic Driver contract", () => {
Expand Down
16 changes: 14 additions & 2 deletions src/core/fake-driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
type Driver,
type DriverCatalogEntry,
type DriverDevice,
type DriverEstimate,
type DriverReality,
type ObservedRunState,
RuntimeMissingError,
Expand Down Expand Up @@ -32,6 +33,12 @@ export interface FakeDriverOptions {
readonly availableOsVersions?: readonly string[];
readonly clock: Clock;
readonly estimateMs?: Partial<Record<DriverEstimateOperation, number>>;
/**
* Reclaim estimate for a `full` clean, when a test needs the two clean levels priced apart
* the way a real driver prices them (an Android `snapshot` against a `wipe`). Falls back to
* `estimateMs.reclaim`, so a test that does not care about the split says nothing.
*/
readonly fullCleanReclaimEstimateMs?: number;
readonly knownModels?: readonly string[];
readonly latencyMs?: Partial<Record<FakeDriverOperation, number>>;
readonly platform: Platform;
Expand All @@ -53,6 +60,7 @@ export class FakeDriver implements Driver {
readonly #calls: FakeDriverCall[] = [];
readonly #clock: Clock;
readonly #estimateMs: FakeDriverOptions["estimateMs"];
readonly #fullCleanReclaimEstimateMs: number | undefined;
readonly #failures = new Map<string, Error>();
#hangMakeReady = false;
readonly #knownModels: Set<string> | undefined;
Expand All @@ -70,6 +78,7 @@ export class FakeDriver implements Driver {
this.#availableOsVersions = new Set(options.availableOsVersions ?? ["latest"]);
this.#clock = options.clock;
this.#estimateMs = options.estimateMs;
this.#fullCleanReclaimEstimateMs = options.fullCleanReclaimEstimateMs;
this.#knownModels =
options.knownModels === undefined ? undefined : new Set(options.knownModels);
this.#latencyMs = options.latencyMs;
Expand Down Expand Up @@ -198,8 +207,11 @@ export class FakeDriver implements Driver {
};
}

estimate(operation: DriverEstimateOperation, _spec: DeviceSpec): number {
return this.#estimateMs?.[operation] ?? 0;
estimate(estimate: DriverEstimate, _spec: DeviceSpec): number {
if (estimate.operation === "reclaim" && estimate.clean === "full") {
return this.#fullCleanReclaimEstimateMs ?? this.#estimateMs?.reclaim ?? 0;
}
return this.#estimateMs?.[estimate.operation] ?? 0;
}

failOn(operation: FakeDriverOperation, callNumber: number, error: Error): void {
Expand Down
1 change: 1 addition & 0 deletions src/core/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ export {
type DriverCatalogEntry,
DriverCrashError,
type DriverDevice,
type DriverEstimate,
type DriverReality,
type ObservedDevice,
type ObservedRunState,
Expand Down
Loading