diff --git a/docs/ABOUT.md b/docs/ABOUT.md index 26eae60..6a19aee 100644 --- a/docs/ABOUT.md +++ b/docs/ABOUT.md @@ -15,7 +15,9 @@ Simlock is a CLI-first control plane (backed by a local daemon) that is the same for both platforms and gives agents one primitive: **lease a device**. An optional local stdio MCP integration exposes the focused lease/release workflow to compatible agent clients; the CLI remains the full operator -interface. +interface. An optional, token-authenticated HTTP API lets remote agents lease +devices from a self-hosted simlock host over the network (see +[HTTP-API.md](HTTP-API.md)). - `simlock lease` returns a *ready* device — booted and health-checked — that no other agent will touch for the duration of the lease. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 14831ce..3f92911 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -6,6 +6,8 @@ agent ──spawns──> simlock CLI ──┐ ├─ shared daemon client ──unix socket──> simlock daemon MCP client ──spawns──> stdio MCP ┘ │ + │ +remote agent ──token auth──> HTTP gateway ──same role interfaces────────┤ ┌───┼─────────────┐ │ core (platform-│ │ agnostic) │ @@ -26,10 +28,22 @@ MCP client ──spawns──> stdio MCP ┘ emulator/adb) ``` -- **CLI and stdio MCP server**: sibling thin frontends over the shared daemon - client and unix socket. The core never knows which frontend made a request. - The CLI is the full operator interface; the MCP server intentionally limits - its tool surface to leasing and releasing for an agent session. +- **CLI, stdio MCP server, and HTTP gateway**: sibling thin frontends. The + core never knows which frontend made a request. The CLI and MCP server sit + over the shared daemon client and unix socket; the CLI is the full operator + interface, and the MCP server intentionally limits its tool surface to + leasing and releasing for an agent session. The HTTP gateway is different in + kind, not just transport: it is the one frontend meant to be reached over a + real network, so it calls the same role interfaces (`LeaseCommands`, + `QueueControl`, `CapacityReader`, `CatalogReader`) in-process rather than + going through the unix socket, requires a bearer token on every route but + `GET /v1/healthz`, and only ever grants detached-style, TTL-renewed leases — + "held lease = live connection" does not survive a real network the way it + does a local process. It starts only after the daemon's own startup + convergence completes (see "Startup: claim first, converge after" below), + so unlike the socket protocol's parked-request behavior during that window, + a request arriving before then is simply refused. See + [HTTP-API.md](HTTP-API.md) for the full route reference. - **CLI**: in the default *held* mode it acquires a lease, prints one JSON result line on stdout, then stays alive holding the daemon connection; the connection is the lease heartbeat. Progress streams as JSON lines on stderr. diff --git a/docs/CLI.md b/docs/CLI.md index 4be9040..7841474 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -336,6 +336,27 @@ under `capacity.config` — see is running, both a global and a per-platform running limit must have room before Simlock provisions or boots a shutdown device. +## `simlock token create --role [--label ]` / `list` / `revoke ` + +Mint and manage bearer tokens for the HTTP API. Operates on +`~/.simlock/tokens.json` directly (under `SIMLOCK_HOME`) — no daemon +round-trip, like `config` reading its file. + +`create` prints the minted secret **once**, alongside the token record: + +```json +{"token":{"id":"tok_9f2c","role":"agent","label":"ci-runner","createdAt":1735689600000},"secret":"slk_Wn9…"} +``` + +Only the secret's SHA-256 hash is ever persisted; there is no way to recover +a lost secret, only to `revoke` the token and `create` a new one. The token +id doubles as the requester identity over HTTP — one token is one requester, +same as the CLI's `--agent-id`. + +`list` prints `{"tokens":[...]}` — the same record shape as `create`, minus +the secret and its hash. `revoke ` prints `{"revoked":true}`, or a +structured `UNKNOWN_TOKEN` error (exit 1) for an id that does not exist. + ## Environment variables ### `SIMLOCK_HOME` diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 39665ad..f61d99c 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -17,6 +17,9 @@ a warning. Inspect the effective, merged configuration at any time with | `lease.heldTtlBackstopMs` | Backstop TTL for held-mode leases, in case the holding process dies without releasing. | `1 hour` | | `lease.detachedTtlMs` | TTL for detached-mode leases before they must be renewed with `simlock lease renew`. | `15 minutes` | | `lease.heartbeatIntervalMs` | How often the daemon pings a held-mode connection that declared the `heartbeat` capability; each pong slides that connection's leases' TTL back out to a full `heldTtlBackstopMs`. Must be `<= lease.heldTtlBackstopMs / 4`. | `5 minutes` | +| `http.enabled` | Master switch for the network-facing HTTP API (see [HTTP-API.md](HTTP-API.md)). Off by default; the daemon binds nothing until this is `true`. | `false` | +| `http.host` | Address the HTTP listener binds. `127.0.0.1` keeps it loopback-only; reaching it remotely is the operator's own tunnel (Tailscale, cloudflared, reverse proxy) — Simlock does no TLS termination in v1. | `127.0.0.1` | +| `http.port` | Port the HTTP listener binds. Must be an integer `1`-`65535`. | `4700` | | `diskPressure.freeBytesThreshold` | Free disk space below which Simlock treats the machine as under disk pressure. | `10 GiB` | | `eventBuffer.capacity` | Number of business events kept in the in-memory ring buffer (see `simlock events`). | `1000` | | `health.enabled` | Master switch for leased-device crash detection and recovery. | `true` | @@ -36,6 +39,8 @@ must be non-negative numbers (milliseconds and bytes, respectively). `health.maxConcurrentRecoveries` must be positive integers. `stalledTransition.thresholdMultiplier` must be a number `>= 1`; `stalledTransition.minimumThresholdMs` must be a non-negative number. +`http.enabled` is a boolean, `http.host` a string, and `http.port` an +integer in `1`-`65535`. See [CLI.md](CLI.md#simlock-config-get-keyset-key-value) for the `simlock config` command itself. diff --git a/docs/EVENTS.md b/docs/EVENTS.md index 643d89d..eeed580 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -17,7 +17,7 @@ in short: `subject.past-tense-fact`, emitted post-commit, facts not commands. | `lease.renewed` | lease id, new deadline | an explicit `simlock lease renew` succeeded (either mode), **or** a held-mode connection that declared the `heartbeat` capability answered a `lease.heartbeat` push (fires once per lease per `lease.heartbeatIntervalMs` while the holder stays alive) | LeaseLifecycle | implemented | | `lease.released` | lease id, device id, reason (closed/explicit/killed/orphaned/device-lost) | holder connection closed, explicit release, (orphaned) a `held` lease found still persisted at daemon startup, which cannot have a live holder across a restart, or (device-lost) a leased device could not be recovered after it stopped running outside simlock | LeaseLifecycle | implemented | | `lease.expired` | lease id, device id | TTL backstop fired without a heartbeat sliding it first — for a capability-declaring holder this means it stopped ponging (crashed, hung, or lost its socket); for one that never declared the capability it means the grant-time TTL (or the last explicit `simlock lease renew`) simply ran out, exactly as before this change | LeaseLifecycle | implemented | -| `lease.rejected` | request spec, reason (timeout/no-wait/unresolvable-spec/already-leased/boot-timeout/killed) | a request ended without a grant | LeaseAcquisitionCoordinator / WaitQueue | implemented | +| `lease.rejected` | request spec, reason (timeout/no-wait/unresolvable-spec/already-leased/boot-timeout/killed/cancelled) | a request ended without a grant; `cancelled` is an explicit single-request cancel (`LeaseEngine#cancelPending`, backing `DELETE /v1/lease-requests/{id}`) of a still-queued waiter -- one with device work already in flight is reported `not-cancellable` instead, the same envelope the queue timeout already uses | LeaseAcquisitionCoordinator / WaitQueue | implemented | ## Device lifecycle diff --git a/docs/HTTP-API.md b/docs/HTTP-API.md new file mode 100644 index 0000000..d1581e8 --- /dev/null +++ b/docs/HTTP-API.md @@ -0,0 +1,274 @@ +# HTTP API reference + +Part of the user manual: the network-facing control-plane API a remote agent +uses instead of the CLI/MCP frontends' unix socket. It is off by default +(`http.enabled: false` — see [CONFIGURATION.md](CONFIGURATION.md)) and, once +enabled, binds `127.0.0.1` unless configured otherwise. Reaching it from +another machine is the operator's own tunnel (Tailscale, cloudflared, a +reverse proxy) — Simlock does no TLS termination in v1, and `Authorization` +is required on every route regardless of how it's reached, loopback included. + +The gateway calls the same role interfaces the CLI and MCP server call +(`LeaseCommands`, `QueueControl`, `CapacityReader`, `CatalogReader`); the core +never knows HTTP exists. See [ARCHITECTURE.md](ARCHITECTURE.md) for how it +fits alongside the other frontends. + +## Leases are detached-only over HTTP + +Every lease granted through this API is **detached**: TTL-bound, kept alive +by `POST /v1/leases/:id/renew`, never by a held connection. HTTP is +stateless — "held lease = live connection" does not survive a real network — +so there is no held mode here and no WebSocket held-mode emulation. A lease +that stops renewing expires via the same TTL machinery `simlock lease +--detach` uses; the device is reclaimed normally. + +Acquisition is an async resource, not a blocking call: `POST +/v1/lease-requests` returns as soon as the request exists (queued, or already +past that), and the client polls, long-polls, or streams its progress to a +terminal state. No route blocks on device work in flight. + +## Authentication + +Every route requires `Authorization: Bearer slk_` except `GET +/v1/healthz`. Missing or unrecognized tokens are `401 UNAUTHENTICATED`. + +Tokens are minted and managed with `simlock token` (see [CLI.md](CLI.md)) — +a local, no-daemon-round-trip command, like `config`. Each token record is +`{ id, role, label?, createdAt }`, hashed at rest in `~/.simlock/tokens.json` +(SHA-256; the plaintext secret is shown exactly once, at `create`, and never +persisted). The token id doubles as the requester identity over HTTP: unlike +the CLI's `--agent-id`/`SIMLOCK_AGENT_ID`, identity is never client-declared +here, so the one-lease-per-requester rule keys off which token authenticated +the request, not anything the request body says. + +Two roles: + +| Role | Can | +|---|---| +| `agent` | catalog, status, its own lease requests and leases | +| `operator` | everything `agent` can, plus every other requester's leases/devices, event replay/stream, and releasing any lease | + +A valid token with the wrong role for a route is `403 FORBIDDEN`, not `401` — +distinct from an unrecognized token. Reaching another requester's own +resource (a lease/request an `agent` token didn't create) is the same `403`, +enforced per-resource rather than as a role gate. + +## Endpoints + +All routes are under `/v1`, JSON bodies both ways, additive evolution only — +new fields, never removed or repurposed ones. + +### `GET /v1/healthz` + +Unauthenticated liveness for tunnels/load balancers. → `200 {"ok":true}`. + +### `GET /v1/status` + +Role: `agent`. The same view `simlock status --json` reads: daemon health +(`starting`/`running`), managed/running capacity per platform, active +leases, managed devices, queue depth. + +### `GET /v1/catalog?platform=ios|android` + +Role: `agent`. Exactly `simlock catalog --json`. Read-only; never triggers a +download. + +### `POST /v1/lease-requests` + +Role: `agent`. Enqueues a device request. + +```json +{ + "platform": "ios", + "device": "iPhone 17 Pro", + "os": "26.5", + "ttlMs": 900000, + "timeoutMs": 300000, + "noWait": false, + "allowDownload": false +} +``` + +`platform` and `device` are required; `os` defaults to the newest installed +runtime; `ttlMs` defaults to `lease.detachedTtlMs`; `timeoutMs` (optional) is +enforced daemon-side so a vanished client can't hold a queue slot forever. +An `Idempotency-Key` header (at most 200 characters) makes a replay of the +same key, from the same requester, return the original request resource +instead of double-queueing — held in memory with a TTL, so a replay after a +daemon restart creates a fresh request (see +[Lifecycle semantics](#lifecycle-semantics) below). + +With `allowDownload: true` the `201` is returned immediately, before the +request is even admitted — resolving a downloadable runtime can take minutes, +so progress (and any admission failure, `REQUESTER_ALREADY_LEASED` included) +surfaces on the request resource instead of on the `POST` itself. + +→ `201`, `Location: /v1/lease-requests/{id}`: + +```json +{ "request": { "id": "req_7d1a", "state": "queued", "queuePosition": 2, "createdAt": "2026-09-01T09:12:00Z" } } +``` + +A rejection that lands before any device work is claimed for the request +fails the `POST` itself instead of the client having to poll to learn about +it: `409 REQUESTER_ALREADY_LEASED` (names the existing lease id), `422` for +an unknown model / missing runtime / no driver, `503 NO_CAPACITY` (with +`Retry-After`) when `noWait` is set. Anything that fails once device work is +already in flight surfaces as the request resource's terminal `failed` state +instead — see the state list below. + +### `GET /v1/lease-requests/{id}` + +Role: `agent` (its own requests; `operator` sees all). Poll the request. +`?wait=` long-polls: returns as soon as the state changes, else +once `wait` elapses. `wait` is capped at 60 seconds — a larger value is +clamped, not rejected, and the poll simply returns (unchanged) sooner than +asked; re-poll to keep waiting. + +States: `queued | reclaiming | provisioning | booting | granted | failed | +cancelled`, carrying `queuePosition` (`queued`) or `etaSeconds` +(`reclaiming`/`provisioning`/`booting`) where the stage has one. Terminal +`granted` embeds the [lease object](#the-lease-object); terminal `failed` +embeds `{ code, message }`. + +### `GET /v1/lease-requests/{id}/events` + +Role: `agent` (ownership as above). Server-Sent Events stream of the same +progress objects, one event per state change, ending with `granted` or +`failed`. A `: keepalive` comment every ~15s keeps idle tunnels from closing +the stream. + +### `DELETE /v1/lease-requests/{id}` + +Role: `agent` (ownership as above). Cancel a pending request. + +→ `204` if it was still cancellable (no device work claimed for it yet). +`409 REQUEST_NOT_CANCELLABLE` once device work is in flight, or the request +already reached a terminal state — the body names the lease id if it was +`granted` (release that instead). `404 UNKNOWN_REQUEST` if unknown. + +### The lease object + +```json +{ "lease": { + "id": "lse_9f2c", "requestId": "req_7d1a", + "platform": "ios", "device": "iPhone 17 Pro", "os": "26.5", + "udid": "ABCD-...", "deviceId": "dev_1a2b", + "createdAt": "2026-09-01T09:14:07Z", + "expiresAt": "2026-09-01T09:29:07Z", "ttlMs": 900000, + "dataPlane": null +} } +``` + +`dataPlane` is **reserved** and always `null` in this version: driving the +leased device remotely (the data plane) is a separate, not-yet-implemented +concern — see [Not implemented](#not-implemented) below. It is in the schema +now so its arrival is additive rather than a breaking shape change. + +### `GET /v1/leases/{id}` + +Role: `agent` (own lease; `operator` any). Re-fetches the lease — a client +that restarts mid-lease recovers its state instead of leaking the lease. +`404 UNKNOWN_LEASE` once it has expired or been released. + +`expiresAt` is always the authoritative deadline. After a **daemon** restart +the gateway no longer remembers a per-request `ttlMs`, so the payload reports +the lease's mode default (the interval a body-less renew applies from then +on) and may omit `requestId`; schedule renewals from `expiresAt`, not from +`ttlMs`. + +### `POST /v1/leases/{id}/renew` + +Role: `agent` (own lease). Body `{ "ttlMs": 900000 }` (optional; defaults to +the lease's own mode default). Resets the deadline to now + ttl. + +→ `200 { "leaseId": "lse_9f2c", "expiresAt": "...", "notices": [] }` + +`notices` carries device-health facts observed since the previous renew for +this lease — `{"event":"device_unhealthy"}`, +`{"event":"device_recovered","attempts":1}` — so a polling-only client +learns its device blinked without holding a stream open. + +### `GET /v1/leases/{id}/events` + +Role: `agent` (own lease). Server-Sent Events for live health pushes on this +lease: `device_unhealthy`, `device_recovered`, `lease_lost` (ends the +stream). The same facts held mode relays on stderr today. + +### `DELETE /v1/leases/{id}` + +Role: `agent` (own lease); `operator` may release any lease. + +→ `202 { "released": true, "device": { "id": "dev_1a2b", "state": "reclaiming" } }` + +The lease is gone the moment this responds; the driver-side purge continues +in the background (existing release semantics — see "Release hands the +purge off" in [ARCHITECTURE.md](ARCHITECTURE.md)), hence `202`, not `200`. + +### Operator routes + +Role: `operator` for all four. + +- `GET /v1/leases` — every active lease (`simlock list --leases`). +- `GET /v1/devices` — every managed device, with state and + `transitionAgeMs` (`simlock list --devices`). +- `GET /v1/events?since=` — replay from the in-memory business-event + ring buffer (`simlock events`). +- `GET /v1/events/stream` — Server-Sent Events follow of the event bus + (`simlock events --follow`). + +## Errors + +Every failure is the same shape the daemon protocol uses: + +```json +{ "error": { "code": "NO_CAPACITY", "message": "..." } } +``` + +| HTTP | Codes | +|---|---| +| 400 | `BAD_REQUEST` (malformed body, bad query param, validation) | +| 401 | `UNAUTHENTICATED` (missing or unrecognized token) | +| 403 | `FORBIDDEN` (role doesn't permit the route; or the lease/request belongs to another requester) | +| 404 | `UNKNOWN_REQUEST`, `UNKNOWN_LEASE` | +| 409 | `REQUESTER_ALREADY_LEASED` (body names the existing lease id), `REQUEST_NOT_CANCELLABLE` (body names the lease id if the request had already been granted) | +| 422 | `UNKNOWN_MODEL`, `RUNTIME_MISSING`, `NO_DRIVER` | +| 503 | `NO_CAPACITY` (only with `noWait: true`; response carries `Retry-After`) | + +## Lifecycle semantics + +- **Daemon restart.** In-flight lease requests are in-memory and do not + survive, same as the socket protocol's queue today. A client polling a + request id from before the restart gets `404 UNKNOWN_REQUEST`; if its + grant had actually landed before the crash, the persisted detached lease + answers a retried `POST` with `409 REQUESTER_ALREADY_LEASED` naming the + lease id, which the client then `GET`s to recover its state. This is the + documented recovery loop: `404` → re-request → (maybe) `409` → `GET`. +- **Idempotency keys** are in-memory with a TTL; a replay after that window + (including across a restart) creates a fresh request rather than erroring + — the `409` above is the real backstop against a double grant, not the + idempotency cache. +- **Startup.** The gateway starts only after the daemon's own startup + convergence finishes (unlike the unix socket, which accepts connections + immediately and parks non-`hello`/`status.get` requests until convergence + completes). A connection refused while the daemon is still starting up is + accepted v1 behavior — there is no HTTP-level "starting" response to + mirror the socket protocol's parked dispatch. +- **Shutdown.** `simlock daemon stop` closes the HTTP listener (and any open + connection, in-flight SSE streams included) before releasing held leases + or tearing down the lease engine, so no HTTP request can run against a + stopping daemon. + +## Not implemented + +- `POST /v1/doctor` and `POST /v1/cleanup` — not part of this version; may + land as a follow-up. +- `nuke` is absent from the HTTP surface entirely, deliberately: a + remote fleet-wipe endpoint is a footgun even behind auth. It stays + SSH/local-only (`simlock nuke`). +- `dataPlane` on the lease object is reserved and always `null` — driving + the leased device remotely is tracked separately (the agent-device + integration work), not in this version. +- MCP-over-HTTP, in-process TLS, and multi-host brokering are all out of + scope for this version too (a per-lease `dataPlane.baseUrl` already leaves + room for the last one, once it exists). diff --git a/e2e/http-api.test.ts b/e2e/http-api.test.ts new file mode 100644 index 0000000..dabcb0f --- /dev/null +++ b/e2e/http-api.test.ts @@ -0,0 +1,207 @@ +import { createServer } from "node:net"; +import { describe, expect, it } from "vitest"; + +import { waitFor, withDaemon } from "./helpers/index.js"; + +/** + * Reserves a free TCP port by binding to port 0 and releasing it immediately. Config + * validation rejects `http.port: 0` (it requires 1-65535, see `docs/CONFIGURATION.md`), + * so the daemon itself can't pick its own ephemeral port -- this is the test's own + * stand-in for that. There is a small window between release and the daemon binding + * the same port, same as any "reserve a port for a subprocess" approach; nothing else + * on this machine is expected to be racing for it during a test run. + */ +async function reservePort(): Promise { + return new Promise((resolve, reject) => { + const probe = createServer(); + probe.once("error", reject); + probe.listen(0, "127.0.0.1", () => { + const address = probe.address(); + probe.close((closeError) => { + if (closeError) { + reject(closeError); + return; + } + if (address === null || typeof address === "string") { + reject(new Error("failed to reserve a port: no AddressInfo")); + return; + } + resolve(address.port); + }); + }); + }); +} + +interface RequestResource { + readonly id: string; + readonly state: string; + readonly createdAt: string; + readonly lease?: LeasePayload; + readonly error?: { readonly code: string; readonly message: string }; +} + +interface LeasePayload { + readonly id: string; + readonly requestId?: string; + readonly platform: string; + readonly device: string; + readonly os: string; + readonly udid: string; + readonly deviceId: string; + readonly createdAt: string; + readonly expiresAt: string; + readonly ttlMs: number; + readonly dataPlane: null; +} + +describe("HTTP API", () => { + it("token create -> healthz -> catalog -> lease-request -> granted -> renew -> release, with auth enforced throughout", async () => { + const port = await reservePort(); + const env = await withDaemon({ + configOverrides: { http: { enabled: true, host: "127.0.0.1", port } }, + }); + await env.driverScript.set({ + ios: { knownModels: ["iPhone 16"], availableOsVersions: ["18.4"] }, + }); + + const baseUrl = `http://127.0.0.1:${port}`; + + // 1. Mint an agent token via the CLI, against the same SIMLOCK_HOME the daemon reads + // tokens.json from -- no daemon round-trip for `token create` (see src/cli/index.ts). + const tokenResult = await env.cli(["token", "create", "--role", "agent"]); + expect(tokenResult.code).toBe(0); + const { secret } = tokenResult.json as { secret: string }; + const agentAuth = { authorization: `Bearer ${secret}` }; + + // 2. The gateway is started only after the daemon's own startup convergence + // finishes (see the comment in src/daemon/main.ts), strictly after the unix + // socket already answers `daemon start` -- so the first HTTP call must tolerate + // a connection refused for a brief window rather than assume the port is live + // the instant `withDaemon()` returns. + await waitFor( + async () => { + try { + const response = await fetch(`${baseUrl}/v1/healthz`); + return response.ok; + } catch { + return false; + } + }, + { label: "HTTP gateway accepting connections" }, + ); + const healthz = await fetch(`${baseUrl}/v1/healthz`); + expect(healthz.status).toBe(200); + expect(await healthz.json()).toEqual({ ok: true }); + + // 3. `GET /v1/catalog` requires auth; unauthenticated and wrong-token requests + // both come back 401 UNAUTHENTICATED. + const catalogNoAuth = await fetch(`${baseUrl}/v1/catalog`); + expect(catalogNoAuth.status).toBe(401); + expect(((await catalogNoAuth.json()) as { error: { code: string } }).error.code).toBe( + "UNAUTHENTICATED", + ); + + const catalogWrongToken = await fetch(`${baseUrl}/v1/catalog`, { + headers: { authorization: "Bearer slk_not-a-real-token" }, + }); + expect(catalogWrongToken.status).toBe(401); + expect(((await catalogWrongToken.json()) as { error: { code: string } }).error.code).toBe( + "UNAUTHENTICATED", + ); + + const catalog = await fetch(`${baseUrl}/v1/catalog`, { headers: agentAuth }); + expect(catalog.status).toBe(200); + expect(await catalog.json()).toMatchObject({ + platforms: expect.arrayContaining([ + expect.objectContaining({ platform: "ios", models: ["iPhone 16"], defaultRuntime: "18.4" }), + ]), + }); + + // 4. An operator-only route rejects an agent token with 403, not 401 -- the token + // is valid, it just doesn't carry the role this route requires. + const operatorRouteAsAgent = await fetch(`${baseUrl}/v1/leases`, { headers: agentAuth }); + expect(operatorRouteAsAgent.status).toBe(403); + expect(((await operatorRouteAsAgent.json()) as { error: { code: string } }).error.code).toBe( + "FORBIDDEN", + ); + + // 5. `POST /v1/lease-requests` enqueues a request resource; poll it (via the `wait` + // long-poll param) until it reaches its terminal `granted` state. + const created = await fetch(`${baseUrl}/v1/lease-requests`, { + method: "POST", + headers: { ...agentAuth, "content-type": "application/json" }, + body: JSON.stringify({ platform: "ios", device: "iPhone 16", os: "18.4" }), + }); + expect(created.status).toBe(201); + expect(created.headers.get("location")).toMatch(/^\/v1\/lease-requests\/req_/); + const createdBody = (await created.json()) as { request: RequestResource }; + const requestId = createdBody.request.id; + + let polled = createdBody.request; + while ( + polled.state !== "granted" && + polled.state !== "failed" && + polled.state !== "cancelled" + ) { + const response = await fetch(`${baseUrl}/v1/lease-requests/${requestId}?wait=10`, { + headers: agentAuth, + }); + expect(response.status).toBe(200); + polled = ((await response.json()) as { request: RequestResource }).request; + } + expect(polled.state, `lease request ${requestId} did not reach granted`).toBe("granted"); + const lease = polled.lease as LeasePayload; + expect(lease).toMatchObject({ + platform: "ios", + device: "iPhone 16", + os: "18.4", + requestId, + dataPlane: null, + }); + + // 6. A second lease request from the same requester (same token) while the first + // is still held is rejected -- 409, naming the existing lease id -- rather than + // queueing a second device for one agent. + const secondRequest = await fetch(`${baseUrl}/v1/lease-requests`, { + method: "POST", + headers: { ...agentAuth, "content-type": "application/json" }, + body: JSON.stringify({ platform: "ios", device: "iPhone 16", os: "18.4" }), + }); + expect(secondRequest.status).toBe(409); + const secondRequestError = (await secondRequest.json()) as { + error: { code: string; existingLeaseId?: string }; + }; + expect(secondRequestError.error.code).toBe("REQUESTER_ALREADY_LEASED"); + expect(secondRequestError.error.existingLeaseId).toBe(lease.id); + + // 7. Renew resets the deadline; the fetched lease also survives a `GET` re-fetch. + const refetched = await fetch(`${baseUrl}/v1/leases/${lease.id}`, { headers: agentAuth }); + expect(refetched.status).toBe(200); + expect(((await refetched.json()) as { lease: LeasePayload }).lease.id).toBe(lease.id); + + const renewed = await fetch(`${baseUrl}/v1/leases/${lease.id}/renew`, { + method: "POST", + headers: { ...agentAuth, "content-type": "application/json" }, + body: JSON.stringify({ ttlMs: 120_000 }), + }); + expect(renewed.status).toBe(200); + const renewedBody = (await renewed.json()) as { + leaseId: string; + expiresAt: string; + notices: unknown[]; + }; + expect(renewedBody).toMatchObject({ leaseId: lease.id, notices: [] }); + expect(renewedBody.expiresAt).not.toBe(lease.expiresAt); + + // 8. Release: 202, purge continues in the background (existing release semantics). + const released = await fetch(`${baseUrl}/v1/leases/${lease.id}`, { + method: "DELETE", + headers: agentAuth, + }); + expect(released.status).toBe(202); + expect(await released.json()).toMatchObject({ released: true, device: { id: lease.deviceId } }); + + const afterRelease = await fetch(`${baseUrl}/v1/leases/${lease.id}`, { headers: agentAuth }); + expect(afterRelease.status).toBe(404); + }); +}); diff --git a/package.json b/package.json index 512a82e..6b76ba2 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,10 @@ "release": "release-it" }, "dependencies": { + "@hono/node-server": "2.0.10", + "@hono/zod-validator": "^0.9.1", "@modelcontextprotocol/sdk": "1.29.0", + "hono": "^4.13.5", "zod": "^3.25.76" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7311066..665bbb9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -211,9 +211,18 @@ importers: .: dependencies: + '@hono/node-server': + specifier: 2.0.10 + version: 2.0.10(hono@4.13.5) + '@hono/zod-validator': + specifier: ^0.9.1 + version: 0.9.1(hono@4.13.5)(zod@3.25.76) '@modelcontextprotocol/sdk': specifier: 1.29.0 version: 1.29.0(zod@3.25.76) + hono: + specifier: ^4.13.5 + version: 4.13.5 zod: specifier: ^3.25.76 version: 3.25.76 @@ -315,6 +324,12 @@ packages: peerDependencies: hono: ^4 + '@hono/zod-validator@0.9.1': + resolution: {integrity: sha512-iiv6w0qrIc0arfvCtUqBWsvl4fXjzaTcQcCJTTtCnkawF9HHGE+KjEl0ox3gQJ6rKZEgE3mLExlQCF72M+mNuw==} + peerDependencies: + hono: '>=4.11.2' + zod: ^3.25.0 || ^4.0.0 + '@inquirer/ansi@2.0.7': resolution: {integrity: sha512-3eTuUO1vH2cZm2ZKHeQxnOqlTi9EfZDGgIe3BL3I4u+rJHocr9Fz86M4fjYABPvFnQG/gGK551HqDiIcETwU6Q==} engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} @@ -1485,8 +1500,8 @@ packages: resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} engines: {node: '>= 0.4'} - hono@4.13.3: - resolution: {integrity: sha512-r8AO2mYHoLxSHkgafNeC/BXyb2vWRxD3jem4Ts+ptav8oTG5FIRifAjuJEmZI4bSvvc2ns0GxmIYiZnHqN3mMw==} + hono@4.13.5: + resolution: {integrity: sha512-O6+/eCYRkzzzy0rPWwKLiGBR1nFuUPZynnwjxN1MBA62NNqbT0wQEzQyK2gSO5yDIDB336sXQleAhOHrzlYyKw==} engines: {node: '>=16.9.0'} hosted-git-info@8.1.0: @@ -2383,9 +2398,14 @@ snapshots: '@fallow-cli/win32-x64-msvc@3.6.0': optional: true - '@hono/node-server@2.0.10(hono@4.13.3)': + '@hono/node-server@2.0.10(hono@4.13.5)': dependencies: - hono: 4.13.3 + hono: 4.13.5 + + '@hono/zod-validator@0.9.1(hono@4.13.5)(zod@3.25.76)': + dependencies: + hono: 4.13.5 + zod: 3.25.76 '@inquirer/ansi@2.0.7': {} @@ -2510,7 +2530,7 @@ snapshots: '@modelcontextprotocol/sdk@1.29.0(zod@3.25.76)': dependencies: - '@hono/node-server': 2.0.10(hono@4.13.3) + '@hono/node-server': 2.0.10(hono@4.13.5) ajv: 8.20.0 ajv-formats: 3.0.1(ajv@8.20.0) content-type: 1.0.5 @@ -2520,7 +2540,7 @@ snapshots: eventsource-parser: 3.1.1 express: 5.2.1 express-rate-limit: 8.6.2(express@5.2.1) - hono: 4.13.3 + hono: 4.13.5 jose: 6.2.9 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 @@ -3362,7 +3382,7 @@ snapshots: dependencies: function-bind: 1.1.2 - hono@4.13.3: {} + hono@4.13.5: {} hosted-git-info@8.1.0: dependencies: diff --git a/src/bus/index.ts b/src/bus/index.ts index 254eb3c..60da4b3 100644 --- a/src/bus/index.ts +++ b/src/bus/index.ts @@ -28,7 +28,8 @@ export interface EventMap { | "unresolvable-spec" | "already-leased" | "boot-timeout" - | "killed"; + | "killed" + | "cancelled"; }; "device.provisioned": { readonly deviceId: string; diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 07ad286..2345051 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -8,6 +8,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { EventBus } from "../bus/index.js"; import { type Config, CleanupReaper, FakeDriver, LeaseEngine, Registry } from "../core/index.js"; import { + CryptoTokenSecrets, FakeClock, FakeParentWatch, FakeSystemStats, @@ -15,6 +16,7 @@ import { NodeFilesystem, NodeIpcTransport, } from "../ports/index.js"; +import { TokenStore } from "../http/token-store.js"; import { DaemonEndpointHost } from "../daemon/connection-host.js"; import { DaemonServer } from "../daemon/server.js"; import { connectExistingDaemon } from "../daemon-client/client.js"; @@ -930,6 +932,117 @@ describe("CLI boundary", () => { }); }); +describe("simlock token", () => { + it("creates an agent token, printing the record and secret on one JSON line", async () => { + const { environment, output } = tokenHarness(); + + await expect( + runCli(["token", "create", "--role", "agent", "--label", "ci-runner"], environment), + ).resolves.toBe(0); + expect(output.stderr).toBe(""); + const parsed = JSON.parse(output.stdout) as { + secret: string; + token: { id: string; role: string; label: string; createdAt: number }; + }; + expect(parsed.token).toEqual({ + createdAt: 1_000, + id: "tok_1", + label: "ci-runner", + role: "agent", + }); + expect(parsed.secret).toMatch(/^slk_/); + expect(JSON.stringify(parsed.token)).not.toContain("hash"); + }); + + it("creates an operator token without a label, omitting the field entirely", async () => { + const { environment, output } = tokenHarness(); + + await expect(runCli(["token", "create", "--role", "operator"], environment)).resolves.toBe(0); + const parsed = JSON.parse(output.stdout) as { token: Record }; + expect(parsed.token.role).toBe("operator"); + expect("label" in parsed.token).toBe(false); + }); + + it("rejects a missing --role as a structured usage error", async () => { + const { environment, output } = tokenHarness(); + + await expect(runCli(["token", "create"], environment)).resolves.toBe(2); + expect(output.stdout).toBe(""); + expect(JSON.parse(output.stderr)).toEqual({ + error: { code: "USAGE", message: expect.stringContaining("--role") }, + }); + }); + + it("rejects an invalid --role as a structured usage error", async () => { + const { environment, output } = tokenHarness(); + + await expect(runCli(["token", "create", "--role", "superuser"], environment)).resolves.toBe(2); + expect(JSON.parse(output.stderr)).toMatchObject({ error: { code: "USAGE" } }); + }); + + it("lists created tokens without exposing their hash", async () => { + const { environment, output, store } = tokenHarness(); + await store.create("agent", "one"); + await store.create("operator", "two"); + + await expect(runCli(["token", "list"], environment)).resolves.toBe(0); + const parsed = JSON.parse(output.stdout) as { tokens: Array> }; + expect(parsed.tokens.map((token) => token.label)).toEqual(["one", "two"]); + expect(JSON.stringify(parsed.tokens)).not.toContain("hash"); + }); + + it("revokes an existing token", async () => { + const { environment, output, store } = tokenHarness(); + const { record } = await store.create("agent"); + + await expect(runCli(["token", "revoke", record.id], environment)).resolves.toBe(0); + expect(JSON.parse(output.stdout)).toEqual({ revoked: true }); + await expect(store.list()).resolves.toEqual([]); + }); + + it("reports UNKNOWN_TOKEN revoking a token id that does not exist", async () => { + const { environment, output } = tokenHarness(); + + await expect(runCli(["token", "revoke", "tok_missing"], environment)).resolves.toBe(1); + expect(output.stdout).toBe(""); + expect(JSON.parse(output.stderr)).toEqual({ + error: { code: "UNKNOWN_TOKEN", message: expect.stringContaining("tok_missing") }, + }); + }); + + it("prints usage for the bare command and --help without touching the store", async () => { + const { environment, output } = tokenHarness(); + + await expect(runCli(["token"], environment)).resolves.toBe(0); + expect(output.stdout).toContain("simlock token create"); + expect(output.stdout).toContain("simlock token list"); + expect(output.stdout).toContain("simlock token revoke"); + }); + + it("rejects an unknown token subcommand", async () => { + const { environment, output } = tokenHarness(); + + await expect(runCli(["token", "bogus"], environment)).resolves.toBe(2); + expect(JSON.parse(output.stderr)).toMatchObject({ error: { code: "USAGE" } }); + }); +}); + +function tokenHarness(): { + environment: CliEnvironment; + output: ReturnType; + store: TokenStore; +} { + const output = outputCapture(); + const store = new TokenStore({ + clock: new FakeClock(1_000), + filesystem: new MemoryFilesystem(), + idGenerator: sequence(), + path: "/tokens.json", + secrets: new CryptoTokenSecrets(), + }); + return { environment: output.environmentWith({ tokenStore: store }), output, store }; +} + class StubConnection implements DaemonConnection { readonly calls: Array<{ readonly payload: unknown; readonly type: string }> = []; closed = false; @@ -1046,6 +1159,7 @@ function testConfig(): Config { stableObservations: 2, }, stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, idle: { deleteAfterMs: 60_000, shutdownAfterMs: 10_000 }, lease: { detachedTtlMs: 60_000, heldTtlBackstopMs: 60_000, heartbeatIntervalMs: 15_000 }, capacity: { diff --git a/src/cli/index.ts b/src/cli/index.ts index 4e8b62b..108a11b 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -3,6 +3,8 @@ import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; import { + CryptoIdGenerator, + CryptoTokenSecrets, NodeDaemonLauncher, NodeFilesystem, NodeIpcTransport, @@ -13,6 +15,7 @@ import { type ParentWatch, type ParentWatchHandle, } from "../ports/index.js"; +import { TokenStore, type TokenRecord, type TokenRole } from "../http/token-store.js"; import { connectDaemon, connectExistingDaemon, @@ -32,7 +35,7 @@ const USAGE = `Usage: simlock [options] Commands: lease, release, status, list, catalog, cleanup, doctor, nuke, events, - daemon, config + daemon, config, token mcp Start the stdio MCP server Run 'simlock --help' for command usage.`; @@ -62,6 +65,23 @@ class UsageError extends Error { } } +/** + * `token` operates on the filesystem directly (no daemon round-trip), so its + * failures need their own structured error code -- not a `DaemonClientError` + * (nothing was sent to the daemon) and not `UsageError` (exit 2 is reserved + * for bad flags/arguments). Defaults to exit 1 like any other non-usage, + * non-daemon failure. + */ +class TokenCliError extends Error { + constructor( + readonly code: string, + message: string, + ) { + super(message); + this.name = "TokenCliError"; + } +} + /** * Points a human at `--help` from inside the single structured stderr line, * for the two usage errors most likely to strand someone at a terminal: an @@ -102,6 +122,13 @@ export interface CliEnvironment { readonly stdout: Output; readonly confirm?: (question: string) => Promise; readonly writeConfigFile: (contents: Record) => Promise; + /** + * `token` reads/writes tokens.json directly, like `config` does with + * config.json -- no daemon round-trip. Optional so most tests (which never + * touch `token`) don't need to fabricate one; `runToken` fails clearly if + * it is missing. + */ + readonly tokenStore?: TokenStore; } /** @@ -122,6 +149,13 @@ function defaultCliEnvironment(env: NodeJS.ProcessEnv = process.env): CliEnviron const socketPath = join(dataDirectory, "daemon.sock"); const configPath = join(dataDirectory, "config.json"); const logPath = join(dataDirectory, "daemon.log"); + const tokenStore = new TokenStore({ + clock, + filesystem, + idGenerator: new CryptoIdGenerator(), + path: join(dataDirectory, "tokens.json"), + secrets: new CryptoTokenSecrets(), + }); return { configPath, connect: (capabilities) => @@ -152,6 +186,7 @@ function defaultCliEnvironment(env: NodeJS.ProcessEnv = process.env): CliEnviron stderr: process.stderr, stdout: process.stdout, confirm: confirmTerminal, + tokenStore, writeConfigFile: async (contents) => { await filesystem.mkdirp(dataDirectory); await filesystem.writeFileAtomic(configPath, `${JSON.stringify(contents, null, 2)}\n`); @@ -195,6 +230,8 @@ export async function runCli( return await runDaemon(argv.slice(1), environment); case "config": return await runConfig(argv.slice(1), environment); + case "token": + return await runToken(argv.slice(1), environment); case "mcp": return await runMcp(argv.slice(1), environment); default: @@ -222,6 +259,7 @@ function writeError(environment: CliEnvironment, error: unknown): void { function cliErrorCode(error: unknown): string { if (error instanceof UsageError) return "USAGE"; if (error instanceof DaemonClientError) return error.code; + if (error instanceof TokenCliError) return error.code; return "INTERNAL"; } @@ -672,6 +710,100 @@ async function runConfig(argv: readonly string[], environment: CliEnvironment): throw new UsageError(`Unknown config command: ${command}`); } +async function runToken(argv: readonly string[], environment: CliEnvironment): Promise { + const command = argv[0]; + if (command === undefined || isHelp(command)) { + environment.stdout.write( + "Usage: simlock token create --role [--label ]\n" + + " simlock token list\n" + + " simlock token revoke \n", + ); + return 0; + } + const tokenStore = requireTokenStore(environment); + if (command === "create") return runTokenCreate(argv.slice(1), tokenStore, environment); + if (command === "list") return runTokenList(argv.slice(1), tokenStore, environment); + if (command === "revoke") return runTokenRevoke(argv.slice(1), tokenStore, environment); + throw new UsageError(withHelpHint(`Unknown token command: ${command}`)); +} + +function requireTokenStore(environment: CliEnvironment): TokenStore { + if (environment.tokenStore === undefined) throw new Error("Token store is unavailable"); + return environment.tokenStore; +} + +async function runTokenCreate( + argv: readonly string[], + tokenStore: TokenStore, + environment: CliEnvironment, +): Promise { + const values = commandArgs(argv, { + help: { type: "boolean", short: "h" }, + label: { type: "string" }, + role: { type: "string" }, + }); + if (values.help) { + environment.stdout.write( + "Usage: simlock token create --role [--label ]\n", + ); + return 0; + } + const role = parseTokenRole(values.role); + const label = typeof values.label === "string" ? values.label : undefined; + if (label === "") throw new UsageError("token create --label must not be empty"); + const { record, secret } = await tokenStore.create(role, label); + writeResult(environment, { secret, token: serializeToken(record) }); + return 0; +} + +async function runTokenList( + argv: readonly string[], + tokenStore: TokenStore, + environment: CliEnvironment, +): Promise { + const values = commandArgs(argv, { help: { type: "boolean", short: "h" } }); + if (values.help) { + environment.stdout.write("Usage: simlock token list\n"); + return 0; + } + const records = await tokenStore.list(); + writeResult(environment, { tokens: records.map(serializeToken) }); + return 0; +} + +async function runTokenRevoke( + argv: readonly string[], + tokenStore: TokenStore, + environment: CliEnvironment, +): Promise { + const values = commandArgs(argv, { help: { type: "boolean", short: "h" } }); + if (values.help) { + environment.stdout.write("Usage: simlock token revoke \n"); + return 0; + } + const id = requiredPositional(values.positionals, "token-id"); + if (!(await tokenStore.revoke(id))) + throw new TokenCliError("UNKNOWN_TOKEN", `Unknown token: ${id}`); + writeResult(environment, { revoked: true }); + return 0; +} + +function parseTokenRole(value: unknown): TokenRole { + if (value !== "agent" && value !== "operator") + throw new UsageError(withHelpHint("token create requires --role ")); + return value; +} + +/** Drops `hash` -- an implementation detail no CLI output needs to expose. */ +function serializeToken(record: TokenRecord): Record { + return { + createdAt: record.createdAt, + id: record.id, + ...(record.label === undefined ? {} : { label: record.label }), + role: record.role, + }; +} + async function requestOnce( environment: CliEnvironment, type: string, diff --git a/src/core/acquisition-planner.test.ts b/src/core/acquisition-planner.test.ts index 9eecea8..701075e 100644 --- a/src/core/acquisition-planner.test.ts +++ b/src/core/acquisition-planner.test.ts @@ -21,6 +21,7 @@ const config: Config = { stableObservations: 2, }, stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, idle: { deleteAfterMs: 60_000, shutdownAfterMs: 10_000 }, warmPool: { quarantine: { diff --git a/src/core/cleanup/idle-destroy.test.ts b/src/core/cleanup/idle-destroy.test.ts index 67dd3c5..744d0f0 100644 --- a/src/core/cleanup/idle-destroy.test.ts +++ b/src/core/cleanup/idle-destroy.test.ts @@ -17,6 +17,7 @@ const config: Config = { stableObservations: 2, }, stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, idle: { deleteAfterMs: 30_000, shutdownAfterMs: 10_000 }, warmPool: { quarantine: { diff --git a/src/core/cleanup/idle-shutdown.test.ts b/src/core/cleanup/idle-shutdown.test.ts index ef3c800..a15ce59 100644 --- a/src/core/cleanup/idle-shutdown.test.ts +++ b/src/core/cleanup/idle-shutdown.test.ts @@ -15,6 +15,7 @@ const config: Config = { stableObservations: 2, }, stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, idle: { deleteAfterMs: 30_000, shutdownAfterMs: 10_000 }, warmPool: { quarantine: { diff --git a/src/core/config.test.ts b/src/core/config.test.ts index a639514..3a42ca2 100644 --- a/src/core/config.test.ts +++ b/src/core/config.test.ts @@ -72,6 +72,7 @@ describe("loadConfig", () => { }, }, log: { level: "info", rotateBytes: 5 * 1024 * 1024 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, warmPool: { quarantine: { maxRetries: 3, @@ -356,6 +357,67 @@ describe("loadConfig", () => { expect(warn).toHaveBeenCalledWith('Unknown config key: "health.maxBoltCount"'); }); + it("applies a file-level http override", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic( + configPath, + JSON.stringify({ http: { enabled: true, host: "0.0.0.0", port: 8080 } }), + ); + + const config = await loadConfig({ configPath, filesystem, systemStats: createStats() }); + expect(config.http).toEqual({ enabled: true, host: "0.0.0.0", port: 8080 }); + }); + + it("applies an override-level http port over the file value", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ http: { port: 5000 } })); + + const config = await loadConfig({ + configPath, + filesystem, + overrides: { http: { port: 6000 } }, + systemStats: createStats(), + }); + expect(config.http.port).toBe(6000); + }); + + it("rejects a non-boolean http.enabled", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ http: { enabled: "yes" } })); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow("http.enabled"); + }); + + it("rejects a non-string http.host", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify({ http: { host: 127 } })); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow("http.host"); + }); + + it.each([ + [{ http: { port: 0 } }, "http.port"], + [{ http: { port: 65536 } }, "http.port"], + [{ http: { port: 1.5 } }, "http.port"], + [{ http: { port: "4700" } }, "http.port"], + ])("rejects an out-of-range or malformed http.port", async (contents, path) => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(configPath, JSON.stringify(contents)); + + await expect( + loadConfig({ configPath, filesystem, systemStats: createStats() }), + ).rejects.toThrow(path); + }); + it("applies a file-level stalledTransition override", async () => { const filesystem = new MemoryFilesystem(); await filesystem.mkdirp("/home/agent/.simlock"); diff --git a/src/core/config.ts b/src/core/config.ts index b426c14..d88efe4 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -14,6 +14,7 @@ import { resourceOptionValidators } from "./capacity/strategies/resource/index.j import { booleanValue, ConfigError, + integerInRange, invalidValue, nonNegativeNumber, numberAtLeast, @@ -22,6 +23,7 @@ import { positiveNumber, requireObject, stringUnion, + stringValue, type Validator, type Warn, } from "./validation.js"; @@ -52,6 +54,11 @@ export interface Config { readonly diskPressure: { readonly freeBytesThreshold: number }; readonly eventBuffer: { readonly capacity: number }; readonly log: { readonly level: LogLevel; readonly rotateBytes: number }; + readonly http: { + readonly enabled: boolean; + readonly host: string; + readonly port: number; + }; readonly health: { readonly enabled: boolean; readonly probeIntervalMs: number; @@ -240,6 +247,7 @@ function defaultConfig(systemStats: SystemStats, strategy: CapacityStrategyName) diskPressure: { freeBytesThreshold: 10 * 1024 ** 3 }, eventBuffer: { capacity: 1_000 }, log: { level: "info", rotateBytes: 5 * 1024 * 1024 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, health: { enabled: true, probeIntervalMs: 30_000, @@ -327,6 +335,11 @@ function configValidators(strategy: CapacityStrategyName): Record minimumThresholdMs: 1_000, ...stalledTransitionOverrides, }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, }; } diff --git a/src/core/index.ts b/src/core/index.ts index 6233662..3627640 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -31,6 +31,7 @@ export { NoCapacityError, NoDriverError, QueueTimeoutError, + RequestCancelledError, RequesterAlreadyLeasedError, } from "./lease-engine.js"; export { LeaseHealthMonitor } from "./lease-health-monitor.js"; diff --git a/src/core/lease-acquisition-coordinator.test.ts b/src/core/lease-acquisition-coordinator.test.ts index 8237eb9..fd405a1 100644 --- a/src/core/lease-acquisition-coordinator.test.ts +++ b/src/core/lease-acquisition-coordinator.test.ts @@ -27,6 +27,7 @@ function config(maxDevices = 1): Config { return { diskPressure: { freeBytesThreshold: 10 * gibibyte }, eventBuffer: { capacity: 100 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, health: { enabled: true, maxConcurrentRecoveries: 1, @@ -373,6 +374,59 @@ describe("LeaseAcquisitionCoordinator", () => { await harness.coordinator.endMaintenance(); }); + it("cancels a queued request, emits lease.rejected(cancelled), and frees the requester immediately", async () => { + const harness = await createHarness(); + await harness.coordinator.request(request, { mode: "held", requesterId: "first" }); + const queued = harness.coordinator.request(request, { mode: "held", requesterId: "queued" }); + await flush(); + expect(harness.coordinator.queueDepth).toBe(1); + + const rejections: unknown[] = []; + harness.bus.subscribe("lease.rejected", (envelope) => rejections.push(envelope.payload)); + + await expect(harness.coordinator.cancelPending("queued")).resolves.toBe("cancelled"); + await expect(queued).rejects.toMatchObject({ name: "RequestCancelledError" }); + expect(harness.coordinator.queueDepth).toBe(0); + expect(rejections).toContainEqual({ requestSpec: request, reason: "cancelled" }); + + // No capacity remains (still held by "first"), but crucially this is NoCapacityError, + // not RequesterAlreadyLeasedError -- the cancelled requester is no longer pending. + await expect( + harness.coordinator.request(request, { mode: "held", noWait: true, requesterId: "queued" }), + ).rejects.toBeInstanceOf(NoCapacityError); + }); + + it("reports not-found for a requester with no pending waiter", async () => { + const harness = await createHarness(); + await expect(harness.coordinator.cancelPending("nobody")).resolves.toBe("not-found"); + }); + + it("reports not-cancellable while device work is already in flight, matching the queue timeout's envelope", async () => { + const harness = await createHarness(); + const shutdown = await seedReady(harness); + await harness.driver.shutdown({ + address: shutdown.address ?? "", + deviceId: shutdown.driverDeviceId, + driverData: shutdown.driverData, + }); + await harness.registry.transitionDevice(shutdown.id, "shutdown", { + event: "device.shutdown", + payload: { deviceId: shutdown.id, initiator: "test" }, + }); + harness.driver.hangMakeReady(); + + const acquisition = harness.coordinator.request(request, { + mode: "held", + requesterId: "booting", + }); + await flush(); + + await expect(harness.coordinator.cancelPending("booting")).resolves.toBe("not-cancellable"); + + harness.driver.releaseMakeReady(); + await expect(acquisition).resolves.toMatchObject({ device: { id: shutdown.id } }); + }); + it("keeps admission closed until concurrent maintenance callers have all finished", async () => { const harness = await createHarness(); await harness.coordinator.beginMaintenance(); diff --git a/src/core/lease-acquisition-coordinator.ts b/src/core/lease-acquisition-coordinator.ts index 68d2c3f..32b896d 100644 --- a/src/core/lease-acquisition-coordinator.ts +++ b/src/core/lease-acquisition-coordinator.ts @@ -20,13 +20,18 @@ import { type LeaseGrant, type LeaseRequestOptions, type LeaseTiming, + RequestCancelledError, RequesterAlreadyLeasedError, type Waiter, type WaitQueue, } from "./wait-queue.js"; export type { LeaseGrant, LeaseRequestOptions } from "./wait-queue.js"; -export { QueueTimeoutError, RequesterAlreadyLeasedError } from "./wait-queue.js"; +export { + QueueTimeoutError, + RequestCancelledError, + RequesterAlreadyLeasedError, +} from "./wait-queue.js"; export { NoDriverError } from "./driver-catalog.js"; export class NoCapacityError extends Error { @@ -61,6 +66,7 @@ export type AcquisitionQueue = Pick< | "depth" | "detachProgress" | "enqueue" + | "findPendingWaiter" | "hasPendingRequester" | "head" | "isQueued" @@ -228,6 +234,27 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { }); } + /** + * Cancels a single pending request. Reuses the queue timeout's safety envelope exactly: + * only a waiter still in `queued` state is safe to reject here. `processing` means device + * work already claimed it (provision/boot/evict in flight, possibly never having touched + * the FIFO list at all on a first-attempt direct dispatch) -- the same state the timeout + * timer leaves untouched -- so this reports `not-cancellable` rather than inventing a new + * rule for tearing down in-flight driver work; nuke's `cancelAll` already owns that harder + * problem, with its own drain of `#activeWorkflows`. + */ + async cancelPending(requesterId: string): Promise<"cancelled" | "not-found" | "not-cancellable"> { + return this.options.decisions.run(async () => { + const waiter = this.options.queue.findPendingWaiter(requesterId) as + | AcquisitionWaiter + | undefined; + if (waiter === undefined) return "not-found"; + if (waiter.state !== "queued") return "not-cancellable"; + this.#reject(waiter, new RequestCancelledError(waiter.id), "cancelled"); + return "cancelled"; + }); + } + /** Direct availability notification for release, cleanup, and queue-timeout callers. */ kick(): void { this.#wakeQueue(); @@ -523,7 +550,8 @@ export class LeaseAcquisitionCoordinator implements AcquisitionMaintenance { | "unresolvable-spec" | "already-leased" | "boot-timeout" - | "killed", + | "killed" + | "cancelled", ): void { if (this.options.queue.reject(waiter, error)) { this.options.eventBus.emit( diff --git a/src/core/lease-engine.test.ts b/src/core/lease-engine.test.ts index 73c7120..75f63e6 100644 --- a/src/core/lease-engine.test.ts +++ b/src/core/lease-engine.test.ts @@ -13,6 +13,7 @@ import { NoCapacityError, QueueTimeoutError, Registry, + RequestCancelledError, } from "./index.js"; const gibibyte = 1024 ** 3; @@ -23,6 +24,7 @@ function config(overrides: Partial = {}): Config { return { diskPressure: { freeBytesThreshold: 10 * gibibyte }, eventBuffer: { capacity: 100 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, health: { enabled: true, maxConcurrentRecoveries: 1, @@ -786,6 +788,22 @@ describe("LeaseEngine", () => { expect(progress).toEqual(["queued"]); }); + it("cancels a queued request through the QueueControl facade and frees the requester for a later grant", async () => { + const harness = await createHarness(); + const holder = await harness.engine.request(request, { mode: "held", requesterId: "holder" }); + const queued = harness.engine.request(request, { mode: "held", requesterId: "queued" }); + await flush(); + + await expect(harness.engine.cancelPending("nobody")).resolves.toBe("not-found"); + await expect(harness.engine.cancelPending("queued")).resolves.toBe("cancelled"); + await expect(queued).rejects.toBeInstanceOf(RequestCancelledError); + + await harness.engine.release(holder.lease.id, "explicit"); + await expect( + harness.engine.request(request, { mode: "held", requesterId: "queued" }), + ).resolves.toMatchObject({ lease: { requesterId: "queued" } }); + }); + it("queues at capacity in FIFO order across three waiters", async () => { const harness = await createHarness(); const first = await harness.engine.request(request, { mode: "held", requesterId: "agent-1" }); diff --git a/src/core/lease-engine.ts b/src/core/lease-engine.ts index bcb2a74..d7d3e0a 100644 --- a/src/core/lease-engine.ts +++ b/src/core/lease-engine.ts @@ -51,6 +51,7 @@ export { NoCapacityError, NoDriverError, QueueTimeoutError, + RequestCancelledError, RequesterAlreadyLeasedError, } from "./lease-acquisition-coordinator.js"; @@ -300,6 +301,12 @@ export class LeaseEngine { await this.#acquisition.detachQueuedProgress(requesterId); } + /** Cancels a single pending request by requester id, for the HTTP lease-request delete route. */ + // fallow-ignore-next-line unused-class-member -- reached through the QueueControl port by DaemonServer (same as the sibling detachQueuedProgress). + async cancelPending(requesterId: string): Promise<"cancelled" | "not-found" | "not-cancellable"> { + return this.#acquisition.cancelPending(requesterId); + } + // fallow-ignore-next-line unused-class-member -- reached through the LeaseCommands port by DaemonServer (same as the sibling heartbeat). async renew(leaseId: string, ttlMs?: number): Promise { return this.#releaseCoordinator.renew(leaseId, ttlMs); diff --git a/src/core/lease-health-monitor.test.ts b/src/core/lease-health-monitor.test.ts index c4291fb..d9cc694 100644 --- a/src/core/lease-health-monitor.test.ts +++ b/src/core/lease-health-monitor.test.ts @@ -51,6 +51,7 @@ function config(overrides: Partial = {}): Config { }, log: { level: "info", rotateBytes: 5 * 1024 * 1024 }, stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, }; } diff --git a/src/core/lease-ports.ts b/src/core/lease-ports.ts index e84d71c..bb15692 100644 --- a/src/core/lease-ports.ts +++ b/src/core/lease-ports.ts @@ -21,6 +21,7 @@ export interface LeaseCommands { export interface QueueControl { readonly queueDepth: number; detachQueuedProgress(requesterId: string): Promise; + cancelPending(requesterId: string): Promise<"cancelled" | "not-found" | "not-cancellable">; } /** Read-only capacity view used by daemon status. */ diff --git a/src/core/nuke.test.ts b/src/core/nuke.test.ts index 6101e7c..ff6f0cc 100644 --- a/src/core/nuke.test.ts +++ b/src/core/nuke.test.ts @@ -156,6 +156,7 @@ function config(): Config { stableObservations: 2, }, stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, idle: { deleteAfterMs: 10, shutdownAfterMs: 5 }, lease: { detachedTtlMs: 60_000, heldTtlBackstopMs: 60_000, heartbeatIntervalMs: 15_000 }, capacity: { diff --git a/src/core/reaper.test.ts b/src/core/reaper.test.ts index 4984d3b..189b17c 100644 --- a/src/core/reaper.test.ts +++ b/src/core/reaper.test.ts @@ -55,6 +55,7 @@ function config(): Config { stableObservations: 2, }, stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, idle: { deleteAfterMs: 30_000, shutdownAfterMs: 10_000 }, lease: { detachedTtlMs: 100, heldTtlBackstopMs: 100, heartbeatIntervalMs: 25 }, capacity: { diff --git a/src/core/validation.ts b/src/core/validation.ts index 318473f..4653e95 100644 --- a/src/core/validation.ts +++ b/src/core/validation.ts @@ -79,6 +79,14 @@ export function booleanValue(value: unknown, path: string): boolean { return value; } +export function stringValue(value: unknown, path: string): string { + if (typeof value !== "string") { + throw invalidValue(path, "a string"); + } + + return value; +} + export function numberAtLeast(minimum: number): Validator { return (value: unknown, path: string) => { if (typeof value !== "number" || !Number.isFinite(value) || value < minimum) { @@ -89,6 +97,21 @@ export function numberAtLeast(minimum: number): Validator { }; } +export function integerInRange(minimum: number, maximum: number): Validator { + return (value: unknown, path: string) => { + if ( + typeof value !== "number" || + !Number.isInteger(value) || + value < minimum || + value > maximum + ) { + throw invalidValue(path, `an integer between ${minimum} and ${maximum}`); + } + + return value; + }; +} + export function stringUnion( allowed: readonly Value[], describe: (allowed: readonly Value[]) => string = (values) => diff --git a/src/core/wait-queue.test.ts b/src/core/wait-queue.test.ts index 1f40b9c..8285b2d 100644 --- a/src/core/wait-queue.test.ts +++ b/src/core/wait-queue.test.ts @@ -5,6 +5,7 @@ import type { DeviceRequest } from "./driver.js"; import { ForeignWaiterError, QueueTimeoutError, + RequestCancelledError, RequesterAlreadyLeasedError, WaitQueue, type LeaseGrant, @@ -159,6 +160,22 @@ describe("WaitQueue", () => { expect(rejected.state).toBe("rejected"); }); + it("finds a pending waiter across queued and not-yet-queued states, for the single-request cancel path", async () => { + const { queue } = createQueue(); + const queued = createWaiter(queue, "queued"); + const processing = createWaiter(queue, "processing"); + queue.enqueue(queued); + queue.markProcessing(processing); + + expect(queue.findPendingWaiter("queued")).toBe(queued); + expect(queue.findPendingWaiter("processing")).toBe(processing); + expect(queue.findPendingWaiter("nobody")).toBeUndefined(); + + expect(queue.reject(queued, new RequestCancelledError(queued.id))).toBe(true); + await expect(queued.promise).rejects.toBeInstanceOf(RequestCancelledError); + expect(queue.findPendingWaiter("queued")).toBeUndefined(); + }); + it("cancels every pending waiter and clears their pending requester state", async () => { const { queue } = createQueue(); const first = createWaiter(queue, "first"); diff --git a/src/core/wait-queue.ts b/src/core/wait-queue.ts index c62a5b1..da1a1e8 100644 --- a/src/core/wait-queue.ts +++ b/src/core/wait-queue.ts @@ -38,6 +38,13 @@ export class QueueTimeoutError extends Error { } } +export class RequestCancelledError extends Error { + constructor(readonly requestId: string) { + super(`Lease request cancelled: ${requestId}`); + this.name = "RequestCancelledError"; + } +} + export class RequesterAlreadyLeasedError extends Error { constructor( readonly requesterId: string, @@ -113,6 +120,19 @@ export class WaitQueue { return this.#pendingRequesters.has(requesterId); } + /** + * Finds a requester's waiter across every non-terminal state, not just the FIFO list -- a + * waiter driven straight to `processing` on its first attempt never gets enqueued at all, so + * a caller deciding cancellability needs this broader membership, unlike `detachProgress` + * which only ever needs to reach an already-queued entry. + */ + findPendingWaiter(requesterId: string): Waiter | undefined { + for (const waiter of this.#pendingWaiters) { + if (waiter.options.requesterId === requesterId) return waiter; + } + return undefined; + } + create(request: DeviceRequest, requestOptions: LeaseRequestOptions): Waiter { if (this.hasPendingRequester(requestOptions.requesterId)) { throw new RequesterAlreadyLeasedError(requestOptions.requesterId); diff --git a/src/core/warm-pool-coordinator.test.ts b/src/core/warm-pool-coordinator.test.ts index fa1a630..31ee926 100644 --- a/src/core/warm-pool-coordinator.test.ts +++ b/src/core/warm-pool-coordinator.test.ts @@ -26,6 +26,7 @@ const config: Config = { stableObservations: 2, }, stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, idle: { deleteAfterMs: 60_000, shutdownAfterMs: 10_000 }, warmPool: { quarantine: { diff --git a/src/daemon/main.ts b/src/daemon/main.ts index 459f372..71904e5 100644 --- a/src/daemon/main.ts +++ b/src/daemon/main.ts @@ -15,8 +15,12 @@ import { } from "../core/index.js"; import { AndroidDriver, SdkMissingError } from "../drivers/android/index.js"; import { IosSimctlDriver } from "../drivers/ios/index.js"; +import { createHttpApp } from "../http/app.js"; +import { HttpGateway } from "../http/server.js"; +import { TokenStore } from "../http/token-store.js"; import { CryptoIdGenerator, + CryptoTokenSecrets, JsonLinesLogger, NodeFileLogSink, type Clock, @@ -122,6 +126,10 @@ export async function startDaemon(options: StartDaemonOptions = {}): Promise Promise) | undefined; const daemon = new DaemonServer({ capacity: leaseEngine, catalog: leaseEngine, @@ -162,8 +170,66 @@ export async function startDaemon(options: StartDaemonOptions = {}): Promise leaseEngine.settle(), dispose: () => leaseEngine.dispose(), + stopAuxiliary: () => stopHttpGateway?.() ?? Promise.resolve(), }); await daemon.start(); + + if (config.http.enabled) { + const httpLogger = logger.child("http"); + const tokens = new TokenStore({ + clock, + filesystem, + idGenerator, + path: join(dataDirectory, "tokens.json"), + secrets: new CryptoTokenSecrets(), + }); + const app = createHttpApp({ + capacity: leaseEngine, + catalog: leaseEngine, + clock, + config, + // Only ever read once `daemon.start()` above has resolved (the gateway starts + // strictly after it, see the comment below), so this is always "running" -- the + // "failed" arm of the underlying `#health` state can only be observed during + // convergence, which is over by the time any HTTP request reaches this closure. + daemonHealth: () => daemon.health as "starting" | "running", + eventBus, + idGenerator, + leases: leaseEngine, + logger: httpLogger, + queue: leaseEngine, + registry, + tokens, + }); + const gateway = new HttpGateway(app, { + host: config.http.host, + logger: httpLogger, + port: config.http.port, + }); + // Started strictly after `daemon.start()` resolves, i.e. after convergence: a + // socket-daemon request parks on `#awaitReady` until convergence completes, but + // HTTP routes call the role interfaces (leaseEngine, registry) directly with no + // equivalent gate, so serving before convergence completes could let a remote + // client observe half-converged state. A connection refused while the daemon is + // still starting up is accepted v1 behavior -- there is no HTTP-level "starting" + // response to mirror the socket protocol's parked dispatch. + try { + await gateway.start(); + } catch (error: unknown) { + // The daemon is already fully started (socket claimed, timers armed) by this point. + // A bind failure -- an occupied port, an invalid host -- must not strand it as a + // half-configured zombie that startDaemon's caller believes failed to start: tear it + // down before rethrowing, so failure means *nothing* is left running. + app.dispose(); + await daemon.stop("http-start-failed").catch(() => undefined); + throw error; + } + stopHttpGateway = async () => { + await gateway.stop(); + app.dispose(); + }; + } + return daemon; } diff --git a/src/daemon/server.test.ts b/src/daemon/server.test.ts index dc823b3..fc880d9 100644 --- a/src/daemon/server.test.ts +++ b/src/daemon/server.test.ts @@ -1010,6 +1010,44 @@ describe("DaemonServer lease heartbeat", () => { expect(order).toEqual(["settle-start", "settle-end", "dispose"]); }); + it("stops an auxiliary frontend before releasing held leases and draining settle", async () => { + const order: string[] = []; + const harness = await createHarness({ + dispose: () => order.push("dispose"), + settle: async () => { + order.push("settle"); + }, + stopAuxiliary: async () => { + order.push("stopAuxiliary"); + }, + }); + const holder = await createClient(harness.socketPath); + await hello(holder); + await holder.request("lease.request", { + mode: "held", + requesterId: "holder", + request: { model: "iPhone 16", osVersion: "26.5", platform: "ios" }, + }); + + await harness.daemon.stop("test-stop-auxiliary"); + + expect(order).toEqual(["stopAuxiliary", "settle", "dispose"]); + // The held lease was still released as part of the same stop -- stopping the + // auxiliary frontend first doesn't skip the socket protocol's own teardown. + expect(harness.registry.snapshot.leases).toHaveLength(0); + }); + + it("reports health via the public accessor across the startup/stop lifecycle", async () => { + const harness = await createHarness({ start: false }); + expect(harness.daemon.health).toBe("starting"); + + await harness.daemon.start(); + expect(harness.daemon.health).toBe("running"); + + await harness.daemon.stop("test-health"); + expect(harness.daemon.health).toBe("running"); + }); + it("logs a clean shutdown", async () => { const { logger: log, sink } = logger(); const harness = await createHarness({ logger: log }); @@ -1088,6 +1126,7 @@ async function createHarness( readonly logger?: Logger; readonly settle?: () => Promise; readonly stateFilesystem?: MemoryFilesystem; + readonly stopAuxiliary?: () => Promise; } = {}, ) { const directory = @@ -1158,6 +1197,7 @@ async function createHarness( registry, settle: options.settle ?? (async () => engine.settle()), ...(options.dispose === undefined ? {} : { dispose: options.dispose }), + ...(options.stopAuxiliary === undefined ? {} : { stopAuxiliary: options.stopAuxiliary }), version: "test", }); runningDaemons.push(daemon); @@ -1273,6 +1313,7 @@ function testConfig(leaseOverrides?: Partial): Config { stableObservations: 2, }, stalledTransition: { thresholdMultiplier: 3, minimumThresholdMs: 60_000 }, + http: { enabled: false, host: "127.0.0.1", port: 4700 }, idle: { deleteAfterMs: 60_000, shutdownAfterMs: 10_000 }, lease: { detachedTtlMs: 60_000, diff --git a/src/daemon/server.ts b/src/daemon/server.ts index 6ddf38e..a6cfa8b 100644 --- a/src/daemon/server.ts +++ b/src/daemon/server.ts @@ -84,6 +84,13 @@ export interface DaemonServerOptions { readonly settle?: () => Promise; /** Cancels any timers the lease subsystem armed (e.g. quarantine retries) on shutdown. */ readonly dispose?: () => void; + /** + * Stops an auxiliary frontend (today: the HTTP gateway's listener, started only after + * `start()` resolves -- see `main.ts`) before anything else in `stop()` runs, so no + * request arriving through it can ever observe a stopping engine. A no-op default when + * no auxiliary frontend is running. + */ + readonly stopAuxiliary?: () => Promise; } type DaemonHealth = "starting" | "running" | "failed"; @@ -118,6 +125,11 @@ export class DaemonServer { return this.options.host.endpoint; } + /** Public read of `#health` for an auxiliary frontend (e.g. the HTTP gateway's `daemonHealth`) that needs it without becoming a privileged internal itself. */ + get health(): DaemonHealth { + return this.#health; + } + /** * Claims the socket first so reachability never depends on startup recovery work: * a lost startup race throws `DaemonAlreadyRunningError` here, before `converge()` @@ -247,6 +259,12 @@ export class DaemonServer { } async #stop(reason: string): Promise { + // Awaited first, before anything below: an auxiliary frontend calls the role + // interfaces directly rather than parking on `#awaitReady`/this method's own + // teardown order, so it must be shut off before held-lease release and + // lease/queue teardown begin -- otherwise a request arriving through it mid-stop + // could run against an engine already being torn down. + await this.options.stopAuxiliary?.(); this.#logger.info("Daemon stopping", { reason }); this.options.eventBus.emit("daemon.stopping", { reason }, "daemon"); if (this.#heartbeatTimer !== undefined) { diff --git a/src/http/app.test.ts b/src/http/app.test.ts new file mode 100644 index 0000000..0338c5f --- /dev/null +++ b/src/http/app.test.ts @@ -0,0 +1,661 @@ +import { describe, expect, it } from "vitest"; + +import { EventBus } from "../bus/index.js"; +import { NoCapacityError, RequesterAlreadyLeasedError } from "../core/index.js"; +import { FakeClock, JsonLinesLogger, MemoryLogSink } from "../ports/index.js"; +import { createHttpApp, type HttpGatewayDeps } from "./app.js"; +import { + FakeCapacityReader, + FakeCatalogReader, + FakeLeaseCommands, + FakeQueueControl, + FakeRegistry, + FakeTokenVerifier, + makeDevice, + makeGrant, + makeLease, + sequenceIdGenerator, + testConfig, + waitForCall, +} from "./test-fakes.js"; + +function buildHarness(overrides: { readonly config?: HttpGatewayDeps["config"] } = {}) { + const clock = new FakeClock(1_000); + const eventBus = new EventBus(clock); + const leases = new FakeLeaseCommands(); + const queue = new FakeQueueControl(); + const capacity = new FakeCapacityReader(); + const catalog = new FakeCatalogReader(); + const registry = new FakeRegistry(); + const tokens = new FakeTokenVerifier(); + tokens.register("slk_agent", { requesterId: "tok_agent", role: "agent" }); + tokens.register("slk_other", { requesterId: "tok_other", role: "agent" }); + tokens.register("slk_operator", { requesterId: "tok_operator", role: "operator" }); + const logSink = new MemoryLogSink(); + const logger = new JsonLinesLogger({ clock, sink: logSink }); + const config = overrides.config ?? testConfig(); + + const app = createHttpApp({ + capacity, + catalog, + clock, + config, + daemonHealth: () => "running", + eventBus, + idGenerator: sequenceIdGenerator("gw"), + leases, + logger, + queue, + registry, + tokens, + }); + + return { app, clock, config, eventBus, leases, logSink, queue, registry, tokens }; +} + +type App = ReturnType["app"]; + +const agentAuth = { authorization: "Bearer slk_agent" }; +const otherAgentAuth = { authorization: "Bearer slk_other" }; +const operatorAuth = { authorization: "Bearer slk_operator" }; +const defaultBody = { device: "iPhone 17 Pro", platform: "ios" }; + +function postLeaseRequest( + app: App, + body: Record, + headers: Record = agentAuth, +): Promise { + return Promise.resolve( + app.request("/v1/lease-requests", { + body: JSON.stringify(body), + headers: { ...headers, "content-type": "application/json" }, + method: "POST", + }), + ); +} + +/** + * Drives a `POST /v1/lease-requests` through to its 201 response and returns the created + * request's id. `LeaseRequestTracker.submit` never settles until `LeaseCommands.request`'s + * first `onProgress` call (or its own grant/rejection) -- see `tracker.ts` -- so this scripts + * one `queued` progress event rather than awaiting the response before that call exists. + */ +async function createLeaseRequest( + app: App, + leases: FakeLeaseCommands, + body: Record = defaultBody, + headers: Record = agentAuth, +): Promise<{ readonly id: string; readonly callIndex: number }> { + const callIndex = leases.calls.length; + const responsePromise = postLeaseRequest(app, body, headers); + await waitForCall(leases, callIndex); + leases.calls[callIndex]?.options.onProgress?.({ queuePosition: 1, stage: "queued" }); + const response = await responsePromise; + const { request } = (await response.json()) as { request: { id: string } }; + return { callIndex, id: request.id }; +} + +interface SseFrame { + readonly event?: string; + readonly data: unknown; +} + +/** A keepalive comment (`: ...`) is not a frame; everything else parses as event+data lines. */ +function parseSseFrame(raw: string): SseFrame | undefined { + if (raw.startsWith(":")) return undefined; + let event: string | undefined; + let data = ""; + for (const line of raw.split("\n")) { + if (line.startsWith("event:")) event = line.slice(6).trim(); + else if (line.startsWith("data:")) data += line.slice(5).trim(); + } + return { data: JSON.parse(data) as unknown, ...(event === undefined ? {} : { event }) }; +} + +/** Splits off every complete (`\n\n`-terminated) frame; the trailing partial stays in `rest`. */ +function splitCompleteFrames(buffer: string): { complete: string[]; rest: string } { + const parts = buffer.split("\n\n"); + const rest = parts.pop() ?? ""; + return { complete: parts, rest }; +} + +async function readSseFrames(response: Response, count: number): Promise { + const reader = response.body?.getReader(); + if (reader === undefined) throw new Error("response has no body"); + const decoder = new TextDecoder(); + let buffer = ""; + const frames: SseFrame[] = []; + while (frames.length < count) { + const { value, done } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const { complete, rest } = splitCompleteFrames(buffer); + buffer = rest; + for (const raw of complete) { + if (frames.length >= count) break; + const frame = parseSseFrame(raw); + if (frame !== undefined) frames.push(frame); + } + } + await reader.cancel(); + return frames; +} + +describe("GET /v1/healthz", () => { + it("answers without authentication", async () => { + const { app } = buildHarness(); + const response = await app.request("/v1/healthz"); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true }); + }); +}); + +describe("authentication and ownership", () => { + it("401s a protected route with no token", async () => { + const { app } = buildHarness(); + const response = await app.request("/v1/status"); + expect(response.status).toBe(401); + }); + + it("403s an agent token reaching an operator-only route", async () => { + const { app } = buildHarness(); + const response = await app.request("/v1/leases", { headers: agentAuth }); + expect(response.status).toBe(403); + }); + + it("403s an agent reaching another requester's lease request", async () => { + const { app, leases } = buildHarness(); + const { id } = await createLeaseRequest(app, leases); + + const response = await app.request(`/v1/lease-requests/${id}`, { headers: otherAgentAuth }); + expect(response.status).toBe(403); + }); +}); + +describe("GET /v1/status, /v1/catalog", () => { + it("reports status shaped like the daemon's status.get", async () => { + const { app, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1", state: "ready" })]; + registry.leases = []; + + const response = await app.request("/v1/status", { headers: agentAuth }); + expect(response.status).toBe(200); + const body = (await response.json()) as { + health: string; + queueDepth: number; + devices: unknown[]; + capacity: { ios: { warm: number } }; + }; + expect(body.health).toBe("running"); + expect(body.queueDepth).toBe(0); + expect(body.devices).toHaveLength(1); + expect(body.capacity.ios.warm).toBe(1); + }); + + it("filters the catalog by platform query", async () => { + const { app } = buildHarness(); + const response = await app.request("/v1/catalog?platform=ios", { headers: agentAuth }); + expect(await response.json()).toEqual({ + platforms: [ + { defaultRuntime: "26.5", models: ["iPhone 17 Pro"], platform: "ios", runtimes: ["26.5"] }, + ], + }); + }); + + it("400s an invalid platform query", async () => { + const { app } = buildHarness(); + const response = await app.request("/v1/catalog?platform=windows", { headers: agentAuth }); + expect(response.status).toBe(400); + expect(((await response.json()) as { error: { code: string } }).error.code).toBe("BAD_REQUEST"); + }); +}); + +describe("POST /v1/lease-requests", () => { + it("creates a request resource, 201 with a Location header", async () => { + const { app, leases } = buildHarness(); + const responsePromise = postLeaseRequest(app, defaultBody); + await waitForCall(leases); + leases.calls[0]?.options.onProgress?.({ queuePosition: 2, stage: "queued" }); + const response = await responsePromise; + + expect(response.status).toBe(201); + expect(response.headers.get("Location")).toMatch(/^\/v1\/lease-requests\/req_/); + const body = (await response.json()) as { + request: { id: string; state: string; queuePosition: number }; + }; + expect(body.request.state).toBe("queued"); + expect(body.request.queuePosition).toBe(2); + }); + + it("400s a malformed body before ever calling LeaseCommands", async () => { + const { app, leases } = buildHarness(); + const response = await postLeaseRequest(app, { platform: "ios" }); + expect(response.status).toBe(400); + expect(leases.calls).toHaveLength(0); + }); + + it("maps a fast RequesterAlreadyLeasedError to 409, naming the existing lease", async () => { + const { app, leases } = buildHarness(); + const responsePromise = postLeaseRequest(app, defaultBody); + await waitForCall(leases); + leases.calls[0]?.reject(new RequesterAlreadyLeasedError("tok_agent", "lse_existing")); + const response = await responsePromise; + + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ + error: { + code: "REQUESTER_ALREADY_LEASED", + existingLeaseId: "lse_existing", + message: expect.any(String), + }, + }); + }); + + it("maps a fast NoCapacityError (noWait) to 503 with Retry-After", async () => { + const { app, leases } = buildHarness(); + const responsePromise = postLeaseRequest(app, { ...defaultBody, noWait: true }); + await waitForCall(leases); + leases.calls[0]?.reject(new NoCapacityError()); + const response = await responsePromise; + + expect(response.status).toBe(503); + expect(response.headers.get("Retry-After")).toBeTruthy(); + }); + + it("replays an identical Idempotency-Key without a second LeaseCommands.request call", async () => { + const { app, leases } = buildHarness(); + const { id: firstId } = await createLeaseRequest(app, leases, defaultBody, { + ...agentAuth, + "idempotency-key": "abc", + }); + + const second = await postLeaseRequest(app, defaultBody, { + ...agentAuth, + "idempotency-key": "abc", + }); + expect(second.status).toBe(201); + const secondBody = (await second.json()) as { request: { id: string } }; + expect(leases.calls).toHaveLength(1); + expect(secondBody.request.id).toBe(firstId); + }); +}); + +describe("full lease-request lifecycle via GET / long-poll / SSE", () => { + it("progresses queued -> booting -> granted, observable through GET", async () => { + const { app, leases } = buildHarness(); + const { id, callIndex } = await createLeaseRequest(app, leases); + + leases.calls[callIndex]?.options.onProgress?.({ etaMs: 30_000, stage: "booting" }); + const midway = await app.request(`/v1/lease-requests/${id}`, { headers: agentAuth }); + expect( + ((await midway.json()) as { request: { state: string; etaSeconds: number } }).request, + ).toEqual({ + createdAt: expect.any(String), + etaSeconds: 30, + id, + state: "booting", + }); + + leases.calls[callIndex]?.resolve(makeGrant({ lease: { id: "lse_final" } })); + const final = await app.request(`/v1/lease-requests/${id}`, { headers: agentAuth }); + const finalBody = (await final.json()) as { + request: { state: string; lease: { id: string; dataPlane: unknown } }; + }; + expect(finalBody.request.state).toBe("granted"); + expect(finalBody.request.lease.id).toBe("lse_final"); + expect(finalBody.request.lease.dataPlane).toBeNull(); + }); + + it("long-polls: ?wait returns early on a state change", async () => { + const { app, leases } = buildHarness(); + const { id, callIndex } = await createLeaseRequest(app, leases); + + const waitPromise = app.request(`/v1/lease-requests/${id}?wait=30`, { headers: agentAuth }); + await Promise.resolve(); + await Promise.resolve(); + leases.calls[callIndex]?.options.onProgress?.({ etaMs: 10_000, stage: "provisioning" }); + const response = await waitPromise; + const body = (await response.json()) as { request: { state: string } }; + expect(body.request.state).toBe("provisioning"); + }); + + it("long-polls: ?wait returns the unchanged state once the timer elapses", async () => { + const { app, clock, leases } = buildHarness(); + const { id } = await createLeaseRequest(app, leases); + + const waitPromise = app.request(`/v1/lease-requests/${id}?wait=5`, { headers: agentAuth }); + await Promise.resolve(); + await Promise.resolve(); + clock.advance(5_000); + const response = await waitPromise; + const body = (await response.json()) as { request: { state: string; queuePosition: number } }; + expect(body.request).toMatchObject({ queuePosition: 1, state: "queued" }); + }); + + it("streams SSE progress events, ending with the terminal granted event", async () => { + const { app, leases } = buildHarness(); + const { id, callIndex } = await createLeaseRequest(app, leases); + + const streamResponse = await app.request(`/v1/lease-requests/${id}/events`, { + headers: agentAuth, + }); + expect(streamResponse.headers.get("content-type")).toContain("text/event-stream"); + + const framesPromise = readSseFrames(streamResponse, 3); + leases.calls[callIndex]?.options.onProgress?.({ etaMs: 20_000, stage: "provisioning" }); + leases.calls[callIndex]?.resolve(makeGrant({ lease: { id: "lse_sse" } })); + const frames = await framesPromise; + + expect(frames.map((frame) => frame.event)).toEqual(["queued", "provisioning", "granted"]); + const last = frames.at(-1)?.data as { lease: { id: string } }; + expect(last.lease.id).toBe("lse_sse"); + }); +}); + +describe("DELETE /v1/lease-requests/:id", () => { + it("404s an unknown request id", async () => { + const { app } = buildHarness(); + const response = await app.request("/v1/lease-requests/req-missing", { + headers: agentAuth, + method: "DELETE", + }); + expect(response.status).toBe(404); + }); + + it("204s when the request was still queued and cancellable", async () => { + const { app, leases, queue } = buildHarness(); + const { id } = await createLeaseRequest(app, leases); + + queue.cancelOutcome = "cancelled"; + const response = await app.request(`/v1/lease-requests/${id}`, { + headers: agentAuth, + method: "DELETE", + }); + expect(response.status).toBe(204); + }); + + it("409s not-cancellable once device work is in flight", async () => { + const { app, leases, queue } = buildHarness(); + const { id } = await createLeaseRequest(app, leases); + + queue.cancelOutcome = "not-cancellable"; + const response = await app.request(`/v1/lease-requests/${id}`, { + headers: agentAuth, + method: "DELETE", + }); + expect(response.status).toBe(409); + expect(((await response.json()) as { error: { code: string } }).error.code).toBe( + "REQUEST_NOT_CANCELLABLE", + ); + }); + + it("409s not-cancellable naming the lease id once already granted", async () => { + const { app, leases } = buildHarness(); + const responsePromise = postLeaseRequest(app, defaultBody); + await waitForCall(leases); + leases.calls[0]?.resolve(makeGrant({ lease: { id: "lse_granted" } })); + const created = await responsePromise; + const { request } = (await created.json()) as { request: { id: string } }; + + const response = await app.request(`/v1/lease-requests/${request.id}`, { + headers: agentAuth, + method: "DELETE", + }); + expect(response.status).toBe(409); + expect(await response.json()).toEqual({ + error: { + code: "REQUEST_NOT_CANCELLABLE", + leaseId: "lse_granted", + message: expect.any(String), + }, + }); + }); +}); + +describe("lease routes", () => { + it("GET /v1/leases/:id returns the acquisition-shaped lease object", async () => { + const { app, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", requesterId: "tok_agent" })]; + + const response = await app.request("/v1/leases/lse_1", { headers: agentAuth }); + expect(response.status).toBe(200); + const body = (await response.json()) as { lease: Record }; + expect(body.lease).toMatchObject({ + dataPlane: null, + deviceId: "dev_1", + device: "iPhone 17 Pro", + id: "lse_1", + platform: "ios", + udid: "ABCD-1234", + }); + }); + + it("404s an unknown lease id", async () => { + const { app } = buildHarness(); + const response = await app.request("/v1/leases/lse-missing", { headers: agentAuth }); + expect(response.status).toBe(404); + }); + + it("403s an agent fetching another requester's lease", async () => { + const { app, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", requesterId: "tok_other" })]; + + const response = await app.request("/v1/leases/lse_1", { headers: agentAuth }); + expect(response.status).toBe(403); + }); + + it("POST /v1/leases/:id/renew drains buffered device-health notices", async () => { + const { app, eventBus, leases, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", requesterId: "tok_agent" })]; + leases.renewImpl = async (leaseId, ttlMs) => ({ + deviceId: "dev_1", + grantedAt: 1_000, + id: leaseId, + mode: "detached", + requesterId: "tok_agent", + ttlDeadline: 2_000 + (ttlMs ?? 900_000), + }); + + eventBus.emit( + "device.crash-detected", + { deviceId: "dev_1", leaseId: "lse_1", observed: "x", platform: "ios" }, + "test", + ); + eventBus.emit( + "device.recovered", + { attempts: 1, deviceId: "dev_1", duration: 500, leaseId: "lse_1" }, + "test", + ); + + const response = await app.request("/v1/leases/lse_1/renew", { + body: JSON.stringify({ ttlMs: 120_000 }), + headers: { ...agentAuth, "content-type": "application/json" }, + method: "POST", + }); + expect(response.status).toBe(200); + const body = (await response.json()) as { leaseId: string; notices: unknown[] }; + expect(body.leaseId).toBe("lse_1"); + expect(body.notices).toEqual([ + { event: "device_unhealthy" }, + { attempts: 1, event: "device_recovered" }, + ]); + expect(leases.renewCalls).toEqual([{ leaseId: "lse_1", ttlMs: 120_000 }]); + }); + + it("POST /v1/leases/:id/renew accepts an empty body", async () => { + const { app, leases, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", requesterId: "tok_agent" })]; + leases.renewImpl = async (leaseId) => ({ + deviceId: "dev_1", + grantedAt: 1_000, + id: leaseId, + mode: "detached", + requesterId: "tok_agent", + ttlDeadline: 2_000, + }); + + const response = await app.request("/v1/leases/lse_1/renew", { + headers: agentAuth, + method: "POST", + }); + expect(response.status).toBe(200); + expect(leases.renewCalls).toEqual([{ leaseId: "lse_1" }]); + }); + + it("404s a renew for an unknown lease", async () => { + const { app } = buildHarness(); + const response = await app.request("/v1/leases/lse-missing/renew", { + headers: agentAuth, + method: "POST", + }); + expect(response.status).toBe(404); + }); + + it("streams live lease notices over SSE, ending on lease_lost", async () => { + const { app, eventBus, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", requesterId: "tok_agent" })]; + + const streamResponse = await app.request("/v1/leases/lse_1/events", { headers: agentAuth }); + const framesPromise = readSseFrames(streamResponse, 2); + await Promise.resolve(); + eventBus.emit( + "device.crash-detected", + { deviceId: "dev_1", leaseId: "lse_1", observed: "x", platform: "ios" }, + "test", + ); + eventBus.emit("lease.expired", { deviceId: "dev_1", leaseId: "lse_1" }, "test"); + const frames = await framesPromise; + + expect(frames.map((frame) => frame.event)).toEqual(["device_unhealthy", "lease_lost"]); + }); + + it("DELETE /v1/leases/:id releases and answers 202 with the device's post-release state", async () => { + const { app, leases, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1", state: "leased" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", requesterId: "tok_agent" })]; + leases.releaseImpl = async (leaseId) => { + registry.leases = registry.leases.filter((lease) => lease.id !== leaseId); + registry.devices = registry.devices.map((device) => + device.id === "dev_1" ? { ...device, state: "reclaiming" } : device, + ); + }; + + const response = await app.request("/v1/leases/lse_1", { + headers: agentAuth, + method: "DELETE", + }); + expect(response.status).toBe(202); + expect(await response.json()).toEqual({ + device: { id: "dev_1", state: "reclaiming" }, + released: true, + }); + expect(leases.releaseCalls).toEqual([{ leaseId: "lse_1", reason: "explicit" }]); + }); + + it("an operator may release another requester's lease", async () => { + const { app, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", requesterId: "tok_agent" })]; + + const response = await app.request("/v1/leases/lse_1", { + headers: operatorAuth, + method: "DELETE", + }); + expect(response.status).toBe(202); + }); +}); + +describe("operator surface", () => { + it("GET /v1/leases lists every lease", async () => { + const { app, registry } = buildHarness(); + registry.leases = [ + makeLease({ id: "lse_1" }), + makeLease({ id: "lse_2", requesterId: "tok_other" }), + ]; + + const response = await app.request("/v1/leases", { headers: operatorAuth }); + const body = (await response.json()) as { leases: Array<{ id: string }> }; + expect(body.leases.map((lease) => lease.id).sort()).toEqual(["lse_1", "lse_2"]); + }); + + it("GET /v1/devices lists every device", async () => { + const { app, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1" }), makeDevice({ id: "dev_2" })]; + + const response = await app.request("/v1/devices", { headers: operatorAuth }); + const body = (await response.json()) as { devices: Array<{ id: string }> }; + expect(body.devices.map((device) => device.id).sort()).toEqual(["dev_1", "dev_2"]); + }); + + it("GET /v1/events replays the ring buffer, optionally filtered by ?since", async () => { + const { app, clock, eventBus } = buildHarness(); + eventBus.emit("daemon.started", { configSnapshot: {}, version: "1" }, "test"); + clock.advance(10_000); + eventBus.emit("daemon.stopping", { reason: "test" }, "test"); + + const all = await app.request("/v1/events", { headers: operatorAuth }); + expect(((await all.json()) as { events: unknown[] }).events).toHaveLength(2); + + const recent = await app.request("/v1/events?since=5s", { headers: operatorAuth }); + const recentBody = (await recent.json()) as { events: Array<{ event: string }> }; + expect(recentBody.events).toEqual([expect.objectContaining({ event: "daemon.stopping" })]); + }); + + it("400s an invalid ?since duration", async () => { + const { app } = buildHarness(); + const response = await app.request("/v1/events?since=nonsense", { headers: operatorAuth }); + expect(response.status).toBe(400); + }); + + it("GET /v1/events/stream follows the event bus live", async () => { + const { app, eventBus } = buildHarness(); + const streamResponse = await app.request("/v1/events/stream", { headers: operatorAuth }); + const framesPromise = readSseFrames(streamResponse, 1); + await Promise.resolve(); + eventBus.emit("daemon.stopping", { reason: "live" }, "test"); + const frames = await framesPromise; + expect(frames[0]?.event).toBe("daemon.stopping"); + }); +}); + +describe("hardening from review", () => { + it("400s an oversized Idempotency-Key", async () => { + const { app } = buildHarness(); + const response = await postLeaseRequest(app, defaultBody, { + ...agentAuth, + "Idempotency-Key": "x".repeat(201), + }); + expect(response.status).toBe(400); + }); + + it("clamps an oversized ?wait to the 60s ceiling", async () => { + const { app, clock, leases } = buildHarness(); + const { id } = await createLeaseRequest(app, leases); + + const waitPromise = app.request(`/v1/lease-requests/${id}?wait=999999`, { + headers: agentAuth, + }); + await Promise.resolve(); + await Promise.resolve(); + clock.advance(60_000); + const response = await waitPromise; + expect(((await response.json()) as { request: { state: string } }).request.state).toBe( + "queued", + ); + }); + + it("reports the mode-default ttlMs for a lease the tracker has no record of", async () => { + const { app, config, registry } = buildHarness(); + registry.devices = [makeDevice({ id: "dev_1" })]; + registry.leases = [makeLease({ deviceId: "dev_1", id: "lse_1", requesterId: "tok_agent" })]; + + const response = await app.request("/v1/leases/lse_1", { headers: agentAuth }); + const body = (await response.json()) as { lease: { ttlMs: number } }; + expect(body.lease.ttlMs).toBe(config.lease.detachedTtlMs); + }); +}); diff --git a/src/http/app.ts b/src/http/app.ts new file mode 100644 index 0000000..58f8d50 --- /dev/null +++ b/src/http/app.ts @@ -0,0 +1,486 @@ +import { zValidator } from "@hono/zod-validator"; +import { Hono } from "hono"; +import { z } from "zod"; + +import type { EventBus } from "../bus/index.js"; +import { + type Config, + type DeviceRecord, + type LeaseRecord, + transitionEnteredAt, +} from "../core/index.js"; +import type { + CapacityReader, + CatalogReader, + LeaseCommands, + QueueControl, +} from "../core/lease-ports.js"; +import type { Clock, IdGenerator, Logger } from "../ports/index.js"; +import { type AuthEnv, requireAuth, requireOwnership } from "./auth.js"; +import { + badRequest, + errorResponse, + mapError, + requestNotCancellable, + unknownLease, + unknownRequest, +} from "./errors.js"; +import { LeaseNoticeBuffer } from "./notices.js"; +import { pipeSse } from "./sse.js"; +import type { TokenIdentity } from "./token-store.js"; +import { + buildLeasePayload, + isTerminalStage, + LeaseRequestTracker, + type TrackedRequestView, +} from "./tracker.js"; + +/** Minimal structural read surface -- narrower than importing the `Registry` class itself. */ +export interface HttpRegistryReader { + readonly snapshot: { + readonly devices: readonly DeviceRecord[]; + readonly leases: readonly LeaseRecord[]; + }; +} + +export interface HttpGatewayDeps { + readonly leases: LeaseCommands; + readonly queue: QueueControl; + readonly capacity: CapacityReader; + readonly catalog: CatalogReader; + readonly registry: HttpRegistryReader; + readonly eventBus: EventBus; + readonly clock: Clock; + readonly idGenerator: IdGenerator; + readonly logger: Logger; + readonly config: Config; + readonly tokens: { verify(secret: string): Promise }; + readonly daemonHealth: () => "starting" | "running"; +} + +type Env = AuthEnv; + +/** Upper bound on `?wait=` long-polls; bounds how long an abandoned poll can pin resources. */ +const MAX_LONG_POLL_SECONDS = 60; +/** Idempotency keys are map keys held for the replay window; unbounded length is a memory lever. */ +const MAX_IDEMPOTENCY_KEY_LENGTH = 200; + +const leaseRequestBodySchema = z.object({ + allowDownload: z.boolean().optional(), + device: z.string().min(1), + noWait: z.boolean().optional(), + os: z.string().min(1).optional(), + platform: z.enum(["ios", "android"]), + timeoutMs: z.number().int().positive().optional(), + ttlMs: z.number().int().positive().optional(), +}); + +/** The gateway-owned subscriptions `createHttpApp` starts, attached to the returned app so a caller can dispose them on shutdown without this module exposing the tracker/notices instances themselves. */ +export interface HttpAppDisposable { + readonly dispose: () => void; +} + +/** Pure `Request -> Response` app: no `node:http`, no `serve()` -- see `server.ts` for that. */ +// fallow-ignore-next-line complexity -- route wiring for one focused resource surface; splitting it would scatter the shared closures (tracker, notices) across files for no clarity gain. +export function createHttpApp(deps: HttpGatewayDeps): Hono & HttpAppDisposable { + const app = new Hono(); + const logger = deps.logger.child("http"); + const tracker = new LeaseRequestTracker({ + clock: deps.clock, + defaultTtlMs: deps.config.lease.detachedTtlMs, + eventBus: deps.eventBus, + idGenerator: deps.idGenerator, + leases: deps.leases, + logger, + queue: deps.queue, + }); + const notices = new LeaseNoticeBuffer(deps.eventBus); + + const agentAuth = requireAuth(deps.tokens); + const operatorAuth = requireAuth(deps.tokens, "operator"); + + // The one error boundary. Hono's `compose` catches a thrown error at the layer that threw + // it and hands it to `app.onError` right there -- it never propagates up through an outer + // middleware's own `await next()` -- so this has to be `onError`, not a try/catch here. + app.onError((error, c) => { + const mapped = mapError(error); + if (mapped.code === "INTERNAL") { + logger.error("Unhandled request error", { message: errorMessage(error), path: c.req.path }); + } + return errorResponse(c, error); + }); + + // "One structured line per request outcome" -- by the time `next()` resolves, `onError` + // above has already run for a thrown error and `c.res` reflects the mapped status. + app.use("*", async (c, next) => { + const start = deps.clock.now(); + await next(); + // The one unauthenticated route is also the one a tunnel/load-balancer polls: logging it + // would let an anonymous flood drive the synchronous log sink from the event loop. + if (c.req.path === "/v1/healthz") return; + const identity = c.get("identity") as TokenIdentity | undefined; + logger.info("request", { + durationMs: deps.clock.now() - start, + method: c.req.method, + path: c.req.path, + requesterId: identity?.requesterId, + status: c.res.status, + }); + }); + + app.get("/v1/healthz", (c) => c.json({ ok: true })); + + app.get("/v1/status", agentAuth, (c) => c.json(buildStatus(deps))); + + app.get("/v1/catalog", agentAuth, async (c) => { + const platform = c.req.query("platform"); + if (platform !== undefined && platform !== "ios" && platform !== "android") { + throw badRequest("platform must be ios or android"); + } + const platforms = await deps.catalog.listCatalog(platform); + return c.json({ platforms }); + }); + + app.post( + "/v1/lease-requests", + agentAuth, + zValidator("json", leaseRequestBodySchema, (result, c) => { + if (!result.success) { + return errorResponse(c, badRequest(formatZodIssues(result.error.issues))); + } + }), + async (c) => { + const identity = c.get("identity"); + const body = c.req.valid("json"); + const idempotencyKey = c.req.header("Idempotency-Key"); + if (idempotencyKey !== undefined && idempotencyKey.length > MAX_IDEMPOTENCY_KEY_LENGTH) { + throw badRequest( + `Idempotency-Key must be at most ${MAX_IDEMPOTENCY_KEY_LENGTH} characters`, + ); + } + + // `allowDownload` passes through unclamped, matching the socket daemon's handling of the + // same flag; if a config-level download policy ever gates it there, this route must gate + // through the same helper. + const outcome = await tracker.submit( + identity, + { + device: body.device, + platform: body.platform, + ...(body.os === undefined ? {} : { os: body.os }), + ...(body.ttlMs === undefined ? {} : { ttlMs: body.ttlMs }), + ...(body.timeoutMs === undefined ? {} : { timeoutMs: body.timeoutMs }), + ...(body.noWait === undefined ? {} : { noWait: body.noWait }), + ...(body.allowDownload === undefined ? {} : { allowDownload: body.allowDownload }), + }, + idempotencyKey, + ); + if (outcome.kind === "rejected") { + return errorResponse(c, outcome.error); + } + c.header("Location", `/v1/lease-requests/${outcome.view.id}`); + return c.json({ request: serializeRequest(outcome.view) }, 201); + }, + ); + + app.get("/v1/lease-requests/:id", agentAuth, async (c) => { + const id = c.req.param("id"); + const initial = tracker.get(id); + if (initial === undefined) throw unknownRequest(id); + requireOwnership(c.get("identity"), initial.requesterId); + + const waitParam = c.req.query("wait"); + if (waitParam !== undefined) { + const seconds = Number(waitParam); + if (!Number.isFinite(seconds) || seconds < 0) { + throw badRequest("wait must be a non-negative number of seconds"); + } + // Clamped, not rejected: an oversized wait still long-polls correctly -- the client + // simply re-polls sooner than it asked -- and the clamp bounds how long an abandoned + // poll can pin its listener and timer. Aborting the request releases them immediately. + await tracker.waitForChange(id, Math.min(seconds, MAX_LONG_POLL_SECONDS), c.req.raw.signal); + } + + const view = tracker.get(id) ?? initial; + return c.json({ request: serializeRequest(view) }); + }); + + app.get("/v1/lease-requests/:id/events", agentAuth, (c) => { + const id = c.req.param("id"); + const initial = tracker.get(id); + if (initial === undefined) throw unknownRequest(id); + requireOwnership(c.get("identity"), initial.requesterId); + + return pipeSse(c, deps.clock, { + subscribe(send, end) { + const current = tracker.get(id) ?? initial; + send({ data: serializeRequest(current), event: current.state.stage }); + if (isTerminalStage(current.state)) { + end(); + return () => {}; + } + const unsubscribe = tracker.subscribe(id, (state) => { + send({ data: serializeRequest({ ...current, state }), event: state.stage }); + if (isTerminalStage(state)) end(); + }); + return unsubscribe ?? (() => {}); + }, + }); + }); + + app.delete("/v1/lease-requests/:id", agentAuth, async (c) => { + const id = c.req.param("id"); + const existing = tracker.get(id); + if (existing === undefined) throw unknownRequest(id); + requireOwnership(c.get("identity"), existing.requesterId); + + const outcome = await tracker.cancel(id); + if (outcome.kind === "cancelled") return c.body(null, 204); + if (outcome.kind === "not-found") throw unknownRequest(id); + if (outcome.leaseId !== undefined) { + throw requestNotCancellable( + `Request already granted lease ${outcome.leaseId}; release it instead`, + { leaseId: outcome.leaseId }, + ); + } + throw requestNotCancellable( + "Request is no longer cancellable -- device work is already in flight for it", + ); + }); + + app.get("/v1/leases/:id", agentAuth, (c) => { + const lease = requireOwnedLease(c.get("identity"), deps, c.req.param("id")); + const device = findDevice(deps, lease.deviceId); + if (device === undefined) throw unknownLease(lease.id); + const requestId = tracker.requestIdForLease(lease.id); + // The tracker's record is gone after a daemon restart; the mode default is then the + // interval that will actually be in force from the next default renew on. `expiresAt` + // stays the authoritative deadline either way -- never derive ttlMs from `grantedAt`, + // which does not move on renewal. + const ttlMs = tracker.effectiveTtlMs(lease.id) ?? modeDefaultTtlMs(lease, deps.config); + return c.json({ + lease: buildLeasePayload(device, lease, { + ...(requestId === undefined ? {} : { requestId }), + ttlMs, + }), + }); + }); + + app.post("/v1/leases/:id/renew", agentAuth, async (c) => { + const current = requireOwnedLease(c.get("identity"), deps, c.req.param("id")); + const id = current.id; + + const ttlMs = await parseRenewBody(c); + if (ttlMs !== undefined && (!Number.isFinite(ttlMs) || ttlMs <= 0)) { + throw badRequest("ttlMs must be a positive number"); + } + + const renewed = await deps.leases.renew(id, ttlMs); + tracker.recordLeaseTtl(id, ttlMs ?? modeDefaultTtlMs(current, deps.config)); + + return c.json({ + expiresAt: new Date(renewed.ttlDeadline).toISOString(), + leaseId: renewed.id, + notices: notices.drain(id), + }); + }); + + app.get("/v1/leases/:id/events", agentAuth, (c) => { + const lease = requireOwnedLease(c.get("identity"), deps, c.req.param("id")); + + return pipeSse(c, deps.clock, { + subscribe(send, end) { + return notices.subscribe(lease.id, (notice) => { + send({ data: notice, event: notice.event }); + if (notice.event === "lease_lost") end(); + }); + }, + }); + }); + + app.delete("/v1/leases/:id", agentAuth, async (c) => { + const lease = requireOwnedLease(c.get("identity"), deps, c.req.param("id")); + + await deps.leases.release(lease.id, "explicit"); + + const device = findDevice(deps, lease.deviceId); + return c.json( + { device: { id: lease.deviceId, state: device?.state ?? "reclaiming" }, released: true }, + 202, + ); + }); + + app.get("/v1/leases", operatorAuth, (c) => + c.json({ + leases: deps.registry.snapshot.leases.map((lease) => decorateLease(lease, deps.config)), + }), + ); + + app.get("/v1/devices", operatorAuth, (c) => + c.json({ + devices: deps.registry.snapshot.devices.map((device) => decorateDevice(device, deps.clock)), + }), + ); + + app.get("/v1/events", operatorAuth, (c) => { + const since = c.req.query("since"); + const sinceTs = since === undefined ? undefined : deps.clock.now() - parseDuration(since); + return c.json({ events: deps.eventBus.replay(sinceTs === undefined ? {} : { sinceTs }) }); + }); + + app.get("/v1/events/stream", operatorAuth, (c) => + pipeSse(c, deps.clock, { + subscribe(send) { + return deps.eventBus.subscribeAll((envelope) => { + send({ data: envelope, event: envelope.event }); + }); + }, + }), + ); + + // `tracker`/`notices` both subscribe to `deps.eventBus` for the app's lifetime -- exposed + // here rather than left to leak, so a caller composing this app into a longer-lived process + // (the daemon) can unsubscribe them on shutdown. Attached to the app object itself instead + // of changing this function's return shape to a `{app, dispose}` pair, which would ripple + // into every existing call site. + return Object.assign(app, { + dispose: () => { + tracker.dispose(); + notices.dispose(); + }, + }); +} + +function serializeRequest( + view: Pick, +): Record { + const base = { createdAt: view.createdAt, id: view.id, state: view.state.stage }; + switch (view.state.stage) { + case "queued": + return { ...base, queuePosition: view.state.queuePosition }; + case "provisioning": + case "booting": + case "reclaiming": + return { ...base, etaSeconds: view.state.etaSeconds }; + case "granted": + return { ...base, lease: view.state.lease }; + case "failed": + return { ...base, error: view.state.error }; + case "cancelled": + return base; + } +} + +/** The interval a default (body-less) renew of this lease applies -- its mode's configured TTL. */ +function modeDefaultTtlMs(lease: LeaseRecord, config: Config): number { + return lease.mode === "held" ? config.lease.heldTtlBackstopMs : config.lease.detachedTtlMs; +} + +/** Shared preamble of every `/v1/leases/:id` route: resolve the lease, then gate on ownership. */ +function requireOwnedLease( + identity: TokenIdentity, + deps: HttpGatewayDeps, + id: string, +): LeaseRecord { + const lease = findLease(deps, id); + if (lease === undefined) throw unknownLease(id); + requireOwnership(identity, lease.requesterId); + return lease; +} + +function findLease(deps: HttpGatewayDeps, id: string): LeaseRecord | undefined { + return deps.registry.snapshot.leases.find((lease) => lease.id === id); +} + +function findDevice(deps: HttpGatewayDeps, id: string): DeviceRecord | undefined { + return deps.registry.snapshot.devices.find((device) => device.id === id); +} + +async function parseRenewBody(c: { + req: { text(): Promise }; +}): Promise { + const raw = await c.req.text(); + if (raw.trim() === "") return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(raw) as unknown; + } catch { + throw badRequest("Invalid JSON body"); + } + if (typeof parsed !== "object" || parsed === null) throw badRequest("Body must be a JSON object"); + const ttlMs = (parsed as Record).ttlMs; + if (ttlMs === undefined) return undefined; + if (typeof ttlMs !== "number") throw badRequest("ttlMs must be a number"); + return ttlMs; +} + +/** Mirrors `DaemonServer#status` exactly -- see server.ts's `#status` -- so `--json` parity holds. */ +function buildStatus(deps: HttpGatewayDeps): unknown { + const snapshot = deps.registry.snapshot; + const running = deps.capacity.runningCapacity; + const warmDevices = snapshot.devices.filter((device) => device.state === "ready"); + const capacity = Object.fromEntries( + (["ios", "android"] as const).map((platform) => [ + platform, + { + limit: deps.capacity.deviceLimit(platform), + ...running[platform], + used: snapshot.devices.filter( + (device) => device.spec.platform === platform && device.state !== "deleted", + ).length, + warm: warmDevices.filter((device) => device.spec.platform === platform).length, + }, + ]), + ); + return { + ...snapshot, + capacity: { ...capacity, global: { ...running.global, warm: warmDevices.length } }, + devices: snapshot.devices.map((device) => decorateDevice(device, deps.clock)), + health: deps.daemonHealth(), + leases: snapshot.leases.map((lease) => decorateLease(lease, deps.config)), + queueDepth: deps.queue.queueDepth, + }; +} + +function decorateDevice( + device: DeviceRecord, + clock: Clock, +): DeviceRecord & { readonly transitionAgeMs?: number } { + const enteredAt = transitionEnteredAt(device); + if (enteredAt === undefined) return device; + return { ...device, transitionAgeMs: clock.now() - enteredAt }; +} + +function decorateLease( + lease: LeaseRecord, + config: Config, +): LeaseRecord & { readonly lastHeartbeatAt?: number } { + if (lease.mode !== "held") return lease; + return { ...lease, lastHeartbeatAt: lease.ttlDeadline - config.lease.heldTtlBackstopMs }; +} + +/** Local re-implementation of the CLI's `parseDuration` -- the HTTP layer never imports `src/cli`. */ +function parseDuration(value: string): number { + const match = /^(\d+)(ms|s|m|h)?$/.exec(value); + if (match === null) throw badRequest(`Invalid duration: ${value}`); + const amount = Number(match[1]); + const unit = match[2] ?? "ms"; + const multiplier = unit === "h" ? 3_600_000 : unit === "m" ? 60_000 : unit === "s" ? 1_000 : 1; + const milliseconds = amount * multiplier; + if (!Number.isSafeInteger(milliseconds)) throw badRequest(`Invalid duration: ${value}`); + return milliseconds; +} + +function formatZodIssues( + issues: readonly { readonly path: readonly PropertyKey[]; readonly message: string }[], +): string { + return issues + .map((issue) => + issue.path.length === 0 ? issue.message : `${issue.path.join(".")}: ${issue.message}`, + ) + .join("; "); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/src/http/auth.test.ts b/src/http/auth.test.ts new file mode 100644 index 0000000..26b2963 --- /dev/null +++ b/src/http/auth.test.ts @@ -0,0 +1,102 @@ +import { Hono } from "hono"; +import { describe, expect, it } from "vitest"; + +import { type AuthEnv, requireAuth, requireOwnership } from "./auth.js"; +import { errorResponse } from "./errors.js"; +import type { TokenIdentity } from "./token-store.js"; + +class FakeTokens { + readonly #identities = new Map(); + + register(secret: string, identity: TokenIdentity): void { + this.#identities.set(secret, identity); + } + + async verify(secret: string): Promise { + return this.#identities.get(secret); + } +} + +function appWithAuth(tokens: FakeTokens, minRole?: "operator") { + const app = new Hono(); + app.onError((error, c) => errorResponse(c, error)); + app.get("/resource", requireAuth(tokens, minRole), (c) => + c.json({ identity: c.get("identity") }), + ); + return app; +} + +describe("requireAuth", () => { + it("401s with no Authorization header", async () => { + const response = await appWithAuth(new FakeTokens()).request("/resource"); + expect(response.status).toBe(401); + expect(await response.json()).toEqual({ + error: { code: "UNAUTHENTICATED", message: expect.any(String) }, + }); + }); + + it("401s on a malformed Authorization header (no Bearer prefix)", async () => { + const response = await appWithAuth(new FakeTokens()).request("/resource", { + headers: { authorization: "slk_secret" }, + }); + expect(response.status).toBe(401); + }); + + it("401s on an unknown bearer token", async () => { + const response = await appWithAuth(new FakeTokens()).request("/resource", { + headers: { authorization: "Bearer slk_unknown" }, + }); + expect(response.status).toBe(401); + }); + + it("admits a known agent token when no minimum role is required", async () => { + const tokens = new FakeTokens(); + tokens.register("slk_agent", { requesterId: "tok_agent", role: "agent" }); + const response = await appWithAuth(tokens).request("/resource", { + headers: { authorization: "Bearer slk_agent" }, + }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + identity: { requesterId: "tok_agent", role: "agent" }, + }); + }); + + it("403s an agent token against an operator-only route", async () => { + const tokens = new FakeTokens(); + tokens.register("slk_agent", { requesterId: "tok_agent", role: "agent" }); + const response = await appWithAuth(tokens, "operator").request("/resource", { + headers: { authorization: "Bearer slk_agent" }, + }); + expect(response.status).toBe(403); + expect(await response.json()).toEqual({ + error: { code: "FORBIDDEN", message: expect.any(String) }, + }); + }); + + it("admits an operator token on an operator-only route", async () => { + const tokens = new FakeTokens(); + tokens.register("slk_op", { requesterId: "tok_op", role: "operator" }); + const response = await appWithAuth(tokens, "operator").request("/resource", { + headers: { authorization: "Bearer slk_op" }, + }); + expect(response.status).toBe(200); + }); +}); + +describe("requireOwnership", () => { + it("allows a requester to access its own resource", () => { + expect(() => requireOwnership({ requesterId: "tok_a", role: "agent" }, "tok_a")).not.toThrow(); + }); + + it("throws 403 when an agent reaches another requester's resource", () => { + expect(() => requireOwnership({ requesterId: "tok_a", role: "agent" }, "tok_b")).toThrowError( + expect.objectContaining({ code: "FORBIDDEN", status: 403 }), + ); + }); + + it("allows an operator to access any requester's resource", () => { + expect(() => + requireOwnership({ requesterId: "tok_op", role: "operator" }, "tok_b"), + ).not.toThrow(); + }); +}); diff --git a/src/http/auth.ts b/src/http/auth.ts new file mode 100644 index 0000000..80f5f39 --- /dev/null +++ b/src/http/auth.ts @@ -0,0 +1,55 @@ +import type { Context, MiddlewareHandler, Next } from "hono"; + +import { forbidden, unauthenticated } from "./errors.js"; +import type { TokenIdentity, TokenRole } from "./token-store.js"; + +export interface AuthEnv { + readonly Variables: { + identity: TokenIdentity; + }; +} + +export interface TokenVerifier { + verify(secret: string): Promise; +} + +const BEARER_PREFIX = "Bearer "; + +function extractSecret(header: string | undefined): string | undefined { + if (header === undefined || !header.startsWith(BEARER_PREFIX)) return undefined; + const secret = header.slice(BEARER_PREFIX.length).trim(); + return secret === "" ? undefined : secret; +} + +/** + * Verifies the bearer token and stores its `TokenIdentity` on the context for downstream + * handlers. `minRole: "operator"` rejects an `agent` token with 403 before the handler runs; + * per-resource ownership (an agent reaching another requester's request/lease) is not a role + * gate and is checked by the handler itself against the stored identity. + */ +export function requireAuth( + tokens: TokenVerifier, + minRole?: TokenRole, +): MiddlewareHandler { + return async (c: Context, next: Next) => { + const secret = extractSecret(c.req.header("authorization")); + if (secret === undefined) throw unauthenticated("Missing bearer token"); + + const identity = await tokens.verify(secret); + if (identity === undefined) throw unauthenticated("Unknown bearer token"); + + if (minRole === "operator" && identity.role !== "operator") { + throw forbidden("Operator role required"); + } + + c.set("identity", identity); + await next(); + }; +} + +/** Throws 403 unless the identity owns `requesterId` or holds the operator role. */ +export function requireOwnership(identity: TokenIdentity, requesterId: string): void { + if (identity.role === "operator") return; + if (identity.requesterId === requesterId) return; + throw forbidden("Not permitted to access another requester's resource"); +} diff --git a/src/http/errors.test.ts b/src/http/errors.test.ts new file mode 100644 index 0000000..1694b76 --- /dev/null +++ b/src/http/errors.test.ts @@ -0,0 +1,116 @@ +import { Hono } from "hono"; +import { describe, expect, it } from "vitest"; + +import { + NoCapacityError, + NoDriverError, + RequesterAlreadyLeasedError, + RuntimeMissingError, + UnknownLeaseError, + UnknownModelError, +} from "../core/index.js"; +import { + errorResponse, + HttpApiError, + mapError, + NO_CAPACITY_RETRY_AFTER_SECONDS, +} from "./errors.js"; + +describe("mapError", () => { + it("passes an HttpApiError's own status/code/extra through unchanged", () => { + const error = new HttpApiError(403, "FORBIDDEN", "nope", { requesterId: "req-1" }); + expect(mapError(error)).toEqual({ + code: "FORBIDDEN", + extra: { requesterId: "req-1" }, + message: "nope", + status: 403, + }); + }); + + it("maps RequesterAlreadyLeasedError to 409, naming the existing lease when there is one", () => { + const withLease = mapError(new RequesterAlreadyLeasedError("agent-1", "lse_1")); + expect(withLease.status).toBe(409); + expect(withLease.code).toBe("REQUESTER_ALREADY_LEASED"); + expect(withLease.extra).toEqual({ existingLeaseId: "lse_1" }); + + const withoutLease = mapError(new RequesterAlreadyLeasedError("agent-1")); + expect(withoutLease.extra).toBeUndefined(); + }); + + it("maps NoCapacityError to 503", () => { + expect(mapError(new NoCapacityError())).toMatchObject({ code: "NO_CAPACITY", status: 503 }); + }); + + it("maps UnknownModelError, RuntimeMissingError, NoDriverError to 422", () => { + expect(mapError(new UnknownModelError("ios", "iPhone 3G"))).toMatchObject({ + code: "UNKNOWN_MODEL", + status: 422, + }); + expect(mapError(new RuntimeMissingError("ios", "9.0"))).toMatchObject({ + code: "RUNTIME_MISSING", + status: 422, + }); + expect(mapError(new NoDriverError("android"))).toMatchObject({ + code: "NO_DRIVER", + status: 422, + }); + }); + + it("maps UnknownLeaseError to 404", () => { + expect(mapError(new UnknownLeaseError("lse_missing"))).toMatchObject({ + code: "UNKNOWN_LEASE", + status: 404, + }); + }); + + it("collapses an unrecognized error to 500 INTERNAL without leaking its message", () => { + const mapped = mapError(new Error("some internal implementation detail, e.g. a stack frame")); + expect(mapped).toEqual({ code: "INTERNAL", message: "Internal error", status: 500 }); + }); + + it("collapses a non-Error thrown value the same way", () => { + expect(mapError("boom")).toEqual({ code: "INTERNAL", message: "Internal error", status: 500 }); + }); +}); + +describe("errorResponse", () => { + function appWithError(error: unknown) { + const app = new Hono(); + app.get("/boom", (c) => errorResponse(c, error)); + return app; + } + + it("writes the standard {error:{code,message}} body", async () => { + const response = await appWithError(new UnknownLeaseError("lse_1")).request("/boom"); + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ + error: { code: "UNKNOWN_LEASE", message: "Unknown lease: lse_1" }, + }); + }); + + it("includes extra fields (e.g. existingLeaseId) alongside code/message", async () => { + const response = await appWithError( + new RequesterAlreadyLeasedError("agent-1", "lse_9"), + ).request("/boom"); + expect(await response.json()).toEqual({ + error: { + code: "REQUESTER_ALREADY_LEASED", + existingLeaseId: "lse_9", + message: expect.any(String), + }, + }); + }); + + it("sets Retry-After on a NO_CAPACITY response", async () => { + const response = await appWithError(new NoCapacityError()).request("/boom"); + expect(response.status).toBe(503); + expect(response.headers.get("Retry-After")).toBe(String(NO_CAPACITY_RETRY_AFTER_SECONDS)); + }); + + it("never includes a stack trace for an unrecognized error", async () => { + const response = await appWithError(new Error("leaked?")).request("/boom"); + const body = (await response.json()) as { error: { message: string } }; + expect(body.error.message).toBe("Internal error"); + expect(JSON.stringify(body)).not.toContain("leaked?"); + }); +}); diff --git a/src/http/errors.ts b/src/http/errors.ts new file mode 100644 index 0000000..275df80 --- /dev/null +++ b/src/http/errors.ts @@ -0,0 +1,134 @@ +import type { Context } from "hono"; + +import { + NoCapacityError, + NoDriverError, + QueueTimeoutError, + RequestCancelledError, + RequesterAlreadyLeasedError, + RuntimeMissingError, + UnknownLeaseError, + UnknownModelError, +} from "../core/index.js"; + +/** Every status this gateway ever answers with; keeps `mapError` exhaustive by construction. */ +export type HttpStatus = 400 | 401 | 403 | 404 | 409 | 422 | 500 | 503; + +/** + * Uniform `{status, code}` pair a route or middleware raises directly (auth, ownership, + * validation, not-found on gateway-owned resources like lease requests). Thrown core errors + * are mapped separately by `mapError` -- this class is for facts only the HTTP layer knows. + */ +export class HttpApiError extends Error { + constructor( + readonly status: HttpStatus, + readonly code: string, + message: string, + readonly extra?: Record, + ) { + super(message); + this.name = "HttpApiError"; + } +} + +export function unauthenticated(message: string): HttpApiError { + return new HttpApiError(401, "UNAUTHENTICATED", message); +} + +export function forbidden(message: string): HttpApiError { + return new HttpApiError(403, "FORBIDDEN", message); +} + +export function badRequest(message: string): HttpApiError { + return new HttpApiError(400, "BAD_REQUEST", message); +} + +export function unknownRequest(id: string): HttpApiError { + return new HttpApiError(404, "UNKNOWN_REQUEST", `Unknown lease request: ${id}`); +} + +export function unknownLease(id: string): HttpApiError { + return new HttpApiError(404, "UNKNOWN_LEASE", `Unknown lease: ${id}`); +} + +export function requestNotCancellable( + message: string, + extra?: Record, +): HttpApiError { + return new HttpApiError(409, "REQUEST_NOT_CANCELLABLE", message, extra); +} + +/** No core signal carries a better estimate; a fixed value is honest about that. */ +export const NO_CAPACITY_RETRY_AFTER_SECONDS = 5; + +export interface MappedError { + readonly status: HttpStatus; + readonly code: string; + readonly message: string; + readonly extra?: Record; +} + +/** + * Maps a thrown error to the response shape the issue's error table specifies. Never echoes + * a stack trace; an error this function doesn't recognize collapses to 500 `INTERNAL` with a + * generic message rather than leaking implementation detail into the response body. + */ +// fallow-ignore-next-line complexity -- one exhaustive table beats scattering this mapping across routes. +export function mapError(error: unknown): MappedError { + if (error instanceof HttpApiError) { + return { + status: error.status, + code: error.code, + message: error.message, + ...(error.extra === undefined ? {} : { extra: error.extra }), + }; + } + if (error instanceof RequesterAlreadyLeasedError) { + return { + status: 409, + code: "REQUESTER_ALREADY_LEASED", + message: error.message, + ...(error.existingLeaseId === undefined + ? {} + : { extra: { existingLeaseId: error.existingLeaseId } }), + }; + } + if (error instanceof NoCapacityError) { + return { status: 503, code: "NO_CAPACITY", message: error.message }; + } + if (error instanceof UnknownModelError) { + return { status: 422, code: "UNKNOWN_MODEL", message: error.message }; + } + if (error instanceof RuntimeMissingError) { + return { status: 422, code: "RUNTIME_MISSING", message: error.message }; + } + if (error instanceof NoDriverError) { + return { status: 422, code: "NO_DRIVER", message: error.message }; + } + if (error instanceof UnknownLeaseError) { + return { status: 404, code: "UNKNOWN_LEASE", message: error.message }; + } + // Neither of these is expected to reach a route handler as a live rejection -- the tracker + // consumes both internally and turns them into a terminal request state -- but map them + // rather than falling through to INTERNAL if that invariant is ever wrong. + if (error instanceof QueueTimeoutError) { + return { status: 500, code: "QUEUE_TIMEOUT", message: error.message }; + } + if (error instanceof RequestCancelledError) { + return { status: 500, code: "REQUEST_CANCELLED", message: error.message }; + } + return { status: 500, code: "INTERNAL", message: "Internal error" }; +} + +/** Writes `mapError`'s result as the standard `{"error":{...}}` body, plus `Retry-After` for NO_CAPACITY. */ +export function errorResponse(c: Context, error: unknown): Response { + const mapped = mapError(error); + const response = c.json( + { error: { code: mapped.code, message: mapped.message, ...mapped.extra } }, + mapped.status, + ); + if (mapped.code === "NO_CAPACITY") { + response.headers.set("Retry-After", String(NO_CAPACITY_RETRY_AFTER_SECONDS)); + } + return response; +} diff --git a/src/http/notices.ts b/src/http/notices.ts new file mode 100644 index 0000000..1ff1adc --- /dev/null +++ b/src/http/notices.ts @@ -0,0 +1,80 @@ +import type { EventBus } from "../bus/index.js"; + +export type LeaseNotice = + | { readonly event: "device_unhealthy" } + | { readonly event: "device_recovered"; readonly attempts: number } + | { readonly event: "lease_lost"; readonly reason: string }; + +/** + * Buffers per-lease health facts between renews and fans them out to any live SSE listener. + * A notice that arrives while a stream is connected is buffered *and* pushed live, so a renew + * and a stream watching the same lease may each deliver it (at-least-once across channels). + * The one asymmetry: `subscribe`'s initial flush drains the buffer, so notices from before + * the stream connected reach only the stream -- they were delivered, just not twice. + * `lease.released` / `lease.expired` clear the buffer for that lease afterwards: once the + * lease is gone, `LeaseCommands.renew` already answers `UNKNOWN_LEASE` before anything here + * would be read again, so there is nothing left worth retaining. + */ +export class LeaseNoticeBuffer { + readonly #buffered = new Map(); + readonly #listeners = new Map void>>(); + readonly #unsubscribers: Array<() => void>; + + constructor(eventBus: EventBus) { + this.#unsubscribers = [ + eventBus.subscribe("device.crash-detected", (envelope) => { + this.#push(envelope.payload.leaseId, { event: "device_unhealthy" }); + }), + eventBus.subscribe("device.recovered", (envelope) => { + this.#push(envelope.payload.leaseId, { + attempts: envelope.payload.attempts, + event: "device_recovered", + }); + }), + eventBus.subscribe("lease.released", (envelope) => { + this.#push(envelope.payload.leaseId, { + event: "lease_lost", + reason: envelope.payload.reason, + }); + this.#buffered.delete(envelope.payload.leaseId); + }), + eventBus.subscribe("lease.expired", (envelope) => { + this.#push(envelope.payload.leaseId, { event: "lease_lost", reason: "expired" }); + this.#buffered.delete(envelope.payload.leaseId); + }), + ]; + } + + /** Drains and clears the buffered notices for `leaseId`, for the renew response's `notices` field. */ + drain(leaseId: string): LeaseNotice[] { + const notices = this.#buffered.get(leaseId) ?? []; + this.#buffered.delete(leaseId); + return notices; + } + + /** + * Live feed for a lease's SSE stream: flushes whatever is already buffered first, so a + * client that connects after a notice fired doesn't miss it, then pushes future notices. + */ + subscribe(leaseId: string, listener: (notice: LeaseNotice) => void): () => void { + for (const notice of this.drain(leaseId)) listener(notice); + const listeners = this.#listeners.get(leaseId) ?? new Set(); + listeners.add(listener); + this.#listeners.set(leaseId, listeners); + return () => { + listeners.delete(listener); + if (listeners.size === 0) this.#listeners.delete(leaseId); + }; + } + + dispose(): void { + for (const unsubscribe of this.#unsubscribers) unsubscribe(); + } + + #push(leaseId: string, notice: LeaseNotice): void { + const buffered = this.#buffered.get(leaseId) ?? []; + buffered.push(notice); + this.#buffered.set(leaseId, buffered); + for (const listener of this.#listeners.get(leaseId) ?? []) listener(notice); + } +} diff --git a/src/http/server.ts b/src/http/server.ts new file mode 100644 index 0000000..f54d958 --- /dev/null +++ b/src/http/server.ts @@ -0,0 +1,99 @@ +import type { Server } from "node:http"; +import type { Socket } from "node:net"; + +import { serve } from "@hono/node-server"; + +import type { Logger } from "../ports/index.js"; + +export interface HttpGatewayOptions { + readonly host: string; + readonly port: number; + readonly logger: Logger; +} + +/** + * The one bit of `Hono` this file depends on -- deliberately structural (not `import + * type { Hono } from "hono"`) so this class doesn't have to match `createHttpApp`'s + * specific `Env` type parameter; `app.fetch` is exactly what `serve()` wants. + */ +export interface FetchApp { + // `any` (not `unknown`) for `env`/`executionCtx` deliberately: Hono's own `fetch` types + // them permissively per its `Env` type parameter, and this interface exists purely to + // let any `Hono<...>` instance satisfy it regardless of that parameter -- `unknown` + // here would make assignment fail on parameter contravariance instead. + readonly fetch: (request: Request, env?: any, executionCtx?: any) => Response | Promise; +} + +/** + * The impure serve adapter -- the only file under `src/http` allowed to import + * `@hono/node-server` (or any `node:http`); everything else stays a pure + * `Request -> Response` function (see `app.ts`'s own comment). Wraps `serve()`/ + * `server.close()` behind `start`/`stop`. + * + * Tracks every accepted socket so `stop()` can force them closed: an SSE stream + * (`GET /v1/lease-requests/:id/events`, `/v1/leases/:id/events`, `/v1/events/stream`) + * never ends on its own from the server side, and `server.close()`'s callback only + * fires once every open connection has ended -- without the destroy step below, a + * single client that never disconnected would hang `daemon stop` forever. + */ +export class HttpGateway { + readonly #app: FetchApp; + readonly #host: string; + readonly #port: number; + readonly #logger: Logger; + readonly #sockets = new Set(); + #server: Server | undefined; + + constructor(app: FetchApp, options: HttpGatewayOptions) { + this.#app = app; + this.#host = options.host; + this.#port = options.port; + this.#logger = options.logger; + } + + /** Resolves once listening, with the actual bound port (matches the configured one in v1, since port 0 is never used here). */ + start(): Promise<{ readonly port: number }> { + return new Promise((resolve, reject) => { + let settled = false; + const server = serve( + { fetch: this.#app.fetch, hostname: this.#host, port: this.#port }, + (info) => { + if (settled) return; + settled = true; + this.#logger.info("HTTP gateway listening", { host: this.#host, port: info.port }); + resolve({ port: info.port }); + }, + ) as Server; + // Only matters before "listening" fires -- e.g. EADDRINUSE. A post-listen socket + // error is a per-connection concern, not a `start()` failure. + server.once("error", (error: unknown) => { + if (settled) return; + settled = true; + reject(error instanceof Error ? error : new Error(String(error))); + }); + server.on("connection", (socket: Socket) => { + this.#sockets.add(socket); + socket.once("close", () => this.#sockets.delete(socket)); + }); + this.#server = server; + }); + } + + /** Closes the listener and destroys any connection still open, in-flight SSE streams included. */ + async stop(): Promise { + const server = this.#server; + if (server === undefined) return; + this.#server = undefined; + await new Promise((resolve, reject) => { + server.close((error) => (error === undefined ? resolve() : reject(error))); + // Destroyed right after asking for the graceful close, not before: an ordinary + // request already in flight gets to finish (its socket isn't in `#sockets` as + // "still open" from the server's perspective any differently than an SSE one, + // but it settles fast enough that the destroy below rarely if ever races it, + // and a request abandoned mid-response is no worse than the daemon stopping + // under it any other way). An SSE stream has no natural end to wait for. + for (const socket of this.#sockets) socket.destroy(); + }); + this.#logger.info("HTTP gateway stopped", { host: this.#host, port: this.#port }); + } +} diff --git a/src/http/sse.ts b/src/http/sse.ts new file mode 100644 index 0000000..342c751 --- /dev/null +++ b/src/http/sse.ts @@ -0,0 +1,77 @@ +import type { Context } from "hono"; +import { streamSSE } from "hono/streaming"; + +import type { Clock } from "../ports/index.js"; + +/** ~15s per the issue spec, so idle tunnels don't close the stream. */ +const KEEPALIVE_MS = 15_000; + +export interface SseEvent { + readonly event: string; + readonly data: unknown; + readonly id?: string; +} + +/** + * One live feed. `subscribe` is called once per connection; it must invoke `send` for every + * event to emit (in order) and `end` exactly once the stream should close after flushing + * whatever `send` calls are already queued. The returned function unsubscribes. + */ +export interface SseSource { + subscribe(send: (event: SseEvent) => void, end: () => void): () => void; +} + +/** + * Bridges an `SseSource` to a hono SSE response. Keepalive comments and the source's own + * lifetime both go through the injected `Clock` -- no `stream.sleep` (real `setTimeout` under + * the hood) and no bare timers. Client abort unsubscribes the source and cancels the timer. + */ +export function pipeSse(c: Context, clock: Clock, source: SseSource): Response { + return streamSSE(c, async (stream) => { + let resolveDone!: () => void; + const done = new Promise((resolve) => { + resolveDone = resolve; + }); + // `resolveDone` is idempotent on its own (a second call is a no-op), so `finish` doesn't + // need an `ended` guard -- and deliberately has none: a terminal event's `send` and its + // `end()` call both happen synchronously in the same source callback (e.g. `lease_lost`), + // so a guard flipped by `end()` before the queued write actually runs would silently drop + // that last write. `stream.closed` (set by the runtime only once this callback returns and + // `run()`'s `finally` closes it -- see hono's `streaming/sse.ts`) is the only signal writes + // below check, and it can't go true until after `writeChain` has already been awaited. + const finish = () => resolveDone(); + + // Serializes writes: `send` can be invoked synchronously (an immediate current-state + // event on subscribe) or later from an event-bus callback, and writes must land in order. + let writeChain: Promise = Promise.resolve(); + const send = (event: SseEvent) => { + if (stream.closed) return; + writeChain = writeChain.then(async () => { + if (stream.closed) return; + await stream.writeSSE({ + data: JSON.stringify(event.data), + event: event.event, + ...(event.id === undefined ? {} : { id: event.id }), + }); + }); + }; + + let keepaliveTimer = clock.setTimer(KEEPALIVE_MS, tickKeepalive); + function tickKeepalive(): void { + if (stream.closed) return; + writeChain = writeChain.then(async () => { + if (stream.closed) return; + await stream.write(": keepalive\n\n"); + }); + keepaliveTimer = clock.setTimer(KEEPALIVE_MS, tickKeepalive); + } + + stream.onAbort(() => finish()); + + const unsubscribe = source.subscribe(send, finish); + await done; + clock.cancel(keepaliveTimer); + await writeChain; + unsubscribe(); + }); +} diff --git a/src/http/test-fakes.ts b/src/http/test-fakes.ts new file mode 100644 index 0000000..8c7504c --- /dev/null +++ b/src/http/test-fakes.ts @@ -0,0 +1,228 @@ +import type { Config, DeviceRecord, DeviceRequest, LeaseRecord } from "../core/index.js"; +import type { + CapacityReader, + CatalogReader, + LeaseCommands, + QueueControl, +} from "../core/lease-ports.js"; +import type { PlatformCatalog } from "../core/driver-catalog.js"; +import type { RunningCapacity } from "../core/capacity/index.js"; +import type { LeaseGrant, LeaseRequestOptions } from "../core/wait-queue.js"; +import type { IdGenerator } from "../ports/index.js"; +import type { TokenIdentity } from "./token-store.js"; + +const gibibyte = 1024 * 1024 * 1024; + +/** Shared config fixture; every field `Config` currently declares (see `src/core/config.ts`). */ +export function testConfig(overrides: Partial = {}): Config { + return { + capacity: { + strategy: "resource", + config: { + limits: { + android: { maxDevices: 4, maxRunning: 4 }, + ios: { maxDevices: 4, maxRunning: 4 }, + maxRunning: 8, + }, + ramBudget: { androidBytesPerDevice: 4 * gibibyte, iosBytesPerDevice: gibibyte }, + }, + }, + diskPressure: { freeBytesThreshold: 10 * gibibyte }, + eventBuffer: { capacity: 100 }, + health: { + enabled: true, + maxConcurrentRecoveries: 1, + maxRecoveryAttempts: 3, + probeIntervalMs: 30_000, + recoveryBackoffMs: 5_000, + stableObservations: 2, + }, + http: { enabled: true, host: "127.0.0.1", port: 4700 }, + idle: { deleteAfterMs: 60_000, shutdownAfterMs: 10_000 }, + lease: { + detachedTtlMs: 900_000, + heartbeatIntervalMs: 5_000, + heldTtlBackstopMs: 3_600_000, + ...overrides, + }, + log: { level: "info", rotateBytes: 5 * 1024 * 1024 }, + stalledTransition: { minimumThresholdMs: 60_000, thresholdMultiplier: 3 }, + warmPool: { + quarantine: { + maxRetries: 3, + maxRetryBackoffMs: 300_000, + retryBackoffMs: 30_000, + retryBackoffMultiplier: 2, + }, + }, + }; +} + +export function sequenceIdGenerator(prefix = "id"): IdGenerator { + let next = 0; + return { + generate: () => { + next += 1; + return `${prefix}-${next}`; + }, + }; +} + +export function makeDevice(overrides: Partial = {}): DeviceRecord { + return { + createdAt: 0, + driverData: undefined, + driverDeviceId: "ABCD-1234", + id: "dev_1", + spec: { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + state: "leased", + ...overrides, + }; +} + +export function makeLease(overrides: Partial = {}): LeaseRecord { + return { + deviceId: "dev_1", + grantedAt: 1_000, + id: "lse_1", + mode: "detached", + requesterId: "tok_agent", + ttlDeadline: 1_000 + 900_000, + ...overrides, + }; +} + +export function makeGrant( + overrides: { + readonly device?: Partial; + readonly lease?: Partial; + } = {}, +): LeaseGrant { + return { + device: makeDevice(overrides.device), + lease: makeLease(overrides.lease), + timing: { + estimatedBootMs: 0, + estimatedProvisionMs: 0, + estimatedReadyMs: 0, + estimatedReclaimMs: 0, + }, + }; +} + +interface PendingRequest { + readonly request: DeviceRequest; + readonly options: LeaseRequestOptions; + readonly resolve: (grant: LeaseGrant) => void; + readonly reject: (error: unknown) => void; +} + +/** Scriptable `LeaseCommands`: every `request()` call parks until the test resolves/rejects it. */ +export class FakeLeaseCommands implements LeaseCommands { + readonly calls: PendingRequest[] = []; + readonly releaseCalls: Array<{ readonly leaseId: string; readonly reason: string }> = []; + readonly renewCalls: Array<{ readonly leaseId: string; readonly ttlMs?: number }> = []; + renewImpl: (leaseId: string, ttlMs?: number) => Promise = () => { + throw new Error("renew not scripted"); + }; + releaseImpl: (leaseId: string, reason: string) => Promise = async () => {}; + + request(request: DeviceRequest, options: LeaseRequestOptions): Promise { + return new Promise((resolve, reject) => { + this.calls.push({ options, reject, request, resolve }); + }); + } + + async release(leaseId: string, reason: "closed" | "explicit" | "killed"): Promise { + this.releaseCalls.push({ leaseId, reason }); + await this.releaseImpl(leaseId, reason); + } + + async releaseAll(): Promise { + return []; + } + + async renew(leaseId: string, ttlMs?: number): Promise { + this.renewCalls.push({ leaseId, ...(ttlMs === undefined ? {} : { ttlMs }) }); + return this.renewImpl(leaseId, ttlMs); + } + + async heartbeat(): Promise { + throw new Error("heartbeat not scripted"); + } +} + +/** + * `FakeLeaseCommands.request` is called synchronously from within `LeaseRequestTracker.submit`, + * but only once the HTTP layer's own async work (body parsing, auth, validation) reaches the + * handler -- so a caller driving a request through `app.request()` must pump the microtask + * queue before `leases.calls[index]` exists. Awaiting the whole response first would deadlock: + * `submit`'s returned promise doesn't settle until a progress/grant/reject callback fires. + */ +export async function waitForCall(leases: FakeLeaseCommands, index = 0): Promise { + for (let attempt = 0; attempt < 100 && leases.calls.length <= index; attempt += 1) { + await Promise.resolve(); + } + if (leases.calls.length <= index) throw new Error("LeaseCommands.request was never called"); +} + +export class FakeQueueControl implements QueueControl { + queueDepth = 0; + cancelOutcome: "cancelled" | "not-found" | "not-cancellable" = "not-found"; + readonly cancelCalls: string[] = []; + + async detachQueuedProgress(): Promise {} + + async cancelPending(requesterId: string): Promise<"cancelled" | "not-found" | "not-cancellable"> { + this.cancelCalls.push(requesterId); + return this.cancelOutcome; + } +} + +export class FakeCapacityReader implements CapacityReader { + runningCapacity: RunningCapacity = { + android: { maxRunning: 4, overLimit: false, reserved: 0, running: 0 }, + global: { maxRunning: 8, overLimit: false, reserved: 0, running: 0 }, + ios: { maxRunning: 4, overLimit: false, reserved: 0, running: 0 }, + }; + + deviceLimit(): number { + return 4; + } +} + +export class FakeCatalogReader implements CatalogReader { + platforms: PlatformCatalog[] = [ + { defaultRuntime: "26.5", models: ["iPhone 17 Pro"], platform: "ios", runtimes: ["26.5"] }, + ]; + + async listCatalog(platform?: "ios" | "android"): Promise { + return this.platforms.filter((entry) => platform === undefined || entry.platform === platform); + } +} + +export class FakeRegistry { + devices: DeviceRecord[] = []; + leases: LeaseRecord[] = []; + + // fallow-ignore-next-line unused-class-member -- reached structurally through HttpGatewayDeps.registry. + get snapshot(): { + readonly devices: readonly DeviceRecord[]; + readonly leases: readonly LeaseRecord[]; + } { + return { devices: this.devices, leases: this.leases }; + } +} + +export class FakeTokenVerifier { + readonly #identities = new Map(); + + register(secret: string, identity: TokenIdentity): void { + this.#identities.set(secret, identity); + } + + // fallow-ignore-next-line unused-class-member -- reached structurally through HttpGatewayDeps.tokens. + async verify(secret: string): Promise { + return this.#identities.get(secret); + } +} diff --git a/src/http/token-store.test.ts b/src/http/token-store.test.ts new file mode 100644 index 0000000..b7572fe --- /dev/null +++ b/src/http/token-store.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, it } from "vitest"; + +import { FakeClock, MemoryFilesystem } from "../ports/index.js"; +import type { TokenSecrets } from "../ports/token-secrets.js"; +import { TokenStore, TokenStoreError } from "./token-store.js"; + +const tokensPath = "/home/agent/.simlock/tokens.json"; + +class FakeTokenSecrets implements TokenSecrets { + #nextSecret = 0; + + generateSecret(): string { + this.#nextSecret += 1; + return `slk_fake-secret-${this.#nextSecret}`; + } + + hash(secret: string): string { + return `hash-of-${[...secret].reverse().join("")}`; + } +} + +function tokenStore( + overrides: { + filesystem?: MemoryFilesystem; + clock?: FakeClock; + secrets?: TokenSecrets; + ids?: readonly string[]; + } = {}, +): { store: TokenStore; filesystem: MemoryFilesystem } { + const filesystem = overrides.filesystem ?? new MemoryFilesystem(); + const ids = [...(overrides.ids ?? ["id-1", "id-2", "id-3"])]; + return { + filesystem, + store: new TokenStore({ + clock: overrides.clock ?? new FakeClock(1_000), + filesystem, + idGenerator: { generate: () => ids.shift() ?? "id-overflow" }, + path: tokensPath, + secrets: overrides.secrets ?? new FakeTokenSecrets(), + }), + }; +} + +describe("TokenStore", () => { + it("lists no tokens when the file does not exist", async () => { + const { store } = tokenStore(); + + await expect(store.list()).resolves.toEqual([]); + }); + + it("creates a token, returning the secret once and persisting only its hash", async () => { + const { store, filesystem } = tokenStore(); + + const { record, secret } = await store.create("agent", "ci-runner"); + + expect(secret).toBe("slk_fake-secret-1"); + expect(record).toEqual({ + id: "tok_id-1", + hash: "hash-of-1-terces-ekaf_kls", + role: "agent", + label: "ci-runner", + createdAt: 1_000, + }); + + const persisted = JSON.parse(await filesystem.readFile(tokensPath)) as unknown[]; + expect(persisted).toEqual([record]); + expect(JSON.stringify(persisted)).not.toContain("fake-secret"); + }); + + it("creates a token without a label when none is given", async () => { + const { store } = tokenStore(); + + const { record } = await store.create("operator"); + + expect(record.label).toBeUndefined(); + expect("label" in record).toBe(false); + }); + + it("lists previously created tokens", async () => { + const { store } = tokenStore(); + await store.create("agent", "one"); + await store.create("operator", "two"); + + const records = await store.list(); + + expect(records.map((record) => record.label)).toEqual(["one", "two"]); + }); + + it("verifies a token by hashing the presented secret against stored hashes", async () => { + const { store } = tokenStore(); + const { record, secret } = await store.create("operator", "root"); + + await expect(store.verify(secret)).resolves.toEqual({ + requesterId: record.id, + role: "operator", + }); + await expect(store.verify("slk_not-a-real-secret")).resolves.toBeUndefined(); + }); + + it("re-reads the file on every verify, seeing tokens created by another store instance", async () => { + const filesystem = new MemoryFilesystem(); + const secrets = new FakeTokenSecrets(); + const writer = tokenStore({ filesystem, secrets, ids: ["writer-id"] }).store; + const reader = tokenStore({ filesystem, secrets, ids: ["unused"] }).store; + + await expect(reader.verify("slk_fake-secret-1")).resolves.toBeUndefined(); + const { secret } = await writer.create("agent"); + + await expect(reader.verify(secret)).resolves.toEqual({ + requesterId: "tok_writer-id", + role: "agent", + }); + }); + + it("revokes an existing token and reports success", async () => { + const { store } = tokenStore(); + const { record } = await store.create("agent"); + + await expect(store.revoke(record.id)).resolves.toBe(true); + await expect(store.list()).resolves.toEqual([]); + }); + + it("returns false revoking an unknown token id", async () => { + const { store } = tokenStore(); + + await expect(store.revoke("tok_does-not-exist")).resolves.toBe(false); + }); + + it("tolerates a missing tokens file as an empty store", async () => { + const { store } = tokenStore(); + + await expect(store.verify("slk_anything")).resolves.toBeUndefined(); + await expect(store.revoke("tok_anything")).resolves.toBe(false); + }); + + it("fails loudly instead of silently resetting on corrupt JSON", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(tokensPath, "{ not valid json"); + const { store } = tokenStore({ filesystem }); + + await expect(store.list()).rejects.toThrow(TokenStoreError); + await expect(filesystem.readFile(tokensPath)).resolves.toBe("{ not valid json"); + }); + + it("fails loudly when the file holds valid JSON that is not a token array", async () => { + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/home/agent/.simlock"); + await filesystem.writeFileAtomic(tokensPath, JSON.stringify({ not: "an array" })); + const { store } = tokenStore({ filesystem }); + + await expect(store.list()).rejects.toThrow(TokenStoreError); + }); +}); diff --git a/src/http/token-store.ts b/src/http/token-store.ts new file mode 100644 index 0000000..227f288 --- /dev/null +++ b/src/http/token-store.ts @@ -0,0 +1,142 @@ +import { dirname } from "node:path"; + +import type { Clock, Filesystem, IdGenerator, TokenSecrets } from "../ports/index.js"; + +export type TokenRole = "agent" | "operator"; + +export interface TokenRecord { + readonly id: string; + readonly hash: string; + readonly role: TokenRole; + readonly label?: string; + readonly createdAt: number; +} + +export interface TokenIdentity { + readonly requesterId: string; + readonly role: TokenRole; +} + +export interface TokenStoreOptions { + readonly filesystem: Filesystem; + readonly clock: Clock; + readonly idGenerator: IdGenerator; + readonly secrets: TokenSecrets; + readonly path: string; +} + +export class TokenStoreError extends Error { + constructor(message: string) { + super(message); + this.name = "TokenStoreError"; + } +} + +/** + * Bearer-token store backed by a JSON file (`tokens.json` under the daemon + * data directory). Holds only SHA-256 hashes of secrets, never plaintext -- + * `create` is the one call that ever sees a secret, and it hands it back to + * the caller without persisting it. + */ +export class TokenStore { + readonly #filesystem: Filesystem; + readonly #clock: Clock; + readonly #idGenerator: IdGenerator; + readonly #secrets: TokenSecrets; + readonly #path: string; + + constructor(options: TokenStoreOptions) { + this.#filesystem = options.filesystem; + this.#clock = options.clock; + this.#idGenerator = options.idGenerator; + this.#secrets = options.secrets; + this.#path = options.path; + } + + async create(role: TokenRole, label?: string): Promise<{ record: TokenRecord; secret: string }> { + const secret = this.#secrets.generateSecret(); + const record: TokenRecord = { + id: `tok_${this.#idGenerator.generate()}`, + hash: this.#secrets.hash(secret), + role, + ...(label === undefined ? {} : { label }), + createdAt: this.#clock.now(), + }; + + const records = await this.#readAll(); + records.push(record); + await this.#writeAll(records); + + return { record, secret }; + } + + async list(): Promise { + return this.#readAll(); + } + + async revoke(id: string): Promise { + const records = await this.#readAll(); + const index = records.findIndex((record) => record.id === id); + if (index === -1) return false; + + records.splice(index, 1); + await this.#writeAll(records); + return true; + } + + /** + * Re-reads tokens.json on every call instead of caching in memory, so a + * `simlock token create`/`revoke` run from another process takes effect + * immediately for a long-running process (the daemon) verifying bearer + * tokens -- there is no in-process invalidation path otherwise. + */ + // fallow-ignore-next-line unused-class-member -- reached structurally through HttpGatewayDeps.tokens (see daemon/main.ts). + async verify(secret: string): Promise { + const hash = this.#secrets.hash(secret); + const record = (await this.#readAll()).find((candidate) => candidate.hash === hash); + if (record === undefined) return undefined; + + return { requesterId: record.id, role: record.role }; + } + + async #readAll(): Promise { + if (!(await this.#filesystem.exists(this.#path))) return []; + + const contents = await this.#filesystem.readFile(this.#path); + let parsed: unknown; + try { + parsed = JSON.parse(contents) as unknown; + } catch (error: unknown) { + throw new TokenStoreError( + `Invalid JSON in token store: ${this.#path} (${errorMessage(error)})`, + ); + } + + if (!Array.isArray(parsed) || !parsed.every(isTokenRecord)) { + throw new TokenStoreError(`Invalid token store: ${this.#path}`); + } + + return parsed; + } + + async #writeAll(records: readonly TokenRecord[]): Promise { + await this.#filesystem.mkdirp(dirname(this.#path)); + await this.#filesystem.writeFileAtomic(this.#path, `${JSON.stringify(records, null, 2)}\n`); + } +} + +function isTokenRecord(value: unknown): value is TokenRecord { + if (typeof value !== "object" || value === null) return false; + const record = value as Record; + return ( + typeof record.id === "string" && + typeof record.hash === "string" && + (record.role === "agent" || record.role === "operator") && + (record.label === undefined || typeof record.label === "string") && + typeof record.createdAt === "number" + ); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/src/http/tracker.test.ts b/src/http/tracker.test.ts new file mode 100644 index 0000000..04c625a --- /dev/null +++ b/src/http/tracker.test.ts @@ -0,0 +1,329 @@ +import { describe, expect, it } from "vitest"; + +import { EventBus } from "../bus/index.js"; +import { RequesterAlreadyLeasedError } from "../core/index.js"; +import { FakeClock } from "../ports/index.js"; +import { + FakeLeaseCommands, + FakeQueueControl, + makeGrant, + sequenceIdGenerator, +} from "./test-fakes.js"; +import { isTerminalStage, LeaseRequestTracker, type TrackedRequestView } from "./tracker.js"; + +function buildTracker(overrides: { readonly defaultTtlMs?: number } = {}) { + const clock = new FakeClock(1_000); + const eventBus = new EventBus(clock); + const leases = new FakeLeaseCommands(); + const queue = new FakeQueueControl(); + const tracker = new LeaseRequestTracker({ + clock, + defaultTtlMs: overrides.defaultTtlMs ?? 900_000, + eventBus, + idGenerator: sequenceIdGenerator("req"), + leases, + queue, + }); + return { clock, eventBus, leases, queue, tracker }; +} + +const identity = { requesterId: "tok_agent" }; +const body = { device: "iPhone 17 Pro", platform: "ios" as const }; + +/** + * `submit`'s returned promise never settles until `LeaseCommands.request`'s first `onProgress` + * call (or its own grant/rejection) -- see `tracker.ts`'s class doc. `FakeLeaseCommands.request` + * runs synchronously (its executor pushes into `calls` before `submit` returns), so scripting + * one `queued` progress event right after calling `submit` -- never awaiting it bare -- is what + * every test below needs to avoid deadlocking on its own promise. + */ +async function createTracked( + tracker: LeaseRequestTracker, + leases: FakeLeaseCommands, + requestBody: typeof body & { readonly ttlMs?: number } = body, +): Promise<{ readonly view: TrackedRequestView; readonly callIndex: number }> { + const callIndex = leases.calls.length; + const outcomePromise = tracker.submit(identity, requestBody); + leases.calls[callIndex]?.options.onProgress?.({ queuePosition: 1, stage: "queued" }); + const outcome = await outcomePromise; + if (outcome.kind !== "created") + throw new Error(`expected created, got rejected: ${String(outcome.error)}`); + return { callIndex, view: outcome.view }; +} + +describe("LeaseRequestTracker.submit", () => { + it("stays pending on the outer promise until the first progress callback, then answers 'created'", async () => { + const { leases, tracker } = buildTracker(); + const outcomePromise = tracker.submit(identity, body); + + let resolved = false; + void outcomePromise.then(() => { + resolved = true; + }); + await Promise.resolve(); + await Promise.resolve(); + expect(resolved).toBe(false); + + leases.calls[0]?.options.onProgress?.({ queuePosition: 3, stage: "queued" }); + const outcome = await outcomePromise; + expect(outcome.kind).toBe("created"); + if (outcome.kind === "created") { + expect(outcome.view.state).toEqual({ queuePosition: 3, stage: "queued" }); + } + }); + + it("answers 'rejected' synchronously when the grant fails before any progress callback", async () => { + const { leases, tracker } = buildTracker(); + const outcomePromise = tracker.submit(identity, body); + leases.calls[0]?.reject(new RequesterAlreadyLeasedError("tok_agent", "lse_9")); + + const outcome = await outcomePromise; + expect(outcome.kind).toBe("rejected"); + if (outcome.kind === "rejected") { + expect(outcome.error).toBeInstanceOf(RequesterAlreadyLeasedError); + } + }); + + it("drops a fast-rejected request from tracking -- it never becomes a gettable resource", async () => { + const { leases, tracker } = buildTracker(); + const outcomePromise = tracker.submit(identity, body); + leases.calls[0]?.reject(new Error("boom")); + await outcomePromise; + + // No id was ever handed to any caller for this failed submission; nothing to assert a `get` + // against, but the tracker must not have grown unboundedly -- covered indirectly by the + // idempotency-replay test below relying on exactly this cleanup. + }); + + it("answers 'created' immediately for an instant grant that never calls onProgress", async () => { + const { leases, tracker } = buildTracker(); + const outcomePromise = tracker.submit(identity, body); + leases.calls[0]?.resolve(makeGrant()); + + const outcome = await outcomePromise; + expect(outcome.kind).toBe("created"); + if (outcome.kind === "created") expect(outcome.view.state.stage).toBe("granted"); + }); + + it("progresses through queued -> booting -> granted, observable via get()", async () => { + const { leases, tracker } = buildTracker(); + const { view, callIndex } = await createTracked(tracker, leases); + const id = view.id; + expect(tracker.get(id)?.state).toEqual({ queuePosition: 1, stage: "queued" }); + + leases.calls[callIndex]?.options.onProgress?.({ etaMs: 60_000, stage: "booting" }); + expect(tracker.get(id)?.state).toEqual({ etaSeconds: 60, stage: "booting" }); + + leases.calls[callIndex]?.resolve(makeGrant({ lease: { id: "lse_42" } })); + await Promise.resolve(); + await Promise.resolve(); + const view2 = tracker.get(id); + expect(view2?.state.stage).toBe("granted"); + if (view2?.state.stage === "granted") expect(view2.state.lease.id).toBe("lse_42"); + }); + + it("renews a freshly granted lease when the body specified a custom ttlMs", async () => { + const { leases, tracker } = buildTracker({ defaultTtlMs: 900_000 }); + const callIndex = leases.calls.length; + const outcomePromise = tracker.submit(identity, { ...body, ttlMs: 60_000 }); + leases.renewImpl = async (leaseId, ttlMs) => ({ + deviceId: "dev_1", + grantedAt: 1_000, + id: leaseId, + mode: "detached", + requesterId: "tok_agent", + ttlDeadline: 1_000 + (ttlMs ?? 0), + }); + leases.calls[callIndex]?.resolve(makeGrant()); + const outcome = await outcomePromise; + if (outcome.kind !== "created") throw new Error("expected created"); + + expect(leases.renewCalls).toEqual([{ leaseId: "lse_1", ttlMs: 60_000 }]); + const view = tracker.get(outcome.view.id); + if (view?.state.stage === "granted") { + expect(view.state.lease.ttlMs).toBe(60_000); + expect(view.state.lease.expiresAt).toBe(new Date(1_000 + 60_000).toISOString()); + } else { + throw new Error("expected granted"); + } + }); + + it("replays an Idempotency-Key for the same requester instead of double-submitting", async () => { + const { leases, tracker } = buildTracker(); + const { view: first } = await createTrackedWithKey(tracker, leases, "key-1"); + + const replay = await tracker.submit(identity, body, "key-1"); + expect(replay.kind).toBe("created"); + if (replay.kind === "created") expect(replay.view.id).toBe(first.id); + expect(leases.calls).toHaveLength(1); + }); + + it("does not replay an Idempotency-Key across different requesters", async () => { + const { leases, tracker } = buildTracker(); + await createTrackedWithKey(tracker, leases, "key-1"); + const callIndex = leases.calls.length; + const outcomePromise = tracker.submit({ requesterId: "tok_other" }, body, "key-1"); + leases.calls[callIndex]?.options.onProgress?.({ queuePosition: 1, stage: "queued" }); + const outcome = await outcomePromise; + expect(outcome.kind).toBe("created"); + expect(leases.calls).toHaveLength(2); + }); +}); + +describe("LeaseRequestTracker.cancel", () => { + it("returns not-found for an unknown id", async () => { + const { tracker } = buildTracker(); + expect(await tracker.cancel("req-missing")).toEqual({ kind: "not-found" }); + }); + + it("cancels a still-queued request and settles it as 'cancelled'", async () => { + const { leases, queue, tracker } = buildTracker(); + const { view } = await createTracked(tracker, leases); + + queue.cancelOutcome = "cancelled"; + const result = await tracker.cancel(view.id); + expect(result).toEqual({ kind: "cancelled" }); + expect(tracker.get(view.id)?.state).toEqual({ stage: "cancelled" }); + }); + + it("reports not-cancellable, naming the lease, once the request is already granted", async () => { + const { leases, tracker } = buildTracker(); + const callIndex = leases.calls.length; + const outcomePromise = tracker.submit(identity, body); + leases.calls[callIndex]?.resolve(makeGrant({ lease: { id: "lse_granted" } })); + const outcome = await outcomePromise; + if (outcome.kind !== "created") throw new Error("expected created"); + + expect(await tracker.cancel(outcome.view.id)).toEqual({ + kind: "not-cancellable", + leaseId: "lse_granted", + }); + }); + + it("reports plain not-cancellable when the queue says device work is already in flight", async () => { + const { leases, queue, tracker } = buildTracker(); + const { view } = await createTracked(tracker, leases); + + queue.cancelOutcome = "not-cancellable"; + expect(await tracker.cancel(view.id)).toEqual({ kind: "not-cancellable" }); + }); +}); + +describe("LeaseRequestTracker.waitForChange", () => { + it("resolves undefined for an unknown id", async () => { + const { tracker } = buildTracker(); + expect(await tracker.waitForChange("req-missing", 30)).toBeUndefined(); + }); + + it("resolves immediately if the request is already terminal", async () => { + const { leases, tracker } = buildTracker(); + const callIndex = leases.calls.length; + const outcomePromise = tracker.submit(identity, body); + leases.calls[callIndex]?.resolve(makeGrant()); + const outcome = await outcomePromise; + if (outcome.kind !== "created") throw new Error("expected created"); + + const view = await tracker.waitForChange(outcome.view.id, 30); + expect(view?.state.stage).toBe("granted"); + }); + + it("resolves early on the next state change", async () => { + const { leases, tracker } = buildTracker(); + const { view, callIndex } = await createTracked(tracker, leases); + + const waitPromise = tracker.waitForChange(view.id, 30); + leases.calls[callIndex]?.options.onProgress?.({ etaMs: 5_000, stage: "provisioning" }); + const changed = await waitPromise; + expect(changed?.state).toEqual({ etaSeconds: 5, stage: "provisioning" }); + }); + + it("resolves with the unchanged state once the wait timer elapses", async () => { + const { clock, leases, tracker } = buildTracker(); + const { view } = await createTracked(tracker, leases); + + const waitPromise = tracker.waitForChange(view.id, 5); + clock.advance(5_000); + const changed = await waitPromise; + expect(changed?.state).toEqual({ queuePosition: 1, stage: "queued" }); + }); +}); + +describe("isTerminalStage", () => { + it("is true only for granted/failed/cancelled", () => { + expect(isTerminalStage({ queuePosition: 1, stage: "queued" })).toBe(false); + expect(isTerminalStage({ stage: "cancelled" })).toBe(true); + expect(isTerminalStage({ error: { code: "X", message: "x" }, stage: "failed" })).toBe(true); + }); +}); + +async function createTrackedWithKey( + tracker: LeaseRequestTracker, + leases: FakeLeaseCommands, + key: string, +): Promise<{ readonly view: TrackedRequestView }> { + const callIndex = leases.calls.length; + const outcomePromise = tracker.submit(identity, body, key); + leases.calls[callIndex]?.options.onProgress?.({ queuePosition: 1, stage: "queued" }); + const outcome = await outcomePromise; + if (outcome.kind !== "created") throw new Error("expected created"); + return { view: outcome.view }; +} + +describe("LeaseRequestTracker.submit with allowDownload", () => { + it("answers 'created' immediately, before any progress callback", async () => { + const { leases, tracker } = buildTracker(); + const outcome = await tracker.submit(identity, { ...body, allowDownload: true }); + expect(outcome.kind).toBe("created"); + expect(leases.calls[0]?.options.allowDownload).toBe(true); + }); + + it("keeps the request visible when the grant later fails -- terminal failed, not a rejected POST", async () => { + const { leases, tracker } = buildTracker(); + const outcome = await tracker.submit(identity, { ...body, allowDownload: true }); + if (outcome.kind !== "created") throw new Error("expected created"); + + leases.calls[0]?.reject(new Error("download failed")); + await Promise.resolve(); + await Promise.resolve(); + expect(tracker.get(outcome.view.id)?.state.stage).toBe("failed"); + }); +}); + +describe("LeaseRequestTracker.waitForChange abort", () => { + it("finishes immediately when the caller's signal aborts", async () => { + const { leases, tracker } = buildTracker(); + const { view } = await createTracked(tracker, leases); + + const controller = new AbortController(); + const wait = tracker.waitForChange(view.id, 30, controller.signal); + controller.abort(); + const result = await wait; + expect(result?.state.stage).toBe("queued"); + }); + + it("finishes immediately for a signal that is already aborted", async () => { + const { leases, tracker } = buildTracker(); + const { view } = await createTracked(tracker, leases); + + const controller = new AbortController(); + controller.abort(); + const result = await tracker.waitForChange(view.id, 30, controller.signal); + expect(result?.state.stage).toBe("queued"); + }); +}); + +describe("LeaseRequestTracker idempotency cleanup", () => { + it("drops the mapping when a submission is rejected before becoming visible -- a replay creates a fresh request", async () => { + const { leases, tracker } = buildTracker(); + const first = tracker.submit(identity, body, "key-1"); + leases.calls[0]?.reject(new RequesterAlreadyLeasedError("tok_agent")); + const firstOutcome = await first; + expect(firstOutcome.kind).toBe("rejected"); + + const second = tracker.submit(identity, body, "key-1"); + leases.calls[1]?.options.onProgress?.({ queuePosition: 1, stage: "queued" }); + const secondOutcome = await second; + expect(secondOutcome.kind).toBe("created"); + expect(leases.calls).toHaveLength(2); + }); +}); diff --git a/src/http/tracker.ts b/src/http/tracker.ts new file mode 100644 index 0000000..38a8b09 --- /dev/null +++ b/src/http/tracker.ts @@ -0,0 +1,493 @@ +import type { EventBus } from "../bus/index.js"; +import { + type DeviceRecord, + type DeviceRequest, + type LeaseRecord, + RequestCancelledError, +} from "../core/index.js"; +import type { LeaseCommands, QueueControl } from "../core/lease-ports.js"; +import type { LeaseGrant, LeaseProgress, LeaseRequestOptions } from "../core/wait-queue.js"; +import type { Clock, IdGenerator, Logger, TimerHandle } from "../ports/index.js"; +import { mapError } from "./errors.js"; + +export interface LeaseRequestInput { + readonly platform: "ios" | "android"; + readonly device: string; + readonly os?: string; + readonly ttlMs?: number; + readonly timeoutMs?: number; + readonly noWait?: boolean; + readonly allowDownload?: boolean; +} + +/** Matches the issue's lease object exactly; `dataPlane` is reserved and always `null` in v1. */ +export interface LeasePayload { + readonly id: string; + readonly requestId?: string; + readonly platform: string; + readonly device: string; + readonly os: string; + readonly udid: string; + readonly deviceId: string; + readonly createdAt: string; + readonly expiresAt: string; + readonly ttlMs: number; + readonly dataPlane: null; +} + +/** + * `LeaseRecord` has no `createdAt` -- `grantedAt` is its equivalent. `ttlMs` must be supplied + * by the caller (the tracker's record of what was applied at grant or last renew, or the + * lease's mode default when that record is gone, e.g. after a daemon restart). It is never + * derived as `ttlDeadline - grantedAt`: `grantedAt` never moves on renewal, so that + * arithmetic reports grant-age plus TTL rather than the interval actually in force -- + * `expiresAt` is the authoritative deadline either way. + */ +export function buildLeasePayload( + device: DeviceRecord, + lease: LeaseRecord, + extra: { readonly requestId?: string; readonly ttlMs: number }, +): LeasePayload { + return { + id: lease.id, + ...(extra.requestId === undefined ? {} : { requestId: extra.requestId }), + platform: device.spec.platform, + device: device.spec.model, + os: device.spec.osVersion, + udid: device.driverDeviceId, + deviceId: device.id, + createdAt: new Date(lease.grantedAt).toISOString(), + expiresAt: new Date(lease.ttlDeadline).toISOString(), + ttlMs: extra.ttlMs, + dataPlane: null, + }; +} + +export type RequestSnapshot = + | { readonly stage: "queued"; readonly queuePosition: number } + | { readonly stage: "reclaiming"; readonly etaSeconds: number } + | { readonly stage: "provisioning"; readonly etaSeconds: number } + | { readonly stage: "booting"; readonly etaSeconds: number } + | { readonly stage: "granted"; readonly lease: LeasePayload } + | { + readonly stage: "failed"; + readonly error: { readonly code: string; readonly message: string }; + } + | { readonly stage: "cancelled" }; + +export function isTerminalStage(state: RequestSnapshot): boolean { + return state.stage === "granted" || state.stage === "failed" || state.stage === "cancelled"; +} + +export interface TrackedRequestView { + readonly id: string; + readonly requesterId: string; + readonly createdAt: string; + readonly state: RequestSnapshot; +} + +export type CancelOutcome = + | { readonly kind: "cancelled" } + | { readonly kind: "not-found" } + | { readonly kind: "not-cancellable"; readonly leaseId?: string }; + +interface TrackedRequest { + readonly id: string; + readonly requesterId: string; + readonly createdAtIso: string; + state: RequestSnapshot; + readonly listeners: Set<(state: RequestSnapshot) => void>; +} + +/** How long a terminal request resource answers `GET` after settling, per the issue spec. */ +const TERMINAL_RETENTION_MS = 5 * 60_000; +/** How long an `Idempotency-Key` replay window stays open. */ +const IDEMPOTENCY_TTL_MS = 10 * 60_000; +/** + * Hard ceiling on live idempotency entries: an authenticated caller can mint a fresh key per + * request without ever occupying its one queue slot, so without a cap this map (and its + * expiry timers) grows without bound. FIFO eviction of the oldest entry only weakens replay + * protection for whoever is flooding, and `RequesterAlreadyLeasedError` remains the backstop + * against a double grant. + */ +const IDEMPOTENCY_MAX_ENTRIES = 10_000; + +export interface LeaseRequestTrackerOptions { + readonly leases: LeaseCommands; + readonly queue: QueueControl; + readonly eventBus: EventBus; + readonly clock: Clock; + readonly idGenerator: IdGenerator; + /** `lease.detachedTtlMs` -- every HTTP lease is detached, so this is the one mode default that applies. */ + readonly defaultTtlMs: number; + readonly logger?: Logger; +} + +/** + * Gateway-layer resource tracking for `POST /v1/lease-requests`. Calls `LeaseCommands.request` + * with an `onProgress` callback and never awaits its returned promise directly -- `submit` + * returns as soon as the request resource exists, matching "acquisition is an async resource, + * no long-blocking POST" from the issue's design principles. `GET`, long-poll, and SSE all + * read the same in-memory state this class owns; no core changes were needed to observe it. + */ +export class LeaseRequestTracker { + readonly #requests = new Map(); + readonly #idempotency = new Map(); + readonly #leaseRequestId = new Map(); + readonly #leaseTtlMs = new Map(); + readonly #unsubscribers: Array<() => void>; + /** + * The retention/idempotency-TTL timers `#setState`/`#registerIdempotency` arm below, + * tracked so `dispose()` can cancel whichever are still outstanding. Without this, a + * `Clock` backed by real timers (the daemon's `SystemClock`, unlike this class's own + * unit tests' `FakeClock`) would keep a real `setTimeout` alive for up to + * `TERMINAL_RETENTION_MS`/`IDEMPOTENCY_TTL_MS` after this instance is otherwise done + * with -- which, for a Node process, means `daemon stop` would not actually exit until + * that timer fires, minutes later. + */ + readonly #activeTimers = new Set(); + + constructor(private readonly options: LeaseRequestTrackerOptions) { + this.#unsubscribers = [ + options.eventBus.subscribe("lease.released", (envelope) => { + this.#leaseRequestId.delete(envelope.payload.leaseId); + this.#leaseTtlMs.delete(envelope.payload.leaseId); + }), + options.eventBus.subscribe("lease.expired", (envelope) => { + this.#leaseRequestId.delete(envelope.payload.leaseId); + this.#leaseTtlMs.delete(envelope.payload.leaseId); + }), + ]; + } + + submit( + identity: { readonly requesterId: string }, + body: LeaseRequestInput, + idempotencyKey?: string, + ): Promise< + | { readonly kind: "created"; readonly view: TrackedRequestView } + | { readonly kind: "rejected"; readonly error: unknown } + > { + const replay = this.#replayIdempotentSubmit(identity.requesterId, idempotencyKey); + if (replay !== undefined) return Promise.resolve({ kind: "created", view: replay }); + + const id = `req_${this.options.idGenerator.generate()}`; + const record: TrackedRequest = { + createdAtIso: new Date(this.options.clock.now()).toISOString(), + id, + listeners: new Set(), + // Best-effort snapshot of current queue depth: on the "admitted, still queued" path + // below this is superseded by the first real `onProgress` call before the POST response + // is even built, so it only matters for the sliver of time before that. + requesterId: identity.requesterId, + state: { queuePosition: this.options.queue.queueDepth + 1, stage: "queued" }, + }; + this.#requests.set(id, record); + this.#registerIdempotency(identity.requesterId, idempotencyKey, id); + + const deviceRequest: DeviceRequest = { + model: body.device, + platform: body.platform, + ...(body.os === undefined ? {} : { osVersion: body.os }), + }; + + // Races the grant/rejection against the request's *first* progress callback. A rejection + // that lands before any progress call (already-leased, unresolvable model/runtime/driver, + // no-capacity-with-noWait -- see `LeaseAcquisitionCoordinator#request`/`#resolveAndDrive`) + // never reached anything the queue considers "in flight", so the POST itself can fail with + // the matching HTTP status instead of the caller polling a request resource just to learn + // that. Once a progress callback fires (or a grant lands without ever needing one, e.g. an + // immediately-ready device), the request is a genuine async resource and always answers + // 201 -- from here on, failures surface only as the resource's terminal `failed` state. + return new Promise((resolve) => { + let settled = false; + const settleCreated = () => { + if (settled) return; + settled = true; + resolve({ kind: "created", view: toView(record) }); + }; + + const requestOptions: LeaseRequestOptions = { + mode: "detached", + onProgress: (progress) => { + this.#applyProgress(record, progress); + settleCreated(); + }, + requesterId: identity.requesterId, + ...(body.noWait === undefined ? {} : { noWait: body.noWait }), + ...(body.allowDownload === undefined ? {} : { allowDownload: body.allowDownload }), + ...(body.timeoutMs === undefined ? {} : { timeoutMs: body.timeoutMs }), + }; + + this.options.leases + .request(deviceRequest, requestOptions) + .then(async (grant) => { + await this.#applyGrant(record, grant, body.ttlMs); + settleCreated(); + }) + .catch((error: unknown) => { + this.#applyFailure(record, error); + if (settled) return; + settled = true; + // Never became visible to any client (the POST itself is about to fail), so it + // shouldn't answer a later GET/replay either -- including through the + // idempotency map, whose entry (and pending expiry timer) would otherwise + // outlive the record it points at. + this.#requests.delete(record.id); + if (idempotencyKey !== undefined) { + this.#dropIdempotency(idempotencyCacheKey(identity.requesterId, idempotencyKey)); + } + resolve({ kind: "rejected", error }); + }); + + // A download-permitted request can spend minutes inside the driver's `resolveSpec` + // (an Android `sdkmanager --install` runs there) before the first progress callback + // -- the one pre-progress stretch that legitimately runs long. Settle the POST now: + // the client polls the resource instead, and even an instant admission rejection + // (already-leased) then surfaces as the resource's terminal `failed` state rather + // than an HTTP error, because by the time it lands the resource is already visible. + if (body.allowDownload === true) settleCreated(); + }); + } + + get(id: string): TrackedRequestView | undefined { + const record = this.#requests.get(id); + return record === undefined ? undefined : toView(record); + } + + /** Registers a listener for future state changes only -- it does not fire for the current state. */ + subscribe(id: string, listener: (state: RequestSnapshot) => void): (() => void) | undefined { + const record = this.#requests.get(id); + if (record === undefined) return undefined; + record.listeners.add(listener); + return () => record.listeners.delete(listener); + } + + /** + * Resolves early on the next state change, else once `seconds` elapses; `undefined` if + * `id` is unknown. An aborted `signal` (the HTTP request's own -- the client hung up) + * also finishes immediately, so a disconnected long-poll releases its listener and timer + * right away instead of pinning them for the full requested wait. + */ + waitForChange( + id: string, + seconds: number, + signal?: AbortSignal, + ): Promise { + const record = this.#requests.get(id); + if (record === undefined) return Promise.resolve(undefined); + if (isTerminalStage(record.state)) return Promise.resolve(toView(record)); + + return new Promise((resolve) => { + let settled = false; + const finish = () => { + if (settled) return; + settled = true; + unsubscribe(); + signal?.removeEventListener("abort", finish); + this.options.clock.cancel(timer); + resolve(toView(record)); + }; + const unsubscribe = this.subscribe(id, finish) ?? (() => {}); + const timer = this.options.clock.setTimer(Math.max(0, seconds) * 1_000, finish); + if (signal?.aborted === true) finish(); + else signal?.addEventListener("abort", finish, { once: true }); + }); + } + + /** + * Cancels a pending request. Reuses `QueueControl.cancelPending`'s safety envelope exactly + * (see its own docs): only a request still queued -- no device work claimed for it yet -- is + * cancellable. The terminal state is applied here, synchronously with the queue's answer, + * rather than waiting for the rejected `LeaseCommands.request` promise's `.catch` to run on a + * later microtask -- a caller awaiting `cancel()` must see the settled state immediately. + */ + async cancel(id: string): Promise { + const record = this.#requests.get(id); + if (record === undefined) return { kind: "not-found" }; + const stateBefore = record.state; + if (stateBefore.stage === "granted") + return { kind: "not-cancellable", leaseId: stateBefore.lease.id }; + if (isTerminalStage(stateBefore)) return { kind: "not-cancellable" }; + + const outcome = await this.options.queue.cancelPending(record.requesterId); + if (outcome === "cancelled") { + this.#setState(record, { stage: "cancelled" }); + return { kind: "cancelled" }; + } + // Settled between the check above and this call (e.g. granted in the interim) -- report + // the now-current state rather than a stale answer. Read fresh rather than reusing + // `stateBefore`: the object identity is the same, but its `stage` may have moved on + // during the `await` above. + const stateAfter = record.state; + if (stateAfter.stage === "granted") + return { kind: "not-cancellable", leaseId: stateAfter.lease.id }; + return { kind: "not-cancellable" }; + } + + /** The tracker's own record of the ttl actually applied to a lease it granted (grant or renew). */ + effectiveTtlMs(leaseId: string): number | undefined { + return this.#leaseTtlMs.get(leaseId); + } + + /** Updates the tracked ttl after a direct (non-tracker) renew, e.g. `POST /v1/leases/:id/renew`. */ + recordLeaseTtl(leaseId: string, ttlMs: number): void { + this.#leaseTtlMs.set(leaseId, ttlMs); + } + + requestIdForLease(leaseId: string): string | undefined { + return this.#leaseRequestId.get(leaseId); + } + + dispose(): void { + for (const unsubscribe of this.#unsubscribers) unsubscribe(); + for (const timer of this.#activeTimers) this.options.clock.cancel(timer); + this.#activeTimers.clear(); + } + + #replayIdempotentSubmit( + requesterId: string, + idempotencyKey: string | undefined, + ): TrackedRequestView | undefined { + if (idempotencyKey === undefined) return undefined; + const entry = this.#idempotency.get(idempotencyCacheKey(requesterId, idempotencyKey)); + if (entry === undefined) return undefined; + const existing = this.#requests.get(entry.requestId); + // The mapping outlived the request's own 5-minute retention: treat this as a fresh key + // rather than returning nothing -- `submit`'s caller falls through to creating a new + // request, and `RequesterAlreadyLeasedError` remains the real backstop against a double + // grant if the original request had already succeeded. + return existing === undefined ? undefined : toView(existing); + } + + #registerIdempotency( + requesterId: string, + idempotencyKey: string | undefined, + requestId: string, + ): void { + if (idempotencyKey === undefined) return; + if (this.#idempotency.size >= IDEMPOTENCY_MAX_ENTRIES) { + const oldest = this.#idempotency.keys().next().value; + if (oldest !== undefined) this.#dropIdempotency(oldest); + } + const cacheKey = idempotencyCacheKey(requesterId, idempotencyKey); + const timer = this.options.clock.setTimer(IDEMPOTENCY_TTL_MS, () => { + this.#activeTimers.delete(timer); + if (this.#idempotency.get(cacheKey)?.requestId === requestId) { + this.#idempotency.delete(cacheKey); + } + }); + this.#activeTimers.add(timer); + this.#idempotency.set(cacheKey, { requestId, timer }); + } + + #dropIdempotency(cacheKey: string): void { + const entry = this.#idempotency.get(cacheKey); + if (entry === undefined) return; + this.options.clock.cancel(entry.timer); + this.#activeTimers.delete(entry.timer); + this.#idempotency.delete(cacheKey); + } + + #applyProgress(record: TrackedRequest, progress: LeaseProgress): void { + switch (progress.stage) { + case "queued": + this.#setState(record, { queuePosition: progress.queuePosition, stage: "queued" }); + return; + case "provisioning": + this.#setState(record, { etaSeconds: toSeconds(progress.etaMs), stage: "provisioning" }); + return; + case "booting": + this.#setState(record, { etaSeconds: toSeconds(progress.etaMs), stage: "booting" }); + return; + case "reclaiming": + this.#setState(record, { etaSeconds: toSeconds(progress.etaMs), stage: "reclaiming" }); + return; + } + } + + async #applyGrant( + record: TrackedRequest, + grant: LeaseGrant, + ttlMs: number | undefined, + ): Promise { + let lease = grant.lease; + let effectiveTtlMs = this.options.defaultTtlMs; + if (ttlMs !== undefined) { + try { + // `LeaseRequestOptions` carries no ttl -- a detached grant always lands on + // `lease.detachedTtlMs` (see `LeaseLifecycle#ttlFor`). A caller-specified `ttlMs` in + // the request body is applied with an explicit renew right after grant; see the class + // doc and this session's report for what was investigated here. + lease = await this.options.leases.renew(lease.id, ttlMs); + effectiveTtlMs = ttlMs; + } catch (error: unknown) { + this.options.logger?.warn( + "Requested ttlMs was not applied; lease keeps the default deadline", + { + leaseId: lease.id, + message: error instanceof Error ? error.message : String(error), + requestedTtlMs: ttlMs, + }, + ); + // Renewing a lease that was just granted failing would be surprising; fall back to the + // grant's own (config-default) deadline rather than losing the lease record entirely. + // `effectiveTtlMs` deliberately stays at the default: the payload must report the ttl + // actually in force, not the one that failed to apply. + } + } + this.#leaseRequestId.set(lease.id, record.id); + this.#leaseTtlMs.set(lease.id, effectiveTtlMs); + this.#setState(record, { + lease: buildLeasePayload(grant.device, lease, { + requestId: record.id, + ttlMs: effectiveTtlMs, + }), + stage: "granted", + }); + } + + #applyFailure(record: TrackedRequest, error: unknown): void { + if (error instanceof RequestCancelledError) { + this.#setState(record, { stage: "cancelled" }); + return; + } + const mapped = mapError(error); + this.#setState(record, { + error: { code: mapped.code, message: mapped.message }, + stage: "failed", + }); + } + + #setState(record: TrackedRequest, state: RequestSnapshot): void { + if (isTerminalStage(record.state)) return; + record.state = state; + // Snapshotted: a listener may synchronously subscribe/unsubscribe (e.g. an SSE stream + // ending itself), which would otherwise mutate `record.listeners` mid-iteration. + for (const listener of Array.from(record.listeners)) listener(state); + if (isTerminalStage(state)) { + const timer = this.options.clock.setTimer(TERMINAL_RETENTION_MS, () => { + this.#activeTimers.delete(timer); + this.#requests.delete(record.id); + }); + this.#activeTimers.add(timer); + } + } +} + +function toView(record: TrackedRequest): TrackedRequestView { + return { + createdAt: record.createdAtIso, + id: record.id, + requesterId: record.requesterId, + state: record.state, + }; +} + +function idempotencyCacheKey(requesterId: string, key: string): string { + return `${requesterId} ${key}`; +} + +function toSeconds(ms: number): number { + return Math.round(ms / 1_000); +} diff --git a/src/ports/index.ts b/src/ports/index.ts index ff9cec2..93ce114 100644 --- a/src/ports/index.ts +++ b/src/ports/index.ts @@ -27,9 +27,9 @@ export type { LogSink } from "./logger.js"; // fallow-ignore-next-line unused-type -- public shape of a parsed log line, for consumers of MemoryLogSink.records. export type { LogRecord } from "./logger.js"; export { CryptoIdGenerator, type IdGenerator } from "./id-generator.js"; +export { CryptoTokenSecrets, type TokenSecrets } from "./token-secrets.js"; export { FakeSystemStats, NodeSystemStats, type SystemStats } from "./system-stats.js"; export { FakeParentWatch, NodeParentWatch, type ParentWatch } from "./parent-watch.js"; -// fallow-ignore-next-line unused-type -- public handle contract returned by ParentWatch.watch(). export type { ParentWatchHandle } from "./parent-watch.js"; export { NodeProcessRunner, diff --git a/src/ports/token-secrets.test.ts b/src/ports/token-secrets.test.ts new file mode 100644 index 0000000..9a5c679 --- /dev/null +++ b/src/ports/token-secrets.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; + +import { CryptoTokenSecrets } from "./index.js"; + +describe("CryptoTokenSecrets", () => { + it("generates secrets prefixed slk_ that differ across calls", () => { + const secrets = new CryptoTokenSecrets(); + + const first = secrets.generateSecret(); + const second = secrets.generateSecret(); + + expect(first).toMatch(/^slk_/); + expect(second).toMatch(/^slk_/); + expect(first).not.toBe(second); + }); + + it("generates secrets carrying at least 32 bytes of entropy", () => { + const secrets = new CryptoTokenSecrets(); + + const secret = secrets.generateSecret().slice("slk_".length); + + // base64url-encodes 32 raw bytes without padding: ceil(32 * 8 / 6) = 43 chars. + expect(secret.length).toBeGreaterThanOrEqual(43); + expect(secret).toMatch(/^[A-Za-z0-9_-]+$/); + }); + + it("hashes deterministically to hex", () => { + const secrets = new CryptoTokenSecrets(); + + const first = secrets.hash("slk_some-secret"); + const second = secrets.hash("slk_some-secret"); + + expect(first).toBe(second); + expect(first).toMatch(/^[0-9a-f]{64}$/); + }); + + it("hashes different secrets to different digests", () => { + const secrets = new CryptoTokenSecrets(); + + expect(secrets.hash("slk_one")).not.toBe(secrets.hash("slk_two")); + }); +}); diff --git a/src/ports/token-secrets.ts b/src/ports/token-secrets.ts new file mode 100644 index 0000000..a75d432 --- /dev/null +++ b/src/ports/token-secrets.ts @@ -0,0 +1,19 @@ +import { createHash, randomBytes } from "node:crypto"; + +export interface TokenSecrets { + generateSecret(): string; + hash(secret: string): string; +} + +const SECRET_PREFIX = "slk_"; +const SECRET_ENTROPY_BYTES = 32; + +export class CryptoTokenSecrets implements TokenSecrets { + generateSecret(): string { + return `${SECRET_PREFIX}${randomBytes(SECRET_ENTROPY_BYTES).toString("base64url")}`; + } + + hash(secret: string): string { + return createHash("sha256").update(secret).digest("hex"); + } +}