From 86567296fea9f12389d07c77833cf1d043cbe2f3 Mon Sep 17 00:00:00 2001 From: Jamie Asefa <13409617+j-asefa@users.noreply.github.com> Date: Fri, 20 Nov 2026 17:47:39 +0000 Subject: [PATCH 1/9] Cloud VM experiments 1/5: storage, state machines and Sailboxes provider Adds the storage and provider layer for VM-backed Experiments, with no routes or UI yet: - Migration 167 adds vm_provider_configs, experiment_vms, experiment_previews and vm_operations, each with a per-table SQL module and QueryRecords schema. - experiment_vms.state and vm_operations.state have one owner each (src/shared/vm-state.ts) and one writer (src/server/vm/machine.ts), documented in docs/state-machines.md with a test that keeps the tables in step. - runVmOperation writes a receipt before every provider call, replays a repeated id, refuses a reused id with a different request, and keeps any unobserved outcome uncertain. reconcileVmCreate settles an uncertain create by the VM's deterministic name and never creates. - The worker tick marks receipts still pending after the longest provider call uncertain. - VmProvider plus a Sailboxes adapter over the HTTPS API (no @sailresearch/sdk, which ships native binaries). - The experiment image is a golden Sailbox checkpoint built from an install script, because Sail builds custom images only through its SDK. Co-Authored-By: Claude Opus 5.5 --- docs/sql-index.md | 35 ++ docs/state-machines.md | 38 ++ migrations/167-experiment-vms.sql | 76 ++++ scripts/check.mjs | 1 + src/server/sql/experiment-previews.ts | 89 +++++ src/server/sql/experiment-vms.ts | 121 ++++++ src/server/sql/rows.ts | 63 +++ src/server/sql/vm-operations.ts | 102 +++++ src/server/sql/vm-provider-configs.ts | 74 ++++ src/server/vm/machine.ts | 112 ++++++ src/server/vm/operations.ts | 260 +++++++++++++ src/server/vm/provider.ts | 118 ++++++ src/server/vm/sailbox-image.ts | 105 +++++ src/server/vm/sailbox-provider.ts | 527 ++++++++++++++++++++++++++ src/server/worker.ts | 2 + src/shared/vm-state.ts | 68 ++++ test/vm-operations-worker.test.ts | 57 +++ test/vm-operations.test.ts | 309 +++++++++++++++ test/vm-sailbox-image.test.ts | 76 ++++ test/vm-sailbox-provider.test.ts | 298 +++++++++++++++ test/vm-state-documentation.test.ts | 46 +++ 21 files changed, 2577 insertions(+) create mode 100644 migrations/167-experiment-vms.sql create mode 100644 src/server/sql/experiment-previews.ts create mode 100644 src/server/sql/experiment-vms.ts create mode 100644 src/server/sql/vm-operations.ts create mode 100644 src/server/sql/vm-provider-configs.ts create mode 100644 src/server/vm/machine.ts create mode 100644 src/server/vm/operations.ts create mode 100644 src/server/vm/provider.ts create mode 100644 src/server/vm/sailbox-image.ts create mode 100644 src/server/vm/sailbox-provider.ts create mode 100644 src/shared/vm-state.ts create mode 100644 test/vm-operations-worker.test.ts create mode 100644 test/vm-operations.test.ts create mode 100644 test/vm-sailbox-image.test.ts create mode 100644 test/vm-sailbox-provider.test.ts create mode 100644 test/vm-state-documentation.test.ts diff --git a/docs/sql-index.md b/docs/sql-index.md index dbd19eff..f7161edb 100644 --- a/docs/sql-index.md +++ b/docs/sql-index.md @@ -511,6 +511,24 @@ Generated by `npm run sql-index` from [src/server/sql](../src/server/sql/); do n - `start(db, owner, command, method, digest, generation, created, updated)` - `unknown(db, time, owner, command, generation)` +## [experiment-previews.ts](../src/server/sql/experiment-previews.ts) + +- `byExperiment(db, owner, experiment)` +- `byPort(db, owner, experiment, port)` +- `deactivate(db, owner, experiment, port, now)` +- `deactivateAll(db, owner, experiment, now)` — A destroyed VM serves nothing, so every link of the experiment is disabled. +- `upsert(db, preview)` — One preview per experiment port; exposing the port again reactivates it. + +## [experiment-vms.ts](../src/server/sql/experiment-vms.ts) + +- `adoptRemote(db, owner, id, remoteId)` — Records the provider's id once; a different id for the same VM is refused. +- `byFolder(db, owner, folder)` +- `byId(db, owner, id)` +- `insert(db, vm)` +- `owned(db, owner)` +- `setState(db, owner, id, from, to, lastError, now)` — Moves `from` to `to`; returns false when the row is no longer in `from`. +- `touch(db, owner, id, now)` + ## [feed-card-dirty.ts](../src/server/sql/feed-card-dirty.ts) - `any(db)` @@ -1639,6 +1657,23 @@ Generated by `npm run sql-index` from [src/server/sql](../src/server/sql/); do n - `profile(db, owner)` - `save(db, owner, name, handle, bio, updated)` +## [vm-operations.ts](../src/server/sql/vm-operations.ts) + +- `byId(db, owner, id)` +- `idTaken(db, id)` — Whether any owner holds this id, without revealing whose it is. +- `insertPending(db, operation)` +- `pendingBefore(db, cutoff)` — Every owner's receipts still pending since before \`cutoff\`, for the recovery sweep only. +- `setState(db, owner, id, from, to, outcome, now)` — Moves `from` to `to`; returns false when the row is no longer in `from`. +- `unsettledByExperiment(db, owner, experiment)` + +## [vm-provider-configs.ts](../src/server/sql/vm-provider-configs.ts) + +- `byOwner(db, owner)` +- `remove(db, owner)` +- `saveKey(db, owner, provider, sealedKey, now)` — Stores the sealed key, keeping the resources already found for the owner. +- `setDefaultHarness(db, owner, harness, now)` +- `setResources(db, owner, resources, now)` + ## [webhook-deliveries.ts](../src/server/sql/webhook-deliveries.ts) - `activation(db, subscription)` diff --git a/docs/state-machines.md b/docs/state-machines.md index 1fae2281..b5d11397 100644 --- a/docs/state-machines.md +++ b/docs/state-machines.md @@ -365,6 +365,44 @@ rolls back local work before recording either a refusal or another attempt. This an uncertain harness delivery. Retry records cascade with their source run or mail; ineligible results retain their metadata until source deletion. +## Cloud VM experiments + +### `experiment_vms.state` + +Owner: `VM_TRANSITIONS` in [vm-state.ts](../src/shared/vm-state.ts), applied +only by `transitionVm()` in [vm/machine.ts](../src/server/vm/machine.ts), which +re-reads the owner's row inside one immediate transaction. A destroyed VM +leaves its project, threads, results and published files readable. + +| From | To | Trigger and guard | +| ---- | -- | ----------------- | +| `provisioning` | `running` / `failed` / `destroying` | The provider reported the VM up, refused the create, or the owner destroyed it; **tx** | +| `running` | `sleeping` / `paused` / `destroying` / `failed` | Sleep observed or requested, pause, owner destroy, or a provider failure observed; **tx** | +| `sleeping` | `running` / `destroying` / `failed` | Wake for a turn or a preview, owner destroy, or a failed wake; **tx** | +| `paused` | `running` / `destroying` / `failed` | Resume, owner destroy, or a failed resume; **tx** | +| `failed` | `provisioning` / `running` / `destroying` | The owner's reconcile or retry, or owner destroy; **tx** | +| `destroying` | `destroyed` / `failed` | The provider confirmed the VM is gone (a missing VM counts), or refused; **tx** | +| **`destroyed`** | — | Terminal | + +### `vm_operations.state` + +Owner: `VM_OPERATION_TRANSITIONS` in the same file, applied only by +`transitionVmOperation()`. The row is a receipt: it is inserted `pending` +before the provider is called, and its id is the caller's idempotency key. + +| From | To | Trigger and guard | +| ---- | -- | ----------------- | +| `pending` | `succeeded` / `failed` / `uncertain` | The provider answered, refused, or gave no observed answer (timeout, disconnect, server error, or the worker's sweep found the row still pending after the longest provider call); **tx** | +| `uncertain` | `succeeded` / `failed` | A late answer arrived, the owner's reconcile found the VM by its `remote_name`, or the provider confirmed it does not exist; **tx** | +| **`succeeded`** | — | Terminal | +| **`failed`** | — | Terminal | + +An uncertain operation is never sent again automatically. A retry is a new +operation with a new id, and is refused while an earlier one for the same +experiment is unsettled. Creates and forks send the provider an idempotency +key scoped to the experiment, so a retry after an "absent" reconcile reaches +the VM a still-running first attempt made instead of starting a second one. + ## Fleets (experimental) | Record | States and edges | Guards | diff --git a/migrations/167-experiment-vms.sql b/migrations/167-experiment-vms.sql new file mode 100644 index 00000000..c7029257 --- /dev/null +++ b/migrations/167-experiment-vms.sql @@ -0,0 +1,76 @@ +-- Cloud VM Experiments. One Experiment is one provider VM linked 1:1 to a +-- project folder. Conversations, results and published files stay in this +-- database and its object store, so no row here is needed to read history. +-- Additive: releases before this migration wrote none of these tables. + +-- The owner's provider account. The key is sealed with the owner-bound +-- secret store and never leaves this host. +CREATE TABLE vm_provider_configs ( + owner_id TEXT PRIMARY KEY, + provider TEXT NOT NULL, + encrypted_api_key TEXT NOT NULL, + app_id TEXT, + auth_volume_id TEXT, + base_image_ref TEXT, + base_image_version TEXT, + default_harness TEXT, + updated_at INTEGER NOT NULL +); + +-- remote_name is chosen before the provider is called, so an uncertain create +-- can be reconciled by looking the name up instead of creating a second VM. +CREATE TABLE experiment_vms ( + id TEXT PRIMARY KEY, + owner_id TEXT NOT NULL, + folder_id TEXT NOT NULL UNIQUE REFERENCES folders(id), + provider TEXT NOT NULL, + remote_id TEXT, + remote_name TEXT NOT NULL UNIQUE, + state TEXT NOT NULL, + auto_approve_local INTEGER NOT NULL DEFAULT 1, + vcpu_limit INTEGER NOT NULL DEFAULT 4, + memory_limit_gib INTEGER NOT NULL DEFAULT 16, + disk_limit_gib INTEGER NOT NULL DEFAULT 64, + auto_sleep_seconds INTEGER NOT NULL DEFAULT 900, + source_experiment_id TEXT REFERENCES experiment_vms(id), + source_checkpoint_id TEXT, + clone_url TEXT, + last_error TEXT, + last_active_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + destroyed_at INTEGER +); +CREATE INDEX experiment_vms_owner ON experiment_vms(owner_id, created_at); + +CREATE TABLE experiment_previews ( + id TEXT PRIMARY KEY, + owner_id TEXT NOT NULL, + experiment_id TEXT NOT NULL REFERENCES experiment_vms(id), + port INTEGER NOT NULL, + label TEXT, + path TEXT NOT NULL DEFAULT '/', + url TEXT NOT NULL, + active INTEGER NOT NULL DEFAULT 1, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(experiment_id, port) +); + +-- A receipt written before every provider call. id is the caller's +-- idempotency key; request_digest refuses the same key for a different +-- request. An operation left pending by a crash becomes uncertain and is +-- never sent again without the owner asking. +CREATE TABLE vm_operations ( + id TEXT PRIMARY KEY, + owner_id TEXT NOT NULL, + experiment_id TEXT NOT NULL REFERENCES experiment_vms(id), + kind TEXT NOT NULL, + state TEXT NOT NULL, + request_digest TEXT NOT NULL, + result_json TEXT, + error_message TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); +CREATE INDEX vm_operations_experiment ON vm_operations(experiment_id, created_at); +CREATE INDEX vm_operations_unsettled ON vm_operations(state) WHERE state IN ('pending', 'uncertain'); diff --git a/scripts/check.mjs b/scripts/check.mjs index 65147a36..96b6c1ed 100644 --- a/scripts/check.mjs +++ b/scripts/check.mjs @@ -15,6 +15,7 @@ const tests = [ "test/json-column-boundaries.test.ts", "test/discriminating-fields.test.ts", "test/run-state-documentation.test.ts", + "test/vm-state-documentation.test.ts", "test/guidance-links.test.ts", "test/migration-policy.test.ts", "test/row-schema-columns.test.ts", diff --git a/src/server/sql/experiment-previews.ts b/src/server/sql/experiment-previews.ts new file mode 100644 index 00000000..fe6187d0 --- /dev/null +++ b/src/server/sql/experiment-previews.ts @@ -0,0 +1,89 @@ +import type Database from "better-sqlite3"; +import { QueryRecords } from "../query-records.js"; +import { parseRecord } from "../row-schemas.js"; +import { statement } from "./statement.js"; + +// Queries for the experiment_previews table (docs/architecture.md#recipe-add-or-extend-a-sql-table-module). + +const parse = (row: unknown) => + parseRecord(QueryRecords.experiment_previews, row, "experiment_previews"); + +const upsertStatement = statement( + "INSERT INTO experiment_previews(id,owner_id,experiment_id,port,label,path,url,active,created_at,updated_at) VALUES(?,?,?,?,?,?,?,1,?,?) ON CONFLICT(experiment_id,port) DO UPDATE SET label=excluded.label,path=excluded.path,url=excluded.url,active=1,updated_at=excluded.updated_at WHERE owner_id=excluded.owner_id", +); +/** One preview per experiment port; exposing the port again reactivates it. */ +export function upsert( + db: Database.Database, + preview: { + id: string; + owner: string; + experimentId: string; + port: number; + label: string | null; + path: string; + url: string; + now: number; + }, +) { + upsertStatement(db).run( + preview.id, + preview.owner, + preview.experimentId, + preview.port, + preview.label, + preview.path, + preview.url, + preview.now, + preview.now, + ); +} + +const byExperimentStatement = statement( + "SELECT * FROM experiment_previews WHERE owner_id=? AND experiment_id=? ORDER BY port", +); +export function byExperiment( + db: Database.Database, + owner: string, + experiment: string, +) { + return byExperimentStatement(db).all(owner, experiment).map(parse); +} + +const byPortStatement = statement( + "SELECT * FROM experiment_previews WHERE owner_id=? AND experiment_id=? AND port=?", +); +export function byPort( + db: Database.Database, + owner: string, + experiment: string, + port: number, +) { + const row = byPortStatement(db).get(owner, experiment, port); + return row === undefined ? undefined : parse(row); +} + +const deactivateStatement = statement( + "UPDATE experiment_previews SET active=0,updated_at=? WHERE owner_id=? AND experiment_id=? AND port=?", +); +export function deactivate( + db: Database.Database, + owner: string, + experiment: string, + port: number, + now: number, +) { + return deactivateStatement(db).run(now, owner, experiment, port).changes; +} + +const deactivateAllStatement = statement( + "UPDATE experiment_previews SET active=0,updated_at=? WHERE owner_id=? AND experiment_id=? AND active=1", +); +/** A destroyed VM serves nothing, so every link of the experiment is disabled. */ +export function deactivateAll( + db: Database.Database, + owner: string, + experiment: string, + now: number, +) { + return deactivateAllStatement(db).run(now, owner, experiment).changes; +} diff --git a/src/server/sql/experiment-vms.ts b/src/server/sql/experiment-vms.ts new file mode 100644 index 00000000..8f12eb95 --- /dev/null +++ b/src/server/sql/experiment-vms.ts @@ -0,0 +1,121 @@ +import type Database from "better-sqlite3"; +import { QueryRecords } from "../query-records.js"; +import { parseRecord } from "../row-schemas.js"; +import { statement } from "./statement.js"; + +// Queries for the experiment_vms table (docs/architecture.md#recipe-add-or-extend-a-sql-table-module). +// state is written only through setState, called by transitionVm() in +// vm/machine.ts (docs/state-machines.md#experiment_vmsstate). + +const parse = (row: unknown) => + parseRecord(QueryRecords.experiment_vms, row, "experiment_vms"); + +const insertStatement = statement( + "INSERT INTO experiment_vms(id,owner_id,folder_id,provider,remote_name,state,auto_approve_local,vcpu_limit,memory_limit_gib,disk_limit_gib,auto_sleep_seconds,source_experiment_id,source_checkpoint_id,clone_url,last_active_at,created_at) VALUES(?,?,?,?,?,'provisioning',?,?,?,?,?,?,?,?,?,?)", +); +export function insert( + db: Database.Database, + vm: { + id: string; + owner: string; + folderId: string; + provider: string; + remoteName: string; + autoApproveLocal: boolean; + vcpuLimit: number; + memoryLimitGib: number; + diskLimitGib: number; + autoSleepSeconds: number; + sourceExperimentId: string | null; + sourceCheckpointId: string | null; + cloneUrl: string | null; + now: number; + }, +) { + insertStatement(db).run( + vm.id, + vm.owner, + vm.folderId, + vm.provider, + vm.remoteName, + vm.autoApproveLocal ? 1 : 0, + vm.vcpuLimit, + vm.memoryLimitGib, + vm.diskLimitGib, + vm.autoSleepSeconds, + vm.sourceExperimentId, + vm.sourceCheckpointId, + vm.cloneUrl, + vm.now, + vm.now, + ); +} + +const byIdStatement = statement( + "SELECT * FROM experiment_vms WHERE owner_id=? AND id=?", +); +export function byId(db: Database.Database, owner: string, id: string) { + const row = byIdStatement(db).get(owner, id); + return row === undefined ? undefined : parse(row); +} + +const byFolderStatement = statement( + "SELECT * FROM experiment_vms WHERE owner_id=? AND folder_id=?", +); +export function byFolder(db: Database.Database, owner: string, folder: string) { + const row = byFolderStatement(db).get(owner, folder); + return row === undefined ? undefined : parse(row); +} + +const ownedStatement = statement( + "SELECT * FROM experiment_vms WHERE owner_id=? ORDER BY created_at DESC, id", +); +export function owned(db: Database.Database, owner: string) { + return ownedStatement(db).all(owner).map(parse); +} + +const setStateStatement = statement( + "UPDATE experiment_vms SET state=?,last_error=?,destroyed_at=CASE WHEN ?='destroyed' THEN ? ELSE destroyed_at END WHERE owner_id=? AND id=? AND state=?", +); +/** Moves `from` to `to`; returns false when the row is no longer in `from`. */ +export function setState( + db: Database.Database, + owner: string, + id: string, + from: string, + to: string, + lastError: string | null, + now: number, +) { + return ( + setStateStatement(db).run(to, lastError, to, now, owner, id, from) + .changes === 1 + ); +} + +const adoptRemoteStatement = statement( + "UPDATE experiment_vms SET remote_id=? WHERE owner_id=? AND id=? AND (remote_id IS NULL OR remote_id=?)", +); +/** Records the provider's id once; a different id for the same VM is refused. */ +export function adoptRemote( + db: Database.Database, + owner: string, + id: string, + remoteId: string, +) { + return ( + adoptRemoteStatement(db).run(remoteId, owner, id, remoteId).changes === 1 + ); +} + +const touchStatement = statement( + "UPDATE experiment_vms SET last_active_at=? WHERE owner_id=? AND id=?", +); +export function touch( + db: Database.Database, + owner: string, + id: string, + now: number, +) { + touchStatement(db).run(now, owner, id); +} diff --git a/src/server/sql/rows.ts b/src/server/sql/rows.ts index fb89000f..ef384960 100644 --- a/src/server/sql/rows.ts +++ b/src/server/sql/rows.ts @@ -7,6 +7,13 @@ import { MessageRole } from "../../shared/message-roles.js"; import { AgentOrigin, ConversationOrigin } from "../../shared/origins.js"; import { ConnectionStatus } from "../../shared/connection-status.js"; import { RunKind } from "../../shared/run-state.js"; +import { + VmHarness, + VmOperationKind, + VmOperationState, + VmProviderKind, + VmState, +} from "../../shared/vm-state.js"; // Scalar SQLite storage shapes. JSON columns remain serialized at this boundary. export const RowId = z.string().min(1); @@ -1401,6 +1408,62 @@ export const QueryRecords = { owner_id: z.string(), card_id: z.string(), }), + vm_provider_configs: z.object({ + owner_id: z.string(), + provider: VmProviderKind, + encrypted_api_key: z.string(), + app_id: z.string().nullable(), + auth_volume_id: z.string().nullable(), + base_image_ref: z.string().nullable(), + base_image_version: z.string().nullable(), + default_harness: VmHarness.nullable(), + updated_at: z.number(), + }), + experiment_vms: z.object({ + id: z.string(), + owner_id: z.string(), + folder_id: z.string(), + provider: VmProviderKind, + remote_id: z.string().nullable(), + remote_name: z.string(), + state: VmState, + auto_approve_local: z.number().int(), + vcpu_limit: z.number().int(), + memory_limit_gib: z.number().int(), + disk_limit_gib: z.number().int(), + auto_sleep_seconds: z.number().int(), + source_experiment_id: z.string().nullable(), + source_checkpoint_id: z.string().nullable(), + clone_url: z.string().nullable(), + last_error: z.string().nullable(), + last_active_at: z.number(), + created_at: z.number(), + destroyed_at: z.number().nullable(), + }), + experiment_previews: z.object({ + id: z.string(), + owner_id: z.string(), + experiment_id: z.string(), + port: z.number().int(), + label: z.string().nullable(), + path: z.string(), + url: z.string(), + active: z.number().int(), + created_at: z.number(), + updated_at: z.number(), + }), + vm_operations: z.object({ + id: z.string(), + owner_id: z.string(), + experiment_id: z.string(), + kind: VmOperationKind, + state: VmOperationState, + request_digest: z.string(), + result_json: z.string().nullable(), + error_message: z.string().nullable(), + created_at: z.number(), + updated_at: z.number(), + }), } as const; export type QueryRow = z.infer< (typeof QueryRecords)[T] diff --git a/src/server/sql/vm-operations.ts b/src/server/sql/vm-operations.ts new file mode 100644 index 00000000..8b699243 --- /dev/null +++ b/src/server/sql/vm-operations.ts @@ -0,0 +1,102 @@ +import type Database from "better-sqlite3"; +import { QueryRecords } from "../query-records.js"; +import { parseRecord } from "../row-schemas.js"; +import { statement } from "./statement.js"; + +// Queries for the vm_operations table (docs/architecture.md#recipe-add-or-extend-a-sql-table-module). +// state is written only through insertPending and setState, called by +// vm/machine.ts (docs/state-machines.md#vm_operationsstate). + +const parse = (row: unknown) => + parseRecord(QueryRecords.vm_operations, row, "vm_operations"); + +const insertPendingStatement = statement( + "INSERT INTO vm_operations(id,owner_id,experiment_id,kind,state,request_digest,created_at,updated_at) VALUES(?,?,?,?,'pending',?,?,?)", +); +export function insertPending( + db: Database.Database, + operation: { + id: string; + owner: string; + experimentId: string; + kind: string; + requestDigest: string; + now: number; + }, +) { + insertPendingStatement(db).run( + operation.id, + operation.owner, + operation.experimentId, + operation.kind, + operation.requestDigest, + operation.now, + operation.now, + ); +} + +const byIdStatement = statement( + "SELECT * FROM vm_operations WHERE owner_id=? AND id=?", +); +export function byId(db: Database.Database, owner: string, id: string) { + const row = byIdStatement(db).get(owner, id); + return row === undefined ? undefined : parse(row); +} + +const idTakenStatement = statement("SELECT 1 FROM vm_operations WHERE id=?"); +/** Whether any owner holds this id, without revealing whose it is. */ +export function idTaken(db: Database.Database, id: string) { + return idTakenStatement(db).get(id) !== undefined; +} + +const unsettledByExperimentStatement = statement( + "SELECT * FROM vm_operations WHERE owner_id=? AND experiment_id=? AND state IN ('pending','uncertain') ORDER BY created_at, id", +); +export function unsettledByExperiment( + db: Database.Database, + owner: string, + experiment: string, +) { + return unsettledByExperimentStatement(db).all(owner, experiment).map(parse); +} + +const setStateStatement = statement( + "UPDATE vm_operations SET state=?,result_json=?,error_message=?,updated_at=? WHERE owner_id=? AND id=? AND state=?", +); +/** Moves `from` to `to`; returns false when the row is no longer in `from`. */ +export function setState( + db: Database.Database, + owner: string, + id: string, + from: string, + to: string, + outcome: { resultJson: string | null; errorMessage: string | null }, + now: number, +) { + return ( + setStateStatement(db).run( + to, + outcome.resultJson, + outcome.errorMessage, + now, + owner, + id, + from, + ).changes === 1 + ); +} + +const pendingIdsStatement = statement( + "SELECT id,owner_id FROM vm_operations WHERE state='pending' AND updated_at parseRecord(pendingIdRow, row, "vm_operations")); +} diff --git a/src/server/sql/vm-provider-configs.ts b/src/server/sql/vm-provider-configs.ts new file mode 100644 index 00000000..076e839e --- /dev/null +++ b/src/server/sql/vm-provider-configs.ts @@ -0,0 +1,74 @@ +import type Database from "better-sqlite3"; +import { QueryRecords } from "../query-records.js"; +import { parseRecord } from "../row-schemas.js"; +import { statement } from "./statement.js"; + +// Queries for the vm_provider_configs table (docs/architecture.md#recipe-add-or-extend-a-sql-table-module). +// encrypted_api_key is sealed by the caller; this module never sees the key. + +const byOwnerStatement = statement( + "SELECT * FROM vm_provider_configs WHERE owner_id=?", +); +export function byOwner(db: Database.Database, owner: string) { + const row = byOwnerStatement(db).get(owner); + return row === undefined + ? undefined + : parseRecord(QueryRecords.vm_provider_configs, row, "vm_provider_configs"); +} + +const saveKeyStatement = statement( + "INSERT INTO vm_provider_configs(owner_id,provider,encrypted_api_key,updated_at) VALUES(?,?,?,?) ON CONFLICT(owner_id) DO UPDATE SET provider=excluded.provider,encrypted_api_key=excluded.encrypted_api_key,updated_at=excluded.updated_at", +); +/** Stores the sealed key, keeping the resources already found for the owner. */ +export function saveKey( + db: Database.Database, + owner: string, + provider: string, + sealedKey: string, + now: number, +) { + saveKeyStatement(db).run(owner, provider, sealedKey, now); +} + +const setResourcesStatement = statement( + "UPDATE vm_provider_configs SET app_id=?,auth_volume_id=?,base_image_ref=?,base_image_version=?,updated_at=? WHERE owner_id=?", +); +export function setResources( + db: Database.Database, + owner: string, + resources: { + appId: string | null; + authVolumeId: string | null; + baseImageRef: string | null; + baseImageVersion: string | null; + }, + now: number, +) { + return setResourcesStatement(db).run( + resources.appId, + resources.authVolumeId, + resources.baseImageRef, + resources.baseImageVersion, + now, + owner, + ).changes; +} + +const setDefaultHarnessStatement = statement( + "UPDATE vm_provider_configs SET default_harness=?,updated_at=? WHERE owner_id=?", +); +export function setDefaultHarness( + db: Database.Database, + owner: string, + harness: string | null, + now: number, +) { + return setDefaultHarnessStatement(db).run(harness, now, owner).changes; +} + +const removeStatement = statement( + "DELETE FROM vm_provider_configs WHERE owner_id=?", +); +export function remove(db: Database.Database, owner: string) { + return removeStatement(db).run(owner).changes; +} diff --git a/src/server/vm/machine.ts b/src/server/vm/machine.ts new file mode 100644 index 00000000..d2544d26 --- /dev/null +++ b/src/server/vm/machine.ts @@ -0,0 +1,112 @@ +import type Database from "better-sqlite3"; +import { + legalVmOperationTransition, + legalVmTransition, + type VmOperationState, + type VmState, +} from "../../shared/vm-state.js"; +import * as experimentVmsQueries from "../sql/experiment-vms.js"; +import * as vmOperationsQueries from "../sql/vm-operations.js"; + +// The only writers of experiment_vms.state and vm_operations.state +// (docs/state-machines.md#cloud-vm-experiments). Each re-reads the row and +// checks its state inside one immediate transaction, so a caller that decided +// from an older read changes nothing. + +export type VmTransitionResult = + | { changed: true; from: VmState } + | { changed: false; reason: "missing" | "illegal"; from?: VmState }; + +/** Moves the owner's VM to `to` when that edge is legal from its current + * state. `expected` narrows the move to a state the caller observed. */ +export function transitionVm( + db: Database.Database, + vm: { owner: string; id: string }, + to: VmState, + options: { expected?: VmState; lastError?: string | null; now: number }, +): VmTransitionResult { + return db + .transaction((): VmTransitionResult => { + const row = experimentVmsQueries.byId(db, vm.owner, vm.id); + if (!row) return { changed: false, reason: "missing" }; + if ( + (options.expected !== undefined && row.state !== options.expected) || + !legalVmTransition(row.state, to) + ) + return { changed: false, reason: "illegal", from: row.state }; + experimentVmsQueries.setState( + db, + vm.owner, + vm.id, + row.state, + to, + options.lastError ?? null, + options.now, + ); + return { changed: true, from: row.state }; + }) + .immediate(); +} + +/** Settles or marks uncertain the owner's operation receipt. Returns false + * when the receipt is missing or the edge is not legal from its state. */ +export function transitionVmOperation( + db: Database.Database, + operation: { owner: string; id: string }, + to: Exclude, + outcome: { result?: unknown; errorMessage?: string | null; now: number }, +): boolean { + return db + .transaction(() => { + const row = vmOperationsQueries.byId(db, operation.owner, operation.id); + if (!row || !legalVmOperationTransition(row.state, to)) return false; + return vmOperationsQueries.setState( + db, + operation.owner, + operation.id, + row.state, + to, + { + resultJson: + outcome.result === undefined + ? row.result_json + : JSON.stringify(outcome.result), + errorMessage: outcome.errorMessage ?? null, + }, + outcome.now, + ); + }) + .immediate(); +} + +/** No provider call Loop makes waits longer than this (the longest is a + * boot, at ten minutes in vm/sailbox-provider.ts). */ +export const VM_OPERATION_MAX_CALL_MS = 15 * 60_000; + +/** A receipt still pending after the longest provider call belongs to a + * caller that died between writing it and observing the answer, whichever + * process it was. Its outcome is unknown, so it becomes uncertain and is + * never sent again automatically. The worker runs this on every tick. */ +export function markStaleVmOperationsUncertain( + db: Database.Database, + now: number, +) { + let marked = 0; + for (const row of vmOperationsQueries.pendingBefore( + db, + now - VM_OPERATION_MAX_CALL_MS, + )) + if ( + transitionVmOperation( + db, + { owner: row.owner_id, id: row.id }, + "uncertain", + { + errorMessage: "Loop stopped before the provider answered", + now, + }, + ) + ) + marked += 1; + return marked; +} diff --git a/src/server/vm/operations.ts b/src/server/vm/operations.ts new file mode 100644 index 00000000..17601d60 --- /dev/null +++ b/src/server/vm/operations.ts @@ -0,0 +1,260 @@ +import { createHash } from "node:crypto"; +import type Database from "better-sqlite3"; +import type { VmOperationKind } from "../../shared/vm-state.js"; +import * as experimentVmsQueries from "../sql/experiment-vms.js"; +import * as vmOperationsQueries from "../sql/vm-operations.js"; +import { transitionVm, transitionVmOperation } from "./machine.js"; +import { VmProviderError, type VmProvider, type VmStatus } from "./provider.js"; + +// Every provider call that changes something runs through runVmOperation: a +// receipt is committed before the call and settled from what was observed. +// Nothing here calls the provider a second time. An unobserved outcome is +// kept as "uncertain" until the owner asks for a reconcile. + +export type VmOperationOutcome = + | { state: "succeeded"; result: unknown; replayed: boolean } + | { state: "failed"; errorMessage: string; replayed: boolean } + | { state: "uncertain" | "pending"; errorMessage: string | null }; + +export type VmOperationRefusal = + /** The id was used for a different request, or belongs to someone else. */ + | "key_conflict" + /** The experiment does not exist for this owner. */ + | "missing_experiment" + /** An earlier operation on this experiment is pending or uncertain. */ + | "unsettled_operation"; + +export class VmOperationRefused extends Error { + constructor(readonly reason: VmOperationRefusal) { + super(`VM operation refused: ${reason}`); + this.name = "VmOperationRefused"; + } +} + +/** SHA-256 of the request with object keys sorted, so the same request + * always has the same digest whatever order its fields were written in. */ +export function vmRequestDigest(request: unknown) { + return createHash("sha256").update(canonical(request)).digest("hex"); +} + +function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + if (value !== null && typeof value === "object") + return `{${Object.entries(value) + .filter(([, item]) => item !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, item]) => `${JSON.stringify(key)}:${canonical(item)}`) + .join(",")}}`; + return JSON.stringify(value ?? null); +} + +function stored( + row: NonNullable>, +): VmOperationOutcome { + if (row.state === "succeeded") + return { + state: "succeeded", + result: row.result_json === null ? null : JSON.parse(row.result_json), + replayed: true, + }; + if (row.state === "failed") + return { + state: "failed", + errorMessage: row.error_message ?? "The provider refused the request", + replayed: true, + }; + return { state: row.state, errorMessage: row.error_message }; +} + +/** Runs `call` at most once for `operation.id`. A repeat of the same id and + * request returns the stored outcome without calling the provider; the same + * id with a different request is refused. */ +export async function runVmOperation( + db: Database.Database, + operation: { + id: string; + owner: string; + experimentId: string; + kind: VmOperationKind; + request: unknown; + }, + call: () => Promise, + clock: () => number, +): Promise { + const requestDigest = vmRequestDigest({ + kind: operation.kind, + experimentId: operation.experimentId, + request: operation.request, + }); + const prior = db + .transaction(() => { + const existing = vmOperationsQueries.byId( + db, + operation.owner, + operation.id, + ); + if (existing) { + if (existing.request_digest !== requestDigest) + throw new VmOperationRefused("key_conflict"); + return existing; + } + if (vmOperationsQueries.idTaken(db, operation.id)) + throw new VmOperationRefused("key_conflict"); + if ( + !experimentVmsQueries.byId(db, operation.owner, operation.experimentId) + ) + throw new VmOperationRefused("missing_experiment"); + if ( + vmOperationsQueries.unsettledByExperiment( + db, + operation.owner, + operation.experimentId, + ).length > 0 + ) + throw new VmOperationRefused("unsettled_operation"); + vmOperationsQueries.insertPending(db, { + id: operation.id, + owner: operation.owner, + experimentId: operation.experimentId, + kind: operation.kind, + requestDigest, + now: clock(), + }); + return undefined; + }) + .immediate(); + if (prior) return stored(prior); + + const receipt = { owner: operation.owner, id: operation.id }; + try { + const result = await call(); + transitionVmOperation(db, receipt, "succeeded", { result, now: clock() }); + return { state: "succeeded", result, replayed: false }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + // Only an answer from the provider proves the call did nothing. Anything + // else, including a bug in the adapter, leaves the outcome unknown. + if (error instanceof VmProviderError && error.outcome === "refused") { + transitionVmOperation(db, receipt, "failed", { + errorMessage, + now: clock(), + }); + return { state: "failed", errorMessage, replayed: false }; + } + transitionVmOperation(db, receipt, "uncertain", { + errorMessage, + now: clock(), + }); + return { state: "uncertain", errorMessage }; + } +} + +const liveState = { + provisioning: undefined, + running: "running", + sleeping: "sleeping", + paused: "paused", + failed: "failed", + terminating: undefined, + terminated: "failed", + unknown: undefined, +} as const satisfies Record< + VmStatus, + "running" | "sleeping" | "paused" | "failed" | undefined +>; + +export type VmCreateReconciliation = + | { outcome: "adopted"; remoteId: string } + | { outcome: "absent" } + | { outcome: "still_uncertain"; reason: string }; + +/** Settles an uncertain create or fork by looking for the VM's name. This + * reads from the provider and never creates: one match is adopted, no match + * fails the operation so the owner can start a new one, and anything else + * leaves it uncertain. */ +export async function reconcileVmCreate( + db: Database.Database, + provider: Pick, + target: { owner: string; operationId: string; appId: string }, + clock: () => number, +): Promise { + const operation = vmOperationsQueries.byId( + db, + target.owner, + target.operationId, + ); + if ( + !operation || + operation.state !== "uncertain" || + (operation.kind !== "create" && operation.kind !== "fork") + ) + return { outcome: "still_uncertain", reason: "not an uncertain create" }; + const vm = experimentVmsQueries.byId( + db, + target.owner, + operation.experiment_id, + ); + if (!vm) return { outcome: "still_uncertain", reason: "experiment missing" }; + + let matches: Awaited>; + try { + matches = await provider.findByName(target.appId, vm.remote_name); + } catch (error) { + return { + outcome: "still_uncertain", + reason: error instanceof Error ? error.message : String(error), + }; + } + if (matches.length > 1) + return { + outcome: "still_uncertain", + reason: `${matches.length} VMs are named ${vm.remote_name}`, + }; + + const receipt = { owner: target.owner, id: operation.id }; + const subject = { owner: target.owner, id: vm.id }; + const [match] = matches; + if (!match) { + if ( + !transitionVmOperation(db, receipt, "failed", { + errorMessage: "The provider has no VM with this name", + now: clock(), + }) + ) + return { outcome: "still_uncertain", reason: "settled concurrently" }; + transitionVm(db, subject, "failed", { + expected: "provisioning", + lastError: "The VM was not created", + now: clock(), + }); + return { outcome: "absent" }; + } + const adopted = db + .transaction(() => { + if ( + !experimentVmsQueries.adoptRemote( + db, + target.owner, + vm.id, + match.remoteId, + ) + ) + return false; + return transitionVmOperation(db, receipt, "succeeded", { + result: { remoteId: match.remoteId }, + now: clock(), + }); + }) + .immediate(); + if (!adopted) + return { outcome: "still_uncertain", reason: "settled concurrently" }; + const state = liveState[match.status]; + if (state) + transitionVm(db, subject, state, { + expected: "provisioning", + lastError: + state === "failed" ? "The provider reports the VM failed" : null, + now: clock(), + }); + return { outcome: "adopted", remoteId: match.remoteId }; +} diff --git a/src/server/vm/provider.ts b/src/server/vm/provider.ts new file mode 100644 index 00000000..ba3350d5 --- /dev/null +++ b/src/server/vm/provider.ts @@ -0,0 +1,118 @@ +// The boundary between Loop and a VM provider. Loop calls a provider only +// after writing a vm_operations receipt (vm/operations.ts); adapters do not +// retry, so an unobserved outcome reaches the caller as VmProviderError with +// outcome "uncertain". + +/** `unknown` is a provider status this build does not recognise; callers + * leave the VM's state alone when they see it. */ +export type VmStatus = + | "provisioning" + | "running" + | "sleeping" + | "paused" + | "failed" + | "terminating" + | "terminated" + | "unknown"; + +export type VmLimits = { + vcpuCount: number; + memoryLimitGib: number; + diskLimitGib: number; +}; + +export type VmVolumeMount = { + volumeId: string; + mountPath: string; + readOnly?: boolean; +}; + +export interface VmProcessHandle { + stdin: NodeJS.WritableStream; + stdout: NodeJS.ReadableStream; + stderr: NodeJS.ReadableStream; + exit: Promise<{ exitCode: number | null; signal: string | null }>; + kill(): Promise; +} + +/** `refused`: the provider answered and did not perform the call. + * `uncertain`: no answer was observed (timeout, disconnect, 5xx), so the call + * may have taken effect and must be reconciled before any retry. */ +export class VmProviderError extends Error { + constructor( + readonly outcome: "refused" | "uncertain", + message: string, + readonly status?: number, + ) { + super(message); + this.name = "VmProviderError"; + } +} + +export interface VmProvider { + readonly kind: "sailbox"; + /** Checks the key and returns the account it belongs to. */ + whoami(): Promise<{ accountId: string }>; + ensureApp(name: string): Promise<{ appId: string }>; + ensureVolume(name: string): Promise<{ volumeId: string }>; + /** Pass a key scoped to the experiment, not to one operation: a retry + * after a reconcile found no VM then reaches the VM a slow first attempt + * made instead of creating a second one. The same applies to + * createFromCheckpoint. */ + create(input: { + idempotencyKey: string; + appId: string; + name: string; + imageRef: string; + limits: VmLimits; + autoSleepSeconds: number; + volumes: VmVolumeMount[]; + }): Promise<{ remoteId: string; status: VmStatus }>; + get(remoteId: string): Promise<{ status: VmStatus } | null>; + /** Live VMs with exactly this name. More than one means Loop cannot tell + * which to adopt, and the caller must leave the operation uncertain. */ + findByName( + appId: string, + name: string, + ): Promise<{ remoteId: string; status: VmStatus }[]>; + exec( + remoteId: string, + command: string[], + options: { + cwd?: string; + env?: Record; + openStdin?: boolean; + }, + ): Promise; + sleep(remoteId: string, idempotencyKey: string): Promise; + resume(remoteId: string, idempotencyKey: string): Promise; + checkpointFilesystem(input: { + idempotencyKey: string; + remoteId: string; + name: string; + ttlSeconds: number; + }): Promise<{ checkpointId: string }>; + /** The new VM keeps the checkpoint's disk, limits and volume mounts; + * `configure` applies the experiment's own limits and idle window. */ + createFromCheckpoint(input: { + idempotencyKey: string; + checkpointId: string; + name: string; + }): Promise<{ remoteId: string; status: VmStatus }>; + /** Sets limits and the idle window. Repeating it with the same values is + * harmless, so it needs no receipt of its own. */ + configure( + remoteId: string, + settings: { limits: VmLimits; autoSleepSeconds: number }, + ): Promise; + exposeHttpPort( + remoteId: string, + port: number, + idempotencyKey: string, + ): Promise<{ url: string }>; + unexposePort(remoteId: string, port: number): Promise; + readFile(remoteId: string, path: string, maxBytes: number): Promise; + writeFile(remoteId: string, path: string, bytes: Buffer): Promise; + /** Succeeds when the VM is already gone. */ + terminate(remoteId: string, idempotencyKey: string): Promise; +} diff --git a/src/server/vm/sailbox-image.ts b/src/server/vm/sailbox-image.ts new file mode 100644 index 00000000..3742d8fc --- /dev/null +++ b/src/server/vm/sailbox-image.ts @@ -0,0 +1,105 @@ +import { createHash } from "node:crypto"; + +// The experiment image. Sail builds custom images only through its SDK, which +// Loop does not bundle, so Loop builds a "golden" Sailbox instead: boot +// SAILBOX_BASE_IMAGE with the auth volume mounted, run INSTALL_SCRIPT over +// exec, take a filesystem-only checkpoint, and start every experiment from +// that checkpoint. Volume contents are not part of a checkpoint, so the +// golden image never holds credentials. The checkpoint id is saved as +// vm_provider_configs.base_image_ref with SAILBOX_IMAGE_VERSION, and a +// version change means the image is rebuilt. +// +// Install sources (checked 2026-11-20): Claude Code +// https://code.claude.com/docs/en/setup, Codex https://github.com/openai/codex, +// Antigravity https://antigravity.google/docs/getting-started?tab=cli, Pi +// https://github.com/earendil-works/pi. The installers fetch their latest +// releases; the image version pins the script, not the harness versions. + +export const AUTH_VOLUME_NAME = "loop-auth"; +export const AUTH_MOUNT_PATH = "/mnt/loop-auth"; +export const WORKSPACE_PATH = "/workspace"; +export const SAIL_APP_NAME = "loop-experiments"; +/** Golden checkpoints stay usable for a year; Sail's default is seven days. */ +export const GOLDEN_CHECKPOINT_TTL_SECONDS = 365 * 24 * 60 * 60; + +/** Credential files, relative to the home directory, kept on the auth volume + * under the harness's directory. Everything else a harness writes, including + * its session transcripts, stays on the VM's own disk. */ +export const HARNESS_CREDENTIALS = { + claude: [".claude/.credentials.json"], + codex: [".codex/auth.json"], + antigravity: [ + ".gemini/oauth_creds.json", + ".gemini/google_accounts.json", + ".gemini/antigravity-cli/antigravity-oauth-token", + ], + pi: [".pi/agent/auth.json"], +} as const; + +export const INSTALL_SCRIPT = `#!/bin/bash +set -euxo pipefail +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y --no-install-recommends \\ + git gh curl ca-certificates ripgrep jq sqlite3 build-essential pkg-config \\ + util-linux openssh-client unzip xz-utils python3 python3-venv python3-pip golang +# Docker ships with the base image; fail early if it does not. +command -v docker +curl -fsSL https://deb.nodesource.com/setup_22.x | bash - +apt-get install -y nodejs +npm install -g pnpm @openai/codex +npm install -g --ignore-scripts @earendil-works/pi-coding-agent +curl -fsSL https://bun.sh/install | bash +curl -LsSf https://astral.sh/uv/install.sh | sh +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal +curl -fsSL https://claude.ai/install.sh | bash +curl -fsSL https://antigravity.google/cli/install.sh | bash +cat > /etc/profile.d/loop-paths.sh <<'PATHS' +export PATH="$HOME/.local/bin:$HOME/.bun/bin:$HOME/.cargo/bin:$PATH" +PATHS +# Claude Code asks onboarding questions until this is recorded; the file is +# local to each VM and holds no credential. +[ -f "$HOME/.claude.json" ] || echo '{"hasCompletedOnboarding":true}' > "$HOME/.claude.json" +install -m 0755 /dev/stdin /usr/local/bin/loop-vm-init <<'INIT' +${initScript()} +INIT +for harness in claude codex agy pi; do + PATH="$HOME/.local/bin:$PATH" command -v "$harness" +done +`; + +/** Runs on every experiment before its first turn. It only creates links and + * the workspace repository, so running it again changes nothing. */ +export function initScript() { + const links = Object.entries(HARNESS_CREDENTIALS) + .flatMap(([harness, files]) => + files.map((file) => `link ${harness} ${file}`), + ) + .join("\n"); + return `#!/bin/bash +set -euo pipefail +auth="\${LOOP_AUTH_MOUNT:-${AUTH_MOUNT_PATH}}" +workspace="\${LOOP_WORKSPACE:-${WORKSPACE_PATH}}" +link() { + local target="$auth/$1/$(basename "$2")" path="$HOME/$2" + mkdir -p "$auth/$1" "$(dirname "$path")" + chmod 700 "$auth/$1" + # A harness that wrote a credential before the link existed keeps it. + if [ -f "$path" ] && [ ! -L "$path" ] && [ ! -e "$target" ]; then + mv "$path" "$target" + fi + ln -sfn "$target" "$path" +} +${links} +mkdir -p "$workspace" +cd "$workspace" +if [ ! -d .git ]; then + git init -q -b main + git -c user.name=Loop -c user.email=loop@localhost commit -q --allow-empty -m "Initial experiment workspace" +fi`; +} + +export const SAILBOX_IMAGE_VERSION = createHash("sha256") + .update(INSTALL_SCRIPT) + .digest("hex") + .slice(0, 16); diff --git a/src/server/vm/sailbox-provider.ts b/src/server/vm/sailbox-provider.ts new file mode 100644 index 00000000..a4692d49 --- /dev/null +++ b/src/server/vm/sailbox-provider.ts @@ -0,0 +1,527 @@ +import { PassThrough, Writable } from "node:stream"; +import { + VmProviderError, + type VmLimits, + type VmProcessHandle, + type VmProvider, + type VmStatus, +} from "./provider.js"; + +// Sailboxes over their HTTPS API (https://docs.sailresearch.com/sailboxes-http-api). +// Loop does not bundle @sailresearch/sdk, which ships native binaries. +// Nothing here retries: each method makes one request and reports what was +// observed, so vm/operations.ts can keep an unanswered call uncertain. + +export const SAILBOX_API = "https://sailbox-api.sailresearch.com/v1"; +export const SAIL_APPS_API = "https://api.sailresearch.com/v1"; + +/** Base image the golden experiment image is installed on: Debian with + * systemd and Docker already running (see vm/sailbox-image.ts). */ +export const SAILBOX_BASE_IMAGE = "BASE_IMAGE_DEVBOX"; + +const CONTROL_TIMEOUT_MS = 60_000; +// Create and start-from-checkpoint wait for the VM to boot. +const BOOT_TIMEOUT_MS = 10 * 60_000; +const MAX_LIST_PAGES = 20; + +type Fetch = typeof fetch; + +const statuses: Record = { + creating: "provisioning", + running: "running", + sleeping: "sleeping", + paused: "paused", + interrupted_restorable: "paused", + terminating: "terminating", + terminated: "terminated", + failed: "failed", + create_failed: "failed", + interrupted_unsafe_to_retry: "failed", +}; +/** Sail documents its status list as open, so a new value is `unknown`. */ +export function sailboxStatus(status: unknown): VmStatus { + return (typeof status === "string" && statuses[status]) || "unknown"; +} + +function message(body: unknown, fallback: string) { + const error = + body && typeof body === "object" && "error" in body + ? (body as { error?: { message?: unknown } }).error + : undefined; + return typeof error?.message === "string" ? error.message : fallback; +} + +// A 4xx answer, including 429 (Sail documents that the request never ran), +// means nothing happened. 501 is a definite refusal too. Any other 5xx, a +// timeout or a dropped connection may follow a change Sail already made. +function refusedStatus(status: number) { + return (status >= 400 && status < 500) || status === 501; +} + +export function createSailboxProvider(options: { + apiKey: string; + fetch?: Fetch; + apiBase?: string; + appsBase?: string; +}): VmProvider { + const send = options.fetch ?? fetch; + const api = options.apiBase ?? SAILBOX_API; + const apps = options.appsBase ?? SAIL_APPS_API; + + async function request( + method: string, + url: string, + init: { + body?: unknown; + raw?: Buffer; + idempotencyKey?: string; + timeoutMs?: number; + accept404?: boolean; + } = {}, + ): Promise { + const headers: Record = { + authorization: `Bearer ${options.apiKey}`, + }; + if (init.idempotencyKey) headers["idempotency-key"] = init.idempotencyKey; + let body: BodyInit | undefined; + if (init.raw !== undefined) { + headers["content-type"] = "application/octet-stream"; + body = new Uint8Array(init.raw); + } else if (init.body !== undefined) { + headers["content-type"] = "application/json"; + body = JSON.stringify(init.body); + } + let response: Response; + try { + response = await send(url, { + method, + headers, + body, + signal: AbortSignal.timeout(init.timeoutMs ?? CONTROL_TIMEOUT_MS), + }); + } catch (error) { + throw new VmProviderError( + "uncertain", + `Sailboxes ${method} did not answer: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if (response.ok) return response; + if (response.status === 404 && init.accept404) return null; + const text = await response.text().catch(() => ""); + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + parsed = undefined; + } + throw new VmProviderError( + refusedStatus(response.status) ? "refused" : "uncertain", + message(parsed, `Sailboxes answered ${response.status}`), + response.status, + ); + } + + async function json( + method: string, + url: string, + init?: Parameters[2], + ): Promise> { + const response = await request(method, url, init); + if (!response) return {}; + try { + const value: unknown = await response.json(); + if (value && typeof value === "object") + return value as Record; + } catch { + // fall through: an answer Loop cannot read is not an observed outcome + } + throw new VmProviderError( + "uncertain", + `Sailboxes ${method} answered with a body Loop could not read`, + ); + } + + const box = (remoteId: string) => + `${api}/sailboxes/${encodeURIComponent(remoteId)}`; + + function field(value: Record, name: string) { + const item = value[name]; + if (typeof item !== "string" || item.length === 0) + throw new VmProviderError( + "uncertain", + `Sailboxes answered without ${name}`, + ); + return item; + } + + // A create that answers 200 with status "failed" did not leave a VM that + // can run; Sail reports the reason in error_message. + function created(value: Record) { + const remoteId = field(value, "sailbox_id"); + return { remoteId, status: sailboxStatus(value.status) }; + } + + return { + kind: "sailbox", + + async whoami() { + const value = await json("GET", `${api}/whoami`); + return { accountId: field(value, "org_id") }; + }, + + async ensureApp(name) { + const value = await json("POST", `${apps}/apps/find`, { + body: { name, mint_if_missing: true }, + }); + return { appId: field(value, "id") }; + }, + + async ensureVolume(name) { + // Creating a volume that already exists returns it. + const value = await json("POST", `${api}/sailbox-volumes`, { + body: { name }, + idempotencyKey: `loop-volume-${name}`, + }); + return { volumeId: field(value, "volume_id") }; + }, + + async create(input) { + return created( + await json("POST", `${api}/sailboxes`, { + idempotencyKey: input.idempotencyKey, + timeoutMs: BOOT_TIMEOUT_MS, + body: { + app_id: input.appId, + name: input.name, + image: { base: input.imageRef }, + visibility: "private", + vcpu_count: input.limits.vcpuCount, + memory_limit_gib: input.limits.memoryLimitGib, + state_disk_limit_gib: input.limits.diskLimitGib, + auto_sleep: { + automatic: true, + min_seconds_before_sleep: input.autoSleepSeconds, + }, + volume_mounts: input.volumes.map((volume) => ({ + volume_id: volume.volumeId, + mount_path: volume.mountPath, + read_only: volume.readOnly ?? false, + })), + }, + }), + ); + }, + + async get(remoteId) { + const response = await request("GET", box(remoteId), { + accept404: true, + }); + if (!response) return null; + const value = (await response.json()) as Record; + return { status: sailboxStatus(value.status) }; + }, + + async findByName(appId, name) { + const matches: { remoteId: string; status: VmStatus }[] = []; + for (let page = 0; page < MAX_LIST_PAGES; page += 1) { + const query = new URLSearchParams({ + app: appId, + name, + exclude_status: "terminated", + limit: "100", + offset: String(page * 100), + }); + const value = await json("GET", `${api}/sailboxes?${query}`); + const data = Array.isArray(value.data) ? value.data : []; + for (const item of data as Record[]) + if (item.name === name) + matches.push({ + remoteId: field(item, "sailbox_id"), + status: sailboxStatus(item.status), + }); + if (value.has_more !== true) return matches; + } + // An incomplete list cannot prove a VM is absent. + throw new VmProviderError( + "uncertain", + "Sailboxes listed more pages than Loop reads", + ); + }, + + exec(remoteId, command, execOptions) { + return startExec(request, box(remoteId), command, execOptions); + }, + + async sleep(remoteId, idempotencyKey) { + await request("POST", `${box(remoteId)}/sleep`, { + body: {}, + idempotencyKey, + }); + }, + + async resume(remoteId, idempotencyKey) { + const value = await json("POST", `${box(remoteId)}/resume`, { + body: {}, + idempotencyKey, + timeoutMs: BOOT_TIMEOUT_MS, + }); + if (value.resume_state === "terminal_unavailable") + throw new VmProviderError( + "refused", + message(value, "The Sailbox can no longer be resumed"), + ); + }, + + async checkpointFilesystem(input) { + const value = await json("POST", `${box(input.remoteId)}/checkpoint`, { + idempotencyKey: input.idempotencyKey, + timeoutMs: BOOT_TIMEOUT_MS, + body: { + include_memory: false, + name: input.name, + ttl_seconds: input.ttlSeconds, + }, + }); + return { checkpointId: field(value, "checkpoint_id") }; + }, + + async createFromCheckpoint(input) { + return created( + await json("POST", `${api}/sailboxes/from_checkpoint`, { + idempotencyKey: input.idempotencyKey, + timeoutMs: BOOT_TIMEOUT_MS, + body: { checkpoint_id: input.checkpointId, name: input.name }, + }), + ); + }, + + async configure( + remoteId, + settings: { limits: VmLimits; autoSleepSeconds: number }, + ) { + await request("POST", `${box(remoteId)}/set-limits`, { + body: { + vcpu_count: settings.limits.vcpuCount, + memory_limit_gib: settings.limits.memoryLimitGib, + state_disk_limit_gib: settings.limits.diskLimitGib, + }, + }); + await request("POST", `${box(remoteId)}/auto_sleep`, { + body: { + automatic: true, + min_seconds_before_sleep: settings.autoSleepSeconds, + }, + }); + }, + + async exposeHttpPort(remoteId, port, idempotencyKey) { + const value = await json("POST", `${box(remoteId)}/listeners`, { + idempotencyKey, + body: { guest_port: port, protocol: "http" }, + }); + return { url: field(value, "public_url") }; + }, + + async unexposePort(remoteId, port) { + await request("DELETE", `${box(remoteId)}/listeners/${port}`, { + accept404: true, + }); + }, + + async readFile(remoteId, path, maxBytes) { + const response = await request( + "GET", + `${box(remoteId)}/files?${new URLSearchParams({ path })}`, + ); + if (!response?.body) + throw new VmProviderError("refused", `No file at ${path}`); + const declared = Number(response.headers.get("content-length")); + if (Number.isFinite(declared) && declared > maxBytes) { + await response.body.cancel(); + throw new VmProviderError( + "refused", + `${path} is larger than ${maxBytes} bytes`, + ); + } + const chunks: Buffer[] = []; + let total = 0; + for await (const chunk of response.body) { + total += chunk.byteLength; + if (total > maxBytes) + throw new VmProviderError( + "refused", + `${path} is larger than ${maxBytes} bytes`, + ); + chunks.push(Buffer.from(chunk)); + } + return Buffer.concat(chunks); + }, + + async writeFile(remoteId, path, bytes) { + await request( + "PUT", + `${box(remoteId)}/files?${new URLSearchParams({ path, mode: "420" })}`, + { raw: bytes }, + ); + }, + + async terminate(remoteId, idempotencyKey) { + // Terminating a Sailbox that is already gone succeeds. + await request("POST", `${box(remoteId)}/terminate`, { + body: {}, + idempotencyKey, + accept404: true, + }); + }, + }; +} + +// Sail applies `cwd` only to string commands. Loop passes argument arrays so +// nothing is shell-parsed, so the working directory is set by a fixed script +// that receives the directory and the command as separate arguments. +export function sailboxArgv(command: string[], cwd: string | undefined) { + if (cwd === undefined) return command; + return [ + "/bin/sh", + "-c", + 'cd -- "$1" || exit 127; shift; exec "$@"', + "loop-exec", + cwd, + ...command, + ]; +} + +type Request = ( + method: string, + url: string, + init?: { + body?: unknown; + raw?: Buffer; + timeoutMs?: number; + }, +) => Promise; + +/** Starts a command and returns its streams. stdout and stderr come from the + * NDJSON event stream; stdin writes are PUTs at absolute offsets. A dropped + * event stream ends both outputs and resolves `exit` with no code. */ +async function startExec( + request: Request, + boxUrl: string, + command: string[], + options: { cwd?: string; env?: Record; openStdin?: boolean }, +): Promise { + const response = await request("POST", `${boxUrl}/exec`, { + timeoutMs: 24 * 60 * 60_000, + body: { + command: sailboxArgv(command, options.cwd), + env: options.env ?? {}, + open_stdin: options.openStdin ?? false, + close_stdin_on_disconnect: true, + }, + }); + if (!response?.body) + throw new VmProviderError("uncertain", "Sailboxes exec sent no stream"); + const stdout = new PassThrough(); + const stderr = new PassThrough(); + let execId: string | undefined; + let started!: () => void; + const ready = new Promise((resolve) => (started = resolve)); + const body = response.body; + + const exit = (async () => { + let result: { exitCode: number | null; signal: string | null } = { + exitCode: null, + signal: null, + }; + let buffered = ""; + const decoder = new TextDecoder(); + const handle = (line: string) => { + if (!line.trim()) return; + let event: Record; + try { + event = JSON.parse(line) as Record; + } catch { + return; + } + if ( + event.type === "started" && + typeof event.exec_request_id === "string" + ) { + execId = event.exec_request_id; + started(); + } else if ( + (event.type === "stdout" || event.type === "stderr") && + typeof event.data === "string" + ) + (event.type === "stdout" ? stdout : stderr).write( + Buffer.from(event.data, "base64"), + ); + else if (event.type === "exit") + result = { + exitCode: + typeof event.return_code === "number" ? event.return_code : null, + signal: typeof event.signal === "string" ? event.signal : null, + }; + }; + try { + for await (const chunk of body) { + buffered += decoder.decode(chunk, { stream: true }); + let newline = buffered.indexOf("\n"); + while (newline >= 0) { + handle(buffered.slice(0, newline)); + buffered = buffered.slice(newline + 1); + newline = buffered.indexOf("\n"); + } + } + handle(buffered); + } catch { + // The stream dropped; the command's outcome is unknown. + } finally { + started(); + stdout.end(); + stderr.end(); + } + return result; + })(); + + let offset = 0; + const putStdin = async (bytes: Buffer, eof: boolean) => { + await ready; + if (!execId) throw new Error("The command ended before it started"); + const query = new URLSearchParams({ + offset: String(offset), + ...(eof ? { eof: "true" } : {}), + }); + await request( + "PUT", + `${boxUrl}/exec/-/stdin?exec_request_id=${encodeURIComponent(execId)}&${query}`, + { raw: bytes }, + ); + offset += bytes.byteLength; + }; + const stdin = new Writable({ + write(chunk: Buffer, _encoding, callback) { + putStdin(Buffer.from(chunk), false).then(() => callback(), callback); + }, + final(callback) { + if (!options.openStdin) return callback(); + putStdin(Buffer.alloc(0), true).then(() => callback(), callback); + }, + }); + + return { + stdin, + stdout, + stderr, + exit, + async kill() { + await ready; + if (!execId) return; + await request( + "POST", + `${boxUrl}/exec/-/cancel?exec_request_id=${encodeURIComponent(execId)}`, + { body: { force: true } }, + ); + }, + }; +} diff --git a/src/server/worker.ts b/src/server/worker.ts index 75b4cec8..32252d37 100644 --- a/src/server/worker.ts +++ b/src/server/worker.ts @@ -34,6 +34,7 @@ import { } from "./queue.js"; import { RunDeferred } from "./run-deferred.js"; import { recoverStuckRuns, sweepLeftoverToolCalls } from "./recover-stuck.js"; +import { markStaleVmOperationsUncertain } from "./vm/machine.js"; import { harnessHolds } from "./harness-holds.js"; import { nativeSessions } from "./native-sessions.js"; import { observeProcess } from "./deployments.js"; @@ -62,6 +63,7 @@ export async function tick( assertExecutionGeneration(workerId); recoverStale(); recoverStuckRuns(); + markStaleVmOperationsUncertain(db, now()); reconcileWorkerCheckouts(); const run = claim(workerId, (candidate) => { if (!accepting()) return false; diff --git a/src/shared/vm-state.ts b/src/shared/vm-state.ts new file mode 100644 index 00000000..db738733 --- /dev/null +++ b/src/shared/vm-state.ts @@ -0,0 +1,68 @@ +import { z } from "zod"; + +// The single owner of the experiment_vms.state vocabulary and its edges. +// transitionVm() in src/server/vm/machine.ts is the only writer. +export const VM_TRANSITIONS = { + provisioning: ["running", "failed", "destroying"], + running: ["sleeping", "paused", "destroying", "failed"], + sleeping: ["running", "destroying", "failed"], + paused: ["running", "destroying", "failed"], + // Only the owner's explicit reconcile, retry or destroy leaves failed. + failed: ["provisioning", "running", "destroying"], + destroying: ["destroyed", "failed"], + // Terminal: the project, its threads, results and files stay readable. + destroyed: [], +} as const; +export type VmState = keyof typeof VM_TRANSITIONS; +export const VmState = z.enum( + Object.keys(VM_TRANSITIONS) as [VmState, ...VmState[]], +); + +export function legalVmTransition(from: VmState, to: VmState) { + const destinations: readonly VmState[] = VM_TRANSITIONS[from]; + return destinations.includes(to); +} + +// vm_operations.state. pending is written before the provider call. A +// pending row found at startup, or a call whose outcome was not observed, +// becomes uncertain; only the owner's reconcile settles it. +export const VM_OPERATION_TRANSITIONS = { + pending: ["succeeded", "failed", "uncertain"], + uncertain: ["succeeded", "failed"], + succeeded: [], + failed: [], +} as const; +export type VmOperationState = keyof typeof VM_OPERATION_TRANSITIONS; +export const VmOperationState = z.enum( + Object.keys(VM_OPERATION_TRANSITIONS) as [ + VmOperationState, + ...VmOperationState[], + ], +); + +export function legalVmOperationTransition( + from: VmOperationState, + to: VmOperationState, +) { + const destinations: readonly VmOperationState[] = + VM_OPERATION_TRANSITIONS[from]; + return destinations.includes(to); +} + +export const VmOperationKinds = [ + "create", + "fork", + "sleep", + "resume", + "expose", + "unexpose", + "destroy", +] as const; +export const VmOperationKind = z.enum(VmOperationKinds); +export type VmOperationKind = z.infer; + +export const VmProviderKind = z.enum(["sailbox"]); +export type VmProviderKind = z.infer; + +export const VmHarness = z.enum(["claude", "codex", "antigravity", "pi"]); +export type VmHarness = z.infer; diff --git a/test/vm-operations-worker.test.ts b/test/vm-operations-worker.test.ts new file mode 100644 index 00000000..df8fb13a --- /dev/null +++ b/test/vm-operations-worker.test.ts @@ -0,0 +1,57 @@ +import { testDirectory } from "./helpers/test-workspace.js"; +import { afterAll, expect, it } from "vitest"; + +testDirectory(); + +const { db, now } = await import("../src/server/db"); +const { tick } = await import("../src/server/worker"); +const experimentVmsQueries = await import("../src/server/sql/experiment-vms"); +const vmOperationsQueries = await import("../src/server/sql/vm-operations"); +const { VM_OPERATION_MAX_CALL_MS } = await import("../src/server/vm/machine"); + +afterAll(() => db.close()); + +// The worker tick is the production caller of the stale-receipt sweep. +it("marks a VM operation receipt left pending by a dead caller uncertain on the next worker tick", async () => { + db.prepare( + "INSERT INTO folders(id,owner_id,kind,name,created_at) VALUES('folder-w','owner-w','project','w',1)", + ).run(); + experimentVmsQueries.insert(db, { + id: "exp-w", + owner: "owner-w", + folderId: "folder-w", + provider: "sailbox", + remoteName: "loop-exp-w", + autoApproveLocal: true, + vcpuLimit: 4, + memoryLimitGib: 16, + diskLimitGib: 64, + autoSleepSeconds: 900, + sourceExperimentId: null, + sourceCheckpointId: null, + cloneUrl: null, + now: now(), + }); + for (const [id, age] of [ + ["stale", VM_OPERATION_MAX_CALL_MS + 60_000], + ["fresh", 1_000], + ] as const) + vmOperationsQueries.insertPending(db, { + id, + owner: "owner-w", + experimentId: "exp-w", + kind: "create", + requestDigest: "d", + now: now() - age, + }); + + await tick(); + + expect(vmOperationsQueries.byId(db, "owner-w", "stale")).toMatchObject({ + state: "uncertain", + error_message: "Loop stopped before the provider answered", + }); + expect(vmOperationsQueries.byId(db, "owner-w", "fresh")?.state).toBe( + "pending", + ); +}); diff --git a/test/vm-operations.test.ts b/test/vm-operations.test.ts new file mode 100644 index 00000000..90549923 --- /dev/null +++ b/test/vm-operations.test.ts @@ -0,0 +1,309 @@ +import Database from "better-sqlite3"; +import { beforeEach, expect, it } from "vitest"; +import { applyMigrations } from "../src/server/migrations.js"; +import * as experimentVmsQueries from "../src/server/sql/experiment-vms.js"; +import * as vmOperationsQueries from "../src/server/sql/vm-operations.js"; +import { + VM_OPERATION_MAX_CALL_MS, + markStaleVmOperationsUncertain, + transitionVm, +} from "../src/server/vm/machine.js"; +import { + reconcileVmCreate, + runVmOperation, +} from "../src/server/vm/operations.js"; +import { VmProviderError, type VmStatus } from "../src/server/vm/provider.js"; + +// Fixture coverage: an in-memory database with the production migrations and +// a scripted provider. No Sailbox is contacted. + +let db: Database.Database; +let time = 1_000; +const clock = () => ++time; + +function experiment(owner: string, id: string) { + db.prepare( + "INSERT INTO folders(id,owner_id,kind,name,created_at) VALUES(?,?,'project',?,1)", + ).run(`folder-${id}`, owner, id); + experimentVmsQueries.insert(db, { + id, + owner, + folderId: `folder-${id}`, + provider: "sailbox", + remoteName: `loop-exp-${id}`, + autoApproveLocal: true, + vcpuLimit: 4, + memoryLimitGib: 16, + diskLimitGib: 64, + autoSleepSeconds: 900, + sourceExperimentId: null, + sourceCheckpointId: null, + cloneUrl: null, + now: clock(), + }); +} + +const create = ( + id: string, + call: () => Promise, + overrides: { owner?: string; experimentId?: string; request?: unknown } = {}, +) => + runVmOperation( + db, + { + id, + owner: overrides.owner ?? "alice", + experimentId: overrides.experimentId ?? "exp1", + kind: "create", + request: overrides.request ?? { name: "loop-exp-exp1" }, + }, + call, + clock, + ); + +const found = (...matches: { remoteId: string; status: VmStatus }[]) => ({ + calls: 0, + async findByName() { + this.calls += 1; + return matches; + }, +}); + +// Migrating once and copying the image keeps each test's database fresh. +const migrated = (() => { + const template = new Database(":memory:"); + applyMigrations(template); + const image = template.serialize(); + template.close(); + return image; +})(); + +beforeEach(() => { + db = new Database(migrated); + db.pragma("foreign_keys=ON"); + experiment("alice", "exp1"); +}); + +it("calls the provider once and replays the stored result for the same id", async () => { + let calls = 0; + const call = async () => ({ remoteId: `sb-${++calls}` }); + expect(await create("op1", call)).toEqual({ + state: "succeeded", + result: { remoteId: "sb-1" }, + replayed: false, + }); + expect(await create("op1", call)).toEqual({ + state: "succeeded", + result: { remoteId: "sb-1" }, + replayed: true, + }); + expect(calls).toBe(1); +}); + +it("refuses an id reused for a different request without calling the provider", async () => { + await create("op1", async () => ({})); + let called = false; + await expect( + create( + "op1", + async () => { + called = true; + }, + { request: { name: "another" } }, + ), + ).rejects.toMatchObject({ reason: "key_conflict" }); + expect(called).toBe(false); +}); + +it("writes the pending receipt before the provider is called", async () => { + let seen: string | undefined; + await create("op1", async () => { + seen = vmOperationsQueries.byId(db, "alice", "op1")?.state; + return {}; + }); + expect(seen).toBe("pending"); +}); + +it("records a provider refusal as failed and an unobserved outcome as uncertain", async () => { + expect( + await create("op1", async () => { + throw new VmProviderError("refused", "quota exceeded", 402); + }), + ).toMatchObject({ state: "failed", errorMessage: "quota exceeded" }); + expect( + await create("op2", async () => { + throw new VmProviderError("uncertain", "timed out"); + }), + ).toEqual({ state: "uncertain", errorMessage: "timed out" }); +}); + +it("treats an error that is not a provider answer as uncertain", async () => { + expect( + await create("op1", async () => { + throw new TypeError("adapter bug"); + }), + ).toEqual({ state: "uncertain", errorMessage: "adapter bug" }); +}); + +it("does not call the provider again for an uncertain operation or start another beside it", async () => { + await create("op1", async () => { + throw new VmProviderError("uncertain", "socket closed"); + }); + let calls = 0; + const call = async () => { + calls += 1; + return {}; + }; + expect(await create("op1", call)).toEqual({ + state: "uncertain", + errorMessage: "socket closed", + }); + await expect(create("op2", call)).rejects.toMatchObject({ + reason: "unsettled_operation", + }); + expect(calls).toBe(0); +}); + +it("keeps another owner's experiment and operation ids out of reach", async () => { + await create("op1", async () => ({ remoteId: "sb-1" })); + let calls = 0; + const call = async () => { + calls += 1; + return {}; + }; + await expect(create("op9", call, { owner: "bob" })).rejects.toMatchObject({ + reason: "missing_experiment", + }); + experiment("bob", "exp2"); + await expect( + create("op1", call, { owner: "bob", experimentId: "exp2" }), + ).rejects.toMatchObject({ reason: "key_conflict" }); + expect(calls).toBe(0); + expect(vmOperationsQueries.byId(db, "bob", "op1")).toBeUndefined(); + expect( + transitionVm(db, { owner: "bob", id: "exp1" }, "running", { now: clock() }), + ).toEqual({ changed: false, reason: "missing" }); +}); + +it("marks a receipt uncertain only once no live call could still be waiting on it", () => { + vmOperationsQueries.insertPending(db, { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "create", + requestDigest: "d", + now: time, + }); + const created = time; + expect( + markStaleVmOperationsUncertain(db, created + VM_OPERATION_MAX_CALL_MS), + ).toBe(0); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("pending"); + expect( + markStaleVmOperationsUncertain(db, created + VM_OPERATION_MAX_CALL_MS + 1), + ).toBe(1); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("uncertain"); +}); + +it("lets a late answer settle a receipt the sweep marked uncertain", async () => { + let answer!: (value: unknown) => void; + const outcome = create( + "op1", + () => new Promise((resolve) => (answer = resolve)), + ); + await new Promise((resolve) => setImmediate(resolve)); + markStaleVmOperationsUncertain(db, time + VM_OPERATION_MAX_CALL_MS + 1); + answer({ remoteId: "sb-late" }); + expect(await outcome).toMatchObject({ state: "succeeded" }); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("succeeded"); +}); + +async function uncertainCreate() { + await create("op1", async () => { + throw new VmProviderError("uncertain", "timed out"); + }); +} +const reconcile = (provider: ReturnType, owner = "alice") => + reconcileVmCreate( + db, + provider, + { owner, operationId: "op1", appId: "app" }, + clock, + ); + +it("adopts the one VM that carries the experiment's name", async () => { + await uncertainCreate(); + expect( + await reconcile(found({ remoteId: "sb-7", status: "running" })), + ).toEqual({ outcome: "adopted", remoteId: "sb-7" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-7", + state: "running", + }); + expect(vmOperationsQueries.byId(db, "alice", "op1")).toMatchObject({ + state: "succeeded", + result_json: '{"remoteId":"sb-7"}', + }); +}); + +it("fails the operation when the provider has no such VM, allowing a new one", async () => { + await uncertainCreate(); + expect(await reconcile(found())).toEqual({ outcome: "absent" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe("failed"); + expect(await create("op2", async () => ({ remoteId: "sb-8" }))).toMatchObject( + { state: "succeeded" }, + ); +}); + +it("stays uncertain when the lookup fails, is ambiguous, or comes from another owner", async () => { + await uncertainCreate(); + expect( + await reconcileVmCreate( + db, + { + async findByName() { + throw new VmProviderError("uncertain", "list timed out"); + }, + }, + { owner: "alice", operationId: "op1", appId: "app" }, + clock, + ), + ).toEqual({ outcome: "still_uncertain", reason: "list timed out" }); + expect( + await reconcile( + found( + { remoteId: "sb-1", status: "running" }, + { remoteId: "sb-2", status: "sleeping" }, + ), + ), + ).toMatchObject({ outcome: "still_uncertain" }); + const other = found({ remoteId: "sb-1", status: "running" }); + expect(await reconcile(other, "bob")).toMatchObject({ + outcome: "still_uncertain", + }); + expect(other.calls).toBe(0); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("uncertain"); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.remote_id).toBeNull(); +}); + +it("refuses illegal and stale VM transitions", () => { + const vm = { owner: "alice", id: "exp1" }; + expect(transitionVm(db, vm, "sleeping", { now: clock() })).toEqual({ + changed: false, + reason: "illegal", + from: "provisioning", + }); + expect(transitionVm(db, vm, "running", { now: clock() }).changed).toBe(true); + expect( + transitionVm(db, vm, "destroying", { + expected: "sleeping", + now: clock(), + }), + ).toEqual({ changed: false, reason: "illegal", from: "running" }); + transitionVm(db, vm, "destroying", { now: clock() }); + transitionVm(db, vm, "destroyed", { now: clock() }); + const row = experimentVmsQueries.byId(db, "alice", "exp1"); + expect(row?.state).toBe("destroyed"); + expect(row?.destroyed_at).toBeGreaterThan(0); + expect(transitionVm(db, vm, "running", { now: clock() }).changed).toBe(false); +}); diff --git a/test/vm-sailbox-image.test.ts b/test/vm-sailbox-image.test.ts new file mode 100644 index 00000000..af3d06e1 --- /dev/null +++ b/test/vm-sailbox-image.test.ts @@ -0,0 +1,76 @@ +import { execFileSync } from "node:child_process"; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { + HARNESS_CREDENTIALS, + INSTALL_SCRIPT, + initScript, +} from "../src/server/vm/sailbox-image.js"; + +// Runs the experiment init script on this machine against temporary +// directories. The install script itself is only syntax-checked here; it has +// not been run on a Sailbox by this test. + +function run(home: string, auth: string, workspace: string) { + const script = join(home, "loop-vm-init"); + writeFileSync(script, initScript(), { mode: 0o755 }); + execFileSync("bash", [script], { + env: { + PATH: process.env.PATH, + HOME: home, + LOOP_AUTH_MOUNT: auth, + LOOP_WORKSPACE: workspace, + GIT_CONFIG_GLOBAL: "/dev/null", + }, + }); +} + +it("links only credential files to the auth volume and initialises the workspace", () => { + const root = mkdtempSync(join(tmpdir(), "loop-vm-init-")); + const [home, auth, workspace] = ["home", "auth", "workspace"].map((name) => + join(root, name), + ); + mkdirSync(home); + mkdirSync(auth); + mkdirSync(join(home, ".codex")); + writeFileSync(join(home, ".codex/auth.json"), "earlier"); + mkdirSync(join(home, ".claude/projects"), { recursive: true }); + + run(home, auth, workspace); + run(home, auth, workspace); + + for (const [harness, files] of Object.entries(HARNESS_CREDENTIALS)) + for (const file of files) { + const path = join(home, file); + expect(lstatSync(path).isSymbolicLink()).toBe(true); + expect(readlinkSync(path).startsWith(join(auth, harness))).toBe(true); + } + // A credential written before the volume was linked moves onto it. + expect(readFileSync(join(auth, "codex/auth.json"), "utf8")).toBe("earlier"); + // Session directories stay on the VM's own disk. + expect(lstatSync(join(home, ".claude/projects")).isSymbolicLink()).toBe( + false, + ); + expect(existsSync(join(auth, "claude/projects"))).toBe(false); + const log = execFileSync("git", ["log", "--format=%s"], { + cwd: workspace, + encoding: "utf8", + }); + expect(log.trim()).toBe("Initial experiment workspace"); +}); + +it("produces an install script bash accepts", () => { + const dir = mkdtempSync(join(tmpdir(), "loop-vm-install-")); + writeFileSync(join(dir, "install.sh"), INSTALL_SCRIPT); + execFileSync("bash", ["-n", join(dir, "install.sh")]); +}); diff --git a/test/vm-sailbox-provider.test.ts b/test/vm-sailbox-provider.test.ts new file mode 100644 index 00000000..9fce66a7 --- /dev/null +++ b/test/vm-sailbox-provider.test.ts @@ -0,0 +1,298 @@ +import { expect, it } from "vitest"; +import { VmProviderError } from "../src/server/vm/provider.js"; +import { + createSailboxProvider, + sailboxArgv, + sailboxStatus, +} from "../src/server/vm/sailbox-provider.js"; + +// Fixture coverage: a scripted fetch stands in for the Sailboxes API, with +// request and response shapes taken from Sail's published OpenAPI document. +// No Sailbox is contacted. + +type Call = { method: string; url: URL; headers: Headers; body: unknown }; + +function fake(answer: (call: Call) => Response | Promise | Error) { + const calls: Call[] = []; + const fetch = (async (input: string | URL, init?: RequestInit) => { + const raw = init?.body; + const call: Call = { + method: init?.method ?? "GET", + url: new URL(String(input)), + headers: new Headers(init?.headers), + body: + typeof raw === "string" + ? JSON.parse(raw) + : raw instanceof Uint8Array + ? Buffer.from(raw).toString() + : undefined, + }; + calls.push(call); + const result = await answer(call); + if (result instanceof Error) throw result; + return result; + }) as typeof globalThis.fetch; + return { + calls, + provider: createSailboxProvider({ apiKey: "sail-secret", fetch }), + }; +} + +const ok = (body: unknown) => Response.json(body); +const failure = (status: number, text = "nope") => + Response.json( + { error: { message: text, type: "x", param: null, code: null } }, + { status }, + ); + +it("creates a private Sailbox with the operation's idempotency key", async () => { + const { calls, provider } = fake(() => + ok({ sailbox_id: "sb_1", status: "running" }), + ); + expect( + await provider.create({ + idempotencyKey: "op-1", + appId: "app_1", + name: "loop-exp-abc", + imageRef: "BASE_IMAGE_DEVBOX", + limits: { vcpuCount: 4, memoryLimitGib: 16, diskLimitGib: 64 }, + autoSleepSeconds: 900, + volumes: [{ volumeId: "vol_1", mountPath: "/mnt/loop-auth" }], + }), + ).toEqual({ remoteId: "sb_1", status: "running" }); + const [call] = calls; + expect(call.method).toBe("POST"); + expect(call.url.href).toBe( + "https://sailbox-api.sailresearch.com/v1/sailboxes", + ); + expect(call.headers.get("authorization")).toBe("Bearer sail-secret"); + expect(call.headers.get("idempotency-key")).toBe("op-1"); + expect(call.body).toEqual({ + app_id: "app_1", + name: "loop-exp-abc", + image: { base: "BASE_IMAGE_DEVBOX" }, + visibility: "private", + vcpu_count: 4, + memory_limit_gib: 16, + state_disk_limit_gib: 64, + auto_sleep: { automatic: true, min_seconds_before_sleep: 900 }, + volume_mounts: [ + { volume_id: "vol_1", mount_path: "/mnt/loop-auth", read_only: false }, + ], + }); + expect(JSON.stringify(call.body)).not.toContain("sail-secret"); +}); + +it("reports a 4xx answer as refused and an unanswered or 5xx call as uncertain", async () => { + const cases: [Response | Error, "refused" | "uncertain"][] = [ + [failure(402, "billing"), "refused"], + [failure(409), "refused"], + [failure(429), "refused"], + [failure(501), "refused"], + [failure(500), "uncertain"], + [failure(503), "uncertain"], + [new TypeError("fetch failed"), "uncertain"], + [new Response("not json"), "uncertain"], + ]; + for (const [answer, outcome] of cases) { + const { provider } = fake(() => answer); + const error = await provider + .ensureApp("loop-experiments") + .catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(VmProviderError); + expect((error as VmProviderError).outcome).toBe(outcome); + } + const { provider } = fake(() => failure(402, "Add a payment method")); + await expect(provider.whoami()).rejects.toThrow("Add a payment method"); +}); + +it("finds live Sailboxes by exact name across pages, and refuses to guess past its page limit", async () => { + const { calls, provider } = fake(({ url }) => + url.searchParams.get("offset") === "0" + ? ok({ + data: [ + { sailbox_id: "sb_1", name: "loop-exp-abc", status: "sleeping" }, + { sailbox_id: "sb_x", name: "loop-exp-abcd", status: "running" }, + ], + has_more: true, + }) + : ok({ + data: [ + { sailbox_id: "sb_2", name: "loop-exp-abc", status: "creating" }, + ], + has_more: false, + }), + ); + expect(await provider.findByName("app_1", "loop-exp-abc")).toEqual([ + { remoteId: "sb_1", status: "sleeping" }, + { remoteId: "sb_2", status: "provisioning" }, + ]); + expect(Object.fromEntries(calls[0].url.searchParams)).toMatchObject({ + app: "app_1", + name: "loop-exp-abc", + exclude_status: "terminated", + }); + const endless = fake(() => ok({ data: [], has_more: true })); + await expect( + endless.provider.findByName("app_1", "loop-exp-abc"), + ).rejects.toMatchObject({ outcome: "uncertain" }); +}); + +it("treats a missing Sailbox as already terminated and as absent on get", async () => { + const { provider } = fake(() => failure(404)); + await expect(provider.terminate("sb_1", "op-2")).resolves.toBeUndefined(); + expect(await provider.get("sb_1")).toBeNull(); +}); + +it("maps Sail's open status set, keeping unrecognised values unknown", () => { + expect(sailboxStatus("create_failed")).toBe("failed"); + expect(sailboxStatus("interrupted_restorable")).toBe("paused"); + expect(sailboxStatus("terminating")).toBe("terminating"); + expect(sailboxStatus("hibernating")).toBe("unknown"); + expect(sailboxStatus(undefined)).toBe("unknown"); +}); + +it("refuses a resume Sail reports as no longer possible", async () => { + const { provider } = fake(() => + ok({ + sailbox_id: "sb_1", + status: "failed", + resume_state: "terminal_unavailable", + }), + ); + await expect(provider.resume("sb_1", "op-3")).rejects.toMatchObject({ + outcome: "refused", + }); +}); + +it("stops reading a file once it passes the limit", async () => { + const { provider } = fake( + () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(6)); + controller.enqueue(new Uint8Array(6)); + controller.close(); + }, + }), + ), + ); + await expect(provider.readFile("sb_1", "/workspace/a", 10)).rejects.toThrow( + "larger than 10 bytes", + ); +}); + +it("sets the working directory without shell-parsing the command", () => { + expect(sailboxArgv(["claude", "-p", "a; rm -rf /"], "/workspace")).toEqual([ + "/bin/sh", + "-c", + 'cd -- "$1" || exit 127; shift; exec "$@"', + "loop-exec", + "/workspace", + "claude", + "-p", + "a; rm -rf /", + ]); + expect(sailboxArgv(["ls"], undefined)).toEqual(["ls"]); +}); + +it("streams exec output, writes stdin at increasing offsets and reports the exit code", async () => { + const encoder = new TextEncoder(); + let release!: () => void; + const stdinDone = new Promise((resolve) => (release = resolve)); + const { calls, provider } = fake(async (call) => { + if (call.url.pathname.endsWith("/exec")) + return new Response( + new ReadableStream({ + async start(controller) { + const send = (event: unknown) => + controller.enqueue(encoder.encode(JSON.stringify(event) + "\n")); + send({ type: "started", exec_request_id: "exec/1" }); + send({ + type: "stdout", + data: Buffer.from("hello ").toString("base64"), + seq: 1, + }); + // An event split across chunks is still read whole. + const line = JSON.stringify({ + type: "stderr", + data: Buffer.from("warn").toString("base64"), + seq: 1, + }); + controller.enqueue(encoder.encode(line.slice(0, 10))); + controller.enqueue(encoder.encode(line.slice(10) + "\n")); + await stdinDone; + send({ type: "heartbeat" }); + send({ type: "exit", status: "failed", return_code: 3 }); + controller.close(); + }, + }), + ); + if (call.url.pathname.endsWith("/stdin")) { + if (call.url.searchParams.get("eof") === "true") release(); + return ok({ accepted_through: 0 }); + } + return ok({}); + }); + const handle = await provider.exec("sb_1", ["cat"], { + cwd: "/workspace", + env: { LOOP_TOOLS_URL: "https://brain.example/api/internal/tool-bridge/t" }, + openStdin: true, + }); + const read = async (stream: NodeJS.ReadableStream) => { + let text = ""; + for await (const chunk of stream) text += String(chunk); + return text; + }; + const output = Promise.all([read(handle.stdout), read(handle.stderr)]); + await new Promise((resolve, reject) => + handle.stdin.write("ab", (error) => (error ? reject(error) : resolve())), + ); + await new Promise((resolve) => handle.stdin.end("cde", resolve)); + expect(await handle.exit).toEqual({ exitCode: 3, signal: null }); + expect(await output).toEqual(["hello ", "warn"]); + + const start = calls[0]; + expect(start.body).toMatchObject({ + command: sailboxArgv(["cat"], "/workspace"), + open_stdin: true, + env: { + LOOP_TOOLS_URL: "https://brain.example/api/internal/tool-bridge/t", + }, + }); + const writes = calls + .filter((call) => call.url.pathname.endsWith("/stdin")) + .map((call) => ({ + id: call.url.searchParams.get("exec_request_id"), + offset: call.url.searchParams.get("offset"), + eof: call.url.searchParams.get("eof"), + body: call.body, + })); + expect(writes).toEqual([ + { id: "exec/1", offset: "0", eof: null, body: "ab" }, + { id: "exec/1", offset: "2", eof: null, body: "cde" }, + { id: "exec/1", offset: "5", eof: "true", body: "" }, + ]); +}); + +it("resolves exit with no code when the event stream drops", async () => { + const { provider } = fake( + () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode( + JSON.stringify({ type: "started", exec_request_id: "e" }) + + "\n", + ), + ); + controller.error(new Error("connection reset")); + }, + }), + ), + ); + const handle = await provider.exec("sb_1", ["sleep", "100"], {}); + expect(await handle.exit).toEqual({ exitCode: null, signal: null }); +}); diff --git a/test/vm-state-documentation.test.ts b/test/vm-state-documentation.test.ts new file mode 100644 index 00000000..1e6029e2 --- /dev/null +++ b/test/vm-state-documentation.test.ts @@ -0,0 +1,46 @@ +import { readFileSync } from "node:fs"; +import { expect, it } from "vitest"; +import { + VM_OPERATION_TRANSITIONS, + VM_TRANSITIONS, +} from "../src/shared/vm-state"; + +const doc = readFileSync( + new URL("../docs/state-machines.md", import.meta.url), + "utf8", +); + +function documented(heading: string) { + // The pinned formatter pads table cells, so rows are found by their + // leading pipe: a header, a rule, then one row per source state. + const section = doc.split(heading)[1].split("\n##")[0]; + const rows = section + .split("\n") + .filter((line) => line.startsWith("|")) + .slice(2); + return Object.fromEntries( + rows.map((line) => { + const [, from, to] = line + .split("|") + .map((part) => part.trim().replaceAll(/[`*]/g, "")); + return [from, to === "—" ? [] : to.split(" / ").sort()]; + }), + ); +} + +const sorted = (edges: Record) => + Object.fromEntries( + Object.entries(edges).map(([from, to]) => [from, [...to].sort()]), + ); + +it("documents every experiment VM transition", () => { + expect(documented("### `experiment_vms.state`")).toEqual( + sorted(VM_TRANSITIONS), + ); +}); + +it("documents every VM operation transition", () => { + expect(documented("### `vm_operations.state`")).toEqual( + sorted(VM_OPERATION_TRANSITIONS), + ); +}); From 942a58b94c6e8243fa525bb91fb26c202f086afc Mon Sep 17 00:00:00 2001 From: Jamie Asefa <13409617+j-asefa@users.noreply.github.com> Date: Fri, 20 Nov 2026 17:48:15 +0000 Subject: [PATCH 2/9] Cloud VM experiments 1/5: fresh volume keys, unreadable get bodies, upgrade test Co-Authored-By: Claude Opus 5.5 --- src/server/vm/sailbox-provider.ts | 15 +++++++++++---- test/vm-operations.test.ts | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/server/vm/sailbox-provider.ts b/src/server/vm/sailbox-provider.ts index a4692d49..9cd0c58b 100644 --- a/src/server/vm/sailbox-provider.ts +++ b/src/server/vm/sailbox-provider.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { PassThrough, Writable } from "node:stream"; import { VmProviderError, @@ -177,10 +178,11 @@ export function createSailboxProvider(options: { }, async ensureVolume(name) { - // Creating a volume that already exists returns it. + // Creating a volume that already exists returns it, so each call gets a + // fresh key: a fixed one would replay a stale answer for 24 hours. const value = await json("POST", `${api}/sailbox-volumes`, { body: { name }, - idempotencyKey: `loop-volume-${name}`, + idempotencyKey: randomUUID(), }); return { volumeId: field(value, "volume_id") }; }, @@ -217,8 +219,13 @@ export function createSailboxProvider(options: { accept404: true, }); if (!response) return null; - const value = (await response.json()) as Record; - return { status: sailboxStatus(value.status) }; + const value: unknown = await response.json().catch(() => undefined); + if (!value || typeof value !== "object") + throw new VmProviderError( + "uncertain", + "Sailboxes GET answered with a body Loop could not read", + ); + return { status: sailboxStatus((value as { status?: unknown }).status) }; }, async findByName(appId, name) { diff --git a/test/vm-operations.test.ts b/test/vm-operations.test.ts index 90549923..88d6e2bf 100644 --- a/test/vm-operations.test.ts +++ b/test/vm-operations.test.ts @@ -1,6 +1,7 @@ import Database from "better-sqlite3"; import { beforeEach, expect, it } from "vitest"; import { applyMigrations } from "../src/server/migrations.js"; +import { applyMigrationsThrough } from "./helpers/migrations.js"; import * as experimentVmsQueries from "../src/server/sql/experiment-vms.js"; import * as vmOperationsQueries from "../src/server/sql/vm-operations.js"; import { @@ -307,3 +308,28 @@ it("refuses illegal and stale VM transitions", () => { expect(row?.destroyed_at).toBeGreaterThan(0); expect(transitionVm(db, vm, "running", { now: clock() }).changed).toBe(false); }); + +it("adds the experiment tables to a database a released version wrote, leaving its rows alone", () => { + const upgrade = new Database(":memory:"); + upgrade.pragma("foreign_keys=ON"); + applyMigrationsThrough(upgrade, 166); + upgrade + .prepare( + "INSERT INTO folders(id,owner_id,kind,name,created_at) VALUES('f1','alice','project','Notes',1)", + ) + .run(); + applyMigrations(upgrade); + expect(upgrade.prepare("SELECT id,name FROM folders").all()).toEqual([ + { id: "f1", name: "Notes" }, + ]); + for (const table of [ + "vm_provider_configs", + "experiment_vms", + "experiment_previews", + "vm_operations", + ]) + expect(upgrade.prepare(`SELECT count(*) AS n FROM ${table}`).get()).toEqual( + { n: 0 }, + ); + upgrade.close(); +}); From 9b5873ec6efd78c1e08ab6d3fc821b76010995fa Mon Sep 17 00:00:00 2001 From: Jamie Asefa <13409617+j-asefa@users.noreply.github.com> Date: Fri, 20 Nov 2026 18:50:17 +0000 Subject: [PATCH 3/9] Cloud VM experiments 1/5: settle reconciles atomically and observe every kind Review pass 1 fixes: - A reconcile adopts the remote id, settles the receipt and applies the observed VM state in one transaction, rolled back when another reconcile settled the receipt first. - A late provider answer no longer contradicts a receipt a reconcile settled; the caller gets the stored outcome. - A listing without a data array or has_more flag is uncertain, never proof that a VM is absent. - Adoption can find a VM already asleep or paused (provisioning -> sleeping/paused). - Exec output honours backpressure: the event stream pauses while Loop's consumer is behind. - Every uncertain operation kind can be reconciled read-only: sleep, resume and destroy from the VM status, expose and unexpose from the published port. vm_operations keeps request_json for that. Co-Authored-By: Claude Opus 5.5 --- docs/state-machines.md | 6 +- migrations/167-experiment-vms.sql | 4 +- src/server/sql/rows.ts | 1 + src/server/sql/vm-operations.ts | 4 +- src/server/vm/operations.ts | 305 +++++++++++++++++++----------- src/server/vm/provider.ts | 2 + src/server/vm/sailbox-provider.ts | 65 +++++-- src/shared/vm-state.ts | 3 +- test/vm-operations-worker.test.ts | 1 + test/vm-operations.test.ts | 203 ++++++++++++++++++-- test/vm-sailbox-provider.test.ts | 48 +++++ 11 files changed, 497 insertions(+), 145 deletions(-) diff --git a/docs/state-machines.md b/docs/state-machines.md index b5d11397..de4377a6 100644 --- a/docs/state-machines.md +++ b/docs/state-machines.md @@ -376,7 +376,7 @@ leaves its project, threads, results and published files readable. | From | To | Trigger and guard | | ---- | -- | ----------------- | -| `provisioning` | `running` / `failed` / `destroying` | The provider reported the VM up, refused the create, or the owner destroyed it; **tx** | +| `provisioning` | `running` / `sleeping` / `paused` / `failed` / `destroying` | The provider reported the VM up (a reconcile can find it already asleep or paused), refused the create, or the owner destroyed it; **tx** | | `running` | `sleeping` / `paused` / `destroying` / `failed` | Sleep observed or requested, pause, owner destroy, or a provider failure observed; **tx** | | `sleeping` | `running` / `destroying` / `failed` | Wake for a turn or a preview, owner destroy, or a failed wake; **tx** | | `paused` | `running` / `destroying` / `failed` | Resume, owner destroy, or a failed resume; **tx** | @@ -393,11 +393,11 @@ before the provider is called, and its id is the caller's idempotency key. | From | To | Trigger and guard | | ---- | -- | ----------------- | | `pending` | `succeeded` / `failed` / `uncertain` | The provider answered, refused, or gave no observed answer (timeout, disconnect, server error, or the worker's sweep found the row still pending after the longest provider call); **tx** | -| `uncertain` | `succeeded` / `failed` | A late answer arrived, the owner's reconcile found the VM by its `remote_name`, or the provider confirmed it does not exist; **tx** | +| `uncertain` | `succeeded` / `failed` | A late answer arrived, or the owner's reconcile observed the outcome: a create or fork by the VM's `remote_name`, any other kind by the VM's status or published port; **tx** | | **`succeeded`** | — | Terminal | | **`failed`** | — | Terminal | -An uncertain operation is never sent again automatically. A retry is a new +An uncertain operation is never sent again automatically. Whichever settles the receipt first wins; a later answer that disagrees returns the stored outcome. A reconcile writes the receipt and the VM's observed state in one transaction. A retry is a new operation with a new id, and is refused while an earlier one for the same experiment is unsettled. Creates and forks send the provider an idempotency key scoped to the experiment, so a retry after an "absent" reconcile reaches diff --git a/migrations/167-experiment-vms.sql b/migrations/167-experiment-vms.sql index c7029257..f956d355 100644 --- a/migrations/167-experiment-vms.sql +++ b/migrations/167-experiment-vms.sql @@ -58,7 +58,8 @@ CREATE TABLE experiment_previews ( -- A receipt written before every provider call. id is the caller's -- idempotency key; request_digest refuses the same key for a different --- request. An operation left pending by a crash becomes uncertain and is +-- request, and request_json keeps the request so a reconcile can check what +-- was asked. An operation left pending by a crash becomes uncertain and is -- never sent again without the owner asking. CREATE TABLE vm_operations ( id TEXT PRIMARY KEY, @@ -67,6 +68,7 @@ CREATE TABLE vm_operations ( kind TEXT NOT NULL, state TEXT NOT NULL, request_digest TEXT NOT NULL, + request_json TEXT NOT NULL, result_json TEXT, error_message TEXT, created_at INTEGER NOT NULL, diff --git a/src/server/sql/rows.ts b/src/server/sql/rows.ts index ef384960..88a25781 100644 --- a/src/server/sql/rows.ts +++ b/src/server/sql/rows.ts @@ -1459,6 +1459,7 @@ export const QueryRecords = { kind: VmOperationKind, state: VmOperationState, request_digest: z.string(), + request_json: z.string(), result_json: z.string().nullable(), error_message: z.string().nullable(), created_at: z.number(), diff --git a/src/server/sql/vm-operations.ts b/src/server/sql/vm-operations.ts index 8b699243..bc4b40d1 100644 --- a/src/server/sql/vm-operations.ts +++ b/src/server/sql/vm-operations.ts @@ -11,7 +11,7 @@ const parse = (row: unknown) => parseRecord(QueryRecords.vm_operations, row, "vm_operations"); const insertPendingStatement = statement( - "INSERT INTO vm_operations(id,owner_id,experiment_id,kind,state,request_digest,created_at,updated_at) VALUES(?,?,?,?,'pending',?,?,?)", + "INSERT INTO vm_operations(id,owner_id,experiment_id,kind,state,request_digest,request_json,created_at,updated_at) VALUES(?,?,?,?,'pending',?,?,?,?)", ); export function insertPending( db: Database.Database, @@ -21,6 +21,7 @@ export function insertPending( experimentId: string; kind: string; requestDigest: string; + requestJson: string; now: number; }, ) { @@ -30,6 +31,7 @@ export function insertPending( operation.experimentId, operation.kind, operation.requestDigest, + operation.requestJson, operation.now, operation.now, ); diff --git a/src/server/vm/operations.ts b/src/server/vm/operations.ts index 17601d60..63050c8f 100644 --- a/src/server/vm/operations.ts +++ b/src/server/vm/operations.ts @@ -1,6 +1,10 @@ import { createHash } from "node:crypto"; import type Database from "better-sqlite3"; -import type { VmOperationKind } from "../../shared/vm-state.js"; +import { + legalVmTransition, + type VmOperationKind, + type VmState, +} from "../../shared/vm-state.js"; import * as experimentVmsQueries from "../sql/experiment-vms.js"; import * as vmOperationsQueries from "../sql/vm-operations.js"; import { transitionVm, transitionVmOperation } from "./machine.js"; @@ -31,26 +35,30 @@ export class VmOperationRefused extends Error { } } -/** SHA-256 of the request with object keys sorted, so the same request - * always has the same digest whatever order its fields were written in. */ -export function vmRequestDigest(request: unknown) { - return createHash("sha256").update(canonical(request)).digest("hex"); -} - -function canonical(value: unknown): string { - if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; +/** The request with object keys sorted, so the same request always has the + * same text, and so the same digest, whatever order its fields were in. */ +export function canonicalVmRequest(value: unknown): string { + if (Array.isArray(value)) + return `[${value.map(canonicalVmRequest).join(",")}]`; if (value !== null && typeof value === "object") return `{${Object.entries(value) .filter(([, item]) => item !== undefined) .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - .map(([key, item]) => `${JSON.stringify(key)}:${canonical(item)}`) + .map( + ([key, item]) => `${JSON.stringify(key)}:${canonicalVmRequest(item)}`, + ) .join(",")}}`; return JSON.stringify(value ?? null); } -function stored( - row: NonNullable>, -): VmOperationOutcome { +/** SHA-256 of the canonical request. */ +export function vmRequestDigest(request: unknown) { + return createHash("sha256").update(canonicalVmRequest(request)).digest("hex"); +} + +type OperationRow = NonNullable>; + +function stored(row: OperationRow): VmOperationOutcome { if (row.state === "succeeded") return { state: "succeeded", @@ -68,7 +76,9 @@ function stored( /** Runs `call` at most once for `operation.id`. A repeat of the same id and * request returns the stored outcome without calling the provider; the same - * id with a different request is refused. */ + * id with a different request is refused. The receipt is the record: when a + * reconcile settled it while the call was still out, the stored outcome is + * returned, not the late answer. */ export async function runVmOperation( db: Database.Database, operation: { @@ -81,6 +91,7 @@ export async function runVmOperation( call: () => Promise, clock: () => number, ): Promise { + const requestJson = canonicalVmRequest(operation.request); const requestDigest = vmRequestDigest({ kind: operation.kind, experimentId: operation.experimentId, @@ -118,6 +129,7 @@ export async function runVmOperation( experimentId: operation.experimentId, kind: operation.kind, requestDigest, + requestJson, now: clock(), }); return undefined; @@ -126,69 +138,90 @@ export async function runVmOperation( if (prior) return stored(prior); const receipt = { owner: operation.owner, id: operation.id }; + const current = () => + stored(vmOperationsQueries.byId(db, receipt.owner, receipt.id)!); + let result: unknown; try { - const result = await call(); - transitionVmOperation(db, receipt, "succeeded", { result, now: clock() }); - return { state: "succeeded", result, replayed: false }; + result = await call(); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); // Only an answer from the provider proves the call did nothing. Anything // else, including a bug in the adapter, leaves the outcome unknown. - if (error instanceof VmProviderError && error.outcome === "refused") { - transitionVmOperation(db, receipt, "failed", { - errorMessage, - now: clock(), - }); - return { state: "failed", errorMessage, replayed: false }; - } - transitionVmOperation(db, receipt, "uncertain", { - errorMessage, - now: clock(), - }); - return { state: "uncertain", errorMessage }; + const to = + error instanceof VmProviderError && error.outcome === "refused" + ? "failed" + : "uncertain"; + if (!transitionVmOperation(db, receipt, to, { errorMessage, now: clock() })) + return current(); + return to === "failed" + ? { state: "failed", errorMessage, replayed: false } + : { state: "uncertain", errorMessage }; } + if ( + !transitionVmOperation(db, receipt, "succeeded", { result, now: clock() }) + ) + return current(); + return { state: "succeeded", result, replayed: false }; } -const liveState = { - provisioning: undefined, - running: "running", - sleeping: "sleeping", - paused: "paused", - failed: "failed", - terminating: undefined, - terminated: "failed", - unknown: undefined, -} as const satisfies Record< - VmStatus, - "running" | "sleeping" | "paused" | "failed" | undefined ->; +/** Moves the VM to the state the provider reports, when that edge is legal + * from where it is. Call inside the transaction that settles the receipt. */ +function applyObservedStatus( + db: Database.Database, + vm: { owner: string; id: string; state: VmState }, + status: VmStatus, + now: number, +) { + const observed: Record = { + provisioning: undefined, + running: "running", + sleeping: "sleeping", + paused: "paused", + failed: "failed", + terminating: undefined, + terminated: vm.state === "destroying" ? "destroyed" : "failed", + unknown: undefined, + }; + const to = observed[status]; + if (!to || to === vm.state || !legalVmTransition(vm.state, to)) return; + transitionVm(db, vm, to, { + expected: vm.state, + lastError: to === "failed" ? `The provider reports the VM ${status}` : null, + now, + }); +} -export type VmCreateReconciliation = - | { outcome: "adopted"; remoteId: string } - | { outcome: "absent" } +export type VmReconciliation = + | { outcome: "succeeded"; result: unknown } + | { outcome: "failed"; errorMessage: string } | { outcome: "still_uncertain"; reason: string }; -/** Settles an uncertain create or fork by looking for the VM's name. This - * reads from the provider and never creates: one match is adopted, no match - * fails the operation so the owner can start a new one, and anything else - * leaves it uncertain. */ -export async function reconcileVmCreate( +type Observation = + | { settle: "succeeded"; result: unknown; status?: VmStatus } + | { settle: "failed"; errorMessage: string; status?: VmStatus } + | { settle: "unknown"; reason: string }; + +class SettledConcurrently extends Error {} + +const unsure = (reason: string): Observation => ({ settle: "unknown", reason }); + +/** Observes the outcome of the owner's uncertain operation and settles it. + * This only reads from the provider: nothing is created, sent or retried. + * The receipt and the VM's observed state are written in one transaction, + * and only while the receipt is still uncertain. */ +export async function reconcileVmOperation( db: Database.Database, - provider: Pick, + provider: Pick, target: { owner: string; operationId: string; appId: string }, clock: () => number, -): Promise { +): Promise { const operation = vmOperationsQueries.byId( db, target.owner, target.operationId, ); - if ( - !operation || - operation.state !== "uncertain" || - (operation.kind !== "create" && operation.kind !== "fork") - ) - return { outcome: "still_uncertain", reason: "not an uncertain create" }; + if (!operation || operation.state !== "uncertain") + return { outcome: "still_uncertain", reason: "not an uncertain operation" }; const vm = experimentVmsQueries.byId( db, target.owner, @@ -196,65 +229,113 @@ export async function reconcileVmCreate( ); if (!vm) return { outcome: "still_uncertain", reason: "experiment missing" }; - let matches: Awaited>; + let observation: Observation; try { - matches = await provider.findByName(target.appId, vm.remote_name); + observation = await observe(provider, operation, vm, target.appId); } catch (error) { - return { - outcome: "still_uncertain", - reason: error instanceof Error ? error.message : String(error), - }; + observation = unsure( + error instanceof Error ? error.message : String(error), + ); } - if (matches.length > 1) - return { - outcome: "still_uncertain", - reason: `${matches.length} VMs are named ${vm.remote_name}`, - }; + if (observation.settle === "unknown") + return { outcome: "still_uncertain", reason: observation.reason }; + const seen = observation; const receipt = { owner: target.owner, id: operation.id }; - const subject = { owner: target.owner, id: vm.id }; - const [match] = matches; - if (!match) { - if ( - !transitionVmOperation(db, receipt, "failed", { - errorMessage: "The provider has no VM with this name", - now: clock(), - }) - ) - return { outcome: "still_uncertain", reason: "settled concurrently" }; - transitionVm(db, subject, "failed", { - expected: "provisioning", - lastError: "The VM was not created", - now: clock(), - }); - return { outcome: "absent" }; - } - const adopted = db - .transaction(() => { + try { + db.transaction(() => { + const fresh = experimentVmsQueries.byId(db, target.owner, vm.id); + if (!fresh) throw new SettledConcurrently(); if ( - !experimentVmsQueries.adoptRemote( - db, - target.owner, - vm.id, - match.remoteId, + seen.settle === "succeeded" && + (operation.kind === "create" || operation.kind === "fork") + ) { + const { remoteId } = seen.result as { remoteId: string }; + if ( + !experimentVmsQueries.adoptRemote(db, target.owner, vm.id, remoteId) ) - ) - return false; - return transitionVmOperation(db, receipt, "succeeded", { - result: { remoteId: match.remoteId }, - now: clock(), - }); - }) - .immediate(); - if (!adopted) - return { outcome: "still_uncertain", reason: "settled concurrently" }; - const state = liveState[match.status]; - if (state) - transitionVm(db, subject, state, { - expected: "provisioning", - lastError: - state === "failed" ? "The provider reports the VM failed" : null, - now: clock(), - }); - return { outcome: "adopted", remoteId: match.remoteId }; + throw new SettledConcurrently(); + } + const settled = + seen.settle === "succeeded" + ? transitionVmOperation(db, receipt, "succeeded", { + result: seen.result, + now: clock(), + }) + : transitionVmOperation(db, receipt, "failed", { + errorMessage: seen.errorMessage, + now: clock(), + }); + if (!settled) throw new SettledConcurrently(); + if (seen.status) + applyObservedStatus( + db, + { owner: target.owner, id: vm.id, state: fresh.state }, + seen.status, + clock(), + ); + }).immediate(); + } catch (error) { + if (error instanceof SettledConcurrently) + return { outcome: "still_uncertain", reason: "settled concurrently" }; + throw error; + } + return seen.settle === "succeeded" + ? { outcome: "succeeded", result: seen.result } + : { outcome: "failed", errorMessage: seen.errorMessage }; +} + +async function observe( + provider: Pick, + operation: OperationRow, + vm: NonNullable>, + appId: string, +): Promise { + if (operation.kind === "create" || operation.kind === "fork") { + const matches = await provider.findByName(appId, vm.remote_name); + if (matches.length > 1) + return unsure(`${matches.length} VMs are named ${vm.remote_name}`); + const [match] = matches; + if (!match) + return { + settle: "failed", + errorMessage: "The provider has no VM with this name", + status: "terminated", + }; + return { + settle: "succeeded", + result: { remoteId: match.remoteId }, + status: match.status, + }; + } + if (!vm.remote_id) return unsure("the VM has no provider id"); + if (operation.kind === "expose" || operation.kind === "unexpose") { + const request: unknown = JSON.parse(operation.request_json); + const port = (request as { port?: unknown } | null)?.port; + if (typeof port !== "number") return unsure("the receipt names no port"); + const published = await provider.getHttpPort(vm.remote_id, port); + if (operation.kind === "expose") + return published + ? { settle: "succeeded", result: published } + : { settle: "failed", errorMessage: "The port is not published" }; + return published + ? { settle: "failed", errorMessage: "The port is still published" } + : { settle: "succeeded", result: null }; + } + const status = (await provider.get(vm.remote_id))?.status ?? "terminated"; + if (status === "unknown" || status === "provisioning") + return unsure(`the provider reports the VM ${status}`); + const reached = { + sleep: status === "sleeping", + resume: status === "running", + destroy: status === "terminated", + }[operation.kind]; + if (reached) return { settle: "succeeded", result: null, status }; + if (operation.kind === "destroy" && status === "terminating") + return unsure("the VM is still shutting down"); + return { + settle: "failed", + errorMessage: `The provider reports the VM ${status}`, + status, + }; } diff --git a/src/server/vm/provider.ts b/src/server/vm/provider.ts index ba3350d5..5b950bd8 100644 --- a/src/server/vm/provider.ts +++ b/src/server/vm/provider.ts @@ -111,6 +111,8 @@ export interface VmProvider { idempotencyKey: string, ): Promise<{ url: string }>; unexposePort(remoteId: string, port: number): Promise; + /** The published HTTP port, or null when the port is not published. */ + getHttpPort(remoteId: string, port: number): Promise<{ url: string } | null>; readFile(remoteId: string, path: string, maxBytes: number): Promise; writeFile(remoteId: string, path: string, bytes: Buffer): Promise; /** Succeeds when the VM is already gone. */ diff --git a/src/server/vm/sailbox-provider.ts b/src/server/vm/sailbox-provider.ts index 9cd0c58b..e06c7ba7 100644 --- a/src/server/vm/sailbox-provider.ts +++ b/src/server/vm/sailbox-provider.ts @@ -239,14 +239,26 @@ export function createSailboxProvider(options: { offset: String(page * 100), }); const value = await json("GET", `${api}/sailboxes?${query}`); - const data = Array.isArray(value.data) ? value.data : []; - for (const item of data as Record[]) - if (item.name === name) + // Absence is only proven by a well-formed final page. + if (!Array.isArray(value.data) || typeof value.has_more !== "boolean") + throw new VmProviderError( + "uncertain", + "Sailboxes listed Sailboxes in a shape Loop could not read", + ); + for (const item of value.data as unknown[]) { + if (!item || typeof item !== "object") + throw new VmProviderError( + "uncertain", + "Sailboxes listed an entry Loop could not read", + ); + const entry = item as Record; + if (entry.name === name) matches.push({ - remoteId: field(item, "sailbox_id"), - status: sailboxStatus(item.status), + remoteId: field(entry, "sailbox_id"), + status: sailboxStatus(entry.status), }); - if (value.has_more !== true) return matches; + } + if (!value.has_more) return matches; } // An incomplete list cannot prove a VM is absent. throw new VmProviderError( @@ -329,6 +341,22 @@ export function createSailboxProvider(options: { return { url: field(value, "public_url") }; }, + async getHttpPort(remoteId, port) { + const response = await request( + "GET", + `${box(remoteId)}/listeners/${port}`, + { accept404: true }, + ); + if (!response) return null; + const value: unknown = await response.json().catch(() => undefined); + if (!value || typeof value !== "object") + throw new VmProviderError( + "uncertain", + "Sailboxes listener answered with a body Loop could not read", + ); + return { url: field(value as Record, "public_url") }; + }, + async unexposePort(remoteId, port) { await request("DELETE", `${box(remoteId)}/listeners/${port}`, { accept404: true, @@ -441,6 +469,7 @@ async function startExec( signal: null, }; let buffered = ""; + let blocked: PassThrough | undefined; const decoder = new TextDecoder(); const handle = (line: string) => { if (!line.trim()) return; @@ -459,11 +488,10 @@ async function startExec( } else if ( (event.type === "stdout" || event.type === "stderr") && typeof event.data === "string" - ) - (event.type === "stdout" ? stdout : stderr).write( - Buffer.from(event.data, "base64"), - ); - else if (event.type === "exit") + ) { + const stream = event.type === "stdout" ? stdout : stderr; + if (!stream.write(Buffer.from(event.data, "base64"))) blocked = stream; + } else if (event.type === "exit") result = { exitCode: typeof event.return_code === "number" ? event.return_code : null, @@ -479,6 +507,21 @@ async function startExec( buffered = buffered.slice(newline + 1); newline = buffered.indexOf("\n"); } + // Output is read from the provider only as fast as Loop's consumer + // reads it: a full stream pauses the event stream until it drains. + if (blocked) { + const waiting: PassThrough = blocked; + blocked = undefined; + await new Promise((resolve) => { + const done = () => { + waiting.off("drain", done); + waiting.off("close", done); + resolve(); + }; + waiting.on("drain", done); + waiting.on("close", done); + }); + } } handle(buffered); } catch { diff --git a/src/shared/vm-state.ts b/src/shared/vm-state.ts index db738733..47fb6f21 100644 --- a/src/shared/vm-state.ts +++ b/src/shared/vm-state.ts @@ -3,7 +3,8 @@ import { z } from "zod"; // The single owner of the experiment_vms.state vocabulary and its edges. // transitionVm() in src/server/vm/machine.ts is the only writer. export const VM_TRANSITIONS = { - provisioning: ["running", "failed", "destroying"], + // sleeping/paused: a reconcile adopted a VM the provider reports asleep. + provisioning: ["running", "sleeping", "paused", "failed", "destroying"], running: ["sleeping", "paused", "destroying", "failed"], sleeping: ["running", "destroying", "failed"], paused: ["running", "destroying", "failed"], diff --git a/test/vm-operations-worker.test.ts b/test/vm-operations-worker.test.ts index df8fb13a..67c0b5da 100644 --- a/test/vm-operations-worker.test.ts +++ b/test/vm-operations-worker.test.ts @@ -42,6 +42,7 @@ it("marks a VM operation receipt left pending by a dead caller uncertain on the experimentId: "exp-w", kind: "create", requestDigest: "d", + requestJson: "{}", now: now() - age, }); diff --git a/test/vm-operations.test.ts b/test/vm-operations.test.ts index 88d6e2bf..d43ea0b3 100644 --- a/test/vm-operations.test.ts +++ b/test/vm-operations.test.ts @@ -10,7 +10,7 @@ import { transitionVm, } from "../src/server/vm/machine.js"; import { - reconcileVmCreate, + reconcileVmOperation, runVmOperation, } from "../src/server/vm/operations.js"; import { VmProviderError, type VmStatus } from "../src/server/vm/provider.js"; @@ -62,12 +62,26 @@ const create = ( clock, ); +// A read-only provider for reconcile: VMs by name, statuses by id and +// published ports. Every lookup is counted. const found = (...matches: { remoteId: string; status: VmStatus }[]) => ({ calls: 0, + statuses: new Map(), + ports: new Map(), async findByName() { this.calls += 1; return matches; }, + async get(remoteId: string) { + this.calls += 1; + const status = this.statuses.get(remoteId); + return status ? { status } : null; + }, + async getHttpPort(_remoteId: string, port: number) { + this.calls += 1; + const url = this.ports.get(port); + return url ? { url } : null; + }, }); // Migrating once and copying the image keeps each test's database fresh. @@ -193,6 +207,7 @@ it("marks a receipt uncertain only once no live call could still be waiting on i experimentId: "exp1", kind: "create", requestDigest: "d", + requestJson: "{}", now: time, }); const created = time; @@ -224,11 +239,15 @@ async function uncertainCreate() { throw new VmProviderError("uncertain", "timed out"); }); } -const reconcile = (provider: ReturnType, owner = "alice") => - reconcileVmCreate( +const reconcile = ( + provider: Parameters[1], + owner = "alice", + operationId = "op1", +) => + reconcileVmOperation( db, provider, - { owner, operationId: "op1", appId: "app" }, + { owner, operationId, appId: "app" }, clock, ); @@ -236,7 +255,7 @@ it("adopts the one VM that carries the experiment's name", async () => { await uncertainCreate(); expect( await reconcile(found({ remoteId: "sb-7", status: "running" })), - ).toEqual({ outcome: "adopted", remoteId: "sb-7" }); + ).toEqual({ outcome: "succeeded", result: { remoteId: "sb-7" } }); expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ remote_id: "sb-7", state: "running", @@ -249,7 +268,7 @@ it("adopts the one VM that carries the experiment's name", async () => { it("fails the operation when the provider has no such VM, allowing a new one", async () => { await uncertainCreate(); - expect(await reconcile(found())).toEqual({ outcome: "absent" }); + expect(await reconcile(found())).toMatchObject({ outcome: "failed" }); expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe("failed"); expect(await create("op2", async () => ({ remoteId: "sb-8" }))).toMatchObject( { state: "succeeded" }, @@ -259,16 +278,12 @@ it("fails the operation when the provider has no such VM, allowing a new one", a it("stays uncertain when the lookup fails, is ambiguous, or comes from another owner", async () => { await uncertainCreate(); expect( - await reconcileVmCreate( - db, - { - async findByName() { - throw new VmProviderError("uncertain", "list timed out"); - }, + await reconcile({ + ...found(), + async findByName() { + throw new VmProviderError("uncertain", "list timed out"); }, - { owner: "alice", operationId: "op1", appId: "app" }, - clock, - ), + }), ).toEqual({ outcome: "still_uncertain", reason: "list timed out" }); expect( await reconcile( @@ -287,9 +302,165 @@ it("stays uncertain when the lookup fails, is ambiguous, or comes from another o expect(experimentVmsQueries.byId(db, "alice", "exp1")?.remote_id).toBeNull(); }); +it("does not adopt a VM when another reconcile settled the receipt during the lookup", async () => { + await uncertainCreate(); + const racing = { + ...found(), + async findByName() { + // The other reconcile wins while this lookup is out. + await reconcile(found()); + return [{ remoteId: "sb-9", status: "running" as const }]; + }, + }; + expect(await reconcile(racing)).toEqual({ + outcome: "still_uncertain", + reason: "settled concurrently", + }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: null, + state: "failed", + }); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("failed"); +}); + +it("returns the stored outcome when a late answer contradicts a reconciled receipt", async () => { + let answer!: (value: unknown) => void; + const outcome = create( + "op1", + () => new Promise((resolve) => (answer = resolve)), + ); + await new Promise((resolve) => setImmediate(resolve)); + markStaleVmOperationsUncertain(db, time + VM_OPERATION_MAX_CALL_MS + 1); + expect(await reconcile(found())).toMatchObject({ outcome: "failed" }); + answer({ remoteId: "sb-late" }); + const late = await outcome; + expect(late).toMatchObject({ state: "failed", replayed: true }); + expect(await create("op1", async () => ({}))).toEqual(late); +}); + +it("adopts a VM found asleep, and settles its receipt and state together", async () => { + await uncertainCreate(); + expect( + await reconcile(found({ remoteId: "sb-7", status: "sleeping" })), + ).toMatchObject({ outcome: "succeeded" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-7", + state: "sleeping", + }); +}); + +it("rolls the adoption back when the VM state cannot be written", async () => { + await uncertainCreate(); + db.exec( + "CREATE TRIGGER refuse_vm_state BEFORE UPDATE OF state ON experiment_vms BEGIN SELECT RAISE(ABORT, 'disk full'); END", + ); + await expect( + reconcile(found({ remoteId: "sb-7", status: "running" })), + ).rejects.toThrow("disk full"); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("uncertain"); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.remote_id).toBeNull(); +}); + +async function runningVm(remoteId = "sb-1") { + await create("op0", async () => ({ remoteId })); + experimentVmsQueries.adoptRemote(db, "alice", "exp1", remoteId); + transitionVm(db, { owner: "alice", id: "exp1" }, "running", { now: clock() }); +} + +it("reconciles an uncertain expose from the published port, unblocking the next operation", async () => { + await runningVm(); + await runVmOperation( + db, + { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "expose", + request: { port: 3000 }, + }, + async () => { + throw new VmProviderError("uncertain", "timed out"); + }, + clock, + ); + const provider = found(); + provider.ports.set(3000, "https://p.example"); + expect(await reconcile(provider)).toEqual({ + outcome: "succeeded", + result: { url: "https://p.example" }, + }); + expect( + await runVmOperation( + db, + { + id: "op2", + owner: "alice", + experimentId: "exp1", + kind: "destroy", + request: {}, + }, + async () => null, + clock, + ), + ).toMatchObject({ state: "succeeded" }); +}); + +it("reconciles an uncertain destroy from the VM's status", async () => { + await runningVm(); + transitionVm(db, { owner: "alice", id: "exp1" }, "destroying", { + now: clock(), + }); + await runVmOperation( + db, + { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "destroy", + request: {}, + }, + async () => { + throw new VmProviderError("uncertain", "timed out"); + }, + clock, + ); + const provider = found(); + provider.statuses.set("sb-1", "terminating"); + expect(await reconcile(provider)).toMatchObject({ + outcome: "still_uncertain", + }); + provider.statuses.delete("sb-1"); + expect(await reconcile(provider)).toMatchObject({ outcome: "succeeded" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe( + "destroyed", + ); +}); + +it("reports an uncertain sleep that did not happen as failed, so it can be asked for again", async () => { + await runningVm(); + await runVmOperation( + db, + { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "sleep", + request: {}, + }, + async () => { + throw new VmProviderError("uncertain", "timed out"); + }, + clock, + ); + const provider = found(); + provider.statuses.set("sb-1", "running"); + expect(await reconcile(provider)).toMatchObject({ outcome: "failed" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe("running"); +}); + it("refuses illegal and stale VM transitions", () => { const vm = { owner: "alice", id: "exp1" }; - expect(transitionVm(db, vm, "sleeping", { now: clock() })).toEqual({ + expect(transitionVm(db, vm, "destroyed", { now: clock() })).toEqual({ changed: false, reason: "illegal", from: "provisioning", diff --git a/test/vm-sailbox-provider.test.ts b/test/vm-sailbox-provider.test.ts index 9fce66a7..ea325db5 100644 --- a/test/vm-sailbox-provider.test.ts +++ b/test/vm-sailbox-provider.test.ts @@ -296,3 +296,51 @@ it("resolves exit with no code when the event stream drops", async () => { const handle = await provider.exec("sb_1", ["sleep", "100"], {}); expect(await handle.exit).toEqual({ exitCode: null, signal: null }); }); + +it("treats a listing it cannot read as uncertain, not as proof of absence", async () => { + for (const page of [ + {}, + { data: [] }, + { data: "none", has_more: false }, + { data: [null], has_more: false }, + ]) { + const { provider } = fake(() => ok(page)); + await expect( + provider.findByName("app_1", "loop-exp-abc"), + ).rejects.toMatchObject({ outcome: "uncertain" }); + } +}); + +it("stops reading exec events while nobody reads the output", async () => { + let pulls = 0; + const encoder = new TextEncoder(); + const chunk = Buffer.alloc(64 * 1024, 1).toString("base64"); + const { provider } = fake( + () => + new Response( + new ReadableStream({ + pull(controller) { + pulls += 1; + controller.enqueue( + encoder.encode( + (pulls === 1 + ? JSON.stringify({ type: "started", exec_request_id: "e" }) + + "\n" + : "") + + JSON.stringify({ type: "stdout", data: chunk, seq: pulls }) + + "\n", + ), + ); + if (pulls === 500) controller.close(); + }, + }), + ), + ); + const handle = await provider.exec("sb_1", ["yes"], {}); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(pulls).toBeLessThan(10); + handle.stdout.resume(); + handle.stderr.resume(); + await handle.exit; + expect(pulls).toBe(500); +}); From fe1a72d33f935a4b73dd98969348441bb774a4f7 Mon Sep 17 00:00:00 2001 From: Jamie Asefa <13409617+j-asefa@users.noreply.github.com> Date: Fri, 20 Nov 2026 18:59:02 +0000 Subject: [PATCH 4/9] Cloud VM experiments 1/5: commit created VMs with their receipts, wait out in-flight calls Review pass 2 fixes: - A reconcile observation that an effect did not happen (no VM, no port, VM not asleep) settles nothing until the longest provider call has passed since the receipt, so a still-running original call cannot be overtaken by a later operation. - runVmOperation takes effects that commit with the settled receipt; createdVmEffects adopts the provider id and observed state in that same transaction. - A create or fork whose VM boots into failed fails its receipt with Sail's reason and keeps the VM id; a reconcile that finds only a failed VM does the same. - A destroy that a reconcile finds unfinished moves the VM from destroying to failed, which allows another destroy. - Exec backpressure waits for every full stream, not only the last. Also from pass 2's open concerns: - The golden image is built with no volume mounted, so third-party installers never run beside harness credentials; experiments mount the auth volume after start (VmProvider.mountVolume). - experiment_vms inserts refuse another owner's folder or source experiment. Co-Authored-By: Claude Opus 5.5 --- docs/sql-index.md | 2 +- docs/state-machines.md | 4 +- src/server/sql/experiment-vms.ts | 43 +++++--- src/server/vm/operations.ts | 152 +++++++++++++++++++++++---- src/server/vm/provider.ts | 16 ++- src/server/vm/sailbox-image.ts | 10 +- src/server/vm/sailbox-provider.ts | 46 +++++--- test/vm-operations.test.ts | 169 ++++++++++++++++++++++++++++++ test/vm-sailbox-provider.test.ts | 73 +++++++++++++ 9 files changed, 452 insertions(+), 63 deletions(-) diff --git a/docs/sql-index.md b/docs/sql-index.md index f7161edb..5b623117 100644 --- a/docs/sql-index.md +++ b/docs/sql-index.md @@ -524,7 +524,7 @@ Generated by `npm run sql-index` from [src/server/sql](../src/server/sql/); do n - `adoptRemote(db, owner, id, remoteId)` — Records the provider's id once; a different id for the same VM is refused. - `byFolder(db, owner, folder)` - `byId(db, owner, id)` -- `insert(db, vm)` +- `insert(db, vm)` — Inserts the owner's experiment. - `owned(db, owner)` - `setState(db, owner, id, from, to, lastError, now)` — Moves `from` to `to`; returns false when the row is no longer in `from`. - `touch(db, owner, id, now)` diff --git a/docs/state-machines.md b/docs/state-machines.md index de4377a6..137688d2 100644 --- a/docs/state-machines.md +++ b/docs/state-machines.md @@ -381,7 +381,7 @@ leaves its project, threads, results and published files readable. | `sleeping` | `running` / `destroying` / `failed` | Wake for a turn or a preview, owner destroy, or a failed wake; **tx** | | `paused` | `running` / `destroying` / `failed` | Resume, owner destroy, or a failed resume; **tx** | | `failed` | `provisioning` / `running` / `destroying` | The owner's reconcile or retry, or owner destroy; **tx** | -| `destroying` | `destroyed` / `failed` | The provider confirmed the VM is gone (a missing VM counts), or refused; **tx** | +| `destroying` | `destroyed` / `failed` | The provider confirmed the VM is gone (a missing VM counts), refused, or a reconcile found the destroy did not finish; **tx** | | **`destroyed`** | — | Terminal | ### `vm_operations.state` @@ -397,7 +397,7 @@ before the provider is called, and its id is the caller's idempotency key. | **`succeeded`** | — | Terminal | | **`failed`** | — | Terminal | -An uncertain operation is never sent again automatically. Whichever settles the receipt first wins; a later answer that disagrees returns the stored outcome. A reconcile writes the receipt and the VM's observed state in one transaction. A retry is a new +An uncertain operation is never sent again automatically. Whichever settles the receipt first wins; a later answer that disagrees returns the stored outcome. A reconcile writes the receipt and the VM's observed state in one transaction, as does a settled create with the VM's provider id. An observation that the effect did not happen settles nothing until the longest provider call has passed since the receipt was written, because the original call may still be in flight. A create whose VM boots into a failed state fails its receipt and keeps the VM's id, so the owner can destroy it. A retry is a new operation with a new id, and is refused while an earlier one for the same experiment is unsettled. Creates and forks send the provider an idempotency key scoped to the experiment, so a retry after an "absent" reconcile reaches diff --git a/src/server/sql/experiment-vms.ts b/src/server/sql/experiment-vms.ts index 8f12eb95..12600883 100644 --- a/src/server/sql/experiment-vms.ts +++ b/src/server/sql/experiment-vms.ts @@ -11,8 +11,10 @@ const parse = (row: unknown) => parseRecord(QueryRecords.experiment_vms, row, "experiment_vms"); const insertStatement = statement( - "INSERT INTO experiment_vms(id,owner_id,folder_id,provider,remote_name,state,auto_approve_local,vcpu_limit,memory_limit_gib,disk_limit_gib,auto_sleep_seconds,source_experiment_id,source_checkpoint_id,clone_url,last_active_at,created_at) VALUES(?,?,?,?,?,'provisioning',?,?,?,?,?,?,?,?,?,?)", + "INSERT INTO experiment_vms(id,owner_id,folder_id,provider,remote_name,state,auto_approve_local,vcpu_limit,memory_limit_gib,disk_limit_gib,auto_sleep_seconds,source_experiment_id,source_checkpoint_id,clone_url,last_active_at,created_at) SELECT ?,?,?,?,?,'provisioning',?,?,?,?,?,?,?,?,?,? WHERE EXISTS(SELECT 1 FROM folders WHERE id=? AND owner_id=?) AND (? IS NULL OR EXISTS(SELECT 1 FROM experiment_vms WHERE id=? AND owner_id=?))", ); +/** Inserts the owner's experiment. Returns false, writing nothing, when the + * folder or the source experiment is not the owner's. */ export function insert( db: Database.Database, vm: { @@ -32,22 +34,29 @@ export function insert( now: number; }, ) { - insertStatement(db).run( - vm.id, - vm.owner, - vm.folderId, - vm.provider, - vm.remoteName, - vm.autoApproveLocal ? 1 : 0, - vm.vcpuLimit, - vm.memoryLimitGib, - vm.diskLimitGib, - vm.autoSleepSeconds, - vm.sourceExperimentId, - vm.sourceCheckpointId, - vm.cloneUrl, - vm.now, - vm.now, + return ( + insertStatement(db).run( + vm.id, + vm.owner, + vm.folderId, + vm.provider, + vm.remoteName, + vm.autoApproveLocal ? 1 : 0, + vm.vcpuLimit, + vm.memoryLimitGib, + vm.diskLimitGib, + vm.autoSleepSeconds, + vm.sourceExperimentId, + vm.sourceCheckpointId, + vm.cloneUrl, + vm.now, + vm.now, + vm.folderId, + vm.owner, + vm.sourceExperimentId, + vm.sourceExperimentId, + vm.owner, + ).changes === 1 ); } diff --git a/src/server/vm/operations.ts b/src/server/vm/operations.ts index 63050c8f..7018d696 100644 --- a/src/server/vm/operations.ts +++ b/src/server/vm/operations.ts @@ -7,7 +7,11 @@ import { } from "../../shared/vm-state.js"; import * as experimentVmsQueries from "../sql/experiment-vms.js"; import * as vmOperationsQueries from "../sql/vm-operations.js"; -import { transitionVm, transitionVmOperation } from "./machine.js"; +import { + VM_OPERATION_MAX_CALL_MS, + transitionVm, + transitionVmOperation, +} from "./machine.js"; import { VmProviderError, type VmProvider, type VmStatus } from "./provider.js"; // Every provider call that changes something runs through runVmOperation: a @@ -90,6 +94,7 @@ export async function runVmOperation( }, call: () => Promise, clock: () => number, + effects: VmOperationEffects = {}, ): Promise { const requestJson = canonicalVmRequest(operation.request); const requestDigest = vmRequestDigest({ @@ -157,11 +162,73 @@ export async function runVmOperation( ? { state: "failed", errorMessage, replayed: false } : { state: "uncertain", errorMessage }; } - if ( - !transitionVmOperation(db, receipt, "succeeded", { result, now: clock() }) - ) - return current(); - return { state: "succeeded", result, replayed: false }; + const errorMessage = effects.failure?.(result) ?? null; + const to = errorMessage === null ? "succeeded" : "failed"; + // The receipt and the local record of what the call did commit together. + // If apply throws, both roll back and the receipt stays pending until the + // sweep marks it uncertain for a reconcile. + const settled = db + .transaction(() => { + if ( + !transitionVmOperation(db, receipt, to, { + result, + errorMessage, + now: clock(), + }) + ) + return false; + effects.apply?.(result); + return true; + }) + .immediate(); + if (!settled) return current(); + return errorMessage === null + ? { state: "succeeded", result, replayed: false } + : { state: "failed", errorMessage, replayed: false }; +} + +/** What a resolved call means locally. */ +export type VmOperationEffects = { + /** A failure the answer itself reports, such as a VM that booted into + * "failed"; null when the call did what was asked. */ + failure?(result: unknown): string | null; + /** Local writes that must commit with the settled receipt. Runs inside + * its transaction. */ + apply?(result: unknown): void; +}; + +type CreatedVm = { remoteId: string; status: VmStatus; errorMessage?: string }; + +/** Effects for a create or fork: the VM's provider id and observed state + * commit with the receipt, and a VM that booted into "failed" fails it. */ +export function createdVmEffects( + db: Database.Database, + vm: { owner: string; id: string }, + clock: () => number, +): VmOperationEffects { + return { + failure: (result) => + (result as CreatedVm).status === "failed" + ? ((result as CreatedVm).errorMessage ?? + "The provider reports the VM failed") + : null, + apply: (result) => { + const created = result as CreatedVm; + if ( + !experimentVmsQueries.adoptRemote(db, vm.owner, vm.id, created.remoteId) + ) + throw new Error("The experiment already records another provider VM"); + const row = experimentVmsQueries.byId(db, vm.owner, vm.id); + if (row) + applyObservedStatus( + db, + { ...vm, state: row.state }, + created.status, + clock(), + created.errorMessage, + ); + }, + }; } /** Moves the VM to the state the provider reports, when that edge is legal @@ -171,6 +238,7 @@ function applyObservedStatus( vm: { owner: string; id: string; state: VmState }, status: VmStatus, now: number, + reason?: string, ) { const observed: Record = { provisioning: undefined, @@ -182,11 +250,20 @@ function applyObservedStatus( terminated: vm.state === "destroying" ? "destroyed" : "failed", unknown: undefined, }; - const to = observed[status]; + let to = observed[status]; + // A destroy that did not finish leaves a VM the owner must destroy again; + // failed is the state that allows it. + if (to && vm.state === "destroying" && to !== "destroyed") to = "failed"; if (!to || to === vm.state || !legalVmTransition(vm.state, to)) return; transitionVm(db, vm, to, { expected: vm.state, - lastError: to === "failed" ? `The provider reports the VM ${status}` : null, + lastError: + to === "failed" + ? (reason ?? + (vm.state === "destroying" + ? `The destroy did not finish; the provider reports the VM ${status}` + : `The provider reports the VM ${status}`)) + : null, now, }); } @@ -196,9 +273,18 @@ export type VmReconciliation = | { outcome: "failed"; errorMessage: string } | { outcome: "still_uncertain"; reason: string }; +// `notYet`: the provider shows the effect did not happen. The original call +// may still be in flight until the longest provider call has passed, so such +// an observation settles nothing before then. type Observation = | { settle: "succeeded"; result: unknown; status?: VmStatus } - | { settle: "failed"; errorMessage: string; status?: VmStatus } + | { + settle: "failed"; + errorMessage: string; + status?: VmStatus; + result?: unknown; + notYet?: true; + } | { settle: "unknown"; reason: string }; class SettledConcurrently extends Error {} @@ -239,6 +325,15 @@ export async function reconcileVmOperation( } if (observation.settle === "unknown") return { outcome: "still_uncertain", reason: observation.reason }; + if ( + observation.settle === "failed" && + observation.notYet && + clock() - operation.created_at <= VM_OPERATION_MAX_CALL_MS + ) + return { + outcome: "still_uncertain", + reason: "the original call may still be in flight", + }; const seen = observation; const receipt = { owner: target.owner, id: operation.id }; @@ -246,16 +341,6 @@ export async function reconcileVmOperation( db.transaction(() => { const fresh = experimentVmsQueries.byId(db, target.owner, vm.id); if (!fresh) throw new SettledConcurrently(); - if ( - seen.settle === "succeeded" && - (operation.kind === "create" || operation.kind === "fork") - ) { - const { remoteId } = seen.result as { remoteId: string }; - if ( - !experimentVmsQueries.adoptRemote(db, target.owner, vm.id, remoteId) - ) - throw new SettledConcurrently(); - } const settled = seen.settle === "succeeded" ? transitionVmOperation(db, receipt, "succeeded", { @@ -263,10 +348,18 @@ export async function reconcileVmOperation( now: clock(), }) : transitionVmOperation(db, receipt, "failed", { + result: seen.result, errorMessage: seen.errorMessage, now: clock(), }); if (!settled) throw new SettledConcurrently(); + const adopted = + (operation.kind === "create" || operation.kind === "fork") && + (seen.result as { remoteId?: unknown } | undefined)?.remoteId; + if (typeof adopted === "string") { + if (!experimentVmsQueries.adoptRemote(db, target.owner, vm.id, adopted)) + throw new SettledConcurrently(); + } if (seen.status) applyObservedStatus( db, @@ -301,6 +394,14 @@ async function observe( settle: "failed", errorMessage: "The provider has no VM with this name", status: "terminated", + notYet: true, + }; + if (match.status === "failed") + return { + settle: "failed", + errorMessage: "The provider reports the VM failed", + result: { remoteId: match.remoteId }, + status: "failed", }; return { settle: "succeeded", @@ -317,9 +418,17 @@ async function observe( if (operation.kind === "expose") return published ? { settle: "succeeded", result: published } - : { settle: "failed", errorMessage: "The port is not published" }; + : { + settle: "failed", + errorMessage: "The port is not published", + notYet: true, + }; return published - ? { settle: "failed", errorMessage: "The port is still published" } + ? { + settle: "failed", + errorMessage: "The port is still published", + notYet: true, + } : { settle: "succeeded", result: null }; } const status = (await provider.get(vm.remote_id))?.status ?? "terminated"; @@ -337,5 +446,6 @@ async function observe( settle: "failed", errorMessage: `The provider reports the VM ${status}`, status, + notYet: true, }; } diff --git a/src/server/vm/provider.ts b/src/server/vm/provider.ts index 5b950bd8..643a37d1 100644 --- a/src/server/vm/provider.ts +++ b/src/server/vm/provider.ts @@ -21,12 +21,22 @@ export type VmLimits = { diskLimitGib: number; }; +/** A VM can exist and still have failed to start; `status` is "failed" and + * `errorMessage` carries the provider's reason. */ +export type VmCreated = { + remoteId: string; + status: VmStatus; + errorMessage?: string; +}; + export type VmVolumeMount = { volumeId: string; mountPath: string; readOnly?: boolean; }; +/** stdout and stderr are paced by their reader: an unread stream stops the + * provider's event stream, including the exit event, so callers read both. */ export interface VmProcessHandle { stdin: NodeJS.WritableStream; stdout: NodeJS.ReadableStream; @@ -67,7 +77,7 @@ export interface VmProvider { limits: VmLimits; autoSleepSeconds: number; volumes: VmVolumeMount[]; - }): Promise<{ remoteId: string; status: VmStatus }>; + }): Promise; get(remoteId: string): Promise<{ status: VmStatus } | null>; /** Live VMs with exactly this name. More than one means Loop cannot tell * which to adopt, and the caller must leave the operation uncertain. */ @@ -98,7 +108,9 @@ export interface VmProvider { idempotencyKey: string; checkpointId: string; name: string; - }): Promise<{ remoteId: string; status: VmStatus }>; + }): Promise; + /** Mounts a volume on a running VM. Repeating the same mount is safe. */ + mountVolume(remoteId: string, mount: VmVolumeMount): Promise; /** Sets limits and the idle window. Repeating it with the same values is * harmless, so it needs no receipt of its own. */ configure( diff --git a/src/server/vm/sailbox-image.ts b/src/server/vm/sailbox-image.ts index 3742d8fc..e3854b4a 100644 --- a/src/server/vm/sailbox-image.ts +++ b/src/server/vm/sailbox-image.ts @@ -2,10 +2,12 @@ import { createHash } from "node:crypto"; // The experiment image. Sail builds custom images only through its SDK, which // Loop does not bundle, so Loop builds a "golden" Sailbox instead: boot -// SAILBOX_BASE_IMAGE with the auth volume mounted, run INSTALL_SCRIPT over -// exec, take a filesystem-only checkpoint, and start every experiment from -// that checkpoint. Volume contents are not part of a checkpoint, so the -// golden image never holds credentials. The checkpoint id is saved as +// SAILBOX_BASE_IMAGE with no volume mounted, run INSTALL_SCRIPT over exec, +// take a filesystem-only checkpoint, and start every experiment from that +// checkpoint. The third-party installers therefore never run beside the +// harness credentials. Each experiment mounts the auth volume after it starts +// (VmProvider.mountVolume); a fork's checkpoint reattaches it, and volume +// contents are never part of a checkpoint. The checkpoint id is saved as // vm_provider_configs.base_image_ref with SAILBOX_IMAGE_VERSION, and a // version change means the image is rebuilt. // diff --git a/src/server/vm/sailbox-provider.ts b/src/server/vm/sailbox-provider.ts index e06c7ba7..fbd37b41 100644 --- a/src/server/vm/sailbox-provider.ts +++ b/src/server/vm/sailbox-provider.ts @@ -159,7 +159,10 @@ export function createSailboxProvider(options: { // can run; Sail reports the reason in error_message. function created(value: Record) { const remoteId = field(value, "sailbox_id"); - return { remoteId, status: sailboxStatus(value.status) }; + const status = sailboxStatus(value.status); + return typeof value.error_message === "string" && value.error_message + ? { remoteId, status, errorMessage: value.error_message } + : { remoteId, status }; } return { @@ -314,6 +317,16 @@ export function createSailboxProvider(options: { ); }, + async mountVolume(remoteId, mount) { + await request("PUT", `${box(remoteId)}/volume-mounts`, { + body: { + volume_id: mount.volumeId, + mount_path: mount.mountPath, + read_only: mount.readOnly ?? false, + }, + }); + }, + async configure( remoteId, settings: { limits: VmLimits; autoSleepSeconds: number }, @@ -469,7 +482,7 @@ async function startExec( signal: null, }; let buffered = ""; - let blocked: PassThrough | undefined; + const blocked = new Set(); const decoder = new TextDecoder(); const handle = (line: string) => { if (!line.trim()) return; @@ -490,7 +503,8 @@ async function startExec( typeof event.data === "string" ) { const stream = event.type === "stdout" ? stdout : stderr; - if (!stream.write(Buffer.from(event.data, "base64"))) blocked = stream; + if (!stream.write(Buffer.from(event.data, "base64"))) + blocked.add(stream); } else if (event.type === "exit") result = { exitCode: @@ -509,19 +523,19 @@ async function startExec( } // Output is read from the provider only as fast as Loop's consumer // reads it: a full stream pauses the event stream until it drains. - if (blocked) { - const waiting: PassThrough = blocked; - blocked = undefined; - await new Promise((resolve) => { - const done = () => { - waiting.off("drain", done); - waiting.off("close", done); - resolve(); - }; - waiting.on("drain", done); - waiting.on("close", done); - }); - } + // Every full stream must drain, not only the last one that filled. + for (const waiting of blocked) + if (waiting.writableNeedDrain && !waiting.destroyed) + await new Promise((resolve) => { + const done = () => { + waiting.off("drain", done); + waiting.off("close", done); + resolve(); + }; + waiting.on("drain", done); + waiting.on("close", done); + }); + blocked.clear(); } handle(buffered); } catch { diff --git a/test/vm-operations.test.ts b/test/vm-operations.test.ts index d43ea0b3..1a21975f 100644 --- a/test/vm-operations.test.ts +++ b/test/vm-operations.test.ts @@ -10,6 +10,7 @@ import { transitionVm, } from "../src/server/vm/machine.js"; import { + createdVmEffects, reconcileVmOperation, runVmOperation, } from "../src/server/vm/operations.js"; @@ -48,6 +49,7 @@ const create = ( id: string, call: () => Promise, overrides: { owner?: string; experimentId?: string; request?: unknown } = {}, + effects?: Parameters[4], ) => runVmOperation( db, @@ -60,6 +62,7 @@ const create = ( }, call, clock, + effects, ); // A read-only provider for reconcile: VMs by name, statuses by id and @@ -239,10 +242,20 @@ async function uncertainCreate() { throw new VmProviderError("uncertain", "timed out"); }); } +// Reconciles once the original call can no longer be in flight; see +// reconcileSoon for the window before that. const reconcile = ( provider: Parameters[1], owner = "alice", operationId = "op1", +) => { + time += VM_OPERATION_MAX_CALL_MS + 1; + return reconcileSoon(provider, owner, operationId); +}; +const reconcileSoon = ( + provider: Parameters[1], + owner = "alice", + operationId = "op1", ) => reconcileVmOperation( db, @@ -458,6 +471,162 @@ it("reports an uncertain sleep that did not happen as failed, so it can be asked expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe("running"); }); +it("records a created VM's id and state in the same transaction as its receipt", async () => { + const vm = { owner: "alice", id: "exp1" }; + expect( + await create( + "op1", + async () => ({ remoteId: "sb-1", status: "running" }), + {}, + createdVmEffects(db, vm, clock), + ), + ).toMatchObject({ state: "succeeded" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-1", + state: "running", + }); + + experiment("alice", "exp2"); + db.exec( + "CREATE TRIGGER refuse_vm_state BEFORE UPDATE OF state ON experiment_vms BEGIN SELECT RAISE(ABORT, 'disk full'); END", + ); + await expect( + create( + "op2", + async () => ({ remoteId: "sb-2", status: "running" }), + { experimentId: "exp2" }, + createdVmEffects(db, { owner: "alice", id: "exp2" }, clock), + ), + ).rejects.toThrow("disk full"); + // Nothing committed: the sweep and a reconcile take it from here. + expect(vmOperationsQueries.byId(db, "alice", "op2")?.state).toBe("pending"); + expect(experimentVmsQueries.byId(db, "alice", "exp2")?.remote_id).toBeNull(); +}); + +it("fails a create whose VM booted into failed, keeping its id so it can be destroyed", async () => { + expect( + await create( + "op1", + async () => ({ + remoteId: "sb-1", + status: "failed", + errorMessage: "image pull failed", + }), + {}, + createdVmEffects(db, { owner: "alice", id: "exp1" }, clock), + ), + ).toEqual({ + state: "failed", + errorMessage: "image pull failed", + replayed: false, + }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-1", + state: "failed", + last_error: "image pull failed", + }); + expect(vmOperationsQueries.byId(db, "alice", "op1")).toMatchObject({ + state: "failed", + result_json: expect.stringContaining("sb-1"), + }); +}); + +it("reconciles a create whose only match failed as failed, adopting the VM", async () => { + await uncertainCreate(); + expect( + await reconcile(found({ remoteId: "sb-7", status: "failed" })), + ).toMatchObject({ outcome: "failed" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-7", + state: "failed", + }); +}); + +it("does not settle on an absent effect while the original call may still be in flight", async () => { + await uncertainCreate(); + expect(await reconcileSoon(found())).toEqual({ + outcome: "still_uncertain", + reason: "the original call may still be in flight", + }); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("uncertain"); + // A VM that is already there settles at once. + expect( + await reconcileSoon(found({ remoteId: "sb-7", status: "running" })), + ).toMatchObject({ outcome: "succeeded" }); +}); + +it("fails the VM, not leaving it destroying, when an uncertain destroy did not happen", async () => { + await runningVm(); + transitionVm(db, { owner: "alice", id: "exp1" }, "destroying", { + now: clock(), + }); + await runVmOperation( + db, + { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "destroy", + request: {}, + }, + async () => { + throw new VmProviderError("uncertain", "timed out"); + }, + clock, + ); + const provider = found(); + provider.statuses.set("sb-1", "running"); + expect(await reconcile(provider)).toMatchObject({ outcome: "failed" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + state: "failed", + last_error: + "The destroy did not finish; the provider reports the VM running", + }); + expect( + transitionVm(db, { owner: "alice", id: "exp1" }, "destroying", { + now: clock(), + }).changed, + ).toBe(true); +}); + +it("refuses an experiment on another owner's folder or forked from another owner's experiment", () => { + db.prepare( + "INSERT INTO folders(id,owner_id,kind,name,created_at) VALUES('folder-bob','bob','project','b',1)", + ).run(); + const base = { + provider: "sailbox", + autoApproveLocal: true, + vcpuLimit: 4, + memoryLimitGib: 16, + diskLimitGib: 64, + autoSleepSeconds: 900, + sourceCheckpointId: null, + cloneUrl: null, + now: clock(), + }; + expect( + experimentVmsQueries.insert(db, { + ...base, + id: "x1", + owner: "alice", + folderId: "folder-bob", + remoteName: "loop-exp-x1", + sourceExperimentId: null, + }), + ).toBe(false); + expect( + experimentVmsQueries.insert(db, { + ...base, + id: "x2", + owner: "bob", + folderId: "folder-bob", + remoteName: "loop-exp-x2", + sourceExperimentId: "exp1", + }), + ).toBe(false); + expect(experimentVmsQueries.byId(db, "bob", "x2")).toBeUndefined(); +}); + it("refuses illegal and stale VM transitions", () => { const vm = { owner: "alice", id: "exp1" }; expect(transitionVm(db, vm, "destroyed", { now: clock() })).toEqual({ diff --git a/test/vm-sailbox-provider.test.ts b/test/vm-sailbox-provider.test.ts index ea325db5..b5cf7fef 100644 --- a/test/vm-sailbox-provider.test.ts +++ b/test/vm-sailbox-provider.test.ts @@ -344,3 +344,76 @@ it("stops reading exec events while nobody reads the output", async () => { await handle.exit; expect(pulls).toBe(500); }); + +it("waits for every full output stream before reading more events", async () => { + let pulls = 0; + const encoder = new TextEncoder(); + const chunk = Buffer.alloc(64 * 1024, 1).toString("base64"); + const { provider } = fake( + () => + new Response( + new ReadableStream({ + pull(controller) { + pulls += 1; + // stdout and stderr fill within one chunk. + controller.enqueue( + encoder.encode( + (pulls === 1 + ? JSON.stringify({ type: "started", exec_request_id: "e" }) + + "\n" + : "") + + JSON.stringify({ type: "stdout", data: chunk, seq: pulls }) + + "\n" + + JSON.stringify({ type: "stderr", data: chunk, seq: pulls }) + + "\n", + ), + ); + if (pulls === 200) controller.close(); + }, + }), + ), + ); + const handle = await provider.exec("sb_1", ["yes"], {}); + // Only stderr is read; stdout stays full. + handle.stderr.resume(); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(pulls).toBeLessThan(10); + handle.stdout.resume(); + await handle.exit; + expect(pulls).toBe(200); +}); + +it("reports a VM that booted into failed with Sail's reason, and mounts volumes on a running VM", async () => { + const { calls, provider } = fake((call) => + call.url.pathname.endsWith("/from_checkpoint") + ? ok({ + sailbox_id: "sb_2", + status: "create_failed", + error_message: "no capacity", + }) + : ok({}), + ); + expect( + await provider.createFromCheckpoint({ + idempotencyKey: "exp-1", + checkpointId: "ckpt_1", + name: "loop-exp-1", + }), + ).toEqual({ + remoteId: "sb_2", + status: "failed", + errorMessage: "no capacity", + }); + await provider.mountVolume("sb_2", { + volumeId: "vol_1", + mountPath: "/mnt/loop-auth", + }); + const mount = calls[1]; + expect(mount.method).toBe("PUT"); + expect(mount.url.pathname).toBe("/v1/sailboxes/sb_2/volume-mounts"); + expect(mount.body).toEqual({ + volume_id: "vol_1", + mount_path: "/mnt/loop-auth", + read_only: false, + }); +}); From a747d16117de5a40c158adb7318e786a5dfb60a0 Mon Sep 17 00:00:00 2001 From: Jamie Asefa <13409617+j-asefa@users.noreply.github.com> Date: Fri, 20 Nov 2026 19:06:43 +0000 Subject: [PATCH 5/9] Cloud VM experiments 1/5: one set of local effects for answers and reconciles Review pass 3 fixes: - A negative observation counts only when its lookup began after the in-flight window, not when its answer arrived. - A create or fork whose VM is still starting (or in an unrecognised state) stays uncertain instead of succeeding. - vmOperationEffects holds each kind's local writes: provider id and VM state for create/fork/sleep/resume/destroy, preview rows for expose and unexpose, and disabling every preview on destroy. runVmOperation and reconcileVmOperation apply the same effects in the settling transaction. - experiment_previews upserts require the owner's experiment. Also from pass 3's open concerns: a refused destroy moves a destroying VM to failed, and a 408 answer is uncertain rather than a refusal. Co-Authored-By: Claude Opus 5.5 --- docs/state-machines.md | 2 +- src/server/sql/experiment-previews.ts | 31 ++-- src/server/vm/operations.ts | 200 +++++++++++++++++++++----- src/server/vm/sailbox-provider.ts | 7 +- test/vm-operations.test.ts | 166 ++++++++++++++++++++- 5 files changed, 345 insertions(+), 61 deletions(-) diff --git a/docs/state-machines.md b/docs/state-machines.md index 137688d2..5341a4f8 100644 --- a/docs/state-machines.md +++ b/docs/state-machines.md @@ -397,7 +397,7 @@ before the provider is called, and its id is the caller's idempotency key. | **`succeeded`** | — | Terminal | | **`failed`** | — | Terminal | -An uncertain operation is never sent again automatically. Whichever settles the receipt first wins; a later answer that disagrees returns the stored outcome. A reconcile writes the receipt and the VM's observed state in one transaction, as does a settled create with the VM's provider id. An observation that the effect did not happen settles nothing until the longest provider call has passed since the receipt was written, because the original call may still be in flight. A create whose VM boots into a failed state fails its receipt and keeps the VM's id, so the owner can destroy it. A retry is a new +An uncertain operation is never sent again automatically. Whichever settles the receipt first wins; a later answer that disagrees returns the stored outcome. `vmOperationEffects` in [vm/operations.ts](../src/server/vm/operations.ts) holds each kind's local writes (provider id, VM state, preview rows), and both an answer and a reconcile apply them in the transaction that settles the receipt. An observation that the effect did not happen settles nothing unless the lookup began after the longest provider call had passed since the receipt was written, because the original call may still be in flight. A create stays uncertain while its VM is still starting. A create whose VM boots into a failed state fails its receipt and keeps the VM's id, so the owner can destroy it. A retry is a new operation with a new id, and is refused while an earlier one for the same experiment is unsettled. Creates and forks send the provider an idempotency key scoped to the experiment, so a retry after an "absent" reconcile reaches diff --git a/src/server/sql/experiment-previews.ts b/src/server/sql/experiment-previews.ts index fe6187d0..29418594 100644 --- a/src/server/sql/experiment-previews.ts +++ b/src/server/sql/experiment-previews.ts @@ -8,10 +8,13 @@ import { statement } from "./statement.js"; const parse = (row: unknown) => parseRecord(QueryRecords.experiment_previews, row, "experiment_previews"); +// INSERT … SELECT needs a WHERE before ON CONFLICT so SQLite does not read +// the upsert clause as a join constraint. const upsertStatement = statement( - "INSERT INTO experiment_previews(id,owner_id,experiment_id,port,label,path,url,active,created_at,updated_at) VALUES(?,?,?,?,?,?,?,1,?,?) ON CONFLICT(experiment_id,port) DO UPDATE SET label=excluded.label,path=excluded.path,url=excluded.url,active=1,updated_at=excluded.updated_at WHERE owner_id=excluded.owner_id", + "INSERT INTO experiment_previews(id,owner_id,experiment_id,port,label,path,url,active,created_at,updated_at) SELECT ?,?,?,?,?,?,?,1,?,? WHERE EXISTS(SELECT 1 FROM experiment_vms WHERE id=? AND owner_id=?) ON CONFLICT(experiment_id,port) DO UPDATE SET label=excluded.label,path=excluded.path,url=excluded.url,active=1,updated_at=excluded.updated_at WHERE experiment_previews.owner_id=excluded.owner_id", ); -/** One preview per experiment port; exposing the port again reactivates it. */ +/** One preview per experiment port; exposing the port again reactivates it. + * Returns false, writing nothing, when the experiment is not the owner's. */ export function upsert( db: Database.Database, preview: { @@ -25,16 +28,20 @@ export function upsert( now: number; }, ) { - upsertStatement(db).run( - preview.id, - preview.owner, - preview.experimentId, - preview.port, - preview.label, - preview.path, - preview.url, - preview.now, - preview.now, + return ( + upsertStatement(db).run( + preview.id, + preview.owner, + preview.experimentId, + preview.port, + preview.label, + preview.path, + preview.url, + preview.now, + preview.now, + preview.experimentId, + preview.owner, + ).changes === 1 ); } diff --git a/src/server/vm/operations.ts b/src/server/vm/operations.ts index 7018d696..c515a664 100644 --- a/src/server/vm/operations.ts +++ b/src/server/vm/operations.ts @@ -5,6 +5,7 @@ import { type VmOperationKind, type VmState, } from "../../shared/vm-state.js"; +import * as experimentPreviewsQueries from "../sql/experiment-previews.js"; import * as experimentVmsQueries from "../sql/experiment-vms.js"; import * as vmOperationsQueries from "../sql/vm-operations.js"; import { @@ -156,8 +157,20 @@ export async function runVmOperation( error instanceof VmProviderError && error.outcome === "refused" ? "failed" : "uncertain"; - if (!transitionVmOperation(db, receipt, to, { errorMessage, now: clock() })) - return current(); + const settled = db + .transaction(() => { + if ( + !transitionVmOperation(db, receipt, to, { + errorMessage, + now: clock(), + }) + ) + return false; + if (to === "failed") effects.refused?.(errorMessage); + return true; + }) + .immediate(); + if (!settled) return current(); return to === "failed" ? { state: "failed", errorMessage, replayed: false } : { state: "uncertain", errorMessage }; @@ -187,46 +200,134 @@ export async function runVmOperation( : { state: "failed", errorMessage, replayed: false }; } -/** What a resolved call means locally. */ +/** The local record cannot take what the provider answered. */ +export class VmRecordConflict extends Error { + constructor(message: string) { + super(message); + this.name = "VmRecordConflict"; + } +} + +/** What a call's answer means locally. Each hook runs inside the + * transaction that settles the receipt, so the local record and the receipt + * commit together or not at all. */ export type VmOperationEffects = { /** A failure the answer itself reports, such as a VM that booted into * "failed"; null when the call did what was asked. */ failure?(result: unknown): string | null; - /** Local writes that must commit with the settled receipt. Runs inside - * its transaction. */ + /** Local writes for a resolved call, including one `failure` failed. */ apply?(result: unknown): void; + /** Local writes for a provider refusal. */ + refused?(errorMessage: string): void; }; type CreatedVm = { remoteId: string; status: VmStatus; errorMessage?: string }; -/** Effects for a create or fork: the VM's provider id and observed state - * commit with the receipt, and a VM that booted into "failed" fails it. */ -export function createdVmEffects( +const kindStatus: Partial> = { + sleep: "sleeping", + resume: "running", + destroy: "terminated", +}; + +/** The local writes for each operation kind. runVmOperation and + * reconcileVmOperation both use them, so an answer and a reconcile leave the + * same record: a create or fork adopts the provider id and observed state, + * sleep/resume/destroy move the VM, expose and unexpose keep the preview + * rows in step, and a destroy disables every preview link. */ +export function vmOperationEffects( db: Database.Database, - vm: { owner: string; id: string }, + operation: { + owner: string; + experimentId: string; + id: string; + kind: VmOperationKind; + request: unknown; + }, clock: () => number, ): VmOperationEffects { + const vm = { owner: operation.owner, id: operation.experimentId }; + const current = () => { + const row = experimentVmsQueries.byId(db, vm.owner, vm.id); + if (!row) throw new VmRecordConflict("The experiment no longer exists"); + return { ...vm, state: row.state }; + }; + const created = operation.kind === "create" || operation.kind === "fork"; return { failure: (result) => - (result as CreatedVm).status === "failed" + created && (result as CreatedVm).status === "failed" ? ((result as CreatedVm).errorMessage ?? "The provider reports the VM failed") : null, apply: (result) => { - const created = result as CreatedVm; - if ( - !experimentVmsQueries.adoptRemote(db, vm.owner, vm.id, created.remoteId) - ) - throw new Error("The experiment already records another provider VM"); - const row = experimentVmsQueries.byId(db, vm.owner, vm.id); - if (row) + if (created) { + const vmResult = result as CreatedVm; + if ( + !experimentVmsQueries.adoptRemote( + db, + vm.owner, + vm.id, + vmResult.remoteId, + ) + ) + throw new VmRecordConflict( + "The experiment already records another provider VM", + ); applyObservedStatus( db, - { ...vm, state: row.state }, - created.status, + current(), + vmResult.status, clock(), - created.errorMessage, + vmResult.errorMessage, ); + return; + } + const status = kindStatus[operation.kind]; + if (status) applyObservedStatus(db, current(), status, clock()); + if (operation.kind === "destroy") + experimentPreviewsQueries.deactivateAll(db, vm.owner, vm.id, clock()); + if (operation.kind === "expose" || operation.kind === "unexpose") { + const request = (operation.request ?? {}) as { + port?: unknown; + label?: unknown; + path?: unknown; + }; + if (typeof request.port !== "number") + throw new VmRecordConflict("The operation names no port"); + if (operation.kind === "unexpose") { + experimentPreviewsQueries.deactivate( + db, + vm.owner, + vm.id, + request.port, + clock(), + ); + return; + } + const { url } = result as { url: string }; + if ( + !experimentPreviewsQueries.upsert(db, { + id: operation.id, + owner: vm.owner, + experimentId: vm.id, + port: request.port, + label: typeof request.label === "string" ? request.label : null, + path: typeof request.path === "string" ? request.path : "/", + url, + now: clock(), + }) + ) + throw new VmRecordConflict("The experiment is not the owner's"); + } + }, + refused: (errorMessage) => { + if (operation.kind !== "destroy") return; + const row = current(); + if (row.state === "destroying") + transitionVm(db, row, "failed", { + expected: "destroying", + lastError: errorMessage, + now: clock(), + }); }, }; } @@ -315,6 +416,10 @@ export async function reconcileVmOperation( ); if (!vm) return { outcome: "still_uncertain", reason: "experiment missing" }; + // The age is taken before the provider is asked: an observation counts as + // "did not happen" only if it was made after the original call could have + // finished. + const observedAt = clock(); let observation: Observation; try { observation = await observe(provider, operation, vm, target.appId); @@ -328,7 +433,7 @@ export async function reconcileVmOperation( if ( observation.settle === "failed" && observation.notYet && - clock() - operation.created_at <= VM_OPERATION_MAX_CALL_MS + observedAt - operation.created_at <= VM_OPERATION_MAX_CALL_MS ) return { outcome: "still_uncertain", @@ -353,20 +458,34 @@ export async function reconcileVmOperation( now: clock(), }); if (!settled) throw new SettledConcurrently(); - const adopted = - (operation.kind === "create" || operation.kind === "fork") && - (seen.result as { remoteId?: unknown } | undefined)?.remoteId; - if (typeof adopted === "string") { - if (!experimentVmsQueries.adoptRemote(db, target.owner, vm.id, adopted)) - throw new SettledConcurrently(); + const effects = vmOperationEffects( + db, + { + owner: target.owner, + experimentId: vm.id, + id: operation.id, + kind: operation.kind, + request: JSON.parse(operation.request_json), + }, + clock, + ); + try { + if (seen.settle === "succeeded" || seen.result !== undefined) + effects.apply?.(seen.result); + else if (seen.status) + applyObservedStatus( + db, + { owner: target.owner, id: vm.id, state: fresh.state }, + seen.status, + clock(), + ); + } catch (error) { + // The local record no longer matches what was observed (another + // writer got there first): keep the receipt uncertain. Any other + // error, such as a failed write, propagates and rolls back. + if (error instanceof VmRecordConflict) throw new SettledConcurrently(); + throw error; } - if (seen.status) - applyObservedStatus( - db, - { owner: target.owner, id: vm.id, state: fresh.state }, - seen.status, - clock(), - ); }).immediate(); } catch (error) { if (error instanceof SettledConcurrently) @@ -400,13 +519,18 @@ async function observe( return { settle: "failed", errorMessage: "The provider reports the VM failed", - result: { remoteId: match.remoteId }, - status: "failed", + result: { remoteId: match.remoteId, status: "failed" }, }; + // Only a VM that finished starting shows the create succeeded. + if ( + match.status !== "running" && + match.status !== "sleeping" && + match.status !== "paused" + ) + return unsure(`the VM is ${match.status}`); return { settle: "succeeded", - result: { remoteId: match.remoteId }, - status: match.status, + result: { remoteId: match.remoteId, status: match.status }, }; } if (!vm.remote_id) return unsure("the VM has no provider id"); diff --git a/src/server/vm/sailbox-provider.ts b/src/server/vm/sailbox-provider.ts index fbd37b41..fe3662ef 100644 --- a/src/server/vm/sailbox-provider.ts +++ b/src/server/vm/sailbox-provider.ts @@ -52,11 +52,12 @@ function message(body: unknown, fallback: string) { return typeof error?.message === "string" ? error.message : fallback; } -// A 4xx answer, including 429 (Sail documents that the request never ran), -// means nothing happened. 501 is a definite refusal too. Any other 5xx, a +// A 4xx answer other than 408, including 429 (Sail documents that the +// request never ran), means nothing happened. 501 is a definite refusal too. Any other 5xx, a // timeout or a dropped connection may follow a change Sail already made. function refusedStatus(status: number) { - return (status >= 400 && status < 500) || status === 501; + // 408 is a timeout, which proves nothing about the mutation. + return (status >= 400 && status < 500 && status !== 408) || status === 501; } export function createSailboxProvider(options: { diff --git a/test/vm-operations.test.ts b/test/vm-operations.test.ts index 1a21975f..188e2779 100644 --- a/test/vm-operations.test.ts +++ b/test/vm-operations.test.ts @@ -2,6 +2,7 @@ import Database from "better-sqlite3"; import { beforeEach, expect, it } from "vitest"; import { applyMigrations } from "../src/server/migrations.js"; import { applyMigrationsThrough } from "./helpers/migrations.js"; +import * as experimentPreviewsQueries from "../src/server/sql/experiment-previews.js"; import * as experimentVmsQueries from "../src/server/sql/experiment-vms.js"; import * as vmOperationsQueries from "../src/server/sql/vm-operations.js"; import { @@ -10,7 +11,7 @@ import { transitionVm, } from "../src/server/vm/machine.js"; import { - createdVmEffects, + vmOperationEffects, reconcileVmOperation, runVmOperation, } from "../src/server/vm/operations.js"; @@ -65,6 +66,13 @@ const create = ( effects, ); +const createEffects = (experimentId: string, id: string) => + vmOperationEffects( + db, + { owner: "alice", experimentId, id, kind: "create", request: {} }, + clock, + ); + // A read-only provider for reconcile: VMs by name, statuses by id and // published ports. Every lookup is counted. const found = (...matches: { remoteId: string; status: VmStatus }[]) => ({ @@ -268,14 +276,17 @@ it("adopts the one VM that carries the experiment's name", async () => { await uncertainCreate(); expect( await reconcile(found({ remoteId: "sb-7", status: "running" })), - ).toEqual({ outcome: "succeeded", result: { remoteId: "sb-7" } }); + ).toEqual({ + outcome: "succeeded", + result: { remoteId: "sb-7", status: "running" }, + }); expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ remote_id: "sb-7", state: "running", }); expect(vmOperationsQueries.byId(db, "alice", "op1")).toMatchObject({ state: "succeeded", - result_json: '{"remoteId":"sb-7"}', + result_json: '{"remoteId":"sb-7","status":"running"}', }); }); @@ -472,13 +483,12 @@ it("reports an uncertain sleep that did not happen as failed, so it can be asked }); it("records a created VM's id and state in the same transaction as its receipt", async () => { - const vm = { owner: "alice", id: "exp1" }; expect( await create( "op1", async () => ({ remoteId: "sb-1", status: "running" }), {}, - createdVmEffects(db, vm, clock), + createEffects("exp1", "op1"), ), ).toMatchObject({ state: "succeeded" }); expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ @@ -495,7 +505,7 @@ it("records a created VM's id and state in the same transaction as its receipt", "op2", async () => ({ remoteId: "sb-2", status: "running" }), { experimentId: "exp2" }, - createdVmEffects(db, { owner: "alice", id: "exp2" }, clock), + createEffects("exp2", "op2"), ), ).rejects.toThrow("disk full"); // Nothing committed: the sweep and a reconcile take it from here. @@ -513,7 +523,7 @@ it("fails a create whose VM booted into failed, keeping its id so it can be dest errorMessage: "image pull failed", }), {}, - createdVmEffects(db, { owner: "alice", id: "exp1" }, clock), + createEffects("exp1", "op1"), ), ).toEqual({ state: "failed", @@ -627,6 +637,148 @@ it("refuses an experiment on another owner's folder or forked from another owner expect(experimentVmsQueries.byId(db, "bob", "x2")).toBeUndefined(); }); +it("counts a lookup as too early when it started inside the in-flight window", async () => { + await uncertainCreate(); + const created = time; + const slow = { + ...found(), + async findByName() { + // The answer arrives after the window, but the lookup began inside it. + time = created + VM_OPERATION_MAX_CALL_MS + 10; + return []; + }, + }; + expect(await reconcileSoon(slow)).toEqual({ + outcome: "still_uncertain", + reason: "the original call may still be in flight", + }); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("uncertain"); +}); + +it("leaves a create uncertain while its VM is still starting", async () => { + await uncertainCreate(); + for (const status of ["provisioning", "unknown", "terminating"] as const) + expect(await reconcile(found({ remoteId: "sb-7", status }))).toMatchObject({ + outcome: "still_uncertain", + }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.remote_id).toBeNull(); +}); + +async function uncertain( + id: string, + kind: "expose" | "unexpose" | "destroy", + request: unknown, +) { + const operation = { id, owner: "alice", experimentId: "exp1", kind, request }; + await runVmOperation( + db, + operation, + async () => { + throw new VmProviderError("uncertain", "timed out"); + }, + clock, + vmOperationEffects(db, operation, clock), + ); +} + +it("keeps preview rows in step when expose, unexpose and destroy are reconciled", async () => { + await runningVm(); + await uncertain("op1", "expose", { port: 3000, label: "Demo" }); + const provider = found(); + provider.ports.set(3000, "https://p.example"); + provider.statuses.set("sb-1", "running"); + await reconcile(provider); + expect( + experimentPreviewsQueries.byPort(db, "alice", "exp1", 3000), + ).toMatchObject({ url: "https://p.example", label: "Demo", active: 1 }); + + await uncertain("op2", "unexpose", { port: 3000 }); + provider.ports.delete(3000); + expect(await reconcile(provider, "alice", "op2")).toMatchObject({ + outcome: "succeeded", + }); + expect( + experimentPreviewsQueries.byPort(db, "alice", "exp1", 3000)?.active, + ).toBe(0); + + experimentPreviewsQueries.upsert(db, { + id: "pv2", + owner: "alice", + experimentId: "exp1", + port: 4000, + label: null, + path: "/", + url: "https://q.example", + now: clock(), + }); + transitionVm(db, { owner: "alice", id: "exp1" }, "destroying", { + now: clock(), + }); + await uncertain("op3", "destroy", {}); + provider.statuses.delete("sb-1"); + expect(await reconcile(provider, "alice", "op3")).toMatchObject({ + outcome: "succeeded", + }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe( + "destroyed", + ); + expect( + experimentPreviewsQueries + .byExperiment(db, "alice", "exp1") + .filter((preview) => preview.active === 1), + ).toEqual([]); +}); + +it("fails a destroying VM when the provider refuses the destroy", async () => { + await runningVm(); + transitionVm(db, { owner: "alice", id: "exp1" }, "destroying", { + now: clock(), + }); + const operation = { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "destroy" as const, + request: {}, + }; + expect( + await runVmOperation( + db, + operation, + async () => { + throw new VmProviderError("refused", "Sailbox is locked", 409); + }, + clock, + vmOperationEffects(db, operation, clock), + ), + ).toMatchObject({ state: "failed" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + state: "failed", + last_error: "Sailbox is locked", + }); +}); + +it("refuses a preview on another owner's experiment", () => { + const preview = { + id: "pv1", + owner: "bob", + experimentId: "exp1", + port: 3000, + label: null, + path: "/", + url: "https://evil.example", + now: clock(), + }; + expect(experimentPreviewsQueries.upsert(db, preview)).toBe(false); + expect( + experimentPreviewsQueries.upsert(db, { ...preview, owner: "alice" }), + ).toBe(true); + expect(experimentPreviewsQueries.byPort(db, "alice", "exp1", 3000)?.url).toBe( + "https://evil.example", + ); + expect(experimentPreviewsQueries.byExperiment(db, "bob", "exp1")).toEqual([]); +}); + it("refuses illegal and stale VM transitions", () => { const vm = { owner: "alice", id: "exp1" }; expect(transitionVm(db, vm, "destroyed", { now: clock() })).toEqual({ From ac62eeedff24369966d10c4764821e6a68beacba Mon Sep 17 00:00:00 2001 From: Jamie Asefa <13409617+j-asefa@users.noreply.github.com> Date: Fri, 20 Nov 2026 19:13:50 +0000 Subject: [PATCH 6/9] Cloud VM experiments 1/5: one reconcile timing rule, owner-scoped ids Review pass 4 fixes: - A reconcile settles only from a lookup that began after the in-flight window, whatever it saw; only adopting an existing VM for a create or fork settles at once, since a late original create reaches the same VM. This replaces the per-case "did not happen" flag that left positive observations (an absent port after unexpose) open to a late call. - A create or fork answered while its VM is still starting keeps its receipt uncertain, with the provider id adopted, for a reconcile. - failed -> sleeping/paused: a retry or reconcile can adopt a VM found asleep after an earlier attempt failed. - A listing entry without a name is uncertain, not skipped. From pass 4's open concerns: - vm_operations ids are unique per owner (PRIMARY KEY (owner_id, id)), so one owner's ids neither collide with nor reveal another's; preview ids are derived from the experiment and port. - Each owner gets its own auth volume name. Co-Authored-By: Claude Opus 5.5 --- docs/sql-index.md | 1 - docs/state-machines.md | 4 +- migrations/167-experiment-vms.sql | 7 +-- src/server/sql/vm-operations.ts | 6 --- src/server/vm/operations.ts | 60 ++++++++++++++++--------- src/server/vm/sailbox-image.ts | 6 ++- src/server/vm/sailbox-provider.ts | 6 ++- src/shared/vm-state.ts | 3 +- test/vm-operations.test.ts | 75 ++++++++++++++++++++++++++++--- test/vm-sailbox-provider.test.ts | 1 + 10 files changed, 127 insertions(+), 42 deletions(-) diff --git a/docs/sql-index.md b/docs/sql-index.md index 5b623117..dc1ebb73 100644 --- a/docs/sql-index.md +++ b/docs/sql-index.md @@ -1660,7 +1660,6 @@ Generated by `npm run sql-index` from [src/server/sql](../src/server/sql/); do n ## [vm-operations.ts](../src/server/sql/vm-operations.ts) - `byId(db, owner, id)` -- `idTaken(db, id)` — Whether any owner holds this id, without revealing whose it is. - `insertPending(db, operation)` - `pendingBefore(db, cutoff)` — Every owner's receipts still pending since before \`cutoff\`, for the recovery sweep only. - `setState(db, owner, id, from, to, outcome, now)` — Moves `from` to `to`; returns false when the row is no longer in `from`. diff --git a/docs/state-machines.md b/docs/state-machines.md index 5341a4f8..406fbb27 100644 --- a/docs/state-machines.md +++ b/docs/state-machines.md @@ -380,7 +380,7 @@ leaves its project, threads, results and published files readable. | `running` | `sleeping` / `paused` / `destroying` / `failed` | Sleep observed or requested, pause, owner destroy, or a provider failure observed; **tx** | | `sleeping` | `running` / `destroying` / `failed` | Wake for a turn or a preview, owner destroy, or a failed wake; **tx** | | `paused` | `running` / `destroying` / `failed` | Resume, owner destroy, or a failed resume; **tx** | -| `failed` | `provisioning` / `running` / `destroying` | The owner's reconcile or retry, or owner destroy; **tx** | +| `failed` | `provisioning` / `running` / `sleeping` / `paused` / `destroying` | The owner's reconcile or retry (which can adopt a VM found asleep or paused), or owner destroy; **tx** | | `destroying` | `destroyed` / `failed` | The provider confirmed the VM is gone (a missing VM counts), refused, or a reconcile found the destroy did not finish; **tx** | | **`destroyed`** | — | Terminal | @@ -397,7 +397,7 @@ before the provider is called, and its id is the caller's idempotency key. | **`succeeded`** | — | Terminal | | **`failed`** | — | Terminal | -An uncertain operation is never sent again automatically. Whichever settles the receipt first wins; a later answer that disagrees returns the stored outcome. `vmOperationEffects` in [vm/operations.ts](../src/server/vm/operations.ts) holds each kind's local writes (provider id, VM state, preview rows), and both an answer and a reconcile apply them in the transaction that settles the receipt. An observation that the effect did not happen settles nothing unless the lookup began after the longest provider call had passed since the receipt was written, because the original call may still be in flight. A create stays uncertain while its VM is still starting. A create whose VM boots into a failed state fails its receipt and keeps the VM's id, so the owner can destroy it. A retry is a new +An uncertain operation is never sent again automatically. Whichever settles the receipt first wins; a later answer that disagrees returns the stored outcome. `vmOperationEffects` in [vm/operations.ts](../src/server/vm/operations.ts) holds each kind's local writes (provider id, VM state, preview rows), and both an answer and a reconcile apply them in the transaction that settles the receipt. A reconcile settles only from a lookup that began after the longest provider call had passed since the receipt was written, because the original call may still land and change what the lookup saw. The exception is a create or fork whose VM exists, which settles at once: the original create reaches the same VM through its experiment-scoped provider key. A create answered while its VM is still starting stays uncertain. The window is Loop's own call timeout; Sail does not document a deadline for applying a request it accepted. A create whose VM boots into a failed state fails its receipt and keeps the VM's id, so the owner can destroy it. A retry is a new operation with a new id, and is refused while an earlier one for the same experiment is unsettled. Creates and forks send the provider an idempotency key scoped to the experiment, so a retry after an "absent" reconcile reaches diff --git a/migrations/167-experiment-vms.sql b/migrations/167-experiment-vms.sql index f956d355..f31d7b5c 100644 --- a/migrations/167-experiment-vms.sql +++ b/migrations/167-experiment-vms.sql @@ -57,12 +57,12 @@ CREATE TABLE experiment_previews ( ); -- A receipt written before every provider call. id is the caller's --- idempotency key; request_digest refuses the same key for a different +-- idempotency key, unique within the owner; request_digest refuses the same key for a different -- request, and request_json keeps the request so a reconcile can check what -- was asked. An operation left pending by a crash becomes uncertain and is -- never sent again without the owner asking. CREATE TABLE vm_operations ( - id TEXT PRIMARY KEY, + id TEXT NOT NULL, owner_id TEXT NOT NULL, experiment_id TEXT NOT NULL REFERENCES experiment_vms(id), kind TEXT NOT NULL, @@ -72,7 +72,8 @@ CREATE TABLE vm_operations ( result_json TEXT, error_message TEXT, created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL + updated_at INTEGER NOT NULL, + PRIMARY KEY (owner_id, id) ); CREATE INDEX vm_operations_experiment ON vm_operations(experiment_id, created_at); CREATE INDEX vm_operations_unsettled ON vm_operations(state) WHERE state IN ('pending', 'uncertain'); diff --git a/src/server/sql/vm-operations.ts b/src/server/sql/vm-operations.ts index bc4b40d1..534305e4 100644 --- a/src/server/sql/vm-operations.ts +++ b/src/server/sql/vm-operations.ts @@ -45,12 +45,6 @@ export function byId(db: Database.Database, owner: string, id: string) { return row === undefined ? undefined : parse(row); } -const idTakenStatement = statement("SELECT 1 FROM vm_operations WHERE id=?"); -/** Whether any owner holds this id, without revealing whose it is. */ -export function idTaken(db: Database.Database, id: string) { - return idTakenStatement(db).get(id) !== undefined; -} - const unsettledByExperimentStatement = statement( "SELECT * FROM vm_operations WHERE owner_id=? AND experiment_id=? AND state IN ('pending','uncertain') ORDER BY created_at, id", ); diff --git a/src/server/vm/operations.ts b/src/server/vm/operations.ts index c515a664..9a818c44 100644 --- a/src/server/vm/operations.ts +++ b/src/server/vm/operations.ts @@ -26,7 +26,7 @@ export type VmOperationOutcome = | { state: "uncertain" | "pending"; errorMessage: string | null }; export type VmOperationRefusal = - /** The id was used for a different request, or belongs to someone else. */ + /** The owner already used the id for a different request. */ | "key_conflict" /** The experiment does not exist for this owner. */ | "missing_experiment" @@ -115,8 +115,6 @@ export async function runVmOperation( throw new VmOperationRefused("key_conflict"); return existing; } - if (vmOperationsQueries.idTaken(db, operation.id)) - throw new VmOperationRefused("key_conflict"); if ( !experimentVmsQueries.byId(db, operation.owner, operation.experimentId) ) @@ -175,8 +173,14 @@ export async function runVmOperation( ? { state: "failed", errorMessage, replayed: false } : { state: "uncertain", errorMessage }; } - const errorMessage = effects.failure?.(result) ?? null; - const to = errorMessage === null ? "succeeded" : "failed"; + const incomplete = effects.incomplete?.(result) ?? null; + const errorMessage = incomplete ?? effects.failure?.(result) ?? null; + const to = + incomplete !== null + ? "uncertain" + : errorMessage === null + ? "succeeded" + : "failed"; // The receipt and the local record of what the call did commit together. // If apply throws, both roll back and the receipt stays pending until the // sweep marks it uncertain for a reconcile. @@ -195,6 +199,7 @@ export async function runVmOperation( }) .immediate(); if (!settled) return current(); + if (to === "uncertain") return { state: "uncertain", errorMessage }; return errorMessage === null ? { state: "succeeded", result, replayed: false } : { state: "failed", errorMessage, replayed: false }; @@ -212,6 +217,10 @@ export class VmRecordConflict extends Error { * transaction that settles the receipt, so the local record and the receipt * commit together or not at all. */ export type VmOperationEffects = { + /** An answer that does not show the outcome yet, such as a VM still + * starting: the receipt stays uncertain (with the answer and `apply`'s + * writes kept) for a reconcile to settle. */ + incomplete?(result: unknown): string | null; /** A failure the answer itself reports, such as a VM that booted into * "failed"; null when the call did what was asked. */ failure?(result: unknown): string | null; @@ -253,6 +262,16 @@ export function vmOperationEffects( }; const created = operation.kind === "create" || operation.kind === "fork"; return { + incomplete: (result) => { + if (!created) return null; + const { status } = result as CreatedVm; + return status === "running" || + status === "sleeping" || + status === "paused" || + status === "failed" + ? null + : `The VM is ${status}`; + }, failure: (result) => created && (result as CreatedVm).status === "failed" ? ((result as CreatedVm).errorMessage ?? @@ -306,7 +325,8 @@ export function vmOperationEffects( const { url } = result as { url: string }; if ( !experimentPreviewsQueries.upsert(db, { - id: operation.id, + // Experiment ids are unique across owners; operation ids are not. + id: `${vm.id}:${request.port}`, owner: vm.owner, experimentId: vm.id, port: request.port, @@ -374,17 +394,21 @@ export type VmReconciliation = | { outcome: "failed"; errorMessage: string } | { outcome: "still_uncertain"; reason: string }; -// `notYet`: the provider shows the effect did not happen. The original call -// may still be in flight until the longest provider call has passed, so such -// an observation settles nothing before then. +// Until the longest provider call has passed, the original call may still +// land and change what a lookup saw: a late delete can remove a listener a +// later expose added, a late sleep can stop a VM a later resume woke. So an +// observation settles only when its lookup began after that window. The one +// exception is `early`: a create or fork whose VM exists, because a late +// arrival of the original create reaches the same VM (its provider +// idempotency key is scoped to the experiment) and cannot undo it. type Observation = - | { settle: "succeeded"; result: unknown; status?: VmStatus } + | { settle: "succeeded"; result: unknown; status?: VmStatus; early?: true } | { settle: "failed"; errorMessage: string; status?: VmStatus; result?: unknown; - notYet?: true; + early?: true; } | { settle: "unknown"; reason: string }; @@ -416,9 +440,8 @@ export async function reconcileVmOperation( ); if (!vm) return { outcome: "still_uncertain", reason: "experiment missing" }; - // The age is taken before the provider is asked: an observation counts as - // "did not happen" only if it was made after the original call could have - // finished. + // The age is taken before the provider is asked, so the lookup itself + // must begin after the original call could have finished. const observedAt = clock(); let observation: Observation; try { @@ -431,8 +454,7 @@ export async function reconcileVmOperation( if (observation.settle === "unknown") return { outcome: "still_uncertain", reason: observation.reason }; if ( - observation.settle === "failed" && - observation.notYet && + !observation.early && observedAt - operation.created_at <= VM_OPERATION_MAX_CALL_MS ) return { @@ -513,13 +535,13 @@ async function observe( settle: "failed", errorMessage: "The provider has no VM with this name", status: "terminated", - notYet: true, }; if (match.status === "failed") return { settle: "failed", errorMessage: "The provider reports the VM failed", result: { remoteId: match.remoteId, status: "failed" }, + early: true, }; // Only a VM that finished starting shows the create succeeded. if ( @@ -531,6 +553,7 @@ async function observe( return { settle: "succeeded", result: { remoteId: match.remoteId, status: match.status }, + early: true, }; } if (!vm.remote_id) return unsure("the VM has no provider id"); @@ -545,13 +568,11 @@ async function observe( : { settle: "failed", errorMessage: "The port is not published", - notYet: true, }; return published ? { settle: "failed", errorMessage: "The port is still published", - notYet: true, } : { settle: "succeeded", result: null }; } @@ -570,6 +591,5 @@ async function observe( settle: "failed", errorMessage: `The provider reports the VM ${status}`, status, - notYet: true, }; } diff --git a/src/server/vm/sailbox-image.ts b/src/server/vm/sailbox-image.ts index e3854b4a..bf033da4 100644 --- a/src/server/vm/sailbox-image.ts +++ b/src/server/vm/sailbox-image.ts @@ -17,7 +17,11 @@ import { createHash } from "node:crypto"; // https://github.com/earendil-works/pi. The installers fetch their latest // releases; the image version pins the script, not the harness versions. -export const AUTH_VOLUME_NAME = "loop-auth"; +/** Each owner's credentials live on their own volume, so two owners sharing + * one Sail account never mount each other's sign-ins. */ +export function authVolumeName(owner: string) { + return `loop-auth-${createHash("sha256").update(owner).digest("hex").slice(0, 16)}`; +} export const AUTH_MOUNT_PATH = "/mnt/loop-auth"; export const WORKSPACE_PATH = "/workspace"; export const SAIL_APP_NAME = "loop-experiments"; diff --git a/src/server/vm/sailbox-provider.ts b/src/server/vm/sailbox-provider.ts index fe3662ef..ff01a3d8 100644 --- a/src/server/vm/sailbox-provider.ts +++ b/src/server/vm/sailbox-provider.ts @@ -250,7 +250,11 @@ export function createSailboxProvider(options: { "Sailboxes listed Sailboxes in a shape Loop could not read", ); for (const item of value.data as unknown[]) { - if (!item || typeof item !== "object") + if ( + !item || + typeof item !== "object" || + typeof (item as { name?: unknown }).name !== "string" + ) throw new VmProviderError( "uncertain", "Sailboxes listed an entry Loop could not read", diff --git a/src/shared/vm-state.ts b/src/shared/vm-state.ts index 47fb6f21..270e7c2f 100644 --- a/src/shared/vm-state.ts +++ b/src/shared/vm-state.ts @@ -9,7 +9,8 @@ export const VM_TRANSITIONS = { sleeping: ["running", "destroying", "failed"], paused: ["running", "destroying", "failed"], // Only the owner's explicit reconcile, retry or destroy leaves failed. - failed: ["provisioning", "running", "destroying"], + // sleeping/paused: a reconcile or retry adopted a VM found asleep. + failed: ["provisioning", "running", "sleeping", "paused", "destroying"], destroying: ["destroyed", "failed"], // Terminal: the project, its threads, results and files stay readable. destroyed: [], diff --git a/test/vm-operations.test.ts b/test/vm-operations.test.ts index 188e2779..3ec3109c 100644 --- a/test/vm-operations.test.ts +++ b/test/vm-operations.test.ts @@ -16,6 +16,7 @@ import { runVmOperation, } from "../src/server/vm/operations.js"; import { VmProviderError, type VmStatus } from "../src/server/vm/provider.js"; +import { authVolumeName } from "../src/server/vm/sailbox-image.js"; // Fixture coverage: an in-memory database with the production migrations and // a scripted provider. No Sailbox is contacted. @@ -190,22 +191,28 @@ it("does not call the provider again for an uncertain operation or start another expect(calls).toBe(0); }); -it("keeps another owner's experiment and operation ids out of reach", async () => { +it("keeps each owner's experiments and operation ids apart", async () => { await create("op1", async () => ({ remoteId: "sb-1" })); let calls = 0; const call = async () => { calls += 1; - return {}; + return { remoteId: "sb-b" }; }; await expect(create("op9", call, { owner: "bob" })).rejects.toMatchObject({ reason: "missing_experiment", }); - experiment("bob", "exp2"); - await expect( - create("op1", call, { owner: "bob", experimentId: "exp2" }), - ).rejects.toMatchObject({ reason: "key_conflict" }); expect(calls).toBe(0); - expect(vmOperationsQueries.byId(db, "bob", "op1")).toBeUndefined(); + // Operation ids are scoped to the owner: bob's "op1" is his own receipt + // and reveals nothing about alice's. + experiment("bob", "exp2"); + expect( + await create("op1", call, { owner: "bob", experimentId: "exp2" }), + ).toMatchObject({ state: "succeeded", replayed: false }); + expect(calls).toBe(1); + expect(vmOperationsQueries.byId(db, "alice", "op1")).toMatchObject({ + experiment_id: "exp1", + result_json: '{"remoteId":"sb-1"}', + }); expect( transitionVm(db, { owner: "bob", id: "exp1" }, "running", { now: clock() }), ).toEqual({ changed: false, reason: "missing" }); @@ -779,6 +786,60 @@ it("refuses a preview on another owner's experiment", () => { expect(experimentPreviewsQueries.byExperiment(db, "bob", "exp1")).toEqual([]); }); +it("does not settle an uncertain unexpose from an absent port until the window has passed", async () => { + await runningVm(); + await uncertain("op1", "unexpose", { port: 3000 }); + const provider = found(); + expect(await reconcileSoon(provider)).toEqual({ + outcome: "still_uncertain", + reason: "the original call may still be in flight", + }); + expect(await reconcile(provider)).toMatchObject({ outcome: "succeeded" }); +}); + +it("keeps a create answered while its VM is still starting uncertain, then settles it by reconcile", async () => { + expect( + await create( + "op1", + async () => ({ remoteId: "sb-1", status: "provisioning" }), + {}, + createEffects("exp1", "op1"), + ), + ).toEqual({ state: "uncertain", errorMessage: "The VM is provisioning" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-1", + state: "provisioning", + }); + expect( + await reconcileSoon(found({ remoteId: "sb-1", status: "running" })), + ).toMatchObject({ outcome: "succeeded" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe("running"); +}); + +it("moves a failed VM to the asleep state a later create adopts", async () => { + await uncertainCreate(); + await reconcile(found()); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe("failed"); + expect( + await create( + "op2", + async () => ({ remoteId: "sb-2", status: "sleeping" }), + {}, + createEffects("exp1", "op2"), + ), + ).toMatchObject({ state: "succeeded" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-2", + state: "sleeping", + }); +}); + +it("gives each owner a separate auth volume", () => { + expect(authVolumeName("alice")).toMatch(/^loop-auth-[0-9a-f]{16}$/); + expect(authVolumeName("alice")).toBe(authVolumeName("alice")); + expect(authVolumeName("alice")).not.toBe(authVolumeName("bob")); +}); + it("refuses illegal and stale VM transitions", () => { const vm = { owner: "alice", id: "exp1" }; expect(transitionVm(db, vm, "destroyed", { now: clock() })).toEqual({ diff --git a/test/vm-sailbox-provider.test.ts b/test/vm-sailbox-provider.test.ts index b5cf7fef..c81eefd2 100644 --- a/test/vm-sailbox-provider.test.ts +++ b/test/vm-sailbox-provider.test.ts @@ -303,6 +303,7 @@ it("treats a listing it cannot read as uncertain, not as proof of absence", asyn { data: [] }, { data: "none", has_more: false }, { data: [null], has_more: false }, + { data: [{ sailbox_id: "sb_1", status: "running" }], has_more: false }, ]) { const { provider } = fake(() => ok(page)); await expect( From 28f251909cb8a3a25df76b23ccfa09068c93896d Mon Sep 17 00:00:00 2001 From: Jamie Asefa <13409617+j-asefa@users.noreply.github.com> Date: Fri, 20 Nov 2026 19:20:14 +0000 Subject: [PATCH 7/9] Cloud VM experiments 1/5: provisional VM ids, complete state edges, readable exec Review pass 5 fixes: - A create answered while its VM is still starting keeps the provider id in the receipt only, so a reconcile that finds it absent leaves the experiment free for a retry. Once a VM id is recorded, another create or fork is refused (vm_exists). - sleeping <-> paused: every live state reaches every other, so a reconcile never drops an observed status. - An exec event Loop cannot read makes the exit code null rather than reporting truncated output as a clean exit; unknown event types are still ignored. From pass 5's open concerns: provider requests refuse redirects, so the API key, file bodies and exec environments are never resent elsewhere. Co-Authored-By: Claude Opus 5.5 --- docs/state-machines.md | 6 +-- src/server/vm/operations.ts | 24 ++++++++-- src/server/vm/sailbox-provider.ts | 17 ++++++- src/shared/vm-state.ts | 7 ++- test/vm-operations.test.ts | 77 ++++++++++++++++++++++++++++++- test/vm-sailbox-provider.test.ts | 61 +++++++++++++++++++++++- 6 files changed, 178 insertions(+), 14 deletions(-) diff --git a/docs/state-machines.md b/docs/state-machines.md index 406fbb27..d394bc9b 100644 --- a/docs/state-machines.md +++ b/docs/state-machines.md @@ -378,8 +378,8 @@ leaves its project, threads, results and published files readable. | ---- | -- | ----------------- | | `provisioning` | `running` / `sleeping` / `paused` / `failed` / `destroying` | The provider reported the VM up (a reconcile can find it already asleep or paused), refused the create, or the owner destroyed it; **tx** | | `running` | `sleeping` / `paused` / `destroying` / `failed` | Sleep observed or requested, pause, owner destroy, or a provider failure observed; **tx** | -| `sleeping` | `running` / `destroying` / `failed` | Wake for a turn or a preview, owner destroy, or a failed wake; **tx** | -| `paused` | `running` / `destroying` / `failed` | Resume, owner destroy, or a failed resume; **tx** | +| `sleeping` | `running` / `paused` / `destroying` / `failed` | Wake for a turn or a preview, the provider reporting it paused, owner destroy, or a failed wake; **tx** | +| `paused` | `running` / `sleeping` / `destroying` / `failed` | Resume, the provider reporting it asleep, owner destroy, or a failed resume; **tx** | | `failed` | `provisioning` / `running` / `sleeping` / `paused` / `destroying` | The owner's reconcile or retry (which can adopt a VM found asleep or paused), or owner destroy; **tx** | | `destroying` | `destroyed` / `failed` | The provider confirmed the VM is gone (a missing VM counts), refused, or a reconcile found the destroy did not finish; **tx** | | **`destroyed`** | — | Terminal | @@ -397,7 +397,7 @@ before the provider is called, and its id is the caller's idempotency key. | **`succeeded`** | — | Terminal | | **`failed`** | — | Terminal | -An uncertain operation is never sent again automatically. Whichever settles the receipt first wins; a later answer that disagrees returns the stored outcome. `vmOperationEffects` in [vm/operations.ts](../src/server/vm/operations.ts) holds each kind's local writes (provider id, VM state, preview rows), and both an answer and a reconcile apply them in the transaction that settles the receipt. A reconcile settles only from a lookup that began after the longest provider call had passed since the receipt was written, because the original call may still land and change what the lookup saw. The exception is a create or fork whose VM exists, which settles at once: the original create reaches the same VM through its experiment-scoped provider key. A create answered while its VM is still starting stays uncertain. The window is Loop's own call timeout; Sail does not document a deadline for applying a request it accepted. A create whose VM boots into a failed state fails its receipt and keeps the VM's id, so the owner can destroy it. A retry is a new +An uncertain operation is never sent again automatically. Whichever settles the receipt first wins; a later answer that disagrees returns the stored outcome. `vmOperationEffects` in [vm/operations.ts](../src/server/vm/operations.ts) holds each kind's local writes (provider id, VM state, preview rows), and both an answer and a reconcile apply them in the transaction that settles the receipt. A reconcile settles only from a lookup that began after the longest provider call had passed since the receipt was written, because the original call may still land and change what the lookup saw. The exception is a create or fork whose VM exists, which settles at once: the original create reaches the same VM through its experiment-scoped provider key. A create answered while its VM is still starting stays uncertain and records nothing on the VM; once the VM's provider id is recorded, a create or fork for that experiment is refused, so a failed VM is destroyed rather than created again. The window is Loop's own call timeout; Sail does not document a deadline for applying a request it accepted. A create whose VM boots into a failed state fails its receipt and keeps the VM's id, so the owner can destroy it. A retry is a new operation with a new id, and is refused while an earlier one for the same experiment is unsettled. Creates and forks send the provider an idempotency key scoped to the experiment, so a retry after an "absent" reconcile reaches diff --git a/src/server/vm/operations.ts b/src/server/vm/operations.ts index 9a818c44..1f5e6317 100644 --- a/src/server/vm/operations.ts +++ b/src/server/vm/operations.ts @@ -31,7 +31,9 @@ export type VmOperationRefusal = /** The experiment does not exist for this owner. */ | "missing_experiment" /** An earlier operation on this experiment is pending or uncertain. */ - | "unsettled_operation"; + | "unsettled_operation" + /** A create or fork for an experiment that already has its VM. */ + | "vm_exists"; export class VmOperationRefused extends Error { constructor(readonly reason: VmOperationRefusal) { @@ -115,10 +117,20 @@ export async function runVmOperation( throw new VmOperationRefused("key_conflict"); return existing; } + const vm = experimentVmsQueries.byId( + db, + operation.owner, + operation.experimentId, + ); + if (!vm) throw new VmOperationRefused("missing_experiment"); + // One experiment has one provider VM. Once it is recorded, a failed VM + // is destroyed, not created again: the provider key is scoped to the + // experiment, so another create would only return the same VM. if ( - !experimentVmsQueries.byId(db, operation.owner, operation.experimentId) + (operation.kind === "create" || operation.kind === "fork") && + vm.remote_id !== null ) - throw new VmOperationRefused("missing_experiment"); + throw new VmOperationRefused("vm_exists"); if ( vmOperationsQueries.unsettledByExperiment( db, @@ -183,7 +195,9 @@ export async function runVmOperation( : "failed"; // The receipt and the local record of what the call did commit together. // If apply throws, both roll back and the receipt stays pending until the - // sweep marks it uncertain for a reconcile. + // sweep marks it uncertain for a reconcile. An incomplete answer writes + // nothing locally: its provider id is provisional until a reconcile sees + // the VM, and lives only in the receipt. const settled = db .transaction(() => { if ( @@ -194,7 +208,7 @@ export async function runVmOperation( }) ) return false; - effects.apply?.(result); + if (to !== "uncertain") effects.apply?.(result); return true; }) .immediate(); diff --git a/src/server/vm/sailbox-provider.ts b/src/server/vm/sailbox-provider.ts index ff01a3d8..bf5c481b 100644 --- a/src/server/vm/sailbox-provider.ts +++ b/src/server/vm/sailbox-provider.ts @@ -100,6 +100,9 @@ export function createSailboxProvider(options: { headers, body, signal: AbortSignal.timeout(init.timeoutMs ?? CONTROL_TIMEOUT_MS), + // A redirect would resend the API key, a request body or a command's + // environment to another address; Sail's API does not redirect. + redirect: "error", }); } catch (error) { throw new VmProviderError( @@ -488,6 +491,9 @@ async function startExec( }; let buffered = ""; const blocked = new Set(); + // An event Loop could not read means the output it passed on may be + // incomplete, so the exit code is not reported as if it were whole. + let corrupt = false; const decoder = new TextDecoder(); const handle = (line: string) => { if (!line.trim()) return; @@ -495,6 +501,11 @@ async function startExec( try { event = JSON.parse(line) as Record; } catch { + corrupt = true; + return; + } + if (!event || typeof event !== "object") { + corrupt = true; return; } if ( @@ -510,7 +521,9 @@ async function startExec( const stream = event.type === "stdout" ? stdout : stderr; if (!stream.write(Buffer.from(event.data, "base64"))) blocked.add(stream); - } else if (event.type === "exit") + } else if (event.type === "stdout" || event.type === "stderr") + corrupt = true; + else if (event.type === "exit") result = { exitCode: typeof event.return_code === "number" ? event.return_code : null, @@ -550,7 +563,7 @@ async function startExec( stdout.end(); stderr.end(); } - return result; + return corrupt ? { exitCode: null, signal: null } : result; })(); let offset = 0; diff --git a/src/shared/vm-state.ts b/src/shared/vm-state.ts index 270e7c2f..265656da 100644 --- a/src/shared/vm-state.ts +++ b/src/shared/vm-state.ts @@ -6,8 +6,11 @@ export const VM_TRANSITIONS = { // sleeping/paused: a reconcile adopted a VM the provider reports asleep. provisioning: ["running", "sleeping", "paused", "failed", "destroying"], running: ["sleeping", "paused", "destroying", "failed"], - sleeping: ["running", "destroying", "failed"], - paused: ["running", "destroying", "failed"], + // sleeping <-> paused: what the provider reports when one becomes the + // other (Sail maps an interrupted, restorable VM to paused). Every live + // state reaches every other, so an observed status is never dropped. + sleeping: ["running", "paused", "destroying", "failed"], + paused: ["running", "sleeping", "destroying", "failed"], // Only the owner's explicit reconcile, retry or destroy leaves failed. // sleeping/paused: a reconcile or retry adopted a VM found asleep. failed: ["provisioning", "running", "sleeping", "paused", "destroying"], diff --git a/test/vm-operations.test.ts b/test/vm-operations.test.ts index 3ec3109c..41bd96ec 100644 --- a/test/vm-operations.test.ts +++ b/test/vm-operations.test.ts @@ -17,6 +17,7 @@ import { } from "../src/server/vm/operations.js"; import { VmProviderError, type VmStatus } from "../src/server/vm/provider.js"; import { authVolumeName } from "../src/server/vm/sailbox-image.js"; +import { legalVmTransition } from "../src/shared/vm-state.js"; // Fixture coverage: an in-memory database with the production migrations and // a scripted provider. No Sailbox is contacted. @@ -806,10 +807,14 @@ it("keeps a create answered while its VM is still starting uncertain, then settl createEffects("exp1", "op1"), ), ).toEqual({ state: "uncertain", errorMessage: "The VM is provisioning" }); + // The id is provisional: it stays in the receipt, not on the VM. expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ - remote_id: "sb-1", + remote_id: null, state: "provisioning", }); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.result_json).toContain( + "sb-1", + ); expect( await reconcileSoon(found({ remoteId: "sb-1", status: "running" })), ).toMatchObject({ outcome: "succeeded" }); @@ -840,6 +845,76 @@ it("gives each owner a separate auth volume", () => { expect(authVolumeName("alice")).not.toBe(authVolumeName("bob")); }); +it("lets a retry create a VM after a still-starting VM turned out absent", async () => { + await create( + "op1", + async () => ({ remoteId: "sb-1", status: "provisioning" }), + {}, + createEffects("exp1", "op1"), + ); + expect(await reconcile(found())).toMatchObject({ outcome: "failed" }); + expect( + await create( + "op2", + async () => ({ remoteId: "sb-2", status: "running" }), + {}, + createEffects("exp1", "op2"), + ), + ).toMatchObject({ state: "succeeded" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-2", + state: "running", + }); +}); + +it("refuses another create once the experiment has its VM, without calling the provider", async () => { + await create( + "op1", + async () => ({ remoteId: "sb-1", status: "failed", errorMessage: "boom" }), + {}, + createEffects("exp1", "op1"), + ); + let called = false; + await expect( + create("op2", async () => { + called = true; + return {}; + }), + ).rejects.toMatchObject({ reason: "vm_exists" }); + expect(called).toBe(false); +}); + +it("records every live status the provider reports, such as a sleeping VM found paused", async () => { + await runningVm(); + transitionVm(db, { owner: "alice", id: "exp1" }, "sleeping", { + now: clock(), + }); + const operation = { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "resume" as const, + request: {}, + }; + await runVmOperation( + db, + operation, + async () => { + throw new VmProviderError("uncertain", "timed out"); + }, + clock, + vmOperationEffects(db, operation, clock), + ); + const provider = found(); + provider.statuses.set("sb-1", "paused"); + expect(await reconcile(provider)).toMatchObject({ outcome: "failed" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe("paused"); + const live = ["running", "sleeping", "paused"] as const; + for (const from of live) + for (const to of live) + if (from !== to) expect(legalVmTransition(from, to)).toBe(true); +}); + it("refuses illegal and stale VM transitions", () => { const vm = { owner: "alice", id: "exp1" }; expect(transitionVm(db, vm, "destroyed", { now: clock() })).toEqual({ diff --git a/test/vm-sailbox-provider.test.ts b/test/vm-sailbox-provider.test.ts index c81eefd2..88d272fc 100644 --- a/test/vm-sailbox-provider.test.ts +++ b/test/vm-sailbox-provider.test.ts @@ -10,7 +10,13 @@ import { // request and response shapes taken from Sail's published OpenAPI document. // No Sailbox is contacted. -type Call = { method: string; url: URL; headers: Headers; body: unknown }; +type Call = { + method: string; + url: URL; + headers: Headers; + body: unknown; + redirect?: RequestRedirect; +}; function fake(answer: (call: Call) => Response | Promise | Error) { const calls: Call[] = []; @@ -20,6 +26,7 @@ function fake(answer: (call: Call) => Response | Promise | Error) { method: init?.method ?? "GET", url: new URL(String(input)), headers: new Headers(init?.headers), + redirect: init?.redirect, body: typeof raw === "string" ? JSON.parse(raw) @@ -418,3 +425,55 @@ it("reports a VM that booted into failed with Sail's reason, and mounts volumes read_only: false, }); }); + +it("never follows a redirect with the API key", async () => { + const { calls, provider } = fake(() => ok({ org_id: "org_1" })); + await provider.whoami(); + expect(calls[0].redirect).toBe("error"); +}); + +it("reports no exit code when an exec event could not be read", async () => { + const encoder = new TextEncoder(); + const { provider } = fake( + () => + new Response( + new ReadableStream({ + start(controller) { + for (const line of [ + JSON.stringify({ type: "started", exec_request_id: "e" }), + "{not json", + JSON.stringify({ type: "stdout", data: 7, seq: 2 }), + JSON.stringify({ type: "progress", note: "new event kind" }), + JSON.stringify({ + type: "exit", + status: "succeeded", + return_code: 0, + }), + ]) + controller.enqueue(encoder.encode(line + "\n")); + controller.close(); + }, + }), + ), + ); + const handle = await provider.exec("sb_1", ["true"], {}); + handle.stdout.resume(); + handle.stderr.resume(); + expect(await handle.exit).toEqual({ exitCode: null, signal: null }); + + // An event type Loop does not know is not corruption. + const clean = fake( + () => + new Response( + [ + JSON.stringify({ type: "started", exec_request_id: "e" }), + JSON.stringify({ type: "progress" }), + JSON.stringify({ type: "exit", status: "succeeded", return_code: 0 }), + ].join("\n") + "\n", + ), + ); + const ok2 = await clean.provider.exec("sb_1", ["true"], {}); + ok2.stdout.resume(); + ok2.stderr.resume(); + expect(await ok2.exit).toEqual({ exitCode: 0, signal: null }); +}); From 937685a79b2664f6e52ca3e9ff3a2b56b62c943a Mon Sep 17 00:00:00 2001 From: Jamie Asefa <13409617+j-asefa@users.noreply.github.com> Date: Fri, 20 Nov 2026 19:25:42 +0000 Subject: [PATCH 8/9] Cloud VM experiments 1/5: keep refreshed credentials, read resume outcomes Review pass 6 fixes: - loop-vm-init moves any regular credential file onto the auth volume before relinking, so a harness that refreshed by replacing its link keeps the new credential instead of being pointed back at the old one. - A resume answer other than running/already_running/terminal_unavailable is uncertain, not success. From pass 6's open concerns: a destroy sent while the VM was still running ends destroyed through destroying, and an exec event longer than 16 MB stops the stream and reports no exit code. Co-Authored-By: Claude Opus 5.5 --- src/server/vm/operations.ts | 10 ++++++++ src/server/vm/sailbox-image.ts | 8 +++--- src/server/vm/sailbox-provider.ts | 21 +++++++++++++++- test/vm-operations.test.ts | 21 ++++++++++++++++ test/vm-sailbox-image.test.ts | 25 +++++++++++++++++++ test/vm-sailbox-provider.test.ts | 41 +++++++++++++++++++++++++++++++ 6 files changed, 122 insertions(+), 4 deletions(-) diff --git a/src/server/vm/operations.ts b/src/server/vm/operations.ts index 1f5e6317..a4e37fef 100644 --- a/src/server/vm/operations.ts +++ b/src/server/vm/operations.ts @@ -314,6 +314,16 @@ export function vmOperationEffects( ); return; } + // A destroy the caller sent without first marking the VM destroying + // still ends destroyed, through destroying, rather than failed. + if (operation.kind === "destroy") { + const row = current(); + if (row.state !== "destroying" && row.state !== "destroyed") + transitionVm(db, row, "destroying", { + expected: row.state, + now: clock(), + }); + } const status = kindStatus[operation.kind]; if (status) applyObservedStatus(db, current(), status, clock()); if (operation.kind === "destroy") diff --git a/src/server/vm/sailbox-image.ts b/src/server/vm/sailbox-image.ts index bf033da4..04da9fce 100644 --- a/src/server/vm/sailbox-image.ts +++ b/src/server/vm/sailbox-image.ts @@ -90,9 +90,11 @@ link() { local target="$auth/$1/$(basename "$2")" path="$HOME/$2" mkdir -p "$auth/$1" "$(dirname "$path")" chmod 700 "$auth/$1" - # A harness that wrote a credential before the link existed keeps it. - if [ -f "$path" ] && [ ! -L "$path" ] && [ ! -e "$target" ]; then - mv "$path" "$target" + # A regular file here is newer than the volume's copy: either the harness + # signed in before the link existed, or it refreshed the credential by + # replacing the link with a file. Either way it is kept, on the volume. + if [ -f "$path" ] && [ ! -L "$path" ]; then + mv -f "$path" "$target" fi ln -sfn "$target" "$path" } diff --git a/src/server/vm/sailbox-provider.ts b/src/server/vm/sailbox-provider.ts index bf5c481b..03f3907e 100644 --- a/src/server/vm/sailbox-provider.ts +++ b/src/server/vm/sailbox-provider.ts @@ -24,6 +24,7 @@ const CONTROL_TIMEOUT_MS = 60_000; // Create and start-from-checkpoint wait for the VM to boot. const BOOT_TIMEOUT_MS = 10 * 60_000; const MAX_LIST_PAGES = 20; +const MAX_EXEC_EVENT_CHARS = 16 * 1024 * 1024; type Fetch = typeof fetch; @@ -295,11 +296,23 @@ export function createSailboxProvider(options: { idempotencyKey, timeoutMs: BOOT_TIMEOUT_MS, }); + if ( + value.resume_state === "running" || + value.resume_state === "already_running" + ) + return; if (value.resume_state === "terminal_unavailable") throw new VmProviderError( "refused", - message(value, "The Sailbox can no longer be resumed"), + typeof value.error_message === "string" && value.error_message + ? value.error_message + : "The Sailbox can no longer be resumed", ); + // Any other answer does not show the VM running. + throw new VmProviderError( + "uncertain", + "Sailboxes answered a resume without a recognised outcome", + ); }, async checkpointFilesystem(input) { @@ -533,6 +546,12 @@ async function startExec( try { for await (const chunk of body) { buffered += decoder.decode(chunk, { stream: true }); + // Sail's output events are small; a line this long is not one. + if (buffered.length > MAX_EXEC_EVENT_CHARS) { + corrupt = true; + await body.cancel().catch(() => undefined); + break; + } let newline = buffered.indexOf("\n"); while (newline >= 0) { handle(buffered.slice(0, newline)); diff --git a/test/vm-operations.test.ts b/test/vm-operations.test.ts index 41bd96ec..1fccbeec 100644 --- a/test/vm-operations.test.ts +++ b/test/vm-operations.test.ts @@ -915,6 +915,27 @@ it("records every live status the provider reports, such as a sleeping VM found if (from !== to) expect(legalVmTransition(from, to)).toBe(true); }); +it("records a destroy sent while the VM was running as destroyed, not failed", async () => { + await runningVm(); + const operation = { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "destroy" as const, + request: {}, + }; + await runVmOperation( + db, + operation, + async () => null, + clock, + vmOperationEffects(db, operation, clock), + ); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe( + "destroyed", + ); +}); + it("refuses illegal and stale VM transitions", () => { const vm = { owner: "alice", id: "exp1" }; expect(transitionVm(db, vm, "destroyed", { now: clock() })).toEqual({ diff --git a/test/vm-sailbox-image.test.ts b/test/vm-sailbox-image.test.ts index af3d06e1..dfef2ab5 100644 --- a/test/vm-sailbox-image.test.ts +++ b/test/vm-sailbox-image.test.ts @@ -6,6 +6,7 @@ import { mkdtempSync, readFileSync, readlinkSync, + rmSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; @@ -74,3 +75,27 @@ it("produces an install script bash accepts", () => { writeFileSync(join(dir, "install.sh"), INSTALL_SCRIPT); execFileSync("bash", ["-n", join(dir, "install.sh")]); }); + +it("keeps a credential the harness refreshed by replacing its link", () => { + const root = mkdtempSync(join(tmpdir(), "loop-vm-refresh-")); + const [home, auth, workspace] = ["home", "auth", "workspace"].map((name) => + join(root, name), + ); + mkdirSync(home); + mkdirSync(auth); + run(home, auth, workspace); + writeFileSync(join(auth, "codex/auth.json"), "old token"); + // The harness writes a new file in place of the link, as an atomic + // rename-based refresh does. + const path = join(home, ".codex/auth.json"); + rmSync(path); + writeFileSync(path, "refreshed token"); + + run(home, auth, workspace); + + expect(lstatSync(path).isSymbolicLink()).toBe(true); + expect(readFileSync(path, "utf8")).toBe("refreshed token"); + expect(readFileSync(join(auth, "codex/auth.json"), "utf8")).toBe( + "refreshed token", + ); +}); diff --git a/test/vm-sailbox-provider.test.ts b/test/vm-sailbox-provider.test.ts index 88d272fc..765d4b70 100644 --- a/test/vm-sailbox-provider.test.ts +++ b/test/vm-sailbox-provider.test.ts @@ -477,3 +477,44 @@ it("reports no exit code when an exec event could not be read", async () => { ok2.stderr.resume(); expect(await ok2.exit).toEqual({ exitCode: 0, signal: null }); }); + +it("treats a resume answer without a recognised outcome as uncertain", async () => { + for (const answer of [{}, { resume_state: "warming" }]) { + const { provider } = fake(() => ok(answer)); + await expect(provider.resume("sb_1", "op-4")).rejects.toMatchObject({ + outcome: "uncertain", + }); + } + for (const resume_state of ["running", "already_running"]) { + const { provider } = fake(() => ok({ resume_state })); + await expect(provider.resume("sb_1", "op-5")).resolves.toBeUndefined(); + } +}); + +it("stops reading an exec stream whose event never ends", async () => { + const encoder = new TextEncoder(); + let pulls = 0; + const { provider } = fake( + () => + new Response( + new ReadableStream({ + pull(controller) { + pulls += 1; + controller.enqueue( + encoder.encode( + pulls === 1 + ? JSON.stringify({ type: "started", exec_request_id: "e" }) + + '\n{"type":"stdout","data":"' + : "A".repeat(1024 * 1024), + ), + ); + }, + }), + ), + ); + const handle = await provider.exec("sb_1", ["yes"], {}); + handle.stdout.resume(); + handle.stderr.resume(); + expect(await handle.exit).toEqual({ exitCode: null, signal: null }); + expect(pulls).toBeLessThan(40); +}); From afabb2a65cd27a00825bf3af3bba83e117342aed Mon Sep 17 00:00:00 2001 From: Jamie Asefa <13409617+j-asefa@users.noreply.github.com> Date: Fri, 20 Nov 2026 19:33:22 +0000 Subject: [PATCH 9/9] Cloud VM experiments 1/5: require the auth mount, finish interrupted init Review pass 7 fixes: - loop-vm-init stops before touching credentials unless the auth volume is mounted, so a skipped mount cannot leave credentials on the VM disk. - The workspace's initial commit is checked by HEAD, not by .git, so a run stopped after git init finishes next time. - The 16 MB exec limit applies to the unfinished event only, after a chunk's complete events are handled. From pass 7's open concerns: runVmOperation applies the kind's own effects unless a caller passes others, so a create cannot settle without recording its VM, and operations other than create/fork are refused (no_vm) before the experiment has a VM. Co-Authored-By: Claude Opus 5.5 --- src/server/vm/operations.ts | 16 +++++++++-- src/server/vm/sailbox-image.ts | 12 +++++++-- src/server/vm/sailbox-provider.ts | 13 ++++----- test/vm-operations.test.ts | 39 ++++++++++++++++++++++++++- test/vm-sailbox-image.test.ts | 45 +++++++++++++++++++++++++++++++ test/vm-sailbox-provider.test.ts | 27 +++++++++++++++++++ 6 files changed, 141 insertions(+), 11 deletions(-) diff --git a/src/server/vm/operations.ts b/src/server/vm/operations.ts index a4e37fef..661ba667 100644 --- a/src/server/vm/operations.ts +++ b/src/server/vm/operations.ts @@ -33,7 +33,9 @@ export type VmOperationRefusal = /** An earlier operation on this experiment is pending or uncertain. */ | "unsettled_operation" /** A create or fork for an experiment that already has its VM. */ - | "vm_exists"; + | "vm_exists" + /** Any other operation on an experiment with no recorded VM. */ + | "no_vm"; export class VmOperationRefused extends Error { constructor(readonly reason: VmOperationRefusal) { @@ -97,7 +99,9 @@ export async function runVmOperation( }, call: () => Promise, clock: () => number, - effects: VmOperationEffects = {}, + // The kind's own local writes unless a caller passes others (tests of the + // receipt alone pass {}), so a create cannot settle without its VM id. + effects: VmOperationEffects = vmOperationEffects(db, operation, clock), ): Promise { const requestJson = canonicalVmRequest(operation.request); const requestDigest = vmRequestDigest({ @@ -131,6 +135,14 @@ export async function runVmOperation( vm.remote_id !== null ) throw new VmOperationRefused("vm_exists"); + // Anything else acts on the recorded VM; without one, an uncertain + // outcome could never be reconciled. + if ( + operation.kind !== "create" && + operation.kind !== "fork" && + vm.remote_id === null + ) + throw new VmOperationRefused("no_vm"); if ( vmOperationsQueries.unsettledByExperiment( db, diff --git a/src/server/vm/sailbox-image.ts b/src/server/vm/sailbox-image.ts index 04da9fce..7e76fec0 100644 --- a/src/server/vm/sailbox-image.ts +++ b/src/server/vm/sailbox-image.ts @@ -86,6 +86,12 @@ export function initScript() { set -euo pipefail auth="\${LOOP_AUTH_MOUNT:-${AUTH_MOUNT_PATH}}" workspace="\${LOOP_WORKSPACE:-${WORKSPACE_PATH}}" +# Without the volume, credentials would land on the VM's own disk, be hidden +# by a later mount, and could be captured by a checkpoint. +if [ "\${LOOP_AUTH_SKIP_MOUNT_CHECK:-0}" != 1 ] && ! mountpoint -q "$auth"; then + echo "loop-vm-init: $auth is not mounted" >&2 + exit 1 +fi link() { local target="$auth/$1/$(basename "$2")" path="$HOME/$2" mkdir -p "$auth/$1" "$(dirname "$path")" @@ -101,8 +107,10 @@ link() { ${links} mkdir -p "$workspace" cd "$workspace" -if [ ! -d .git ]; then - git init -q -b main +# Checked by commit, not by .git, so a run stopped after git init finishes +# the job next time. +[ -d .git ] || git init -q -b main +if ! git rev-parse -q --verify HEAD >/dev/null; then git -c user.name=Loop -c user.email=loop@localhost commit -q --allow-empty -m "Initial experiment workspace" fi`; } diff --git a/src/server/vm/sailbox-provider.ts b/src/server/vm/sailbox-provider.ts index 03f3907e..dbf8e9bd 100644 --- a/src/server/vm/sailbox-provider.ts +++ b/src/server/vm/sailbox-provider.ts @@ -546,18 +546,19 @@ async function startExec( try { for await (const chunk of body) { buffered += decoder.decode(chunk, { stream: true }); - // Sail's output events are small; a line this long is not one. - if (buffered.length > MAX_EXEC_EVENT_CHARS) { - corrupt = true; - await body.cancel().catch(() => undefined); - break; - } let newline = buffered.indexOf("\n"); while (newline >= 0) { handle(buffered.slice(0, newline)); buffered = buffered.slice(newline + 1); newline = buffered.indexOf("\n"); } + // Only the unfinished event is bounded: Sail's output events are + // small, and one this long is not one of them. + if (buffered.length > MAX_EXEC_EVENT_CHARS) { + corrupt = true; + await body.cancel().catch(() => undefined); + break; + } // Output is read from the provider only as fast as Loop's consumer // reads it: a full stream pauses the event stream until it drains. // Every full stream must drain, not only the last one that filled. diff --git a/test/vm-operations.test.ts b/test/vm-operations.test.ts index 1fccbeec..05b01324 100644 --- a/test/vm-operations.test.ts +++ b/test/vm-operations.test.ts @@ -65,7 +65,8 @@ const create = ( }, call, clock, - effects, + // Receipt-only tests opt out of the create's local writes. + effects ?? {}, ); const createEffects = (experimentId: string, id: string) => @@ -936,6 +937,42 @@ it("records a destroy sent while the VM was running as destroyed, not failed", a ); }); +it("refuses an operation on an experiment with no recorded VM", async () => { + await expect( + runVmOperation( + db, + { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "sleep", + request: {}, + }, + async () => null, + clock, + ), + ).rejects.toMatchObject({ reason: "no_vm" }); +}); + +it("records a create's VM by default, without the caller passing effects", async () => { + await runVmOperation( + db, + { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "create", + request: {}, + }, + async () => ({ remoteId: "sb-1", status: "running" }), + clock, + ); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-1", + state: "running", + }); +}); + it("refuses illegal and stale VM transitions", () => { const vm = { owner: "alice", id: "exp1" }; expect(transitionVm(db, vm, "destroyed", { now: clock() })).toEqual({ diff --git a/test/vm-sailbox-image.test.ts b/test/vm-sailbox-image.test.ts index dfef2ab5..2f9c030c 100644 --- a/test/vm-sailbox-image.test.ts +++ b/test/vm-sailbox-image.test.ts @@ -31,6 +31,8 @@ function run(home: string, auth: string, workspace: string) { HOME: home, LOOP_AUTH_MOUNT: auth, LOOP_WORKSPACE: workspace, + // The fixture's auth directory is not a mount point. + LOOP_AUTH_SKIP_MOUNT_CHECK: "1", GIT_CONFIG_GLOBAL: "/dev/null", }, }); @@ -99,3 +101,46 @@ it("keeps a credential the harness refreshed by replacing its link", () => { "refreshed token", ); }); + +it("refuses to place credentials when the auth volume is not mounted", () => { + const root = mkdtempSync(join(tmpdir(), "loop-vm-unmounted-")); + const [home, auth, workspace] = ["home", "auth", "workspace"].map((name) => + join(root, name), + ); + mkdirSync(join(home, ".codex"), { recursive: true }); + mkdirSync(auth); + writeFileSync(join(home, ".codex/auth.json"), "token"); + const script = join(root, "loop-vm-init"); + writeFileSync(script, initScript(), { mode: 0o755 }); + expect(() => + execFileSync("bash", [script], { + stdio: "pipe", + env: { + PATH: process.env.PATH, + HOME: home, + LOOP_AUTH_MOUNT: auth, + LOOP_WORKSPACE: workspace, + }, + }), + ).toThrow(/is not mounted/); + expect(readFileSync(join(home, ".codex/auth.json"), "utf8")).toBe("token"); + expect(existsSync(join(auth, "codex"))).toBe(false); +}); + +it("finishes the initial commit when an earlier run stopped after git init", () => { + const root = mkdtempSync(join(tmpdir(), "loop-vm-unborn-")); + const [home, auth, workspace] = ["home", "auth", "workspace"].map((name) => + join(root, name), + ); + mkdirSync(home); + mkdirSync(auth); + mkdirSync(workspace); + execFileSync("git", ["init", "-q", "-b", "main"], { cwd: workspace }); + run(home, auth, workspace); + expect( + execFileSync("git", ["log", "--format=%s"], { + cwd: workspace, + encoding: "utf8", + }).trim(), + ).toBe("Initial experiment workspace"); +}); diff --git a/test/vm-sailbox-provider.test.ts b/test/vm-sailbox-provider.test.ts index 765d4b70..fa430c0f 100644 --- a/test/vm-sailbox-provider.test.ts +++ b/test/vm-sailbox-provider.test.ts @@ -518,3 +518,30 @@ it("stops reading an exec stream whose event never ends", async () => { expect(await handle.exit).toEqual({ exitCode: null, signal: null }); expect(pulls).toBeLessThan(40); }); + +it("keeps every event of a large chunk made of small events", async () => { + const line = (event: unknown) => JSON.stringify(event) + "\n"; + const data = Buffer.alloc(1024, 65).toString("base64"); + let text = line({ type: "started", exec_request_id: "e" }); + // ~20 MB in one chunk, every event well under the per-event limit. + for (let seq = 1; seq <= 15_000; seq += 1) + text += line({ type: "stdout", data, seq }); + text += line({ type: "exit", status: "succeeded", return_code: 0 }); + const { provider } = fake( + () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(text)); + controller.close(); + }, + }), + ), + ); + const handle = await provider.exec("sb_1", ["cat", "big"], {}); + let bytes = 0; + handle.stdout.on("data", (chunk: Buffer) => (bytes += chunk.length)); + handle.stderr.resume(); + expect(await handle.exit).toEqual({ exitCode: 0, signal: null }); + expect(bytes).toBe(15_000 * 1024); +});