diff --git a/docs/sql-index.md b/docs/sql-index.md index dbd19eff..dc1ebb73 100644 --- a/docs/sql-index.md +++ b/docs/sql-index.md @@ -511,6 +511,24 @@ Generated by `npm run sql-index` from [src/server/sql](../src/server/sql/); do n - `start(db, owner, command, method, digest, generation, created, updated)` - `unknown(db, time, owner, command, generation)` +## [experiment-previews.ts](../src/server/sql/experiment-previews.ts) + +- `byExperiment(db, owner, experiment)` +- `byPort(db, owner, experiment, port)` +- `deactivate(db, owner, experiment, port, now)` +- `deactivateAll(db, owner, experiment, now)` — A destroyed VM serves nothing, so every link of the experiment is disabled. +- `upsert(db, preview)` — One preview per experiment port; exposing the port again reactivates it. + +## [experiment-vms.ts](../src/server/sql/experiment-vms.ts) + +- `adoptRemote(db, owner, id, remoteId)` — Records the provider's id once; a different id for the same VM is refused. +- `byFolder(db, owner, folder)` +- `byId(db, owner, id)` +- `insert(db, vm)` — Inserts the owner's experiment. +- `owned(db, owner)` +- `setState(db, owner, id, from, to, lastError, now)` — Moves `from` to `to`; returns false when the row is no longer in `from`. +- `touch(db, owner, id, now)` + ## [feed-card-dirty.ts](../src/server/sql/feed-card-dirty.ts) - `any(db)` @@ -1639,6 +1657,22 @@ Generated by `npm run sql-index` from [src/server/sql](../src/server/sql/); do n - `profile(db, owner)` - `save(db, owner, name, handle, bio, updated)` +## [vm-operations.ts](../src/server/sql/vm-operations.ts) + +- `byId(db, owner, id)` +- `insertPending(db, operation)` +- `pendingBefore(db, cutoff)` — Every owner's receipts still pending since before \`cutoff\`, for the recovery sweep only. +- `setState(db, owner, id, from, to, outcome, now)` — Moves `from` to `to`; returns false when the row is no longer in `from`. +- `unsettledByExperiment(db, owner, experiment)` + +## [vm-provider-configs.ts](../src/server/sql/vm-provider-configs.ts) + +- `byOwner(db, owner)` +- `remove(db, owner)` +- `saveKey(db, owner, provider, sealedKey, now)` — Stores the sealed key, keeping the resources already found for the owner. +- `setDefaultHarness(db, owner, harness, now)` +- `setResources(db, owner, resources, now)` + ## [webhook-deliveries.ts](../src/server/sql/webhook-deliveries.ts) - `activation(db, subscription)` diff --git a/docs/state-machines.md b/docs/state-machines.md index 1fae2281..d394bc9b 100644 --- a/docs/state-machines.md +++ b/docs/state-machines.md @@ -365,6 +365,44 @@ rolls back local work before recording either a refusal or another attempt. This an uncertain harness delivery. Retry records cascade with their source run or mail; ineligible results retain their metadata until source deletion. +## Cloud VM experiments + +### `experiment_vms.state` + +Owner: `VM_TRANSITIONS` in [vm-state.ts](../src/shared/vm-state.ts), applied +only by `transitionVm()` in [vm/machine.ts](../src/server/vm/machine.ts), which +re-reads the owner's row inside one immediate transaction. A destroyed VM +leaves its project, threads, results and published files readable. + +| From | To | Trigger and guard | +| ---- | -- | ----------------- | +| `provisioning` | `running` / `sleeping` / `paused` / `failed` / `destroying` | The provider reported the VM up (a reconcile can find it already asleep or paused), refused the create, or the owner destroyed it; **tx** | +| `running` | `sleeping` / `paused` / `destroying` / `failed` | Sleep observed or requested, pause, owner destroy, or a provider failure observed; **tx** | +| `sleeping` | `running` / `paused` / `destroying` / `failed` | Wake for a turn or a preview, the provider reporting it paused, owner destroy, or a failed wake; **tx** | +| `paused` | `running` / `sleeping` / `destroying` / `failed` | Resume, the provider reporting it asleep, owner destroy, or a failed resume; **tx** | +| `failed` | `provisioning` / `running` / `sleeping` / `paused` / `destroying` | The owner's reconcile or retry (which can adopt a VM found asleep or paused), or owner destroy; **tx** | +| `destroying` | `destroyed` / `failed` | The provider confirmed the VM is gone (a missing VM counts), refused, or a reconcile found the destroy did not finish; **tx** | +| **`destroyed`** | — | Terminal | + +### `vm_operations.state` + +Owner: `VM_OPERATION_TRANSITIONS` in the same file, applied only by +`transitionVmOperation()`. The row is a receipt: it is inserted `pending` +before the provider is called, and its id is the caller's idempotency key. + +| From | To | Trigger and guard | +| ---- | -- | ----------------- | +| `pending` | `succeeded` / `failed` / `uncertain` | The provider answered, refused, or gave no observed answer (timeout, disconnect, server error, or the worker's sweep found the row still pending after the longest provider call); **tx** | +| `uncertain` | `succeeded` / `failed` | A late answer arrived, or the owner's reconcile observed the outcome: a create or fork by the VM's `remote_name`, any other kind by the VM's status or published port; **tx** | +| **`succeeded`** | — | Terminal | +| **`failed`** | — | Terminal | + +An uncertain operation is never sent again automatically. Whichever settles the receipt first wins; a later answer that disagrees returns the stored outcome. `vmOperationEffects` in [vm/operations.ts](../src/server/vm/operations.ts) holds each kind's local writes (provider id, VM state, preview rows), and both an answer and a reconcile apply them in the transaction that settles the receipt. A reconcile settles only from a lookup that began after the longest provider call had passed since the receipt was written, because the original call may still land and change what the lookup saw. The exception is a create or fork whose VM exists, which settles at once: the original create reaches the same VM through its experiment-scoped provider key. A create answered while its VM is still starting stays uncertain and records nothing on the VM; once the VM's provider id is recorded, a create or fork for that experiment is refused, so a failed VM is destroyed rather than created again. The window is Loop's own call timeout; Sail does not document a deadline for applying a request it accepted. A create whose VM boots into a failed state fails its receipt and keeps the VM's id, so the owner can destroy it. A retry is a new +operation with a new id, and is refused while an earlier one for the same +experiment is unsettled. Creates and forks send the provider an idempotency +key scoped to the experiment, so a retry after an "absent" reconcile reaches +the VM a still-running first attempt made instead of starting a second one. + ## Fleets (experimental) | Record | States and edges | Guards | diff --git a/migrations/167-experiment-vms.sql b/migrations/167-experiment-vms.sql new file mode 100644 index 00000000..f31d7b5c --- /dev/null +++ b/migrations/167-experiment-vms.sql @@ -0,0 +1,79 @@ +-- Cloud VM Experiments. One Experiment is one provider VM linked 1:1 to a +-- project folder. Conversations, results and published files stay in this +-- database and its object store, so no row here is needed to read history. +-- Additive: releases before this migration wrote none of these tables. + +-- The owner's provider account. The key is sealed with the owner-bound +-- secret store and never leaves this host. +CREATE TABLE vm_provider_configs ( + owner_id TEXT PRIMARY KEY, + provider TEXT NOT NULL, + encrypted_api_key TEXT NOT NULL, + app_id TEXT, + auth_volume_id TEXT, + base_image_ref TEXT, + base_image_version TEXT, + default_harness TEXT, + updated_at INTEGER NOT NULL +); + +-- remote_name is chosen before the provider is called, so an uncertain create +-- can be reconciled by looking the name up instead of creating a second VM. +CREATE TABLE experiment_vms ( + id TEXT PRIMARY KEY, + owner_id TEXT NOT NULL, + folder_id TEXT NOT NULL UNIQUE REFERENCES folders(id), + provider TEXT NOT NULL, + remote_id TEXT, + remote_name TEXT NOT NULL UNIQUE, + state TEXT NOT NULL, + auto_approve_local INTEGER NOT NULL DEFAULT 1, + vcpu_limit INTEGER NOT NULL DEFAULT 4, + memory_limit_gib INTEGER NOT NULL DEFAULT 16, + disk_limit_gib INTEGER NOT NULL DEFAULT 64, + auto_sleep_seconds INTEGER NOT NULL DEFAULT 900, + source_experiment_id TEXT REFERENCES experiment_vms(id), + source_checkpoint_id TEXT, + clone_url TEXT, + last_error TEXT, + last_active_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + destroyed_at INTEGER +); +CREATE INDEX experiment_vms_owner ON experiment_vms(owner_id, created_at); + +CREATE TABLE experiment_previews ( + id TEXT PRIMARY KEY, + owner_id TEXT NOT NULL, + experiment_id TEXT NOT NULL REFERENCES experiment_vms(id), + port INTEGER NOT NULL, + label TEXT, + path TEXT NOT NULL DEFAULT '/', + url TEXT NOT NULL, + active INTEGER NOT NULL DEFAULT 1, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(experiment_id, port) +); + +-- A receipt written before every provider call. id is the caller's +-- idempotency key, unique within the owner; request_digest refuses the same key for a different +-- request, and request_json keeps the request so a reconcile can check what +-- was asked. An operation left pending by a crash becomes uncertain and is +-- never sent again without the owner asking. +CREATE TABLE vm_operations ( + id TEXT NOT NULL, + owner_id TEXT NOT NULL, + experiment_id TEXT NOT NULL REFERENCES experiment_vms(id), + kind TEXT NOT NULL, + state TEXT NOT NULL, + request_digest TEXT NOT NULL, + request_json TEXT NOT NULL, + result_json TEXT, + error_message TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (owner_id, id) +); +CREATE INDEX vm_operations_experiment ON vm_operations(experiment_id, created_at); +CREATE INDEX vm_operations_unsettled ON vm_operations(state) WHERE state IN ('pending', 'uncertain'); diff --git a/scripts/check.mjs b/scripts/check.mjs index 65147a36..96b6c1ed 100644 --- a/scripts/check.mjs +++ b/scripts/check.mjs @@ -15,6 +15,7 @@ const tests = [ "test/json-column-boundaries.test.ts", "test/discriminating-fields.test.ts", "test/run-state-documentation.test.ts", + "test/vm-state-documentation.test.ts", "test/guidance-links.test.ts", "test/migration-policy.test.ts", "test/row-schema-columns.test.ts", diff --git a/src/server/sql/experiment-previews.ts b/src/server/sql/experiment-previews.ts new file mode 100644 index 00000000..29418594 --- /dev/null +++ b/src/server/sql/experiment-previews.ts @@ -0,0 +1,96 @@ +import type Database from "better-sqlite3"; +import { QueryRecords } from "../query-records.js"; +import { parseRecord } from "../row-schemas.js"; +import { statement } from "./statement.js"; + +// Queries for the experiment_previews table (docs/architecture.md#recipe-add-or-extend-a-sql-table-module). + +const parse = (row: unknown) => + parseRecord(QueryRecords.experiment_previews, row, "experiment_previews"); + +// INSERT … SELECT needs a WHERE before ON CONFLICT so SQLite does not read +// the upsert clause as a join constraint. +const upsertStatement = statement( + "INSERT INTO experiment_previews(id,owner_id,experiment_id,port,label,path,url,active,created_at,updated_at) SELECT ?,?,?,?,?,?,?,1,?,? WHERE EXISTS(SELECT 1 FROM experiment_vms WHERE id=? AND owner_id=?) ON CONFLICT(experiment_id,port) DO UPDATE SET label=excluded.label,path=excluded.path,url=excluded.url,active=1,updated_at=excluded.updated_at WHERE experiment_previews.owner_id=excluded.owner_id", +); +/** One preview per experiment port; exposing the port again reactivates it. + * Returns false, writing nothing, when the experiment is not the owner's. */ +export function upsert( + db: Database.Database, + preview: { + id: string; + owner: string; + experimentId: string; + port: number; + label: string | null; + path: string; + url: string; + now: number; + }, +) { + return ( + upsertStatement(db).run( + preview.id, + preview.owner, + preview.experimentId, + preview.port, + preview.label, + preview.path, + preview.url, + preview.now, + preview.now, + preview.experimentId, + preview.owner, + ).changes === 1 + ); +} + +const byExperimentStatement = statement( + "SELECT * FROM experiment_previews WHERE owner_id=? AND experiment_id=? ORDER BY port", +); +export function byExperiment( + db: Database.Database, + owner: string, + experiment: string, +) { + return byExperimentStatement(db).all(owner, experiment).map(parse); +} + +const byPortStatement = statement( + "SELECT * FROM experiment_previews WHERE owner_id=? AND experiment_id=? AND port=?", +); +export function byPort( + db: Database.Database, + owner: string, + experiment: string, + port: number, +) { + const row = byPortStatement(db).get(owner, experiment, port); + return row === undefined ? undefined : parse(row); +} + +const deactivateStatement = statement( + "UPDATE experiment_previews SET active=0,updated_at=? WHERE owner_id=? AND experiment_id=? AND port=?", +); +export function deactivate( + db: Database.Database, + owner: string, + experiment: string, + port: number, + now: number, +) { + return deactivateStatement(db).run(now, owner, experiment, port).changes; +} + +const deactivateAllStatement = statement( + "UPDATE experiment_previews SET active=0,updated_at=? WHERE owner_id=? AND experiment_id=? AND active=1", +); +/** A destroyed VM serves nothing, so every link of the experiment is disabled. */ +export function deactivateAll( + db: Database.Database, + owner: string, + experiment: string, + now: number, +) { + return deactivateAllStatement(db).run(now, owner, experiment).changes; +} diff --git a/src/server/sql/experiment-vms.ts b/src/server/sql/experiment-vms.ts new file mode 100644 index 00000000..12600883 --- /dev/null +++ b/src/server/sql/experiment-vms.ts @@ -0,0 +1,130 @@ +import type Database from "better-sqlite3"; +import { QueryRecords } from "../query-records.js"; +import { parseRecord } from "../row-schemas.js"; +import { statement } from "./statement.js"; + +// Queries for the experiment_vms table (docs/architecture.md#recipe-add-or-extend-a-sql-table-module). +// state is written only through setState, called by transitionVm() in +// vm/machine.ts (docs/state-machines.md#experiment_vmsstate). + +const parse = (row: unknown) => + parseRecord(QueryRecords.experiment_vms, row, "experiment_vms"); + +const insertStatement = statement( + "INSERT INTO experiment_vms(id,owner_id,folder_id,provider,remote_name,state,auto_approve_local,vcpu_limit,memory_limit_gib,disk_limit_gib,auto_sleep_seconds,source_experiment_id,source_checkpoint_id,clone_url,last_active_at,created_at) SELECT ?,?,?,?,?,'provisioning',?,?,?,?,?,?,?,?,?,? WHERE EXISTS(SELECT 1 FROM folders WHERE id=? AND owner_id=?) AND (? IS NULL OR EXISTS(SELECT 1 FROM experiment_vms WHERE id=? AND owner_id=?))", +); +/** Inserts the owner's experiment. Returns false, writing nothing, when the + * folder or the source experiment is not the owner's. */ +export function insert( + db: Database.Database, + vm: { + id: string; + owner: string; + folderId: string; + provider: string; + remoteName: string; + autoApproveLocal: boolean; + vcpuLimit: number; + memoryLimitGib: number; + diskLimitGib: number; + autoSleepSeconds: number; + sourceExperimentId: string | null; + sourceCheckpointId: string | null; + cloneUrl: string | null; + now: number; + }, +) { + return ( + insertStatement(db).run( + vm.id, + vm.owner, + vm.folderId, + vm.provider, + vm.remoteName, + vm.autoApproveLocal ? 1 : 0, + vm.vcpuLimit, + vm.memoryLimitGib, + vm.diskLimitGib, + vm.autoSleepSeconds, + vm.sourceExperimentId, + vm.sourceCheckpointId, + vm.cloneUrl, + vm.now, + vm.now, + vm.folderId, + vm.owner, + vm.sourceExperimentId, + vm.sourceExperimentId, + vm.owner, + ).changes === 1 + ); +} + +const byIdStatement = statement( + "SELECT * FROM experiment_vms WHERE owner_id=? AND id=?", +); +export function byId(db: Database.Database, owner: string, id: string) { + const row = byIdStatement(db).get(owner, id); + return row === undefined ? undefined : parse(row); +} + +const byFolderStatement = statement( + "SELECT * FROM experiment_vms WHERE owner_id=? AND folder_id=?", +); +export function byFolder(db: Database.Database, owner: string, folder: string) { + const row = byFolderStatement(db).get(owner, folder); + return row === undefined ? undefined : parse(row); +} + +const ownedStatement = statement( + "SELECT * FROM experiment_vms WHERE owner_id=? ORDER BY created_at DESC, id", +); +export function owned(db: Database.Database, owner: string) { + return ownedStatement(db).all(owner).map(parse); +} + +const setStateStatement = statement( + "UPDATE experiment_vms SET state=?,last_error=?,destroyed_at=CASE WHEN ?='destroyed' THEN ? ELSE destroyed_at END WHERE owner_id=? AND id=? AND state=?", +); +/** Moves `from` to `to`; returns false when the row is no longer in `from`. */ +export function setState( + db: Database.Database, + owner: string, + id: string, + from: string, + to: string, + lastError: string | null, + now: number, +) { + return ( + setStateStatement(db).run(to, lastError, to, now, owner, id, from) + .changes === 1 + ); +} + +const adoptRemoteStatement = statement( + "UPDATE experiment_vms SET remote_id=? WHERE owner_id=? AND id=? AND (remote_id IS NULL OR remote_id=?)", +); +/** Records the provider's id once; a different id for the same VM is refused. */ +export function adoptRemote( + db: Database.Database, + owner: string, + id: string, + remoteId: string, +) { + return ( + adoptRemoteStatement(db).run(remoteId, owner, id, remoteId).changes === 1 + ); +} + +const touchStatement = statement( + "UPDATE experiment_vms SET last_active_at=? WHERE owner_id=? AND id=?", +); +export function touch( + db: Database.Database, + owner: string, + id: string, + now: number, +) { + touchStatement(db).run(now, owner, id); +} diff --git a/src/server/sql/rows.ts b/src/server/sql/rows.ts index fb89000f..88a25781 100644 --- a/src/server/sql/rows.ts +++ b/src/server/sql/rows.ts @@ -7,6 +7,13 @@ import { MessageRole } from "../../shared/message-roles.js"; import { AgentOrigin, ConversationOrigin } from "../../shared/origins.js"; import { ConnectionStatus } from "../../shared/connection-status.js"; import { RunKind } from "../../shared/run-state.js"; +import { + VmHarness, + VmOperationKind, + VmOperationState, + VmProviderKind, + VmState, +} from "../../shared/vm-state.js"; // Scalar SQLite storage shapes. JSON columns remain serialized at this boundary. export const RowId = z.string().min(1); @@ -1401,6 +1408,63 @@ export const QueryRecords = { owner_id: z.string(), card_id: z.string(), }), + vm_provider_configs: z.object({ + owner_id: z.string(), + provider: VmProviderKind, + encrypted_api_key: z.string(), + app_id: z.string().nullable(), + auth_volume_id: z.string().nullable(), + base_image_ref: z.string().nullable(), + base_image_version: z.string().nullable(), + default_harness: VmHarness.nullable(), + updated_at: z.number(), + }), + experiment_vms: z.object({ + id: z.string(), + owner_id: z.string(), + folder_id: z.string(), + provider: VmProviderKind, + remote_id: z.string().nullable(), + remote_name: z.string(), + state: VmState, + auto_approve_local: z.number().int(), + vcpu_limit: z.number().int(), + memory_limit_gib: z.number().int(), + disk_limit_gib: z.number().int(), + auto_sleep_seconds: z.number().int(), + source_experiment_id: z.string().nullable(), + source_checkpoint_id: z.string().nullable(), + clone_url: z.string().nullable(), + last_error: z.string().nullable(), + last_active_at: z.number(), + created_at: z.number(), + destroyed_at: z.number().nullable(), + }), + experiment_previews: z.object({ + id: z.string(), + owner_id: z.string(), + experiment_id: z.string(), + port: z.number().int(), + label: z.string().nullable(), + path: z.string(), + url: z.string(), + active: z.number().int(), + created_at: z.number(), + updated_at: z.number(), + }), + vm_operations: z.object({ + id: z.string(), + owner_id: z.string(), + experiment_id: z.string(), + kind: VmOperationKind, + state: VmOperationState, + request_digest: z.string(), + request_json: z.string(), + result_json: z.string().nullable(), + error_message: z.string().nullable(), + created_at: z.number(), + updated_at: z.number(), + }), } as const; export type QueryRow = z.infer< (typeof QueryRecords)[T] diff --git a/src/server/sql/vm-operations.ts b/src/server/sql/vm-operations.ts new file mode 100644 index 00000000..534305e4 --- /dev/null +++ b/src/server/sql/vm-operations.ts @@ -0,0 +1,98 @@ +import type Database from "better-sqlite3"; +import { QueryRecords } from "../query-records.js"; +import { parseRecord } from "../row-schemas.js"; +import { statement } from "./statement.js"; + +// Queries for the vm_operations table (docs/architecture.md#recipe-add-or-extend-a-sql-table-module). +// state is written only through insertPending and setState, called by +// vm/machine.ts (docs/state-machines.md#vm_operationsstate). + +const parse = (row: unknown) => + parseRecord(QueryRecords.vm_operations, row, "vm_operations"); + +const insertPendingStatement = statement( + "INSERT INTO vm_operations(id,owner_id,experiment_id,kind,state,request_digest,request_json,created_at,updated_at) VALUES(?,?,?,?,'pending',?,?,?,?)", +); +export function insertPending( + db: Database.Database, + operation: { + id: string; + owner: string; + experimentId: string; + kind: string; + requestDigest: string; + requestJson: string; + now: number; + }, +) { + insertPendingStatement(db).run( + operation.id, + operation.owner, + operation.experimentId, + operation.kind, + operation.requestDigest, + operation.requestJson, + operation.now, + operation.now, + ); +} + +const byIdStatement = statement( + "SELECT * FROM vm_operations WHERE owner_id=? AND id=?", +); +export function byId(db: Database.Database, owner: string, id: string) { + const row = byIdStatement(db).get(owner, id); + return row === undefined ? undefined : parse(row); +} + +const unsettledByExperimentStatement = statement( + "SELECT * FROM vm_operations WHERE owner_id=? AND experiment_id=? AND state IN ('pending','uncertain') ORDER BY created_at, id", +); +export function unsettledByExperiment( + db: Database.Database, + owner: string, + experiment: string, +) { + return unsettledByExperimentStatement(db).all(owner, experiment).map(parse); +} + +const setStateStatement = statement( + "UPDATE vm_operations SET state=?,result_json=?,error_message=?,updated_at=? WHERE owner_id=? AND id=? AND state=?", +); +/** Moves `from` to `to`; returns false when the row is no longer in `from`. */ +export function setState( + db: Database.Database, + owner: string, + id: string, + from: string, + to: string, + outcome: { resultJson: string | null; errorMessage: string | null }, + now: number, +) { + return ( + setStateStatement(db).run( + to, + outcome.resultJson, + outcome.errorMessage, + now, + owner, + id, + from, + ).changes === 1 + ); +} + +const pendingIdsStatement = statement( + "SELECT id,owner_id FROM vm_operations WHERE state='pending' AND updated_at parseRecord(pendingIdRow, row, "vm_operations")); +} diff --git a/src/server/sql/vm-provider-configs.ts b/src/server/sql/vm-provider-configs.ts new file mode 100644 index 00000000..076e839e --- /dev/null +++ b/src/server/sql/vm-provider-configs.ts @@ -0,0 +1,74 @@ +import type Database from "better-sqlite3"; +import { QueryRecords } from "../query-records.js"; +import { parseRecord } from "../row-schemas.js"; +import { statement } from "./statement.js"; + +// Queries for the vm_provider_configs table (docs/architecture.md#recipe-add-or-extend-a-sql-table-module). +// encrypted_api_key is sealed by the caller; this module never sees the key. + +const byOwnerStatement = statement( + "SELECT * FROM vm_provider_configs WHERE owner_id=?", +); +export function byOwner(db: Database.Database, owner: string) { + const row = byOwnerStatement(db).get(owner); + return row === undefined + ? undefined + : parseRecord(QueryRecords.vm_provider_configs, row, "vm_provider_configs"); +} + +const saveKeyStatement = statement( + "INSERT INTO vm_provider_configs(owner_id,provider,encrypted_api_key,updated_at) VALUES(?,?,?,?) ON CONFLICT(owner_id) DO UPDATE SET provider=excluded.provider,encrypted_api_key=excluded.encrypted_api_key,updated_at=excluded.updated_at", +); +/** Stores the sealed key, keeping the resources already found for the owner. */ +export function saveKey( + db: Database.Database, + owner: string, + provider: string, + sealedKey: string, + now: number, +) { + saveKeyStatement(db).run(owner, provider, sealedKey, now); +} + +const setResourcesStatement = statement( + "UPDATE vm_provider_configs SET app_id=?,auth_volume_id=?,base_image_ref=?,base_image_version=?,updated_at=? WHERE owner_id=?", +); +export function setResources( + db: Database.Database, + owner: string, + resources: { + appId: string | null; + authVolumeId: string | null; + baseImageRef: string | null; + baseImageVersion: string | null; + }, + now: number, +) { + return setResourcesStatement(db).run( + resources.appId, + resources.authVolumeId, + resources.baseImageRef, + resources.baseImageVersion, + now, + owner, + ).changes; +} + +const setDefaultHarnessStatement = statement( + "UPDATE vm_provider_configs SET default_harness=?,updated_at=? WHERE owner_id=?", +); +export function setDefaultHarness( + db: Database.Database, + owner: string, + harness: string | null, + now: number, +) { + return setDefaultHarnessStatement(db).run(harness, now, owner).changes; +} + +const removeStatement = statement( + "DELETE FROM vm_provider_configs WHERE owner_id=?", +); +export function remove(db: Database.Database, owner: string) { + return removeStatement(db).run(owner).changes; +} diff --git a/src/server/vm/machine.ts b/src/server/vm/machine.ts new file mode 100644 index 00000000..d2544d26 --- /dev/null +++ b/src/server/vm/machine.ts @@ -0,0 +1,112 @@ +import type Database from "better-sqlite3"; +import { + legalVmOperationTransition, + legalVmTransition, + type VmOperationState, + type VmState, +} from "../../shared/vm-state.js"; +import * as experimentVmsQueries from "../sql/experiment-vms.js"; +import * as vmOperationsQueries from "../sql/vm-operations.js"; + +// The only writers of experiment_vms.state and vm_operations.state +// (docs/state-machines.md#cloud-vm-experiments). Each re-reads the row and +// checks its state inside one immediate transaction, so a caller that decided +// from an older read changes nothing. + +export type VmTransitionResult = + | { changed: true; from: VmState } + | { changed: false; reason: "missing" | "illegal"; from?: VmState }; + +/** Moves the owner's VM to `to` when that edge is legal from its current + * state. `expected` narrows the move to a state the caller observed. */ +export function transitionVm( + db: Database.Database, + vm: { owner: string; id: string }, + to: VmState, + options: { expected?: VmState; lastError?: string | null; now: number }, +): VmTransitionResult { + return db + .transaction((): VmTransitionResult => { + const row = experimentVmsQueries.byId(db, vm.owner, vm.id); + if (!row) return { changed: false, reason: "missing" }; + if ( + (options.expected !== undefined && row.state !== options.expected) || + !legalVmTransition(row.state, to) + ) + return { changed: false, reason: "illegal", from: row.state }; + experimentVmsQueries.setState( + db, + vm.owner, + vm.id, + row.state, + to, + options.lastError ?? null, + options.now, + ); + return { changed: true, from: row.state }; + }) + .immediate(); +} + +/** Settles or marks uncertain the owner's operation receipt. Returns false + * when the receipt is missing or the edge is not legal from its state. */ +export function transitionVmOperation( + db: Database.Database, + operation: { owner: string; id: string }, + to: Exclude, + outcome: { result?: unknown; errorMessage?: string | null; now: number }, +): boolean { + return db + .transaction(() => { + const row = vmOperationsQueries.byId(db, operation.owner, operation.id); + if (!row || !legalVmOperationTransition(row.state, to)) return false; + return vmOperationsQueries.setState( + db, + operation.owner, + operation.id, + row.state, + to, + { + resultJson: + outcome.result === undefined + ? row.result_json + : JSON.stringify(outcome.result), + errorMessage: outcome.errorMessage ?? null, + }, + outcome.now, + ); + }) + .immediate(); +} + +/** No provider call Loop makes waits longer than this (the longest is a + * boot, at ten minutes in vm/sailbox-provider.ts). */ +export const VM_OPERATION_MAX_CALL_MS = 15 * 60_000; + +/** A receipt still pending after the longest provider call belongs to a + * caller that died between writing it and observing the answer, whichever + * process it was. Its outcome is unknown, so it becomes uncertain and is + * never sent again automatically. The worker runs this on every tick. */ +export function markStaleVmOperationsUncertain( + db: Database.Database, + now: number, +) { + let marked = 0; + for (const row of vmOperationsQueries.pendingBefore( + db, + now - VM_OPERATION_MAX_CALL_MS, + )) + if ( + transitionVmOperation( + db, + { owner: row.owner_id, id: row.id }, + "uncertain", + { + errorMessage: "Loop stopped before the provider answered", + now, + }, + ) + ) + marked += 1; + return marked; +} diff --git a/src/server/vm/operations.ts b/src/server/vm/operations.ts new file mode 100644 index 00000000..661ba667 --- /dev/null +++ b/src/server/vm/operations.ts @@ -0,0 +1,631 @@ +import { createHash } from "node:crypto"; +import type Database from "better-sqlite3"; +import { + legalVmTransition, + type VmOperationKind, + type VmState, +} from "../../shared/vm-state.js"; +import * as experimentPreviewsQueries from "../sql/experiment-previews.js"; +import * as experimentVmsQueries from "../sql/experiment-vms.js"; +import * as vmOperationsQueries from "../sql/vm-operations.js"; +import { + VM_OPERATION_MAX_CALL_MS, + transitionVm, + transitionVmOperation, +} from "./machine.js"; +import { VmProviderError, type VmProvider, type VmStatus } from "./provider.js"; + +// Every provider call that changes something runs through runVmOperation: a +// receipt is committed before the call and settled from what was observed. +// Nothing here calls the provider a second time. An unobserved outcome is +// kept as "uncertain" until the owner asks for a reconcile. + +export type VmOperationOutcome = + | { state: "succeeded"; result: unknown; replayed: boolean } + | { state: "failed"; errorMessage: string; replayed: boolean } + | { state: "uncertain" | "pending"; errorMessage: string | null }; + +export type VmOperationRefusal = + /** The owner already used the id for a different request. */ + | "key_conflict" + /** The experiment does not exist for this owner. */ + | "missing_experiment" + /** An earlier operation on this experiment is pending or uncertain. */ + | "unsettled_operation" + /** A create or fork for an experiment that already has its VM. */ + | "vm_exists" + /** Any other operation on an experiment with no recorded VM. */ + | "no_vm"; + +export class VmOperationRefused extends Error { + constructor(readonly reason: VmOperationRefusal) { + super(`VM operation refused: ${reason}`); + this.name = "VmOperationRefused"; + } +} + +/** The request with object keys sorted, so the same request always has the + * same text, and so the same digest, whatever order its fields were in. */ +export function canonicalVmRequest(value: unknown): string { + if (Array.isArray(value)) + return `[${value.map(canonicalVmRequest).join(",")}]`; + if (value !== null && typeof value === "object") + return `{${Object.entries(value) + .filter(([, item]) => item !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map( + ([key, item]) => `${JSON.stringify(key)}:${canonicalVmRequest(item)}`, + ) + .join(",")}}`; + return JSON.stringify(value ?? null); +} + +/** SHA-256 of the canonical request. */ +export function vmRequestDigest(request: unknown) { + return createHash("sha256").update(canonicalVmRequest(request)).digest("hex"); +} + +type OperationRow = NonNullable>; + +function stored(row: OperationRow): VmOperationOutcome { + if (row.state === "succeeded") + return { + state: "succeeded", + result: row.result_json === null ? null : JSON.parse(row.result_json), + replayed: true, + }; + if (row.state === "failed") + return { + state: "failed", + errorMessage: row.error_message ?? "The provider refused the request", + replayed: true, + }; + return { state: row.state, errorMessage: row.error_message }; +} + +/** Runs `call` at most once for `operation.id`. A repeat of the same id and + * request returns the stored outcome without calling the provider; the same + * id with a different request is refused. The receipt is the record: when a + * reconcile settled it while the call was still out, the stored outcome is + * returned, not the late answer. */ +export async function runVmOperation( + db: Database.Database, + operation: { + id: string; + owner: string; + experimentId: string; + kind: VmOperationKind; + request: unknown; + }, + call: () => Promise, + clock: () => number, + // The kind's own local writes unless a caller passes others (tests of the + // receipt alone pass {}), so a create cannot settle without its VM id. + effects: VmOperationEffects = vmOperationEffects(db, operation, clock), +): Promise { + const requestJson = canonicalVmRequest(operation.request); + const requestDigest = vmRequestDigest({ + kind: operation.kind, + experimentId: operation.experimentId, + request: operation.request, + }); + const prior = db + .transaction(() => { + const existing = vmOperationsQueries.byId( + db, + operation.owner, + operation.id, + ); + if (existing) { + if (existing.request_digest !== requestDigest) + throw new VmOperationRefused("key_conflict"); + return existing; + } + const vm = experimentVmsQueries.byId( + db, + operation.owner, + operation.experimentId, + ); + if (!vm) throw new VmOperationRefused("missing_experiment"); + // One experiment has one provider VM. Once it is recorded, a failed VM + // is destroyed, not created again: the provider key is scoped to the + // experiment, so another create would only return the same VM. + if ( + (operation.kind === "create" || operation.kind === "fork") && + vm.remote_id !== null + ) + throw new VmOperationRefused("vm_exists"); + // Anything else acts on the recorded VM; without one, an uncertain + // outcome could never be reconciled. + if ( + operation.kind !== "create" && + operation.kind !== "fork" && + vm.remote_id === null + ) + throw new VmOperationRefused("no_vm"); + if ( + vmOperationsQueries.unsettledByExperiment( + db, + operation.owner, + operation.experimentId, + ).length > 0 + ) + throw new VmOperationRefused("unsettled_operation"); + vmOperationsQueries.insertPending(db, { + id: operation.id, + owner: operation.owner, + experimentId: operation.experimentId, + kind: operation.kind, + requestDigest, + requestJson, + now: clock(), + }); + return undefined; + }) + .immediate(); + if (prior) return stored(prior); + + const receipt = { owner: operation.owner, id: operation.id }; + const current = () => + stored(vmOperationsQueries.byId(db, receipt.owner, receipt.id)!); + let result: unknown; + try { + result = await call(); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + // Only an answer from the provider proves the call did nothing. Anything + // else, including a bug in the adapter, leaves the outcome unknown. + const to = + error instanceof VmProviderError && error.outcome === "refused" + ? "failed" + : "uncertain"; + const settled = db + .transaction(() => { + if ( + !transitionVmOperation(db, receipt, to, { + errorMessage, + now: clock(), + }) + ) + return false; + if (to === "failed") effects.refused?.(errorMessage); + return true; + }) + .immediate(); + if (!settled) return current(); + return to === "failed" + ? { state: "failed", errorMessage, replayed: false } + : { state: "uncertain", errorMessage }; + } + const incomplete = effects.incomplete?.(result) ?? null; + const errorMessage = incomplete ?? effects.failure?.(result) ?? null; + const to = + incomplete !== null + ? "uncertain" + : errorMessage === null + ? "succeeded" + : "failed"; + // The receipt and the local record of what the call did commit together. + // If apply throws, both roll back and the receipt stays pending until the + // sweep marks it uncertain for a reconcile. An incomplete answer writes + // nothing locally: its provider id is provisional until a reconcile sees + // the VM, and lives only in the receipt. + const settled = db + .transaction(() => { + if ( + !transitionVmOperation(db, receipt, to, { + result, + errorMessage, + now: clock(), + }) + ) + return false; + if (to !== "uncertain") effects.apply?.(result); + return true; + }) + .immediate(); + if (!settled) return current(); + if (to === "uncertain") return { state: "uncertain", errorMessage }; + return errorMessage === null + ? { state: "succeeded", result, replayed: false } + : { state: "failed", errorMessage, replayed: false }; +} + +/** The local record cannot take what the provider answered. */ +export class VmRecordConflict extends Error { + constructor(message: string) { + super(message); + this.name = "VmRecordConflict"; + } +} + +/** What a call's answer means locally. Each hook runs inside the + * transaction that settles the receipt, so the local record and the receipt + * commit together or not at all. */ +export type VmOperationEffects = { + /** An answer that does not show the outcome yet, such as a VM still + * starting: the receipt stays uncertain (with the answer and `apply`'s + * writes kept) for a reconcile to settle. */ + incomplete?(result: unknown): string | null; + /** A failure the answer itself reports, such as a VM that booted into + * "failed"; null when the call did what was asked. */ + failure?(result: unknown): string | null; + /** Local writes for a resolved call, including one `failure` failed. */ + apply?(result: unknown): void; + /** Local writes for a provider refusal. */ + refused?(errorMessage: string): void; +}; + +type CreatedVm = { remoteId: string; status: VmStatus; errorMessage?: string }; + +const kindStatus: Partial> = { + sleep: "sleeping", + resume: "running", + destroy: "terminated", +}; + +/** The local writes for each operation kind. runVmOperation and + * reconcileVmOperation both use them, so an answer and a reconcile leave the + * same record: a create or fork adopts the provider id and observed state, + * sleep/resume/destroy move the VM, expose and unexpose keep the preview + * rows in step, and a destroy disables every preview link. */ +export function vmOperationEffects( + db: Database.Database, + operation: { + owner: string; + experimentId: string; + id: string; + kind: VmOperationKind; + request: unknown; + }, + clock: () => number, +): VmOperationEffects { + const vm = { owner: operation.owner, id: operation.experimentId }; + const current = () => { + const row = experimentVmsQueries.byId(db, vm.owner, vm.id); + if (!row) throw new VmRecordConflict("The experiment no longer exists"); + return { ...vm, state: row.state }; + }; + const created = operation.kind === "create" || operation.kind === "fork"; + return { + incomplete: (result) => { + if (!created) return null; + const { status } = result as CreatedVm; + return status === "running" || + status === "sleeping" || + status === "paused" || + status === "failed" + ? null + : `The VM is ${status}`; + }, + failure: (result) => + created && (result as CreatedVm).status === "failed" + ? ((result as CreatedVm).errorMessage ?? + "The provider reports the VM failed") + : null, + apply: (result) => { + if (created) { + const vmResult = result as CreatedVm; + if ( + !experimentVmsQueries.adoptRemote( + db, + vm.owner, + vm.id, + vmResult.remoteId, + ) + ) + throw new VmRecordConflict( + "The experiment already records another provider VM", + ); + applyObservedStatus( + db, + current(), + vmResult.status, + clock(), + vmResult.errorMessage, + ); + return; + } + // A destroy the caller sent without first marking the VM destroying + // still ends destroyed, through destroying, rather than failed. + if (operation.kind === "destroy") { + const row = current(); + if (row.state !== "destroying" && row.state !== "destroyed") + transitionVm(db, row, "destroying", { + expected: row.state, + now: clock(), + }); + } + const status = kindStatus[operation.kind]; + if (status) applyObservedStatus(db, current(), status, clock()); + if (operation.kind === "destroy") + experimentPreviewsQueries.deactivateAll(db, vm.owner, vm.id, clock()); + if (operation.kind === "expose" || operation.kind === "unexpose") { + const request = (operation.request ?? {}) as { + port?: unknown; + label?: unknown; + path?: unknown; + }; + if (typeof request.port !== "number") + throw new VmRecordConflict("The operation names no port"); + if (operation.kind === "unexpose") { + experimentPreviewsQueries.deactivate( + db, + vm.owner, + vm.id, + request.port, + clock(), + ); + return; + } + const { url } = result as { url: string }; + if ( + !experimentPreviewsQueries.upsert(db, { + // Experiment ids are unique across owners; operation ids are not. + id: `${vm.id}:${request.port}`, + owner: vm.owner, + experimentId: vm.id, + port: request.port, + label: typeof request.label === "string" ? request.label : null, + path: typeof request.path === "string" ? request.path : "/", + url, + now: clock(), + }) + ) + throw new VmRecordConflict("The experiment is not the owner's"); + } + }, + refused: (errorMessage) => { + if (operation.kind !== "destroy") return; + const row = current(); + if (row.state === "destroying") + transitionVm(db, row, "failed", { + expected: "destroying", + lastError: errorMessage, + now: clock(), + }); + }, + }; +} + +/** Moves the VM to the state the provider reports, when that edge is legal + * from where it is. Call inside the transaction that settles the receipt. */ +function applyObservedStatus( + db: Database.Database, + vm: { owner: string; id: string; state: VmState }, + status: VmStatus, + now: number, + reason?: string, +) { + const observed: Record = { + provisioning: undefined, + running: "running", + sleeping: "sleeping", + paused: "paused", + failed: "failed", + terminating: undefined, + terminated: vm.state === "destroying" ? "destroyed" : "failed", + unknown: undefined, + }; + let to = observed[status]; + // A destroy that did not finish leaves a VM the owner must destroy again; + // failed is the state that allows it. + if (to && vm.state === "destroying" && to !== "destroyed") to = "failed"; + if (!to || to === vm.state || !legalVmTransition(vm.state, to)) return; + transitionVm(db, vm, to, { + expected: vm.state, + lastError: + to === "failed" + ? (reason ?? + (vm.state === "destroying" + ? `The destroy did not finish; the provider reports the VM ${status}` + : `The provider reports the VM ${status}`)) + : null, + now, + }); +} + +export type VmReconciliation = + | { outcome: "succeeded"; result: unknown } + | { outcome: "failed"; errorMessage: string } + | { outcome: "still_uncertain"; reason: string }; + +// Until the longest provider call has passed, the original call may still +// land and change what a lookup saw: a late delete can remove a listener a +// later expose added, a late sleep can stop a VM a later resume woke. So an +// observation settles only when its lookup began after that window. The one +// exception is `early`: a create or fork whose VM exists, because a late +// arrival of the original create reaches the same VM (its provider +// idempotency key is scoped to the experiment) and cannot undo it. +type Observation = + | { settle: "succeeded"; result: unknown; status?: VmStatus; early?: true } + | { + settle: "failed"; + errorMessage: string; + status?: VmStatus; + result?: unknown; + early?: true; + } + | { settle: "unknown"; reason: string }; + +class SettledConcurrently extends Error {} + +const unsure = (reason: string): Observation => ({ settle: "unknown", reason }); + +/** Observes the outcome of the owner's uncertain operation and settles it. + * This only reads from the provider: nothing is created, sent or retried. + * The receipt and the VM's observed state are written in one transaction, + * and only while the receipt is still uncertain. */ +export async function reconcileVmOperation( + db: Database.Database, + provider: Pick, + target: { owner: string; operationId: string; appId: string }, + clock: () => number, +): Promise { + const operation = vmOperationsQueries.byId( + db, + target.owner, + target.operationId, + ); + if (!operation || operation.state !== "uncertain") + return { outcome: "still_uncertain", reason: "not an uncertain operation" }; + const vm = experimentVmsQueries.byId( + db, + target.owner, + operation.experiment_id, + ); + if (!vm) return { outcome: "still_uncertain", reason: "experiment missing" }; + + // The age is taken before the provider is asked, so the lookup itself + // must begin after the original call could have finished. + const observedAt = clock(); + let observation: Observation; + try { + observation = await observe(provider, operation, vm, target.appId); + } catch (error) { + observation = unsure( + error instanceof Error ? error.message : String(error), + ); + } + if (observation.settle === "unknown") + return { outcome: "still_uncertain", reason: observation.reason }; + if ( + !observation.early && + observedAt - operation.created_at <= VM_OPERATION_MAX_CALL_MS + ) + return { + outcome: "still_uncertain", + reason: "the original call may still be in flight", + }; + const seen = observation; + + const receipt = { owner: target.owner, id: operation.id }; + try { + db.transaction(() => { + const fresh = experimentVmsQueries.byId(db, target.owner, vm.id); + if (!fresh) throw new SettledConcurrently(); + const settled = + seen.settle === "succeeded" + ? transitionVmOperation(db, receipt, "succeeded", { + result: seen.result, + now: clock(), + }) + : transitionVmOperation(db, receipt, "failed", { + result: seen.result, + errorMessage: seen.errorMessage, + now: clock(), + }); + if (!settled) throw new SettledConcurrently(); + const effects = vmOperationEffects( + db, + { + owner: target.owner, + experimentId: vm.id, + id: operation.id, + kind: operation.kind, + request: JSON.parse(operation.request_json), + }, + clock, + ); + try { + if (seen.settle === "succeeded" || seen.result !== undefined) + effects.apply?.(seen.result); + else if (seen.status) + applyObservedStatus( + db, + { owner: target.owner, id: vm.id, state: fresh.state }, + seen.status, + clock(), + ); + } catch (error) { + // The local record no longer matches what was observed (another + // writer got there first): keep the receipt uncertain. Any other + // error, such as a failed write, propagates and rolls back. + if (error instanceof VmRecordConflict) throw new SettledConcurrently(); + throw error; + } + }).immediate(); + } catch (error) { + if (error instanceof SettledConcurrently) + return { outcome: "still_uncertain", reason: "settled concurrently" }; + throw error; + } + return seen.settle === "succeeded" + ? { outcome: "succeeded", result: seen.result } + : { outcome: "failed", errorMessage: seen.errorMessage }; +} + +async function observe( + provider: Pick, + operation: OperationRow, + vm: NonNullable>, + appId: string, +): Promise { + if (operation.kind === "create" || operation.kind === "fork") { + const matches = await provider.findByName(appId, vm.remote_name); + if (matches.length > 1) + return unsure(`${matches.length} VMs are named ${vm.remote_name}`); + const [match] = matches; + if (!match) + return { + settle: "failed", + errorMessage: "The provider has no VM with this name", + status: "terminated", + }; + if (match.status === "failed") + return { + settle: "failed", + errorMessage: "The provider reports the VM failed", + result: { remoteId: match.remoteId, status: "failed" }, + early: true, + }; + // Only a VM that finished starting shows the create succeeded. + if ( + match.status !== "running" && + match.status !== "sleeping" && + match.status !== "paused" + ) + return unsure(`the VM is ${match.status}`); + return { + settle: "succeeded", + result: { remoteId: match.remoteId, status: match.status }, + early: true, + }; + } + if (!vm.remote_id) return unsure("the VM has no provider id"); + if (operation.kind === "expose" || operation.kind === "unexpose") { + const request: unknown = JSON.parse(operation.request_json); + const port = (request as { port?: unknown } | null)?.port; + if (typeof port !== "number") return unsure("the receipt names no port"); + const published = await provider.getHttpPort(vm.remote_id, port); + if (operation.kind === "expose") + return published + ? { settle: "succeeded", result: published } + : { + settle: "failed", + errorMessage: "The port is not published", + }; + return published + ? { + settle: "failed", + errorMessage: "The port is still published", + } + : { settle: "succeeded", result: null }; + } + const status = (await provider.get(vm.remote_id))?.status ?? "terminated"; + if (status === "unknown" || status === "provisioning") + return unsure(`the provider reports the VM ${status}`); + const reached = { + sleep: status === "sleeping", + resume: status === "running", + destroy: status === "terminated", + }[operation.kind]; + if (reached) return { settle: "succeeded", result: null, status }; + if (operation.kind === "destroy" && status === "terminating") + return unsure("the VM is still shutting down"); + return { + settle: "failed", + errorMessage: `The provider reports the VM ${status}`, + status, + }; +} diff --git a/src/server/vm/provider.ts b/src/server/vm/provider.ts new file mode 100644 index 00000000..643a37d1 --- /dev/null +++ b/src/server/vm/provider.ts @@ -0,0 +1,132 @@ +// The boundary between Loop and a VM provider. Loop calls a provider only +// after writing a vm_operations receipt (vm/operations.ts); adapters do not +// retry, so an unobserved outcome reaches the caller as VmProviderError with +// outcome "uncertain". + +/** `unknown` is a provider status this build does not recognise; callers + * leave the VM's state alone when they see it. */ +export type VmStatus = + | "provisioning" + | "running" + | "sleeping" + | "paused" + | "failed" + | "terminating" + | "terminated" + | "unknown"; + +export type VmLimits = { + vcpuCount: number; + memoryLimitGib: number; + diskLimitGib: number; +}; + +/** A VM can exist and still have failed to start; `status` is "failed" and + * `errorMessage` carries the provider's reason. */ +export type VmCreated = { + remoteId: string; + status: VmStatus; + errorMessage?: string; +}; + +export type VmVolumeMount = { + volumeId: string; + mountPath: string; + readOnly?: boolean; +}; + +/** stdout and stderr are paced by their reader: an unread stream stops the + * provider's event stream, including the exit event, so callers read both. */ +export interface VmProcessHandle { + stdin: NodeJS.WritableStream; + stdout: NodeJS.ReadableStream; + stderr: NodeJS.ReadableStream; + exit: Promise<{ exitCode: number | null; signal: string | null }>; + kill(): Promise; +} + +/** `refused`: the provider answered and did not perform the call. + * `uncertain`: no answer was observed (timeout, disconnect, 5xx), so the call + * may have taken effect and must be reconciled before any retry. */ +export class VmProviderError extends Error { + constructor( + readonly outcome: "refused" | "uncertain", + message: string, + readonly status?: number, + ) { + super(message); + this.name = "VmProviderError"; + } +} + +export interface VmProvider { + readonly kind: "sailbox"; + /** Checks the key and returns the account it belongs to. */ + whoami(): Promise<{ accountId: string }>; + ensureApp(name: string): Promise<{ appId: string }>; + ensureVolume(name: string): Promise<{ volumeId: string }>; + /** Pass a key scoped to the experiment, not to one operation: a retry + * after a reconcile found no VM then reaches the VM a slow first attempt + * made instead of creating a second one. The same applies to + * createFromCheckpoint. */ + create(input: { + idempotencyKey: string; + appId: string; + name: string; + imageRef: string; + limits: VmLimits; + autoSleepSeconds: number; + volumes: VmVolumeMount[]; + }): Promise; + get(remoteId: string): Promise<{ status: VmStatus } | null>; + /** Live VMs with exactly this name. More than one means Loop cannot tell + * which to adopt, and the caller must leave the operation uncertain. */ + findByName( + appId: string, + name: string, + ): Promise<{ remoteId: string; status: VmStatus }[]>; + exec( + remoteId: string, + command: string[], + options: { + cwd?: string; + env?: Record; + openStdin?: boolean; + }, + ): Promise; + sleep(remoteId: string, idempotencyKey: string): Promise; + resume(remoteId: string, idempotencyKey: string): Promise; + checkpointFilesystem(input: { + idempotencyKey: string; + remoteId: string; + name: string; + ttlSeconds: number; + }): Promise<{ checkpointId: string }>; + /** The new VM keeps the checkpoint's disk, limits and volume mounts; + * `configure` applies the experiment's own limits and idle window. */ + createFromCheckpoint(input: { + idempotencyKey: string; + checkpointId: string; + name: string; + }): Promise; + /** Mounts a volume on a running VM. Repeating the same mount is safe. */ + mountVolume(remoteId: string, mount: VmVolumeMount): Promise; + /** Sets limits and the idle window. Repeating it with the same values is + * harmless, so it needs no receipt of its own. */ + configure( + remoteId: string, + settings: { limits: VmLimits; autoSleepSeconds: number }, + ): Promise; + exposeHttpPort( + remoteId: string, + port: number, + idempotencyKey: string, + ): Promise<{ url: string }>; + unexposePort(remoteId: string, port: number): Promise; + /** The published HTTP port, or null when the port is not published. */ + getHttpPort(remoteId: string, port: number): Promise<{ url: string } | null>; + readFile(remoteId: string, path: string, maxBytes: number): Promise; + writeFile(remoteId: string, path: string, bytes: Buffer): Promise; + /** Succeeds when the VM is already gone. */ + terminate(remoteId: string, idempotencyKey: string): Promise; +} diff --git a/src/server/vm/sailbox-image.ts b/src/server/vm/sailbox-image.ts new file mode 100644 index 00000000..7e76fec0 --- /dev/null +++ b/src/server/vm/sailbox-image.ts @@ -0,0 +1,121 @@ +import { createHash } from "node:crypto"; + +// The experiment image. Sail builds custom images only through its SDK, which +// Loop does not bundle, so Loop builds a "golden" Sailbox instead: boot +// SAILBOX_BASE_IMAGE with no volume mounted, run INSTALL_SCRIPT over exec, +// take a filesystem-only checkpoint, and start every experiment from that +// checkpoint. The third-party installers therefore never run beside the +// harness credentials. Each experiment mounts the auth volume after it starts +// (VmProvider.mountVolume); a fork's checkpoint reattaches it, and volume +// contents are never part of a checkpoint. The checkpoint id is saved as +// vm_provider_configs.base_image_ref with SAILBOX_IMAGE_VERSION, and a +// version change means the image is rebuilt. +// +// Install sources (checked 2026-11-20): Claude Code +// https://code.claude.com/docs/en/setup, Codex https://github.com/openai/codex, +// Antigravity https://antigravity.google/docs/getting-started?tab=cli, Pi +// https://github.com/earendil-works/pi. The installers fetch their latest +// releases; the image version pins the script, not the harness versions. + +/** Each owner's credentials live on their own volume, so two owners sharing + * one Sail account never mount each other's sign-ins. */ +export function authVolumeName(owner: string) { + return `loop-auth-${createHash("sha256").update(owner).digest("hex").slice(0, 16)}`; +} +export const AUTH_MOUNT_PATH = "/mnt/loop-auth"; +export const WORKSPACE_PATH = "/workspace"; +export const SAIL_APP_NAME = "loop-experiments"; +/** Golden checkpoints stay usable for a year; Sail's default is seven days. */ +export const GOLDEN_CHECKPOINT_TTL_SECONDS = 365 * 24 * 60 * 60; + +/** Credential files, relative to the home directory, kept on the auth volume + * under the harness's directory. Everything else a harness writes, including + * its session transcripts, stays on the VM's own disk. */ +export const HARNESS_CREDENTIALS = { + claude: [".claude/.credentials.json"], + codex: [".codex/auth.json"], + antigravity: [ + ".gemini/oauth_creds.json", + ".gemini/google_accounts.json", + ".gemini/antigravity-cli/antigravity-oauth-token", + ], + pi: [".pi/agent/auth.json"], +} as const; + +export const INSTALL_SCRIPT = `#!/bin/bash +set -euxo pipefail +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y --no-install-recommends \\ + git gh curl ca-certificates ripgrep jq sqlite3 build-essential pkg-config \\ + util-linux openssh-client unzip xz-utils python3 python3-venv python3-pip golang +# Docker ships with the base image; fail early if it does not. +command -v docker +curl -fsSL https://deb.nodesource.com/setup_22.x | bash - +apt-get install -y nodejs +npm install -g pnpm @openai/codex +npm install -g --ignore-scripts @earendil-works/pi-coding-agent +curl -fsSL https://bun.sh/install | bash +curl -LsSf https://astral.sh/uv/install.sh | sh +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal +curl -fsSL https://claude.ai/install.sh | bash +curl -fsSL https://antigravity.google/cli/install.sh | bash +cat > /etc/profile.d/loop-paths.sh <<'PATHS' +export PATH="$HOME/.local/bin:$HOME/.bun/bin:$HOME/.cargo/bin:$PATH" +PATHS +# Claude Code asks onboarding questions until this is recorded; the file is +# local to each VM and holds no credential. +[ -f "$HOME/.claude.json" ] || echo '{"hasCompletedOnboarding":true}' > "$HOME/.claude.json" +install -m 0755 /dev/stdin /usr/local/bin/loop-vm-init <<'INIT' +${initScript()} +INIT +for harness in claude codex agy pi; do + PATH="$HOME/.local/bin:$PATH" command -v "$harness" +done +`; + +/** Runs on every experiment before its first turn. It only creates links and + * the workspace repository, so running it again changes nothing. */ +export function initScript() { + const links = Object.entries(HARNESS_CREDENTIALS) + .flatMap(([harness, files]) => + files.map((file) => `link ${harness} ${file}`), + ) + .join("\n"); + return `#!/bin/bash +set -euo pipefail +auth="\${LOOP_AUTH_MOUNT:-${AUTH_MOUNT_PATH}}" +workspace="\${LOOP_WORKSPACE:-${WORKSPACE_PATH}}" +# Without the volume, credentials would land on the VM's own disk, be hidden +# by a later mount, and could be captured by a checkpoint. +if [ "\${LOOP_AUTH_SKIP_MOUNT_CHECK:-0}" != 1 ] && ! mountpoint -q "$auth"; then + echo "loop-vm-init: $auth is not mounted" >&2 + exit 1 +fi +link() { + local target="$auth/$1/$(basename "$2")" path="$HOME/$2" + mkdir -p "$auth/$1" "$(dirname "$path")" + chmod 700 "$auth/$1" + # A regular file here is newer than the volume's copy: either the harness + # signed in before the link existed, or it refreshed the credential by + # replacing the link with a file. Either way it is kept, on the volume. + if [ -f "$path" ] && [ ! -L "$path" ]; then + mv -f "$path" "$target" + fi + ln -sfn "$target" "$path" +} +${links} +mkdir -p "$workspace" +cd "$workspace" +# Checked by commit, not by .git, so a run stopped after git init finishes +# the job next time. +[ -d .git ] || git init -q -b main +if ! git rev-parse -q --verify HEAD >/dev/null; then + git -c user.name=Loop -c user.email=loop@localhost commit -q --allow-empty -m "Initial experiment workspace" +fi`; +} + +export const SAILBOX_IMAGE_VERSION = createHash("sha256") + .update(INSTALL_SCRIPT) + .digest("hex") + .slice(0, 16); diff --git a/src/server/vm/sailbox-provider.ts b/src/server/vm/sailbox-provider.ts new file mode 100644 index 00000000..dbf8e9bd --- /dev/null +++ b/src/server/vm/sailbox-provider.ts @@ -0,0 +1,629 @@ +import { randomUUID } from "node:crypto"; +import { PassThrough, Writable } from "node:stream"; +import { + VmProviderError, + type VmLimits, + type VmProcessHandle, + type VmProvider, + type VmStatus, +} from "./provider.js"; + +// Sailboxes over their HTTPS API (https://docs.sailresearch.com/sailboxes-http-api). +// Loop does not bundle @sailresearch/sdk, which ships native binaries. +// Nothing here retries: each method makes one request and reports what was +// observed, so vm/operations.ts can keep an unanswered call uncertain. + +export const SAILBOX_API = "https://sailbox-api.sailresearch.com/v1"; +export const SAIL_APPS_API = "https://api.sailresearch.com/v1"; + +/** Base image the golden experiment image is installed on: Debian with + * systemd and Docker already running (see vm/sailbox-image.ts). */ +export const SAILBOX_BASE_IMAGE = "BASE_IMAGE_DEVBOX"; + +const CONTROL_TIMEOUT_MS = 60_000; +// Create and start-from-checkpoint wait for the VM to boot. +const BOOT_TIMEOUT_MS = 10 * 60_000; +const MAX_LIST_PAGES = 20; +const MAX_EXEC_EVENT_CHARS = 16 * 1024 * 1024; + +type Fetch = typeof fetch; + +const statuses: Record = { + creating: "provisioning", + running: "running", + sleeping: "sleeping", + paused: "paused", + interrupted_restorable: "paused", + terminating: "terminating", + terminated: "terminated", + failed: "failed", + create_failed: "failed", + interrupted_unsafe_to_retry: "failed", +}; +/** Sail documents its status list as open, so a new value is `unknown`. */ +export function sailboxStatus(status: unknown): VmStatus { + return (typeof status === "string" && statuses[status]) || "unknown"; +} + +function message(body: unknown, fallback: string) { + const error = + body && typeof body === "object" && "error" in body + ? (body as { error?: { message?: unknown } }).error + : undefined; + return typeof error?.message === "string" ? error.message : fallback; +} + +// A 4xx answer other than 408, including 429 (Sail documents that the +// request never ran), means nothing happened. 501 is a definite refusal too. Any other 5xx, a +// timeout or a dropped connection may follow a change Sail already made. +function refusedStatus(status: number) { + // 408 is a timeout, which proves nothing about the mutation. + return (status >= 400 && status < 500 && status !== 408) || status === 501; +} + +export function createSailboxProvider(options: { + apiKey: string; + fetch?: Fetch; + apiBase?: string; + appsBase?: string; +}): VmProvider { + const send = options.fetch ?? fetch; + const api = options.apiBase ?? SAILBOX_API; + const apps = options.appsBase ?? SAIL_APPS_API; + + async function request( + method: string, + url: string, + init: { + body?: unknown; + raw?: Buffer; + idempotencyKey?: string; + timeoutMs?: number; + accept404?: boolean; + } = {}, + ): Promise { + const headers: Record = { + authorization: `Bearer ${options.apiKey}`, + }; + if (init.idempotencyKey) headers["idempotency-key"] = init.idempotencyKey; + let body: BodyInit | undefined; + if (init.raw !== undefined) { + headers["content-type"] = "application/octet-stream"; + body = new Uint8Array(init.raw); + } else if (init.body !== undefined) { + headers["content-type"] = "application/json"; + body = JSON.stringify(init.body); + } + let response: Response; + try { + response = await send(url, { + method, + headers, + body, + signal: AbortSignal.timeout(init.timeoutMs ?? CONTROL_TIMEOUT_MS), + // A redirect would resend the API key, a request body or a command's + // environment to another address; Sail's API does not redirect. + redirect: "error", + }); + } catch (error) { + throw new VmProviderError( + "uncertain", + `Sailboxes ${method} did not answer: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if (response.ok) return response; + if (response.status === 404 && init.accept404) return null; + const text = await response.text().catch(() => ""); + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + parsed = undefined; + } + throw new VmProviderError( + refusedStatus(response.status) ? "refused" : "uncertain", + message(parsed, `Sailboxes answered ${response.status}`), + response.status, + ); + } + + async function json( + method: string, + url: string, + init?: Parameters[2], + ): Promise> { + const response = await request(method, url, init); + if (!response) return {}; + try { + const value: unknown = await response.json(); + if (value && typeof value === "object") + return value as Record; + } catch { + // fall through: an answer Loop cannot read is not an observed outcome + } + throw new VmProviderError( + "uncertain", + `Sailboxes ${method} answered with a body Loop could not read`, + ); + } + + const box = (remoteId: string) => + `${api}/sailboxes/${encodeURIComponent(remoteId)}`; + + function field(value: Record, name: string) { + const item = value[name]; + if (typeof item !== "string" || item.length === 0) + throw new VmProviderError( + "uncertain", + `Sailboxes answered without ${name}`, + ); + return item; + } + + // A create that answers 200 with status "failed" did not leave a VM that + // can run; Sail reports the reason in error_message. + function created(value: Record) { + const remoteId = field(value, "sailbox_id"); + const status = sailboxStatus(value.status); + return typeof value.error_message === "string" && value.error_message + ? { remoteId, status, errorMessage: value.error_message } + : { remoteId, status }; + } + + return { + kind: "sailbox", + + async whoami() { + const value = await json("GET", `${api}/whoami`); + return { accountId: field(value, "org_id") }; + }, + + async ensureApp(name) { + const value = await json("POST", `${apps}/apps/find`, { + body: { name, mint_if_missing: true }, + }); + return { appId: field(value, "id") }; + }, + + async ensureVolume(name) { + // Creating a volume that already exists returns it, so each call gets a + // fresh key: a fixed one would replay a stale answer for 24 hours. + const value = await json("POST", `${api}/sailbox-volumes`, { + body: { name }, + idempotencyKey: randomUUID(), + }); + return { volumeId: field(value, "volume_id") }; + }, + + async create(input) { + return created( + await json("POST", `${api}/sailboxes`, { + idempotencyKey: input.idempotencyKey, + timeoutMs: BOOT_TIMEOUT_MS, + body: { + app_id: input.appId, + name: input.name, + image: { base: input.imageRef }, + visibility: "private", + vcpu_count: input.limits.vcpuCount, + memory_limit_gib: input.limits.memoryLimitGib, + state_disk_limit_gib: input.limits.diskLimitGib, + auto_sleep: { + automatic: true, + min_seconds_before_sleep: input.autoSleepSeconds, + }, + volume_mounts: input.volumes.map((volume) => ({ + volume_id: volume.volumeId, + mount_path: volume.mountPath, + read_only: volume.readOnly ?? false, + })), + }, + }), + ); + }, + + async get(remoteId) { + const response = await request("GET", box(remoteId), { + accept404: true, + }); + if (!response) return null; + const value: unknown = await response.json().catch(() => undefined); + if (!value || typeof value !== "object") + throw new VmProviderError( + "uncertain", + "Sailboxes GET answered with a body Loop could not read", + ); + return { status: sailboxStatus((value as { status?: unknown }).status) }; + }, + + async findByName(appId, name) { + const matches: { remoteId: string; status: VmStatus }[] = []; + for (let page = 0; page < MAX_LIST_PAGES; page += 1) { + const query = new URLSearchParams({ + app: appId, + name, + exclude_status: "terminated", + limit: "100", + offset: String(page * 100), + }); + const value = await json("GET", `${api}/sailboxes?${query}`); + // Absence is only proven by a well-formed final page. + if (!Array.isArray(value.data) || typeof value.has_more !== "boolean") + throw new VmProviderError( + "uncertain", + "Sailboxes listed Sailboxes in a shape Loop could not read", + ); + for (const item of value.data as unknown[]) { + if ( + !item || + typeof item !== "object" || + typeof (item as { name?: unknown }).name !== "string" + ) + throw new VmProviderError( + "uncertain", + "Sailboxes listed an entry Loop could not read", + ); + const entry = item as Record; + if (entry.name === name) + matches.push({ + remoteId: field(entry, "sailbox_id"), + status: sailboxStatus(entry.status), + }); + } + if (!value.has_more) return matches; + } + // An incomplete list cannot prove a VM is absent. + throw new VmProviderError( + "uncertain", + "Sailboxes listed more pages than Loop reads", + ); + }, + + exec(remoteId, command, execOptions) { + return startExec(request, box(remoteId), command, execOptions); + }, + + async sleep(remoteId, idempotencyKey) { + await request("POST", `${box(remoteId)}/sleep`, { + body: {}, + idempotencyKey, + }); + }, + + async resume(remoteId, idempotencyKey) { + const value = await json("POST", `${box(remoteId)}/resume`, { + body: {}, + idempotencyKey, + timeoutMs: BOOT_TIMEOUT_MS, + }); + if ( + value.resume_state === "running" || + value.resume_state === "already_running" + ) + return; + if (value.resume_state === "terminal_unavailable") + throw new VmProviderError( + "refused", + typeof value.error_message === "string" && value.error_message + ? value.error_message + : "The Sailbox can no longer be resumed", + ); + // Any other answer does not show the VM running. + throw new VmProviderError( + "uncertain", + "Sailboxes answered a resume without a recognised outcome", + ); + }, + + async checkpointFilesystem(input) { + const value = await json("POST", `${box(input.remoteId)}/checkpoint`, { + idempotencyKey: input.idempotencyKey, + timeoutMs: BOOT_TIMEOUT_MS, + body: { + include_memory: false, + name: input.name, + ttl_seconds: input.ttlSeconds, + }, + }); + return { checkpointId: field(value, "checkpoint_id") }; + }, + + async createFromCheckpoint(input) { + return created( + await json("POST", `${api}/sailboxes/from_checkpoint`, { + idempotencyKey: input.idempotencyKey, + timeoutMs: BOOT_TIMEOUT_MS, + body: { checkpoint_id: input.checkpointId, name: input.name }, + }), + ); + }, + + async mountVolume(remoteId, mount) { + await request("PUT", `${box(remoteId)}/volume-mounts`, { + body: { + volume_id: mount.volumeId, + mount_path: mount.mountPath, + read_only: mount.readOnly ?? false, + }, + }); + }, + + async configure( + remoteId, + settings: { limits: VmLimits; autoSleepSeconds: number }, + ) { + await request("POST", `${box(remoteId)}/set-limits`, { + body: { + vcpu_count: settings.limits.vcpuCount, + memory_limit_gib: settings.limits.memoryLimitGib, + state_disk_limit_gib: settings.limits.diskLimitGib, + }, + }); + await request("POST", `${box(remoteId)}/auto_sleep`, { + body: { + automatic: true, + min_seconds_before_sleep: settings.autoSleepSeconds, + }, + }); + }, + + async exposeHttpPort(remoteId, port, idempotencyKey) { + const value = await json("POST", `${box(remoteId)}/listeners`, { + idempotencyKey, + body: { guest_port: port, protocol: "http" }, + }); + return { url: field(value, "public_url") }; + }, + + async getHttpPort(remoteId, port) { + const response = await request( + "GET", + `${box(remoteId)}/listeners/${port}`, + { accept404: true }, + ); + if (!response) return null; + const value: unknown = await response.json().catch(() => undefined); + if (!value || typeof value !== "object") + throw new VmProviderError( + "uncertain", + "Sailboxes listener answered with a body Loop could not read", + ); + return { url: field(value as Record, "public_url") }; + }, + + async unexposePort(remoteId, port) { + await request("DELETE", `${box(remoteId)}/listeners/${port}`, { + accept404: true, + }); + }, + + async readFile(remoteId, path, maxBytes) { + const response = await request( + "GET", + `${box(remoteId)}/files?${new URLSearchParams({ path })}`, + ); + if (!response?.body) + throw new VmProviderError("refused", `No file at ${path}`); + const declared = Number(response.headers.get("content-length")); + if (Number.isFinite(declared) && declared > maxBytes) { + await response.body.cancel(); + throw new VmProviderError( + "refused", + `${path} is larger than ${maxBytes} bytes`, + ); + } + const chunks: Buffer[] = []; + let total = 0; + for await (const chunk of response.body) { + total += chunk.byteLength; + if (total > maxBytes) + throw new VmProviderError( + "refused", + `${path} is larger than ${maxBytes} bytes`, + ); + chunks.push(Buffer.from(chunk)); + } + return Buffer.concat(chunks); + }, + + async writeFile(remoteId, path, bytes) { + await request( + "PUT", + `${box(remoteId)}/files?${new URLSearchParams({ path, mode: "420" })}`, + { raw: bytes }, + ); + }, + + async terminate(remoteId, idempotencyKey) { + // Terminating a Sailbox that is already gone succeeds. + await request("POST", `${box(remoteId)}/terminate`, { + body: {}, + idempotencyKey, + accept404: true, + }); + }, + }; +} + +// Sail applies `cwd` only to string commands. Loop passes argument arrays so +// nothing is shell-parsed, so the working directory is set by a fixed script +// that receives the directory and the command as separate arguments. +export function sailboxArgv(command: string[], cwd: string | undefined) { + if (cwd === undefined) return command; + return [ + "/bin/sh", + "-c", + 'cd -- "$1" || exit 127; shift; exec "$@"', + "loop-exec", + cwd, + ...command, + ]; +} + +type Request = ( + method: string, + url: string, + init?: { + body?: unknown; + raw?: Buffer; + timeoutMs?: number; + }, +) => Promise; + +/** Starts a command and returns its streams. stdout and stderr come from the + * NDJSON event stream; stdin writes are PUTs at absolute offsets. A dropped + * event stream ends both outputs and resolves `exit` with no code. */ +async function startExec( + request: Request, + boxUrl: string, + command: string[], + options: { cwd?: string; env?: Record; openStdin?: boolean }, +): Promise { + const response = await request("POST", `${boxUrl}/exec`, { + timeoutMs: 24 * 60 * 60_000, + body: { + command: sailboxArgv(command, options.cwd), + env: options.env ?? {}, + open_stdin: options.openStdin ?? false, + close_stdin_on_disconnect: true, + }, + }); + if (!response?.body) + throw new VmProviderError("uncertain", "Sailboxes exec sent no stream"); + const stdout = new PassThrough(); + const stderr = new PassThrough(); + let execId: string | undefined; + let started!: () => void; + const ready = new Promise((resolve) => (started = resolve)); + const body = response.body; + + const exit = (async () => { + let result: { exitCode: number | null; signal: string | null } = { + exitCode: null, + signal: null, + }; + let buffered = ""; + const blocked = new Set(); + // An event Loop could not read means the output it passed on may be + // incomplete, so the exit code is not reported as if it were whole. + let corrupt = false; + const decoder = new TextDecoder(); + const handle = (line: string) => { + if (!line.trim()) return; + let event: Record; + try { + event = JSON.parse(line) as Record; + } catch { + corrupt = true; + return; + } + if (!event || typeof event !== "object") { + corrupt = true; + return; + } + if ( + event.type === "started" && + typeof event.exec_request_id === "string" + ) { + execId = event.exec_request_id; + started(); + } else if ( + (event.type === "stdout" || event.type === "stderr") && + typeof event.data === "string" + ) { + const stream = event.type === "stdout" ? stdout : stderr; + if (!stream.write(Buffer.from(event.data, "base64"))) + blocked.add(stream); + } else if (event.type === "stdout" || event.type === "stderr") + corrupt = true; + else if (event.type === "exit") + result = { + exitCode: + typeof event.return_code === "number" ? event.return_code : null, + signal: typeof event.signal === "string" ? event.signal : null, + }; + }; + try { + for await (const chunk of body) { + buffered += decoder.decode(chunk, { stream: true }); + let newline = buffered.indexOf("\n"); + while (newline >= 0) { + handle(buffered.slice(0, newline)); + buffered = buffered.slice(newline + 1); + newline = buffered.indexOf("\n"); + } + // Only the unfinished event is bounded: Sail's output events are + // small, and one this long is not one of them. + if (buffered.length > MAX_EXEC_EVENT_CHARS) { + corrupt = true; + await body.cancel().catch(() => undefined); + break; + } + // Output is read from the provider only as fast as Loop's consumer + // reads it: a full stream pauses the event stream until it drains. + // Every full stream must drain, not only the last one that filled. + for (const waiting of blocked) + if (waiting.writableNeedDrain && !waiting.destroyed) + await new Promise((resolve) => { + const done = () => { + waiting.off("drain", done); + waiting.off("close", done); + resolve(); + }; + waiting.on("drain", done); + waiting.on("close", done); + }); + blocked.clear(); + } + handle(buffered); + } catch { + // The stream dropped; the command's outcome is unknown. + } finally { + started(); + stdout.end(); + stderr.end(); + } + return corrupt ? { exitCode: null, signal: null } : result; + })(); + + let offset = 0; + const putStdin = async (bytes: Buffer, eof: boolean) => { + await ready; + if (!execId) throw new Error("The command ended before it started"); + const query = new URLSearchParams({ + offset: String(offset), + ...(eof ? { eof: "true" } : {}), + }); + await request( + "PUT", + `${boxUrl}/exec/-/stdin?exec_request_id=${encodeURIComponent(execId)}&${query}`, + { raw: bytes }, + ); + offset += bytes.byteLength; + }; + const stdin = new Writable({ + write(chunk: Buffer, _encoding, callback) { + putStdin(Buffer.from(chunk), false).then(() => callback(), callback); + }, + final(callback) { + if (!options.openStdin) return callback(); + putStdin(Buffer.alloc(0), true).then(() => callback(), callback); + }, + }); + + return { + stdin, + stdout, + stderr, + exit, + async kill() { + await ready; + if (!execId) return; + await request( + "POST", + `${boxUrl}/exec/-/cancel?exec_request_id=${encodeURIComponent(execId)}`, + { body: { force: true } }, + ); + }, + }; +} diff --git a/src/server/worker.ts b/src/server/worker.ts index 75b4cec8..32252d37 100644 --- a/src/server/worker.ts +++ b/src/server/worker.ts @@ -34,6 +34,7 @@ import { } from "./queue.js"; import { RunDeferred } from "./run-deferred.js"; import { recoverStuckRuns, sweepLeftoverToolCalls } from "./recover-stuck.js"; +import { markStaleVmOperationsUncertain } from "./vm/machine.js"; import { harnessHolds } from "./harness-holds.js"; import { nativeSessions } from "./native-sessions.js"; import { observeProcess } from "./deployments.js"; @@ -62,6 +63,7 @@ export async function tick( assertExecutionGeneration(workerId); recoverStale(); recoverStuckRuns(); + markStaleVmOperationsUncertain(db, now()); reconcileWorkerCheckouts(); const run = claim(workerId, (candidate) => { if (!accepting()) return false; diff --git a/src/shared/vm-state.ts b/src/shared/vm-state.ts new file mode 100644 index 00000000..265656da --- /dev/null +++ b/src/shared/vm-state.ts @@ -0,0 +1,73 @@ +import { z } from "zod"; + +// The single owner of the experiment_vms.state vocabulary and its edges. +// transitionVm() in src/server/vm/machine.ts is the only writer. +export const VM_TRANSITIONS = { + // sleeping/paused: a reconcile adopted a VM the provider reports asleep. + provisioning: ["running", "sleeping", "paused", "failed", "destroying"], + running: ["sleeping", "paused", "destroying", "failed"], + // sleeping <-> paused: what the provider reports when one becomes the + // other (Sail maps an interrupted, restorable VM to paused). Every live + // state reaches every other, so an observed status is never dropped. + sleeping: ["running", "paused", "destroying", "failed"], + paused: ["running", "sleeping", "destroying", "failed"], + // Only the owner's explicit reconcile, retry or destroy leaves failed. + // sleeping/paused: a reconcile or retry adopted a VM found asleep. + failed: ["provisioning", "running", "sleeping", "paused", "destroying"], + destroying: ["destroyed", "failed"], + // Terminal: the project, its threads, results and files stay readable. + destroyed: [], +} as const; +export type VmState = keyof typeof VM_TRANSITIONS; +export const VmState = z.enum( + Object.keys(VM_TRANSITIONS) as [VmState, ...VmState[]], +); + +export function legalVmTransition(from: VmState, to: VmState) { + const destinations: readonly VmState[] = VM_TRANSITIONS[from]; + return destinations.includes(to); +} + +// vm_operations.state. pending is written before the provider call. A +// pending row found at startup, or a call whose outcome was not observed, +// becomes uncertain; only the owner's reconcile settles it. +export const VM_OPERATION_TRANSITIONS = { + pending: ["succeeded", "failed", "uncertain"], + uncertain: ["succeeded", "failed"], + succeeded: [], + failed: [], +} as const; +export type VmOperationState = keyof typeof VM_OPERATION_TRANSITIONS; +export const VmOperationState = z.enum( + Object.keys(VM_OPERATION_TRANSITIONS) as [ + VmOperationState, + ...VmOperationState[], + ], +); + +export function legalVmOperationTransition( + from: VmOperationState, + to: VmOperationState, +) { + const destinations: readonly VmOperationState[] = + VM_OPERATION_TRANSITIONS[from]; + return destinations.includes(to); +} + +export const VmOperationKinds = [ + "create", + "fork", + "sleep", + "resume", + "expose", + "unexpose", + "destroy", +] as const; +export const VmOperationKind = z.enum(VmOperationKinds); +export type VmOperationKind = z.infer; + +export const VmProviderKind = z.enum(["sailbox"]); +export type VmProviderKind = z.infer; + +export const VmHarness = z.enum(["claude", "codex", "antigravity", "pi"]); +export type VmHarness = z.infer; diff --git a/test/vm-operations-worker.test.ts b/test/vm-operations-worker.test.ts new file mode 100644 index 00000000..67c0b5da --- /dev/null +++ b/test/vm-operations-worker.test.ts @@ -0,0 +1,58 @@ +import { testDirectory } from "./helpers/test-workspace.js"; +import { afterAll, expect, it } from "vitest"; + +testDirectory(); + +const { db, now } = await import("../src/server/db"); +const { tick } = await import("../src/server/worker"); +const experimentVmsQueries = await import("../src/server/sql/experiment-vms"); +const vmOperationsQueries = await import("../src/server/sql/vm-operations"); +const { VM_OPERATION_MAX_CALL_MS } = await import("../src/server/vm/machine"); + +afterAll(() => db.close()); + +// The worker tick is the production caller of the stale-receipt sweep. +it("marks a VM operation receipt left pending by a dead caller uncertain on the next worker tick", async () => { + db.prepare( + "INSERT INTO folders(id,owner_id,kind,name,created_at) VALUES('folder-w','owner-w','project','w',1)", + ).run(); + experimentVmsQueries.insert(db, { + id: "exp-w", + owner: "owner-w", + folderId: "folder-w", + provider: "sailbox", + remoteName: "loop-exp-w", + autoApproveLocal: true, + vcpuLimit: 4, + memoryLimitGib: 16, + diskLimitGib: 64, + autoSleepSeconds: 900, + sourceExperimentId: null, + sourceCheckpointId: null, + cloneUrl: null, + now: now(), + }); + for (const [id, age] of [ + ["stale", VM_OPERATION_MAX_CALL_MS + 60_000], + ["fresh", 1_000], + ] as const) + vmOperationsQueries.insertPending(db, { + id, + owner: "owner-w", + experimentId: "exp-w", + kind: "create", + requestDigest: "d", + requestJson: "{}", + now: now() - age, + }); + + await tick(); + + expect(vmOperationsQueries.byId(db, "owner-w", "stale")).toMatchObject({ + state: "uncertain", + error_message: "Loop stopped before the provider answered", + }); + expect(vmOperationsQueries.byId(db, "owner-w", "fresh")?.state).toBe( + "pending", + ); +}); diff --git a/test/vm-operations.test.ts b/test/vm-operations.test.ts new file mode 100644 index 00000000..05b01324 --- /dev/null +++ b/test/vm-operations.test.ts @@ -0,0 +1,1021 @@ +import Database from "better-sqlite3"; +import { beforeEach, expect, it } from "vitest"; +import { applyMigrations } from "../src/server/migrations.js"; +import { applyMigrationsThrough } from "./helpers/migrations.js"; +import * as experimentPreviewsQueries from "../src/server/sql/experiment-previews.js"; +import * as experimentVmsQueries from "../src/server/sql/experiment-vms.js"; +import * as vmOperationsQueries from "../src/server/sql/vm-operations.js"; +import { + VM_OPERATION_MAX_CALL_MS, + markStaleVmOperationsUncertain, + transitionVm, +} from "../src/server/vm/machine.js"; +import { + vmOperationEffects, + reconcileVmOperation, + runVmOperation, +} from "../src/server/vm/operations.js"; +import { VmProviderError, type VmStatus } from "../src/server/vm/provider.js"; +import { authVolumeName } from "../src/server/vm/sailbox-image.js"; +import { legalVmTransition } from "../src/shared/vm-state.js"; + +// Fixture coverage: an in-memory database with the production migrations and +// a scripted provider. No Sailbox is contacted. + +let db: Database.Database; +let time = 1_000; +const clock = () => ++time; + +function experiment(owner: string, id: string) { + db.prepare( + "INSERT INTO folders(id,owner_id,kind,name,created_at) VALUES(?,?,'project',?,1)", + ).run(`folder-${id}`, owner, id); + experimentVmsQueries.insert(db, { + id, + owner, + folderId: `folder-${id}`, + provider: "sailbox", + remoteName: `loop-exp-${id}`, + autoApproveLocal: true, + vcpuLimit: 4, + memoryLimitGib: 16, + diskLimitGib: 64, + autoSleepSeconds: 900, + sourceExperimentId: null, + sourceCheckpointId: null, + cloneUrl: null, + now: clock(), + }); +} + +const create = ( + id: string, + call: () => Promise, + overrides: { owner?: string; experimentId?: string; request?: unknown } = {}, + effects?: Parameters[4], +) => + runVmOperation( + db, + { + id, + owner: overrides.owner ?? "alice", + experimentId: overrides.experimentId ?? "exp1", + kind: "create", + request: overrides.request ?? { name: "loop-exp-exp1" }, + }, + call, + clock, + // Receipt-only tests opt out of the create's local writes. + effects ?? {}, + ); + +const createEffects = (experimentId: string, id: string) => + vmOperationEffects( + db, + { owner: "alice", experimentId, id, kind: "create", request: {} }, + clock, + ); + +// A read-only provider for reconcile: VMs by name, statuses by id and +// published ports. Every lookup is counted. +const found = (...matches: { remoteId: string; status: VmStatus }[]) => ({ + calls: 0, + statuses: new Map(), + ports: new Map(), + async findByName() { + this.calls += 1; + return matches; + }, + async get(remoteId: string) { + this.calls += 1; + const status = this.statuses.get(remoteId); + return status ? { status } : null; + }, + async getHttpPort(_remoteId: string, port: number) { + this.calls += 1; + const url = this.ports.get(port); + return url ? { url } : null; + }, +}); + +// Migrating once and copying the image keeps each test's database fresh. +const migrated = (() => { + const template = new Database(":memory:"); + applyMigrations(template); + const image = template.serialize(); + template.close(); + return image; +})(); + +beforeEach(() => { + db = new Database(migrated); + db.pragma("foreign_keys=ON"); + experiment("alice", "exp1"); +}); + +it("calls the provider once and replays the stored result for the same id", async () => { + let calls = 0; + const call = async () => ({ remoteId: `sb-${++calls}` }); + expect(await create("op1", call)).toEqual({ + state: "succeeded", + result: { remoteId: "sb-1" }, + replayed: false, + }); + expect(await create("op1", call)).toEqual({ + state: "succeeded", + result: { remoteId: "sb-1" }, + replayed: true, + }); + expect(calls).toBe(1); +}); + +it("refuses an id reused for a different request without calling the provider", async () => { + await create("op1", async () => ({})); + let called = false; + await expect( + create( + "op1", + async () => { + called = true; + }, + { request: { name: "another" } }, + ), + ).rejects.toMatchObject({ reason: "key_conflict" }); + expect(called).toBe(false); +}); + +it("writes the pending receipt before the provider is called", async () => { + let seen: string | undefined; + await create("op1", async () => { + seen = vmOperationsQueries.byId(db, "alice", "op1")?.state; + return {}; + }); + expect(seen).toBe("pending"); +}); + +it("records a provider refusal as failed and an unobserved outcome as uncertain", async () => { + expect( + await create("op1", async () => { + throw new VmProviderError("refused", "quota exceeded", 402); + }), + ).toMatchObject({ state: "failed", errorMessage: "quota exceeded" }); + expect( + await create("op2", async () => { + throw new VmProviderError("uncertain", "timed out"); + }), + ).toEqual({ state: "uncertain", errorMessage: "timed out" }); +}); + +it("treats an error that is not a provider answer as uncertain", async () => { + expect( + await create("op1", async () => { + throw new TypeError("adapter bug"); + }), + ).toEqual({ state: "uncertain", errorMessage: "adapter bug" }); +}); + +it("does not call the provider again for an uncertain operation or start another beside it", async () => { + await create("op1", async () => { + throw new VmProviderError("uncertain", "socket closed"); + }); + let calls = 0; + const call = async () => { + calls += 1; + return {}; + }; + expect(await create("op1", call)).toEqual({ + state: "uncertain", + errorMessage: "socket closed", + }); + await expect(create("op2", call)).rejects.toMatchObject({ + reason: "unsettled_operation", + }); + expect(calls).toBe(0); +}); + +it("keeps each owner's experiments and operation ids apart", async () => { + await create("op1", async () => ({ remoteId: "sb-1" })); + let calls = 0; + const call = async () => { + calls += 1; + return { remoteId: "sb-b" }; + }; + await expect(create("op9", call, { owner: "bob" })).rejects.toMatchObject({ + reason: "missing_experiment", + }); + expect(calls).toBe(0); + // Operation ids are scoped to the owner: bob's "op1" is his own receipt + // and reveals nothing about alice's. + experiment("bob", "exp2"); + expect( + await create("op1", call, { owner: "bob", experimentId: "exp2" }), + ).toMatchObject({ state: "succeeded", replayed: false }); + expect(calls).toBe(1); + expect(vmOperationsQueries.byId(db, "alice", "op1")).toMatchObject({ + experiment_id: "exp1", + result_json: '{"remoteId":"sb-1"}', + }); + expect( + transitionVm(db, { owner: "bob", id: "exp1" }, "running", { now: clock() }), + ).toEqual({ changed: false, reason: "missing" }); +}); + +it("marks a receipt uncertain only once no live call could still be waiting on it", () => { + vmOperationsQueries.insertPending(db, { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "create", + requestDigest: "d", + requestJson: "{}", + now: time, + }); + const created = time; + expect( + markStaleVmOperationsUncertain(db, created + VM_OPERATION_MAX_CALL_MS), + ).toBe(0); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("pending"); + expect( + markStaleVmOperationsUncertain(db, created + VM_OPERATION_MAX_CALL_MS + 1), + ).toBe(1); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("uncertain"); +}); + +it("lets a late answer settle a receipt the sweep marked uncertain", async () => { + let answer!: (value: unknown) => void; + const outcome = create( + "op1", + () => new Promise((resolve) => (answer = resolve)), + ); + await new Promise((resolve) => setImmediate(resolve)); + markStaleVmOperationsUncertain(db, time + VM_OPERATION_MAX_CALL_MS + 1); + answer({ remoteId: "sb-late" }); + expect(await outcome).toMatchObject({ state: "succeeded" }); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("succeeded"); +}); + +async function uncertainCreate() { + await create("op1", async () => { + throw new VmProviderError("uncertain", "timed out"); + }); +} +// Reconciles once the original call can no longer be in flight; see +// reconcileSoon for the window before that. +const reconcile = ( + provider: Parameters[1], + owner = "alice", + operationId = "op1", +) => { + time += VM_OPERATION_MAX_CALL_MS + 1; + return reconcileSoon(provider, owner, operationId); +}; +const reconcileSoon = ( + provider: Parameters[1], + owner = "alice", + operationId = "op1", +) => + reconcileVmOperation( + db, + provider, + { owner, operationId, appId: "app" }, + clock, + ); + +it("adopts the one VM that carries the experiment's name", async () => { + await uncertainCreate(); + expect( + await reconcile(found({ remoteId: "sb-7", status: "running" })), + ).toEqual({ + outcome: "succeeded", + result: { remoteId: "sb-7", status: "running" }, + }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-7", + state: "running", + }); + expect(vmOperationsQueries.byId(db, "alice", "op1")).toMatchObject({ + state: "succeeded", + result_json: '{"remoteId":"sb-7","status":"running"}', + }); +}); + +it("fails the operation when the provider has no such VM, allowing a new one", async () => { + await uncertainCreate(); + expect(await reconcile(found())).toMatchObject({ outcome: "failed" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe("failed"); + expect(await create("op2", async () => ({ remoteId: "sb-8" }))).toMatchObject( + { state: "succeeded" }, + ); +}); + +it("stays uncertain when the lookup fails, is ambiguous, or comes from another owner", async () => { + await uncertainCreate(); + expect( + await reconcile({ + ...found(), + async findByName() { + throw new VmProviderError("uncertain", "list timed out"); + }, + }), + ).toEqual({ outcome: "still_uncertain", reason: "list timed out" }); + expect( + await reconcile( + found( + { remoteId: "sb-1", status: "running" }, + { remoteId: "sb-2", status: "sleeping" }, + ), + ), + ).toMatchObject({ outcome: "still_uncertain" }); + const other = found({ remoteId: "sb-1", status: "running" }); + expect(await reconcile(other, "bob")).toMatchObject({ + outcome: "still_uncertain", + }); + expect(other.calls).toBe(0); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("uncertain"); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.remote_id).toBeNull(); +}); + +it("does not adopt a VM when another reconcile settled the receipt during the lookup", async () => { + await uncertainCreate(); + const racing = { + ...found(), + async findByName() { + // The other reconcile wins while this lookup is out. + await reconcile(found()); + return [{ remoteId: "sb-9", status: "running" as const }]; + }, + }; + expect(await reconcile(racing)).toEqual({ + outcome: "still_uncertain", + reason: "settled concurrently", + }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: null, + state: "failed", + }); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("failed"); +}); + +it("returns the stored outcome when a late answer contradicts a reconciled receipt", async () => { + let answer!: (value: unknown) => void; + const outcome = create( + "op1", + () => new Promise((resolve) => (answer = resolve)), + ); + await new Promise((resolve) => setImmediate(resolve)); + markStaleVmOperationsUncertain(db, time + VM_OPERATION_MAX_CALL_MS + 1); + expect(await reconcile(found())).toMatchObject({ outcome: "failed" }); + answer({ remoteId: "sb-late" }); + const late = await outcome; + expect(late).toMatchObject({ state: "failed", replayed: true }); + expect(await create("op1", async () => ({}))).toEqual(late); +}); + +it("adopts a VM found asleep, and settles its receipt and state together", async () => { + await uncertainCreate(); + expect( + await reconcile(found({ remoteId: "sb-7", status: "sleeping" })), + ).toMatchObject({ outcome: "succeeded" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-7", + state: "sleeping", + }); +}); + +it("rolls the adoption back when the VM state cannot be written", async () => { + await uncertainCreate(); + db.exec( + "CREATE TRIGGER refuse_vm_state BEFORE UPDATE OF state ON experiment_vms BEGIN SELECT RAISE(ABORT, 'disk full'); END", + ); + await expect( + reconcile(found({ remoteId: "sb-7", status: "running" })), + ).rejects.toThrow("disk full"); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("uncertain"); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.remote_id).toBeNull(); +}); + +async function runningVm(remoteId = "sb-1") { + await create("op0", async () => ({ remoteId })); + experimentVmsQueries.adoptRemote(db, "alice", "exp1", remoteId); + transitionVm(db, { owner: "alice", id: "exp1" }, "running", { now: clock() }); +} + +it("reconciles an uncertain expose from the published port, unblocking the next operation", async () => { + await runningVm(); + await runVmOperation( + db, + { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "expose", + request: { port: 3000 }, + }, + async () => { + throw new VmProviderError("uncertain", "timed out"); + }, + clock, + ); + const provider = found(); + provider.ports.set(3000, "https://p.example"); + expect(await reconcile(provider)).toEqual({ + outcome: "succeeded", + result: { url: "https://p.example" }, + }); + expect( + await runVmOperation( + db, + { + id: "op2", + owner: "alice", + experimentId: "exp1", + kind: "destroy", + request: {}, + }, + async () => null, + clock, + ), + ).toMatchObject({ state: "succeeded" }); +}); + +it("reconciles an uncertain destroy from the VM's status", async () => { + await runningVm(); + transitionVm(db, { owner: "alice", id: "exp1" }, "destroying", { + now: clock(), + }); + await runVmOperation( + db, + { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "destroy", + request: {}, + }, + async () => { + throw new VmProviderError("uncertain", "timed out"); + }, + clock, + ); + const provider = found(); + provider.statuses.set("sb-1", "terminating"); + expect(await reconcile(provider)).toMatchObject({ + outcome: "still_uncertain", + }); + provider.statuses.delete("sb-1"); + expect(await reconcile(provider)).toMatchObject({ outcome: "succeeded" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe( + "destroyed", + ); +}); + +it("reports an uncertain sleep that did not happen as failed, so it can be asked for again", async () => { + await runningVm(); + await runVmOperation( + db, + { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "sleep", + request: {}, + }, + async () => { + throw new VmProviderError("uncertain", "timed out"); + }, + clock, + ); + const provider = found(); + provider.statuses.set("sb-1", "running"); + expect(await reconcile(provider)).toMatchObject({ outcome: "failed" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe("running"); +}); + +it("records a created VM's id and state in the same transaction as its receipt", async () => { + expect( + await create( + "op1", + async () => ({ remoteId: "sb-1", status: "running" }), + {}, + createEffects("exp1", "op1"), + ), + ).toMatchObject({ state: "succeeded" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-1", + state: "running", + }); + + experiment("alice", "exp2"); + db.exec( + "CREATE TRIGGER refuse_vm_state BEFORE UPDATE OF state ON experiment_vms BEGIN SELECT RAISE(ABORT, 'disk full'); END", + ); + await expect( + create( + "op2", + async () => ({ remoteId: "sb-2", status: "running" }), + { experimentId: "exp2" }, + createEffects("exp2", "op2"), + ), + ).rejects.toThrow("disk full"); + // Nothing committed: the sweep and a reconcile take it from here. + expect(vmOperationsQueries.byId(db, "alice", "op2")?.state).toBe("pending"); + expect(experimentVmsQueries.byId(db, "alice", "exp2")?.remote_id).toBeNull(); +}); + +it("fails a create whose VM booted into failed, keeping its id so it can be destroyed", async () => { + expect( + await create( + "op1", + async () => ({ + remoteId: "sb-1", + status: "failed", + errorMessage: "image pull failed", + }), + {}, + createEffects("exp1", "op1"), + ), + ).toEqual({ + state: "failed", + errorMessage: "image pull failed", + replayed: false, + }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-1", + state: "failed", + last_error: "image pull failed", + }); + expect(vmOperationsQueries.byId(db, "alice", "op1")).toMatchObject({ + state: "failed", + result_json: expect.stringContaining("sb-1"), + }); +}); + +it("reconciles a create whose only match failed as failed, adopting the VM", async () => { + await uncertainCreate(); + expect( + await reconcile(found({ remoteId: "sb-7", status: "failed" })), + ).toMatchObject({ outcome: "failed" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-7", + state: "failed", + }); +}); + +it("does not settle on an absent effect while the original call may still be in flight", async () => { + await uncertainCreate(); + expect(await reconcileSoon(found())).toEqual({ + outcome: "still_uncertain", + reason: "the original call may still be in flight", + }); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("uncertain"); + // A VM that is already there settles at once. + expect( + await reconcileSoon(found({ remoteId: "sb-7", status: "running" })), + ).toMatchObject({ outcome: "succeeded" }); +}); + +it("fails the VM, not leaving it destroying, when an uncertain destroy did not happen", async () => { + await runningVm(); + transitionVm(db, { owner: "alice", id: "exp1" }, "destroying", { + now: clock(), + }); + await runVmOperation( + db, + { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "destroy", + request: {}, + }, + async () => { + throw new VmProviderError("uncertain", "timed out"); + }, + clock, + ); + const provider = found(); + provider.statuses.set("sb-1", "running"); + expect(await reconcile(provider)).toMatchObject({ outcome: "failed" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + state: "failed", + last_error: + "The destroy did not finish; the provider reports the VM running", + }); + expect( + transitionVm(db, { owner: "alice", id: "exp1" }, "destroying", { + now: clock(), + }).changed, + ).toBe(true); +}); + +it("refuses an experiment on another owner's folder or forked from another owner's experiment", () => { + db.prepare( + "INSERT INTO folders(id,owner_id,kind,name,created_at) VALUES('folder-bob','bob','project','b',1)", + ).run(); + const base = { + provider: "sailbox", + autoApproveLocal: true, + vcpuLimit: 4, + memoryLimitGib: 16, + diskLimitGib: 64, + autoSleepSeconds: 900, + sourceCheckpointId: null, + cloneUrl: null, + now: clock(), + }; + expect( + experimentVmsQueries.insert(db, { + ...base, + id: "x1", + owner: "alice", + folderId: "folder-bob", + remoteName: "loop-exp-x1", + sourceExperimentId: null, + }), + ).toBe(false); + expect( + experimentVmsQueries.insert(db, { + ...base, + id: "x2", + owner: "bob", + folderId: "folder-bob", + remoteName: "loop-exp-x2", + sourceExperimentId: "exp1", + }), + ).toBe(false); + expect(experimentVmsQueries.byId(db, "bob", "x2")).toBeUndefined(); +}); + +it("counts a lookup as too early when it started inside the in-flight window", async () => { + await uncertainCreate(); + const created = time; + const slow = { + ...found(), + async findByName() { + // The answer arrives after the window, but the lookup began inside it. + time = created + VM_OPERATION_MAX_CALL_MS + 10; + return []; + }, + }; + expect(await reconcileSoon(slow)).toEqual({ + outcome: "still_uncertain", + reason: "the original call may still be in flight", + }); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.state).toBe("uncertain"); +}); + +it("leaves a create uncertain while its VM is still starting", async () => { + await uncertainCreate(); + for (const status of ["provisioning", "unknown", "terminating"] as const) + expect(await reconcile(found({ remoteId: "sb-7", status }))).toMatchObject({ + outcome: "still_uncertain", + }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.remote_id).toBeNull(); +}); + +async function uncertain( + id: string, + kind: "expose" | "unexpose" | "destroy", + request: unknown, +) { + const operation = { id, owner: "alice", experimentId: "exp1", kind, request }; + await runVmOperation( + db, + operation, + async () => { + throw new VmProviderError("uncertain", "timed out"); + }, + clock, + vmOperationEffects(db, operation, clock), + ); +} + +it("keeps preview rows in step when expose, unexpose and destroy are reconciled", async () => { + await runningVm(); + await uncertain("op1", "expose", { port: 3000, label: "Demo" }); + const provider = found(); + provider.ports.set(3000, "https://p.example"); + provider.statuses.set("sb-1", "running"); + await reconcile(provider); + expect( + experimentPreviewsQueries.byPort(db, "alice", "exp1", 3000), + ).toMatchObject({ url: "https://p.example", label: "Demo", active: 1 }); + + await uncertain("op2", "unexpose", { port: 3000 }); + provider.ports.delete(3000); + expect(await reconcile(provider, "alice", "op2")).toMatchObject({ + outcome: "succeeded", + }); + expect( + experimentPreviewsQueries.byPort(db, "alice", "exp1", 3000)?.active, + ).toBe(0); + + experimentPreviewsQueries.upsert(db, { + id: "pv2", + owner: "alice", + experimentId: "exp1", + port: 4000, + label: null, + path: "/", + url: "https://q.example", + now: clock(), + }); + transitionVm(db, { owner: "alice", id: "exp1" }, "destroying", { + now: clock(), + }); + await uncertain("op3", "destroy", {}); + provider.statuses.delete("sb-1"); + expect(await reconcile(provider, "alice", "op3")).toMatchObject({ + outcome: "succeeded", + }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe( + "destroyed", + ); + expect( + experimentPreviewsQueries + .byExperiment(db, "alice", "exp1") + .filter((preview) => preview.active === 1), + ).toEqual([]); +}); + +it("fails a destroying VM when the provider refuses the destroy", async () => { + await runningVm(); + transitionVm(db, { owner: "alice", id: "exp1" }, "destroying", { + now: clock(), + }); + const operation = { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "destroy" as const, + request: {}, + }; + expect( + await runVmOperation( + db, + operation, + async () => { + throw new VmProviderError("refused", "Sailbox is locked", 409); + }, + clock, + vmOperationEffects(db, operation, clock), + ), + ).toMatchObject({ state: "failed" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + state: "failed", + last_error: "Sailbox is locked", + }); +}); + +it("refuses a preview on another owner's experiment", () => { + const preview = { + id: "pv1", + owner: "bob", + experimentId: "exp1", + port: 3000, + label: null, + path: "/", + url: "https://evil.example", + now: clock(), + }; + expect(experimentPreviewsQueries.upsert(db, preview)).toBe(false); + expect( + experimentPreviewsQueries.upsert(db, { ...preview, owner: "alice" }), + ).toBe(true); + expect(experimentPreviewsQueries.byPort(db, "alice", "exp1", 3000)?.url).toBe( + "https://evil.example", + ); + expect(experimentPreviewsQueries.byExperiment(db, "bob", "exp1")).toEqual([]); +}); + +it("does not settle an uncertain unexpose from an absent port until the window has passed", async () => { + await runningVm(); + await uncertain("op1", "unexpose", { port: 3000 }); + const provider = found(); + expect(await reconcileSoon(provider)).toEqual({ + outcome: "still_uncertain", + reason: "the original call may still be in flight", + }); + expect(await reconcile(provider)).toMatchObject({ outcome: "succeeded" }); +}); + +it("keeps a create answered while its VM is still starting uncertain, then settles it by reconcile", async () => { + expect( + await create( + "op1", + async () => ({ remoteId: "sb-1", status: "provisioning" }), + {}, + createEffects("exp1", "op1"), + ), + ).toEqual({ state: "uncertain", errorMessage: "The VM is provisioning" }); + // The id is provisional: it stays in the receipt, not on the VM. + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: null, + state: "provisioning", + }); + expect(vmOperationsQueries.byId(db, "alice", "op1")?.result_json).toContain( + "sb-1", + ); + expect( + await reconcileSoon(found({ remoteId: "sb-1", status: "running" })), + ).toMatchObject({ outcome: "succeeded" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe("running"); +}); + +it("moves a failed VM to the asleep state a later create adopts", async () => { + await uncertainCreate(); + await reconcile(found()); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe("failed"); + expect( + await create( + "op2", + async () => ({ remoteId: "sb-2", status: "sleeping" }), + {}, + createEffects("exp1", "op2"), + ), + ).toMatchObject({ state: "succeeded" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-2", + state: "sleeping", + }); +}); + +it("gives each owner a separate auth volume", () => { + expect(authVolumeName("alice")).toMatch(/^loop-auth-[0-9a-f]{16}$/); + expect(authVolumeName("alice")).toBe(authVolumeName("alice")); + expect(authVolumeName("alice")).not.toBe(authVolumeName("bob")); +}); + +it("lets a retry create a VM after a still-starting VM turned out absent", async () => { + await create( + "op1", + async () => ({ remoteId: "sb-1", status: "provisioning" }), + {}, + createEffects("exp1", "op1"), + ); + expect(await reconcile(found())).toMatchObject({ outcome: "failed" }); + expect( + await create( + "op2", + async () => ({ remoteId: "sb-2", status: "running" }), + {}, + createEffects("exp1", "op2"), + ), + ).toMatchObject({ state: "succeeded" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-2", + state: "running", + }); +}); + +it("refuses another create once the experiment has its VM, without calling the provider", async () => { + await create( + "op1", + async () => ({ remoteId: "sb-1", status: "failed", errorMessage: "boom" }), + {}, + createEffects("exp1", "op1"), + ); + let called = false; + await expect( + create("op2", async () => { + called = true; + return {}; + }), + ).rejects.toMatchObject({ reason: "vm_exists" }); + expect(called).toBe(false); +}); + +it("records every live status the provider reports, such as a sleeping VM found paused", async () => { + await runningVm(); + transitionVm(db, { owner: "alice", id: "exp1" }, "sleeping", { + now: clock(), + }); + const operation = { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "resume" as const, + request: {}, + }; + await runVmOperation( + db, + operation, + async () => { + throw new VmProviderError("uncertain", "timed out"); + }, + clock, + vmOperationEffects(db, operation, clock), + ); + const provider = found(); + provider.statuses.set("sb-1", "paused"); + expect(await reconcile(provider)).toMatchObject({ outcome: "failed" }); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe("paused"); + const live = ["running", "sleeping", "paused"] as const; + for (const from of live) + for (const to of live) + if (from !== to) expect(legalVmTransition(from, to)).toBe(true); +}); + +it("records a destroy sent while the VM was running as destroyed, not failed", async () => { + await runningVm(); + const operation = { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "destroy" as const, + request: {}, + }; + await runVmOperation( + db, + operation, + async () => null, + clock, + vmOperationEffects(db, operation, clock), + ); + expect(experimentVmsQueries.byId(db, "alice", "exp1")?.state).toBe( + "destroyed", + ); +}); + +it("refuses an operation on an experiment with no recorded VM", async () => { + await expect( + runVmOperation( + db, + { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "sleep", + request: {}, + }, + async () => null, + clock, + ), + ).rejects.toMatchObject({ reason: "no_vm" }); +}); + +it("records a create's VM by default, without the caller passing effects", async () => { + await runVmOperation( + db, + { + id: "op1", + owner: "alice", + experimentId: "exp1", + kind: "create", + request: {}, + }, + async () => ({ remoteId: "sb-1", status: "running" }), + clock, + ); + expect(experimentVmsQueries.byId(db, "alice", "exp1")).toMatchObject({ + remote_id: "sb-1", + state: "running", + }); +}); + +it("refuses illegal and stale VM transitions", () => { + const vm = { owner: "alice", id: "exp1" }; + expect(transitionVm(db, vm, "destroyed", { now: clock() })).toEqual({ + changed: false, + reason: "illegal", + from: "provisioning", + }); + expect(transitionVm(db, vm, "running", { now: clock() }).changed).toBe(true); + expect( + transitionVm(db, vm, "destroying", { + expected: "sleeping", + now: clock(), + }), + ).toEqual({ changed: false, reason: "illegal", from: "running" }); + transitionVm(db, vm, "destroying", { now: clock() }); + transitionVm(db, vm, "destroyed", { now: clock() }); + const row = experimentVmsQueries.byId(db, "alice", "exp1"); + expect(row?.state).toBe("destroyed"); + expect(row?.destroyed_at).toBeGreaterThan(0); + expect(transitionVm(db, vm, "running", { now: clock() }).changed).toBe(false); +}); + +it("adds the experiment tables to a database a released version wrote, leaving its rows alone", () => { + const upgrade = new Database(":memory:"); + upgrade.pragma("foreign_keys=ON"); + applyMigrationsThrough(upgrade, 166); + upgrade + .prepare( + "INSERT INTO folders(id,owner_id,kind,name,created_at) VALUES('f1','alice','project','Notes',1)", + ) + .run(); + applyMigrations(upgrade); + expect(upgrade.prepare("SELECT id,name FROM folders").all()).toEqual([ + { id: "f1", name: "Notes" }, + ]); + for (const table of [ + "vm_provider_configs", + "experiment_vms", + "experiment_previews", + "vm_operations", + ]) + expect(upgrade.prepare(`SELECT count(*) AS n FROM ${table}`).get()).toEqual( + { n: 0 }, + ); + upgrade.close(); +}); diff --git a/test/vm-sailbox-image.test.ts b/test/vm-sailbox-image.test.ts new file mode 100644 index 00000000..2f9c030c --- /dev/null +++ b/test/vm-sailbox-image.test.ts @@ -0,0 +1,146 @@ +import { execFileSync } from "node:child_process"; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readlinkSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { + HARNESS_CREDENTIALS, + INSTALL_SCRIPT, + initScript, +} from "../src/server/vm/sailbox-image.js"; + +// Runs the experiment init script on this machine against temporary +// directories. The install script itself is only syntax-checked here; it has +// not been run on a Sailbox by this test. + +function run(home: string, auth: string, workspace: string) { + const script = join(home, "loop-vm-init"); + writeFileSync(script, initScript(), { mode: 0o755 }); + execFileSync("bash", [script], { + env: { + PATH: process.env.PATH, + HOME: home, + LOOP_AUTH_MOUNT: auth, + LOOP_WORKSPACE: workspace, + // The fixture's auth directory is not a mount point. + LOOP_AUTH_SKIP_MOUNT_CHECK: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + }, + }); +} + +it("links only credential files to the auth volume and initialises the workspace", () => { + const root = mkdtempSync(join(tmpdir(), "loop-vm-init-")); + const [home, auth, workspace] = ["home", "auth", "workspace"].map((name) => + join(root, name), + ); + mkdirSync(home); + mkdirSync(auth); + mkdirSync(join(home, ".codex")); + writeFileSync(join(home, ".codex/auth.json"), "earlier"); + mkdirSync(join(home, ".claude/projects"), { recursive: true }); + + run(home, auth, workspace); + run(home, auth, workspace); + + for (const [harness, files] of Object.entries(HARNESS_CREDENTIALS)) + for (const file of files) { + const path = join(home, file); + expect(lstatSync(path).isSymbolicLink()).toBe(true); + expect(readlinkSync(path).startsWith(join(auth, harness))).toBe(true); + } + // A credential written before the volume was linked moves onto it. + expect(readFileSync(join(auth, "codex/auth.json"), "utf8")).toBe("earlier"); + // Session directories stay on the VM's own disk. + expect(lstatSync(join(home, ".claude/projects")).isSymbolicLink()).toBe( + false, + ); + expect(existsSync(join(auth, "claude/projects"))).toBe(false); + const log = execFileSync("git", ["log", "--format=%s"], { + cwd: workspace, + encoding: "utf8", + }); + expect(log.trim()).toBe("Initial experiment workspace"); +}); + +it("produces an install script bash accepts", () => { + const dir = mkdtempSync(join(tmpdir(), "loop-vm-install-")); + writeFileSync(join(dir, "install.sh"), INSTALL_SCRIPT); + execFileSync("bash", ["-n", join(dir, "install.sh")]); +}); + +it("keeps a credential the harness refreshed by replacing its link", () => { + const root = mkdtempSync(join(tmpdir(), "loop-vm-refresh-")); + const [home, auth, workspace] = ["home", "auth", "workspace"].map((name) => + join(root, name), + ); + mkdirSync(home); + mkdirSync(auth); + run(home, auth, workspace); + writeFileSync(join(auth, "codex/auth.json"), "old token"); + // The harness writes a new file in place of the link, as an atomic + // rename-based refresh does. + const path = join(home, ".codex/auth.json"); + rmSync(path); + writeFileSync(path, "refreshed token"); + + run(home, auth, workspace); + + expect(lstatSync(path).isSymbolicLink()).toBe(true); + expect(readFileSync(path, "utf8")).toBe("refreshed token"); + expect(readFileSync(join(auth, "codex/auth.json"), "utf8")).toBe( + "refreshed token", + ); +}); + +it("refuses to place credentials when the auth volume is not mounted", () => { + const root = mkdtempSync(join(tmpdir(), "loop-vm-unmounted-")); + const [home, auth, workspace] = ["home", "auth", "workspace"].map((name) => + join(root, name), + ); + mkdirSync(join(home, ".codex"), { recursive: true }); + mkdirSync(auth); + writeFileSync(join(home, ".codex/auth.json"), "token"); + const script = join(root, "loop-vm-init"); + writeFileSync(script, initScript(), { mode: 0o755 }); + expect(() => + execFileSync("bash", [script], { + stdio: "pipe", + env: { + PATH: process.env.PATH, + HOME: home, + LOOP_AUTH_MOUNT: auth, + LOOP_WORKSPACE: workspace, + }, + }), + ).toThrow(/is not mounted/); + expect(readFileSync(join(home, ".codex/auth.json"), "utf8")).toBe("token"); + expect(existsSync(join(auth, "codex"))).toBe(false); +}); + +it("finishes the initial commit when an earlier run stopped after git init", () => { + const root = mkdtempSync(join(tmpdir(), "loop-vm-unborn-")); + const [home, auth, workspace] = ["home", "auth", "workspace"].map((name) => + join(root, name), + ); + mkdirSync(home); + mkdirSync(auth); + mkdirSync(workspace); + execFileSync("git", ["init", "-q", "-b", "main"], { cwd: workspace }); + run(home, auth, workspace); + expect( + execFileSync("git", ["log", "--format=%s"], { + cwd: workspace, + encoding: "utf8", + }).trim(), + ).toBe("Initial experiment workspace"); +}); diff --git a/test/vm-sailbox-provider.test.ts b/test/vm-sailbox-provider.test.ts new file mode 100644 index 00000000..fa430c0f --- /dev/null +++ b/test/vm-sailbox-provider.test.ts @@ -0,0 +1,547 @@ +import { expect, it } from "vitest"; +import { VmProviderError } from "../src/server/vm/provider.js"; +import { + createSailboxProvider, + sailboxArgv, + sailboxStatus, +} from "../src/server/vm/sailbox-provider.js"; + +// Fixture coverage: a scripted fetch stands in for the Sailboxes API, with +// request and response shapes taken from Sail's published OpenAPI document. +// No Sailbox is contacted. + +type Call = { + method: string; + url: URL; + headers: Headers; + body: unknown; + redirect?: RequestRedirect; +}; + +function fake(answer: (call: Call) => Response | Promise | Error) { + const calls: Call[] = []; + const fetch = (async (input: string | URL, init?: RequestInit) => { + const raw = init?.body; + const call: Call = { + method: init?.method ?? "GET", + url: new URL(String(input)), + headers: new Headers(init?.headers), + redirect: init?.redirect, + body: + typeof raw === "string" + ? JSON.parse(raw) + : raw instanceof Uint8Array + ? Buffer.from(raw).toString() + : undefined, + }; + calls.push(call); + const result = await answer(call); + if (result instanceof Error) throw result; + return result; + }) as typeof globalThis.fetch; + return { + calls, + provider: createSailboxProvider({ apiKey: "sail-secret", fetch }), + }; +} + +const ok = (body: unknown) => Response.json(body); +const failure = (status: number, text = "nope") => + Response.json( + { error: { message: text, type: "x", param: null, code: null } }, + { status }, + ); + +it("creates a private Sailbox with the operation's idempotency key", async () => { + const { calls, provider } = fake(() => + ok({ sailbox_id: "sb_1", status: "running" }), + ); + expect( + await provider.create({ + idempotencyKey: "op-1", + appId: "app_1", + name: "loop-exp-abc", + imageRef: "BASE_IMAGE_DEVBOX", + limits: { vcpuCount: 4, memoryLimitGib: 16, diskLimitGib: 64 }, + autoSleepSeconds: 900, + volumes: [{ volumeId: "vol_1", mountPath: "/mnt/loop-auth" }], + }), + ).toEqual({ remoteId: "sb_1", status: "running" }); + const [call] = calls; + expect(call.method).toBe("POST"); + expect(call.url.href).toBe( + "https://sailbox-api.sailresearch.com/v1/sailboxes", + ); + expect(call.headers.get("authorization")).toBe("Bearer sail-secret"); + expect(call.headers.get("idempotency-key")).toBe("op-1"); + expect(call.body).toEqual({ + app_id: "app_1", + name: "loop-exp-abc", + image: { base: "BASE_IMAGE_DEVBOX" }, + visibility: "private", + vcpu_count: 4, + memory_limit_gib: 16, + state_disk_limit_gib: 64, + auto_sleep: { automatic: true, min_seconds_before_sleep: 900 }, + volume_mounts: [ + { volume_id: "vol_1", mount_path: "/mnt/loop-auth", read_only: false }, + ], + }); + expect(JSON.stringify(call.body)).not.toContain("sail-secret"); +}); + +it("reports a 4xx answer as refused and an unanswered or 5xx call as uncertain", async () => { + const cases: [Response | Error, "refused" | "uncertain"][] = [ + [failure(402, "billing"), "refused"], + [failure(409), "refused"], + [failure(429), "refused"], + [failure(501), "refused"], + [failure(500), "uncertain"], + [failure(503), "uncertain"], + [new TypeError("fetch failed"), "uncertain"], + [new Response("not json"), "uncertain"], + ]; + for (const [answer, outcome] of cases) { + const { provider } = fake(() => answer); + const error = await provider + .ensureApp("loop-experiments") + .catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(VmProviderError); + expect((error as VmProviderError).outcome).toBe(outcome); + } + const { provider } = fake(() => failure(402, "Add a payment method")); + await expect(provider.whoami()).rejects.toThrow("Add a payment method"); +}); + +it("finds live Sailboxes by exact name across pages, and refuses to guess past its page limit", async () => { + const { calls, provider } = fake(({ url }) => + url.searchParams.get("offset") === "0" + ? ok({ + data: [ + { sailbox_id: "sb_1", name: "loop-exp-abc", status: "sleeping" }, + { sailbox_id: "sb_x", name: "loop-exp-abcd", status: "running" }, + ], + has_more: true, + }) + : ok({ + data: [ + { sailbox_id: "sb_2", name: "loop-exp-abc", status: "creating" }, + ], + has_more: false, + }), + ); + expect(await provider.findByName("app_1", "loop-exp-abc")).toEqual([ + { remoteId: "sb_1", status: "sleeping" }, + { remoteId: "sb_2", status: "provisioning" }, + ]); + expect(Object.fromEntries(calls[0].url.searchParams)).toMatchObject({ + app: "app_1", + name: "loop-exp-abc", + exclude_status: "terminated", + }); + const endless = fake(() => ok({ data: [], has_more: true })); + await expect( + endless.provider.findByName("app_1", "loop-exp-abc"), + ).rejects.toMatchObject({ outcome: "uncertain" }); +}); + +it("treats a missing Sailbox as already terminated and as absent on get", async () => { + const { provider } = fake(() => failure(404)); + await expect(provider.terminate("sb_1", "op-2")).resolves.toBeUndefined(); + expect(await provider.get("sb_1")).toBeNull(); +}); + +it("maps Sail's open status set, keeping unrecognised values unknown", () => { + expect(sailboxStatus("create_failed")).toBe("failed"); + expect(sailboxStatus("interrupted_restorable")).toBe("paused"); + expect(sailboxStatus("terminating")).toBe("terminating"); + expect(sailboxStatus("hibernating")).toBe("unknown"); + expect(sailboxStatus(undefined)).toBe("unknown"); +}); + +it("refuses a resume Sail reports as no longer possible", async () => { + const { provider } = fake(() => + ok({ + sailbox_id: "sb_1", + status: "failed", + resume_state: "terminal_unavailable", + }), + ); + await expect(provider.resume("sb_1", "op-3")).rejects.toMatchObject({ + outcome: "refused", + }); +}); + +it("stops reading a file once it passes the limit", async () => { + const { provider } = fake( + () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(6)); + controller.enqueue(new Uint8Array(6)); + controller.close(); + }, + }), + ), + ); + await expect(provider.readFile("sb_1", "/workspace/a", 10)).rejects.toThrow( + "larger than 10 bytes", + ); +}); + +it("sets the working directory without shell-parsing the command", () => { + expect(sailboxArgv(["claude", "-p", "a; rm -rf /"], "/workspace")).toEqual([ + "/bin/sh", + "-c", + 'cd -- "$1" || exit 127; shift; exec "$@"', + "loop-exec", + "/workspace", + "claude", + "-p", + "a; rm -rf /", + ]); + expect(sailboxArgv(["ls"], undefined)).toEqual(["ls"]); +}); + +it("streams exec output, writes stdin at increasing offsets and reports the exit code", async () => { + const encoder = new TextEncoder(); + let release!: () => void; + const stdinDone = new Promise((resolve) => (release = resolve)); + const { calls, provider } = fake(async (call) => { + if (call.url.pathname.endsWith("/exec")) + return new Response( + new ReadableStream({ + async start(controller) { + const send = (event: unknown) => + controller.enqueue(encoder.encode(JSON.stringify(event) + "\n")); + send({ type: "started", exec_request_id: "exec/1" }); + send({ + type: "stdout", + data: Buffer.from("hello ").toString("base64"), + seq: 1, + }); + // An event split across chunks is still read whole. + const line = JSON.stringify({ + type: "stderr", + data: Buffer.from("warn").toString("base64"), + seq: 1, + }); + controller.enqueue(encoder.encode(line.slice(0, 10))); + controller.enqueue(encoder.encode(line.slice(10) + "\n")); + await stdinDone; + send({ type: "heartbeat" }); + send({ type: "exit", status: "failed", return_code: 3 }); + controller.close(); + }, + }), + ); + if (call.url.pathname.endsWith("/stdin")) { + if (call.url.searchParams.get("eof") === "true") release(); + return ok({ accepted_through: 0 }); + } + return ok({}); + }); + const handle = await provider.exec("sb_1", ["cat"], { + cwd: "/workspace", + env: { LOOP_TOOLS_URL: "https://brain.example/api/internal/tool-bridge/t" }, + openStdin: true, + }); + const read = async (stream: NodeJS.ReadableStream) => { + let text = ""; + for await (const chunk of stream) text += String(chunk); + return text; + }; + const output = Promise.all([read(handle.stdout), read(handle.stderr)]); + await new Promise((resolve, reject) => + handle.stdin.write("ab", (error) => (error ? reject(error) : resolve())), + ); + await new Promise((resolve) => handle.stdin.end("cde", resolve)); + expect(await handle.exit).toEqual({ exitCode: 3, signal: null }); + expect(await output).toEqual(["hello ", "warn"]); + + const start = calls[0]; + expect(start.body).toMatchObject({ + command: sailboxArgv(["cat"], "/workspace"), + open_stdin: true, + env: { + LOOP_TOOLS_URL: "https://brain.example/api/internal/tool-bridge/t", + }, + }); + const writes = calls + .filter((call) => call.url.pathname.endsWith("/stdin")) + .map((call) => ({ + id: call.url.searchParams.get("exec_request_id"), + offset: call.url.searchParams.get("offset"), + eof: call.url.searchParams.get("eof"), + body: call.body, + })); + expect(writes).toEqual([ + { id: "exec/1", offset: "0", eof: null, body: "ab" }, + { id: "exec/1", offset: "2", eof: null, body: "cde" }, + { id: "exec/1", offset: "5", eof: "true", body: "" }, + ]); +}); + +it("resolves exit with no code when the event stream drops", async () => { + const { provider } = fake( + () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode( + JSON.stringify({ type: "started", exec_request_id: "e" }) + + "\n", + ), + ); + controller.error(new Error("connection reset")); + }, + }), + ), + ); + const handle = await provider.exec("sb_1", ["sleep", "100"], {}); + expect(await handle.exit).toEqual({ exitCode: null, signal: null }); +}); + +it("treats a listing it cannot read as uncertain, not as proof of absence", async () => { + for (const page of [ + {}, + { data: [] }, + { data: "none", has_more: false }, + { data: [null], has_more: false }, + { data: [{ sailbox_id: "sb_1", status: "running" }], has_more: false }, + ]) { + const { provider } = fake(() => ok(page)); + await expect( + provider.findByName("app_1", "loop-exp-abc"), + ).rejects.toMatchObject({ outcome: "uncertain" }); + } +}); + +it("stops reading exec events while nobody reads the output", async () => { + let pulls = 0; + const encoder = new TextEncoder(); + const chunk = Buffer.alloc(64 * 1024, 1).toString("base64"); + const { provider } = fake( + () => + new Response( + new ReadableStream({ + pull(controller) { + pulls += 1; + controller.enqueue( + encoder.encode( + (pulls === 1 + ? JSON.stringify({ type: "started", exec_request_id: "e" }) + + "\n" + : "") + + JSON.stringify({ type: "stdout", data: chunk, seq: pulls }) + + "\n", + ), + ); + if (pulls === 500) controller.close(); + }, + }), + ), + ); + const handle = await provider.exec("sb_1", ["yes"], {}); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(pulls).toBeLessThan(10); + handle.stdout.resume(); + handle.stderr.resume(); + await handle.exit; + expect(pulls).toBe(500); +}); + +it("waits for every full output stream before reading more events", async () => { + let pulls = 0; + const encoder = new TextEncoder(); + const chunk = Buffer.alloc(64 * 1024, 1).toString("base64"); + const { provider } = fake( + () => + new Response( + new ReadableStream({ + pull(controller) { + pulls += 1; + // stdout and stderr fill within one chunk. + controller.enqueue( + encoder.encode( + (pulls === 1 + ? JSON.stringify({ type: "started", exec_request_id: "e" }) + + "\n" + : "") + + JSON.stringify({ type: "stdout", data: chunk, seq: pulls }) + + "\n" + + JSON.stringify({ type: "stderr", data: chunk, seq: pulls }) + + "\n", + ), + ); + if (pulls === 200) controller.close(); + }, + }), + ), + ); + const handle = await provider.exec("sb_1", ["yes"], {}); + // Only stderr is read; stdout stays full. + handle.stderr.resume(); + await new Promise((resolve) => setTimeout(resolve, 200)); + expect(pulls).toBeLessThan(10); + handle.stdout.resume(); + await handle.exit; + expect(pulls).toBe(200); +}); + +it("reports a VM that booted into failed with Sail's reason, and mounts volumes on a running VM", async () => { + const { calls, provider } = fake((call) => + call.url.pathname.endsWith("/from_checkpoint") + ? ok({ + sailbox_id: "sb_2", + status: "create_failed", + error_message: "no capacity", + }) + : ok({}), + ); + expect( + await provider.createFromCheckpoint({ + idempotencyKey: "exp-1", + checkpointId: "ckpt_1", + name: "loop-exp-1", + }), + ).toEqual({ + remoteId: "sb_2", + status: "failed", + errorMessage: "no capacity", + }); + await provider.mountVolume("sb_2", { + volumeId: "vol_1", + mountPath: "/mnt/loop-auth", + }); + const mount = calls[1]; + expect(mount.method).toBe("PUT"); + expect(mount.url.pathname).toBe("/v1/sailboxes/sb_2/volume-mounts"); + expect(mount.body).toEqual({ + volume_id: "vol_1", + mount_path: "/mnt/loop-auth", + read_only: false, + }); +}); + +it("never follows a redirect with the API key", async () => { + const { calls, provider } = fake(() => ok({ org_id: "org_1" })); + await provider.whoami(); + expect(calls[0].redirect).toBe("error"); +}); + +it("reports no exit code when an exec event could not be read", async () => { + const encoder = new TextEncoder(); + const { provider } = fake( + () => + new Response( + new ReadableStream({ + start(controller) { + for (const line of [ + JSON.stringify({ type: "started", exec_request_id: "e" }), + "{not json", + JSON.stringify({ type: "stdout", data: 7, seq: 2 }), + JSON.stringify({ type: "progress", note: "new event kind" }), + JSON.stringify({ + type: "exit", + status: "succeeded", + return_code: 0, + }), + ]) + controller.enqueue(encoder.encode(line + "\n")); + controller.close(); + }, + }), + ), + ); + const handle = await provider.exec("sb_1", ["true"], {}); + handle.stdout.resume(); + handle.stderr.resume(); + expect(await handle.exit).toEqual({ exitCode: null, signal: null }); + + // An event type Loop does not know is not corruption. + const clean = fake( + () => + new Response( + [ + JSON.stringify({ type: "started", exec_request_id: "e" }), + JSON.stringify({ type: "progress" }), + JSON.stringify({ type: "exit", status: "succeeded", return_code: 0 }), + ].join("\n") + "\n", + ), + ); + const ok2 = await clean.provider.exec("sb_1", ["true"], {}); + ok2.stdout.resume(); + ok2.stderr.resume(); + expect(await ok2.exit).toEqual({ exitCode: 0, signal: null }); +}); + +it("treats a resume answer without a recognised outcome as uncertain", async () => { + for (const answer of [{}, { resume_state: "warming" }]) { + const { provider } = fake(() => ok(answer)); + await expect(provider.resume("sb_1", "op-4")).rejects.toMatchObject({ + outcome: "uncertain", + }); + } + for (const resume_state of ["running", "already_running"]) { + const { provider } = fake(() => ok({ resume_state })); + await expect(provider.resume("sb_1", "op-5")).resolves.toBeUndefined(); + } +}); + +it("stops reading an exec stream whose event never ends", async () => { + const encoder = new TextEncoder(); + let pulls = 0; + const { provider } = fake( + () => + new Response( + new ReadableStream({ + pull(controller) { + pulls += 1; + controller.enqueue( + encoder.encode( + pulls === 1 + ? JSON.stringify({ type: "started", exec_request_id: "e" }) + + '\n{"type":"stdout","data":"' + : "A".repeat(1024 * 1024), + ), + ); + }, + }), + ), + ); + const handle = await provider.exec("sb_1", ["yes"], {}); + handle.stdout.resume(); + handle.stderr.resume(); + expect(await handle.exit).toEqual({ exitCode: null, signal: null }); + expect(pulls).toBeLessThan(40); +}); + +it("keeps every event of a large chunk made of small events", async () => { + const line = (event: unknown) => JSON.stringify(event) + "\n"; + const data = Buffer.alloc(1024, 65).toString("base64"); + let text = line({ type: "started", exec_request_id: "e" }); + // ~20 MB in one chunk, every event well under the per-event limit. + for (let seq = 1; seq <= 15_000; seq += 1) + text += line({ type: "stdout", data, seq }); + text += line({ type: "exit", status: "succeeded", return_code: 0 }); + const { provider } = fake( + () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(text)); + controller.close(); + }, + }), + ), + ); + const handle = await provider.exec("sb_1", ["cat", "big"], {}); + let bytes = 0; + handle.stdout.on("data", (chunk: Buffer) => (bytes += chunk.length)); + handle.stderr.resume(); + expect(await handle.exit).toEqual({ exitCode: 0, signal: null }); + expect(bytes).toBe(15_000 * 1024); +}); diff --git a/test/vm-state-documentation.test.ts b/test/vm-state-documentation.test.ts new file mode 100644 index 00000000..1e6029e2 --- /dev/null +++ b/test/vm-state-documentation.test.ts @@ -0,0 +1,46 @@ +import { readFileSync } from "node:fs"; +import { expect, it } from "vitest"; +import { + VM_OPERATION_TRANSITIONS, + VM_TRANSITIONS, +} from "../src/shared/vm-state"; + +const doc = readFileSync( + new URL("../docs/state-machines.md", import.meta.url), + "utf8", +); + +function documented(heading: string) { + // The pinned formatter pads table cells, so rows are found by their + // leading pipe: a header, a rule, then one row per source state. + const section = doc.split(heading)[1].split("\n##")[0]; + const rows = section + .split("\n") + .filter((line) => line.startsWith("|")) + .slice(2); + return Object.fromEntries( + rows.map((line) => { + const [, from, to] = line + .split("|") + .map((part) => part.trim().replaceAll(/[`*]/g, "")); + return [from, to === "—" ? [] : to.split(" / ").sort()]; + }), + ); +} + +const sorted = (edges: Record) => + Object.fromEntries( + Object.entries(edges).map(([from, to]) => [from, [...to].sort()]), + ); + +it("documents every experiment VM transition", () => { + expect(documented("### `experiment_vms.state`")).toEqual( + sorted(VM_TRANSITIONS), + ); +}); + +it("documents every VM operation transition", () => { + expect(documented("### `vm_operations.state`")).toEqual( + sorted(VM_OPERATION_TRANSITIONS), + ); +});