diff --git a/apps/api/esbuild.config.mjs b/apps/api/esbuild.config.mjs index 4d9c8c06b..1003499a4 100644 --- a/apps/api/esbuild.config.mjs +++ b/apps/api/esbuild.config.mjs @@ -59,7 +59,9 @@ const noRelativeIntegrationsImports = { }; await build({ - entryPoints: ["src/server.ts"], + // migrate.ts is the one-shot migration step `openmapx services update` runs + // before it replaces app-api. + entryPoints: ["src/server.ts", "src/migrate.ts"], bundle: true, platform: "node", target: "node24", @@ -68,6 +70,6 @@ await build({ define: { __OPENMAPX_PRIVACY_SOURCE_FINGERPRINT__: JSON.stringify(privacySourceFingerprint), }, - outfile: "dist/server.js", + outdir: "dist", plugins: [noRelativeIntegrationsImports], }); diff --git a/apps/api/src/migrate.ts b/apps/api/src/migrate.ts new file mode 100644 index 000000000..99839a4b9 --- /dev/null +++ b/apps/api/src/migrate.ts @@ -0,0 +1,66 @@ +/** + * Applies the pending database migrations and exits. + * + * `openmapx services update` runs this in a one-off app-api container before + * it replaces app-api, so a long migration is its own step with its own output + * rather than part of app-api's startup and healthcheck. app-api still applies + * any pending migrations when it starts; after this step there are none. + */ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { envString } from "@openmapx/core/server-env"; +import { openMigrationSession, sql } from "./db/index"; +import { type MigrationRun, startMigrations } from "./db/migration-runner"; +import { applyRequiredMigrations } from "./startup-migrations"; +import { createSafePinoOptions } from "./utils/safe-log-fields"; + +const { default: pino } = await import("pino"); +const log = pino(createSafePinoOptions(envString("LOG_LEVEL", "info"))); + +// The one-off container runs this as PID 1, which ignores SIGTERM without a +// handler. Stopping the container must stop the migration in Postgres too. +let run: MigrationRun | undefined; +let stopping = false; +const stop = (signal: NodeJS.Signals) => { + if (stopping) return; + stopping = true; + log.warn({ signal }, "Stopping the database migration"); + void (run?.cancel() ?? Promise.resolve()) + .catch((err) => log.warn(err, "Could not stop the running database migration")) + .finally(() => process.exit(signal === "SIGTERM" ? 143 : 130)); +}; +process.on("SIGTERM", stop); +process.on("SIGINT", stop); + +const migrationsDir = join(import.meta.dirname ?? ".", "db", "migrations"); +const startedAt = Date.now(); +try { + let applied: string[] = []; + await applyRequiredMigrations({ + migrationsDirectory: migrationsDir, + directoryExists: existsSync, + migrate: async () => { + run = startMigrations({ + session: openMigrationSession(), + control: sql, + migrationsFolder: migrationsDir, + log, + }); + applied = await run.done; + }, + }); + log.info( + { migrations: applied, durationMs: Date.now() - startedAt }, + "Database migrations applied", + ); + await sql.end({ timeout: 5 }); + process.exit(0); +} catch (err) { + if (stopping) { + log.warn("Database migration stopped before it finished; it rolled back"); + // The stop handler exits once the migration has rolled back. + await new Promise(() => {}); + } + log.fatal(err, "Database migration failed"); + process.exit(1); +} diff --git a/docs/docs/developer/cli-reference.md b/docs/docs/developer/cli-reference.md index 2862568fe..97b39c7c3 100644 --- a/docs/docs/developer/cli-reference.md +++ b/docs/docs/developer/cli-reference.md @@ -62,22 +62,22 @@ Discover, inspect, select, build, and run the backend services that make up a deployment. The selection, preset, build, and lifecycle semantics are documented in full under [Managing services](../install/managing-services.md). -| Command | Description | -| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `services list` | List discovered services as a table. Filters: `--capability `, `--quality `, `--enabled`. | -| `services get ` | Print one service's full manifest as JSON. | -| `services selected` | Show the requested roots and the effective set after dependency expansion. | -| `services enable ` | Persistently add root services to `infra/docker/state/ops-agent/service-selection.json`. | -| `services disable ` | Persistently remove root services from the selection file. | -| `services build ` | Build prepared artifacts for the named services, then re-render and re-link. Flags: `--region `, `--continue-on-error`, `--import` (MOTIS only: also run `motis import`). | -| `services build-all` | Build every service that declares a build, in order. Flags: `--region `, `--fail-fast`. | -| `services start [ids...]` | Render, apply hardlinks, then `docker compose up -d` the named services. Flag: `--preset `. | -| `services stop [ids...]` | `docker compose stop` the named services. Flag: `--preset `. | -| `services restart [ids...]` | In-place reboot (`docker compose restart`); does **not** re-render. Flag: `--preset `. | -| `services update [ids...]` | Re-render, link, pull the configured images, then replace containers with `up -d --force-recreate`. Flag: `--preset `. For core OpenMapX app releases, use the aggregate release-manifest procedure in [Upgrading](../install/upgrading.md). | -| `services status [id]` | Container status table for one or all services (`docker compose ps`). | -| `services logs ` | Stream service logs. Flags: `--tail ` (default `100`), `--follow`. | -| `services capabilities` | Inventory the capability and data-type vocabulary across the registry. Flag: `--unrecognised`. | +| Command | Description | +| --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `services list` | List discovered services as a table. Filters: `--capability `, `--quality `, `--enabled`. | +| `services get ` | Print one service's full manifest as JSON. | +| `services selected` | Show the requested roots and the effective set after dependency expansion. | +| `services enable ` | Persistently add root services to `infra/docker/state/ops-agent/service-selection.json`. | +| `services disable ` | Persistently remove root services from the selection file. | +| `services build ` | Build prepared artifacts for the named services, then re-render and re-link. Flags: `--region `, `--continue-on-error`, `--import` (MOTIS only: also run `motis import`). | +| `services build-all` | Build every service that declares a build, in order. Flags: `--region `, `--fail-fast`. | +| `services start [ids...]` | Render, apply hardlinks, then `docker compose up -d` the named services. Flag: `--preset `. | +| `services stop [ids...]` | `docker compose stop` the named services. Flag: `--preset `. | +| `services restart [ids...]` | In-place reboot (`docker compose restart`); does **not** re-render. Flag: `--preset `. | +| `services update [ids...]` | Re-render, link, pull the configured images, then replace containers with `up -d --force-recreate`. When `app-api` is among them, it first stops `app-api` and applies the new image's database migrations in a one-off container. Flag: `--preset `. For core OpenMapX app releases, use the aggregate release-manifest procedure in [Upgrading](../install/upgrading.md). | +| `services status [id]` | Container status table for one or all services (`docker compose ps`). | +| `services logs ` | Stream service logs. Flags: `--tail ` (default `100`), `--follow`. | +| `services capabilities` | Inventory the capability and data-type vocabulary across the registry. Flag: `--unrecognised`. | `services enable` / `disable` refuse to edit the selection file when `OPENMAPX_ENABLED_SERVICES` is set, so the source of truth stays unambiguous. diff --git a/docs/docs/install/upgrading.md b/docs/docs/install/upgrading.md index d6ae9f581..d8544ceb5 100644 --- a/docs/docs/install/upgrading.md +++ b/docs/docs/install/upgrading.md @@ -391,14 +391,15 @@ dedicated hostname or port. ## 5. Database migrations apply automatically OpenMapX does not have a separate "run migrations" command for a self-host -upgrade. The `app-api` container **applies pending Drizzle migrations on every -boot**: at startup it runs the migration files bundled in the image against the -database, and the migrator is **idempotent**; it skips migrations that are +upgrade. When `services update` replaces `app-api` (step 4), it first stops the +old `app-api` and applies the new image's pending Drizzle migrations in a one-off +container. Their output is part of the update's output. Only then does it start +the new containers. The migrator is **idempotent**; it skips migrations that are already applied and only runs the new ones. -So once the new `app-api` image starts (step 4), any schema changes that shipped -with the release are applied automatically. You can confirm it ran cleanly in the -API logs: +`app-api` also applies pending migrations whenever it starts, so a container +started any other way catches up too. After the update's migration step there +are none left. You can confirm it ran cleanly in the API logs: ```bash pnpm openmapx services logs app-api --tail 100 @@ -418,15 +419,13 @@ the deployment contains state you cannot readily recreate. ### While a migration runs -`app-api` logs `Applying N database migrations` with their names before it -starts. It answers its healthcheck only after they finish, so the container -shows as `starting` meanwhile, and `app-web` waits for it. Docker allows it 20 -minutes before it counts failed healthchecks. A migration that changes many rows -can take a while. To follow it: +The migration step logs `Applying N database migrations` with their names before +it starts. The API is unavailable until it finishes and the new containers have +started. A migration that changes many rows can take a while. To follow it: -- Every minute, `app-api` logs `Still applying database migrations` with the - statement it is running and for how long. If that statement is waiting for a - lock another database session holds, it logs that instead. +- Every minute, it logs `Still applying database migrations` with the statement + it is running and for how long. If that statement is waiting for a lock another + database session holds, it logs that instead. - In PostgreSQL the migration's session is named `openmapx-api-migrations`. From `psql` in the `postgis` container, connected to the `openmapx` database: @@ -436,11 +435,14 @@ can take a while. To follow it: ``` Let a migration that is making progress finish. All pending migrations run in -one transaction, so stopping `app-api` rolls every one of them back, and the next -start repeats them from the beginning. The admin-panel updater does the same -when `app-api` is not healthy in time: it returns the services to their previous -images, which stops the migration. A second `app-api` started meanwhile waits for -the first to finish instead of migrating the database at the same time. +one transaction, so stopping the update rolls every one of them back. The +previous `app-api` then starts again, nothing is replaced, and the next update +repeats the migrations from the beginning. A migration that fails does the same. + +An update from the admin panel stops after 30 minutes, including its backup and +image pull, and its migration with it. Apply a release whose migration needs +longer from the host with `pnpm openmapx services update`, which has no time +limit. ## 6. Verify the upgrade diff --git a/packages/cli/__tests__/api-migrations.test.ts b/packages/cli/__tests__/api-migrations.test.ts new file mode 100644 index 000000000..5708c9135 --- /dev/null +++ b/packages/cli/__tests__/api-migrations.test.ts @@ -0,0 +1,149 @@ +import { execa } from "execa"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { migrateBeforeApiUpdate, stopMigrationArgs } from "../src/lib/api-migrations"; +import { dockerCompose, dockerComposeStream } from "../src/lib/docker"; + +vi.mock("execa", () => ({ execa: vi.fn() })); +vi.mock("node:child_process", () => ({ spawn: vi.fn(() => ({ unref: vi.fn() })) })); +vi.mock("../src/lib/docker", () => ({ dockerCompose: vi.fn(), dockerComposeStream: vi.fn() })); +vi.mock("../src/lib/paths", () => ({ + repoPaths: () => ({ infraDir: "/srv/openmapx/infra/docker" }), +})); +vi.mock("../src/lib/output", () => ({ + log: { warn: vi.fn(), err: vi.fn(), dim: vi.fn(), ok: vi.fn() }, +})); + +type Docker = (args: string[]) => { stdout?: string; exitCode?: number }; + +function docker(handle: Docker) { + vi.mocked(execa).mockImplementation((async (_file: string, args: string[]) => ({ + stdout: "", + stderr: "", + exitCode: 0, + ...handle(args), + })) as never); +} + +const composeCalls = () => vi.mocked(dockerComposeStream).mock.calls.map(([args]) => args); +const dockerCalls = () => vi.mocked(execa).mock.calls.map(([, args]) => args); + +const PROJECT_LABEL = "com.docker.compose.project=openmapx"; +const MARKER = "org.openmapx.migration=app-api"; +const FILTERS = ["--filter", `label=${PROJECT_LABEL}`, "--filter", `label=${MARKER}`]; + +beforeEach(() => { + vi.mocked(dockerCompose) + .mockReset() + .mockResolvedValue({ + stdout: JSON.stringify({ name: "openmapx", services: {} }), + stderr: "", + exitCode: 0, + }); + vi.mocked(dockerComposeStream).mockReset().mockResolvedValue(0); + docker(() => ({})); +}); +afterEach(() => vi.clearAllMocks()); + +describe("migrateBeforeApiUpdate", () => { + it("stops app-api and migrates with the new image in a labelled one-off container", async () => { + expect(await migrateBeforeApiUpdate({ deps: false })).toBe(0); + + const [stop, run] = composeCalls(); + expect(stop).toEqual(["stop", "app-api"]); + expect(run.slice(0, 4)).toEqual(["run", "--rm", "-T", "--no-deps"]); + expect(run[run.indexOf("--label") + 1]).toBe(MARKER); + expect(run.slice(run.indexOf("app-api"))).toEqual([ + "app-api", + "sh", + "-c", + "if [ -f dist/migrate.js ]; then exec node --import tsx/esm dist/migrate.js; fi; exit 78", + ]); + expect(vi.mocked(dockerCompose).mock.calls).toEqual([[["config", "--format", "json"]]]); + expect(dockerCalls()).toEqual([["ps", "-aq", ...FILTERS]]); + expect(composeCalls()).toHaveLength(2); + }); + + it("lets the one-off container start app-api's dependencies when the update does", async () => { + await migrateBeforeApiUpdate({ deps: true }); + expect(composeCalls()[1]).not.toContain("--no-deps"); + }); + + it("first stops a migration an earlier update left running", async () => { + docker((args) => (args[0] === "ps" ? { stdout: "abc123\ndef456\n" } : {})); + await migrateBeforeApiUpdate({ deps: false }); + expect(dockerCalls().slice(1)).toEqual([ + ["stop", "-t", "60", "abc123", "def456"], + ["rm", "-f", "abc123", "def456"], + ]); + expect(composeCalls()[0]).toEqual(["stop", "app-api"]); + }); + + it("starts the previous app-api again and reports the failure when the migration fails", async () => { + vi.mocked(dockerComposeStream).mockImplementation(async (args) => (args[0] === "run" ? 1 : 0)); + expect(await migrateBeforeApiUpdate({ deps: false })).toBe(1); + expect(composeCalls().map((args) => args[0])).toEqual(["stop", "run", "start"]); + expect(composeCalls()[2]).toEqual(["start", "app-api"]); + }); + + it("leaves migrating to app-api's startup when its image has no migration entry point", async () => { + vi.mocked(dockerComposeStream).mockImplementation(async (args) => (args[0] === "run" ? 78 : 0)); + expect(await migrateBeforeApiUpdate({ deps: false })).toBe(0); + expect(composeCalls().map((args) => args[0])).toEqual(["stop", "run"]); + }); + + it("migrates nothing when the Compose project cannot be read", async () => { + vi.mocked(dockerCompose).mockResolvedValueOnce({ stdout: "", stderr: "boom", exitCode: 1 }); + await expect(migrateBeforeApiUpdate({ deps: false })).rejects.toThrow( + "Could not read the Compose project name", + ); + expect(dockerComposeStream).not.toHaveBeenCalled(); + }); + + it("migrates nothing when app-api cannot be stopped", async () => { + vi.mocked(dockerComposeStream).mockResolvedValueOnce(5); + expect(await migrateBeforeApiUpdate({ deps: false })).toBe(5); + expect(composeCalls()).toEqual([["stop", "app-api"]]); + }); + + it("listens for a stop request only while the migration container runs", async () => { + const before = process.listenerCount("SIGTERM"); + let during = 0; + vi.mocked(dockerComposeStream).mockImplementation(async (args) => { + if (args[0] === "run") during = process.listenerCount("SIGTERM"); + return 0; + }); + await migrateBeforeApiUpdate({ deps: false }); + expect(during).toBe(before + 1); + expect(process.listenerCount("SIGTERM")).toBe(before); + }); +}); + +describe("stopMigrationArgs", () => { + it("stops the labelled migration and then starts app-api through Compose", async () => { + const { execFileSync } = + await vi.importActual("node:child_process"); + // Run the script with a stub `docker` that records each invocation. + const { mkdtempSync, writeFileSync, chmodSync, readFileSync } = await import("node:fs"); + const { join } = await import("node:path"); + const { tmpdir } = await import("node:os"); + const bin = mkdtempSync(join(tmpdir(), "stop-migration-")); + const record = join(bin, "calls"); + writeFileSync( + join(bin, "docker"), + `#!/bin/sh\nprintf '%s|' "$@" >> "${record}"; echo >> "${record}"\n[ "$1" = ps ] && echo abc123\nexit 0\n`, + ); + chmodSync(join(bin, "docker"), 0o755); + + const args = stopMigrationArgs( + [PROJECT_LABEL, MARKER], + ["compose", "-f", "/srv/open mapx/docker-compose.yml"], + ); + execFileSync("sh", args, { env: { ...process.env, PATH: `${bin}:${process.env.PATH}` } }); + + expect(readFileSync(record, "utf8").trim().split("\n")).toEqual([ + `ps|-q|--filter|label=${PROJECT_LABEL}|--filter|label=${MARKER}|`, + "stop|-t|60|abc123|", + "compose|-f|/srv/open mapx/docker-compose.yml|start|app-api|", + ]); + }); +}); diff --git a/packages/cli/__tests__/cli-services-update.test.ts b/packages/cli/__tests__/cli-services-update.test.ts index 4f2401e81..70f0ce091 100644 --- a/packages/cli/__tests__/cli-services-update.test.ts +++ b/packages/cli/__tests__/cli-services-update.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { renderComposeForRepo } from "../src/commands/compose"; import { registerServicesCommands } from "../src/commands/services"; import { refuseAgentSwitchOrExit } from "../src/lib/agent-layout"; +import { migrateBeforeApiUpdate } from "../src/lib/api-migrations"; import { dockerComposeStream } from "../src/lib/docker"; import { ensureReleaseOverlay } from "../src/lib/release"; @@ -20,6 +21,11 @@ vi.mock("../src/lib/docker", () => ({ dockerComposeStream: vi.fn() })); // Whether recreating the agent would switch it to its narrow mounts unready // is agent-layout.test.ts's; here it lets the update go on. vi.mock("../src/lib/agent-layout", () => ({ refuseAgentSwitchOrExit: vi.fn(async () => {}) })); +// The migration step's own Docker calls are api-migrations.test.ts's. +vi.mock("../src/lib/api-migrations", () => ({ + API_SERVICE_ID: "app-api", + migrateBeforeApiUpdate: vi.fn(async () => 0), +})); vi.mock("../src/lib/release", async (importOriginal) => ({ ...(await importOriginal()), ensureReleaseOverlay: vi.fn(), @@ -28,6 +34,7 @@ vi.mock("../src/lib/release", async (importOriginal) => ({ beforeEach(() => { vi.mocked(dockerComposeStream).mockReset().mockResolvedValue(0); vi.mocked(ensureReleaseOverlay).mockReset(); + vi.mocked(migrateBeforeApiUpdate).mockReset().mockResolvedValue(0); vi.spyOn(console, "log").mockImplementation(() => {}); vi.spyOn(console, "error").mockImplementation(() => {}); vi.spyOn(console, "warn").mockImplementation(() => {}); @@ -148,3 +155,38 @@ describe("services update and the operations agent's mounts", () => { expect(vi.mocked(dockerComposeStream).mock.calls).toEqual([[["pull", "ops-agent"]]]); }); }); + +describe("services update and app-api's database migrations", () => { + beforeEach(() => { + vi.mocked(ensureReleaseOverlay).mockResolvedValue({ status: "present", path: "release.yml" }); + }); + + it("migrates after the pull and before recreation when app-api is replaced", async () => { + const order: string[] = []; + vi.mocked(dockerComposeStream).mockImplementation(async (args) => { + order.push(args[0] as string); + return 0; + }); + vi.mocked(migrateBeforeApiUpdate).mockImplementation(async () => { + order.push("migrate"); + return 0; + }); + await expect(update("--no-deps", "app-web", "app-api")).rejects.toThrow("exit:0"); + expect(order).toEqual(["pull", "migrate", "up"]); + expect(migrateBeforeApiUpdate).toHaveBeenCalledWith({ deps: false }); + + await expect(update("app-api")).rejects.toThrow("exit:0"); + expect(migrateBeforeApiUpdate).toHaveBeenLastCalledWith({ deps: true }); + }); + + it("recreates nothing when the migration step fails", async () => { + vi.mocked(migrateBeforeApiUpdate).mockResolvedValue(1); + await expect(update("--no-deps", "app-web", "app-api")).rejects.toThrow("exit:1"); + expect(vi.mocked(dockerComposeStream).mock.calls).toEqual([[["pull", "app-web", "app-api"]]]); + }); + + it("does not migrate when app-api is not among the services", async () => { + await expect(update("app-web")).rejects.toThrow("exit:0"); + expect(migrateBeforeApiUpdate).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/src/commands/services.ts b/packages/cli/src/commands/services.ts index b7cc742ee..0d6777687 100644 --- a/packages/cli/src/commands/services.ts +++ b/packages/cli/src/commands/services.ts @@ -2,6 +2,7 @@ import { services as coreServices } from "@openmapx/core/server"; import type { Command } from "commander"; import kleur from "kleur"; import { refuseAgentSwitchOrExit } from "../lib/agent-layout"; +import { API_SERVICE_ID, migrateBeforeApiUpdate } from "../lib/api-migrations"; import { dockerComposeStream } from "../lib/docker"; import { applyGeneratedHardlinks, @@ -506,6 +507,16 @@ export function registerServicesCommands(program: Command): void { // With its dependencies, the agent may be recreated too. await refuseAgentSwitchOrExit(options.deps === false ? allIds : "all"); + if (allIds.includes(API_SERVICE_ID)) { + let migrationCode: number; + try { + migrationCode = await migrateBeforeApiUpdate({ deps: options.deps !== false }); + } catch (err) { + log.err(`Database migration step failed: ${(err as Error).message}`); + process.exit(1); + } + if (migrationCode !== 0) process.exit(migrationCode); + } const code = await dockerComposeStream([ "up", "-d", diff --git a/packages/cli/src/lib/api-migrations.ts b/packages/cli/src/lib/api-migrations.ts new file mode 100644 index 000000000..b53fc60fb --- /dev/null +++ b/packages/cli/src/lib/api-migrations.ts @@ -0,0 +1,142 @@ +import { spawn } from "node:child_process"; +import { services as coreServices } from "@openmapx/core/server"; +import { execa } from "execa"; +import { dockerCompose, dockerComposeStream } from "./docker"; +import { log } from "./output"; +import { repoPaths } from "./paths"; + +export const API_SERVICE_ID = "app-api"; + +/** The app-api image's migration entry point, relative to its working directory. */ +const MIGRATE_SCRIPT = "dist/migrate.js"; +/** Exit status of the migration container when its image has no migration entry point. */ +const NO_MIGRATION_STEP = 78; +/** The migration cancels its statement within seconds of SIGTERM; allow for its rollback. */ +const STOP_TIMEOUT_SECONDS = "60"; + +/** Marks migration containers among the one-off containers of app-api. */ +const MIGRATION_MARKER = "org.openmapx.migration=app-api"; + +/** + * The labels of this stack's migration containers. `docker compose run` + * containers outlive a stopped or killed `docker compose` client, so each + * migration step first stops any its predecessors left behind. The Compose + * project is the same whether the host or the operations agent runs this. + */ +async function migrationLabels(): Promise<[string, string]> { + const result = await dockerCompose(["config", "--format", "json"]); + const project = + result.exitCode === 0 ? (JSON.parse(result.stdout) as { name?: unknown }).name : undefined; + if (typeof project !== "string" || project === "") { + throw new Error(`Could not read the Compose project name: ${result.stderr}`); + } + return [`com.docker.compose.project=${project}`, MIGRATION_MARKER]; +} + +const labelFilters = (labels: string[]) => + labels.flatMap((label) => ["--filter", `label=${label}`]); + +async function migrationContainers(labels: string[]): Promise { + const result = await execa("docker", ["ps", "-aq", ...labelFilters(labels)], { reject: false }); + if (result.exitCode !== 0) throw new Error(`docker ps failed: ${result.stderr}`); + return result.stdout.split("\n").filter(Boolean); +} + +async function stopMigrationContainers(ids: string[]): Promise { + if (ids.length === 0) return; + await execa("docker", ["stop", "-t", STOP_TIMEOUT_SECONDS, ...ids], { reject: false }); + await execa("docker", ["rm", "-f", ...ids], { reject: false }); +} + +/** + * `sh` arguments that stop this stack's migration containers and then start + * app-api with `docker start`. The labels and the Compose + * arguments are positional parameters, never part of the script text. + */ +export function stopMigrationArgs(labels: [string, string], composeArgs: string[]): string[] { + return [ + "-c", + 'docker stop -t "$1" $(docker ps -q --filter "label=$2" --filter "label=$3"); shift 3; exec docker "$@"', + "stop-migration", + STOP_TIMEOUT_SECONDS, + ...labels, + ...composeArgs, + "start", + API_SERVICE_ID, + ]; +} + +/** + * Applies app-api's pending database migrations before app-api is replaced. + * + * The old app-api stops first, so it never runs against the migrated schema, + * then a one-off container of the new image migrates the database. Its output + * is this command's output. On failure the old app-api container starts again + * and the nonzero status is returned: nothing has been recreated, and the + * migration rolled back as a whole. + * + * Images without the migration entry point skip the step: app-api applies its + * migrations when it starts, as earlier releases did. + */ +export async function migrateBeforeApiUpdate(options: { deps: boolean }): Promise { + const labels = await migrationLabels(); + const leftovers = await migrationContainers(labels); + if (leftovers.length > 0) { + log.warn("Stopping a database migration an earlier update left running…"); + await stopMigrationContainers(leftovers); + } + + const stopCode = await dockerComposeStream(["stop", API_SERVICE_ID]); + if (stopCode !== 0) { + log.err("Could not stop app-api before migrating the database; nothing was replaced."); + return stopCode; + } + + // The `docker compose` client does not pass a stop request on to the + // container it runs, and the operations agent kills this command shortly + // after it asks it to stop. A separate process group outlives it: it stops + // the migration, which rolls back, and starts the previous app-api again. + const paths = repoPaths(); + const stopOnSignal = () => { + spawn("sh", stopMigrationArgs(labels, coreServices.stackComposeArgs(paths)), { + cwd: paths.infraDir, + detached: true, + stdio: "ignore", + }).unref(); + }; + process.once("SIGTERM", stopOnSignal); + process.once("SIGINT", stopOnSignal); + + log.dim("Applying database migrations before replacing app-api…"); + let code: number; + try { + code = await dockerComposeStream([ + "run", + "--rm", + "-T", + ...(options.deps ? [] : ["--no-deps"]), + "--label", + MIGRATION_MARKER, + API_SERVICE_ID, + "sh", + "-c", + `if [ -f ${MIGRATE_SCRIPT} ]; then exec node --import tsx/esm ${MIGRATE_SCRIPT}; fi; exit ${NO_MIGRATION_STEP}`, + ]); + } finally { + process.off("SIGTERM", stopOnSignal); + process.off("SIGINT", stopOnSignal); + } + + if (code === NO_MIGRATION_STEP) { + log.dim("This app-api image applies its migrations when it starts."); + return 0; + } + if (code !== 0) { + log.err( + "The database migration did not complete, and whatever part of it ran was rolled back. Starting the previous app-api again; nothing was replaced.", + ); + await dockerComposeStream(["start", API_SERVICE_ID]); + return code; + } + return 0; +} diff --git a/services/app-api/service.json b/services/app-api/service.json index 9b97eb80a..18d7e2b49 100644 --- a/services/app-api/service.json +++ b/services/app-api/service.json @@ -142,7 +142,7 @@ "interval": "15s", "timeout": "3s", "retries": 10, - "startPeriod": "20m" + "startPeriod": "30s" } }, "bindMounts": [