From 66f76a4a18054c7eb84ffb4165edeb3f71d824b6 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 12:34:50 -0400 Subject: [PATCH 01/39] feat(migrate): add export clerk and export supabase, and import by export run ID `clerk migrate export clerk` reads users over BAPI, and `export supabase` reads auth.users over a connection string (Bun.sql or bun:sqlite), so the bcrypt digests come across. Each export is a run holding export.json, an envelope that names its source. `clerk migrate import ` then needs no file path and no --source, records the export as fromExport, and names the export folder to delete once the import is done. A connection string the database rejects is asked for again at a terminal. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 4 + .../cli-core/src/commands/migrate/README.md | 298 +++++++++++--- .../migrate/export/clerk-source.test.ts | 241 +++++++++++ .../commands/migrate/export/clerk-source.ts | 188 +++++++++ .../src/commands/migrate/export/clerk.test.ts | 375 +++++++++++++++++ .../src/commands/migrate/export/clerk.ts | 284 +++++++++++++ .../migrate/export/db-exports.test.ts | 187 +++++++++ .../src/commands/migrate/export/db-options.ts | 171 ++++++++ .../src/commands/migrate/export/index.ts | 135 +++++++ .../commands/migrate/export/registry.test.ts | 32 ++ .../src/commands/migrate/export/registry.ts | 48 +++ .../commands/migrate/export/shared.test.ts | 124 ++++++ .../src/commands/migrate/export/shared.ts | 198 +++++++++ .../src/commands/migrate/export/supabase.ts | 150 +++++++ .../src/commands/migrate/index.test.ts | 87 +++- .../cli-core/src/commands/migrate/index.ts | 41 +- .../src/commands/migrate/lib/db.test.ts | 325 +++++++++++++++ .../cli-core/src/commands/migrate/lib/db.ts | 379 ++++++++++++++++++ .../src/commands/migrate/lib/export-file.ts | 48 ++- .../commands/migrate/lib/input-retry.test.ts | 234 +++++++++++ .../src/commands/migrate/lib/input-retry.ts | 84 ++++ .../commands/migrate/lib/transform.test.ts | 16 + .../src/commands/migrate/lib/transform.ts | 5 +- .../commands/migrate/run-interactive.test.ts | 30 +- .../cli-core/src/commands/migrate/run.test.ts | 69 +++- packages/cli-core/src/commands/migrate/run.ts | 126 ++++-- scripts/check-bun-version.ts | 7 + 27 files changed, 3772 insertions(+), 114 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/export/clerk-source.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/clerk-source.ts create mode 100644 packages/cli-core/src/commands/migrate/export/clerk.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/clerk.ts create mode 100644 packages/cli-core/src/commands/migrate/export/db-exports.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/db-options.ts create mode 100644 packages/cli-core/src/commands/migrate/export/index.ts create mode 100644 packages/cli-core/src/commands/migrate/export/registry.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/registry.ts create mode 100644 packages/cli-core/src/commands/migrate/export/shared.test.ts create mode 100644 packages/cli-core/src/commands/migrate/export/shared.ts create mode 100644 packages/cli-core/src/commands/migrate/export/supabase.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/db.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/db.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/input-retry.test.ts create mode 100644 packages/cli-core/src/commands/migrate/lib/input-retry.ts diff --git a/.gitignore b/.gitignore index 91df74696..1caf6ad7e 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,7 @@ test/e2e/.har # Local planning/spec docs docs/superpowers/ + +# Local migration exports and the credentials that produced them +exports/ +*service-account*.json diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index bf28cdaf3..f8933f960 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -15,17 +15,19 @@ Migrate users into a Clerk instance from another auth provider, or from another Clerk instance. ``` -clerk migrate import [--source ] [--dry-run] [--allow-partial] [--new-run] [--yes] [--json] +clerk migrate export [-o ] [--json] +clerk migrate import [--source ] [--dry-run] [--allow-partial] [--new-run] [--yes] [--json] clerk migrate runs [run-id] [--json] clerk migrate undo [--dry-run] [--yes] [--include-signed-in] [--json] clerk migrate help ``` -An import is usually two steps: +A migration is usually three steps: ```sh -clerk migrate import users.json --source supabase --dry-run # 1. check it against the instance -clerk migrate import users.json --source supabase --yes # 2. import it +clerk migrate export supabase # 1. a run, holding export.json +clerk migrate import 20260929-141502-a1b2 --dry-run # 2. check it against the instance +clerk migrate import 20260929-141502-a1b2 --yes # 3. import it ``` `clerk migrate undo ` takes an import back out, and `clerk migrate runs` @@ -48,8 +50,8 @@ Every command follows these: be dropped are warnings. 3. **State lives in one place: the [run store](#the-run-store).** Each run records its target, its file, and every source ID → Clerk ID outcome, - including the error for each user who failed. `runs`, `undo` and re-runs all - read or write it. + including the error for each user who failed. `runs`, `undo`, re-runs and + exports all read or write it. 4. **Every command prints its target first:** the environment, app and instance, and where the key came from. 5. **Every subcommand takes `--json`.** Exit codes: `0` all good, `1` some users @@ -98,8 +100,9 @@ instance — its environment, its app when the key came from one, and its ID from `GET /v1/instance` — and where the key came from: `--secret-key`, `--app`, the `CLERK_SECRET_KEY` env var, an accountless app's `.env.local`, or the linked -profile. `runs` names the runs folder. `--json` carries the same facts as -`target`. +profile. An export names its source platform instead, and `export clerk` the +instance it reads. `runs` names the runs folder. `--json` carries the same +facts as `target`. ``` Target: My App (app_2x9k…), production instance ins_2x9k… @@ -111,10 +114,10 @@ the key now in use still addresses the same instance. ## The run store -Every import and undo that gets as far as writing is a **run**, and the run store is -the one place `clerk migrate` keeps state. A dry run, a refusal, a run that -needs consent and an empty file write none (`run: null`). An import whose -users `--require-password` all leaves out still writes one, recording them +Every import, export and undo that gets as far as writing is a **run**, and the +run store is the one place `clerk migrate` keeps state. A dry run, a refusal, a +run that needs consent and an empty file write none (`run: null`). An import +whose users `--require-password` all leaves out still writes one, recording them as skipped. ### Where runs are kept @@ -147,10 +150,11 @@ On a `failed` line, `code` is the HTTP status, such as `422` or `429`, or the checks skipped for the file's own data (invalid, a duplicate within it, an email Clerk refuses, a malformed hash). -A user's status is `creating`, `created`, `failed`, `skipped` or `deleted`. The last line -for each `sourceId` wins. A `429` retry, an extra email or phone that did not -attach, a first phone Clerk refused (which the summary also counts), and a -validation failure all land in the line's `error` field. +A user's status is `creating`, `created`, `failed`, `skipped`, `deleted` or +`exported`. The last line for each `sourceId` wins. A `429` retry, an extra +email or phone that did not attach, a first phone Clerk refused (which the +summary also counts), and a validation failure all land in the line's `error` +field. `creating` is written as a user's `POST /v1/users` goes out. It stays the latest status when no answer says whether the create landed: an abort, a network error, @@ -169,11 +173,13 @@ the lock file to delete if that process is not a migrate run. A lock holding this process's own PID is stale: in a container the CLI often gets the same PID every run. -Run folders are created owner-only (`0700`), because they hold user data. +Run folders are created owner-only (`0700`), and export files `0600`: they hold +password hashes and user data. `users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C still leaves a complete record of everything already processed. A line that -cannot be written stops that user's create from going out. +cannot be written stops that user's create from going out. An export's file +lands in its run folder as `export.json` unless `--output` says otherwise. ### Why `users.ndjson` is NDJSON @@ -196,6 +202,182 @@ grep '"sourceId":"user_123"' .clerk/migrate/20260929-141502-a1b2/users.ndjson ## Commands +The direction is always spelled out — `migrate import` moves users **into** +Clerk, `migrate export` gets them **out** of a source platform — so neither is +implied by the group. + +### `clerk migrate export` + +Gets users **out** of a source platform, so there is something to feed +`clerk migrate import`. + +```sh +clerk migrate export # pick a platform +clerk migrate export clerk --output users.json +clerk migrate export supabase --db-url "postgres://postgres:...@db.xxx.supabase.co:5432/postgres" +``` + +The platform is an optional positional. Omitted, you get a picker built from +the registry; given, it runs directly. Each platform resolves its own flags — +what a Clerk export needs (a secret key) has nothing in common with what a +database export needs. + +**A credential the far end rejects is asked for again.** A connection string is +long, pasted by hand, masked as it is typed, and wrong in ways nothing local can +check: a typo'd host, a revoked password, the right server but the wrong +database. Only the connection can say, and by then the operator has answered +every other question the command asked. So that step — and only that step, +never a fetch already under way or a file already written — runs inside a +retry: the failure is explained, the prompt comes back, and the rest of the +export continues against whichever credential worked. Agent mode and a non-TTY +fail outright instead, having nobody to ask, and `-y` fails too, having been +told not to. + +| Platform | Source | Feeds | +| ---------- | -------------------------------- | ------------------- | +| `clerk` | Clerk Backend API | `--source clerk` | +| `supabase` | Supabase Postgres (`auth.users`) | `--source supabase` | + +Every export is a [run](#the-run-store), and the file lands in the run +folder as `export.json`. `--output` writes it somewhere else instead, +resolved against the **current directory** like every other path flag here; +the run still records where. Nothing is asked about where the file goes. + +The file is an envelope around the users: + +```json +{ + "clerkMigrate": 1, + "source": "clerk", + "exportedAt": "2026-09-29T14:15:02.000Z", + "runId": "20260929-141502-a1b2", + "users": [ … ] +} +``` + +`source` is what lets `clerk migrate import ` run with no +`--source`. + +`--json` prints the result on stdout instead — `{ target, run, output, users, +coverage, next }` — and never prompts, so a missing credential exits 2 naming +the flag to pass. + +| Flag | Platforms | Description | +| --------------------- | ---------- | --------------------------------------------------------- | +| `-o, --output ` | all | Write the export here instead of the run folder | +| `-y, --yes` | all | Do not prompt: fail on a bad credential | +| `--json` | all | Print the result as JSON; never prompts | +| `--runs-dir ` | all | Where runs are kept (see [the run store](#the-run-store)) | +| `--db-url ` | `supabase` | Postgres, MySQL, libsql/Turso or SQLite connection string | + +`export clerk` also takes the targeting flags — it reads from a Clerk instance, +so it resolves a key the same way `clerk migrate import` does, with one extra +step. The linked project is usually the migration's _destination_, so taking it +as the source without asking is how a run exports an instance and imports it +back into itself. Instead: + +- **Naming the instance runs unquestioned.** `--secret-key `, `--app`, + `--instance`, or an exported `CLERK_SECRET_KEY` — any of them is a sentence + you typed for this run, so none of them opens a picker. That is what makes + the export scriptable outside agent mode, and it keeps an exported key + outranking the linked profile here the way it does everywhere else in the + CLI. +- Anything resolved on your behalf — the linked project, a keyless app — is + never taken silently. A picker of every **instance** on your account opens + instead — one flat row each, `my-app - Production instance (ins_…)`, not an + application picker followed by an instance picker — with the resolved + application's instances listed **first** so taking one is still a single + Enter. Only when there are no instances to offer does it stop and list + `--secret-key`, `--app`/`--instance` and `clerk link` instead. +- With nothing to resolve at all (no link, no key, no flags), you get the + application picker `clerk users` uses — `Select a Clerk application to use:`, + followed by an instance picker when the application has more than one — + rather than an error about an unlinked directory. That is `clerk link`'s + picker, so it does offer `+ Create a new application`; a brand-new + application has no users to export, so it is never the answer here. + +The instance picker (the second tier) has no "create a new application" choice. +Its rows are searchable by what they show, so typing an application name, +`production`, or an instance id all narrow it. + +In agent mode the resolved instance is used without a prompt; pass +`--secret-key` or `--app`/`--instance` to be explicit. + +After each export you get a field-coverage table — which Clerk-relevant fields +were present on how many users — so you know the data is thin _before_ you +import it, not after: + +``` +Field coverage + ✓ 3/3 have an email address + ✗ 0/3 have a phone number + ! 1/3 have a username + ! 2/3 have a password (not exportable — see below) + +Exported 3 users to /project/.clerk/migrate/20260929-141502-a1b2/export.json +Run 20260929-141502-a1b2. See each user with `clerk migrate runs 20260929-141502-a1b2`. + +Import them with: + clerk migrate import 20260929-141502-a1b2 + + Imports into whichever instance the resolved secret key belongs to. + For production, add `--instance prod` or use a production secret key. +``` + +The import command prints through the same channel as the coverage table +rather than the gutter's **Next steps** outro, which is human-only — an agent +would otherwise be told what was exported and never how to import it. + +There is one command, not a development and a production variant, because no +flag's absence means "development" — the resolved key decides, through +`--secret-key`, `--app`, `CLERK_SECRET_KEY`, the keyless project and the linked +profile in that order. + +The export run has one line per exported user, so `clerk migrate runs` lists it +alongside imports. + +#### Clerk exports no passwords + +Clerk never returns password digests, TOTP secrets or backup codes over the API +— only the `*_enabled` booleans. Migrated users must reset their password in +the destination instance. The export says so on every run, and the coverage row +counts users who _have_ a password, so the size of the gap is visible up front. + +#### `supabase` reads the database + +```sh +clerk migrate export supabase --db-url "postgres://postgres:...@db.xxx.supabase.co:5432/postgres" +``` + +Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`; +`libsql://` (Turso) over the server's HTTP pipeline endpoint, since `bun:sqlite` +only opens local files and `@libsql/client` ships native optional dependencies. +Nothing native ships in the binary — that is the whole reason the `engines.bun` +floor exists. Resolution is `--db-url`, then `SUPABASE_DB_URL`, then a masked +prompt, since a connection string carries the password inline. A password +pasted unencoded (`#`, `@`, `/` and the like) is percent-encoded for you. + +**Connection strings are redacted everywhere.** Errors show +`postgres://***@host/db`, including when the password itself contains an +unencoded `@` — the most common mistake, and exactly when the string ends up in +an error message. + +Connection failures get a hint rather than a driver error. Bun reports both an +unreachable host and a closed port as "Connection closed", so: + +| Situation | What you are told | +| ------------------------ | ------------------------------------------------------------------------------------------------- | +| Host or port unreachable | Check the host and port. On Supabase: use the pooler connection string, or enable the IPv4 add-on | +| Credentials rejected | Check the user and password | +| Table missing | Check the database name and SELECT permission. On Supabase: enable Auth, connect as `postgres` | +| SQLite file missing | Check the path and that the file is readable | + +**It reads the database rather than the Admin API** because +`encrypted_password` exists only there. An API-based export would force every +user to reset their password; this one carries the bcrypt digests across. It +also keeps `raw_app_meta_data`, which is what the import's +[checks](#checks) read for each user's providers. + ### `clerk migrate import` Reads an exported user file, maps it onto Clerk's user schema, checks every @@ -203,8 +385,9 @@ user against the destination instance, and creates them through the Backend API. ```sh -clerk migrate import users.json --source supabase --dry-run # check, write nothing -clerk migrate import users.json --source supabase --yes # import +clerk migrate import 20260929-141502-a1b2 --dry-run # check, write nothing +clerk migrate import 20260929-141502-a1b2 --yes # an export run +clerk migrate import users.json --source supabase --yes # any other file clerk migrate import users.json --source clerk --allow-partial --yes clerk migrate import users.json --source clerk --new-run --yes clerk migrate import users.json --source clerk --json --yes @@ -216,31 +399,35 @@ clerk migrate import users.json --source clerk --secret-key sk_test_... -y clerk migrate import # a human is asked ``` -| Flag | Description | -| --------------------- | ------------------------------------------------------------------- | -| `[file]` | A JSON or CSV export | -| `--source ` | Where the file came from: one of the [sources](#sources) | -| `--dry-run` | Run the [checks](#checks) against the instance, and write nothing | -| `--allow-partial` | Import the users that pass, and record the rest as skipped | -| `--new-run` | Start a new run instead of [continuing](#re-running) an earlier one | -| `--require-password` | Import only users that carry a password digest | -| `--skip-legal-checks` | Import users with no legal acceptance into an instance requiring it | -| `-y, --yes` | Import without prompting | -| `--json` | Output as JSON. Never prompts, so importing needs `--yes` | -| `--runs-dir ` | Where runs are kept (see [the run store](#the-run-store)) | +| Flag | Description | +| ----------------------- | ------------------------------------------------------------------- | +| `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | +| `--source ` | Where the file came from: one of the [sources](#sources) | +| `--dry-run` | Run the [checks](#checks) against the instance, and write nothing | +| `--allow-partial` | Import the users that pass, and record the rest as skipped | +| `--new-run` | Start a new run instead of [continuing](#re-running) an earlier one | +| `--require-password` | Import only users that carry a password digest | +| `--skip-legal-checks` | Import users with no legal acceptance into an instance requiring it | +| `-y, --yes` | Import without prompting | +| `--json` | Output as JSON. Never prompts, so importing needs `--yes` | +| `--runs-dir ` | Where runs are kept (see [the run store](#the-run-store)) | Plus the targeting flags from the table above: `--secret-key`, `--app` and `--instance`. -A file is a JSON array, a CSV, or NDJSON, one user per line (what Auth0's bulk -export job writes). NDJSON is read always for `.ndjson` and `.jsonl`, and for a -`.json` file that doesn't parse whole. A leading BOM is ignored in JSON and -CSV. A file that isn't valid JSON is named in the error. +An export run ID stands for the file that run wrote, and the import records it +as `fromExport`. A file `clerk migrate export` wrote carries its source, so it +needs no `--source`, and a `--source` that contradicts it exits 2. Any other +file needs `--source`: a JSON array, a CSV, or NDJSON, one user per line (what +Auth0's bulk export job writes). NDJSON is read always for `.ndjson` and +`.jsonl`, and for a `.json` file that doesn't parse whole. A leading BOM is +ignored in JSON and CSV. A file that isn't valid JSON is named in the error. **What a human is asked, and what an agent is told.** A human at a terminal who -leaves out the file is asked for its path, and then for its source. An agent, a -non-TTY run, or `--json` without the file or `--source` exits 2 naming what to -pass. +leaves out the file is asked for its path, and is asked for a source only when +the file does not name one. An agent, a non-TTY run, or `--json` without the +file, or without `--source` for a file that does not name one, exits 2 naming +what to pass. **Nothing is written to Clerk without consent.** After the checks, a human is asked `Import N users?`, and declining writes nothing. `--yes` skips the question. @@ -315,8 +502,9 @@ the user into the source instance. It is how a cut-off create is told apart from a user an app or another tool made with the same `external_id`, and it stays on the user. A run recorded before the marker existed adopts nothing. -When an import completes, it names its run folder, which only `undo` needs and -which holds user data, with the `rm -rf` to remove it. +When an import completes, it names the folders it no longer needs: the export it +read, which holds your users' data, and its own run, which only `undo` needs. +Each comes with the `rm -rf` to remove it. #### Checks @@ -740,19 +928,21 @@ stamping every user with today's. ## API endpoints -| Method | Path | Used by | -| -------- | ------------------------- | ------------------------------------------------------------------------------- | -| `POST` | `/v1/users` | `migrate import` — creates each user | -| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | -| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | -| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | -| `GET` | `/v1/users?external_id=…` | `migrate import` — checks for users already in the instance, 100 values a call | -| `GET` | `/v1/users?external_id=…` | `migrate import` — a continue looks up users whose create was in flight | -| `GET` | `/v1/users?external_id=…` | `migrate undo` — finds users whose create was in flight when the import stopped | -| `GET` | `/v1/users?user_id=…` | `migrate undo` — reads the imported users back, 100 a call | -| `DELETE` | `/v1/users/{user_id}` | `migrate undo` — deletes one user | -| `GET` | `/v1/instance` | `migrate import`, `undo` — names the instance behind the key | -| `GET` | `/v1/domains` | `migrate import` checks — resolves the Frontend API host | +| Method | Path | Used by | +| -------- | ----------------------------------------------- | ------------------------------------------------------------------------------- | +| `POST` | `/v1/users` | `migrate import` — creates each user | +| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | +| `GET` | `/v1/users?limit=&offset=&order_by=+created_at` | `migrate export clerk` — pages the whole instance, oldest first, 500 at a time | +| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | +| `GET` | `/v1/users?external_id=…` | `migrate import` — checks for users already in the instance, 100 values a call | +| `GET` | `/v1/users?external_id=…` | `migrate import` — a continue looks up users whose create was in flight | +| `GET` | `/v1/users?external_id=…` | `migrate undo` — finds users whose create was in flight when the import stopped | +| `GET` | `/v1/users?user_id=…` | `migrate undo` — reads the imported users back, 100 a call | +| `DELETE` | `/v1/users/{user_id}` | `migrate undo` — deletes one user | +| `GET` | `/v1/instance` | `migrate import`, `undo` — names the instance behind the key | +| `GET` | `/v1/instance` | `migrate import`, `undo`, `export clerk` — names the instance behind the key | +| `GET` | `/v1/domains` | `migrate import` checks — resolves the Frontend API host | The checks also read the instance's Frontend API `GET /v1/environment` (bootstrapping a dev browser first on development instances) for its diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts new file mode 100644 index 000000000..1e5d95566 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts @@ -0,0 +1,241 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +const mockDescribeBapiTarget = mock(); +const mockResolveBapiSecretKey = mock(); +mock.module("../../../lib/bapi-command.ts", () => ({ + describeBapiTarget: (...args: unknown[]) => mockDescribeBapiTarget(...args), + resolveBapiSecretKey: (...args: unknown[]) => mockResolveBapiSecretKey(...args), +})); + +const mockResolveProfile = mock(); +mock.module("../../../lib/config.ts", () => ({ + resolveProfile: (...args: unknown[]) => mockResolveProfile(...args), +})); + +const mockFetchApps = mock(); +mock.module("../../../lib/app-picker.ts", () => ({ + fetchAppsTolerantly: (...args: unknown[]) => mockFetchApps(...args), +})); + +const mockSearch = mock(); +mock.module("../../../lib/listage.ts", () => ({ + search: (...args: unknown[]) => mockSearch(...args), +})); + +const mockResolveUsersInstanceContext = mock(); +mock.module("../../users/interactive/instance-context.ts", () => ({ + resolveUsersInstanceContext: (...args: unknown[]) => mockResolveUsersInstanceContext(...args), +})); + +let human = true; +mock.module("../../../mode.ts", () => ({ + isHuman: () => human, + isAgent: () => !human, + getMode: () => (human ? "human" : "agent"), + setMode: () => {}, +})); + +const { resolveClerkSource } = await import("./clerk-source.ts"); + +const captured = useCaptureLog(); + +beforeEach(() => { + human = true; + mockDescribeBapiTarget.mockReset(); + mockResolveBapiSecretKey.mockReset(); + mockResolveProfile.mockReset(); + mockResolveProfile.mockResolvedValue(undefined); + mockResolveUsersInstanceContext.mockReset(); + mockFetchApps.mockReset(); + mockSearch.mockReset(); + delete process.env.CLERK_SECRET_KEY; +}); + +/** The linked-project case: something resolved, nobody asked for it. */ +function stubResolved(target: string | undefined, secretKey = "sk_test_resolved") { + mockDescribeBapiTarget.mockResolvedValue(target); + mockResolveBapiSecretKey.mockResolvedValue(secretKey); +} + +describe("resolveClerkSource", () => { + test("--secret-key names the instance outright and is never questioned", async () => { + stubResolved(undefined, "sk_test_explicit"); + + const source = await resolveClerkSource({ secretKey: "sk_test_explicit" }); + + expect(source).toEqual({ secretKey: "sk_test_explicit", target: undefined }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + // An export has to be scriptable outside agent mode. Before this, `chosen` + // keyed off `--secret-key` alone, so every other way of naming an instance + // still stopped at a picker no flag could answer. + test.each([ + ["--app", { app: "app_1" }], + ["--instance", { instance: "prod" }], + ["--app and --instance", { app: "app_1", instance: "prod" }], + ])("%s names the instance, so nothing is asked", async (_label, options) => { + stubResolved("my-app (production)"); + + const source = await resolveClerkSource(options); + + expect(source).toEqual({ secretKey: "sk_test_resolved", target: "my-app (production)" }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + // `resolveBapiSecretKey` puts an exported key above the linked profile for + // every other command in this family. Opening a picker here — and then + // overriding the key with whatever it returned — made this the one command + // where exporting CLERK_SECRET_KEY did less than not exporting it. + test("an exported CLERK_SECRET_KEY is not second-guessed in a linked directory", async () => { + process.env.CLERK_SECRET_KEY = "sk_test_from_env"; + stubResolved("my-app (development)", "sk_test_from_env"); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_test_from_env", target: "my-app (development)" }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + // Exporting the instance that is about to be imported *into* is the failure + // this whole module exists to prevent, so a resolved instance is offered as + // one choice among the account's applications rather than taken silently. + test("offers every instance, flat, with the linked application's first", async () => { + stubResolved("my-app (development)"); + mockResolveProfile.mockResolvedValue({ profile: { appId: "app_2" } }); + mockFetchApps.mockResolvedValue([ + { + application_id: "app_1", + name: "my-app", + instances: [ + { instance_id: "ins_1d", environment_type: "development" }, + { instance_id: "ins_1p", environment_type: "production" }, + ], + }, + { + application_id: "app_2", + name: "other-app", + instances: [{ instance_id: "ins_2d", environment_type: "development" }], + }, + ]); + mockSearch.mockResolvedValue({ app: "app_1", instance: "ins_1p" }); + mockResolveUsersInstanceContext.mockResolvedValue({ + secretKey: "sk_live_other", + appLabel: "my-app", + instanceLabel: "production", + }); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_live_other", target: "my-app (production)" }); + // One row per instance, not per application: dev and prod are different + // user pools, and exporting the wrong one is silent. + const { message, source: listSource } = mockSearch.mock.calls[0]![0]; + expect(message).toBe("What Clerk instance do you want to export users from?"); + expect(listSource("")).toEqual([ + { + name: "other-app - Development instance (ins_2d)", + value: { app: "app_2", instance: "ins_2d" }, + }, + { + name: "my-app - Development instance (ins_1d)", + value: { app: "app_1", instance: "ins_1d" }, + }, + { + name: "my-app - Production instance (ins_1p)", + value: { app: "app_1", instance: "ins_1p" }, + }, + ]); + // Both halves are handed on, so the secret-key lookup runs against exactly + // the instance that was chosen and nothing prompts a second time. + expect(mockResolveUsersInstanceContext).toHaveBeenCalledWith({ + app: "app_1", + instance: "ins_1p", + }); + expect(captured.err).toBe(""); + }); + + // The list is searched by its rendered label, so an application id typed from + // a dashboard URL still finds its instances. + test("filters on the rendered label", async () => { + stubResolved("my-app (development)"); + mockFetchApps.mockResolvedValue([ + { + application_id: "app_1", + name: "my-app", + instances: [{ instance_id: "ins_1p", environment_type: "production" }], + }, + { + application_id: "app_2", + name: "other-app", + instances: [{ instance_id: "ins_2d", environment_type: "development" }], + }, + ]); + mockSearch.mockResolvedValue({ app: "app_1", instance: "ins_1p" }); + mockResolveUsersInstanceContext.mockResolvedValue({ secretKey: "sk_live_other" }); + + await resolveClerkSource({}); + + const { source: listSource } = mockSearch.mock.calls[0]![0]; + expect(listSource("ins_2d")).toEqual([ + { + name: "other-app - Development instance (ins_2d)", + value: { app: "app_2", instance: "ins_2d" }, + }, + ]); + expect(listSource("production")).toHaveLength(1); + }); + + // An empty list is not a picker. PLAPI being degraded looks the same as an + // account with no applications, and neither one has an instance to offer. + test("no instances to offer falls back to the flags", async () => { + stubResolved("my-app (development)"); + // An application with no instances is not an offer either. + mockFetchApps.mockResolvedValue([{ application_id: "app_1", name: "my-app", instances: [] }]); + + await expect(resolveClerkSource({})).rejects.toBeInstanceOf(UserAbortError); + + expect(mockSearch).not.toHaveBeenCalled(); + expect(captured.err).toContain("--secret-key"); + expect(captured.err).toContain("--app"); + expect(captured.err).toContain("clerk link"); + }); + + test("agent mode takes the resolved instance without prompting", async () => { + human = false; + stubResolved("my-app (production)"); + + const source = await resolveClerkSource({}); + + expect(source.secretKey).toBe("sk_test_resolved"); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("an unlinked directory picks an application instead of failing", async () => { + mockDescribeBapiTarget.mockRejectedValue( + new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }), + ); + mockResolveUsersInstanceContext.mockResolvedValue({ + secretKey: "sk_test_picked", + appLabel: "other-app", + instanceLabel: "production", + }); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_test_picked", target: "other-app (production)" }); + // The picker just asked which application; asking again is noise. + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("an explicit --app that fails to resolve surfaces the error, not the picker", async () => { + const failure = new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }); + mockDescribeBapiTarget.mockRejectedValue(failure); + + await expect(resolveClerkSource({ app: "app_123" })).rejects.toThrow(failure); + + expect(mockResolveUsersInstanceContext).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.ts new file mode 100644 index 000000000..78dfa201c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.ts @@ -0,0 +1,188 @@ +/** + * Which Clerk instance `migrate export clerk` reads *from*. + * + * Every other resolver in the CLI answers "where do I operate?" with the linked + * project, silently. For an export that default is actively dangerous: the + * linked instance is normally the migration's *destination*, so taking it + * without asking is how a run ends up exporting an instance and importing it + * straight back into itself. + * + * So the source is resolved in three tiers: + * + * 1. The user named the instance — `--secret-key`, `--app`, `--instance`, or + * an exported `CLERK_SECRET_KEY` — and it runs unquestioned. An exported + * key outranks the linked profile everywhere else in this CLI + * (`resolveBapiSecretKey`), and this command is not the one place that + * should differ. + * 2. Anything the CLI resolved on the user's behalf (the linked project, a + * keyless app) is not taken silently: every instance on the account is + * offered, with the resolved application's instances first so "yes, that + * one" is still a single Enter. + * 3. Nothing to resolve at all — no link, no key, no flags — falls back to the + * application picker `users list` uses, rather than failing on an unlinked + * directory. That one is `clerk link`'s, so it also offers "create a new + * application"; an empty application is never the right source, but the + * alternative here is an error, not a better list. + */ + +import { fetchAppsTolerantly } from "../../../lib/app-picker.ts"; +import { describeBapiTarget, resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; +import { resolveProfile } from "../../../lib/config.ts"; +import { CliError, ERROR_CODE, throwUserAbort } from "../../../lib/errors.ts"; +import { search } from "../../../lib/listage.ts"; +import type { ApplicationInstance } from "../../../lib/plapi.ts"; +import { log } from "../../../lib/log.ts"; +import { isHuman } from "../../../mode.ts"; +import { resolveUsersInstanceContext } from "../../users/interactive/instance-context.ts"; + +/** e.g. `Development instance`. Unknown environment types print as-is. */ +function instanceLabel(instance: ApplicationInstance): string { + const type = instance.environment_type; + if (!type) return "instance"; + return `${type.charAt(0).toUpperCase()}${type.slice(1)} instance`; +} + +export type ResolveClerkSourceOptions = { + secretKey?: string; + app?: string; + instance?: string; + cwd?: string; +}; + +export type ClerkExportSource = { + secretKey: string; + /** + * Human-readable target, e.g. `my-app (production)`. Absent when + * `--secret-key` (or `CLERK_SECRET_KEY`) named the instance directly, since + * a bare key carries no application context to describe. + */ + target?: string; +}; + +/** {@link ClerkExportSource} plus whether the user already chose it out loud. */ +type ResolvedSource = ClerkExportSource & { chosen: boolean }; + +async function resolveSource(options: ResolveClerkSourceOptions): Promise { + // What separates the two tiers is not "did the CLI have to look anything up" + // but "did the user say which instance". A flag or an exported key is a + // sentence they typed for this run; the linked project is a choice they made + // for some other purpose, possibly months ago, and normally names the + // migration's destination rather than its source. Only the second is worth + // asking about. + // + // `--instance` counts on its own: paired with the linked application it + // names one instance, and pairing it with `--app` addresses any instance on + // the account. Without this an export could not be scripted at all outside + // agent mode — every run would stop at a picker no flag could answer. + const named = + Boolean(options.secretKey) || + Boolean(options.app) || + Boolean(options.instance) || + Boolean(process.env.CLERK_SECRET_KEY); + + try { + return { + target: await describeBapiTarget(options), + secretKey: await resolveBapiSecretKey(options), + chosen: named, + }; + } catch (error) { + if ( + !isHuman() || + named || + !(error instanceof CliError) || + error.code !== ERROR_CODE.NO_SECRET_KEY + ) { + throw error; + } + + const ctx = await resolveUsersInstanceContext({}); + return { + secretKey: ctx.secretKey, + target: ctx.appLabel ? `${ctx.appLabel} (${ctx.instanceLabel})` : undefined, + // The picker just asked. Confirming the answer to a question the user + // answered one prompt ago is noise. + chosen: true, + }; + } +} + +/** + * Offers every instance on the account, flat — one row per instance rather than + * an application picker followed by an instance picker. + * + * An application is not what an export reads from; an instance is. Picking + * "Migration Test" and then "development" is two questions with one answer, and + * it hides the thing that actually matters — dev and prod are different user + * pools, and exporting the wrong one is silent. + * + * Deliberately not `pickOrCreateApp`: its "+ Create a new application" choice + * makes sense when you are choosing somewhere to *write*, and no sense at all + * as an export source — a brand-new application has no users in it. + * + * @param currentAppId the application the CLI resolved on the user's behalf. + * Its instances lead the list, because they are the likeliest answer. + * @returns undefined when there is nothing to offer, so the caller can fall + * back to telling the user which flags to pass instead of showing an empty + * list. `fetchAppsTolerantly` returns empty on a degraded PLAPI, not just on + * an account with no applications. + */ +async function pickInstance(currentAppId?: string): Promise { + const apps = await fetchAppsTolerantly(); + + const ordered = currentAppId + ? [ + ...apps.filter((app) => app.application_id === currentAppId), + ...apps.filter((app) => app.application_id !== currentAppId), + ] + : apps; + + const choices = ordered.flatMap((app) => + (app.instances ?? []).map((instance) => ({ + name: `${app.name || app.application_id} - ${instanceLabel(instance)} (${instance.instance_id})`, + value: { app: app.application_id, instance: instance.instance_id }, + })), + ); + if (choices.length === 0) return undefined; + + const picked = await search<{ app: string; instance: string }>({ + message: "What Clerk instance do you want to export users from?", + source: (term) => + term + ? choices.filter((choice) => choice.name.toLowerCase().includes(term.toLowerCase())) + : choices, + }); + + // Both halves are passed on, so the secret-key lookup runs against exactly + // the instance that was chosen and nothing prompts a second time. + const ctx = await resolveUsersInstanceContext(picked); + return { + secretKey: ctx.secretKey, + target: ctx.appLabel ? `${ctx.appLabel} (${ctx.instanceLabel})` : undefined, + }; +} + +/** The application the CLI resolved on the user's behalf, if it knows one. */ +async function currentAppId(options: ResolveClerkSourceOptions): Promise { + if (options.app) return options.app; + const resolved = await resolveProfile(options.cwd ?? process.cwd()).catch(() => undefined); + return resolved?.profile.appId; +} + +export async function resolveClerkSource( + options: ResolveClerkSourceOptions, +): Promise { + const { chosen, ...source } = await resolveSource(options); + if (chosen || !source.target || !isHuman()) return source; + + const picked = await pickInstance(await currentAppId(options)); + if (picked) return picked; + + log.info( + "Export from a different instance with one of:\n" + + " `--secret-key ` — the source instance's secret key\n" + + " `--app --instance ` — another application on your account\n" + + " `clerk link` — link this directory to a different application first", + ); + throwUserAbort(); +} diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts new file mode 100644 index 000000000..992b78841 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -0,0 +1,375 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { UserLine } from "../lib/run-store.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { + buildClerkExport, + exportClerk, + fetchAllClerkUsers, + mapClerkUserToExport, +} from "./clerk.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: string[]; + +let originalMode: string | undefined; + +beforeAll(() => { + originalMode = process.env.CLERK_MODE; + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-expclerk-"))); + process.chdir(workDir); +}); + +afterAll(() => { + if (originalMode === undefined) delete process.env.CLERK_MODE; + else process.env.CLERK_MODE = originalMode; + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a + // leaked "human" from an earlier test stops a later one on the destination prompt. + process.env.CLERK_MODE = "agent"; + requests = []; + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +/** Answers `GET /v1/users` from `pages`, one page per call. */ +function stubPages(pages: unknown[][]) { + let call = 0; + globalThis.fetch = (async (input: string | URL | Request) => { + // The export names the instance it reads before paging through it. + if (new URL(input.toString()).pathname === "/v1/instance") { + return Response.json({ object: "instance", id: "ins_src", environment_type: "production" }); + } + requests.push(input.toString()); + return Response.json(pages[call++] ?? []); + }) as unknown as typeof fetch; +} + +const user = (overrides: Record = {}) => ({ + id: "user_1", + primary_email_address_id: "idn_1", + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + ], + phone_numbers: [], + ...overrides, +}); + +describe("mapClerkUserToExport", () => { + test("writes the field names the clerk transformer reads", () => { + expect( + mapClerkUserToExport(user({ first_name: "Ada", last_name: "L", username: "ada" })), + ).toMatchObject({ + id: "user_1", + primary_email_address: "a@x.dev", + first_name: "Ada", + last_name: "L", + username: "ada", + }); + }); + + // `migrate import` puts the first entry on POST /v1/users and attaches the rest + // afterwards, so a reordered list would change which address signs the user in. + test("keeps the primary identifier out of the additional list", () => { + const mapped = mapClerkUserToExport( + user({ + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "b@x.dev", verification: { status: "verified" } }, + ], + }), + ); + expect(mapped.primary_email_address).toBe("a@x.dev"); + expect(mapped.verified_email_addresses).toEqual(["b@x.dev"]); + }); + + test("separates unverified identifiers", () => { + const mapped = mapClerkUserToExport( + user({ + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "c@x.dev", verification: { status: "unverified" } }, + ], + }), + ); + expect(mapped.unverified_email_addresses).toEqual(["c@x.dev"]); + expect(mapped.verified_email_addresses).toBeUndefined(); + }); + + // Verify-at-sign-up off lets a primary stay unverified. Exported as primary, + // the import would create it verified. + test("keeps an unverified primary unverified, and promotes a verified one", () => { + const mapped = mapClerkUserToExport( + user({ + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "unverified" } }, + { id: "idn_2", email_address: "b@x.dev", verification: { status: "verified" } }, + ], + }), + ); + expect(mapped.primary_email_address).toBe("b@x.dev"); + expect(mapped.unverified_email_addresses).toEqual(["a@x.dev"]); + }); + + // The import puts the old Clerk ID in external_id, so the app's own value + // would otherwise be lost. + test("moves external_id into private metadata, keeping what is there", () => { + const mapped = mapClerkUserToExport( + user({ external_id: "acct_9", private_metadata: { tier: "gold" } }), + ); + expect(mapped.private_metadata).toEqual({ tier: "gold", clerkExternalId: "acct_9" }); + }); + + test("promotes the first verified address when none is flagged primary", () => { + const mapped = mapClerkUserToExport( + user({ + primary_email_address_id: null, + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "b@x.dev", verification: { status: "verified" } }, + ], + }), + ); + expect(mapped.primary_email_address).toBe("a@x.dev"); + expect(mapped.verified_email_addresses).toEqual(["b@x.dev"]); + }); + + test("maps phone numbers the same way", () => { + const mapped = mapClerkUserToExport( + user({ + primary_phone_number_id: "pn_1", + phone_numbers: [ + { id: "pn_1", phone_number: "+15555550100", verification: { status: "verified" } }, + { id: "pn_2", phone_number: "+15555550101", verification: { status: "unverified" } }, + ], + }), + ); + expect(mapped.primary_phone_number).toBe("+15555550100"); + expect(mapped.unverified_phone_numbers).toEqual(["+15555550101"]); + }); + + test("converts BAPI's Unix-millisecond timestamps to RFC3339", () => { + const mapped = mapClerkUserToExport(user({ created_at: 1704067200000 })); + expect(mapped.created_at).toBe("2024-01-01T00:00:00.000Z"); + }); + + test("omits empty metadata rather than writing empty objects", () => { + const mapped = mapClerkUserToExport( + user({ public_metadata: {}, private_metadata: { plan: "pro" } }), + ); + expect("public_metadata" in mapped).toBe(false); + expect(mapped.private_metadata).toEqual({ plan: "pro" }); + }); + + test("carries the account-state fields the import accepts", () => { + const mapped = mapClerkUserToExport( + user({ + banned: true, + create_organization_enabled: false, + create_organizations_limit: 3, + delete_self_enabled: true, + }), + ); + expect(mapped).toMatchObject({ + banned: true, + create_organization_enabled: false, + create_organizations_limit: 3, + delete_self_enabled: true, + }); + }); +}); + +describe("fetchAllClerkUsers", () => { + test("pages until a short page arrives", async () => { + stubPages([ + Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), + Array.from({ length: 12 }, (_, i) => user({ id: `v${i}` })), + ]); + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all).toHaveLength(512); + expect(requests).toHaveLength(2); + expect(requests[1]).toContain("offset=500"); + }); + + // Oldest first, so a sign-up mid-export lands at the end; a row a deletion + // shifts onto the next page is dropped as a repeat. + test("pages oldest first and drops a user a shifted page repeats", async () => { + stubPages([ + Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), + [user({ id: "u499" }), user({ id: "v0" })], + ]); + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all).toHaveLength(501); + expect(requests[0]).toContain("order_by=%2Bcreated_at"); + }); + + // A full final page must still trigger one more request, or an instance whose + // size is an exact multiple of the page size would look short by one page. + test("makes one more request when the last page is exactly full", async () => { + stubPages([Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), []]); + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all).toHaveLength(500); + expect(requests).toHaveLength(2); + }); + + test("asks for BAPI's maximum page size", async () => { + stubPages([[]]); + await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + expect(requests[0]).toContain("limit=500"); + }); + + test("copes with an instance that has no users", async () => { + stubPages([[]]); + expect(await fetchAllClerkUsers({ secretKey: "sk_test_x" })).toEqual([]); + }); +}); + +describe("buildClerkExport", () => { + test("counts coverage per field", () => { + const { coverage } = buildClerkExport([ + user({ id: "u1", first_name: "Ada", password_enabled: true }), + user({ id: "u2" }), + ]); + + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(2); + expect(byLabel["have a first name"]).toBe(1); + expect(byLabel["have a password (not exportable — see below)"]).toBe(1); + }); + + test("records one line per exported user", () => { + const lines: UserLine[] = []; + buildClerkExport([user({ id: "u1" }), user({ id: "u2" })], (line) => lines.push(line)); + + expect(lines).toEqual([ + { sourceId: "u1", status: "exported" }, + { sourceId: "u2", status: "exported" }, + ]); + }); +}); + +/** The envelope the one export run in this project wrote. */ +function onlyExportFile(): string { + const dir = path.join(workDir, ".clerk", "migrate"); + const entries = fs.readdirSync(dir); + expect(entries).toHaveLength(1); + return path.join(dir, entries[0] as string, "export.json"); +} + +/** The users inside that envelope. */ +function exportedUsers(): Record[] { + return ( + JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as { users: Record[] } + ).users; +} + +describe("exportClerk", () => { + test("writes the default path and reports coverage", async () => { + stubPages([[user({ id: "u1", first_name: "Ada" })], []]); + + await exportClerk({ secretKey: "sk_test_x" }); + expect(JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8"))).toMatchObject({ + source: "clerk", + }); + const written = exportedUsers(); + expect(written).toHaveLength(1); + expect(written[0]?.id).toBe("u1"); + expect(captured.err).toContain("Field coverage"); + expect(captured.err).toContain("Exported 1 user"); + }); + + test("names the command that consumes the file", async () => { + stubPages([[user()], []]); + // The suggestion now rides the gutter's Next steps block, which only + // renders in human mode. + process.env.CLERK_MODE = "human"; + await exportClerk({ secretKey: "sk_test_x", output: "exports/mine.json" }); + expect(captured.err).toMatch(/clerk migrate import \d{8}-\d{6}-[0-9a-f]{4}/); + }); + + test("names the instance it reads from first, and records it on the run", async () => { + stubPages([[user()], []]); + + await exportClerk({ secretKey: "sk_test_x" }); + + expect(Bun.stripANSI(captured.err)).toContain("Source: Clerk, production instance ins_src"); + expect( + JSON.parse(fs.readFileSync(onlyExportFile().replace("export.json", "run.json"), "utf-8")), + ).toMatchObject({ + target: { + platform: "clerk", + instanceId: "ins_src", + }, + }); + }); + + test("--output controls the destination, relative to the working directory", async () => { + stubPages([[user()], []]); + + await exportClerk({ secretKey: "sk_test_x", output: "somewhere/mine.json" }); + + expect(fs.existsSync(path.join(workDir, "somewhere", "mine.json"))).toBe(true); + expect(fs.existsSync(path.join(workDir, "exports"))).toBe(false); + }); + + // Silence here would be the worst outcome: the operator finds out when + // nobody can sign in to the destination instance. + test("says plainly that passwords are not in the file", async () => { + stubPages([[user({ password_enabled: true })], []]); + await exportClerk({ secretKey: "sk_test_x" }); + expect(captured.err).toContain("never returns password digests"); + }); + + test("writes an empty file and says so when the instance has no users", async () => { + stubPages([[]]); + + await exportClerk({ secretKey: "sk_test_x" }); + + expect(captured.err).toContain("No users found to export"); + expect(exportedUsers()).toEqual([]); + }); + + test("an empty export warns but does not suggest importing it", async () => { + stubPages([[]]); + process.env.CLERK_MODE = "human"; + await exportClerk({ secretKey: "sk_test_x", output: "exports/mine.json" }); + + expect(captured.err).toContain("No users found to export"); + expect(captured.err).not.toContain("Next steps"); + expect(captured.err).not.toContain("Import them with"); + }); + + test("agent mode suppresses the Next steps block", async () => { + stubPages([[user()], []]); + + await exportClerk({ secretKey: "sk_test_x" }); + + expect(captured.err).toContain("Exported 1 user"); + expect(captured.err).not.toContain("Next steps"); + // The import command still prints, where an agent can read it. + expect(captured.err).toContain("Import them with"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts new file mode 100644 index 000000000..58945c1d9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -0,0 +1,284 @@ +/** + * `clerk migrate export clerk` — pull users out of a Clerk instance. + * + * Ported from the standalone migration-tool's `src/export/clerk.ts`, rewritten + * onto `bapiRequest` instead of `@clerk/backend` so it shares the CLI's auth + * resolution, `--verbose` request tracing and error taxonomy. + * + * The output feeds `clerk migrate import --transformer clerk` unedited, which is + * what makes development → production a two-command operation. + * + * **Passwords do not come out of this endpoint.** Clerk never returns password + * digests, TOTP secrets or backup codes over the API; only the `*_enabled` + * booleans. The coverage report says how many users *have* a password so the + * gap is visible before the import, not after. + */ + +import { bapiRequest } from "../../../lib/bapi.ts"; +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import type { UserLine } from "../lib/run-store.ts"; +import { retryOn429 } from "../lib/retry.ts"; +import { fetchInstanceIdentity, printTarget } from "../lib/target.ts"; +import { resolveClerkSource } from "./clerk-source.ts"; +import { finishExport, startExportRun } from "./shared.ts"; + +/** BAPI's maximum page size for `GET /v1/users`. */ +const PAGE_SIZE = 500; + +export type ExportClerkOptions = { + output?: string; + secretKey?: string; + app?: string; + instance?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; + /** Print the result as JSON on stdout; never prompts. */ + json?: boolean; +}; + +type BapiIdentifier = { + email_address?: string; + phone_number?: string; + verification?: { status?: string } | null; +}; + +type BapiUser = { + id: string; + external_id?: string | null; + username?: string | null; + first_name?: string | null; + last_name?: string | null; + email_addresses?: BapiIdentifier[]; + phone_numbers?: BapiIdentifier[]; + primary_email_address_id?: string | null; + primary_phone_number_id?: string | null; + public_metadata?: Record; + private_metadata?: Record; + unsafe_metadata?: Record; + password_enabled?: boolean; + totp_enabled?: boolean; + banned?: boolean; + create_organization_enabled?: boolean; + create_organizations_limit?: number | null; + delete_self_enabled?: boolean; + created_at?: number; + legal_accepted_at?: number | null; +}; + +type IdentifierWithId = BapiIdentifier & { id?: string }; + +/** + * Splits identifiers into verified and unverified, primary first. + * + * The primary has to lead: `migrate import` puts the first entry on + * `POST /v1/users` and attaches the rest afterwards, so a reordered list would + * silently change which address the user signs in with. + */ +function splitIdentifiers( + entries: IdentifierWithId[] | undefined, + primaryId: string | null | undefined, + read: (entry: BapiIdentifier) => string | undefined, +): { primary?: string; verified: string[]; unverified: string[] } { + const verified: string[] = []; + const unverified: string[] = []; + let primary: string | undefined; + + for (const entry of entries ?? []) { + const value = read(entry); + if (!value) continue; + + const isVerified = entry.verification?.status === "verified"; + // An unverified primary stays unverified: the import puts a primary on + // POST /v1/users, which creates it verified. + if (entry.id && entry.id === primaryId && isVerified) { + primary = value; + continue; + } + if (isVerified) verified.push(value); + else unverified.push(value); + } + + // No verified primary: promote the first verified one so the export still + // has an identifier the import can lead with. + if (!primary && verified.length > 0) primary = verified.shift(); + + return { primary, verified, unverified }; +} + +/** Maps a BAPI user onto the shape the `clerk` transformer reads. */ +export function mapClerkUserToExport(user: BapiUser): Record { + const exported: Record = { id: user.id }; + + const emails = splitIdentifiers( + user.email_addresses, + user.primary_email_address_id, + (entry) => entry.email_address, + ); + if (emails.primary) exported.primary_email_address = emails.primary; + if (emails.verified.length > 0) exported.verified_email_addresses = emails.verified; + if (emails.unverified.length > 0) exported.unverified_email_addresses = emails.unverified; + + const phones = splitIdentifiers( + user.phone_numbers, + user.primary_phone_number_id, + (entry) => entry.phone_number, + ); + if (phones.primary) exported.primary_phone_number = phones.primary; + if (phones.verified.length > 0) exported.verified_phone_numbers = phones.verified; + if (phones.unverified.length > 0) exported.unverified_phone_numbers = phones.unverified; + + if (user.username) exported.username = user.username; + if (user.first_name) exported.first_name = user.first_name; + if (user.last_name) exported.last_name = user.last_name; + + for (const [source, target] of [ + ["public_metadata", "public_metadata"], + ["private_metadata", "private_metadata"], + ["unsafe_metadata", "unsafe_metadata"], + ] as const) { + const value = user[source]; + if (value && Object.keys(value).length > 0) exported[target] = value; + } + + // The import sets external_id to the old Clerk ID, so an app's own + // external_id moves to private metadata rather than being lost. + if (user.external_id) { + exported.private_metadata = { + ...(exported.private_metadata as Record | undefined), + clerkExternalId: user.external_id, + }; + } + + if (user.banned) exported.banned = true; + if (user.create_organization_enabled !== undefined) { + exported.create_organization_enabled = user.create_organization_enabled; + } + if (user.create_organizations_limit !== null && user.create_organizations_limit !== undefined) { + exported.create_organizations_limit = user.create_organizations_limit; + } + if (user.delete_self_enabled !== undefined) { + exported.delete_self_enabled = user.delete_self_enabled; + } + + // BAPI reports timestamps as Unix milliseconds; the schema wants RFC3339. + if (user.created_at) exported.created_at = new Date(user.created_at).toISOString(); + if (user.legal_accepted_at) { + exported.legal_accepted_at = new Date(user.legal_accepted_at).toISOString(); + } + + return exported; +} + +/** Pages through every user in the instance. */ +export async function fetchAllClerkUsers(options: { + secretKey: string; + spinner?: SpinnerControls; +}): Promise { + const all = new Map(); + + // Oldest first, so a sign-up during the export lands at the end instead of + // shifting every later page by one (BAPI's default is newest first). A + // deletion can still shift a page; the Map drops the repeat that causes. + // ponytail: offset paging; /v1/users has no cursor to page by instead. + for (let offset = 0; ; offset += PAGE_SIZE) { + const response = await retryOn429(async () => + bapiRequest({ + method: "GET", + path: `/v1/users?limit=${PAGE_SIZE}&offset=${offset}&order_by=%2Bcreated_at`, + secretKey: options.secretKey, + }), + ); + + const page = Array.isArray(response.body) ? (response.body as BapiUser[]) : []; + for (const user of page) all.set(user.id, user); + options.spinner?.update(`Fetching users from Clerk: ${all.size} so far...`); + + // A short page means the end; anything else would loop forever on an + // instance whose size happens to be a multiple of the page size. + if (page.length < PAGE_SIZE) break; + } + + return [...all.values()]; +} + +export type ClerkExportResult = { + users: Record[]; + coverage: { label: string; count: number }[]; +}; + +/** Maps every user and counts what the export actually contains. */ +export function buildClerkExport( + users: BapiUser[], + record: (line: UserLine) => void = () => {}, +): ClerkExportResult { + const exported: Record[] = []; + const counts = { email: 0, username: 0, firstName: 0, lastName: 0, phone: 0, password: 0 }; + + for (const user of users) { + try { + const mapped = mapClerkUserToExport(user); + exported.push(mapped); + + if (mapped.primary_email_address) counts.email++; + if (mapped.username) counts.username++; + if (mapped.first_name) counts.firstName++; + if (mapped.last_name) counts.lastName++; + if (mapped.primary_phone_number) counts.phone++; + if (user.password_enabled) counts.password++; + + record({ sourceId: user.id, status: "exported" }); + } catch (error) { + record({ sourceId: user.id, status: "skipped", error: (error as Error).message }); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a phone number", count: counts.phone }, + { label: "have a username", count: counts.username }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + { label: "have a password (not exportable — see below)", count: counts.password }, + ], + }; +} + +export async function exportClerk(options: ExportClerkOptions): Promise { + // Resolved before the gutter opens, the way `export supabase` resolves its + // connection string: confirming the source is a question about whether to run at + // all, not a step of the run. + const source = await resolveClerkSource({ + secretKey: options.secretKey, + app: options.app, + instance: options.instance, + }); + + await withGutter("Exporting users from Clerk", async () => { + const identity = await fetchInstanceIdentity(source.secretKey); + const target = { + platform: "clerk", + env: identity.env, + instanceId: identity.instanceId, + ...(source.target ? { keySource: source.target } : {}), + }; + if (!options.json) printTarget(target); + + const users = await withSpinner("Fetching users from Clerk...", async (spinner) => + fetchAllClerkUsers({ secretKey: source.secretKey, spinner }), + ); + + const run = await startExportRun(options, target); + const { users: exported, coverage } = buildClerkExport(users, run.append); + finishExport({ run, options, users: exported, coverage }); + + if (exported.length > 0) { + log.warn( + "Clerk's API never returns password digests, TOTP secrets or backup codes, so they are not in this file. " + + "Users will need to reset their password in the destination instance.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts new file mode 100644 index 000000000..138aca86b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -0,0 +1,187 @@ +/** + * The database-backed exports: resolving the connection string, the Supabase + * export's row mapping, and connection failures against a real SQLite file. + * + * SQLite because it is the one engine that needs no container, and it + * exercises the same client. Postgres and MySQL are covered by the manual + * matrix run recorded in the ticket. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import type { UserLine } from "../lib/run-store.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { buildSupabaseExport, exportSupabase } from "./supabase.ts"; +import { + looksLikeConnectionString, + normalizeConnectionString, + resolveDbUrl, +} from "./db-options.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-dbexp-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); +}); + +describe("looksLikeConnectionString", () => { + test.each([ + ["postgres://u:p@h:5432/db", true], + ["mysql://u:p@h:3306/db", true], + ["./db.sqlite", true], + ["file:./db.sqlite", true], + ["/abs/app.db", true], + ["", false], + [" ", false], + ["just some words", false], + ["postgres://", false], + ])("%p -> %p", (input, expected) => { + expect(looksLikeConnectionString(input)).toBe(expected); + }); +}); + +describe("normalizeConnectionString", () => { + test("encodes a password pasted in raw", () => { + const raw = "postgres://postgres:aB#c%92^d@db.example.supabase.co:5432/postgres"; + const normalized = normalizeConnectionString(raw); + + expect(looksLikeConnectionString(normalized)).toBe(true); + expect(decodeURIComponent(new URL(normalized).password)).toBe("aB#c%92^d"); + expect(new URL(normalized).hostname).toBe("db.example.supabase.co"); + }); + + test("encodes an unencoded @ in the password", () => { + const normalized = normalizeConnectionString("postgres://u:p@ss@host:5432/db"); + + expect(decodeURIComponent(new URL(normalized).password)).toBe("p@ss"); + expect(new URL(normalized).hostname).toBe("host"); + }); + + // A bare `%` parses as a URL but fails the driver's decode with "URI error". + test("encodes a bare % in a password", () => { + const normalized = normalizeConnectionString("postgres://u:50%off@host:5432/db"); + + expect(decodeURIComponent(new URL(normalized).password)).toBe("50%off"); + }); + + test("leaves an already-valid string alone", () => { + const encoded = "postgres://u:p%40ss@host:5432/db"; + expect(normalizeConnectionString(encoded)).toBe(encoded); + }); + + test("leaves non-URL forms alone", () => { + expect(normalizeConnectionString(" ./db.sqlite ")).toBe("./db.sqlite"); + }); +}); + +describe("resolveDbUrl", () => { + const config = { platform: "supabase" as const, envVar: "SUPABASE_DB_URL", prompt: "url" }; + + test("prefers the flag", async () => { + const url = await resolveDbUrl({ dbUrl: "postgres://u:p@h/db" }, config, { + SUPABASE_DB_URL: "mysql://u:p@h/db", + }); + expect(url).toBe("postgres://u:p@h/db"); + }); + + test("falls back to the environment variable", async () => { + expect(await resolveDbUrl({}, config, { SUPABASE_DB_URL: "mysql://u:p@h/db" })).toBe( + "mysql://u:p@h/db", + ); + }); + + test("encodes a raw password passed to the flag", async () => { + const url = await resolveDbUrl({ dbUrl: "postgres://u:p#ss@host:5432/db" }, config, {}); + expect(decodeURIComponent(new URL(url).password)).toBe("p#ss"); + }); + + test("rejects a flag that is not a connection string, naming the encoding trap", async () => { + await expect(resolveDbUrl({ dbUrl: "not a url" }, config, {})).rejects.toThrow(/URL-encode it/); + }); + + test("warns and moves on when the environment variable is unusable", async () => { + // Tests run non-TTY, so it then hits the agent-mode branch. + await expect(resolveDbUrl({}, config, { SUPABASE_DB_URL: "garbage" })).rejects.toThrow( + /cannot prompt here/, + ); + expect(captured.err).toContain("SUPABASE_DB_URL is not a valid connection string"); + }); + + test("names both the flag and the variable when it cannot prompt", async () => { + await expect(resolveDbUrl({}, config, {})).rejects.toThrow(/--db-url.*SUPABASE_DB_URL/s); + }); +}); + +describe("supabase export", () => { + test("serializes timestamps the transformer can parse", () => { + const { users } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", created_at: new Date("2024-01-01T00:00:00Z") }, + ]); + expect(users[0]?.created_at).toBe("2024-01-01T00:00:00.000Z"); + }); + + test("omits null columns rather than exporting them", () => { + const { users } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", phone: null, last_name: null }, + ]); + expect("phone" in (users[0] ?? {})).toBe(false); + expect("last_name" in (users[0] ?? {})).toBe(false); + }); + + test("counts the password hashes, the reason this reads the database", () => { + const { coverage } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", encrypted_password: "$2b$10$x" }, + { id: "u2", email: "b@x.dev" }, + ]); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have a password hash"]).toBe(1); + }); + + test("keeps raw_app_meta_data, which the import checks read for providers", () => { + const { users } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", raw_app_meta_data: { providers: ["discord"] } }, + ]); + expect(users[0]?.raw_app_meta_data).toEqual({ providers: ["discord"] }); + }); + + test("records one line per exported user", () => { + const lines: UserLine[] = []; + buildSupabaseExport([{ id: "u1" }, { id: "u2" }], (line) => lines.push(line)); + + expect(lines).toHaveLength(2); + }); +}); + +describe("connection failures", () => { + test("a missing SQLite file fails before a run is written", async () => { + await expect(exportSupabase({ dbUrl: "./definitely-not-here.sqlite" })).rejects.toThrow( + CliError, + ); + expect(fs.existsSync(path.join(workDir, ".clerk"))).toBe(false); + }); + + test("the failure never contains the password", async () => { + await expect( + exportSupabase({ dbUrl: "postgres://user:hunter2@127.0.0.1:1/db" }), + ).rejects.toThrow( + expect.objectContaining({ message: expect.not.stringContaining("hunter2") }) as Error, + ); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts new file mode 100644 index 000000000..3c0ea5a4c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -0,0 +1,171 @@ +/** + * Resolving a `--db-url` for the database-backed exports. + * + * Shared by the database-backed exports: each takes one connection string, + * from a flag, an environment variable, or a prompt. + */ + +import { throwUsageError } from "../../../lib/errors.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { password as passwordPrompt } from "../../../lib/prompts.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { detectDbType, isLibsqlUrl, redactConnectionString, type DbPlatform } from "../lib/db.ts"; + +export type DbExportOptions = { + dbUrl?: string; + output?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; + /** Print the result as JSON on stdout; never prompts. */ + json?: boolean; +}; + +export type ResolveConfig = { + platform: DbPlatform; + /** Environment variable checked when `--db-url` is absent. */ + envVar: string; + prompt: string; + /** Extra guidance shown before prompting. */ + hint?: string; +}; + +const URL_SCHEME = /^(postgresql|postgres|mysql|mysql2|libsql):\/\//i; + +/** + * True when the string parses as a URL with a host. + * + * A hostname is required: `postgres://` alone parses as a valid URL, and + * accepting it only defers the failure into the driver. + */ +function parsesAsUrl(value: string): boolean { + try { + const url = new URL(value); + // A bare `%` in the password parses, then fails the driver's decode with a + // cryptic "URI error"; treat it as unparsed so it gets encoded. + decodeURIComponent(url.username); + decodeURIComponent(url.password); + return url.hostname.length > 0; + } catch { + return false; + } +} + +/** + * Percent-encodes the credentials when the raw string will not parse as a URL. + * + * Dashboards hand out `postgres://user:[YOUR-PASSWORD]@host/db` and people + * paste their real password in verbatim. A `#`, `@`, `/` or `^` in it makes the + * whole string unparseable — here and later inside `Bun.SQL` — so encode it for + * them rather than bouncing a paste they cannot even see (the prompt is + * masked). Strings that already parse are returned untouched, so a password + * that was correctly encoded is never double-encoded. + */ +export function normalizeConnectionString(value: string): string { + const trimmed = value.trim(); + if (!URL_SCHEME.test(trimmed) || parsesAsUrl(trimmed)) return trimmed; + + // Greedy up to the LAST `@`: everything before it is userinfo, so an + // unencoded `@` inside the password does not split the string early. + const match = /^([a-z0-9+]+:\/\/)(.*)@([^@]*)$/i.exec(trimmed); + if (!match) return trimmed; + + const [, scheme = "", userinfo = "", rest = ""] = match; + const separator = userinfo.indexOf(":"); + const user = separator === -1 ? userinfo : userinfo.slice(0, separator); + const secret = separator === -1 ? undefined : userinfo.slice(separator + 1); + const credentials = + secret === undefined + ? encodeURIComponent(user) + : `${encodeURIComponent(user)}:${encodeURIComponent(secret)}`; + + const encoded = `${scheme}${credentials}@${rest}`; + return parsesAsUrl(encoded) ? encoded : trimmed; +} + +/** True for something that could plausibly be a connection string. */ +export function looksLikeConnectionString(value: string): boolean { + const trimmed = value.trim(); + if (!trimmed) return false; + + if (URL_SCHEME.test(trimmed)) return parsesAsUrl(trimmed); + + return ( + trimmed.startsWith("file:") || /\.(sqlite3?|db)$/i.test(trimmed) || trimmed.startsWith("./") + ); +} + +/** + * Resolves the connection string: flag, then environment, then a prompt. + * + * Prompted as a password so it is not echoed — a connection string carries the + * database password inline. + */ +export async function resolveDbUrl( + options: DbExportOptions, + config: ResolveConfig, + env: Record = process.env, +): Promise { + const fromFlag = options.dbUrl ? normalizeConnectionString(options.dbUrl) : undefined; + if (fromFlag) { + if (!looksLikeConnectionString(fromFlag)) { + throwUsageError( + `--db-url does not look like a connection string. Expected postgres://…, mysql://…, libsql://… or a SQLite file path.\n` + + "If the password contains @, # or /, URL-encode it.", + ); + } + return fromFlag; + } + + const raw = env[config.envVar]; + const fromEnv = raw ? normalizeConnectionString(raw) : undefined; + if (fromEnv) { + if (looksLikeConnectionString(fromEnv)) return fromEnv; + // Falling through silently would make the prompt look unexplained. + log.warn(`${config.envVar} is not a valid connection string; ignoring it.`); + } + + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export ${config.platform}\` needs a database connection and cannot prompt here.\n` + + `Pass --db-url, or set ${config.envVar}.`, + undefined, + undefined, + [ + { + command: `clerk migrate export ${config.platform} --db-url "postgres://user:password@host:5432/db"`, + description: "Export from Postgres", + }, + ], + ); + } + + if (config.hint) log.info(dim(config.hint)); + + return promptDbUrl(config); +} + +/** + * Asks for a connection string, masked. + * + * Masked because a connection string carries the database password inline. The + * validator runs on the normalized value, so a password that needed encoding is + * judged as the driver will see it, not as it was typed. + */ +export async function promptDbUrl(config: ResolveConfig): Promise { + const answer = await passwordPrompt({ + message: config.prompt, + validate: (value) => + looksLikeConnectionString(normalizeConnectionString(value ?? "")) + ? undefined + : "Expected postgres://…, mysql://…, libsql://… or a SQLite file path", + }); + + return normalizeConnectionString(answer); +} + +/** Describes the target for the run's opening line, credentials removed. */ +export function describeTarget(connectionString: string): string { + const label = isLibsqlUrl(connectionString) ? "libsql" : detectDbType(connectionString); + return `${label} at ${redactConnectionString(connectionString)}`; +} diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts new file mode 100644 index 000000000..898604c6c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -0,0 +1,135 @@ +import type { Command } from "@commander-js/extra-typings"; +import { throwUsageError } from "../../../lib/errors.ts"; +import { select } from "../../../lib/listage.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { exportClerk } from "./clerk.ts"; +import { exportSupabase } from "./supabase.ts"; +import type { DbExportOptions } from "./db-options.ts"; +import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "../lib/run-store.ts"; +import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./registry.ts"; + +/** + * Bare `clerk migrate export` — pick a platform, then run its export. + * + * The picker is built from the registry, so a new platform appears without a + * second place to update. Whatever the chosen platform needs beyond the + * platform name, it prompts for itself. + */ +export async function exportPicker(options: Record = {}): Promise { + if (isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export\` needs a platform and cannot prompt here. Name one: ${exportPlatformKeys().join(", ")}.`, + undefined, + undefined, + exportPlatforms.map((entry) => ({ + command: `clerk migrate export ${entry.key}`, + description: entry.description, + })), + ); + } + + const platform = await select({ + message: "Which platform are you exporting from?", + choices: exportPlatforms.map((entry) => ({ + name: entry.label, + value: entry.key, + description: entry.description, + })), + }); + + const entry = getExportPlatform(platform); + // Unreachable via the picker; a guard so a registry edit cannot silently + // produce a choice with nothing behind it. + if (!entry) throwUsageError(`Unknown export platform "${platform}".`); + + await entry.run(options); +} + +const handlers = { + picker: exportPicker, + clerk: exportClerk, + supabase: exportSupabase, +}; + +/** The platforms that read a database, which share `--db-url`. */ +const DB_PLATFORMS = [ + { + key: "supabase", + summary: "Export users from a Supabase Postgres database", + envVar: "SUPABASE_DB_URL", + example: "postgres://postgres:password@db.xxx.supabase.co:5432/postgres", + }, +] as const; + +/** Registers `export [platform]` under the `migrate` group. */ +export function registerMigrateExport(migrateCommand: Command<[], Record>): void { + const exportCommand = migrateCommand + .command("export") + .description("Export users from a source platform, ready for `clerk migrate import`") + .setExamples([ + { command: "clerk migrate export", description: "Pick a platform interactively" }, + { + command: "clerk migrate export clerk", + description: "Export from a Clerk instance into a new run", + }, + { + command: "clerk migrate export supabase", + description: "Export from a Supabase database", + }, + ]) + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") + .action(async (_opts, cmd) => + handlers.picker(cmd.optsWithGlobals() as Record), + ); + + exportCommand + .command("clerk") + .description("Export users from a Clerk instance") + .option("-o, --output ", "Write the export here instead of the run folder") + .option("-y, --yes", "Do not prompt: fail on a rejected credential") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") + .option("--secret-key ", "Backend API secret key to use") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .setExamples([ + { + command: "clerk migrate export clerk", + description: "Prompts for the source instance and where to save the file", + }, + { + command: "clerk migrate export clerk --secret-key sk_live_… --output prod-users.json", + description: "Name the source instance outright, skipping the picker", + }, + ]) + .action(async (_opts, cmd) => + handlers.clerk(cmd.optsWithGlobals() as Parameters[0]), + ); + + // Each takes exactly one connection string, so they are registered from a + // table rather than near-identical blocks. + for (const platform of DB_PLATFORMS) { + exportCommand + .command(platform.key) + .description(platform.summary) + .option("--db-url ", "Postgres, MySQL, libsql/Turso or SQLite connection string") + .option("-o, --output ", "Write the export here instead of the run folder") + .option("-y, --yes", "Do not prompt: fail on a rejected credential") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") + .setExamples([ + { + command: `clerk migrate export ${platform.key} --db-url "${platform.example}"`, + description: "Export from an explicit database", + }, + { + command: `clerk migrate export ${platform.key}`, + description: `Read ${platform.envVar}, or prompt`, + }, + ]) + .action(async (_opts, cmd) => + handlers[platform.key](cmd.optsWithGlobals() as DbExportOptions), + ); + } +} diff --git a/packages/cli-core/src/commands/migrate/export/registry.test.ts b/packages/cli-core/src/commands/migrate/export/registry.test.ts new file mode 100644 index 000000000..3131db58e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/registry.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, test } from "bun:test"; +import { sourceKeys } from "../sources/registry.ts"; +import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./registry.ts"; + +describe("export registry", () => { + test("registers every source platform", () => { + expect(exportPlatformKeys()).toEqual(["clerk", "supabase"]); + }); + + test.each([...exportPlatforms])("$key carries a label and description", (entry) => { + expect(entry.label.length).toBeGreaterThan(0); + expect(entry.description.length).toBeGreaterThan(0); + }); + + // The picker, the docs and the "what next" line all read this, so a typo + // would send someone to a transformer that does not exist. + test.each([...exportPlatforms])("$key names a real transformer", (entry) => { + expect(sourceKeys()).toContain(entry.sourceKey); + }); + + test.each([...exportPlatforms])("$key has something to run", (entry) => { + expect(typeof entry.run).toBe("function"); + }); + + test("looks a platform up by key", () => { + expect(getExportPlatform("supabase")?.label).toBe("Supabase"); + }); + + test("returns nothing for a platform that is not registered", () => { + expect(getExportPlatform("okta")).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/registry.ts b/packages/cli-core/src/commands/migrate/export/registry.ts new file mode 100644 index 000000000..57b4b6cd9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/registry.ts @@ -0,0 +1,48 @@ +/** + * Export registry. + * + * The picker behind a bare `clerk migrate export` is built from this array, so + * adding a platform is one file plus one entry — the same shape the source + * registry uses. + * + * `run` takes no arguments on purpose: each platform resolves its own flags, + * environment variables and prompts, because what Auth0 needs (a tenant domain + * and M2M credentials) has nothing in common with what a database export needs. + */ + +import { exportClerk } from "./clerk.ts"; +import { exportSupabase } from "./supabase.ts"; + +export type ExportRegistryEntry = { + key: string; + label: string; + description: string; + /** Which source reads the file this export writes. */ + sourceKey: string; + run: (options: Record) => Promise; +}; + +export const exportPlatforms: ExportRegistryEntry[] = [ + { + key: "clerk", + label: "Clerk", + description: "Another Clerk instance, e.g. development → production", + sourceKey: "clerk", + run: async (options) => exportClerk(options), + }, + { + key: "supabase", + label: "Supabase", + description: "A Supabase Postgres database — includes password hashes", + sourceKey: "supabase", + run: async (options) => exportSupabase(options), + }, +]; + +export function exportPlatformKeys(): string[] { + return exportPlatforms.map((entry) => entry.key); +} + +export function getExportPlatform(key: string): ExportRegistryEntry | undefined { + return exportPlatforms.find((entry) => entry.key === key); +} diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts new file mode 100644 index 000000000..2270948df --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -0,0 +1,124 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { readEnvelope } from "../lib/export-file.ts"; +import { latestUserLines, readRun } from "../lib/run-store.ts"; +import { finishExport, formatImportCommand, startExportRun } from "./shared.ts"; + +const captured = useCaptureLog(); + +let runsDir: string; +let originalCwd: string; + +beforeEach(() => { + originalCwd = process.cwd(); + runsDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-export-shared-"))); + process.chdir(runsDir); +}); + +afterEach(() => { + process.chdir(originalCwd); + fs.rmSync(runsDir, { recursive: true, force: true }); +}); + +const users = [{ id: "u1", email: "a@x.dev" }]; +const coverage = [{ label: "have an email address", count: 1 }]; + +describe("finishExport", () => { + test("writes the envelope into the run folder by default", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + run.append({ sourceId: "u1", status: "exported" }); + + const { record, outputPath } = finishExport({ run, options: {}, users, coverage }); + + expect(outputPath).toBe(path.join(runsDir, record.id, "export.json")); + expect(readEnvelope(outputPath)).toMatchObject({ + clerkMigrate: 1, + source: "supabase", + runId: record.id, + users, + }); + expect(readRun(runsDir, record.id)).toMatchObject({ + kind: "export", + status: "complete", + file: { path: outputPath }, + }); + expect(latestUserLines(runsDir, record.id).get("u1")?.status).toBe("exported"); + }); + + // The file holds password hashes and PII, so no other local account reads it. + test("writes the file owner-only, in an owner-only run folder", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + + const { outputPath } = finishExport({ run, options: {}, users, coverage }); + + expect(fs.statSync(outputPath).mode & 0o777).toBe(0o600); + expect(fs.statSync(path.dirname(outputPath)).mode & 0o777).toBe(0o700); + }); + + test("--output writes somewhere else, and the run still records where", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + + const { record, outputPath } = finishExport({ + run, + options: { output: "mine/users.json" }, + users, + coverage, + }); + + expect(outputPath).toBe(path.join(runsDir, "mine", "users.json")); + expect(readRun(runsDir, record.id)?.file?.path).toBe(outputPath); + }); + + test("carries Firebase's hash parameters to the import", async () => { + const run = await startExportRun({ runsDir }, { platform: "firebase" }); + const firebase = { + base64_signer_key: "k", + base64_salt_separator: "s", + rounds: 8, + mem_cost: 14, + }; + + const { outputPath } = finishExport({ run, options: {}, users, coverage, firebase }); + + expect(readEnvelope(outputPath)?.firebase).toEqual(firebase); + }); + + test("prints the import command by run ID", async () => { + const run = await startExportRun({ runsDir }, { platform: "clerk" }); + + const { record } = finishExport({ run, options: {}, users, coverage }); + + expect(captured.err).toContain(`clerk migrate import ${record.id}`); + }); + + test("--json returns the result on stdout instead", async () => { + const run = await startExportRun({ runsDir }, { platform: "clerk" }); + + const { record, outputPath } = finishExport({ run, options: { json: true }, users, coverage }); + + expect(JSON.parse(captured.out)).toMatchObject({ + run: { id: record.id, kind: "export" }, + output: outputPath, + users: 1, + next: `clerk migrate import ${record.id}`, + }); + expect(captured.err).not.toContain("Field coverage"); + }); +}); + +describe("formatImportCommand", () => { + const render = () => Bun.stripANSI(formatImportCommand("20260929-141502-a1b2").join("\n")); + + test("names the export run, which carries its own source", () => { + expect(render()).toContain("clerk migrate import 20260929-141502-a1b2"); + }); + + // The instance comes from the resolved key, so there is no flag whose + // absence means development — the note says what actually decides. + test("says how to reach production", () => { + expect(render()).toContain("--instance prod"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts new file mode 100644 index 000000000..c7c550a23 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -0,0 +1,198 @@ +/** + * Shared plumbing for the export modules: the run that records each user, + * where the file lands, and what the user is told about it. + * + * Every export is a run. The file lands in that run's folder as + * `export.json` unless `--output` names somewhere else, and `--output` + * resolves against the **current working directory**, the way every other + * path flag in this CLI does. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { dim, green, yellow } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { ENVELOPE_VERSION, type ExportEnvelope } from "../lib/export-file.ts"; +import { ACCOUNT_LINKING_NOTE } from "../sources/registry.ts"; +import { + resolveRunsDir, + sha256File, + startRun, + type Run, + type RunRecord, + type RunTarget, +} from "../lib/run-store.ts"; +import type { FirebaseHashConfig } from "../types.ts"; + +/** What every export command takes on top of its own credentials. */ +export type ExportCommonOptions = { + /** Where to write the file, instead of the run folder. */ + output?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; + json?: boolean; +}; + +/** + * Starts the export run that records each user as it is exported. + * + * Started once the users are in hand, so a rejected credential leaves no run + * behind. + */ +export async function startExportRun( + options: ExportCommonOptions, + target: RunTarget, +): Promise { + const runsDir = await resolveRunsDir(options.runsDir, { write: true }); + return startRun(runsDir, { kind: "export", target, source: target.platform }); +} + +/** Where the file lands: `--output`, or `export.json` in the run folder. */ +export function exportPath(run: Run, output: string | undefined): string { + return output ? path.resolve(process.cwd(), output) : path.join(run.dir, "export.json"); +} + +/** + * Writes the envelope, creating any missing parent directories. + * + * @returns The absolute path written. + */ +export function writeExportFile(file: string, envelope: ExportEnvelope): string { + // Password hashes, PII and a Firebase signer key: owner-only. + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); + fs.writeFileSync(file, JSON.stringify(envelope, null, 2), { mode: 0o600 }); + return file; +} + +export type CoverageField = { label: string; count: number }; + +/** + * How complete an export is, per field. + * + * ✓ every user, ! some, dim ✗ none. The point is to see *before* importing + * that, say, only 3 of 400 users have a password — which changes what the + * migration means. + */ +export function formatFieldCoverage(fields: CoverageField[], total: number): string[] { + return fields.map(({ label, count }) => { + const icon = count === total ? green("✓") : count > 0 ? yellow("!") : dim("✗"); + return ` ${icon} ${dim(`${count}/${total} ${label}`)}`; + }); +} + +/** + * An extra block printed under the coverage table. + * + * For a breakdown that is not "how many users have this field" — WorkOS's + * OAuth providers, where one user can appear in two rows and the denominator + * is not the user count. Folding that into the coverage table would put rows + * of two different kinds under one heading. + */ +export type ExportSection = { title: string; rows: string[] }; + +/** + * The import command for an export run, and what it will target. + * + * One command rather than a development and a production variant, because + * there is no flag whose absence means "development": the key decides, through + * `--secret-key`, `--app`, `CLERK_SECRET_KEY`, the keyless project and the + * linked profile in that order. A line labelled "development" would be wrong + * for anyone holding `CLERK_SECRET_KEY=sk_live_…`, which is the reader who can + * least afford it. So the note names what picks the instance instead. + */ +export function formatImportCommand(runId: string): string[] { + return [ + "Import them with:", + dim(` clerk migrate import ${runId}`), + "", + dim(" Imports into whichever instance the resolved secret key belongs to."), + dim(" For production, add `--instance prod` or use a production secret key."), + ]; +} + +export type FinishExportInput = { + run: Run; + options: ExportCommonOptions; + users: Record[]; + coverage: CoverageField[]; + /** Extra blocks, printed under the coverage table in order. */ + sections?: ExportSection[]; + /** Firebase's hash parameters, carried to the import in the envelope. */ + firebase?: FirebaseHashConfig; + /** The platform stopped short of every user; `--json` says so. */ + truncated?: boolean; +}; + +export type FinishedExport = { record: RunRecord; outputPath: string }; + +/** + * Writes the envelope, finishes the run, and reports it. + * + * The import command prints through `log.info` rather than the gutter's + * next-steps outro, which is human-only: this is the one line that says what + * to do with the file, and an agent that cannot see it has to guess. `--json` + * returns the same facts on stdout instead. + */ +export function finishExport(input: FinishExportInput): FinishedExport { + const { run, options, users, coverage } = input; + const platform = run.record.target.platform ?? run.record.source ?? ""; + const outputPath = writeExportFile(exportPath(run, options.output), { + clerkMigrate: ENVELOPE_VERSION, + source: run.record.source ?? platform, + exportedAt: new Date().toISOString(), + runId: run.record.id, + ...(input.firebase ? { firebase: input.firebase } : {}), + users, + }); + run.update({ file: { path: outputPath, sha256: sha256File(outputPath) } }); + const record = run.finish(); + const next = `clerk migrate import ${record.id}`; + + if (options.json) { + log.data( + JSON.stringify( + { + target: record.target, + run: record, + output: outputPath, + users: users.length, + coverage, + ...(input.sections?.length ? { sections: input.sections } : {}), + ...(input.truncated ? { truncated: true } : {}), + next, + }, + null, + 2, + ), + ); + return { record, outputPath }; + } + + log.blank(); + if (users.length === 0) { + log.warn(`No users found to export. Wrote an empty file to ${outputPath}.`); + log.info(dim(`Run ${record.id}`)); + return { record, outputPath }; + } + + log.info("Field coverage"); + for (const line of formatFieldCoverage(coverage, users.length)) log.info(line); + + for (const section of input.sections ?? []) { + log.blank(); + log.info(section.title); + for (const row of section.rows) log.info(row); + } + + log.blank(); + log.success(`Exported ${users.length} user${users.length === 1 ? "" : "s"} to ${outputPath}`); + log.info(dim(`Run ${record.id}. See each user with \`clerk migrate runs ${record.id}\`.`)); + + log.blank(); + log.info(dim(ACCOUNT_LINKING_NOTE)); + + log.blank(); + for (const line of formatImportCommand(record.id)) log.info(line); + + return { record, outputPath }; +} diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts new file mode 100644 index 000000000..abea84f7e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -0,0 +1,150 @@ +/** + * `clerk migrate export supabase` — read users straight out of `auth.users`. + * + * Ported from the standalone migration-tool's `src/export/supabase.ts`, on + * `Bun.sql` instead of `pg`. + * + * The database rather than the Admin API because **`encrypted_password` only + * exists here**. Supabase's API does not return password hashes, so an + * API-based export forces every user to reset their password; this one carries + * the bcrypt digests across. + */ + +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner } from "../../../lib/spinner.ts"; +import type { UserLine } from "../lib/run-store.ts"; +import { printTarget } from "../lib/target.ts"; +import { withDbClient, type DbClient } from "../lib/db.ts"; +import { finishExport, startExportRun } from "./shared.ts"; +import { + promptDbUrl, + resolveDbUrl, + type DbExportOptions, + type ResolveConfig, +} from "./db-options.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; + +/** + * `display_name` is coalesced into `first_name` here rather than in the + * transformer so a user who writes their own SQL sees the shape the + * transformer expects. + */ +const EXPORT_QUERY = ` + SELECT + id, + email, + email_confirmed_at, + encrypted_password, + phone, + phone_confirmed_at, + COALESCE( + raw_user_meta_data->>'display_name', + raw_user_meta_data->>'first_name', + raw_user_meta_data->>'name' + ) AS first_name, + raw_user_meta_data->>'last_name' AS last_name, + raw_user_meta_data, + raw_app_meta_data, + banned_until, + -- Through to_jsonb so an older auth.users without the column still reads. + to_jsonb(u)->>'deleted_at' AS deleted_at, + created_at + FROM auth.users u + ORDER BY created_at +`; + +type SupabaseRow = Record & { + id?: unknown; + email?: string | null; + encrypted_password?: string | null; + raw_app_meta_data?: unknown; +}; + +/** Serializes values the JSON export cannot carry as-is. */ +function normalizeRow(row: SupabaseRow): Record { + const normalized: Record = {}; + + for (const [key, value] of Object.entries(row)) { + if (value === null || value === undefined) continue; + // Postgres returns timestamps as Date objects; the transformer parses + // strings, and JSON.stringify would otherwise bury the format difference. + normalized[key] = value instanceof Date ? value.toISOString() : value; + } + + return normalized; +} + +export async function fetchSupabaseUsers(client: DbClient): Promise { + return client.query(EXPORT_QUERY); +} + +export function buildSupabaseExport( + rows: SupabaseRow[], + record: (line: UserLine) => void = () => {}, +) { + const users: Record[] = []; + const counts = { email: 0, emailConfirmed: 0, password: 0, phone: 0, firstName: 0, lastName: 0 }; + + for (const row of rows) { + const userId = String(row.id ?? ""); + try { + users.push(normalizeRow(row)); + + if (row.email) counts.email++; + if (row.email_confirmed_at) counts.emailConfirmed++; + if (row.encrypted_password) counts.password++; + if (row.phone) counts.phone++; + if (row.first_name) counts.firstName++; + if (row.last_name) counts.lastName++; + + record({ sourceId: userId, status: "exported" }); + } catch (error) { + record({ sourceId: userId, status: "skipped", error: (error as Error).message }); + } + } + + return { + users, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a confirmed email", count: counts.emailConfirmed }, + { label: "have a password hash", count: counts.password }, + { label: "have a phone number", count: counts.phone }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + ], + }; +} + +const SUPABASE_DB = { + platform: "supabase", + envVar: "SUPABASE_DB_URL", + prompt: "Supabase Postgres connection string", + hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", +} as const satisfies ResolveConfig; + +export async function exportSupabase(options: DbExportOptions): Promise { + const dbUrl = await resolveDbUrl(options, SUPABASE_DB); + + await withGutter("Exporting users from Supabase", async () => { + if (!options.json) printTarget({ platform: "supabase" }); + const { value: rows } = await withInputRetry( + dbUrl, + async () => promptDbUrl(SUPABASE_DB), + async (connectionString) => + withSpinner("Reading auth.users...", async () => + withDbClient(connectionString, "supabase", fetchSupabaseUsers), + ), + ); + + const run = await startExportRun(options, { platform: "supabase" }); + const { users, coverage } = buildSupabaseExport(rows, run.append); + finishExport({ run, options, users, coverage }); + + if (users.length > 0) { + log.info( + "Password hashes are included — this is why the export reads the database rather than the Admin API.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index 7d742c553..56dca9681 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -4,6 +4,7 @@ import os from "node:os"; import path from "node:path"; import { getMode, setMode } from "../../mode.ts"; import { createProgram } from "../../cli-program.ts"; +import { exportPlatformKeys } from "./export/registry.ts"; import { isAssumeYes, setAssumeYes } from "./lib/assume-yes.ts"; let saved: string | undefined; @@ -31,9 +32,10 @@ describe("registerMigrate", () => { expect(migrate?.description()).toContain("Migrate users"); }); - test("registers import, undo and runs", () => { + test("registers import, export, undo and runs", () => { expect(findCommand(["migrate"])?.commands.map((cmd) => cmd.name())).toEqual([ "import", + "export", "undo", "runs", ]); @@ -61,6 +63,50 @@ describe("registerMigrate", () => { ]); }); + test("registers an export subcommand per registered platform", () => { + expect(findCommand(["migrate", "export"])?.commands.map((cmd) => cmd.name())).toEqual( + exportPlatformKeys(), + ); + }); + + // Bare `migrate export` runs the picker rather than defaulting to a + // platform, so nobody exports from the wrong place by pressing enter. + test("leaves export with no default subcommand", () => { + const group = findCommand(["migrate", "export"]) as unknown as { + _defaultCommandName?: string; + }; + expect(group._defaultCommandName).toBeFalsy(); + }); + + test("migrate export takes exactly its flags", () => { + expect(findCommand(["migrate", "export"])?.options.map((option) => option.long)).toEqual([ + "--runs-dir", + "--json", + ]); + }); + + test("migrate export clerk takes exactly its flags", () => { + expect( + findCommand(["migrate", "export", "clerk"])?.options.map((option) => option.long), + ).toEqual(["--output", "--yes", "--runs-dir", "--json", "--secret-key", "--app", "--instance"]); + }); + + test("migrate export supabase takes exactly its flags", () => { + expect( + findCommand(["migrate", "export", "supabase"])?.options.map((option) => option.long), + ).toEqual(["--db-url", "--output", "--yes", "--runs-dir", "--json"]); + }); + + test.each(exportPlatformKeys())( + "migrate export %s names the run folder as the default output", + (platform) => { + const output = findCommand(["migrate", "export", platform])?.options.find( + (option) => option.long === "--output", + ); + expect(output?.description).toContain("instead of the run folder"); + }, + ); + test("registers runs with an optional run ID", () => { const runsCommand = findCommand(["migrate", "runs"]); expect(runsCommand?.registeredArguments[0]?.required).toBe(false); @@ -89,9 +135,9 @@ describe("registerMigrate", () => { expect(option?.argChoices).toBeUndefined(); }); - test("takes the file as an optional argument", () => { + test("takes the file, or the export run that wrote it, as an optional argument", () => { const [argument] = findCommand(["migrate", "import"])?.registeredArguments ?? []; - expect(argument?.name()).toBe("file"); + expect(argument?.name()).toBe("file|export-run-id"); expect(argument?.required).toBe(false); }); @@ -110,8 +156,11 @@ describe("registerMigrate", () => { */ describe("the migrate group's -y hook", () => { let missing: string; + let missingDb: string; beforeAll(() => { - missing = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-hook-")), "none.json"); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-hook-")); + missing = path.join(dir, "none.json"); + missingDb = path.join(dir, "none.sqlite"); }); async function parse(argv: string[]) { @@ -133,6 +182,10 @@ describe("the migrate group's -y hook", () => { ); }); + test("records -y on an export", async () => { + expect(await parse(["migrate", "export", "supabase", "-y", "--db-url", missingDb])).toBe(true); + }); + // `--json` means nobody reads a prompt, and agent mode is how every prompt in // this tree already knows to stand down. test("--json runs the command in agent mode", async () => { @@ -146,23 +199,29 @@ describe("the migrate group's -y hook", () => { } }); + // The export group declares --json too, and takes the flag; the hook has to + // read the group's options as well as the subcommand's. + test("--json on an export subcommand runs it in agent mode", async () => { + const original = getMode(); + try { + setMode("human"); + await parse(["migrate", "export", "supabase", "--json", "--db-url", missingDb]); + expect(getMode()).toBe("agent"); + } finally { + setMode(original); + } + }); + test("an explicit --mode wins over --json", async () => { const original = getMode(); + const runsDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-json-")); try { setMode("agent"); - await parse([ - "--mode", - "human", - "migrate", - "import", - missing, - "--json", - "--secret-key", - "sk_test_x", - ]); + await parse(["--mode", "human", "migrate", "runs", "--json", "--runs-dir", runsDir]); expect(getMode()).toBe("human"); } finally { setMode(original); + fs.rmSync(runsDir, { recursive: true, force: true }); } }); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts index cbf062084..98ffcb17d 100644 --- a/packages/cli-core/src/commands/migrate/index.ts +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -2,6 +2,7 @@ import type { Program } from "../../cli-program.ts"; import { isExperimentEnabled, requireExperiment } from "../../lib/experimental.ts"; import { setMode } from "../../mode.ts"; import { setAssumeYes } from "./lib/assume-yes.ts"; +import { registerMigrateExport } from "./export/index.ts"; import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "./lib/run-store.ts"; import { run } from "./run.ts"; import { runs } from "./runs.ts"; @@ -33,11 +34,15 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce .description("Migrate users into Clerk from another auth provider or another Clerk instance") .setExamples([ { - command: "clerk migrate import users.json --source supabase --dry-run", - description: "Check a Supabase export against the instance, and write nothing", + command: "clerk migrate export supabase", + description: "Export users from Supabase into a new run", }, { - command: "clerk migrate import users.json --source supabase --yes", + command: "clerk migrate import 20260929-141502-a1b2 --dry-run", + description: "Check an export against the instance, and write nothing", + }, + { + command: "clerk migrate import 20260929-141502-a1b2 --yes", description: "Import it. Run it again to continue after a failure", }, { command: "clerk migrate runs", description: "List every migration run" }, @@ -47,20 +52,25 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce }, ]); - // `-y` is resolved once here, for code below a command's options to read - // rather than having it passed down. Hooks are inherited, so this fires for - // every subcommand under `migrate`; one that declares no `-y` resolves to false. + // `-y` is read several layers down — by the credential-retry loop and the + // import — so it is resolved once here rather than threaded through every + // call. Hooks are inherited, so this fires for every + // subcommand under `migrate`; one that declares no `-y` resolves to false. // // `--json` means nobody is reading a prompt, so it runs the command in agent // mode: every prompt in this tree already stands down for an agent, with the // usage error naming what to pass instead. An explicit `--mode` still wins. migrateCommand.hook("preAction", (_thisCommand, actionCommand) => { + // With globals: `export supabase --json` lands on the export group's own + // --json, which the subcommand's opts() never sees. const opts = actionCommand.optsWithGlobals(); setAssumeYes(Boolean(opts.yes)); if (opts.json && !opts.mode) setMode("agent"); }); - // Named, not `isDefault`: bare `clerk migrate` prints help. + // Named, not `isDefault`. `import` and `export` are the two directions this + // group moves users in, and neither is implied by the bare group name. + // Bare `clerk migrate` prints help. // // The flags stay here rather than on `migrate`, matching how `config` keeps // its own on `pull`/`patch`/`put` — a group's help is a list of subcommands @@ -68,8 +78,11 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce migrateCommand .command("import") .description("Import users from an exported JSON or CSV file") - .argument("[file]", "A JSON or CSV export") - .option("--source ", "Where the file came from") + .argument("[file|export-run-id]", "The export file, or the ID of the export run that wrote it") + .option( + "--source ", + "Where the file came from. Not needed for a file from `clerk migrate export`", + ) .option("--dry-run", "Check the file against the instance, report, and write nothing") .option("--allow-partial", "Import the users that pass the checks, and skip the rest") .option("--new-run", "Start a new run instead of continuing an earlier one of this file") @@ -86,8 +99,12 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) .setExamples([ { - command: "clerk migrate import users.json --source supabase --dry-run", - description: "Check the file against the instance, and write nothing", + command: "clerk migrate import 20260929-141502-a1b2 --dry-run", + description: "Check what an export run wrote, and write nothing", + }, + { + command: "clerk migrate import users.json --source supabase --yes", + description: "Import a file. Run it again to continue after a failure", }, { command: "clerk migrate import users.json --source clerk --allow-partial --yes", @@ -101,6 +118,8 @@ export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = proce }), ); + registerMigrateExport(migrateCommand); + // Flat, not under a noun group: this is the one command in the tree that // destroys data in Clerk, and it is worth keeping short and prominent. migrateCommand diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts new file mode 100644 index 000000000..b7ea84c9c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -0,0 +1,325 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError, EXIT_CODE } from "../../../lib/errors.ts"; +import { + createDbClient, + describeDbError, + detectDbType, + redactConnectionString, + sqlitePath, + withDbClient, +} from "./db.ts"; + +let workDir: string; +let dbPath: string; + +beforeAll(() => { + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-db-"))); + dbPath = path.join(workDir, "test.sqlite"); + + const db = new Database(dbPath, { create: true }); + db.run(`CREATE TABLE "user" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" INTEGER)`); + db.run(`INSERT INTO "user" VALUES (?, ?, ?)`, ["u1", "a@x.dev", 1]); + db.run(`INSERT INTO "user" VALUES (?, ?, ?)`, ["u2", "b@x.dev", 0]); + db.close(); +}); + +afterAll(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("detectDbType", () => { + test.each([ + ["postgres://u:p@h/db", "postgres"], + ["postgresql://u:p@h/db", "postgres"], + ["POSTGRES://u:p@h/db", "postgres"], + ["mysql://u:p@h/db", "mysql"], + ["mysql2://u:p@h/db", "mysql"], + ["./db.sqlite", "sqlite"], + ["libsql://app-org.turso.io", "sqlite"], + ["file:./db.sqlite", "sqlite"], + ["/abs/path.db", "sqlite"], + [" postgres://u:p@h/db ", "postgres"], + ])("%s -> %s", (input, expected) => { + expect(detectDbType(input)).toBe(expected as never); + }); +}); + +describe("redactConnectionString", () => { + test.each([ + ["postgres://user:secret@host:5432/db", "postgres://***@host:5432/db"], + ["mysql://root:hunter2@127.0.0.1:3306/app", "mysql://***@127.0.0.1:3306/app"], + ["postgres://host/db", "postgres://host/db"], + ["libsql://app.turso.io?authToken=secret", "libsql://app.turso.io?authToken=***"], + ["postgres://u@host/db?password=secret", "postgres://***@host/db?password=***"], + [ + "postgres://host/db?sslmode=require&password=secret", + "postgres://host/db?sslmode=require&password=***", + ], + ])("%s -> %s", (input, expected) => { + expect(redactConnectionString(input)).toBe(expected); + }); + + // An unencoded `@` in the password is the most common mistake, and it is + // exactly when the string ends up in an error message. Matching the first + // `@` would leave the rest of the password visible. + test("redacts a password containing an unencoded @", () => { + const redacted = redactConnectionString("postgres://user:pa@ss@host/db"); + expect(redacted).toBe("postgres://***@host/db"); + expect(redacted).not.toContain("ss"); + }); + + test("redacts a password containing a colon", () => { + expect(redactConnectionString("postgres://user:a:b:c@host/db")).toBe("postgres://***@host/db"); + }); + + test.each([["./db.sqlite"], ["/var/data/app.db"], ["file:./local.sqlite"]])( + "leaves the credential-free path %s alone", + (input) => { + expect(redactConnectionString(input)).toBe(input); + }, + ); +}); + +describe("sqlitePath", () => { + test.each([ + ["./db.sqlite", "./db.sqlite"], + ["file:./db.sqlite", "./db.sqlite"], + ["file:/abs/db.sqlite", "/abs/db.sqlite"], + ["./db.sqlite?mode=ro", "./db.sqlite"], + [" ./db.sqlite ", "./db.sqlite"], + ])("%s -> %s", (input, expected) => { + expect(sqlitePath(input)).toBe(expected); + }); +}); + +describe("a libsql client", () => { + const originalFetch = globalThis.fetch; + let requests: { url: string; token?: string; body: any }[] = []; + + function stubFetch(result: unknown) { + requests = []; + globalThis.fetch = (async (url: string, init: RequestInit) => { + requests.push({ + url: String(url), + token: (init.headers as Record).authorization, + body: JSON.parse(String(init.body)), + }); + return new Response(JSON.stringify({ results: [result, { type: "ok" }] }), { + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + } + + const okRows = (cols: string[], rows: unknown[][]) => ({ + type: "ok", + response: { type: "execute", result: { cols: cols.map((name) => ({ name })), rows } }, + }); + + afterAll(() => { + globalThis.fetch = originalFetch; + }); + + test("posts to the pipeline endpoint with the URL's token and decodes rows", async () => { + stubFetch( + okRows( + ["id", "count", "verified", "missing", "hash"], + [ + [ + { type: "text", value: "u1" }, + { type: "integer", value: "12" }, + { type: "float", value: 1.5 }, + { type: "null" }, + { type: "blob", base64: Buffer.from("hash").toString("base64") }, + ], + ], + ), + ); + + const client = await createDbClient("libsql://app-org.turso.io?authToken=t0ken"); + const rows = await client.query('SELECT * FROM "user" WHERE id = ?', ["u1"]); + await client.close(); + + expect(requests[0]?.url).toBe("https://app-org.turso.io/v2/pipeline"); + expect(requests[0]?.token).toBe("Bearer t0ken"); + expect(requests.at(-1)?.body.requests[0].stmt.args).toEqual([{ type: "text", value: "u1" }]); + expect(rows).toEqual([ + { + id: "u1", + count: 12, + verified: 1.5, + missing: null, + hash: Buffer.from("hash"), + }, + ] as never); + expect(client.dbType).toBe("sqlite"); + }); + + test("reports a server-side error", async () => { + stubFetch({ type: "error", error: { message: "no such table: user" } }); + + await expect(createDbClient("libsql://app-org.turso.io")).rejects.toThrow(CliError); + }); +}); + +describe("a sqlite client", () => { + test("connects and queries", async () => { + const client = await createDbClient(dbPath); + try { + const rows = await client.query<{ id: string }>(`SELECT id FROM "user" ORDER BY id`); + expect(rows.map((row) => row.id)).toEqual(["u1", "u2"]); + } finally { + await client.close(); + } + }); + + test("binds parameters", async () => { + const client = await createDbClient(dbPath); + try { + const rows = await client.query<{ email: string }>(`SELECT email FROM "user" WHERE id = ?`, [ + "u2", + ]); + expect(rows[0]?.email).toBe("b@x.dev"); + } finally { + await client.close(); + } + }); + + test("reports its dialect's placeholder and quoting", async () => { + const client = await createDbClient(dbPath); + try { + expect(client.dbType).toBe("sqlite"); + expect(client.placeholder(1)).toBe("?"); + expect(client.quote("user")).toBe('"user"'); + } finally { + await client.close(); + } + }); + + test("accepts a file: URL", async () => { + const client = await createDbClient(`file:${dbPath}`); + try { + expect(await client.query(`SELECT 1 AS n`)).toHaveLength(1); + } finally { + await client.close(); + } + }); + + // bun:sqlite opens lazily, so without an explicit probe a missing file would + // surface at the first real query, long after "connecting" finished. + test("fails at connect time when the file is missing, not mid-export", async () => { + await expect(createDbClient(path.join(workDir, "nope.sqlite"))).rejects.toThrow(CliError); + }); + + test("names the file in the failure", async () => { + await expect(createDbClient(path.join(workDir, "nope.sqlite"))).rejects.toThrow( + /Could not open the SQLite file/, + ); + }); +}); + +describe("withDbClient", () => { + test("returns the work's value", async () => { + expect(await withDbClient(dbPath, undefined, async () => "done")).toBe("done"); + }); + + test("closes the client even when the work throws", async () => { + // A leaked handle keeps the process alive after the export has written its + // file, which reads as a hang. + await expect( + withDbClient(dbPath, undefined, async () => { + throw new Error("boom"); + }), + ).rejects.toThrow(/boom/); + + // The file is still usable, so nothing is holding it open. + expect(await withDbClient(dbPath, undefined, async () => "reopened")).toBe("reopened"); + }); + + test("attaches a hint to a query failure, not just a connection failure", async () => { + await expect( + withDbClient(dbPath, undefined, (client) => client.query(`SELECT * FROM missing_table`)), + ).rejects.toThrow(/expected table was not found/); + }); + + // A schema the export cannot read is a refusal (exit 2), and the advice is + // about the columns, not the connection. + test("names a missing column, and exits 2", async () => { + const error = (await withDbClient(dbPath, "supabase", (client) => + client.query(`SELECT no_such_column FROM sqlite_master`), + ).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("missing a column the export reads"); + expect(error.message).not.toContain("Check the connection string"); + }); + + test("passes a CliError through unchanged", async () => { + await expect( + withDbClient(dbPath, undefined, async () => { + throw new CliError("already explained"); + }), + ).rejects.toThrow(/already explained/); + }); +}); + +describe("describeDbError", () => { + const withCode = (code: string, message = "") => Object.assign(new Error(message), { code }); + + // Bun reports an unreachable host and a closed port identically, as + // "Connection closed" — precisely where a bare driver error helps least. + test.each([ + ["ERR_POSTGRES_CONNECTION_CLOSED", "Connection closed"], + ["ERR_MYSQL_CONNECTION_CLOSED", "Connection closed"], + ])("turns %s into host/port guidance", (code, message) => { + expect(describeDbError(withCode(code, message))).toMatch(/Check the host and port/); + }); + + test("gives Supabase the IPv4 add-on hint, which is the usual cause there", () => { + const hint = describeDbError( + withCode("ERR_POSTGRES_CONNECTION_CLOSED", "Connection closed"), + "supabase", + ); + expect(hint).toMatch(/pooler connection string/); + expect(hint).toMatch(/IPv4/); + }); + + test.each([ + ['password authentication failed for user "postgres"'], + ["Access denied for user 'root'@'localhost' (using password: YES)"], + ])("recognizes the rejected credentials in %p", (message) => { + expect(describeDbError(new Error(message))).toMatch(/rejected those credentials/); + }); + + test.each([ + ['relation "auth.users" does not exist'], + ["no such table: user"], + ["permission denied for table users"], + ])("recognizes the missing table in %p", (message) => { + expect(describeDbError(new Error(message))).toMatch(/table was not found|cannot read it/); + }); + + test("points Supabase at Auth being enabled and the postgres role", () => { + const hint = describeDbError(new Error('relation "auth.users" does not exist'), "supabase"); + expect(hint).toMatch(/Supabase Auth is enabled/); + expect(hint).toMatch(/postgres` role/); + }); + + test("recognizes an unopenable SQLite file", () => { + expect(describeDbError(new Error("unable to open database file"))).toMatch( + /Could not open the SQLite file/, + ); + }); + + test("still says something useful for an error it does not recognize", () => { + expect(describeDbError(new Error("something odd"))).toMatch(/Check the connection string/); + }); + + test("never echoes the error's own text, which could carry a connection string", () => { + const hint = describeDbError(new Error("failed for postgres://user:secret@host/db")); + expect(hint).not.toContain("secret"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts new file mode 100644 index 000000000..b3dca8c43 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -0,0 +1,379 @@ +/** + * One query interface over Postgres, MySQL and SQLite. + * + * Rewritten from the standalone migration-tool's `src/lib/db.ts`, which used + * `pg`, `mysql2` and `better-sqlite3`. None of those belong in a statically + * compiled binary — `better-sqlite3` is a native addon outright — so this runs + * on `Bun.sql` (Postgres and MySQL) and `bun:sqlite`, both built into the + * runtime. That swap is the entire reason the `engines.bun` floor exists. + * + * **Placeholders are not unified.** `Bun.sql` passes the query through to each + * server as written, so Postgres wants `$1` and MySQL wants `?` — verified + * against both. Rather than rewrite SQL strings (Postgres uses `?` as a JSONB + * operator, so a naive rewriter would corrupt real queries), callers ask the + * client for the placeholder and the identifier quoting they need. They already + * build per-dialect SQL for table casing, so this adds no new branching. + */ + +import { Database } from "bun:sqlite"; +import { SQL } from "bun"; +import { CliError, ERROR_CODE, EXIT_CODE } from "../../../lib/errors.ts"; + +export type DbType = "postgres" | "mysql" | "sqlite"; + +export interface DbClient { + dbType: DbType; + query>(sql: string, params?: unknown[]): Promise; + /** The bind placeholder for the 1-indexed `position`. */ + placeholder(position: number): string; + /** Quotes an identifier for this dialect. */ + quote(identifier: string): string; + close(): Promise; +} + +/** + * Reads the database type from a connection string. + * + * Anything that is not a recognized URL scheme is treated as a SQLite path, + * matching how the standalone tool behaved and how users actually pass + * `./db.sqlite`. `libsql://` (Turso) is SQLite too — it only differs in how + * the rows are fetched, so callers that branch on the dialect want "sqlite". + */ +export function detectDbType(connectionString: string): DbType { + const lower = connectionString.trim().toLowerCase(); + if (lower.startsWith("postgresql://") || lower.startsWith("postgres://")) return "postgres"; + if (lower.startsWith("mysql://") || lower.startsWith("mysql2://")) return "mysql"; + return "sqlite"; +} + +/** True for a remote libsql/Turso URL, which is read over HTTP rather than opened. */ +export function isLibsqlUrl(connectionString: string): boolean { + return /^libsql:\/\//i.test(connectionString.trim()); +} + +/** + * Replaces any credentials in a connection string with `***`. + * + * Connection strings reach the CLI on the command line and end up in error + * messages and `--verbose` output. Bun's own errors do not echo them, and + * nothing here should either. + */ +export function redactConnectionString(connectionString: string): string { + // Greedy up to the LAST `@`: an unencoded `@` in the password is the most + // common connection-string mistake, and matching the first one would leave + // the rest of the password in the message. Everything before the final `@` + // is userinfo, so redacting all of it is always safe. + // Non-URL forms (SQLite paths) have no `://` and are left alone. + // Turso carries its credential as `?authToken=`, not as userinfo, and a + // `?password=` (which Bun.SQL ignores) still must not be printed. + return connectionString + .replace(/^([a-z0-9+]+:\/\/)(.*)@/i, "$1***@") + .replace(/([?&](?:authToken|password)=)[^&]*/gi, "$1***"); +} + +/** Strips a `file:` prefix and any URL query, leaving a filesystem path. */ +export function sqlitePath(connectionString: string): string { + const trimmed = connectionString.trim(); + const withoutScheme = trimmed.startsWith("file:") ? trimmed.slice("file:".length) : trimmed; + return withoutScheme.split("?")[0] ?? withoutScheme; +} + +const QUOTING: Record string> = { + // Doubling the delimiter is the escape in every dialect here, so an + // identifier containing one cannot break out of the quotes. + postgres: (identifier) => `"${identifier.replace(/"/g, '""')}"`, + sqlite: (identifier) => `"${identifier.replace(/"/g, '""')}"`, + mysql: (identifier) => `\`${identifier.replace(/`/g, "``")}\``, +}; + +function bunSqlClient(connectionString: string, dbType: "postgres" | "mysql"): DbClient { + const sql = new SQL(connectionString); + + return { + dbType, + async query>(query: string, params: unknown[] = []) { + const rows = await sql.unsafe(query, params); + return (Array.isArray(rows) ? rows : []) as T[]; + }, + placeholder: dbType === "postgres" ? (position) => `$${position}` : () => "?", + quote: QUOTING[dbType], + async close() { + await sql.close(); + }, + }; +} + +/** + * One value in Hrana's wire format, the protocol libsql servers speak. + * + * Integers arrive as strings so 64-bit values survive JSON. + */ +type HranaValue = { type: string; value?: string | number; base64?: string }; + +function decodeHrana(value: HranaValue): unknown { + switch (value.type) { + case "null": + return null; + case "integer": { + const raw = String(value.value ?? "0"); + const asNumber = Number(raw); + // Past 2^53 a number would silently lose digits; ids can get that big. + return Number.isSafeInteger(asNumber) ? asNumber : BigInt(raw); + } + case "float": + return Number(value.value); + case "blob": + // bun:sqlite hands back bytes for a BLOB, so this does too. + return Buffer.from(value.base64 ?? "", "base64"); + default: + return value.value ?? null; + } +} + +function encodeHrana(param: unknown): HranaValue { + if (param === null || param === undefined) return { type: "null" }; + if (typeof param === "bigint") return { type: "integer", value: param.toString() }; + if (typeof param === "boolean") return { type: "integer", value: param ? "1" : "0" }; + if (typeof param === "number") { + return Number.isInteger(param) + ? { type: "integer", value: String(param) } + : { type: "float", value: param }; + } + if (param instanceof Uint8Array) { + return { type: "blob", base64: Buffer.from(param).toString("base64") }; + } + return { type: "text", value: String(param) }; +} + +/** + * Talks to a libsql server (Turso) over its HTTP pipeline endpoint. + * + * `bun:sqlite` opens local files and cannot reach a remote database, and + * `@libsql/client` ships native optional dependencies that do not survive + * `bun build --compile`. The protocol is one POST per statement, so it is + * fewer lines to speak it directly than to carry the dependency. + * + * The token comes from `?authToken=` on the URL — the form the Turso CLI + * prints — or from `TURSO_AUTH_TOKEN`/`LIBSQL_AUTH_TOKEN`. A self-hosted sqld + * with auth disabled needs neither, so a missing token is not an error here. + */ +function libsqlClient( + connectionString: string, + env: Record = process.env, +): DbClient { + const url = new URL(connectionString.trim()); + const token = url.searchParams.get("authToken") || env.TURSO_AUTH_TOKEN || env.LIBSQL_AUTH_TOKEN; + const endpoint = `https://${url.host}/v2/pipeline`; + + return { + dbType: "sqlite", + async query>(query: string, params: unknown[] = []) { + const response = await fetch(endpoint, { + method: "POST", + headers: { + "content-type": "application/json", + ...(token ? { authorization: `Bearer ${token}` } : {}), + }, + // `close` keeps every request stateless: no baton to carry forward. + body: JSON.stringify({ + requests: [ + { type: "execute", stmt: { sql: query, args: params.map(encodeHrana) } }, + { type: "close" }, + ], + }), + }); + + if (!response.ok) { + throw new Error(`libsql request failed: ${response.status} ${response.statusText}`.trim()); + } + + const body = (await response.json()) as { + results?: { + type: string; + error?: { message?: string }; + response?: { result?: { cols?: { name?: string }[]; rows?: HranaValue[][] } }; + }[]; + }; + + const first = body.results?.[0]; + if (!first || first.type === "error") { + throw new Error(first?.error?.message ?? "libsql returned no result"); + } + + const cols = first.response?.result?.cols ?? []; + const rows = first.response?.result?.rows ?? []; + return rows.map( + (row) => + Object.fromEntries( + row.map((value, index) => [cols[index]?.name ?? String(index), decodeHrana(value)]), + ) as T, + ); + }, + placeholder: () => "?", + quote: QUOTING.sqlite, + async close() { + return Promise.resolve(); + }, + }; +} + +function sqliteClient(connectionString: string): DbClient { + const database = new Database(sqlitePath(connectionString), { readonly: true }); + + return { + dbType: "sqlite", + async query>(query: string, params: unknown[] = []) { + // bun:sqlite is synchronous; the Promise keeps one interface for callers. + return Promise.resolve(database.query(query).all(...(params as never[])) as T[]); + }, + placeholder: () => "?", + quote: QUOTING.sqlite, + async close() { + database.close(); + return Promise.resolve(); + }, + }; +} + +/** + * Connects to the database a connection string names. + * + * @param platform - Tailors the failure hint; the same "Connection closed" + * means something different on Supabase than on a local SQLite file. + */ +export async function createDbClient( + connectionString: string, + platform?: DbPlatform, +): Promise { + const dbType = detectDbType(connectionString); + + try { + if (isLibsqlUrl(connectionString)) { + const client = libsqlClient(connectionString); + await client.query("SELECT 1"); + return client; + } + + if (dbType === "sqlite") { + const client = sqliteClient(connectionString); + // bun:sqlite opens lazily, so a missing file would not surface until the + // first real query — long after the "connecting" spinner has stopped. + await client.query("SELECT 1"); + return client; + } + + const client = bunSqlClient(connectionString, dbType); + await client.query("SELECT 1"); + return client; + } catch (error) { + throw connectionError(error, connectionString, platform); + } +} + +export type DbPlatform = "supabase"; + +/** + * Turns a driver error into something a user can act on. + * + * Rewritten rather than ported: the standalone tool matched on `pg`'s + * `ENOTFOUND`/`ETIMEDOUT`, which `Bun.sql` never emits. Bun reports both an + * unreachable host and a closed port as `ERR_*_CONNECTION_CLOSED` with the + * message "Connection closed" — exactly the case where a bare driver error + * helps least. + */ +export function describeDbError(error: unknown, platform?: DbPlatform): string { + const message = error instanceof Error ? error.message : String(error); + const code = (error as { code?: string })?.code ?? ""; + + if (code.includes("CONNECTION_CLOSED") || /connection closed|econnrefused/i.test(message)) { + if (platform === "supabase") { + return ( + "Could not reach the database. Check the host and port in the connection string.\n" + + "Supabase direct connections need the IPv4 add-on — use the pooler connection string\n" + + "(Dashboard → Connect → Session pooler), or enable IPv4 under Settings → Add-Ons." + ); + } + return "Could not reach the database. Check the host and port, and that the server accepts connections from here."; + } + + // Turso resolves every `*.turso.io` name, so a typo'd database does not fail + // to connect — it answers 404. "Check the host" would send the reader after + // the half that is right. + if (/\b404\b/.test(message)) { + return ( + "No database at that libsql host. Check the database name in the URL —\n" + + "`turso db show ` prints the URL to use." + ); + } + + if (/\b401\b|unauthorized|not authorized/i.test(message)) { + return ( + "The libsql server rejected that token.\n" + + "Append ?authToken=… to the URL, or set TURSO_AUTH_TOKEN (`turso db tokens create `)." + ); + } + + if (/password authentication failed|access denied/i.test(message)) { + return "The database rejected those credentials. Check the user and password in the connection string."; + } + + // Before the table check: Postgres words a missing column "does not exist" too. + if (/no such column|column .* does not exist|unknown column/i.test(message)) { + return "The user table is missing a column the export reads. Check the schema matches the platform's default."; + } + + if (/does not exist|unknown database|no such table|permission denied/i.test(message)) { + if (platform === "supabase") { + return ( + "The auth.users table was not readable. It is created automatically when Supabase Auth is enabled.\n" + + "Check Authentication is enabled, and connect as the `postgres` role rather than an application role." + ); + } + return "The expected table was not found, or the user cannot read it. Check the database name and the user's SELECT permission."; + } + + if (/unable to open database|sqlitecantopen|no such file/i.test(message)) { + return "Could not open the SQLite file. Check the path, and that the file exists and is readable."; + } + + return "Check the connection string, that the server is running, and that it is reachable from here."; +} + +function connectionError( + error: unknown, + connectionString: string, + platform?: DbPlatform, +): CliError { + const message = error instanceof Error ? error.message : String(error); + return new CliError( + `Could not connect to ${redactConnectionString(connectionString)}: ${message}\n\n${describeDbError(error, platform)}`, + { code: ERROR_CODE.USAGE_ERROR, exitCode: EXIT_CODE.USAGE }, + ); +} + +/** + * Runs `work` against a fresh client and always closes it. + * + * A leaked connection keeps the process alive after the export has written its + * file, which looks like a hang. + */ +export async function withDbClient( + connectionString: string, + platform: DbPlatform | undefined, + work: (client: DbClient) => Promise, +): Promise { + const client = await createDbClient(connectionString, platform); + try { + return await work(client); + } catch (error) { + // A query failure carries the same actionable hints as a connection one: + // a missing table is the most common thing that goes wrong here. + if (error instanceof CliError) throw error; + throw new CliError( + `${error instanceof Error ? error.message : String(error)}\n\n${describeDbError(error, platform)}`, + { code: ERROR_CODE.USAGE_ERROR, exitCode: EXIT_CODE.USAGE }, + ); + } finally { + await client.close().catch(() => {}); + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/export-file.ts b/packages/cli-core/src/commands/migrate/lib/export-file.ts index 63bfd0862..c9ebb499d 100644 --- a/packages/cli-core/src/commands/migrate/lib/export-file.ts +++ b/packages/cli-core/src/commands/migrate/lib/export-file.ts @@ -1,9 +1,55 @@ /** - * Reading an export file the way an import does. + * The file `clerk migrate export` writes: the users, wrapped in an envelope + * that says where they came from. + * + * The envelope is what lets `clerk migrate import ` run with no + * `--source`: the source, and Firebase's hash parameters, travel with the + * users. A bare array, a CSV, or Firebase's own `{ users: [...] }` still + * import, but need the source named. */ import fs from "node:fs"; import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import type { FirebaseHashConfig } from "../types.ts"; + +export const ENVELOPE_VERSION = 1; + +export type ExportEnvelope = { + clerkMigrate: typeof ENVELOPE_VERSION; + source: string; + exportedAt: string; + runId: string; + firebase?: FirebaseHashConfig; + users: Record[]; +}; + +export function isEnvelope(value: unknown): value is ExportEnvelope { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const candidate = value as Partial; + return ( + candidate.clerkMigrate === ENVELOPE_VERSION && + typeof candidate.source === "string" && + Array.isArray(candidate.users) + ); +} + +/** + * The envelope in a file, when it has one. + * + * `undefined` for a CSV, a bare array, or anything unreadable: those are the + * shapes that need `--source`, and the load that follows reports what is + * actually wrong with a broken file. + */ +// ponytail: parses the file here and again when the users load; cache by path if exports get large enough to notice. +export function readEnvelope(file: string): ExportEnvelope | undefined { + if (!file.toLowerCase().endsWith(".json")) return undefined; + try { + const parsed = readJsonFile(file); + return isEnvelope(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} /** * A JSON file's contents, as an import reads it. diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts new file mode 100644 index 000000000..d77c09506 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts @@ -0,0 +1,234 @@ +/** + * `withInputRetry` — the loop that puts a credential prompt back up when the + * far end rejects what it was given. + * + * Its own file because `mock.module` registrations last for the process, and + * `bun test --parallel` puts several files in each worker — a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import { CliError, ERROR_CODE, EXIT_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +let answers: string[] = []; +let cancelPrompt = false; + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../../lib/prompts.ts", () => ({ + password: async () => { + if (cancelPrompt) throw new UserAbortError(); + return answers.shift() ?? ""; + }, + text: async () => answers.shift() ?? "", + confirm: async () => true, + multiselect: async () => [], + select: async () => "", + editor: async () => "{}", + note: () => {}, +})); + +const { withInputRetry } = await import("./input-retry.ts"); +const { promptDbUrl } = await import("../export/db-options.ts"); +const { setAssumeYes } = await import("./assume-yes.ts"); + +const captured = useCaptureLog(); + +const CONFIG = { + platform: "supabase", + envVar: "SUPABASE_DB_URL", + prompt: "Supabase Postgres connection string", +} as const; + +const FIRST = "libsql://typo.turso.io?authToken=t"; +const SECOND = "libsql://right.turso.io?authToken=t"; + +const rejected = () => + new CliError("Could not reach it", { code: ERROR_CODE.USAGE_ERROR, exitCode: EXIT_CODE.USAGE }); + +let originalMode: string | undefined; + +beforeAll(() => { + originalMode = process.env.CLERK_MODE; +}); + +afterAll(() => { + if (originalMode === undefined) delete process.env.CLERK_MODE; + else process.env.CLERK_MODE = originalMode; +}); + +beforeEach(() => { + process.env.CLERK_MODE = "human"; + setAssumeYes(false); + answers = []; + cancelPrompt = false; +}); + +describe("withInputRetry", () => { + test("returns the first result without prompting when the work succeeds", async () => { + const seen: string[] = []; + + const { value, input } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + seen.push(url); + return Promise.resolve("rows"); + }, + ); + + expect(value).toBe("rows"); + expect(input).toBe(FIRST); + expect(seen).toEqual([FIRST]); + }); + + test("asks again after a failure and runs with the new input", async () => { + answers = [SECOND]; + const seen: string[] = []; + + const { value } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + seen.push(url); + if (url === FIRST) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(value).toBe("rows"); + expect(seen).toEqual([FIRST, SECOND]); + // The operator has to be told what was wrong with a string they cannot see. + expect(captured.err).toContain("Could not reach it"); + }); + + // Later steps run against the credential that worked, not the one first tried + // — a Firebase export reads its project id off the key that Google accepted. + test("reports the input that finally worked", async () => { + answers = [SECOND]; + + const { input } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + if (url === FIRST) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(input).toBe(SECOND); + }); + + test("keeps asking until an input works", async () => { + answers = [FIRST, FIRST, SECOND]; + let attempts = 0; + + await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + attempts++; + if (url !== SECOND) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(attempts).toBe(4); + }); + + // Cancelling the prompt is an answer: it ends the command rather than looping + // on a question the operator has already declined. + test("lets a cancelled prompt out of the loop", async () => { + cancelPrompt = true; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + throw rejected(); + }, + ), + ).rejects.toThrow(UserAbortError); + }); + + test("throws without prompting when there is nobody to ask", async () => { + process.env.CLERK_MODE = "agent"; + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw rejected(); + }, + ), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); + + // `-y` is a human on a TTY who could be asked and said not to. Agent mode + // cannot reach the prompt at all; this one can and declines to, so it needs + // its own check rather than riding on the mode assertion above. + test("throws without prompting when `-y` said not to ask", async () => { + setAssumeYes(true); + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw rejected(); + }, + ), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); + + // A bug inside the work, or an interrupt, is not a wrong answer to a prompt. + // Another credential would not fix an outage, a 429 or a refused connection. + test.each([ + ["a refused connection", new CliError("Could not reach x")], + ["a 503", new CliError("The server returned 503 listing users")], + ])("does not ask again after %s", async (_label, failure) => { + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw failure; + }, + ), + ).rejects.toBe(failure); + + expect(attempts).toBe(1); + }); + + test("does not retry an error the database layer did not raise", async () => { + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw new TypeError("undefined is not a function"); + }, + ), + ).rejects.toThrow(TypeError); + + expect(attempts).toBe(1); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.ts new file mode 100644 index 000000000..13ec1b2c0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.ts @@ -0,0 +1,84 @@ +/** + * Retrying the *answer*, not the request. + * + * Distinct from `retry.ts`, which re-sends an identical request after a 429: + * there the request was right and the server was busy. Here the request was + * fine and the input was wrong, so nothing changes until the operator supplies + * something better. + * + * Every credential a migration takes — a connection string, a Firebase service + * account key, an Auth0 client secret — is long, pasted by hand, masked as it + * is typed, and wrong in ways nothing local can check: a typo'd host, an + * expired token, a key that was revoked, the right server but the wrong + * database. Only the remote end can say, and by then the operator has already + * answered every other question the command asked. Ending there charges them a + * full re-run for one line they could not see. + */ + +import { CliError, EXIT_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { isAssumeYes } from "./assume-yes.ts"; + +/** + * Runs `work`, and on failure asks for the input again and runs it once more. + * + * Keep `work` to the step that *proves* the input — the connection, the token + * exchange. Everything inside runs again on each attempt, so work that has + * already written a file, or a long fetch the credential has already been + * accepted for, does not belong in here. + * + * `-y`, agent mode and a non-TTY get the failure unchanged: there is nobody to + * ask, and a loop that cannot prompt is a loop that cannot end. A cancelled + * prompt throws {@link UserAbortError}, which is not a `CliError` and so leaves + * the loop — declining the question is an answer. + * + * @param input - What to try first: a flag, an environment value, or the + * answer to the prompt the caller has already put up. + * @param reprompt - Asks for a replacement. Called once per failure. + * @param work - The step the input has to survive. + * @returns The result, and the input that produced it — which is not `input` + * when it took a retry, and later steps need the one that worked. + */ +export async function withInputRetry( + input: I, + reprompt: () => Promise, + work: (input: I) => Promise, +): Promise<{ value: T; input: I }> { + let candidate = input; + + for (;;) { + try { + return { value: await work(candidate), input: candidate }; + } catch (error) { + // Everything these steps raise for a bad credential is a usage error + // carrying its own explanation. An outage, a 429 or a refused + // connection is not fixed by another credential, and anything else (an + // interrupt, a bug) is not ours to retry. + const badInput = error instanceof CliError && error.exitCode === EXIT_CODE.USAGE; + if (!badInput || !isHuman() || isAgent() || isAssumeYes()) throw error; + + log.error(error.message); + candidate = await reprompt(); + } + } +} + +/** Statuses that mean the credential is wrong, rather than the service being down. */ +const CREDENTIAL_STATUSES = new Set([400, 401, 403]); + +/** True when an API's status says the credential, not the service, is the problem. */ +export function isCredentialStatus(status: number): boolean { + return CREDENTIAL_STATUSES.has(status); +} + +/** + * Throws an API failure: a usage error (exit 2, and a fresh prompt under + * {@link withInputRetry}) when the status blames the credential; a plain + * error (exit 1) for a 429, a 5xx or anything else another credential would + * not fix. + */ +export function throwApiFailure(status: number, message: string, docsUrl?: string): never { + if (isCredentialStatus(status)) throwUsageError(message, docsUrl); + throw new CliError(message, { docsUrl }); +} diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts index 47110010b..561ad655e 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -309,4 +309,20 @@ describe("loadUsersFromFile", () => { new CliError("null-row.json: row 2 is not a user object."), ); }); + + test("reads the users out of an export's envelope", async () => { + const envelope = (users: unknown[]) => + JSON.stringify({ clerkMigrate: 1, source: "clerk", exportedAt: "", runId: "r", users }); + fs.writeFileSync( + path.join(workDir, "envelope.json"), + envelope([{ id: "u1", primary_email_address: "a@x.dev" }]), + ); + const { users } = await loadUsersFromFile("envelope.json", "clerk"); + expect(users.map((user) => user.userId)).toEqual(["u1"]); + + fs.writeFileSync(path.join(workDir, "envelope-null.json"), envelope([null])); + await expect(loadUsersFromFile("envelope-null.json", "clerk")).rejects.toThrow( + new CliError("envelope-null.json: row 1 is not a user object."), + ); + }); }); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts index 82ea8c5ad..385a4891a 100644 --- a/packages/cli-core/src/commands/migrate/lib/transform.ts +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -15,7 +15,7 @@ import { getSource } from "../sources/registry.ts"; import { normalizeBooleanField } from "../sources/shared.ts"; import { type TransformContext, type SourceEntry, type User } from "../types.ts"; import { userSchema } from "../validator.ts"; -import { readJsonFile } from "./export-file.ts"; +import { isEnvelope, readJsonFile } from "./export-file.ts"; export type FileType = "application/json" | "text/csv"; @@ -441,8 +441,11 @@ async function readUsersFromFile( let preExtracted: Record[] | undefined; let csvHeaders: string[] | undefined; + // An export's envelope already holds the users in the source's own shape, + // so there is nothing left for a pre-transform to unwrap. if (type === "application/json") { const parsed = readJsonFile(filePath); + if (isEnvelope(parsed)) return assertUserRows(parsed.users, file); if (!transformer.preTransform) return assertUserRows(parsed, file); } diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index 473b56ce0..9fd2e1ea4 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -57,6 +57,7 @@ const { terminal } = await import("./lib/terminal.ts"); const terminalAvailable = spyOn(terminal, "available"); const { UserAbortError } = await import("../../lib/errors.ts"); const { _setConfigDir } = await import("../../lib/config.ts"); +const { listRuns, startRun } = await import("./lib/run-store.ts"); const captured = useCaptureLog(); @@ -174,6 +175,33 @@ describe("prompts for what was not passed", () => { expect(mockText).not.toHaveBeenCalled(); expect(mockSelect).not.toHaveBeenCalled(); }); + + // The envelope names the source, so asking would only invite a wrong answer. + test("does not ask for a source when the file names its own", async () => { + const exportRun = startRun(runsDir(), { + kind: "export", + target: { platform: "clerk" }, + source: "clerk", + }); + const file = path.join(exportRun.dir, "export.json"); + fs.writeFileSync( + file, + JSON.stringify({ + clerkMigrate: 1, + source: "clerk", + exportedAt: "2026-09-01T00:00:00.000Z", + runId: exportRun.record.id, + users: EXPORT, + }), + ); + exportRun.update({ file: { path: file, sha256: "x" } }); + exportRun.finish(); + + await run({ input: exportRun.record.id, secretKey: "sk_test_x" }); + + expect(mockSelect).not.toHaveBeenCalled(); + expect(created()).toHaveLength(2); + }); }); // `CLERK_MODE=human` with no terminal: the prompt would fail to open one and spin. @@ -212,7 +240,7 @@ describe("consent", () => { await expect(run(importOptions)).rejects.toThrow(UserAbortError); expect(created()).toHaveLength(0); - expect(fs.existsSync(runsDir())).toBe(false); + expect(listRuns(runsDir())).toHaveLength(0); }); test("--yes does not ask", async () => { diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 85b4f0f38..03929ac11 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -299,6 +299,73 @@ describe("run", () => { expect(captured.err).toContain(`rm -rf '${path.join(spaced, record!.id)}'`); }); + describe("export envelopes", () => { + /** An export run whose envelope holds `users`, as `clerk migrate export` writes it. */ + function exportRun(source: string, rows: unknown[]) { + const run = startRun(runsDir(), { kind: "export", target: { platform: source }, source }); + const file = path.join(run.dir, "export.json"); + fs.writeFileSync( + file, + JSON.stringify({ + clerkMigrate: 1, + source, + exportedAt: "2026-09-01T00:00:00.000Z", + runId: run.record.id, + users: rows, + }), + ); + run.update({ file: { path: file, sha256: "x" } }); + return { record: run.finish(), file }; + } + + const { source: _source, input: _input, ...noSource } = baseOptions; + + test("imports by export run ID, with the source the envelope names", async () => { + const { record } = exportRun("clerk", export2); + + await run({ ...noSource, input: record.id }); + + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); + const imported = listRuns(runsDir()).find((candidate) => candidate.kind === "import"); + expect(imported).toMatchObject({ source: "clerk", fromExport: record.id }); + // The export holds password hashes and PII, so the cleanup names it too. + expect(captured.err).toContain(`The export in run ${record.id} holds your users' data`); + }); + + test("imports an envelope file with no source named", async () => { + const { file } = exportRun("clerk", export2); + + await run({ ...noSource, input: file }); + + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); + }); + + test("refuses a source that contradicts the envelope", async () => { + const { record } = exportRun("clerk", export2); + + await expect(run({ ...noSource, source: "supabase", input: record.id })).rejects.toThrow( + /exported from clerk, but --source names supabase/, + ); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); + }); + + test("refuses a run ID that is not an export", async () => { + await run(baseOptions); + const [imported] = listRuns(runsDir()); + + await expect(run({ ...noSource, input: imported!.id })).rejects.toThrow( + /is an import run, which has no file to import/, + ); + }); + + test("refuses a run ID with no run behind it", async () => { + await expect(run({ ...noSource, input: "20260101-000000-abcd" })).rejects.toThrow( + /No run `20260101-000000-abcd`/, + ); + expect(requests).toHaveLength(0); + }); + }); + test("gitignores the project's .clerk folder before writing a run", async () => { await run(baseOptions); expect(fs.readFileSync(path.join(workDir, ".gitignore"), "utf-8")).toContain(".clerk/"); @@ -643,7 +710,7 @@ describe("run", () => { describe("without a file or a source", () => { test("names what to pass rather than prompting for the file", async () => { await expect(run({ source: "clerk", yes: true, secretKey: "sk_test_x" })).rejects.toThrow( - /needs the file to import, and cannot prompt here/, + /needs the file to import, or the export run that wrote it, and cannot prompt here/, ); expect(requests).toHaveLength(0); }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 08270ea5f..099615b3e 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -1,13 +1,14 @@ /** - * `clerk migrate import ` — the user import itself. + * `clerk migrate import ` — the user import itself. * * Registered as the `import` subcommand. The exported handler keeps the name * `run` because `import` is a reserved word. * * An import goes through the same steps every time: * - * 1. Settle the file, the source and the target instance, and print the - * target. + * 1. Settle the file (an export run ID stands for the file that run wrote), + * the source (a file from `clerk migrate export` names its own) and the + * target instance, and print the target. * 2. Decide whether this continues an earlier run of the same file, source and * instance, from the run store. * 3. Run `checkImport()` against the real instance. `--dry-run` stops here. @@ -46,13 +47,16 @@ import { withGutter, withSpinner } from "../../lib/spinner.ts"; import { importUsers } from "./import-users.ts"; import { ALREADY_IMPORTED, checkImport, type ImportChecks } from "./lib/checks.ts"; import { fetchInstanceSettings, fetchUserCount } from "./lib/clerk-config.ts"; +import { readEnvelope, type ExportEnvelope } from "./lib/export-file.ts"; import { DEV_USER_LIMIT, resolveLimits, type InstanceType } from "./lib/instance.ts"; import { continueRun, latestUserLines, listRuns, + readRun, readUserLines, resolveRunsDir, + RUN_ID_PATTERN, runDir, runState, sha256File, @@ -95,7 +99,7 @@ import { login } from "../auth/login.ts"; import { link } from "../link/index.ts"; export type MigrateRunOptions = { - /** The export file. */ + /** An export file, or the ID of the export run that wrote one. */ input?: string; /** A built-in source key. */ source?: string; @@ -149,7 +153,7 @@ export function validateRunOptions(options: MigrateRunOptions): { } if (!options.file) { throwUsageError( - "Missing the file to import (a JSON or CSV export).", + "Missing the file to import (a JSON or CSV export, or an export run ID).", undefined, ERROR_CODE.USAGE_ERROR, [ @@ -273,24 +277,46 @@ async function ensureImportTarget(options: MigrateRunOptions): Promise { } /** - * Settles which file to read. A human who gave none is asked for a path. + * Settles which file to read. The argument may name the export run that wrote + * the file; a human who gave none is asked for a path. */ -async function resolveInput(options: MigrateRunOptions): Promise { - if (options.input) return options.input; - if (!canPrompt(options)) { +async function resolveInput( + options: MigrateRunOptions, +): Promise<{ file: string; fromExport?: string }> { + const value = options.input; + if (!value) { + if (!canPrompt(options)) { + throwUsageError( + "`clerk migrate import` needs the file to import, or the export run that wrote it, and cannot prompt here.", + undefined, + undefined, + [ + { + command: "clerk migrate import 20260929-141502-a1b2 --yes", + description: "Import what an export run wrote", + }, + { + command: "clerk migrate import users.json --source clerk --yes", + description: "Import a file", + }, + ], + ); + } + return { file: await promptForFile() }; + } + if (!RUN_ID_PATTERN.test(value) || fileExists(value)) return { file: value }; + + const runsDir = await resolveRunsDir(options.runsDir); + const record = readRun(runsDir, value); + if (!record) { + throwUsageError(`No run \`${value}\` in ${runsDir}. Run \`clerk migrate runs\` to list them.`); + } + if (record.kind !== "export" || !record.file) { throwUsageError( - "`clerk migrate import` needs the file to import, and cannot prompt here.", - undefined, - undefined, - [ - { - command: "clerk migrate import users.json --source clerk --yes", - description: "Import a file", - }, - ], + `Run ${value} is an ${record.kind} run, which has no file to import. Name an export run, or a file.`, ); } - return promptForFile(); + return { file: record.file.path, fromExport: record.id }; } /** @@ -304,6 +330,27 @@ async function applySource(options: MigrateRunOptions): Promise"; + const input = fromExport ?? options.input ?? options.file ?? ""; const parts = ["clerk migrate import", quoteArg(input)]; - if (options.source) parts.push("--source", quoteArg(options.source)); + if (!fromExport && options.source) parts.push("--source", quoteArg(options.source)); if (options.newRun) parts.push("--new-run"); parts.push(...importFlags(options)); parts.push(...carry); @@ -620,9 +677,14 @@ export async function run(rawOptions: MigrateRunOptions): Promise { await ensureImportTarget(options); const input = await resolveInput(options); - options = { ...options, file: input }; - - if (!options.source && canPrompt(options) && fileExists(input)) { + options = { ...options, file: input.file }; + const envelope = fileExists(input.file) + ? readEnvelope(resolveImportFilePath(input.file)) + : undefined; + options = applyEnvelope(options, envelope); + + // Asked only when the file does not say where it came from. + if (!options.source && canPrompt(options) && fileExists(input.file)) { options = { ...options, source: await promptForSource() }; } @@ -849,6 +911,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { allowPartial: options.allowPartial || recorded.allowPartial, skipLegalChecks: options.skipLegalChecks || recorded.skipLegalChecks, }, + input.fromExport, ["--yes"], ) : undefined; @@ -882,7 +945,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { ? [] : [ { - command: commandFor(options, ["--allow-partial", "--yes"]), + command: commandFor(options, input.fromExport, ["--allow-partial", "--yes"]), description: "Import the users that pass", }, ]), @@ -933,7 +996,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { undefined, [ { - command: commandFor(options, ["--yes"]), + command: commandFor(options, input.fromExport, ["--yes"]), description: "Run the import", }, ], @@ -958,6 +1021,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { target, source, file: { path: filePath, sha256 }, + ...(input.fromExport ? { fromExport: input.fromExport } : {}), flags, }); // Printed now, not on the way out: on a Ctrl-C the signal handler exits @@ -1071,7 +1135,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { runsStep, // The same import again continues this run; --new-run would not. // Run as printed: consent was given to this run, at the prompt or by -y. - commandFor({ ...options, newRun: false }, ["--yes"], later), + commandFor({ ...options, newRun: false }, input.fromExport, ["--yes"], later), undo, ) : NEXT_STEPS.MIGRATE_DONE(runsStep, undo); diff --git a/scripts/check-bun-version.ts b/scripts/check-bun-version.ts index a05b2d4b7..74f576892 100644 --- a/scripts/check-bun-version.ts +++ b/scripts/check-bun-version.ts @@ -9,6 +9,13 @@ * producing hundreds of order-dependent failures. Bun does not enforce * `engines.bun` at install time, so this preflight fails loudly instead. * + * A second, lower constraint rides along: the DB-backed `clerk migrate export` + * commands read MySQL through `Bun.sql` rather than `mysql2`. Verified against + * MySQL 8.4 -- the adapter landed in Bun 1.2.21, but VARBINARY/BLOB columns + * came back as lossily decoded strings until 1.3.6, which would silently + * corrupt exported password hashes. The 1.3.13 floor above already covers it; + * do not drop below 1.3.6 if the `--parallel` requirement ever goes away. + * * Usage: * bun run scripts/check-bun-version.ts */ From 6e747cde91e33f1fd61722ecd68f02165ab0ed50 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 12:34:51 -0400 Subject: [PATCH 02/39] docs(changeset): Add clerk migrate export for Clerk and Supabase, and import by export run ID Co-Authored-By: Claude Opus 5.5 --- .changeset/migrate-export.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/migrate-export.md diff --git a/.changeset/migrate-export.md b/.changeset/migrate-export.md new file mode 100644 index 000000000..df70f88da --- /dev/null +++ b/.changeset/migrate-export.md @@ -0,0 +1,5 @@ +--- +"clerk": minor +--- + +Add `clerk migrate export clerk` and `clerk migrate export supabase`, and let `clerk migrate import` take the export's run ID in place of a file and `--source`. `clerk migrate` is still experimental and needs `CLERK_EXPERIMENTAL=migrate`. From d42b5d92a9fc61515d167fef78e07b62c806b0ad Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 20:07:37 -0400 Subject: [PATCH 03/39] fix(build): keep the bun builtins external in the node bundle Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli-core/package.json b/packages/cli-core/package.json index 7dfba9886..2d9e785bc 100644 --- a/packages/cli-core/package.json +++ b/packages/cli-core/package.json @@ -7,7 +7,7 @@ }, "type": "module", "scripts": { - "build": "bun build ./src/cli.ts --outfile ./dist/cli.js --target node --external @napi-rs/keyring --minify", + "build": "bun build ./src/cli.ts --outfile ./dist/cli.js --target node --external @napi-rs/keyring --external bun --external bun:sqlite --minify", "build:compile": "bun build --compile --minify --no-compile-autoload-dotenv --no-compile-autoload-bunfig ./src/cli.ts --outfile ./dist/clerk", "dev": "bun run ./src/cli.ts", "typecheck": "tsc --noEmit -p tsconfig.json", From 0887f04812b0e01b598f9fa91108c2319a702a6f Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 20:07:37 -0400 Subject: [PATCH 04/39] fix(migrate): make an existing --output file owner-only too Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/shared.test.ts | 10 ++++++++++ .../cli-core/src/commands/migrate/export/shared.ts | 2 ++ 2 files changed, 12 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts index 2270948df..8939368b8 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.test.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -58,6 +58,16 @@ describe("finishExport", () => { expect(fs.statSync(path.dirname(outputPath)).mode & 0o777).toBe(0o700); }); + test("tightens an existing --output file to owner-only", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + const output = path.join(runsDir, "existing.json"); + fs.writeFileSync(output, "", { mode: 0o644 }); + + const { outputPath } = finishExport({ run, options: { output }, users, coverage }); + + expect(fs.statSync(outputPath).mode & 0o777).toBe(0o600); + }); + test("--output writes somewhere else, and the run still records where", async () => { const run = await startExportRun({ runsDir }, { platform: "supabase" }); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index c7c550a23..98fcb75c6 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -61,6 +61,8 @@ export function writeExportFile(file: string, envelope: ExportEnvelope): string // Password hashes, PII and a Firebase signer key: owner-only. fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); fs.writeFileSync(file, JSON.stringify(envelope, null, 2), { mode: 0o600 }); + // `mode` only applies on create; an existing `--output` keeps its own. + fs.chmodSync(file, 0o600); return file; } From 65f41fb9d9d3411c3fc5e170cd771847de0c4c8a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 20:07:38 -0400 Subject: [PATCH 05/39] fix(migrate): time out a libsql request that stops responding Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/lib/db.test.ts | 4 +++- packages/cli-core/src/commands/migrate/lib/db.ts | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts index b7ea84c9c..a262644d6 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -98,7 +98,7 @@ describe("sqlitePath", () => { describe("a libsql client", () => { const originalFetch = globalThis.fetch; - let requests: { url: string; token?: string; body: any }[] = []; + let requests: { url: string; token?: string; body: any; signal?: AbortSignal | null }[] = []; function stubFetch(result: unknown) { requests = []; @@ -107,6 +107,7 @@ describe("a libsql client", () => { url: String(url), token: (init.headers as Record).authorization, body: JSON.parse(String(init.body)), + signal: init.signal, }); return new Response(JSON.stringify({ results: [result, { type: "ok" }] }), { headers: { "content-type": "application/json" }, @@ -145,6 +146,7 @@ describe("a libsql client", () => { expect(requests[0]?.url).toBe("https://app-org.turso.io/v2/pipeline"); expect(requests[0]?.token).toBe("Bearer t0ken"); + expect(requests[0]?.signal).toBeInstanceOf(AbortSignal); expect(requests.at(-1)?.body.requests[0].stmt.args).toEqual([{ type: "text", value: "u1" }]); expect(rows).toEqual([ { diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index b3dca8c43..c2d386fef 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -157,6 +157,8 @@ function encodeHrana(param: unknown): HranaValue { * prints — or from `TURSO_AUTH_TOKEN`/`LIBSQL_AUTH_TOKEN`. A self-hosted sqld * with auth disabled needs neither, so a missing token is not an error here. */ +const LIBSQL_TIMEOUT_MS = 120_000; + function libsqlClient( connectionString: string, env: Record = process.env, @@ -170,6 +172,8 @@ function libsqlClient( async query>(query: string, params: unknown[] = []) { const response = await fetch(endpoint, { method: "POST", + // Generous, since one unpaginated SELECT can be a whole users table. + signal: AbortSignal.timeout(LIBSQL_TIMEOUT_MS), headers: { "content-type": "application/json", ...(token ? { authorization: `Bearer ${token}` } : {}), From 65c30f56135f29f983b8e04dee26efce44cd1250 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 6 Oct 2026 20:15:38 -0400 Subject: [PATCH 06/39] fix(migrate): redact credentials for schemes with - or . Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/lib/db.test.ts | 1 + packages/cli-core/src/commands/migrate/lib/db.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts index a262644d6..80afe6b3d 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -53,6 +53,7 @@ describe("redactConnectionString", () => { ["postgres://user:secret@host:5432/db", "postgres://***@host:5432/db"], ["mysql://root:hunter2@127.0.0.1:3306/app", "mysql://***@127.0.0.1:3306/app"], ["postgres://host/db", "postgres://host/db"], + ["postgres-x://u:secret@host/db", "postgres-x://***@host/db"], ["libsql://app.turso.io?authToken=secret", "libsql://app.turso.io?authToken=***"], ["postgres://u@host/db?password=secret", "postgres://***@host/db?password=***"], [ diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index c2d386fef..958bc80bd 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -67,7 +67,7 @@ export function redactConnectionString(connectionString: string): string { // Turso carries its credential as `?authToken=`, not as userinfo, and a // `?password=` (which Bun.SQL ignores) still must not be printed. return connectionString - .replace(/^([a-z0-9+]+:\/\/)(.*)@/i, "$1***@") + .replace(/^([a-z][a-z0-9+.-]*:\/\/)(.*)@/i, "$1***@") .replace(/([?&](?:authToken|password)=)[^&]*/gi, "$1***"); } From efb7a481b3ef96095f7722d03e1041fe587a4aa5 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 14:46:19 -0400 Subject: [PATCH 07/39] fix(migrate): never prompt under --json, even at a terminal `--json` is documented as non-interactive, but the export credential resolvers, the export picker, the Clerk instance picker and withInputRetry only checked the mode. With a human at the TTY they still prompted. Each now treats `--json` the way it treats agent mode. Co-Authored-By: Claude Opus 5.5 --- .../migrate/export/clerk-source.test.ts | 19 +++++++++++ .../commands/migrate/export/clerk-source.ts | 5 ++- .../src/commands/migrate/export/clerk.ts | 1 + .../src/commands/migrate/export/db-options.ts | 2 +- .../src/commands/migrate/export/index.ts | 2 +- .../src/commands/migrate/export/supabase.ts | 1 + .../commands/migrate/lib/input-retry.test.ts | 32 ++++++++++++++++++- .../src/commands/migrate/lib/input-retry.ts | 8 +++-- 8 files changed, 63 insertions(+), 7 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts index 1e5d95566..cab31394e 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts @@ -213,6 +213,25 @@ describe("resolveClerkSource", () => { expect(mockSearch).not.toHaveBeenCalled(); }); + // `--json` never prompts, even at a terminal: it takes what an agent would. + test("--json takes the resolved instance without prompting", async () => { + stubResolved("my-app (production)"); + + const source = await resolveClerkSource({ json: true }); + + expect(source.secretKey).toBe("sk_test_resolved"); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("--json in an unlinked directory fails rather than opening the picker", async () => { + const failure = new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }); + mockDescribeBapiTarget.mockRejectedValue(failure); + + await expect(resolveClerkSource({ json: true })).rejects.toThrow(failure); + + expect(mockResolveUsersInstanceContext).not.toHaveBeenCalled(); + }); + test("an unlinked directory picks an application instead of failing", async () => { mockDescribeBapiTarget.mockRejectedValue( new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }), diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.ts index 78dfa201c..57b9b1fb6 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk-source.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.ts @@ -47,6 +47,8 @@ export type ResolveClerkSourceOptions = { app?: string; instance?: string; cwd?: string; + /** `--json` never prompts. */ + json?: boolean; }; export type ClerkExportSource = { @@ -88,6 +90,7 @@ async function resolveSource(options: ResolveClerkSourceOptions): Promise { const { chosen, ...source } = await resolveSource(options); - if (chosen || !source.target || !isHuman()) return source; + if (chosen || !source.target || options.json || !isHuman()) return source; const picked = await pickInstance(await currentAppId(options)); if (picked) return picked; diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 58945c1d9..12e1f064e 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -254,6 +254,7 @@ export async function exportClerk(options: ExportClerkOptions): Promise { secretKey: options.secretKey, app: options.app, instance: options.instance, + json: options.json, }); await withGutter("Exporting users from Clerk", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 3c0ea5a4c..69868a107 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -125,7 +125,7 @@ export async function resolveDbUrl( log.warn(`${config.envVar} is not a valid connection string; ignoring it.`); } - if (isAgent() || !isHuman()) { + if (options.json || isAgent() || !isHuman()) { throwUsageError( `\`clerk migrate export ${config.platform}\` needs a database connection and cannot prompt here.\n` + `Pass --db-url, or set ${config.envVar}.`, diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index 898604c6c..0330f4f7d 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -16,7 +16,7 @@ import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./regist * platform name, it prompts for itself. */ export async function exportPicker(options: Record = {}): Promise { - if (isAgent() || !isHuman()) { + if (options.json || isAgent() || !isHuman()) { throwUsageError( `\`clerk migrate export\` needs a platform and cannot prompt here. Name one: ${exportPlatformKeys().join(", ")}.`, undefined, diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index abea84f7e..5e87898d4 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -135,6 +135,7 @@ export async function exportSupabase(options: DbExportOptions): Promise { withSpinner("Reading auth.users...", async () => withDbClient(connectionString, "supabase", fetchSupabaseUsers), ), + options, ); const run = await startExportRun(options, { platform: "supabase" }); diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts index d77c09506..e1c15e0e2 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts @@ -31,7 +31,7 @@ mock.module("../../../lib/prompts.ts", () => ({ })); const { withInputRetry } = await import("./input-retry.ts"); -const { promptDbUrl } = await import("../export/db-options.ts"); +const { promptDbUrl, resolveDbUrl } = await import("../export/db-options.ts"); const { setAssumeYes } = await import("./assume-yes.ts"); const captured = useCaptureLog(); @@ -172,6 +172,25 @@ describe("withInputRetry", () => { expect(attempts).toBe(1); }); + // `--json` is non-interactive by contract, even when a human is at the TTY. + test("throws without prompting under --json", async () => { + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw rejected(); + }, + { json: true }, + ), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); + // `-y` is a human on a TTY who could be asked and said not to. Agent mode // cannot reach the prompt at all; this one can and declines to, so it needs // its own check rather than riding on the mode assertion above. @@ -232,3 +251,14 @@ describe("withInputRetry", () => { expect(attempts).toBe(1); }); }); + +// Here rather than in `db-exports.test.ts` because the prompt is mocked: a +// missed guard reaches it and returns, where a real prompt would hang the run. +describe("resolveDbUrl", () => { + test("does not prompt under --json, even with a human at the TTY", async () => { + answers = [FIRST]; + + await expect(resolveDbUrl({ json: true }, CONFIG, {})).rejects.toThrow(/cannot prompt here/); + expect(answers).toEqual([FIRST]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.ts index 13ec1b2c0..c8aed0bb4 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.ts @@ -28,8 +28,8 @@ import { isAssumeYes } from "./assume-yes.ts"; * already written a file, or a long fetch the credential has already been * accepted for, does not belong in here. * - * `-y`, agent mode and a non-TTY get the failure unchanged: there is nobody to - * ask, and a loop that cannot prompt is a loop that cannot end. A cancelled + * `-y`, `--json`, agent mode and a non-TTY get the failure unchanged: there is + * nobody to ask, and a loop that cannot prompt is a loop that cannot end. A cancelled * prompt throws {@link UserAbortError}, which is not a `CliError` and so leaves * the loop — declining the question is an answer. * @@ -37,6 +37,7 @@ import { isAssumeYes } from "./assume-yes.ts"; * answer to the prompt the caller has already put up. * @param reprompt - Asks for a replacement. Called once per failure. * @param work - The step the input has to survive. + * @param options - The command's options; `--json` never prompts. * @returns The result, and the input that produced it — which is not `input` * when it took a retry, and later steps need the one that worked. */ @@ -44,6 +45,7 @@ export async function withInputRetry( input: I, reprompt: () => Promise, work: (input: I) => Promise, + options: { json?: boolean } = {}, ): Promise<{ value: T; input: I }> { let candidate = input; @@ -56,7 +58,7 @@ export async function withInputRetry( // connection is not fixed by another credential, and anything else (an // interrupt, a bug) is not ours to retry. const badInput = error instanceof CliError && error.exitCode === EXIT_CODE.USAGE; - if (!badInput || !isHuman() || isAgent() || isAssumeYes()) throw error; + if (!badInput || options.json || !isHuman() || isAgent() || isAssumeYes()) throw error; log.error(error.message); candidate = await reprompt(); From 14c6763d234f87f11a25e9f5a08bc013cca5da6b Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Wed, 7 Oct 2026 17:51:15 -0400 Subject: [PATCH 08/39] fix(migrate): redact credentials from database error messages The connection string was redacted where we print it, but the driver's or server's own message was appended as is, and a libsql server or a driver can quote the URL, the password or the token. Connection and query failures now pass that message through redactDbMessage too. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/lib/db.test.ts | 53 +++++++++++++++ .../cli-core/src/commands/migrate/lib/db.ts | 64 +++++++++++++++++-- 2 files changed, 113 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts index 80afe6b3d..7290720d8 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -9,6 +9,7 @@ import { describeDbError, detectDbType, redactConnectionString, + redactDbMessage, sqlitePath, withDbClient, } from "./db.ts"; @@ -85,6 +86,29 @@ describe("redactConnectionString", () => { ); }); +describe("redactDbMessage", () => { + test.each([ + ["the whole URL", "failed: postgres://u:pw%40x@h/db", "failed: postgres://***@h/db"], + ["the password, raw", 'auth failed for "pw%40x"', 'auth failed for "***"'], + ["the password, decoded", 'auth failed for "pw@x"', 'auth failed for "***"'], + ])("redacts %s", (_label, message, expected) => { + expect(redactDbMessage(message, "postgres://u:pw%40x@h/db")).toBe(expected); + }); + + test("redacts a libsql token from the URL or the environment", () => { + expect(redactDbMessage("token abc is bad", "libsql://h.turso.io?authToken=abc")).toBe( + "token *** is bad", + ); + expect( + redactDbMessage("token envtok is bad", "libsql://h.turso.io", { TURSO_AUTH_TOKEN: "envtok" }), + ).toBe("token *** is bad"); + }); + + test("leaves a message with no credentials alone", () => { + expect(redactDbMessage("no such table: user", "./db.sqlite")).toBe("no such table: user"); + }); +}); + describe("sqlitePath", () => { test.each([ ["./db.sqlite", "./db.sqlite"], @@ -161,6 +185,35 @@ describe("a libsql client", () => { expect(client.dbType).toBe("sqlite"); }); + // A server that echoes the token must not get it printed. + test("redacts the token from a server error it echoes", async () => { + stubFetch({ type: "error", error: { message: "bad auth token t0ken for app-org" } }); + + const error = (await createDbClient("libsql://app-org.turso.io?authToken=t0ken").catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error).toBeInstanceOf(CliError); + expect(error.message).not.toContain("t0ken"); + expect(error.message).toContain("bad auth token *** for app-org"); + }); + + test("redacts the token from a query failure after connecting", async () => { + stubFetch(okRows(["1"], [[{ type: "integer", value: "1" }]])); + + const error = (await withDbClient( + "libsql://app-org.turso.io?authToken=t0ken", + undefined, + async () => { + throw new Error("query rejected for token t0ken"); + }, + ).catch((caught: unknown) => caught)) as CliError; + + expect(error).toBeInstanceOf(CliError); + expect(error.message).toContain("query rejected for token ***"); + expect(error.message).not.toContain("t0ken"); + }); + test("reports a server-side error", async () => { stubFetch({ type: "error", error: { message: "no such table: user" } }); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index 958bc80bd..e7b75c8af 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -71,6 +71,55 @@ export function redactConnectionString(connectionString: string): string { .replace(/([?&](?:authToken|password)=)[^&]*/gi, "$1***"); } +/** + * A driver's error message with every credential from `connectionString` + * replaced by `***`. + * + * {@link redactConnectionString} covers the string we print; this covers what + * the server or driver says back, which can quote the URL, the password or a + * libsql token. A short password can blank an unrelated word too, and a + * mangled message is the better failure. + */ +export function redactDbMessage( + message: string, + connectionString: string, + env: Record = process.env, +): string { + const trimmed = connectionString.trim(); + let redacted = message.split(trimmed).join(redactConnectionString(trimmed)); + + const secrets = new Set(); + try { + const url = new URL(trimmed); + for (const value of [ + url.password, + url.searchParams.get("authToken"), + url.searchParams.get("password"), + ]) { + if (!value) continue; + secrets.add(value); + try { + secrets.add(decodeURIComponent(value)); + } catch { + // Not valid percent-encoding: the raw form is the only one there is. + } + } + } catch { + // A SQLite path is not a URL, and carries no credentials. + } + if (isLibsqlUrl(trimmed)) { + for (const value of [env.TURSO_AUTH_TOKEN, env.LIBSQL_AUTH_TOKEN]) { + if (value) secrets.add(value); + } + } + + // Longest first, so a secret that contains another is replaced whole. + for (const secret of [...secrets].sort((a, b) => b.length - a.length)) { + redacted = redacted.split(secret).join("***"); + } + return redacted; +} + /** Strips a `file:` prefix and any URL query, leaving a filesystem path. */ export function sqlitePath(connectionString: string): string { const trimmed = connectionString.trim(); @@ -348,7 +397,10 @@ function connectionError( connectionString: string, platform?: DbPlatform, ): CliError { - const message = error instanceof Error ? error.message : String(error); + const message = redactDbMessage( + error instanceof Error ? error.message : String(error), + connectionString, + ); return new CliError( `Could not connect to ${redactConnectionString(connectionString)}: ${message}\n\n${describeDbError(error, platform)}`, { code: ERROR_CODE.USAGE_ERROR, exitCode: EXIT_CODE.USAGE }, @@ -373,10 +425,14 @@ export async function withDbClient( // A query failure carries the same actionable hints as a connection one: // a missing table is the most common thing that goes wrong here. if (error instanceof CliError) throw error; - throw new CliError( - `${error instanceof Error ? error.message : String(error)}\n\n${describeDbError(error, platform)}`, - { code: ERROR_CODE.USAGE_ERROR, exitCode: EXIT_CODE.USAGE }, + const message = redactDbMessage( + error instanceof Error ? error.message : String(error), + connectionString, ); + throw new CliError(`${message}\n\n${describeDbError(error, platform)}`, { + code: ERROR_CODE.USAGE_ERROR, + exitCode: EXIT_CODE.USAGE, + }); } finally { await client.close().catch(() => {}); } From c8b99975f26eb403c828889ad34b71ddc67865fc Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 8 Oct 2026 13:53:43 -0400 Subject: [PATCH 09/39] fix(migrate): give MySQL's missing-table errors the table advice MySQL says "doesn't exist" and "Unknown table" where Postgres says "does not exist", so a missing table on MySQL fell through to the connection advice, though the connection had worked. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/lib/db.test.ts | 2 ++ packages/cli-core/src/commands/migrate/lib/db.ts | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts index 7290720d8..a4c34bba0 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -354,6 +354,8 @@ describe("describeDbError", () => { ['relation "auth.users" does not exist'], ["no such table: user"], ["permission denied for table users"], + ["Table 'app.users' doesn't exist"], + ["Unknown table 'users'"], ])("recognizes the missing table in %p", (message) => { expect(describeDbError(new Error(message))).toMatch(/table was not found|cannot read it/); }); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts index e7b75c8af..ab362ab5a 100644 --- a/packages/cli-core/src/commands/migrate/lib/db.ts +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -375,7 +375,12 @@ export function describeDbError(error: unknown, platform?: DbPlatform): string { return "The user table is missing a column the export reads. Check the schema matches the platform's default."; } - if (/does not exist|unknown database|no such table|permission denied/i.test(message)) { + // MySQL says "doesn't exist" and "Unknown table" where Postgres says "does not exist". + if ( + /does not exist|doesn't exist|unknown table|unknown database|no such table|permission denied/i.test( + message, + ) + ) { if (platform === "supabase") { return ( "The auth.users table was not readable. It is created automatically when Supabase Auth is enabled.\n" + From 9a1c151e5cec56ec005d68ccbc6b1edbeb3b133f Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 8 Oct 2026 15:58:10 -0400 Subject: [PATCH 10/39] docs(migrate): say why -y is held per run, now that withInputRetry reads it Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/lib/assume-yes.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/assume-yes.ts b/packages/cli-core/src/commands/migrate/lib/assume-yes.ts index c61c7ef86..dfcd748fe 100644 --- a/packages/cli-core/src/commands/migrate/lib/assume-yes.ts +++ b/packages/cli-core/src/commands/migrate/lib/assume-yes.ts @@ -2,13 +2,16 @@ * Whether this run was given `-y`. * * Not the same question as {@link isAgent}: agent mode says the CLI *cannot* - * prompt; `-y` says the operator does not want it to. + * prompt; `-y` says the operator does not want it to. A confirm is skipped by + * either, but `withInputRetry` needs to know a human chose not to be asked: + * under `-y` a rejected credential fails instead of being asked for again. * * Set once by the `migrate` group's `preAction` hook, so every subcommand under * it is covered whether or not it declares the flag — one that does not simply - * resolves to `false`. Held per run, the way `mode.ts` holds `--mode`, so code - * below a command's options can read it without the flag being passed down. - * Nothing in `import` needs that: it reads `options.yes` directly. + * resolves to `false`. Held per run, the way `mode.ts` holds `--mode`, because + * its readers sit layers below the command that parses it: `withInputRetry` + * runs under every credential prompt, and passing `yes` down would put it on + * every export handler on the way. */ let assumeYes = false; From 8c2c0191f2719fe75c0900bd9d45b6b125dbb3bb Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 8 Oct 2026 16:23:15 -0400 Subject: [PATCH 11/39] fix(migrate): let the Supabase source split names instead of the export's SQL The export coalesced display_name, first_name and name into first_name. The source works names out of the metadata only when first_name is empty, so an OAuth "Jane Doe" came across as one first name, and a display name outranked a real first name. The export now selects only the metadata's own first_name. Co-Authored-By: Claude Opus 5.5 --- .../migrate/export/db-exports.test.ts | 17 +++++++++++++++- .../src/commands/migrate/export/supabase.ts | 14 ++++++------- .../commands/migrate/sources/sources.test.ts | 20 +++++++++++++++++++ 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 138aca86b..b8908e84a 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -12,9 +12,10 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { CliError } from "../../../lib/errors.ts"; +import type { DbClient } from "../lib/db.ts"; import type { UserLine } from "../lib/run-store.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; -import { buildSupabaseExport, exportSupabase } from "./supabase.ts"; +import { buildSupabaseExport, exportSupabase, fetchSupabaseUsers } from "./supabase.ts"; import { looksLikeConnectionString, normalizeConnectionString, @@ -130,6 +131,20 @@ describe("resolveDbUrl", () => { }); describe("supabase export", () => { + // The source splits a display or OAuth name, but only when first_name is + // empty; coalescing them into first_name here would stop it. + test("selects only the metadata's own first_name as first_name", async () => { + let sql = ""; + await fetchSupabaseUsers({ + query: async (query: string) => { + sql = query; + return []; + }, + } as unknown as DbClient); + expect(sql).toContain("raw_user_meta_data->>'first_name' AS first_name"); + expect(sql).not.toContain("display_name"); + }); + test("serializes timestamps the transformer can parse", () => { const { users } = buildSupabaseExport([ { id: "u1", email: "a@x.dev", created_at: new Date("2024-01-01T00:00:00Z") }, diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 5e87898d4..2bc1f42fd 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -25,9 +25,11 @@ import { import { withInputRetry } from "../lib/input-retry.ts"; /** - * `display_name` is coalesced into `first_name` here rather than in the - * transformer so a user who writes their own SQL sees the shape the - * transformer expects. + * Only the metadata's own `first_name` and `last_name` are pulled out as + * columns. A `display_name`, `full_name` or OAuth `name` stays in + * `raw_user_meta_data` for the Supabase source to split, which it does only + * when `first_name` is empty: coalescing them in here would keep "Jane Doe" + * as one first name, and let a display name outrank a real first name. */ const EXPORT_QUERY = ` SELECT @@ -37,11 +39,7 @@ const EXPORT_QUERY = ` encrypted_password, phone, phone_confirmed_at, - COALESCE( - raw_user_meta_data->>'display_name', - raw_user_meta_data->>'first_name', - raw_user_meta_data->>'name' - ) AS first_name, + raw_user_meta_data->>'first_name' AS first_name, raw_user_meta_data->>'last_name' AS last_name, raw_user_meta_data, raw_app_meta_data, diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts index 4b023d882..5a5881207 100644 --- a/packages/cli-core/src/commands/migrate/sources/sources.test.ts +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -105,6 +105,26 @@ describe("isVerified", () => { }); describe("supabase", () => { + // What `export supabase` hands over: first_name only when the metadata has + // one, the rest left in the metadata for this source to split. + test.each([ + ["an OAuth name", { name: "Jane Doe" }, "Jane", "Doe"], + [ + "a display name beside a real last name", + { display_name: "Jane Doe", last_name: "Doe" }, + "Jane", + "Doe", + ], + ])("splits %s from the metadata", (_label, meta, firstName, lastName) => { + const user = one("supabase", { + id: "s1", + email: "a@x.dev", + last_name: (meta as { last_name?: string }).last_name, + raw_user_meta_data: meta, + }); + expect(user).toMatchObject({ firstName, lastName }); + }); + // `NULL` is how SQL tools write an empty column into a CSV. test("reads NULL cells in a CSV as empty, not as values", async () => { const { users, failures } = await load( From 5c41a6f28df1416f567942653a172e8be74a45ae Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 8 Oct 2026 16:24:28 -0400 Subject: [PATCH 12/39] fix(migrate): refuse an export clerk --instance the key does not address A key from --secret-key or CLERK_SECRET_KEY picks the instance and ignores --instance, so a dev key with --instance prod read the dev users while the run was labelled production. The export now runs the import's check, and records where the key came from. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/clerk.test.ts | 11 +++++++++++ .../src/commands/migrate/export/clerk.ts | 19 +++++++++++++++++-- .../src/commands/migrate/lib/target.ts | 2 +- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 992b78841..2ea77d1d4 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -310,6 +310,17 @@ describe("exportClerk", () => { expect(captured.err).toMatch(/clerk migrate import \d{8}-\d{6}-[0-9a-f]{4}/); }); + // The key decides the instance; --instance naming a different one would + // label a read of one user pool with another's name. + test("refuses an --instance the key does not address, before reading any user", async () => { + stubPages([[user()], []]); + + await expect(exportClerk({ secretKey: "sk_test_x", instance: "dev" })).rejects.toThrow( + /--instance dev does not match the key from --secret-key/, + ); + expect(requests).toEqual([]); + }); + test("names the instance it reads from first, and records it on the run", async () => { stubPages([[user()], []]); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 12e1f064e..52fc2a03f 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -19,7 +19,12 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import type { UserLine } from "../lib/run-store.ts"; import { retryOn429 } from "../lib/retry.ts"; -import { fetchInstanceIdentity, printTarget } from "../lib/target.ts"; +import { + assertInstanceFlagMatches, + fetchInstanceIdentity, + KEY_SOURCE, + printTarget, +} from "../lib/target.ts"; import { resolveClerkSource } from "./clerk-source.ts"; import { finishExport, startExportRun } from "./shared.ts"; @@ -259,11 +264,21 @@ export async function exportClerk(options: ExportClerkOptions): Promise { await withGutter("Exporting users from Clerk", async () => { const identity = await fetchInstanceIdentity(source.secretKey); + // A key picks the instance on its own and ignores --instance, so the two + // must agree, as on import: otherwise this reads the wrong user pool under + // the requested instance's name. + const keyFrom = options.secretKey + ? KEY_SOURCE.secretKeyFlag + : !options.app && process.env.CLERK_SECRET_KEY + ? KEY_SOURCE.envVar + : undefined; + if (keyFrom) assertInstanceFlagMatches({ instance: options.instance }, keyFrom, identity); + const keySource = keyFrom ?? source.target; const target = { platform: "clerk", env: identity.env, instanceId: identity.instanceId, - ...(source.target ? { keySource: source.target } : {}), + ...(keySource ? { keySource } : {}), }; if (!options.json) printTarget(target); diff --git a/packages/cli-core/src/commands/migrate/lib/target.ts b/packages/cli-core/src/commands/migrate/lib/target.ts index 01a8798e4..2b8af2742 100644 --- a/packages/cli-core/src/commands/migrate/lib/target.ts +++ b/packages/cli-core/src/commands/migrate/lib/target.ts @@ -212,7 +212,7 @@ export function keyInstanceId(secretKey: string): string { * writes and deletes in bulk, so `--instance dev` next to an exported * `sk_live_` key must not reach production. */ -function assertInstanceFlagMatches( +export function assertInstanceFlagMatches( options: TargetOptions, keySource: string, identity: InstanceIdentity, From a7c2d8d5665d10504aaf4b549791501edd9aaa2d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 8 Oct 2026 16:25:51 -0400 Subject: [PATCH 13/39] fix(migrate): retry only the connect for a Supabase export, not the read The whole read sat inside the credential retry, and withDbClient turns every query error into a usage error, so a statement timeout partway through auth.users asked for the connection string again, though it had been accepted, and exited 2. withDbConnection retries the connect alone, then runs the read: a failure there exits 1 with its hint. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/db-options.ts | 52 ++++++++++++++++++- .../src/commands/migrate/export/supabase.ts | 15 ++---- .../commands/migrate/lib/input-retry.test.ts | 44 +++++++++++++++- 3 files changed, 97 insertions(+), 14 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 69868a107..4c8900e93 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -5,12 +5,23 @@ * from a flag, an environment variable, or a prompt. */ -import { throwUsageError } from "../../../lib/errors.ts"; +import { CliError, throwUsageError } from "../../../lib/errors.ts"; import { dim } from "../../../lib/color.ts"; import { log } from "../../../lib/log.ts"; import { password as passwordPrompt } from "../../../lib/prompts.ts"; +import { withSpinner } from "../../../lib/spinner.ts"; import { isAgent, isHuman } from "../../../mode.ts"; -import { detectDbType, isLibsqlUrl, redactConnectionString, type DbPlatform } from "../lib/db.ts"; +import { + createDbClient, + describeDbError, + detectDbType, + isLibsqlUrl, + redactConnectionString, + redactDbMessage, + type DbClient, + type DbPlatform, +} from "../lib/db.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; export type DbExportOptions = { dbUrl?: string; @@ -169,3 +180,40 @@ export function describeTarget(connectionString: string): string { const label = isLibsqlUrl(connectionString) ? "libsql" : detectDbType(connectionString); return `${label} at ${redactConnectionString(connectionString)}`; } + +/** + * Connects, asking for the connection string again if the server refuses it, + * then runs `work` on the client and closes it. + * + * Only the connect is retried. Once it has worked, the string is right: a + * statement timeout or a dropped connection partway through a large read is + * not fixed by asking for it again, so it fails (exit 1) with the hint for it. + */ +export async function withDbConnection( + dbUrl: string, + config: ResolveConfig, + options: { json?: boolean }, + work: (client: DbClient) => Promise, +): Promise { + const { value: client, input: connectionString } = await withInputRetry( + dbUrl, + async () => promptDbUrl(config), + async (candidate) => + withSpinner("Connecting to the database...", async () => + createDbClient(candidate, config.platform), + ), + options, + ); + try { + return await work(client); + } catch (error) { + if (error instanceof CliError) throw error; + const message = redactDbMessage( + error instanceof Error ? error.message : String(error), + connectionString, + ); + throw new CliError(`${message}\n\n${describeDbError(error, config.platform)}`); + } finally { + await client.close().catch(() => {}); + } +} diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 2bc1f42fd..263c97d35 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -14,15 +14,14 @@ import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import type { UserLine } from "../lib/run-store.ts"; import { printTarget } from "../lib/target.ts"; -import { withDbClient, type DbClient } from "../lib/db.ts"; +import type { DbClient } from "../lib/db.ts"; import { finishExport, startExportRun } from "./shared.ts"; import { - promptDbUrl, resolveDbUrl, + withDbConnection, type DbExportOptions, type ResolveConfig, } from "./db-options.ts"; -import { withInputRetry } from "../lib/input-retry.ts"; /** * Only the metadata's own `first_name` and `last_name` are pulled out as @@ -126,14 +125,8 @@ export async function exportSupabase(options: DbExportOptions): Promise { await withGutter("Exporting users from Supabase", async () => { if (!options.json) printTarget({ platform: "supabase" }); - const { value: rows } = await withInputRetry( - dbUrl, - async () => promptDbUrl(SUPABASE_DB), - async (connectionString) => - withSpinner("Reading auth.users...", async () => - withDbClient(connectionString, "supabase", fetchSupabaseUsers), - ), - options, + const rows = await withDbConnection(dbUrl, SUPABASE_DB, options, async (client) => + withSpinner("Reading auth.users...", async () => fetchSupabaseUsers(client)), ); const run = await startExportRun(options, { platform: "supabase" }); diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts index e1c15e0e2..fa511394d 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts @@ -9,6 +9,10 @@ */ import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { CliError, ERROR_CODE, EXIT_CODE, UserAbortError } from "../../../lib/errors.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; @@ -31,7 +35,7 @@ mock.module("../../../lib/prompts.ts", () => ({ })); const { withInputRetry } = await import("./input-retry.ts"); -const { promptDbUrl, resolveDbUrl } = await import("../export/db-options.ts"); +const { promptDbUrl, resolveDbUrl, withDbConnection } = await import("../export/db-options.ts"); const { setAssumeYes } = await import("./assume-yes.ts"); const captured = useCaptureLog(); @@ -262,3 +266,41 @@ describe("resolveDbUrl", () => { expect(answers).toEqual([FIRST]); }); }); + +// Only the connect proves the connection string. A failure after it is not a +// wrong string, so it is not asked for again, and it is not a usage error. +describe("withDbConnection", () => { + const sqlite = () => { + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-dbconn-")), "db.sqlite"); + new Database(file, { create: true }).close(); + return file; + }; + + test("asks again when the connection fails, then runs the work once", async () => { + answers = [sqlite()]; + let runs = 0; + + const value = await withDbConnection("./no-such-dir/missing.sqlite", CONFIG, {}, async () => { + runs++; + return "rows"; + }); + + expect(value).toBe("rows"); + expect(runs).toBe(1); + expect(answers).toEqual([]); + }); + + test("does not ask again when the read fails after connecting, and exits 1", async () => { + answers = [sqlite()]; + const file = sqlite(); + + const error = (await withDbConnection(file, CONFIG, {}, async () => { + throw new Error("canceling statement due to statement timeout"); + }).catch((caught: unknown) => caught)) as CliError; + + expect(error).toBeInstanceOf(CliError); + expect(error.exitCode).toBe(EXIT_CODE.GENERAL); + expect(error.message).toContain("statement timeout"); + expect(answers).toHaveLength(1); + }); +}); From d79727e376fa14abaabe01de0246f7a9d1625099 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 8 Oct 2026 16:26:42 -0400 Subject: [PATCH 14/39] fix(migrate): name an export file outside its run folder in the cleanup hint With export --output, the file holding the password hashes is not in the export run's folder, so deleting the folder as suggested left it on disk. The hint now adds an rm for that file. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/run.test.ts | 11 +++++++++++ packages/cli-core/src/commands/migrate/run.ts | 9 ++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 03929ac11..066ae1063 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -320,6 +320,17 @@ describe("run", () => { const { source: _source, input: _input, ...noSource } = baseOptions; + test("names an --output export file outside the run folder for deletion too", async () => { + const { record, file } = exportRun("clerk", export2); + const outside = path.join(workDir, `users-${record.id}.json`); + fs.renameSync(file, outside); + patchRun(runsDir(), record.id, { file: { path: outside, sha256: "x" } }); + + await run({ ...noSource, input: record.id }); + + expect(captured.err).toContain(`rm ${outside}`); + }); + test("imports by export run ID, with the source the envelope names", async () => { const { record } = exportRun("clerk", export2); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 099615b3e..95f9812f4 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -17,6 +17,7 @@ * 5. Import, one run line per user. */ +import path from "node:path"; import { bold, dim, green, red, yellow } from "../../lib/color.ts"; import { resolveProfile } from "../../lib/config.ts"; import { hasAccountCredentials } from "../../lib/credential-store.ts"; @@ -547,9 +548,15 @@ function formatSummary( function cleanupLines(runsDir: string, record: RunRecord): string[] { const lines: string[] = []; if (record.fromExport) { + const exportDir = runDir(runsDir, record.fromExport); + // `export --output` writes the file outside the run folder, and deleting + // the folder alone would leave the password hashes on disk. + const file = record.file?.path; + const outside = file && path.relative(exportDir, file).startsWith(".."); lines.push( `The export in run ${record.fromExport} holds your users' data. Once you have checked the import, delete it:`, - dim(` rm -rf ${quoteArg(runDir(runsDir, record.fromExport))}`), + dim(` rm -rf ${quoteArg(exportDir)}`), + ...(outside ? [dim(` rm ${quoteArg(file)}`)] : []), ); } lines.push( From e9d59c19cb117a161290167b2abce77077914a7f Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 8 Oct 2026 16:28:11 -0400 Subject: [PATCH 15/39] fix(migrate): overlap export clerk's pages so a deletion can't skip a user With oldest-first offset paging, a user deleted after an earlier page shifts every later row back one, so the row at the next page boundary was skipped, not repeated as the comment said, and the export reported success one user short. A count check can't see this: the deletion and the skip cancel out. Each page now re-reads the last 20 rows of the one before, and the Map drops the repeats. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/clerk.test.ts | 28 +++++++++++++++++-- .../src/commands/migrate/export/clerk.ts | 15 ++++++++-- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 2ea77d1d4..07afeb40d 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -207,11 +207,33 @@ describe("fetchAllClerkUsers", () => { expect(all).toHaveLength(512); expect(requests).toHaveLength(2); - expect(requests[1]).toContain("offset=500"); + expect(requests[1]).toContain("offset=480"); }); - // Oldest first, so a sign-up mid-export lands at the end; a row a deletion - // shifts onto the next page is dropped as a repeat. + // A user deleted after the first page shifts every later row back one. The + // overlap re-reads the boundary, so the row that slid onto it isn't lost. + test("keeps the user a mid-export deletion slides onto a page boundary", async () => { + const users = Array.from({ length: 600 }, (_, i) => user({ id: `u${i}` })); + let pages = 0; + // A server that answers by offset, and loses u10 after the first page. + globalThis.fetch = (async (input: string | URL | Request) => { + const url = new URL(input.toString()); + if (url.pathname === "/v1/instance") { + return Response.json({ object: "instance", id: "ins_src", environment_type: "production" }); + } + const offset = Number(url.searchParams.get("offset")); + const rows = pages++ === 0 ? users : users.filter((entry) => entry.id !== "u10"); + return Response.json(rows.slice(offset, offset + 500)); + }) as unknown as typeof fetch; + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all.map((entry) => entry.id)).toContain("u500"); + expect(all).toHaveLength(600); + }); + + // Oldest first, so a sign-up mid-export lands at the end; a repeat the + // overlap reads again is dropped. test("pages oldest first and drops a user a shifted page repeats", async () => { stubPages([ Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 52fc2a03f..6f66aeb01 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -31,6 +31,13 @@ import { finishExport, startExportRun } from "./shared.ts"; /** BAPI's maximum page size for `GET /v1/users`. */ const PAGE_SIZE = 500; +/** + * How many rows each page re-reads from the end of the one before. A user + * deleted mid-export shifts every later row back one, so without it the row + * at each page boundary would be skipped; the Map drops the repeats. + */ +const PAGE_OVERLAP = 20; + export type ExportClerkOptions = { output?: string; secretKey?: string; @@ -184,9 +191,11 @@ export async function fetchAllClerkUsers(options: { // Oldest first, so a sign-up during the export lands at the end instead of // shifting every later page by one (BAPI's default is newest first). A - // deletion can still shift a page; the Map drops the repeat that causes. - // ponytail: offset paging; /v1/users has no cursor to page by instead. - for (let offset = 0; ; offset += PAGE_SIZE) { + // deletion shifts later rows back, which would skip the row at the next + // page boundary, so each page overlaps the last. + // ponytail: offset paging, safe for up to PAGE_OVERLAP deletions per page + // mid-export; /v1/users has no cursor to page by instead. + for (let offset = 0; ; offset += PAGE_SIZE - PAGE_OVERLAP) { const response = await retryOn429(async () => bapiRequest({ method: "GET", From 658fa6dd7d700e28b4a8f1697269ec473910cfe5 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 8 Oct 2026 16:28:59 -0400 Subject: [PATCH 16/39] fix(migrate): let -y stand the exports' prompts down, as it says -y reads "Do not prompt", but only the credential retry honored it: in a linked directory the Clerk export still showed its instance picker, and the Supabase export still asked for a connection string. Under -y the picker takes the resolved instance, as an agent does, and a missing connection string is the usage error naming --db-url. Co-Authored-By: Claude Opus 5.5 --- .../migrate/export/clerk-source.test.ts | 22 +++++++++++++++++++ .../commands/migrate/export/clerk-source.ts | 5 ++++- .../src/commands/migrate/export/db-options.ts | 4 +++- .../commands/migrate/lib/input-retry.test.ts | 9 ++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts index cab31394e..1a3105ecd 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts @@ -38,11 +38,13 @@ mock.module("../../../mode.ts", () => ({ })); const { resolveClerkSource } = await import("./clerk-source.ts"); +const { setAssumeYes } = await import("../lib/assume-yes.ts"); const captured = useCaptureLog(); beforeEach(() => { human = true; + setAssumeYes(false); mockDescribeBapiTarget.mockReset(); mockResolveBapiSecretKey.mockReset(); mockResolveProfile.mockReset(); @@ -214,6 +216,26 @@ describe("resolveClerkSource", () => { }); // `--json` never prompts, even at a terminal: it takes what an agent would. + // `-y` is "do not prompt": it takes the resolved instance as an agent does. + test("-y takes the resolved instance without the picker", async () => { + stubResolved("my-app (production)"); + setAssumeYes(true); + + const source = await resolveClerkSource({}); + + expect(source.secretKey).toBe("sk_test_resolved"); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("-y in an unlinked directory fails rather than opening the picker", async () => { + const failure = new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }); + mockDescribeBapiTarget.mockRejectedValue(failure); + setAssumeYes(true); + + await expect(resolveClerkSource({})).rejects.toThrow(failure); + expect(mockResolveUsersInstanceContext).not.toHaveBeenCalled(); + }); + test("--json takes the resolved instance without prompting", async () => { stubResolved("my-app (production)"); diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.ts index 57b9b1fb6..897c44170 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk-source.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.ts @@ -33,6 +33,7 @@ import { search } from "../../../lib/listage.ts"; import type { ApplicationInstance } from "../../../lib/plapi.ts"; import { log } from "../../../lib/log.ts"; import { isHuman } from "../../../mode.ts"; +import { isAssumeYes } from "../lib/assume-yes.ts"; import { resolveUsersInstanceContext } from "../../users/interactive/instance-context.ts"; /** e.g. `Development instance`. Unknown environment types print as-is. */ @@ -91,6 +92,7 @@ async function resolveSource(options: ResolveClerkSourceOptions): Promise { const { chosen, ...source } = await resolveSource(options); - if (chosen || !source.target || options.json || !isHuman()) return source; + // `-y` takes the resolved instance without the picker, as an agent does. + if (chosen || !source.target || options.json || isAssumeYes() || !isHuman()) return source; const picked = await pickInstance(await currentAppId(options)); if (picked) return picked; diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 4c8900e93..a045a859b 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -21,6 +21,7 @@ import { type DbClient, type DbPlatform, } from "../lib/db.ts"; +import { isAssumeYes } from "../lib/assume-yes.ts"; import { withInputRetry } from "../lib/input-retry.ts"; export type DbExportOptions = { @@ -136,7 +137,8 @@ export async function resolveDbUrl( log.warn(`${config.envVar} is not a valid connection string; ignoring it.`); } - if (options.json || isAgent() || !isHuman()) { + // `-y` is "do not prompt": it gets the usage error naming --db-url too. + if (options.json || isAssumeYes() || isAgent() || !isHuman()) { throwUsageError( `\`clerk migrate export ${config.platform}\` needs a database connection and cannot prompt here.\n` + `Pass --db-url, or set ${config.envVar}.`, diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts index fa511394d..67bd157c3 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts @@ -265,6 +265,15 @@ describe("resolveDbUrl", () => { await expect(resolveDbUrl({ json: true }, CONFIG, {})).rejects.toThrow(/cannot prompt here/); expect(answers).toEqual([FIRST]); }); + + // `-y` is "do not prompt", at a terminal too. + test("does not prompt under -y, even with a human at the TTY", async () => { + answers = [FIRST]; + setAssumeYes(true); + + await expect(resolveDbUrl({}, CONFIG, {})).rejects.toThrow(/cannot prompt here/); + expect(answers).toEqual([FIRST]); + }); }); // Only the connect proves the connection string. A failure after it is not a From d0540cfc003b58d26bbcd818845684eed4da0177 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 8 Oct 2026 16:41:06 -0400 Subject: [PATCH 17/39] fix(migrate): refuse an export run ID whose file changed since the export An export run ID resolved to whatever sat at the path it recorded. With --output, a later export or an edit can overwrite that path, and the old run ID then imported different users under its name. The file's hash and envelope run ID are now checked against the run, and a file that is gone or changed exits 2 with nothing imported. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 4 +- .../commands/migrate/run-interactive.test.ts | 4 +- .../cli-core/src/commands/migrate/run.test.ts | 44 +++++++++++++++++-- packages/cli-core/src/commands/migrate/run.ts | 16 ++++++- 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index f8933f960..bc18a33a0 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -415,7 +415,9 @@ clerk migrate import # a human is asked Plus the targeting flags from the table above: `--secret-key`, `--app` and `--instance`. -An export run ID stands for the file that run wrote, and the import records it +An export run ID stands for the file that run wrote: if that file is gone, or +has changed since (an `--output` path another export or an edit overwrote), the +import exits 2 and imports nothing. The import records it as `fromExport`. A file `clerk migrate export` wrote carries its source, so it needs no `--source`, and a `--source` that contradicts it exits 2. Any other file needs `--source`: a JSON array, a CSV, or NDJSON, one user per line (what diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts index 9fd2e1ea4..786c02cb7 100644 --- a/packages/cli-core/src/commands/migrate/run-interactive.test.ts +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -57,7 +57,7 @@ const { terminal } = await import("./lib/terminal.ts"); const terminalAvailable = spyOn(terminal, "available"); const { UserAbortError } = await import("../../lib/errors.ts"); const { _setConfigDir } = await import("../../lib/config.ts"); -const { listRuns, startRun } = await import("./lib/run-store.ts"); +const { listRuns, sha256File, startRun } = await import("./lib/run-store.ts"); const captured = useCaptureLog(); @@ -194,7 +194,7 @@ describe("prompts for what was not passed", () => { users: EXPORT, }), ); - exportRun.update({ file: { path: file, sha256: "x" } }); + exportRun.update({ file: { path: file, sha256: sha256File(file) } }); exportRun.finish(); await run({ input: exportRun.record.id, secretKey: "sk_test_x" }); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 066ae1063..de764f329 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -21,7 +21,14 @@ import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; // signed-in check — except the one that asserts what happens without it. mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); import { importFlags } from "./lib/carry-flags.ts"; -import { latestUserLines, listRuns, patchRun, readRun, startRun } from "./lib/run-store.ts"; +import { + latestUserLines, + listRuns, + patchRun, + readRun, + sha256File, + startRun, +} from "./lib/run-store.ts"; import { explainErrors, run, validateRunOptions } from "./run.ts"; // After run.ts: imported first, signals.ts loads version.ts ahead of its Bun // macro here, and the file fails to load. @@ -314,17 +321,48 @@ describe("run", () => { users: rows, }), ); - run.update({ file: { path: file, sha256: "x" } }); + run.update({ file: { path: file, sha256: sha256File(file) } }); return { record: run.finish(), file }; } const { source: _source, input: _input, ...noSource } = baseOptions; + // The run ID stands for the file that run wrote: a later export or an edit + // at the same path must not import under the old run's name. + test("refuses an export file that changed since its run wrote it", async () => { + const { record, file } = exportRun("clerk", export2); + const envelope = JSON.parse(fs.readFileSync(file, "utf-8")); + fs.writeFileSync(file, JSON.stringify({ ...envelope, users: [envelope.users[0]] })); + + await expect(run({ ...noSource, input: record.id })).rejects.toThrow( + /has changed since it was exported/, + ); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); + }); + + test("refuses an export file another run's export overwrote", async () => { + const { record, file } = exportRun("clerk", export2); + const envelope = JSON.parse(fs.readFileSync(file, "utf-8")); + fs.writeFileSync(file, JSON.stringify({ ...envelope, runId: "20260101-000000-abcd" })); + patchRun(runsDir(), record.id, { file: { path: file, sha256: sha256File(file) } }); + + await expect(run({ ...noSource, input: record.id })).rejects.toThrow( + /has changed since it was exported/, + ); + }); + + test("refuses an export run whose file is gone", async () => { + const { record, file } = exportRun("clerk", export2); + fs.rmSync(file); + + await expect(run({ ...noSource, input: record.id })).rejects.toThrow(/is gone/); + }); + test("names an --output export file outside the run folder for deletion too", async () => { const { record, file } = exportRun("clerk", export2); const outside = path.join(workDir, `users-${record.id}.json`); fs.renameSync(file, outside); - patchRun(runsDir(), record.id, { file: { path: outside, sha256: "x" } }); + patchRun(runsDir(), record.id, { file: { path: outside, sha256: sha256File(outside) } }); await run({ ...noSource, input: record.id }); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 95f9812f4..fa9434f6e 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -317,7 +317,21 @@ async function resolveInput( `Run ${value} is an ${record.kind} run, which has no file to import. Name an export run, or a file.`, ); } - return { file: record.file.path, fromExport: record.id }; + // `--output` can point a later export, or an edit, at the same path. The run + // ID has to still mean the file that run wrote, not whatever is there now. + const { path: file, sha256 } = record.file; + if (!fileExists(file)) { + throwUsageError( + `The file run ${value} wrote, ${quoteArg(file)}, is gone. Export again, or import a file by its path.`, + ); + } + if (sha256File(file) !== sha256 || readEnvelope(file)?.runId !== record.id) { + throwUsageError( + `The file run ${value} wrote, ${quoteArg(file)}, has changed since it was exported. Nothing was imported. ` + + "Import it by its path if you mean its current contents, or export again.", + ); + } + return { file, fromExport: record.id }; } /** From 4967bc894fdf5a9131378b46402f9ccf6dc08c91 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 8 Oct 2026 17:19:28 -0400 Subject: [PATCH 18/39] fix(migrate): check an export run's file again where its users are read The run-ID check hashed the file in resolveInput, but the users load later, and an export overwriting a shared --output path in between was imported under the old run's name. The recorded hash now comes along: it is compared with the hash taken for resume matching, and again right after the users load, before the checks or any create. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/run.test.ts | 22 ++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 26 ++++++++++++++----- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index de764f329..39590bac2 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -340,6 +340,28 @@ describe("run", () => { expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); }); + // A shared --output path can be overwritten after the run-ID check, while + // the import is still naming its target. + test("refuses an export file overwritten after the run ID was checked", async () => { + const { record, file } = exportRun("clerk", export2); + const served = globalThis.fetch; + let overwritten = false; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + if (!overwritten && new URL(input.toString()).pathname === "/v1/instance") { + overwritten = true; + const envelope = JSON.parse(fs.readFileSync(file, "utf-8")); + fs.writeFileSync(file, JSON.stringify({ ...envelope, users: [envelope.users[0]] })); + } + return served(input, init); + }) as typeof fetch; + + await expect(run({ ...noSource, input: record.id })).rejects.toThrow( + /has changed since it was exported/, + ); + expect(overwritten).toBe(true); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); + }); + test("refuses an export file another run's export overwrote", async () => { const { record, file } = exportRun("clerk", export2); const envelope = JSON.parse(fs.readFileSync(file, "utf-8")); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index fa9434f6e..5a486c796 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -283,7 +283,7 @@ async function ensureImportTarget(options: MigrateRunOptions): Promise { */ async function resolveInput( options: MigrateRunOptions, -): Promise<{ file: string; fromExport?: string }> { +): Promise<{ file: string; fromExport?: string; exportSha256?: string }> { const value = options.input; if (!value) { if (!canPrompt(options)) { @@ -326,12 +326,18 @@ async function resolveInput( ); } if (sha256File(file) !== sha256 || readEnvelope(file)?.runId !== record.id) { - throwUsageError( - `The file run ${value} wrote, ${quoteArg(file)}, has changed since it was exported. Nothing was imported. ` + - "Import it by its path if you mean its current contents, or export again.", - ); + throwChangedExport(value, file); } - return { file, fromExport: record.id }; + // Carried on, so the file is checked again where the users are read: another + // export can still overwrite the path after this. + return { file, fromExport: record.id, exportSha256: sha256 }; +} + +function throwChangedExport(runId: string, file: string): never { + throwUsageError( + `The file run ${runId} wrote, ${quoteArg(file)}, has changed since it was exported. Nothing was imported. ` + + "Import it by its path if you mean its current contents, or export again.", + ); } /** @@ -722,6 +728,9 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const filePath = resolveImportFilePath(file); const sha256 = sha256File(filePath); + if (input.fromExport && sha256 !== input.exportSha256) { + throwChangedExport(input.fromExport, filePath); + } const runsDir = await resolveRunsDir(options.runsDir); const match = { @@ -798,6 +807,11 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const loaded = await withSpinner(`Loading users from ${file}...`, async () => loadUsersFromFile(file, source), ); + // Read, and checked again before anything is created: the users loaded + // came from the file the export run recorded, not one written since. + if (input.fromExport && sha256File(filePath) !== input.exportSha256) { + throwChangedExport(input.fromExport, filePath); + } let users = loaded.users.filter((user) => !done.has(user.userId)); const failures = loaded.failures.filter((failure) => !done.has(failure.userId)); From c21e843afe475af4a3e1ecb878bea890e5a0087e Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 8 Oct 2026 17:35:16 -0400 Subject: [PATCH 19/39] fix(migrate): check an export run's file after the last read of it The recheck came after the user load, but a Supabase import reads the file again for its provider rows, so an export overwriting the path in between paired one export's users with another's providers. The check now runs after that read, outside its try. Co-Authored-By: Claude Opus 5.5 --- .../migrate/run-export-recheck.test.ts | 93 +++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 12 ++- 2 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 packages/cli-core/src/commands/migrate/run-export-recheck.test.ts diff --git a/packages/cli-core/src/commands/migrate/run-export-recheck.test.ts b/packages/cli-core/src/commands/migrate/run-export-recheck.test.ts new file mode 100644 index 000000000..156df76be --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run-export-recheck.test.ts @@ -0,0 +1,93 @@ +/** + * An export run's file checked again after the import's last read of it. + * + * Its own file because `mock.module` lasts for the file: this one swaps in a + * provider read that overwrites the export, as a concurrent export to a shared + * `--output` path would between the user load and the provider read. + */ + +import { afterAll, beforeAll, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; +import * as providers from "./lib/supabase-providers.ts"; + +let exportFile = ""; + +mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); +mock.module("./lib/supabase-providers.ts", () => ({ + ...providers, + readSupabaseRows: async () => { + const envelope = JSON.parse(fs.readFileSync(exportFile, "utf-8")); + fs.writeFileSync(exportFile, JSON.stringify({ ...envelope, users: [] })); + return []; + }, +})); + +const { _setConfigDir } = await import("../../lib/config.ts"); +const { sha256File, startRun } = await import("./lib/run-store.ts"); +const { run } = await import("./run.ts"); + +useCaptureLog(); + +let workDir: string; +let configDir: string; +let originalCwd: string; +const originalFetch = globalThis.fetch; +const creates: string[] = []; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-recheck-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-recheck-config-")); + _setConfigDir(configDir); + process.chdir(workDir); + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(input.toString()); + if (url.pathname === "/v1/instance") { + return Response.json({ object: "instance", id: "ins_1", environment_type: "development" }); + } + if (url.pathname === "/v1/users/count") return Response.json({ total_count: 0 }); + if (url.pathname === "/v1/users" && init?.method === "POST") { + creates.push(url.pathname); + return Response.json({ id: "user_created" }); + } + if (url.pathname === "/v1/users") return Response.json([]); + return new Response("unavailable", { status: 503 }); + }) as typeof fetch; +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +test("refuses an export overwritten between the user load and the provider read", async () => { + const exportRun = startRun(path.join(workDir, ".clerk", "migrate"), { + kind: "export", + target: { platform: "supabase" }, + source: "supabase", + }); + exportFile = path.join(exportRun.dir, "export.json"); + fs.writeFileSync( + exportFile, + JSON.stringify({ + clerkMigrate: 1, + source: "supabase", + exportedAt: "2026-09-01T00:00:00.000Z", + runId: exportRun.record.id, + users: [{ id: "s1", email: "a@x.dev", email_confirmed_at: "2024-01-01" }], + }), + ); + exportRun.update({ file: { path: exportFile, sha256: sha256File(exportFile) } }); + exportRun.finish(); + + await expect( + run({ input: exportRun.record.id, yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/has changed since it was exported/); + expect(creates).toEqual([]); +}); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 5a486c796..ac459cd77 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -807,11 +807,6 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const loaded = await withSpinner(`Loading users from ${file}...`, async () => loadUsersFromFile(file, source), ); - // Read, and checked again before anything is created: the users loaded - // came from the file the export run recorded, not one written since. - if (input.fromExport && sha256File(filePath) !== input.exportSha256) { - throwChangedExport(input.fromExport, filePath); - } let users = loaded.users.filter((user) => !done.has(user.userId)); const failures = loaded.failures.filter((failure) => !done.has(failure.userId)); @@ -840,6 +835,13 @@ export async function run(rawOptions: MigrateRunOptions): Promise { } } + // Checked again after the last read of the file, before anything is + // created: the users and the provider rows came from the file the export + // run recorded, not one written since. + if (input.fromExport && sha256File(filePath) !== input.exportSha256) { + throwChangedExport(input.fromExport, filePath); + } + const [settings, existingUsers] = await withSpinner("Checking the instance...", async () => Promise.all([ fetchInstanceSettings(secretKey), From 9286e234070ab6546d1dc4823df61120e0b7d0e3 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 00:01:00 -0400 Subject: [PATCH 20/39] fix(migrate): read a path import's envelope and users from one revision A path import read the file's envelope, then its users, with nothing tying the two reads together: a file replaced in between could pair the old envelope (its source, a Firebase export's hash parameters) with the new users. The file is now hashed before the envelope is read, and the existing checks before and after the last read compare against that revision for every import, not only one by export run ID. Co-Authored-By: Claude Opus 5.5 --- .../migrate/run-export-recheck.test.ts | 15 ++++++++++ packages/cli-core/src/commands/migrate/run.ts | 30 ++++++++++++++----- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run-export-recheck.test.ts b/packages/cli-core/src/commands/migrate/run-export-recheck.test.ts index 156df76be..672b03e75 100644 --- a/packages/cli-core/src/commands/migrate/run-export-recheck.test.ts +++ b/packages/cli-core/src/commands/migrate/run-export-recheck.test.ts @@ -91,3 +91,18 @@ test("refuses an export overwritten between the user load and the provider read" ).rejects.toThrow(/has changed since it was exported/); expect(creates).toEqual([]); }); + +// A path import reads the envelope and the users separately, so a file +// rewritten between them would mix two revisions. +test("refuses a path import whose file is rewritten mid-import", async () => { + exportFile = path.join(workDir, "supabase-users.json"); + fs.writeFileSync( + exportFile, + JSON.stringify([{ id: "s2", email: "b@x.dev", email_confirmed_at: "2024-01-01" }]), + ); + + await expect( + run({ input: exportFile, source: "supabase", yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/changed while it was being imported/); + expect(creates).toEqual([]); +}); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index ac459cd77..29215f2d2 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -333,6 +333,14 @@ async function resolveInput( return { file, fromExport: record.id, exportSha256: sha256 }; } +/** A file the import read changed before it finished reading it. */ +function throwChangedFile(file: string): never { + throwUsageError( + `${quoteArg(file)} changed while it was being imported. Nothing was imported. ` + + "Run the import again once nothing else is writing to it.", + ); +} + function throwChangedExport(runId: string, file: string): never { throwUsageError( `The file run ${runId} wrote, ${quoteArg(file)}, has changed since it was exported. Nothing was imported. ` + @@ -705,9 +713,19 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const input = await resolveInput(options); options = { ...options, file: input.file }; + // Hashed before the envelope is read: the envelope and the users are read + // separately, so each later read is checked against this revision. + const readSha256 = fileExists(input.file) + ? sha256File(resolveImportFilePath(input.file)) + : undefined; const envelope = fileExists(input.file) ? readEnvelope(resolveImportFilePath(input.file)) : undefined; + // The revision the import must read throughout: the one an export run + // recorded, or the one the envelope came from. + const expectedSha256 = input.exportSha256 ?? readSha256; + const throwChanged = (filePath: string): never => + input.fromExport ? throwChangedExport(input.fromExport, filePath) : throwChangedFile(filePath); options = applyEnvelope(options, envelope); // Asked only when the file does not say where it came from. @@ -728,9 +746,7 @@ export async function run(rawOptions: MigrateRunOptions): Promise { const filePath = resolveImportFilePath(file); const sha256 = sha256File(filePath); - if (input.fromExport && sha256 !== input.exportSha256) { - throwChangedExport(input.fromExport, filePath); - } + if (expectedSha256 !== undefined && sha256 !== expectedSha256) throwChanged(filePath); const runsDir = await resolveRunsDir(options.runsDir); const match = { @@ -836,10 +852,10 @@ export async function run(rawOptions: MigrateRunOptions): Promise { } // Checked again after the last read of the file, before anything is - // created: the users and the provider rows came from the file the export - // run recorded, not one written since. - if (input.fromExport && sha256File(filePath) !== input.exportSha256) { - throwChangedExport(input.fromExport, filePath); + // created: the envelope, the users and the provider rows all came from + // one revision, the one an export run recorded if there is one. + if (expectedSha256 !== undefined && sha256File(filePath) !== expectedSha256) { + throwChanged(filePath); } const [settings, existingUsers] = await withSpinner("Checking the instance...", async () => From 2437b1e363872b333a9dbd1555a5d758a8a20848 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 14:23:44 -0400 Subject: [PATCH 21/39] fix(migrate): make an existing --output owner-only before writing the export to it The export was written to an existing `--output` file and only then chmod'ed to 0600, so its password hashes and PII sat at the file's old mode in between, or for good if the write failed first. The file is now opened without truncating, made owner-only, then truncated and written. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/export/shared.test.ts | 24 ++++++++++++++++++- .../src/commands/migrate/export/shared.ts | 14 ++++++++--- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts index 8939368b8..4ac5a6cab 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.test.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -68,6 +68,28 @@ describe("finishExport", () => { expect(fs.statSync(outputPath).mode & 0o777).toBe(0o600); }); + // Tightened before a byte lands: a chmod after the write leaves the data + // readable at the old mode in between, or for good if the write fails. + test("an existing --output is owner-only before the export is written to it", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + const output = path.join(runsDir, "existing-before.json"); + fs.writeFileSync(output, "old", { mode: 0o644 }); + const modesAtWrite: number[] = []; + const write = fs.writeSync; + const spy = spyOn(fs, "writeSync").mockImplementation(((fd: number, ...rest: unknown[]) => { + modesAtWrite.push(fs.fstatSync(fd).mode & 0o777); + return (write as (...args: unknown[]) => number)(fd, ...rest); + }) as typeof fs.writeSync); + try { + finishExport({ run, options: { output }, users, coverage }); + } finally { + spy.mockRestore(); + } + + expect(modesAtWrite).toEqual([0o600]); + expect(fs.readFileSync(output, "utf-8")).not.toContain("old"); + }); + test("--output writes somewhere else, and the run still records where", async () => { const run = await startExportRun({ runsDir }, { platform: "supabase" }); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index 98fcb75c6..2af4e1cc7 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -60,9 +60,17 @@ export function exportPath(run: Run, output: string | undefined): string { export function writeExportFile(file: string, envelope: ExportEnvelope): string { // Password hashes, PII and a Firebase signer key: owner-only. fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); - fs.writeFileSync(file, JSON.stringify(envelope, null, 2), { mode: 0o600 }); - // `mode` only applies on create; an existing `--output` keeps its own. - fs.chmodSync(file, 0o600); + // Made owner-only before a byte is written: `mode` applies only on create, + // so an existing `--output` would otherwise hold the data at its own mode + // until a later chmod, or keep it if the write failed first. + const fd = fs.openSync(file, fs.constants.O_WRONLY | fs.constants.O_CREAT, 0o600); + try { + fs.fchmodSync(fd, 0o600); + fs.ftruncateSync(fd, 0); + fs.writeSync(fd, JSON.stringify(envelope, null, 2)); + } finally { + fs.closeSync(fd); + } return file; } From e9f467e13865263097845cc71c3788b22a84c584 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 14:23:44 -0400 Subject: [PATCH 22/39] fix(migrate): link a run continued by export run ID to its export An import started by path and continued by export run ID kept a record with no `fromExport`, since only a new run set it. The cleanup at the end then left out the note to delete the export that holds the users' data. A continue by export run ID now records the link. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/run.test.ts | 17 +++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 15 ++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 39590bac2..b6c565b45 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -403,6 +403,23 @@ describe("run", () => { expect(captured.err).toContain(`The export in run ${record.id} holds your users' data`); }); + // Started by path, continued by run ID: the run still names its export. + test("a continue by export run ID links the run to the export", async () => { + const { record, file } = exportRun("clerk", export2); + stubClerk({ failing: new Set(["u2"]) }); + await run({ ...noSource, input: file }); + expect(listRuns(runsDir()).find((c) => c.kind === "import")?.fromExport).toBeUndefined(); + + stubClerk(); + process.exitCode = 0; + await run({ ...noSource, input: record.id }); + + const imported = listRuns(runsDir()).filter((c) => c.kind === "import"); + expect(imported).toHaveLength(1); + expect(imported[0]).toMatchObject({ fromExport: record.id }); + expect(captured.err).toContain(`The export in run ${record.id} holds your users' data`); + }); + test("imports an envelope file with no source named", async () => { const { file } = exportRun("clerk", export2); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 29215f2d2..487bc3ec0 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -691,6 +691,16 @@ function continuesWithoutNewRun( return retries ? resume.record : undefined; } +/** + * A continued run, linked to the export run it now reads from: one started + * by path and continued by export run ID would otherwise lack the link, and + * the cleanup would not name the export holding the users' data. + */ +function continueWithExport(run: Run, fromExport: string | undefined): Run { + if (fromExport && run.record.fromExport !== fromExport) run.update({ fromExport }); + return run; +} + /** * Records the checks' rejects as skipped users, so the run says who they were. * The checks never reject an adopted user: it is created already. @@ -1068,7 +1078,10 @@ export async function run(rawOptions: MigrateRunOptions): Promise { await resolveRunsDir(options.runsDir, { write: true }); const flags = pickImportFlags(options); const run = continued - ? continueRun(runsDir, withAttempt(continued, target, flags), plannedLines, carry) + ? continueWithExport( + continueRun(runsDir, withAttempt(continued, target, flags), plannedLines, carry), + input.fromExport, + ) : startRun(runsDir, { kind: "import", target, From e79294838b6a8e23e926166c6e3f62d5a7e849db Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 14:23:45 -0400 Subject: [PATCH 23/39] fix(migrate): refuse a non-Postgres URL for a Supabase export before connecting The Supabase export reads `auth.users` with Postgres syntax, but took any connection string the database helper accepts: a MySQL or SQLite URL connected and then failed with nothing exported. Anything but postgres:// or postgresql:// is now refused up front, naming where to find the right one. Co-Authored-By: Claude Opus 5.5 --- .../src/commands/migrate/export/db-exports.test.ts | 14 ++++++++------ .../src/commands/migrate/export/supabase.ts | 11 ++++++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index b8908e84a..2f00bda4f 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -185,12 +185,14 @@ describe("supabase export", () => { }); describe("connection failures", () => { - test("a missing SQLite file fails before a run is written", async () => { - await expect(exportSupabase({ dbUrl: "./definitely-not-here.sqlite" })).rejects.toThrow( - CliError, - ); - expect(fs.existsSync(path.join(workDir, ".clerk"))).toBe(false); - }); + // Supabase is Postgres: anything else would connect and then fail. + test.each([["mysql://u:p@127.0.0.1:1/db"], ["./definitely-not-here.sqlite"]])( + "refuses %s before connecting or writing a run", + async (dbUrl) => { + await expect(exportSupabase({ dbUrl })).rejects.toThrow(/Supabase's database is Postgres/); + expect(fs.existsSync(path.join(workDir, ".clerk"))).toBe(false); + }, + ); test("the failure never contains the password", async () => { await expect( diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 263c97d35..0a4f7769f 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -10,11 +10,12 @@ * the bcrypt digests across. */ +import { throwUsageError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import type { UserLine } from "../lib/run-store.ts"; import { printTarget } from "../lib/target.ts"; -import type { DbClient } from "../lib/db.ts"; +import { detectDbType, type DbClient } from "../lib/db.ts"; import { finishExport, startExportRun } from "./shared.ts"; import { resolveDbUrl, @@ -122,6 +123,14 @@ const SUPABASE_DB = { export async function exportSupabase(options: DbExportOptions): Promise { const dbUrl = await resolveDbUrl(options, SUPABASE_DB); + // The query reads `auth.users` with Postgres syntax: a MySQL or SQLite URL + // would connect, then fail with nothing exported. + if (detectDbType(dbUrl) !== "postgres") { + throwUsageError( + "Supabase's database is Postgres: pass a postgres:// or postgresql:// connection string. Nothing was exported.\n" + + `Find it under ${SUPABASE_DB.hint}`, + ); + } await withGutter("Exporting users from Supabase", async () => { if (!options.json) printTarget({ platform: "supabase" }); From 2fa63fddf0d0c790267eac44ca166eb39cd244b5 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 15:00:52 -0400 Subject: [PATCH 24/39] fix(migrate): write an export to a temp file and rename it over --output Truncating an existing `--output` before writing it meant a failed write destroyed the export that was there, and a short write could leave a partial file recorded as complete. The export is now written whole to a new owner-only file beside it, then renamed into place; a failure removes the temp file and leaves the old one as it was. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/export/shared.test.ts | 31 +++++++++++-------- .../src/commands/migrate/export/shared.ts | 19 ++++++------ 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts index 4ac5a6cab..8a22300c6 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.test.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -68,26 +68,31 @@ describe("finishExport", () => { expect(fs.statSync(outputPath).mode & 0o777).toBe(0o600); }); - // Tightened before a byte lands: a chmod after the write leaves the data - // readable at the old mode in between, or for good if the write fails. - test("an existing --output is owner-only before the export is written to it", async () => { + // Written to a new file and renamed into place: a failed write leaves an + // existing export as it was, and no half-written file behind. + test("a failed write keeps an existing --output whole", async () => { const run = await startExportRun({ runsDir }, { platform: "supabase" }); const output = path.join(runsDir, "existing-before.json"); - fs.writeFileSync(output, "old", { mode: 0o644 }); - const modesAtWrite: number[] = []; - const write = fs.writeSync; - const spy = spyOn(fs, "writeSync").mockImplementation(((fd: number, ...rest: unknown[]) => { - modesAtWrite.push(fs.fstatSync(fd).mode & 0o777); - return (write as (...args: unknown[]) => number)(fd, ...rest); - }) as typeof fs.writeSync); + fs.writeFileSync(output, "old", { mode: 0o600 }); + const write = fs.writeFileSync; + const spy = spyOn(fs, "writeFileSync").mockImplementation((( + target: unknown, + ...rest: unknown[] + ) => { + if (String(target).endsWith(".tmp")) { + (write as (...args: unknown[]) => void)(target, "{ half", ...rest.slice(1)); + throw new Error("ENOSPC: no space left on device"); + } + return (write as (...args: unknown[]) => void)(target, ...rest); + }) as typeof fs.writeFileSync); try { - finishExport({ run, options: { output }, users, coverage }); + expect(() => finishExport({ run, options: { output }, users, coverage })).toThrow(/ENOSPC/); } finally { spy.mockRestore(); } - expect(modesAtWrite).toEqual([0o600]); - expect(fs.readFileSync(output, "utf-8")).not.toContain("old"); + expect(fs.readFileSync(output, "utf-8")).toBe("old"); + expect(fs.readdirSync(runsDir).filter((name) => name.endsWith(".tmp"))).toEqual([]); }); test("--output writes somewhere else, and the run still records where", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index 2af4e1cc7..d329a6d25 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -60,16 +60,17 @@ export function exportPath(run: Run, output: string | undefined): string { export function writeExportFile(file: string, envelope: ExportEnvelope): string { // Password hashes, PII and a Firebase signer key: owner-only. fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); - // Made owner-only before a byte is written: `mode` applies only on create, - // so an existing `--output` would otherwise hold the data at its own mode - // until a later chmod, or keep it if the write failed first. - const fd = fs.openSync(file, fs.constants.O_WRONLY | fs.constants.O_CREAT, 0o600); + // Written whole to a new owner-only file, then renamed over the output: an + // existing `--output` keeps its old contents if the write fails, and the + // data is never at that file's own mode (`mode` applies only on create; + // the rename keeps the new file's). + const temp = `${file}.${process.pid}.tmp`; try { - fs.fchmodSync(fd, 0o600); - fs.ftruncateSync(fd, 0); - fs.writeSync(fd, JSON.stringify(envelope, null, 2)); - } finally { - fs.closeSync(fd); + fs.writeFileSync(temp, JSON.stringify(envelope, null, 2), { mode: 0o600, flag: "wx" }); + fs.renameSync(temp, file); + } catch (error) { + fs.rmSync(temp, { force: true }); + throw error; } return file; } From 0cece066277096693d8e012ade56b5505e618072 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 15:00:52 -0400 Subject: [PATCH 25/39] docs(migrate): note the Supabase export's memory ceiling The export reads every `auth.users` row at once. The import reads the file whole as well, so paging one side alone would not help; the comment says where to page both if that ever matters. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/export/supabase.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 0a4f7769f..321902519 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -72,6 +72,13 @@ function normalizeRow(row: SupabaseRow): Record { return normalized; } +/** + * Every `auth.users` row, at once. + * + * ponytail: the whole table in memory, then the whole envelope. The import + * reads the file whole too, so streaming this side alone would buy nothing; + * page both (keyset on `created_at, id`) if a project's users outgrow memory. + */ export async function fetchSupabaseUsers(client: DbClient): Promise { return client.query(EXPORT_QUERY); } From 140dcdfc40932889e01e13d8c1abd238da3eead1 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 17:16:47 -0400 Subject: [PATCH 26/39] docs(migrate): say export supabase takes a Postgres connection string only Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index bc18a33a0..f87662056 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -268,7 +268,7 @@ the flag to pass. | `-y, --yes` | all | Do not prompt: fail on a bad credential | | `--json` | all | Print the result as JSON; never prompts | | `--runs-dir ` | all | Where runs are kept (see [the run store](#the-run-store)) | -| `--db-url ` | `supabase` | Postgres, MySQL, libsql/Turso or SQLite connection string | +| `--db-url ` | `supabase` | Postgres connection string | `export clerk` also takes the targeting flags — it reads from a Clerk instance, so it resolves a key the same way `clerk migrate import` does, with one extra @@ -349,11 +349,9 @@ counts users who _have_ a password, so the size of the gap is visible up front. clerk migrate export supabase --db-url "postgres://postgres:...@db.xxx.supabase.co:5432/postgres" ``` -Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`; -`libsql://` (Turso) over the server's HTTP pipeline endpoint, since `bun:sqlite` -only opens local files and `@libsql/client` ships native optional dependencies. -Nothing native ships in the binary — that is the whole reason the `engines.bun` -floor exists. Resolution is `--db-url`, then `SUPABASE_DB_URL`, then a masked +Supabase's database is Postgres, read through `Bun.sql`: any URL but +`postgres://` or `postgresql://` is refused before connecting. Nothing native +ships in the binary — that is the whole reason the `engines.bun` floor exists. Resolution is `--db-url`, then `SUPABASE_DB_URL`, then a masked prompt, since a connection string carries the password inline. A password pasted unencoded (`#`, `@`, `/` and the like) is percent-encoded for you. From fee9c62ddbf08159b2164340cc1f415f5c888fff Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 9 Oct 2026 17:20:05 -0400 Subject: [PATCH 27/39] chore(migrate): drop the CliError import the Postgres-only refusal left unused Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/export/db-exports.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 2f00bda4f..12afc39ab 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -11,7 +11,6 @@ import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:tes import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { CliError } from "../../../lib/errors.ts"; import type { DbClient } from "../lib/db.ts"; import type { UserLine } from "../lib/run-store.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; From 73b1f87789db8facdd3b75ddefe612cc98529840 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 13 Oct 2026 21:10:07 -0400 Subject: [PATCH 28/39] fix(migrate): refuse a non-Postgres --db-url for Supabase on every path, not just the first The check only saw the first connection string. A retry after a refused connect took a MySQL URL or a SQLite path, which connected and then failed the auth.users query. The dialect now lives in the platform's config, so the flag, the environment variable and every prompt check it, and the help and messages name only Postgres. Co-Authored-By: Claude Opus 5.5 --- .../migrate/export/db-exports.test.ts | 26 +++++++- .../src/commands/migrate/export/db-options.ts | 59 +++++++++++++++++-- .../src/commands/migrate/export/index.ts | 3 +- .../src/commands/migrate/export/supabase.ts | 15 ++--- 4 files changed, 83 insertions(+), 20 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 12afc39ab..ff83bbc6b 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -14,8 +14,14 @@ import path from "node:path"; import type { DbClient } from "../lib/db.ts"; import type { UserLine } from "../lib/run-store.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; -import { buildSupabaseExport, exportSupabase, fetchSupabaseUsers } from "./supabase.ts"; import { + buildSupabaseExport, + exportSupabase, + fetchSupabaseUsers, + SUPABASE_DB, +} from "./supabase.ts"; +import { + dialectRefusal, looksLikeConnectionString, normalizeConnectionString, resolveDbUrl, @@ -188,11 +194,27 @@ describe("connection failures", () => { test.each([["mysql://u:p@127.0.0.1:1/db"], ["./definitely-not-here.sqlite"]])( "refuses %s before connecting or writing a run", async (dbUrl) => { - await expect(exportSupabase({ dbUrl })).rejects.toThrow(/Supabase's database is Postgres/); + await expect(exportSupabase({ dbUrl })).rejects.toThrow( + /`clerk migrate export supabase` reads a Postgres database/, + ); expect(fs.existsSync(path.join(workDir, ".clerk"))).toBe(false); }, ); + // Each retry of a refused connect is a fresh prompt, checked the same way. + test.each([["mysql://u:p@127.0.0.1:1/db"], ["libsql://db.turso.io"], ["./users.db"]])( + "the prompt refuses %s for Supabase", + (dbUrl) => { + expect(dialectRefusal(SUPABASE_DB, dbUrl)).toContain("reads a Postgres database"); + }, + ); + + test("the prompt takes a Postgres URL for Supabase", () => { + expect(dialectRefusal(SUPABASE_DB, "postgresql://u:p@db.x.supabase.co:5432/postgres")).toBe( + undefined, + ); + }); + test("the failure never contains the password", async () => { await expect( exportSupabase({ dbUrl: "postgres://user:hunter2@127.0.0.1:1/db" }), diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index a045a859b..98b73e027 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -20,6 +20,7 @@ import { redactDbMessage, type DbClient, type DbPlatform, + type DbType, } from "../lib/db.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; import { withInputRetry } from "../lib/input-retry.ts"; @@ -40,8 +41,47 @@ export type ResolveConfig = { prompt: string; /** Extra guidance shown before prompting. */ hint?: string; + /** The one dialect the platform's database speaks: any other is refused before connecting. */ + dialect?: DbType; }; +const DIALECTS: Record = { + postgres: { label: "Postgres", expected: "postgres://… or postgresql://…" }, + mysql: { label: "MySQL", expected: "mysql://…" }, + sqlite: { label: "SQLite", expected: "libsql://… or a SQLite file path" }, +}; + +/** The connection strings `config` accepts, for messages. */ +function expectedUrl(config: ResolveConfig): string { + return config.dialect + ? DIALECTS[config.dialect].expected + : "postgres://…, mysql://…, libsql://… or a SQLite file path"; +} + +/** + * Why `config` cannot use a connection string that parses: it names a dialect + * the platform's database doesn't speak, which would connect, then fail with + * nothing exported. Undefined when it can. + */ +export function dialectRefusal( + config: ResolveConfig, + connectionString: string, +): string | undefined { + if (!config.dialect) return undefined; + const dialect = isLibsqlUrl(connectionString) ? "sqlite" : detectDbType(connectionString); + if (dialect === config.dialect) return undefined; + return `\`clerk migrate export ${config.platform}\` reads a ${DIALECTS[config.dialect].label} database: pass ${expectedUrl(config)}`; +} + +/** Refuses a connection string the platform's database cannot be, before connecting. */ +function assertDialect(config: ResolveConfig, connectionString: string): void { + const refused = dialectRefusal(config, connectionString); + if (!refused) return; + throwUsageError( + `${refused}. Nothing was exported.${config.hint ? `\nFind it under ${config.hint}` : ""}`, + ); +} + const URL_SCHEME = /^(postgresql|postgres|mysql|mysql2|libsql):\/\//i; /** @@ -122,17 +162,21 @@ export async function resolveDbUrl( if (fromFlag) { if (!looksLikeConnectionString(fromFlag)) { throwUsageError( - `--db-url does not look like a connection string. Expected postgres://…, mysql://…, libsql://… or a SQLite file path.\n` + + `--db-url does not look like a connection string. Expected ${expectedUrl(config)}.\n` + "If the password contains @, # or /, URL-encode it.", ); } + assertDialect(config, fromFlag); return fromFlag; } const raw = env[config.envVar]; const fromEnv = raw ? normalizeConnectionString(raw) : undefined; if (fromEnv) { - if (looksLikeConnectionString(fromEnv)) return fromEnv; + if (looksLikeConnectionString(fromEnv)) { + assertDialect(config, fromEnv); + return fromEnv; + } // Falling through silently would make the prompt look unexplained. log.warn(`${config.envVar} is not a valid connection string; ignoring it.`); } @@ -168,10 +212,13 @@ export async function resolveDbUrl( export async function promptDbUrl(config: ResolveConfig): Promise { const answer = await passwordPrompt({ message: config.prompt, - validate: (value) => - looksLikeConnectionString(normalizeConnectionString(value ?? "")) - ? undefined - : "Expected postgres://…, mysql://…, libsql://… or a SQLite file path", + // Every retry of a refused connect comes back through here, so a pasted + // string of the wrong dialect is caught on each one. + validate: (value) => { + const candidate = normalizeConnectionString(value ?? ""); + if (!looksLikeConnectionString(candidate)) return `Expected ${expectedUrl(config)}`; + return dialectRefusal(config, candidate); + }, }); return normalizeConnectionString(answer); diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index 0330f4f7d..306dd3eca 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -57,6 +57,7 @@ const DB_PLATFORMS = [ key: "supabase", summary: "Export users from a Supabase Postgres database", envVar: "SUPABASE_DB_URL", + dbUrl: "Postgres connection string", example: "postgres://postgres:password@db.xxx.supabase.co:5432/postgres", }, ] as const; @@ -113,7 +114,7 @@ export function registerMigrateExport(migrateCommand: Command<[], Record", "Postgres, MySQL, libsql/Turso or SQLite connection string") + .option("--db-url ", platform.dbUrl) .option("-o, --output ", "Write the export here instead of the run folder") .option("-y, --yes", "Do not prompt: fail on a rejected credential") .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 321902519..72d8267d6 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -10,12 +10,11 @@ * the bcrypt digests across. */ -import { throwUsageError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner } from "../../../lib/spinner.ts"; import type { UserLine } from "../lib/run-store.ts"; import { printTarget } from "../lib/target.ts"; -import { detectDbType, type DbClient } from "../lib/db.ts"; +import type { DbClient } from "../lib/db.ts"; import { finishExport, startExportRun } from "./shared.ts"; import { resolveDbUrl, @@ -121,23 +120,17 @@ export function buildSupabaseExport( }; } -const SUPABASE_DB = { +export const SUPABASE_DB = { platform: "supabase", envVar: "SUPABASE_DB_URL", prompt: "Supabase Postgres connection string", hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", + // The query reads `auth.users` with Postgres syntax. + dialect: "postgres", } as const satisfies ResolveConfig; export async function exportSupabase(options: DbExportOptions): Promise { const dbUrl = await resolveDbUrl(options, SUPABASE_DB); - // The query reads `auth.users` with Postgres syntax: a MySQL or SQLite URL - // would connect, then fail with nothing exported. - if (detectDbType(dbUrl) !== "postgres") { - throwUsageError( - "Supabase's database is Postgres: pass a postgres:// or postgresql:// connection string. Nothing was exported.\n" + - `Find it under ${SUPABASE_DB.hint}`, - ); - } await withGutter("Exporting users from Supabase", async () => { if (!options.json) printTarget({ platform: "supabase" }); From 0a13f0da5b198852e2c61d7d3a7b1ae086bfdb90 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 13 Oct 2026 21:10:37 -0400 Subject: [PATCH 29/39] fix(migrate): refuse an export file whose source this CLI does not know, before any request `--source` goes through the registry, but the envelope's source did not. A file from a newer CLI, or an edited one, reached BAPI and then failed in getSource with a plain Error and no list of valid sources. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/run.test.ts | 9 +++++++++ packages/cli-core/src/commands/migrate/run.ts | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index b6c565b45..848bb231b 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -327,6 +327,15 @@ describe("run", () => { const { source: _source, input: _input, ...noSource } = baseOptions; + test("refuses a file whose source this CLI does not know, before any request", async () => { + const { record } = exportRun("okta", export2); + + await expect(run({ ...noSource, input: record.id })).rejects.toThrow( + /exported from "okta", which this version of the CLI does not know\. Valid sources: /, + ); + expect(requests).toHaveLength(0); + }); + // The run ID stands for the file that run wrote: a later export or an edit // at the same path must not import under the old run's name. test("refuses an export file that changed since its run wrote it", async () => { diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 487bc3ec0..6af6ee57e 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -377,6 +377,14 @@ function applyEnvelope( "Drop --source: the file already says where it came from.", ); } + // `--source` goes through the registry; the file's own word must too, or a + // file from a newer CLI fails late, inside the import, with no list. + if (!sourceKeys().includes(envelope.source)) { + throwUsageError( + `The file says it was exported from "${envelope.source}", which this version of the CLI does not know. ` + + `Valid sources: ${sourceKeys().join(", ")}.`, + ); + } return { ...options, source: envelope.source }; } From 8c5a04bee69505923e2fb9fdae54438cc2ef0d3a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 13 Oct 2026 21:11:44 -0400 Subject: [PATCH 30/39] fix(migrate): name the path when an export's write fails, and release its run A write to a directory or an unwritable --output escaped as a raw fs error and left the run locked, with every user's line but no file. An import by that run ID then called it a run with no file to import. Now the write failure names the path, the run is released, and the import says the export never wrote its file. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/export/shared.test.ts | 13 +++++++++ .../src/commands/migrate/export/shared.ts | 29 ++++++++++++++----- .../cli-core/src/commands/migrate/run.test.ts | 13 +++++++++ packages/cli-core/src/commands/migrate/run.ts | 7 ++++- 4 files changed, 53 insertions(+), 9 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts index 8a22300c6..e9793b4c2 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.test.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -95,6 +95,19 @@ describe("finishExport", () => { expect(fs.readdirSync(runsDir).filter((name) => name.endsWith(".tmp"))).toEqual([]); }); + // The run is left behind with every user's line but no file. + test("a write that fails says where, and releases the run", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + const output = path.join(runsDir, "a-directory.json"); + fs.mkdirSync(output); + + expect(() => finishExport({ run, options: { output }, users, coverage })).toThrow( + new RegExp(`Could not write the export to ${output}: .*\\n.*with another --output`), + ); + expect(fs.existsSync(path.join(run.dir, "lock"))).toBe(false); + expect(readRun(runsDir, run.record.id)?.file).toBeUndefined(); + }); + test("--output writes somewhere else, and the run still records where", async () => { const run = await startExportRun({ runsDir }, { platform: "supabase" }); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index d329a6d25..2cac9bf55 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -11,6 +11,7 @@ import fs from "node:fs"; import path from "node:path"; import { dim, green, yellow } from "../../../lib/color.ts"; +import { CliError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; import { ENVELOPE_VERSION, type ExportEnvelope } from "../lib/export-file.ts"; import { ACCOUNT_LINKING_NOTE } from "../sources/registry.ts"; @@ -147,14 +148,26 @@ export type FinishedExport = { record: RunRecord; outputPath: string }; export function finishExport(input: FinishExportInput): FinishedExport { const { run, options, users, coverage } = input; const platform = run.record.target.platform ?? run.record.source ?? ""; - const outputPath = writeExportFile(exportPath(run, options.output), { - clerkMigrate: ENVELOPE_VERSION, - source: run.record.source ?? platform, - exportedAt: new Date().toISOString(), - runId: run.record.id, - ...(input.firebase ? { firebase: input.firebase } : {}), - users, - }); + const target = exportPath(run, options.output); + let outputPath: string; + try { + outputPath = writeExportFile(target, { + clerkMigrate: ENVELOPE_VERSION, + source: run.record.source ?? platform, + exportedAt: new Date().toISOString(), + runId: run.record.id, + ...(input.firebase ? { firebase: input.firebase } : {}), + users, + }); + } catch (error) { + // Released with no file, the run reads as an export that never finished, + // and an import by its ID says so. + run.release(); + throw new CliError( + `Could not write the export to ${target}: ${(error as Error).message}\n` + + `Nothing was written. Run the export again${options.output ? " with another --output" : ""}.`, + ); + } run.update({ file: { path: outputPath, sha256: sha256File(outputPath) } }); const record = run.finish(); const next = `clerk migrate import ${record.id}`; diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 848bb231b..f05ee2666 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -336,6 +336,19 @@ describe("run", () => { expect(requests).toHaveLength(0); }); + test("an export run that never wrote its file says so", async () => { + const unfinished = startRun(runsDir(), { + kind: "export", + target: { platform: "clerk" }, + source: "clerk", + }); + unfinished.release(); + + await expect(run({ ...noSource, input: unfinished.record.id })).rejects.toThrow( + /never wrote its file: it was interrupted, or the write failed/, + ); + }); + // The run ID stands for the file that run wrote: a later export or an edit // at the same path must not import under the old run's name. test("refuses an export file that changed since its run wrote it", async () => { diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 6af6ee57e..9c288aa53 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -312,11 +312,16 @@ async function resolveInput( if (!record) { throwUsageError(`No run \`${value}\` in ${runsDir}. Run \`clerk migrate runs\` to list them.`); } - if (record.kind !== "export" || !record.file) { + if (record.kind !== "export") { throwUsageError( `Run ${value} is an ${record.kind} run, which has no file to import. Name an export run, or a file.`, ); } + if (!record.file) { + throwUsageError( + `Export run ${value} never wrote its file: it was interrupted, or the write failed. Run the export again.`, + ); + } // `--output` can point a later export, or an edit, at the same path. The run // ID has to still mean the file that run wrote, not whatever is there now. const { path: file, sha256 } = record.file; From c2212ec57910268d88280dfad20b58d10e224735 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 13 Oct 2026 21:12:17 -0400 Subject: [PATCH 31/39] fix(migrate): refuse an export --output that does not end in .json The import reads a file by its extension, so an envelope written to users.ndjson or users.csv could not be imported by its run ID. The export now refuses such a path before it reads anything. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 3 ++- .../src/commands/migrate/export/index.ts | 12 ++++++++++++ .../src/commands/migrate/index.test.ts | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index f87662056..655647cab 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -241,7 +241,8 @@ told not to. Every export is a [run](#the-run-store), and the file lands in the run folder as `export.json`. `--output` writes it somewhere else instead, resolved against the **current directory** like every other path flag here; -the run still records where. Nothing is asked about where the file goes. +the run still records where. It must end in `.json`, since the import reads a +file by its extension. Nothing is asked about where the file goes. The file is an envelope around the users: diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts index 306dd3eca..944f5799d 100644 --- a/packages/cli-core/src/commands/migrate/export/index.ts +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -84,6 +84,18 @@ export function registerMigrateExport(migrateCommand: Command<[], Record), ); + // The export is one JSON document, and the import reads a file by its + // extension: an envelope written to users.ndjson could not be imported by + // run ID. Checked before anything is read. + exportCommand.hook("preAction", (_thisCommand, actionCommand) => { + const { output } = actionCommand.opts() as { output?: unknown }; + if (typeof output === "string" && !output.toLowerCase().endsWith(".json")) { + throwUsageError( + `--output must end in .json: the export is one JSON document, and \`clerk migrate import\` reads a file by its extension. Got ${output}.`, + ); + } + }); + exportCommand .command("clerk") .description("Export users from a Clerk instance") diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts index 56dca9681..962624b7f 100644 --- a/packages/cli-core/src/commands/migrate/index.test.ts +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -225,6 +225,25 @@ describe("the migrate group's -y hook", () => { } }); + test("export refuses an --output that is not .json, before reading anything", async () => { + const program = createProgram(); + program.exitOverride(); + await expect( + program.parseAsync([ + "node", + "clerk", + "migrate", + "export", + "supabase", + "--db-url", + "postgres://u:p@127.0.0.1:1/db", + "--output", + "users.ndjson", + "--json", + ]), + ).rejects.toThrow(/--output must end in \.json/); + }); + test("records its absence, so a previous run cannot leak into this one", async () => { setAssumeYes(true); expect(await parse(["migrate", "import", missing, "--secret-key", "sk_test_x"])).toBe(false); From 12d243fd1967e5ba4d396d0df6bb9df98c10311d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 13 Oct 2026 21:12:42 -0400 Subject: [PATCH 32/39] docs(migrate): say why an unreachable database is asked for again The retry comment said a refused connection is not fixed by another credential, but connect failures are usage errors and are re-asked. That is deliberate: Bun reports a mistyped host or port the same way as a server that is down. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/lib/input-retry.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.ts index c8aed0bb4..edc34be74 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.ts @@ -54,9 +54,12 @@ export async function withInputRetry( return { value: await work(candidate), input: candidate }; } catch (error) { // Everything these steps raise for a bad credential is a usage error - // carrying its own explanation. An outage, a 429 or a refused - // connection is not fixed by another credential, and anything else (an - // interrupt, a bug) is not ours to retry. + // carrying its own explanation. An outage or a 429 is not fixed by + // another credential, and anything else (an interrupt, a bug) is not ours + // to retry. A database that cannot be reached is the exception: Bun + // reports a mistyped host or port exactly as it does a server that is + // down ("Connection closed"), and the string is the likelier culprit, so + // it is asked for again and, unanswered, exits 2. const badInput = error instanceof CliError && error.exitCode === EXIT_CODE.USAGE; if (!badInput || options.json || !isHuman() || isAgent() || isAssumeYes()) throw error; From fdffc347d16ea3f0a02399abd3908d1bb172de60 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 13 Oct 2026 21:13:02 -0400 Subject: [PATCH 33/39] fix(migrate): name an --output export on another Windows drive in the cleanup path.relative returns an absolute path across drives, so the `rm` line for the file was left out. Co-Authored-By: Claude Opus 5.5 --- packages/cli-core/src/commands/migrate/run.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 9c288aa53..91ea5e680 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -593,7 +593,9 @@ function cleanupLines(runsDir: string, record: RunRecord): string[] { // `export --output` writes the file outside the run folder, and deleting // the folder alone would leave the password hashes on disk. const file = record.file?.path; - const outside = file && path.relative(exportDir, file).startsWith(".."); + // On Windows a file on another drive comes back absolute, not as `..`. + const relative = file ? path.relative(exportDir, file) : ""; + const outside = file && (path.isAbsolute(relative) || relative.startsWith("..")); lines.push( `The export in run ${record.fromExport} holds your users' data. Once you have checked the import, delete it:`, dim(` rm -rf ${quoteArg(exportDir)}`), From aede72cf83174d72017462e6c1a2d49a56db7e60 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Tue, 13 Oct 2026 21:13:41 -0400 Subject: [PATCH 34/39] fix(migrate): export Supabase's is_anonymous, so the import skips anonymous users The import skips an anonymous user, but the export never selected the column, so those users came through as having no identifier. Co-Authored-By: Claude Opus 5.5 --- .../migrate/export/db-exports.test.ts | 20 +++++++++++++++++++ .../src/commands/migrate/export/supabase.ts | 2 ++ 2 files changed, 22 insertions(+) diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index ff83bbc6b..9f7fa2142 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -27,6 +27,8 @@ import { resolveDbUrl, } from "./db-options.ts"; +import { transformUsers } from "../lib/transform.ts"; + const captured = useCaptureLog(); let workDir: string; @@ -150,6 +152,24 @@ describe("supabase export", () => { expect(sql).not.toContain("display_name"); }); + // Without the column, the import would fail an anonymous user as having no + // identifier rather than skip it. + test("carries is_anonymous, so the import skips an anonymous user", async () => { + let sql = ""; + await fetchSupabaseUsers({ + query: async (query: string) => { + sql = query; + return []; + }, + } as unknown as DbClient); + expect(sql).toContain("AS is_anonymous"); + + const { users } = buildSupabaseExport([{ id: "anon", is_anonymous: true }]); + const result = transformUsers(users, "supabase"); + expect(result.validationFailed).toBe(0); + expect(result.transformedData[0]?.skipReason).toContain("anonymous"); + }); + test("serializes timestamps the transformer can parse", () => { const { users } = buildSupabaseExport([ { id: "u1", email: "a@x.dev", created_at: new Date("2024-01-01T00:00:00Z") }, diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 72d8267d6..2e40c22dc 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -45,6 +45,8 @@ const EXPORT_QUERY = ` banned_until, -- Through to_jsonb so an older auth.users without the column still reads. to_jsonb(u)->>'deleted_at' AS deleted_at, + -- The Supabase source skips an anonymous user: it has nothing to sign in with. + (to_jsonb(u)->>'is_anonymous')::boolean AS is_anonymous, created_at FROM auth.users u ORDER BY created_at From a8e179899a8e82fb3b6af88a29ce8b59ecf772d6 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 15 Oct 2026 01:07:49 -0400 Subject: [PATCH 35/39] fix(migrate): page the Clerk export by cursor, ask before exporting the destination, and keep --runs-dir in export hints From Michael's review of #524: - export clerk pages GET /v1/users with starting_after the last user of the page before, so a deletion can't slide a user past a page boundary. A deleted cursor user (422) stops the export instead of skipping anyone. - With nobody to ask (agent mode, --json, -y), export clerk refuses an instance resolved on the user's behalf, which is usually the destination, and names the flags. An unlinked directory with no key says "not linked" first, and now opens the picker for a human or exits 2 otherwise. No instances to pick from exits 2 too, and when applications can't be listed (an accountless app with no login) it asks about the resolved instance. - import refuses an export run of the instance it would import into. - The export's next, "Import them with" and runs hints keep --runs-dir, and only that: its target flags name the source. - A failed write leaves no run behind, the summary and --json count skipped users, the password warning shows only when someone had one, and an app's external_id moving to private_metadata.clerkExternalId is said up front. - The Supabase export counts names the source splits out of the metadata, keeps a ban stored as 'infinity', and says when SUPABASE_DB_URL is set but unusable. - withInputRetry won't re-prompt without a terminal, which forced human mode with no TTY used to spin on. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 56 ++++---- .../migrate/export/clerk-source.test.ts | 120 +++++++++++------- .../commands/migrate/export/clerk-source.ts | 71 ++++++++--- .../src/commands/migrate/export/clerk.test.ts | 58 ++++++--- .../src/commands/migrate/export/clerk.ts | 73 +++++++---- .../migrate/export/db-exports.test.ts | 27 ++++ .../src/commands/migrate/export/db-options.ts | 4 +- .../commands/migrate/export/shared.test.ts | 33 ++++- .../src/commands/migrate/export/shared.ts | 30 ++++- .../src/commands/migrate/export/supabase.ts | 27 +++- .../commands/migrate/lib/input-retry.test.ts | 26 +++- .../src/commands/migrate/lib/input-retry.ts | 19 +++ .../cli-core/src/commands/migrate/run.test.ts | 29 +++++ packages/cli-core/src/commands/migrate/run.ts | 24 +++- 14 files changed, 444 insertions(+), 153 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 655647cab..0c7139850 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -15,7 +15,7 @@ Migrate users into a Clerk instance from another auth provider, or from another Clerk instance. ``` -clerk migrate export [-o ] [--json] +clerk migrate export [source] [-o ] [--json] clerk migrate import [--source ] [--dry-run] [--allow-partial] [--new-run] [--yes] [--json] clerk migrate runs [run-id] [--json] clerk migrate undo [--dry-run] [--yes] [--include-signed-in] [--json] @@ -260,8 +260,9 @@ The file is an envelope around the users: `--source`. `--json` prints the result on stdout instead — `{ target, run, output, users, -coverage, next }` — and never prompts, so a missing credential exits 2 naming -the flag to pass. +skipped, coverage, next }` — and never prompts, so a missing credential exits 2 +naming the flag to pass. `next`, and the commands printed after a human run, +keep `--runs-dir`: the import looks the run ID up there. | Flag | Platforms | Description | | --------------------- | ---------- | --------------------------------------------------------- | @@ -288,8 +289,10 @@ back into itself. Instead: instead — one flat row each, `my-app - Production instance (ins_…)`, not an application picker followed by an instance picker — with the resolved application's instances listed **first** so taking one is still a single - Enter. Only when there are no instances to offer does it stop and list - `--secret-key`, `--app`/`--instance` and `clerk link` instead. + Enter. When there are no instances to offer, it exits 2 naming + `--secret-key`, `--app`/`--instance` and `CLERK_SECRET_KEY`. When your + applications can't be listed (an accountless app with no login), it asks + whether to export the resolved instance instead. - With nothing to resolve at all (no link, no key, no flags), you get the application picker `clerk users` uses — `Select a Clerk application to use:`, followed by an instance picker when the application has more than one — @@ -301,8 +304,13 @@ The instance picker (the second tier) has no "create a new application" choice. Its rows are searchable by what they show, so typing an application name, `production`, or an instance id all narrow it. -In agent mode the resolved instance is used without a prompt; pass -`--secret-key` or `--app`/`--instance` to be explicit. +With nobody to ask — agent mode, `--json` or `-y` — an instance resolved on +your behalf is never taken: the export exits 2 and names the flags, since +exporting the destination and importing it back is the mistake this guards +against. The same goes for nothing to resolve at all. Name the source with +`--secret-key`, `--app`/`--instance` or `CLERK_SECRET_KEY`. And `clerk migrate +import` refuses an export run of the instance it would import into: every user +is already there. After each export you get a field-coverage table — which Clerk-relevant fields were present on how many users — so you know the data is thin _before_ you @@ -505,7 +513,8 @@ stays on the user. A run recorded before the marker existed adopts nothing. When an import completes, it names the folders it no longer needs: the export it read, which holds your users' data, and its own run, which only `undo` needs. -Each comes with the `rm -rf` to remove it. +Each comes with the command to remove it: `rm -rf` for a run folder, and a +plain `rm` for an export written with `--output` outside one. #### Checks @@ -929,21 +938,22 @@ stamping every user with today's. ## API endpoints -| Method | Path | Used by | -| -------- | ----------------------------------------------- | ------------------------------------------------------------------------------- | -| `POST` | `/v1/users` | `migrate import` — creates each user | -| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | -| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | -| `GET` | `/v1/users?limit=&offset=&order_by=+created_at` | `migrate export clerk` — pages the whole instance, oldest first, 500 at a time | -| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | -| `GET` | `/v1/users?external_id=…` | `migrate import` — checks for users already in the instance, 100 values a call | -| `GET` | `/v1/users?external_id=…` | `migrate import` — a continue looks up users whose create was in flight | -| `GET` | `/v1/users?external_id=…` | `migrate undo` — finds users whose create was in flight when the import stopped | -| `GET` | `/v1/users?user_id=…` | `migrate undo` — reads the imported users back, 100 a call | -| `DELETE` | `/v1/users/{user_id}` | `migrate undo` — deletes one user | -| `GET` | `/v1/instance` | `migrate import`, `undo` — names the instance behind the key | -| `GET` | `/v1/instance` | `migrate import`, `undo`, `export clerk` — names the instance behind the key | -| `GET` | `/v1/domains` | `migrate import` checks — resolves the Frontend API host | +| Method | Path | Used by | +| -------- | ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `POST` | `/v1/users` | `migrate import` — creates each user | +| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | +| `GET` | `/v1/users?limit=&offset=&order_by=+created_at` | `migrate export clerk` — pages the whole instance, oldest first, 500 at a time | +| `GET` | `/v1/users?limit=&starting_after=&order_by=+created_at` | `migrate export clerk` — pages the whole instance, oldest first, 500 at a time, each page from the last user of the one before | +| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | +| `GET` | `/v1/users?external_id=…` | `migrate import` — checks for users already in the instance, 100 values a call | +| `GET` | `/v1/users?external_id=…` | `migrate import` — a continue looks up users whose create was in flight | +| `GET` | `/v1/users?external_id=…` | `migrate undo` — finds users whose create was in flight when the import stopped | +| `GET` | `/v1/users?user_id=…` | `migrate undo` — reads the imported users back, 100 a call | +| `DELETE` | `/v1/users/{user_id}` | `migrate undo` — deletes one user | +| `GET` | `/v1/instance` | `migrate import`, `undo` — names the instance behind the key | +| `GET` | `/v1/instance` | `migrate import`, `undo`, `export clerk` — names the instance behind the key | +| `GET` | `/v1/domains` | `migrate import` checks — resolves the Frontend API host | The checks also read the instance's Frontend API `GET /v1/environment` (bootstrapping a dev browser first on development instances) for its diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts index 1a3105ecd..d6021f91d 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, mock, test } from "bun:test"; -import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { CliError, ERROR_CODE, EXIT_CODE, UserAbortError } from "../../../lib/errors.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; const mockDescribeBapiTarget = mock(); @@ -29,6 +29,11 @@ mock.module("../../users/interactive/instance-context.ts", () => ({ resolveUsersInstanceContext: (...args: unknown[]) => mockResolveUsersInstanceContext(...args), })); +const mockConfirm = mock(); +mock.module("../../../lib/prompts.ts", () => ({ + confirm: (...args: unknown[]) => mockConfirm(...args), +})); + let human = true; mock.module("../../../mode.ts", () => ({ isHuman: () => human, @@ -52,6 +57,7 @@ beforeEach(() => { mockResolveUsersInstanceContext.mockReset(); mockFetchApps.mockReset(); mockSearch.mockReset(); + mockConfirm.mockReset(); delete process.env.CLERK_SECRET_KEY; }); @@ -192,71 +198,99 @@ describe("resolveClerkSource", () => { // An empty list is not a picker. PLAPI being degraded looks the same as an // account with no applications, and neither one has an instance to offer. - test("no instances to offer falls back to the flags", async () => { + test("no instances to offer is a usage error naming the flags", async () => { stubResolved("my-app (development)"); // An application with no instances is not an offer either. mockFetchApps.mockResolvedValue([{ application_id: "app_1", name: "my-app", instances: [] }]); - await expect(resolveClerkSource({})).rejects.toBeInstanceOf(UserAbortError); + const error = (await resolveClerkSource({}).catch((e: unknown) => e)) as CliError; + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("There are no instances to pick from"); + expect(error.message).toContain("--secret-key"); + expect(error.message).toContain("--app"); expect(mockSearch).not.toHaveBeenCalled(); - expect(captured.err).toContain("--secret-key"); - expect(captured.err).toContain("--app"); - expect(captured.err).toContain("clerk link"); }); - test("agent mode takes the resolved instance without prompting", async () => { - human = false; - stubResolved("my-app (production)"); + // An accountless app's key is resolved; listing applications needs a login. + test("offers the resolved instance when applications can't be listed", async () => { + stubResolved("this accountless application (secret key from .env)"); + mockFetchApps.mockRejectedValue(new CliError("Not authenticated. Run `clerk auth login`.")); + mockConfirm.mockResolvedValue(true); const source = await resolveClerkSource({}); expect(source.secretKey).toBe("sk_test_resolved"); - expect(mockSearch).not.toHaveBeenCalled(); - }); - - // `--json` never prompts, even at a terminal: it takes what an agent would. - // `-y` is "do not prompt": it takes the resolved instance as an agent does. - test("-y takes the resolved instance without the picker", async () => { - stubResolved("my-app (production)"); - setAssumeYes(true); - - const source = await resolveClerkSource({}); - - expect(source.secretKey).toBe("sk_test_resolved"); - expect(mockSearch).not.toHaveBeenCalled(); - }); - - test("-y in an unlinked directory fails rather than opening the picker", async () => { - const failure = new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }); - mockDescribeBapiTarget.mockRejectedValue(failure); - setAssumeYes(true); - - await expect(resolveClerkSource({})).rejects.toThrow(failure); - expect(mockResolveUsersInstanceContext).not.toHaveBeenCalled(); + expect(mockConfirm).toHaveBeenCalledTimes(1); }); - test("--json takes the resolved instance without prompting", async () => { - stubResolved("my-app (production)"); - - const source = await resolveClerkSource({ json: true }); + test("a no to the resolved instance, when applications can't be listed, exports nothing", async () => { + stubResolved("this accountless application (secret key from .env)"); + mockFetchApps.mockRejectedValue(new CliError("Not authenticated. Run `clerk auth login`.")); + mockConfirm.mockResolvedValue(false); - expect(source.secretKey).toBe("sk_test_resolved"); - expect(mockSearch).not.toHaveBeenCalled(); + await expect(resolveClerkSource({})).rejects.toBeInstanceOf(UserAbortError); }); - test("--json in an unlinked directory fails rather than opening the picker", async () => { - const failure = new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }); - mockDescribeBapiTarget.mockRejectedValue(failure); + // Nobody to ask, and the resolved instance is usually the destination: an + // agent would export it and import it straight back. + test.each([ + [ + "agent mode", + (): void => { + human = false; + }, + {}, + ], + ["-y", (): void => setAssumeYes(true), {}], + ["--json", (): void => {}, { json: true }], + ] as const)( + "%s refuses the resolved instance, naming the flags", + async (_label, arrange, options) => { + stubResolved("my-app (production)"); + arrange(); + + const error = (await resolveClerkSource(options).catch((e: unknown) => e)) as CliError; + + expect(error).toBeInstanceOf(CliError); + expect(error.message).toContain("won't export my-app (production) without asking"); + expect(error.message).toContain("--secret-key"); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(mockSearch).not.toHaveBeenCalled(); + }, + ); + + test("agent mode still takes a named source", async () => { + human = false; + stubResolved(undefined, "sk_test_explicit"); - await expect(resolveClerkSource({ json: true })).rejects.toThrow(failure); + const source = await resolveClerkSource({ secretKey: "sk_test_explicit" }); - expect(mockResolveUsersInstanceContext).not.toHaveBeenCalled(); + expect(source.secretKey).toBe("sk_test_explicit"); }); + // Unlinked with no key, the resolver says "not linked" before "no key". + test.each([ + ["-y", (): void => setAssumeYes(true), {}, ERROR_CODE.NOT_LINKED], + ["--json", (): void => {}, { json: true }, ERROR_CODE.NOT_LINKED], + ["--json, no key", (): void => {}, { json: true }, ERROR_CODE.NO_SECRET_KEY], + ] as const)( + "%s in an unlinked directory is a usage error naming the flags", + async (_label, arrange, options, code) => { + mockDescribeBapiTarget.mockRejectedValue(new CliError("No Clerk project linked.", { code })); + arrange(); + + await expect(resolveClerkSource(options)).rejects.toThrow( + /has no source instance, and can't ask here/, + ); + expect(mockResolveUsersInstanceContext).not.toHaveBeenCalled(); + }, + ); + + // What the real resolver throws in an unlinked directory with no key. test("an unlinked directory picks an application instead of failing", async () => { mockDescribeBapiTarget.mockRejectedValue( - new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }), + new CliError("No Clerk project linked to this directory.", { code: ERROR_CODE.NOT_LINKED }), ); mockResolveUsersInstanceContext.mockResolvedValue({ secretKey: "sk_test_picked", diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.ts index 897c44170..54fc4abb3 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk-source.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.ts @@ -28,10 +28,10 @@ import { fetchAppsTolerantly } from "../../../lib/app-picker.ts"; import { describeBapiTarget, resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; import { resolveProfile } from "../../../lib/config.ts"; -import { CliError, ERROR_CODE, throwUserAbort } from "../../../lib/errors.ts"; +import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; import { search } from "../../../lib/listage.ts"; import type { ApplicationInstance } from "../../../lib/plapi.ts"; -import { log } from "../../../lib/log.ts"; +import { confirm } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; import { resolveUsersInstanceContext } from "../../users/interactive/instance-context.ts"; @@ -62,6 +62,18 @@ export type ClerkExportSource = { target?: string; }; +/** The flags that name a source instance, for a refusal that can't ask. */ +const NAME_A_SOURCE = + "Name the source instance with one of:\n" + + " `--secret-key ` — the source instance's secret key\n" + + " `--app --instance ` — an application on your account\n" + + " `CLERK_SECRET_KEY` — the source instance's secret key, exported"; + +/** Whether anyone is there to answer a picker. `-y` takes no question. */ +function canAsk(options: ResolveClerkSourceOptions): boolean { + return !options.json && !isAssumeYes() && isHuman(); +} + /** {@link ClerkExportSource} plus whether the user already chose it out loud. */ type ResolvedSource = ClerkExportSource & { chosen: boolean }; @@ -90,15 +102,16 @@ async function resolveSource(options: ResolveClerkSourceOptions): Promise { const { chosen, ...source } = await resolveSource(options); - // `-y` takes the resolved instance without the picker, as an agent does. - if (chosen || !source.target || options.json || isAssumeYes() || !isHuman()) return source; + if (chosen) return source; + + // What the CLI resolved on its own (the linked project, a key in `.env`) is + // normally the migration's destination. With nobody to ask, exporting it + // and printing `clerk migrate import ` points the import straight back + // at it, so the source has to be named. + if (!canAsk(options)) { + throwUsageError( + `\`clerk migrate export clerk\` won't export ${source.target ?? "the resolved instance"} without asking: it is usually where the users are going. ${NAME_A_SOURCE}`, + ); + } - const picked = await pickInstance(await currentAppId(options)); + let picked: ClerkExportSource | undefined; + try { + picked = await pickInstance(await currentAppId(options)); + } catch (error) { + // Listing applications needs an account. An accountless app's key is + // already resolved, so offer it rather than stop at a login. + if (!(error instanceof CliError)) throw error; + const use = await confirm({ + message: `Your applications can't be listed (${error.message.split("\n")[0]}). Export from ${source.target ?? "the resolved instance"}?`, + default: false, + }); + if (use) return source; + throwUserAbort(); + } if (picked) return picked; - log.info( - "Export from a different instance with one of:\n" + - " `--secret-key ` — the source instance's secret key\n" + - " `--app --instance ` — another application on your account\n" + - " `clerk link` — link this directory to a different application first", - ); - throwUserAbort(); + throwUsageError(`There are no instances to pick from. ${NAME_A_SOURCE}`); } diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts index 07afeb40d..1acc6aa5f 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -197,7 +197,7 @@ describe("mapClerkUserToExport", () => { }); describe("fetchAllClerkUsers", () => { - test("pages until a short page arrives", async () => { + test("pages until a short page arrives, each page from the last user of the one before", async () => { stubPages([ Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), Array.from({ length: 12 }, (_, i) => user({ id: `v${i}` })), @@ -207,23 +207,25 @@ describe("fetchAllClerkUsers", () => { expect(all).toHaveLength(512); expect(requests).toHaveLength(2); - expect(requests[1]).toContain("offset=480"); + expect(requests[0]).toContain("order_by=%2Bcreated_at"); + expect(requests[0]).not.toContain("starting_after"); + expect(requests[1]).toContain("starting_after=u499"); }); - // A user deleted after the first page shifts every later row back one. The - // overlap re-reads the boundary, so the row that slid onto it isn't lost. - test("keeps the user a mid-export deletion slides onto a page boundary", async () => { + // An offset would slide u500 back onto the page already read and skip it. + test("keeps every user when one already read is deleted mid-export", async () => { const users = Array.from({ length: 600 }, (_, i) => user({ id: `u${i}` })); let pages = 0; - // A server that answers by offset, and loses u10 after the first page. + // A server that answers by cursor, and loses u10 after the first page. globalThis.fetch = (async (input: string | URL | Request) => { const url = new URL(input.toString()); if (url.pathname === "/v1/instance") { return Response.json({ object: "instance", id: "ins_src", environment_type: "production" }); } - const offset = Number(url.searchParams.get("offset")); const rows = pages++ === 0 ? users : users.filter((entry) => entry.id !== "u10"); - return Response.json(rows.slice(offset, offset + 500)); + const after = url.searchParams.get("starting_after"); + const from = after ? rows.findIndex((entry) => entry.id === after) + 1 : 0; + return Response.json(rows.slice(from, from + 500)); }) as unknown as typeof fetch; const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); @@ -232,18 +234,20 @@ describe("fetchAllClerkUsers", () => { expect(all).toHaveLength(600); }); - // Oldest first, so a sign-up mid-export lands at the end; a repeat the - // overlap reads again is dropped. - test("pages oldest first and drops a user a shifted page repeats", async () => { - stubPages([ - Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), - [user({ id: "u499" }), user({ id: "v0" })], - ]); - - const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); - - expect(all).toHaveLength(501); - expect(requests[0]).toContain("order_by=%2Bcreated_at"); + // Clerk refuses a cursor user that is gone; the export must not carry on as if complete. + test("stops, saying so, when the user it continues from was deleted", async () => { + let call = 0; + globalThis.fetch = (async () => + call++ === 0 + ? Response.json(Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` }))) + : Response.json( + { errors: [{ code: "form_param_value_invalid", message: "invalid starting_after" }] }, + { status: 422 }, + )) as unknown as typeof fetch; + + await expect(fetchAllClerkUsers({ secretKey: "sk_test_x" })).rejects.toThrow( + /Users changed during the export/, + ); }); // A full final page must still trigger one more request, or an instance whose @@ -376,6 +380,20 @@ describe("exportClerk", () => { expect(captured.err).toContain("never returns password digests"); }); + test("says nothing about passwords when nobody had one", async () => { + stubPages([[user({ password_enabled: false })], []]); + await exportClerk({ secretKey: "sk_test_x" }); + expect(captured.err).not.toContain("never returns password digests"); + }); + + // An app keyed on external_id breaks after the move unless it knows. + test("says an app's external_id moves to private metadata", async () => { + stubPages([[user({ id: "u1", external_id: "app-1" }), user({ id: "u2" })], []]); + await exportClerk({ secretKey: "sk_test_x" }); + expect(captured.err).toContain("1 user has an external_id"); + expect(captured.err).toContain("private_metadata.clerkExternalId"); + }); + test("writes an empty file and says so when the instance has no users", async () => { stubPages([[]]); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts index 6f66aeb01..f2f8c161a 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -15,6 +15,7 @@ */ import { bapiRequest } from "../../../lib/bapi.ts"; +import { ApiError, CliError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; import type { UserLine } from "../lib/run-store.ts"; @@ -31,13 +32,6 @@ import { finishExport, startExportRun } from "./shared.ts"; /** BAPI's maximum page size for `GET /v1/users`. */ const PAGE_SIZE = 500; -/** - * How many rows each page re-reads from the end of the one before. A user - * deleted mid-export shifts every later row back one, so without it the row - * at each page boundary would be skipped; the Map drops the repeats. - */ -const PAGE_OVERLAP = 20; - export type ExportClerkOptions = { output?: string; secretKey?: string; @@ -182,38 +176,51 @@ export function mapClerkUserToExport(user: BapiUser): Record { return exported; } -/** Pages through every user in the instance. */ +/** + * Pages through every user in the instance, oldest first, each page from the + * last user of the one before (`starting_after`). A sign-up mid-export lands + * at the end, and a deletion can't slide a user past a page boundary the way + * it would with an offset. + * + * @throws CliError when the user a page continues from was deleted before the + * request: Clerk refuses the cursor (422), and the export stops rather than + * guess where to carry on. + */ export async function fetchAllClerkUsers(options: { secretKey: string; spinner?: SpinnerControls; }): Promise { - const all = new Map(); - - // Oldest first, so a sign-up during the export lands at the end instead of - // shifting every later page by one (BAPI's default is newest first). A - // deletion shifts later rows back, which would skip the row at the next - // page boundary, so each page overlaps the last. - // ponytail: offset paging, safe for up to PAGE_OVERLAP deletions per page - // mid-export; /v1/users has no cursor to page by instead. - for (let offset = 0; ; offset += PAGE_SIZE - PAGE_OVERLAP) { - const response = await retryOn429(async () => - bapiRequest({ - method: "GET", - path: `/v1/users?limit=${PAGE_SIZE}&offset=${offset}&order_by=%2Bcreated_at`, - secretKey: options.secretKey, - }), - ); + const all: BapiUser[] = []; + let after: string | undefined; + + for (;;) { + const params = new URLSearchParams({ limit: String(PAGE_SIZE), order_by: "+created_at" }); + if (after) params.set("starting_after", after); + let response; + try { + response = await retryOn429(async () => + bapiRequest({ method: "GET", path: `/v1/users?${params}`, secretKey: options.secretKey }), + ); + } catch (error) { + if (after && error instanceof ApiError && error.status === 422) { + throw new CliError( + "Users changed during the export: the user the next page continues from was deleted. Nothing was exported. Run the export again.", + ); + } + throw error; + } const page = Array.isArray(response.body) ? (response.body as BapiUser[]) : []; - for (const user of page) all.set(user.id, user); - options.spinner?.update(`Fetching users from Clerk: ${all.size} so far...`); + all.push(...page); + options.spinner?.update(`Fetching users from Clerk: ${all.length} so far...`); // A short page means the end; anything else would loop forever on an // instance whose size happens to be a multiple of the page size. if (page.length < PAGE_SIZE) break; + after = page.at(-1)!.id; } - return [...all.values()]; + return all; } export type ClerkExportResult = { @@ -299,11 +306,21 @@ export async function exportClerk(options: ExportClerkOptions): Promise { const { users: exported, coverage } = buildClerkExport(users, run.append); finishExport({ run, options, users: exported, coverage }); - if (exported.length > 0) { + if (users.some((user) => user.password_enabled)) { log.warn( "Clerk's API never returns password digests, TOTP secrets or backup codes, so they are not in this file. " + "Users will need to reset their password in the destination instance.", ); } + // The import makes each user's ID here its external_id, which undo and a + // re-run rely on, so an app that keys its own records on external_id has + // to look in the metadata after the move. + const withExternalId = users.filter((user) => user.external_id).length; + if (withExternalId > 0) { + log.warn( + `${withExternalId} user${withExternalId === 1 ? " has" : "s have"} an external_id. The import sets each user's external_id to its ID in this instance, ` + + "and keeps the app's own value as `private_metadata.clerkExternalId`: update anything that looks users up by external_id.", + ); + } }); } diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts index 9f7fa2142..eb08624ff 100644 --- a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -132,6 +132,12 @@ describe("resolveDbUrl", () => { expect(captured.err).toContain("SUPABASE_DB_URL is not a valid connection string"); }); + test("says the variable is set but unusable, rather than asking for it", async () => { + await expect(resolveDbUrl({}, config, { [config.envVar]: "garbage" })).rejects.toThrow( + `${config.envVar} is set, but isn't a connection string`, + ); + }); + test("names both the flag and the variable when it cannot prompt", async () => { await expect(resolveDbUrl({}, config, {})).rejects.toThrow(/--db-url.*SUPABASE_DB_URL/s); }); @@ -194,6 +200,27 @@ describe("supabase export", () => { expect(byLabel["have a password hash"]).toBe(1); }); + // The source splits a metadata name into first and last; so does the count. + test("counts a first and last name split from the metadata", () => { + const { coverage } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", raw_user_meta_data: { full_name: "Gus Smith" } }, + { id: "u2", email: "b@x.dev", raw_user_meta_data: { name: "Ada" } }, + { id: "u3", email: "c@x.dev" }, + ]); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have a first name"]).toBe(2); + expect(byLabel["have a last name"]).toBe(1); + }); + + test("keeps a ban Postgres stores as 'infinity', which JSON would write as null", () => { + const { users } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", banned_until: Infinity }, + { id: "u2", email: "b@x.dev", banned_until: -Infinity }, + ]); + expect(users[0]?.banned_until).toBe("9999-12-31T23:59:59.999Z"); + expect(users[1]).not.toHaveProperty("banned_until"); + }); + test("keeps raw_app_meta_data, which the import checks read for providers", () => { const { users } = buildSupabaseExport([ { id: "u1", email: "a@x.dev", raw_app_meta_data: { providers: ["discord"] } }, diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts index 98b73e027..8abb2ff1c 100644 --- a/packages/cli-core/src/commands/migrate/export/db-options.ts +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -185,7 +185,9 @@ export async function resolveDbUrl( if (options.json || isAssumeYes() || isAgent() || !isHuman()) { throwUsageError( `\`clerk migrate export ${config.platform}\` needs a database connection and cannot prompt here.\n` + - `Pass --db-url, or set ${config.envVar}.`, + (fromEnv + ? `${config.envVar} is set, but isn't a connection string. Fix it, or pass --db-url.` + : `Pass --db-url, or set ${config.envVar}.`), undefined, undefined, [ diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts index e9793b4c2..84f161f0b 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.test.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { quoteArg } from "../../../lib/json-body.ts"; import { useCaptureLog } from "../../../test/lib/stubs.ts"; import { readEnvelope } from "../lib/export-file.ts"; import { latestUserLines, readRun } from "../lib/run-store.ts"; @@ -96,7 +97,7 @@ describe("finishExport", () => { }); // The run is left behind with every user's line but no file. - test("a write that fails says where, and releases the run", async () => { + test("a write that fails says where, and leaves no run behind", async () => { const run = await startExportRun({ runsDir }, { platform: "supabase" }); const output = path.join(runsDir, "a-directory.json"); fs.mkdirSync(output); @@ -104,8 +105,34 @@ describe("finishExport", () => { expect(() => finishExport({ run, options: { output }, users, coverage })).toThrow( new RegExp(`Could not write the export to ${output}: .*\\n.*with another --output`), ); - expect(fs.existsSync(path.join(run.dir, "lock"))).toBe(false); - expect(readRun(runsDir, run.record.id)?.file).toBeUndefined(); + expect(fs.existsSync(run.dir)).toBe(false); + expect(readRun(runsDir, run.record.id)).toBeUndefined(); + }); + + // Import looks the run ID up in the runs folder it is given. + test("the printed commands keep --runs-dir", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + + finishExport({ run, options: { runsDir }, users, coverage }); + + expect(captured.err).toContain( + `clerk migrate import ${run.record.id} --runs-dir ${quoteArg(runsDir)}`, + ); + expect(captured.err).toContain( + `clerk migrate runs ${run.record.id} --runs-dir ${quoteArg(runsDir)}`, + ); + }); + + test("--json's next keeps --runs-dir, and counts the users skipped", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + run.append({ sourceId: "bad", status: "skipped", error: "no id" }); + + finishExport({ run, options: { runsDir, json: true }, users, coverage }); + + expect(JSON.parse(captured.out)).toMatchObject({ + next: `clerk migrate import ${run.record.id} --runs-dir ${quoteArg(runsDir)}`, + skipped: 1, + }); }); test("--output writes somewhere else, and the run still records where", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts index 2cac9bf55..7f1b17246 100644 --- a/packages/cli-core/src/commands/migrate/export/shared.ts +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -13,6 +13,7 @@ import path from "node:path"; import { dim, green, yellow } from "../../../lib/color.ts"; import { CliError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; +import { carryFlags, withFlags } from "../lib/carry-flags.ts"; import { ENVELOPE_VERSION, type ExportEnvelope } from "../lib/export-file.ts"; import { ACCOUNT_LINKING_NOTE } from "../sources/registry.ts"; import { @@ -112,10 +113,10 @@ export type ExportSection = { title: string; rows: string[] }; * for anyone holding `CLERK_SECRET_KEY=sk_live_…`, which is the reader who can * least afford it. So the note names what picks the instance instead. */ -export function formatImportCommand(runId: string): string[] { +export function formatImportCommand(runId: string, carry: string[] = []): string[] { return [ "Import them with:", - dim(` clerk migrate import ${runId}`), + dim(` ${withFlags(`clerk migrate import ${runId}`, carry)}`), "", dim(" Imports into whichever instance the resolved secret key belongs to."), dim(" For production, add `--instance prod` or use a production secret key."), @@ -160,9 +161,10 @@ export function finishExport(input: FinishExportInput): FinishedExport { users, }); } catch (error) { - // Released with no file, the run reads as an export that never finished, - // and an import by its ID says so. + // Nothing was written, so there is no run either: left behind, it would + // list as an export that stopped part-way. run.release(); + fs.rmSync(run.dir, { recursive: true, force: true }); throw new CliError( `Could not write the export to ${target}: ${(error as Error).message}\n` + `Nothing was written. Run the export again${options.output ? " with another --output" : ""}.`, @@ -170,7 +172,11 @@ export function finishExport(input: FinishExportInput): FinishedExport { } run.update({ file: { path: outputPath, sha256: sha256File(outputPath) } }); const record = run.finish(); - const next = `clerk migrate import ${record.id}`; + // Only the runs folder: the export's --secret-key, --app and --instance name + // the source, and would aim the import at it. + const carry = carryFlags({ runsDir: options.runsDir }); + const next = withFlags(`clerk migrate import ${record.id}`, carry); + const skipped = record.counts.skipped ?? 0; if (options.json) { log.data( @@ -180,6 +186,7 @@ export function finishExport(input: FinishExportInput): FinishedExport { run: record, output: outputPath, users: users.length, + skipped, coverage, ...(input.sections?.length ? { sections: input.sections } : {}), ...(input.truncated ? { truncated: true } : {}), @@ -210,13 +217,22 @@ export function finishExport(input: FinishExportInput): FinishedExport { log.blank(); log.success(`Exported ${users.length} user${users.length === 1 ? "" : "s"} to ${outputPath}`); - log.info(dim(`Run ${record.id}. See each user with \`clerk migrate runs ${record.id}\`.`)); + if (skipped > 0) { + log.warn( + `Skipped ${skipped} user${skipped === 1 ? "" : "s"} the export couldn't read. Each is a \`skipped\` line in the run, with why.`, + ); + } + log.info( + dim( + `Run ${record.id}. See each user with \`${withFlags(`clerk migrate runs ${record.id}`, carry)}\`.`, + ), + ); log.blank(); log.info(dim(ACCOUNT_LINKING_NOTE)); log.blank(); - for (const line of formatImportCommand(record.id)) log.info(line); + for (const line of formatImportCommand(record.id, carry)) log.info(line); return { record, outputPath }; } diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts index 2e40c22dc..efe8427da 100644 --- a/packages/cli-core/src/commands/migrate/export/supabase.ts +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -59,12 +59,33 @@ type SupabaseRow = Record & { raw_app_meta_data?: unknown; }; +/** + * The words of a display name in the user's metadata, as the Supabase source + * splits it into first and last name; empty when there is none. + */ +function metadataName(meta: unknown): string[] { + if (!meta || typeof meta !== "object") return []; + const { display_name, full_name, name } = meta as Record; + const value = display_name ?? full_name ?? name; + return typeof value === "string" ? value.trim().split(/\s+/).filter(Boolean) : []; +} + +/** How a ban Postgres stores as `'infinity'` is written: the last date JSON can hold. */ +const FOREVER = "9999-12-31T23:59:59.999Z"; + /** Serializes values the JSON export cannot carry as-is. */ function normalizeRow(row: SupabaseRow): Record { const normalized: Record = {}; for (const [key, value] of Object.entries(row)) { if (value === null || value === undefined) continue; + // Bun.SQL returns a timestamp of 'infinity' as the number Infinity, which + // JSON writes as null: a ban set by hand to last forever would be lost. + if (value === Infinity) { + normalized[key] = FOREVER; + continue; + } + if (value === -Infinity) continue; // Postgres returns timestamps as Date objects; the transformer parses // strings, and JSON.stringify would otherwise bury the format difference. normalized[key] = value instanceof Date ? value.toISOString() : value; @@ -100,8 +121,10 @@ export function buildSupabaseExport( if (row.email_confirmed_at) counts.emailConfirmed++; if (row.encrypted_password) counts.password++; if (row.phone) counts.phone++; - if (row.first_name) counts.firstName++; - if (row.last_name) counts.lastName++; + // The source splits a name it finds in the metadata, so count those too. + const [first, ...rest] = metadataName(row.raw_user_meta_data); + if (row.first_name || first) counts.firstName++; + if (row.last_name || rest.length > 0) counts.lastName++; record({ sourceId: userId, status: "exported" }); } catch (error) { diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts index 67bd157c3..f2260cd04 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts @@ -34,7 +34,7 @@ mock.module("../../../lib/prompts.ts", () => ({ note: () => {}, })); -const { withInputRetry } = await import("./input-retry.ts"); +const { terminal, withInputRetry } = await import("./input-retry.ts"); const { promptDbUrl, resolveDbUrl, withDbConnection } = await import("../export/db-options.ts"); const { setAssumeYes } = await import("./assume-yes.ts"); @@ -63,14 +63,38 @@ afterAll(() => { else process.env.CLERK_MODE = originalMode; }); +// The prompts here are stubs; whether this process has a terminal is not +// what most of these tests are about. +const realTerminal = terminal.available; + beforeEach(() => { process.env.CLERK_MODE = "human"; setAssumeYes(false); answers = []; cancelPrompt = false; + terminal.available = () => true; +}); + +afterAll(() => { + terminal.available = realTerminal; }); describe("withInputRetry", () => { + // `CLERK_MODE=human` with no terminal: a prompt would fail to open one and spin. + test("fails rather than asking again when there is no terminal to ask on", async () => { + terminal.available = () => false; + answers = [SECOND]; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => Promise.reject(rejected()), + ), + ).rejects.toThrow("Could not reach it"); + expect(answers).toEqual([SECOND]); + }); + test("returns the first result without prompting when the work succeeds", async () => { const seen: string[] = []; diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.ts index edc34be74..a85d738da 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.ts @@ -15,11 +15,29 @@ * full re-run for one line they could not see. */ +import fs from "node:fs"; import { CliError, EXIT_CODE, throwUsageError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { isAssumeYes } from "./assume-yes.ts"; +/** + * Whether a prompt has a terminal to read: stdin, or `/dev/tty` behind a piped + * stdin. `CLERK_MODE=human` forces human mode without one, and a prompt with + * no terminal fails to open it and spins. An object so tests can swap it. + */ +export const terminal = { + available(): boolean { + if (process.stdin.isTTY) return true; + try { + fs.closeSync(fs.openSync(process.platform === "win32" ? "CONIN$" : "/dev/tty", "r")); + return true; + } catch { + return false; + } + }, +}; + /** * Runs `work`, and on failure asks for the input again and runs it once more. * @@ -62,6 +80,7 @@ export async function withInputRetry( // it is asked for again and, unanswered, exits 2. const badInput = error instanceof CliError && error.exitCode === EXIT_CODE.USAGE; if (!badInput || options.json || !isHuman() || isAgent() || isAssumeYes()) throw error; + if (!terminal.available()) throw error; log.error(error.message); candidate = await reprompt(); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index f05ee2666..a5caccec7 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -459,6 +459,35 @@ describe("run", () => { expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); }); + /** Records which instance an export run read, as `export clerk` does. */ + const exportedFrom = (id: string, instanceId: string) => { + const record = readRun(runsDir(), id)!; + fs.writeFileSync( + path.join(runsDir(), id, "run.json"), + JSON.stringify({ ...record, target: { ...record.target, instanceId } }), + ); + }; + + // Every user is already there; each would come back a duplicate. + test("refuses to import an export into the instance it came from", async () => { + const { record } = exportRun("clerk", export2); + exportedFrom(record.id, "ins_1"); + + await expect(run({ ...noSource, input: record.id })).rejects.toThrow( + /exported this instance's own users/, + ); + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(0); + }); + + test("imports an export of another instance", async () => { + const { record } = exportRun("clerk", export2); + exportedFrom(record.id, "ins_other"); + + await run({ ...noSource, input: record.id }); + + expect(requests.filter((r) => r.url.endsWith("/v1/users"))).toHaveLength(2); + }); + test("refuses a run ID that is not an export", async () => { await run(baseOptions); const [imported] = listRuns(runsDir()); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 91ea5e680..d7c1460b0 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -281,9 +281,13 @@ async function ensureImportTarget(options: MigrateRunOptions): Promise { * Settles which file to read. The argument may name the export run that wrote * the file; a human who gave none is asked for a path. */ -async function resolveInput( - options: MigrateRunOptions, -): Promise<{ file: string; fromExport?: string; exportSha256?: string }> { +async function resolveInput(options: MigrateRunOptions): Promise<{ + file: string; + fromExport?: string; + exportSha256?: string; + /** The instance an export run read its users from. */ + exportInstanceId?: string; +}> { const value = options.input; if (!value) { if (!canPrompt(options)) { @@ -335,7 +339,12 @@ async function resolveInput( } // Carried on, so the file is checked again where the users are read: another // export can still overwrite the path after this. - return { file, fromExport: record.id, exportSha256: sha256 }; + return { + file, + fromExport: record.id, + exportSha256: sha256, + ...(record.target.instanceId ? { exportInstanceId: record.target.instanceId } : {}), + }; } /** A file the import read changed before it finished reading it. */ @@ -767,6 +776,13 @@ export async function run(rawOptions: MigrateRunOptions): Promise { async ({ setNextSteps }) => { const { secretKey, target } = await resolveClerkTarget(options); if (!options.json) printTarget(target); + // Every user is already here, so each would be refused as a duplicate. + if (input.exportInstanceId && input.exportInstanceId === target.instanceId) { + throwUsageError( + `Run ${input.fromExport} exported this instance's own users, so importing it here would put them back where they came from. Nothing was imported.\n` + + "Import into another instance: pass --instance prod, --app --instance , or that instance's --secret-key.", + ); + } const limits = resolveLimits(secretKey); const filePath = resolveImportFilePath(file); From 782ae64423d34efebc907640995e079bd4b44e5a Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 15 Oct 2026 11:48:07 -0400 Subject: [PATCH 36/39] refactor(migrate): read the terminal check from lib/terminal.ts in the credential retry Slice 2 now keeps the check there for import's and undo's consent prompts, so the retry shares it rather than keeping a copy. Co-Authored-By: Claude Opus 5.5 --- .../commands/migrate/lib/input-retry.test.ts | 3 ++- .../src/commands/migrate/lib/input-retry.ts | 19 +------------------ 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts index f2260cd04..db1c30c85 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts @@ -34,7 +34,8 @@ mock.module("../../../lib/prompts.ts", () => ({ note: () => {}, })); -const { terminal, withInputRetry } = await import("./input-retry.ts"); +const { withInputRetry } = await import("./input-retry.ts"); +const { terminal } = await import("./terminal.ts"); const { promptDbUrl, resolveDbUrl, withDbConnection } = await import("../export/db-options.ts"); const { setAssumeYes } = await import("./assume-yes.ts"); diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.ts index a85d738da..4598f7113 100644 --- a/packages/cli-core/src/commands/migrate/lib/input-retry.ts +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.ts @@ -15,28 +15,11 @@ * full re-run for one line they could not see. */ -import fs from "node:fs"; import { CliError, EXIT_CODE, throwUsageError } from "../../../lib/errors.ts"; import { log } from "../../../lib/log.ts"; import { isAgent, isHuman } from "../../../mode.ts"; import { isAssumeYes } from "./assume-yes.ts"; - -/** - * Whether a prompt has a terminal to read: stdin, or `/dev/tty` behind a piped - * stdin. `CLERK_MODE=human` forces human mode without one, and a prompt with - * no terminal fails to open it and spins. An object so tests can swap it. - */ -export const terminal = { - available(): boolean { - if (process.stdin.isTTY) return true; - try { - fs.closeSync(fs.openSync(process.platform === "win32" ? "CONIN$" : "/dev/tty", "r")); - return true; - } catch { - return false; - } - }, -}; +import { terminal } from "./terminal.ts"; /** * Runs `work`, and on failure asks for the input again and runs it once more. From c27bf1b696f4cf881cfe9f80ede5320023c6c16d Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Thu, 15 Oct 2026 12:48:21 -0400 Subject: [PATCH 37/39] fix(migrate): offer the same flat instance list when unlinked, put the linked instance last, and name the export's own file in cleanup From Michael's re-test of #524, and CodeRabbit on the cleanup hint: - An unlinked directory opened `clerk link`'s app picker, whose first row creates an empty app. It now gets the flat instance list the linked case uses; not logged in, it exits 2 naming the flags. - Only a missing or expired login while listing applications offers the resolved instance; a failure after picking another one goes through. The prompt says listing needs a login, after the spinner's failure line. - The linked application's instances come last, marked "(linked: usually the destination)", instead of being the preselected first row. - The cleanup hint reads the export file's path from the export run's own record, and names the copy this import read separately when it differs. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 27 +++-- .../migrate/export/clerk-source.test.ts | 100 ++++++++++++++++-- .../commands/migrate/export/clerk-source.ts | 95 ++++++++++------- .../cli-core/src/commands/migrate/run.test.ts | 65 ++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 26 +++-- 5 files changed, 243 insertions(+), 70 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 0c7139850..4f3f1b311 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -287,20 +287,19 @@ back into itself. Instead: - Anything resolved on your behalf — the linked project, a keyless app — is never taken silently. A picker of every **instance** on your account opens instead — one flat row each, `my-app - Production instance (ins_…)`, not an - application picker followed by an instance picker — with the resolved - application's instances listed **first** so taking one is still a single - Enter. When there are no instances to offer, it exits 2 naming - `--secret-key`, `--app`/`--instance` and `CLERK_SECRET_KEY`. When your - applications can't be listed (an accountless app with no login), it asks - whether to export the resolved instance instead. -- With nothing to resolve at all (no link, no key, no flags), you get the - application picker `clerk users` uses — `Select a Clerk application to use:`, - followed by an instance picker when the application has more than one — - rather than an error about an unlinked directory. That is `clerk link`'s - picker, so it does offer `+ Create a new application`; a brand-new - application has no users to export, so it is never the answer here. - -The instance picker (the second tier) has no "create a new application" choice. + application picker followed by an instance picker. The resolved + application's instances come **last**, marked `(linked: usually the +destination)`: it is usually where the users are going, and the first row is + what a reflexive Enter takes. When there are no instances to offer, it exits + 2 naming `--secret-key`, `--app`/`--instance` and `CLERK_SECRET_KEY`. When + listing your applications needs a login (an accountless app, or an expired + session), it asks whether to export the resolved instance instead. +- With nothing to resolve at all (no link, no key, no flags), you get the same + flat list of instances, rather than an error about an unlinked directory. + Not logged in, there is nothing to list, so it exits 2 naming the same flags. + +The instance picker has no "create a new application" choice: a brand-new +application has no users to export. Its rows are searchable by what they show, so typing an application name, `production`, or an instance id all narrow it. diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts index d6021f91d..f82d89f7d 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts @@ -110,7 +110,9 @@ describe("resolveClerkSource", () => { // Exporting the instance that is about to be imported *into* is the failure // this whole module exists to prevent, so a resolved instance is offered as // one choice among the account's applications rather than taken silently. - test("offers every instance, flat, with the linked application's first", async () => { + // The linked application is usually the destination, and the first row is + // what a reflexive Enter takes: it goes last, marked. + test("offers every instance, flat, with the linked application's last and marked", async () => { stubResolved("my-app (development)"); mockResolveProfile.mockResolvedValue({ profile: { appId: "app_2" } }); mockFetchApps.mockResolvedValue([ @@ -143,10 +145,6 @@ describe("resolveClerkSource", () => { const { message, source: listSource } = mockSearch.mock.calls[0]![0]; expect(message).toBe("What Clerk instance do you want to export users from?"); expect(listSource("")).toEqual([ - { - name: "other-app - Development instance (ins_2d)", - value: { app: "app_2", instance: "ins_2d" }, - }, { name: "my-app - Development instance (ins_1d)", value: { app: "app_1", instance: "ins_1d" }, @@ -155,6 +153,10 @@ describe("resolveClerkSource", () => { name: "my-app - Production instance (ins_1p)", value: { app: "app_1", instance: "ins_1p" }, }, + { + name: "other-app - Development instance (ins_2d) (linked: usually the destination)", + value: { app: "app_2", instance: "ins_2d" }, + }, ]); // Both halves are handed on, so the secret-key lookup runs against exactly // the instance that was chosen and nothing prompts a second time. @@ -215,7 +217,11 @@ describe("resolveClerkSource", () => { // An accountless app's key is resolved; listing applications needs a login. test("offers the resolved instance when applications can't be listed", async () => { stubResolved("this accountless application (secret key from .env)"); - mockFetchApps.mockRejectedValue(new CliError("Not authenticated. Run `clerk auth login`.")); + mockFetchApps.mockRejectedValue( + new CliError("Not authenticated. Run `clerk auth login`.", { + code: ERROR_CODE.AUTH_REQUIRED, + }), + ); mockConfirm.mockResolvedValue(true); const source = await resolveClerkSource({}); @@ -226,7 +232,11 @@ describe("resolveClerkSource", () => { test("a no to the resolved instance, when applications can't be listed, exports nothing", async () => { stubResolved("this accountless application (secret key from .env)"); - mockFetchApps.mockRejectedValue(new CliError("Not authenticated. Run `clerk auth login`.")); + mockFetchApps.mockRejectedValue( + new CliError("Not authenticated. Run `clerk auth login`.", { + code: ERROR_CODE.AUTH_REQUIRED, + }), + ); mockConfirm.mockResolvedValue(false); await expect(resolveClerkSource({})).rejects.toBeInstanceOf(UserAbortError); @@ -287,11 +297,20 @@ describe("resolveClerkSource", () => { }, ); - // What the real resolver throws in an unlinked directory with no key. - test("an unlinked directory picks an application instead of failing", async () => { + // What the real resolver throws in an unlinked directory with no key. The + // same flat list as the linked case, with no "create a new application" row. + test("an unlinked directory offers every instance, flat, instead of failing", async () => { mockDescribeBapiTarget.mockRejectedValue( new CliError("No Clerk project linked to this directory.", { code: ERROR_CODE.NOT_LINKED }), ); + mockFetchApps.mockResolvedValue([ + { + application_id: "app_1", + name: "other-app", + instances: [{ instance_id: "ins_1p", environment_type: "production" }], + }, + ]); + mockSearch.mockResolvedValue({ app: "app_1", instance: "ins_1p" }); mockResolveUsersInstanceContext.mockResolvedValue({ secretKey: "sk_test_picked", appLabel: "other-app", @@ -301,8 +320,67 @@ describe("resolveClerkSource", () => { const source = await resolveClerkSource({}); expect(source).toEqual({ secretKey: "sk_test_picked", target: "other-app (production)" }); - // The picker just asked which application; asking again is noise. - expect(mockSearch).not.toHaveBeenCalled(); + const { source: listSource } = mockSearch.mock.calls[0]![0]; + expect(listSource("").map((choice: { name: string }) => choice.name)).toEqual([ + "other-app - Production instance (ins_1p)", + ]); + expect(mockResolveUsersInstanceContext).toHaveBeenCalledWith({ + app: "app_1", + instance: "ins_1p", + }); + }); + + test("an unlinked directory with no login is a usage error naming the flags", async () => { + mockDescribeBapiTarget.mockRejectedValue( + new CliError("No Clerk project linked to this directory.", { code: ERROR_CODE.NOT_LINKED }), + ); + mockFetchApps.mockRejectedValue( + new CliError("Not authenticated. Run `clerk auth login`.", { + code: ERROR_CODE.AUTH_REQUIRED, + }), + ); + + const error = (await resolveClerkSource({}).catch((e: unknown) => e)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("Listing your applications needs `clerk auth login`"); + expect(error.message).toContain("--secret-key"); + expect(mockConfirm).not.toHaveBeenCalled(); + }); + + // Only "can't list" offers the resolved instance. A failure after picking + // another one is that failure, not a reason to offer the destination. + test("a failure after picking an instance goes through, not to the fallback", async () => { + stubResolved("my-app (development)"); + mockFetchApps.mockResolvedValue([ + { + application_id: "app_1", + name: "other-app", + instances: [{ instance_id: "ins_1p", environment_type: "production" }], + }, + ]); + mockSearch.mockResolvedValue({ app: "app_1", instance: "ins_1p" }); + const failure = new CliError("Could not fetch that instance's secret key."); + mockResolveUsersInstanceContext.mockRejectedValue(failure); + + await expect(resolveClerkSource({})).rejects.toThrow(failure); + expect(mockConfirm).not.toHaveBeenCalled(); + }); + + test("names why it offers the resolved instance when applications can't be listed", async () => { + stubResolved("this accountless application (secret key from .env)"); + mockFetchApps.mockRejectedValue( + new CliError("Not authenticated. Run `clerk auth login`.", { + code: ERROR_CODE.AUTH_REQUIRED, + }), + ); + mockConfirm.mockResolvedValue(true); + + await resolveClerkSource({}); + + expect(mockConfirm.mock.calls[0]![0].message).toContain( + "Listing your applications needs a login", + ); }); test("an explicit --app that fails to resolve surfaces the error, not the picker", async () => { diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.ts index 54fc4abb3..b125ea05c 100644 --- a/packages/cli-core/src/commands/migrate/export/clerk-source.ts +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.ts @@ -28,9 +28,15 @@ import { fetchAppsTolerantly } from "../../../lib/app-picker.ts"; import { describeBapiTarget, resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; import { resolveProfile } from "../../../lib/config.ts"; -import { CliError, ERROR_CODE, throwUsageError, throwUserAbort } from "../../../lib/errors.ts"; +import { + CliError, + ERROR_CODE, + isAuthError, + throwUsageError, + throwUserAbort, +} from "../../../lib/errors.ts"; import { search } from "../../../lib/listage.ts"; -import type { ApplicationInstance } from "../../../lib/plapi.ts"; +import type { Application, ApplicationInstance } from "../../../lib/plapi.ts"; import { confirm } from "../../../lib/prompts.ts"; import { isHuman } from "../../../mode.ts"; import { isAssumeYes } from "../lib/assume-yes.ts"; @@ -77,7 +83,9 @@ function canAsk(options: ResolveClerkSourceOptions): boolean { /** {@link ClerkExportSource} plus whether the user already chose it out loud. */ type ResolvedSource = ClerkExportSource & { chosen: boolean }; -async function resolveSource(options: ResolveClerkSourceOptions): Promise { +async function resolveSource( + options: ResolveClerkSourceOptions, +): Promise { // What separates the two tiers is not "did the CLI have to look anything up" // but "did the user say which instance". A flag or an exported key is a // sentence they typed for this run; the linked project is a choice they made @@ -113,15 +121,22 @@ async function resolveSource(options: ResolveClerkSourceOptions): Promise { + try { + return await fetchAppsTolerantly(); + } catch (error) { + // Only a missing or expired login is "can't list"; anything else is a + // failure of its own, and goes through as one. + if (isAuthError(error)) return undefined; + throw error; } } @@ -138,26 +153,25 @@ async function resolveSource(options: ResolveClerkSourceOptions): Promise { - const apps = await fetchAppsTolerantly(); - - const ordered = currentAppId - ? [ - ...apps.filter((app) => app.application_id === currentAppId), - ...apps.filter((app) => app.application_id !== currentAppId), - ] - : apps; +async function pickInstance( + apps: Application[], + linkedAppId?: string, +): Promise { + const linked = (app: Application) => app.application_id === linkedAppId; + const ordered = [...apps.filter((app) => !linked(app)), ...apps.filter(linked)]; const choices = ordered.flatMap((app) => (app.instances ?? []).map((instance) => ({ - name: `${app.name || app.application_id} - ${instanceLabel(instance)} (${instance.instance_id})`, + name: + `${app.name || app.application_id} - ${instanceLabel(instance)} (${instance.instance_id})` + + (linked(app) ? " (linked: usually the destination)" : ""), value: { app: app.application_id, instance: instance.instance_id }, })), ); @@ -190,8 +204,9 @@ async function currentAppId(options: ResolveClerkSourceOptions): Promise { - const { chosen, ...source } = await resolveSource(options); - if (chosen) return source; + const resolved = await resolveSource(options); + if (resolved?.chosen) return { secretKey: resolved.secretKey, target: resolved.target }; + const source = resolved && { secretKey: resolved.secretKey, target: resolved.target }; // What the CLI resolved on its own (the linked project, a key in `.env`) is // normally the migration's destination. With nobody to ask, exporting it @@ -199,24 +214,28 @@ export async function resolveClerkSource( // at it, so the source has to be named. if (!canAsk(options)) { throwUsageError( - `\`clerk migrate export clerk\` won't export ${source.target ?? "the resolved instance"} without asking: it is usually where the users are going. ${NAME_A_SOURCE}`, + `\`clerk migrate export clerk\` won't export ${source?.target ?? "the resolved instance"} without asking: it is usually where the users are going. ${NAME_A_SOURCE}`, ); } - let picked: ClerkExportSource | undefined; - try { - picked = await pickInstance(await currentAppId(options)); - } catch (error) { - // Listing applications needs an account. An accountless app's key is - // already resolved, so offer it rather than stop at a login. - if (!(error instanceof CliError)) throw error; + const apps = await listApps(); + if (!apps) { + // Listing applications needs a login. An accountless app's key is + // already resolved, so it can still be offered on its own. + if (!source) { + throwUsageError( + `Listing your applications needs \`clerk auth login\`, and nothing here names a source instance. ${NAME_A_SOURCE}`, + ); + } const use = await confirm({ - message: `Your applications can't be listed (${error.message.split("\n")[0]}). Export from ${source.target ?? "the resolved instance"}?`, + message: `Listing your applications needs a login, so they can't be offered. Export from ${source.target ?? "the resolved instance"} instead?`, default: false, }); if (use) return source; throwUserAbort(); } + + const picked = await pickInstance(apps, await currentAppId(options)); if (picked) return picked; throwUsageError(`There are no instances to pick from. ${NAME_A_SOURCE}`); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index a5caccec7..1dd968ff6 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -413,6 +413,71 @@ describe("run", () => { expect(captured.err).toContain(`rm ${outside}`); }); + // Started from a copy, continued by the export's ID: the run kept the + // copy's path, but the export's own file outside its folder must go too. + test("names the export's own file, and the copy the import read, when they differ", async () => { + const { record, file } = exportRun("clerk", export2); + const exported = path.join(workDir, `users-${record.id}.json`); + fs.renameSync(file, exported); + patchRun(runsDir(), record.id, { file: { path: exported, sha256: sha256File(exported) } }); + const copy = path.join(workDir, "copy.json"); + fs.copyFileSync(exported, copy); + stubClerk({ failing: new Set(["u2"]) }); + await run({ ...noSource, input: copy }); + + stubClerk(); + process.exitCode = 0; + captured.clear(); + await run({ ...noSource, input: record.id }); + + expect(captured.err).toContain(`rm -rf ${path.join(runsDir(), record.id)}`); + expect(captured.err).toContain(`rm ${exported}`); + expect(captured.err).toContain("The file this import read holds the same data:"); + expect(captured.err).toContain(`rm ${copy}`); + }); + + test("names one file once when the import read the export's own", async () => { + const { record, file } = exportRun("clerk", export2); + const exported = path.join(workDir, `users-${record.id}.json`); + fs.renameSync(file, exported); + patchRun(runsDir(), record.id, { file: { path: exported, sha256: sha256File(exported) } }); + + await run({ ...noSource, input: record.id }); + + expect(captured.err.split(`rm ${exported}`)).toHaveLength(2); + expect(captured.err).not.toContain("The file this import read"); + }); + + test("names no file inside the export's folder, which the folder's rm covers", async () => { + const { record } = exportRun("clerk", export2); + + await run({ ...noSource, input: record.id }); + + expect(captured.err).not.toMatch(/\n\s*rm (?!-rf)/); + }); + + // The copy is still named, but not as the export. + test("names the copy on its own when the export's record is gone", async () => { + const { record, file } = exportRun("clerk", export2); + const copy = path.join(workDir, "copy.json"); + fs.copyFileSync(file, copy); + stubClerk({ failing: new Set(["u2"]) }); + await run({ ...noSource, input: copy }); + stubClerk(); + process.exitCode = 0; + await run({ ...noSource, input: record.id }); + fs.rmSync(path.join(runsDir(), record.id, "run.json")); + const [imported] = listRuns(runsDir()); + captured.clear(); + + // The cleanup reads the export's record as it prints, so print it again. + expect(imported?.fromExport).toBe(record.id); + const { cleanupLines } = await import("./run.ts"); + const lines = cleanupLines(runsDir(), imported!).join("\n"); + expect(lines).toContain("The file this import read holds the same data:"); + expect(lines).toContain(`rm ${copy}`); + }); + test("imports by export run ID, with the source the envelope names", async () => { const { record } = exportRun("clerk", export2); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index d7c1460b0..87087d032 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -595,21 +595,33 @@ function formatSummary( * export carries user data and password hashes, and the run is only needed * for `undo`. */ -function cleanupLines(runsDir: string, record: RunRecord): string[] { +export function cleanupLines(runsDir: string, record: RunRecord): string[] { const lines: string[] = []; if (record.fromExport) { const exportDir = runDir(runsDir, record.fromExport); // `export --output` writes the file outside the run folder, and deleting - // the folder alone would leave the password hashes on disk. - const file = record.file?.path; - // On Windows a file on another drive comes back absolute, not as `..`. - const relative = file ? path.relative(exportDir, file) : ""; - const outside = file && (path.isAbsolute(relative) || relative.startsWith("..")); + // the folder alone would leave the password hashes on disk. The export + // run's own record says where: this run may have started from a copy, + // and kept the copy's path when continued by the export's ID. + const exportFile = readRun(runsDir, record.fromExport)?.file?.path; + const importFile = record.file?.path; + const outside = (file: string | undefined): file is string => { + if (!file) return false; + // On Windows a file on another drive comes back absolute, not as `..`. + const relative = path.relative(exportDir, file); + return path.isAbsolute(relative) || relative.startsWith(".."); + }; lines.push( `The export in run ${record.fromExport} holds your users' data. Once you have checked the import, delete it:`, dim(` rm -rf ${quoteArg(exportDir)}`), - ...(outside ? [dim(` rm ${quoteArg(file)}`)] : []), + ...(outside(exportFile) ? [dim(` rm ${quoteArg(exportFile)}`)] : []), ); + if (importFile !== exportFile && outside(importFile)) { + lines.push( + "The file this import read holds the same data:", + dim(` rm ${quoteArg(importFile)}`), + ); + } } lines.push( `Keep run ${record.id} while you might still undo it. After that:`, From aa1d65cdb996bbc21b085ed5e2c86d1fac6e1398 Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 16 Oct 2026 16:11:47 -0400 Subject: [PATCH 38/39] fix(migrate): link a continue by export run ID with nothing left to its export The no-users-left branch finished a continued run without continueWithExport, so a run started by path and continued by export run ID never recorded the export, and no cleanup line named it. It now links the run and prints the cleanup lines once the run is complete. The README's API table loses a duplicate GET /v1/instance row. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/README.md | 1 - .../cli-core/src/commands/migrate/run.test.ts | 25 +++++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 22 +++++++++++----- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md index 4f3f1b311..a8ffca9a0 100644 --- a/packages/cli-core/src/commands/migrate/README.md +++ b/packages/cli-core/src/commands/migrate/README.md @@ -950,7 +950,6 @@ stamping every user with today's. | `GET` | `/v1/users?external_id=…` | `migrate undo` — finds users whose create was in flight when the import stopped | | `GET` | `/v1/users?user_id=…` | `migrate undo` — reads the imported users back, 100 a call | | `DELETE` | `/v1/users/{user_id}` | `migrate undo` — deletes one user | -| `GET` | `/v1/instance` | `migrate import`, `undo` — names the instance behind the key | | `GET` | `/v1/instance` | `migrate import`, `undo`, `export clerk` — names the instance behind the key | | `GET` | `/v1/domains` | `migrate import` checks — resolves the Frontend API host | diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 1dd968ff6..5e34417ed 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -507,6 +507,31 @@ describe("run", () => { expect(captured.err).toContain(`The export in run ${record.id} holds your users' data`); }); + // Stopped after its last create, the run has nothing left, but the + // continue by export run ID still links it and names the export. + test("a continue by export run ID with nothing left links the run to the export", async () => { + const { record, file } = exportRun("clerk", export2); + await run({ ...noSource, input: file }); + const imported = listRuns(runsDir()).find((c) => c.kind === "import")!; + // A crash after the last create writes no finish time. + const crashed = readRun(runsDir(), imported.id)!; + delete crashed.finishedAt; + fs.writeFileSync( + path.join(runsDir(), imported.id, "run.json"), + JSON.stringify({ ...crashed, status: "running" }), + ); + captured.clear(); + + await run({ ...noSource, input: record.id }); + + expect(captured.err).toContain("No users left to import."); + expect(readRun(runsDir(), imported.id)).toMatchObject({ + status: "complete", + fromExport: record.id, + }); + expect(captured.err).toContain(`The export in run ${record.id} holds your users' data`); + }); + test("imports an envelope file with no source named", async () => { const { file } = exportRun("clerk", export2); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index 87087d032..c30cd2f68 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -1070,17 +1070,27 @@ export async function run(rawOptions: MigrateRunOptions): Promise { ) { // Settled, so a continued run is finished rather than left interrupted, // and the JSON reports it as finished, as run.json now does. + // Continued by export run ID, the run records the export it now reads. const finished = continued - ? continueRun( - runsDir, - withAttempt(continued, target, pickImportFlags(options)), - plannedLines, - carry, + ? continueWithExport( + continueRun( + runsDir, + withAttempt(continued, target, pickImportFlags(options)), + plannedLines, + carry, + ), + input.fromExport, ).finish() : undefined; if (options.json) preview({ nothingToImport: true, ...(finished ? { run: finished } : {}) }); - else log.warn("No users left to import."); + else { + log.warn("No users left to import."); + if (finished?.status === "complete") { + log.blank(); + for (const line of cleanupLines(runsDir, finished)) log.info(line); + } + } return; } From 9da0cfa631c055ff15b7d5bb6e7633520a718ecd Mon Sep 17 00:00:00 2001 From: Roy Anger Date: Fri, 16 Oct 2026 18:59:37 -0400 Subject: [PATCH 39/39] fix(migrate): link a complete run named by its export run ID to the export An import finished by path, then run again by export run ID, returned "Already imported" before recording the export, so the cleanup never named it. The complete branch now records fromExport on the run's own record and prints the cleanup lines. Co-Authored-By: Claude Opus 5.5 --- .../cli-core/src/commands/migrate/run.test.ts | 17 +++++++++++++++++ packages/cli-core/src/commands/migrate/run.ts | 16 ++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts index 5e34417ed..17af33d43 100644 --- a/packages/cli-core/src/commands/migrate/run.test.ts +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -532,6 +532,23 @@ describe("run", () => { expect(captured.err).toContain(`The export in run ${record.id} holds your users' data`); }); + // Finished by path, the run is complete: re-run by export run ID, it is + // already imported, but still links the export and names it for cleanup. + test("a complete run re-run by export run ID links the run to the export", async () => { + const { record, file } = exportRun("clerk", export2); + await run({ ...noSource, input: file }); + const imported = listRuns(runsDir()).find((c) => c.kind === "import")!; + expect(imported).toMatchObject({ status: "complete" }); + expect(imported.fromExport).toBeUndefined(); + captured.clear(); + + await run({ ...noSource, input: record.id }); + + expect(captured.err).toContain(`Already imported in run ${imported.id}`); + expect(readRun(runsDir(), imported.id)?.fromExport).toBe(record.id); + expect(captured.err).toContain(`The export in run ${record.id} holds your users' data`); + }); + test("imports an envelope file with no source named", async () => { const { file } = exportRun("clerk", export2); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts index c30cd2f68..6297d958e 100644 --- a/packages/cli-core/src/commands/migrate/run.ts +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -71,6 +71,7 @@ import { unfinishedUndoMessage, inUseMessage, openUndoOf, + patchRun, importsOfFile, undosOf, withAttempt, @@ -813,12 +814,19 @@ export async function run(rawOptions: MigrateRunOptions): Promise { : findResume(runsDir, match, carry); if (resume.kind === "complete") { + // Imported by path and named now by its export run ID: the record + // links the export, so the cleanup names the folder holding the data. + const record = + input.fromExport && resume.record.fromExport !== input.fromExport + ? (patchRun(runsDir, resume.record.id, { fromExport: input.fromExport }) ?? + resume.record) + : resume.record; if (options.json) { - log.data(JSON.stringify({ target, run: resume.record, alreadyImported: true }, null, 2)); + log.data(JSON.stringify({ target, run: record, alreadyImported: true }, null, 2)); } else { - log.success( - `Already imported in run ${resume.record.id}. Pass --new-run to import it again.`, - ); + log.success(`Already imported in run ${record.id}. Pass --new-run to import it again.`); + log.blank(); + for (const line of cleanupLines(runsDir, record)) log.info(line); } return; }