feat(utilities): Backup & Restore — commands per engine, with saved defaults

apps/e2e/package.json

@@ -13,6 +13,7 @@
     "test:headed": "HEADLESS=false vitest run --config vitest.config.ts",
     "test:smoke": "vitest run --config vitest.config.ts src/tests/smoke.test.ts",
     "test:git": "vitest run --config vitest.config.ts src/tests/git-migrations.test.ts",
+    "test:backup": "vitest run --config vitest.config.ts src/tests/backup-restore-ui.test.ts src/tests/backup-commands-dialects.test.ts",
     "test:smoke:headed": "HEADLESS=false vitest run --config vitest.config.ts src/tests/smoke.test.ts",
     "test:auth": "vitest run --config vitest.config.ts src/tests/auth-recovery.test.ts",
     "test:auth:headed": "HEADLESS=false vitest run --config vitest.config.ts src/tests/auth-recovery.test.ts",

apps/e2e/scripts/run-all.mjs

@@ -179,6 +179,24 @@ const ALWAYS = [
           // the teammate's run); the 300s default would kill it mid-run.
           timeoutMs: 720_000,
         },
+        // Backup & Restore: the panel against a saved Postgres connection
+        // (and SQL Server's SQL editor hand-off when configured).
+        {
+          key: 'backup-ui',
+          file: 'src/tests/backup-restore-ui.test.ts',
+          label: 'Backup & Restore',
+        },
+      ]
+    : []),
+  // The SQL-language backups (SQL Server, CockroachDB, ClickHouse, DuckDB),
+  // run for real against whichever of those engines are configured.
+  ...(configured.length
+    ? [
+        {
+          key: 'backup-commands',
+          file: 'src/tests/backup-commands-dialects.test.ts',
+          label: 'Backup commands',
+        },
       ]
     : []),
 ];

apps/e2e/src/generated/test-ids.ts

@@ -2610,6 +2610,60 @@ export const TestIds = {
     },
   },
   utilities: {
+    BackupRestorePanel: {
+      /** section · Backup */
+      backupCommand: 'backup-command',
+      /** button · Backup */
+      backupCommandCopy: 'backup-command-copy',
+      /** button · Open in SQL Editor */
+      backupCommandOpenSql: 'backup-command-open-sql',
+      /** pre · Backup */
+      backupCommandText: 'backup-command-text',
+      /** input */
+      backupCompress: 'backup-compress',
+      /** p */
+      backupError: 'backup-error',
+      /** input */
+      backupFileName: 'backup-file-name',
+      /** input */
+      backupFolder: 'backup-folder',
+      /** div */
+      backupFormat: 'backup-format',
+      /** button */
+      backupFormatForValue: (value: string | number) => `backup-format-${value}`,
+      /** input */
+      backupLimitSchema: 'backup-limit-schema',
+      /** div · Choose a connection */
+      backupNoConnection: 'backup-no-connection',
+      /** ul */
+      backupNotes: 'backup-notes',
+      /** div */
+      backupPanel: 'backup-panel',
+      /** button */
+      backupReset: 'backup-reset',
+      /** div */
+      backupRunsOn: 'backup-runs-on',
+      /** button · Save as my default for */
+      backupSaveDefault: 'backup-save-default',
+      /** span */
+      backupSaveStatus: 'backup-save-status',
+      /** div */
+      backupScope: 'backup-scope',
+      /** button */
+      backupScopeForValue: (value: string | number) => `backup-scope-${value}`,
+      /** textarea · orders, customers */
+      backupTables: 'backup-tables',
+      /** div · No backup commands */
+      backupUnsupported: 'backup-unsupported',
+      /** section · Restore */
+      restoreCommand: 'restore-command',
+      /** button · Restore */
+      restoreCommandCopy: 'restore-command-copy',
+      /** button · Open in SQL Editor */
+      restoreCommandOpenSql: 'restore-command-open-sql',
+      /** pre · Restore */
+      restoreCommandText: 'restore-command-text',
+    },
     CloneTableModal: {
       /** button · Apply clone */
       cloneApply: 'clone-apply',
@@ -2895,6 +2949,8 @@ export const TestIds = {
       indexManagementModal: 'index-management-modal',
       /** nav · Database utilities */
       sqlSidebarUtilities: 'sql-sidebar-utilities',
+      /** data · Backup & Restore */
+      utilitiesBackup: 'utilities-backup',
       /** data · Clone Table */
       utilitiesCloneTable: 'utilities-clone-table',
       /** select */
@@ -3392,6 +3448,28 @@ export type TestId =
   | `auth-sso-buttons-continue-${string}`
   | 'backend-offline-banner'
   | 'backend-offline-retry'
+  | 'backup-command'
+  | 'backup-command-copy'
+  | 'backup-command-open-sql'
+  | 'backup-command-text'
+  | 'backup-compress'
+  | 'backup-error'
+  | 'backup-file-name'
+  | 'backup-folder'
+  | 'backup-format'
+  | `backup-format-${string}`
+  | 'backup-limit-schema'
+  | 'backup-no-connection'
+  | 'backup-notes'
+  | 'backup-panel'
+  | 'backup-reset'
+  | 'backup-runs-on'
+  | 'backup-save-default'
+  | 'backup-save-status'
+  | 'backup-scope'
+  | `backup-scope-${string}`
+  | 'backup-tables'
+  | 'backup-unsupported'
   | 'blueprint-add-column'
   | 'blueprint-add-fk'
   | 'blueprint-add-index'
@@ -4105,6 +4183,10 @@ export type TestId =
   | 'report-table'
   | 'report-tiles'
   | 'report-who'
+  | 'restore-command'
+  | 'restore-command-copy'
+  | 'restore-command-open-sql'
+  | 'restore-command-text'
   | `rf-kinds-${string}`
   | `rf-object-${string}-${string}`
   | `rf-version-${string}`
@@ -4577,6 +4659,7 @@ export type TestId =
   | 'user-unsupported'
   | 'user-valid-until'
   | 'user-warnings'
+  | 'utilities-backup'
   | 'utilities-clone-table'
   | 'utilities-connection'
   | 'utilities-connection-pool'

apps/e2e/src/tests/backup-commands-dialects.test.ts

@@ -0,0 +1,127 @@
+/**
+ * Backup & Restore · the SQL-language engines, run for real.
+ *
+ * Fox Schema writes backup commands and never runs them, so nothing in the
+ * product finds out whether an engine accepts them. For the engines whose
+ * backup *is* SQL — SQL Server, CockroachDB, ClickHouse, DuckDB — this runs the
+ * generated statements through the product's own SQL runner, against the
+ * configured databases:
+ *
+ *   - SQL Server backs up into its data folder, then RESTORE VERIFYONLY reads
+ *     the file back. The generated restore itself would overwrite the test
+ *     database, so it is not run; VERIFYONLY proves the file and the RESTORE
+ *     grammar without that.
+ *   - CockroachDB backs up and restores under a new name, which is dropped.
+ *   - ClickHouse and DuckDB run when configured. A server without a backup
+ *     disk is an environment answer, reported as a skip, never as a pass.
+ *
+ * The shell-tool engines (pg_dump, mysqldump, Data Pump…) are checked by the
+ * unit tests and by hand; this suite holds only what the SQL runner can run.
+ */
+import fs from 'node:fs';
+import { afterAll, describe, expect, it } from 'vitest';
+import { buildBackupCommands, type BackupCommands } from '@foxschema/sql';
+import { getSourceConfig, hasConfig } from '../helpers/db-config.js';
+import { isEnvironmentFailure, runStatements, tryCleanup } from '../helpers/sql-exec.js';
+
+/** Answers that are about this server's setup, not the statement. */
+const NOT_ABOUT_THE_BACKUP = [
+  /not allowed for backups/i, //  ClickHouse: no <backups><allowed_disk>
+  /allowed_disk/i,
+  /Unknown disk/i,
+  /Operating system error/i, //  SQL Server: folder missing or not writable
+  /Cannot open backup device/i,
+  /external.?io|nodelocal/i, //  CockroachDB started without an external-io directory
+];
+const environmental = (message: string) => isEnvironmentFailure(message) || NOT_ABOUT_THE_BACKUP.some((re) => re.test(message));
+
+/** A folder each dev container already has. */
+const FOLDER: Record<string, string> = {
+  sqlserver: '/var/opt/mssql/data',
+  cockroachdb: 'nodelocal://1/fox-e2e-backups',
+  clickhouse: 'backups',
+  duckdb: '/tmp',
+};
+
+const NOW = new Date();
+const cleanups: Array<() => Promise<void>> = [];
+afterAll(async () => {
+  for (const cleanup of cleanups) await cleanup();
+});
+
+function commandsFor(dialect: string): BackupCommands {
+  const cfg = getSourceConfig(dialect)!;
+  const result = buildBackupCommands(
+    { dialect, host: cfg.host, port: cfg.port, database: cfg.database, schema: cfg.schema, username: cfg.username },
+    { folder: FOLDER[dialect], fileName: `fox_e2e_${NOW.getTime()}` },
+    NOW
+  );
+  if ('error' in result) throw new Error(result.error);
+  return result;
+}
+
+/** Statements in a generated script, one per `;`-terminated line group. */
+const statementsOf = (script: string) =>
+  script
+    .split(/;\s*(?:\n|$)/)
+    .map((s) => s.trim())
+    .filter(Boolean);
+
+/** Run statements in order; undefined when they all ran, the first real error otherwise, or a skip. */
+async function run(dialect: string, statements: string[]): Promise<{ rejected?: string; skipped?: string }> {
+  const results = await runStatements(dialect, statements);
+  for (let i = 0; i < statements.length; i++) {
+    const r = results[i];
+    if (r?.ok) continue;
+    const message = r?.error ?? 'no result';
+    if (environmental(message)) return { skipped: `${dialect}: ${message.split('\n')[0]!.slice(0, 200)}` };
+    return { rejected: `${dialect} rejected:\n  ${statements[i]}\n  ${message.split('\n')[0]!.slice(0, 300)}` };
+  }
+  return {};
+}
+
+describe.skipIf(!hasConfig('sqlserver'))('SQL Server backup', () => {
+  it('writes a .bak the server can read back', async () => {
+    const c = commandsFor('sqlserver');
+    const backup = await run('sqlserver', statementsOf(c.backup));
+    expect(backup.rejected).toBeUndefined();
+    if (backup.skipped) return console.warn(backup.skipped);
+    const verify = await run('sqlserver', [`RESTORE VERIFYONLY FROM DISK = N'${c.location.replace(/'/g, "''")}'`]);
+    expect(verify.rejected).toBeUndefined();
+    // The .bak stays in the container's data folder (SQL cannot delete it):
+    // docker exec foxschema-sqlserver sh -c 'rm -f /var/opt/mssql/data/fox_e2e_*.bak'
+  }, 120_000);
+});
+
+describe.skipIf(!hasConfig('cockroachdb'))('CockroachDB backup', () => {
+  it('backs up and restores under a new name', async () => {
+    const cfg = getSourceConfig('cockroachdb')!;
+    const c = commandsFor('cockroachdb');
+    cleanups.push(() => tryCleanup('cockroachdb', [`DROP DATABASE IF EXISTS "${cfg.database}_restored" CASCADE`]));
+    const backup = await run('cockroachdb', statementsOf(c.backup));
+    expect(backup.rejected).toBeUndefined();
+    if (backup.skipped) return console.warn(backup.skipped);
+    const restore = await run('cockroachdb', statementsOf(c.restore));
+    expect(restore.rejected).toBeUndefined();
+  }, 180_000);
+});
+
+describe.skipIf(!hasConfig('clickhouse'))('ClickHouse backup', () => {
+  it('is accepted by the server, or refused only for want of a backup disk', async () => {
+    const c = commandsFor('clickhouse');
+    const backup = await run('clickhouse', statementsOf(c.backup));
+    expect(backup.rejected).toBeUndefined();
+    if (backup.skipped) console.warn(backup.skipped);
+  }, 120_000);
+});
+
+describe.skipIf(!hasConfig('duckdb'))('DuckDB backup', () => {
+  it('exports the database to a folder', async () => {
+    const c = commandsFor('duckdb');
+    // DuckDB runs inside the API server, on this machine in a local run.
+    cleanups.push(async () => fs.rmSync(c.location, { recursive: true, force: true }));
+    const backup = await run('duckdb', statementsOf(c.backup));
+    expect(backup.rejected).toBeUndefined();
+    if (backup.skipped) console.warn(backup.skipped);
+  }, 120_000);
+});

apps/e2e/src/tests/backup-restore-ui.test.ts

@@ -0,0 +1,113 @@
+/**
+ * Backup & Restore in the Utilities workspace, in a real browser.
+ *
+ * The commands are unit-tested per engine; this checks the screen around
+ * them: it says where the command runs, writes it for the chosen connection
+ * without a password, keeps a saved folder as the user's default across a
+ * reload, and hands SQL Server's BACKUP DATABASE to the SQL editor.
+ */
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import type { Page } from 'playwright';
+import { buildDriver, quitDriver, clickWhen, waitFor } from '../helpers/driver.js';
+import { getSourceConfig, hasConfig } from '../helpers/db-config.js';
+import { saveScreenshot } from '../helpers/screenshot.js';
+import { deleteSavedConnections } from '../helpers/sql-exec.js';
+import { byTestId } from '../helpers/test-ids.js';
+import { AppPage } from '../pages/AppPage.js';
+import { SqlEditorPage } from '../pages/SqlEditorPage.js';
+
+const runId = Date.now().toString(36);
+const names: Record<string, string> = {};
+let driver: Page;
+let sql: SqlEditorPage;
+
+async function openBackupFor(dialect: string): Promise<void> {
+  await sql.openUtilitiesView();
+  await clickWhen(driver, byTestId('utilities-backup'));
+  // Saved connections load after the page does; after a reload the list can
+  // still be empty when the workspace first paints.
+  await driver.waitForFunction(
+    (name) =>
+      [...(document.querySelector('[data-testid="utilities-connection"]') as HTMLSelectElement | null)?.options ?? []].some((o) =>
+        (o.textContent ?? '').includes(name)
+      ),
+    names[dialect]!,
+    { timeout: 15_000 }
+  );
+  await sql.selectUtilityConnection(names[dialect]!, 'utilities-connection');
+  await waitFor(driver, byTestId('backup-command-text'), 15_000);
+}
+
+const commandText = () => driver.locator(byTestId('backup-command-text')).innerText();
+
+describe.skipIf(!hasConfig('postgres'))('Backup & Restore', () => {
+  beforeAll(async () => {
+    driver = await buildDriver();
+    sql = new SqlEditorPage(driver);
+    await new AppPage(driver).open();
+    for (const dialect of ['postgres', 'sqlserver']) {
+      if (!hasConfig(dialect)) continue;
+      names[dialect] = `E2E Backup ${dialect} ${runId}`;
+      await sql.addCredential(names[dialect]!, getSourceConfig(dialect)!);
+    }
+  }, 300_000);
+
+  afterAll(async () => {
+    await deleteSavedConnections(Object.values(names));
+    if (driver) await quitDriver(driver);
+  });
+
+  it('says where pg_dump runs and writes it for this connection, with no password', async () => {
+    const cfg = getSourceConfig('postgres')!;
+    await openBackupFor('postgres');
+    expect(await driver.locator(byTestId('backup-runs-on')).innerText()).toMatch(/Runs on your machine/);
+    const command = await commandText();
+    expect(command).toMatch(/^pg_dump /);
+    expect(command).toContain(`--username ${cfg.username}`);
+    expect(command).not.toContain(cfg.password || '\u0000never');
+    expect(await driver.locator(byTestId('restore-command-text')).innerText()).toMatch(/^pg_restore /);
+    await saveScreenshot(driver, 'backup-restore-postgres');
+  });
+
+  it('keeps a saved folder as the default for the engine, across a reload', async () => {
+    // Unique per run: a run that died before resetting leaves its folder saved,
+    // and typing the saved value again leaves nothing to save.
+    const folder = `/nas/e2e-${runId}`;
+    await openBackupFor('postgres');
+    await driver.locator(byTestId('backup-folder')).fill(folder);
+    await driver.locator(byTestId('backup-save-default')).click();
+    await driver.waitForFunction(
+      () => /^Saved/.test(document.querySelector('[data-testid="backup-save-status"]')?.textContent ?? ''),
+      undefined,
+      { timeout: 10_000 }
+    );
+
+    await driver.reload();
+    await openBackupFor('postgres');
+    await driver.waitForFunction(
+      () => /saved default/.test(document.querySelector('[data-testid="backup-save-status"]')?.textContent ?? ''),
+      undefined,
+      { timeout: 10_000 }
+    );
+    expect(await driver.locator(byTestId('backup-folder')).inputValue()).toBe(folder);
+    expect(await commandText()).toContain(`--file=${folder}/`);
+
+    // Leave the e2e account's default as the engine's own.
+    await driver.locator(byTestId('backup-folder')).fill('./backups');
+    await driver.locator(byTestId('backup-save-default')).click();
+    await driver.waitForFunction(
+      () => /^Saved/.test(document.querySelector('[data-testid="backup-save-status"]')?.textContent ?? ''),
+      undefined,
+      { timeout: 10_000 }
+    );
+  });
+
+  it.skipIf(!hasConfig('sqlserver'))('runs SQL Server’s backup on the server, and opens it in the SQL editor', async () => {
+    await openBackupFor('sqlserver');
+    expect(await driver.locator(byTestId('backup-runs-on')).innerText()).toMatch(/database server/);
+    expect(await commandText()).toMatch(/^BACKUP DATABASE \[/);
+    await saveScreenshot(driver, 'backup-restore-sqlserver');
+    await driver.locator(byTestId('backup-command-open-sql')).click();
+    await waitFor(driver, byTestId('sql-editor-view'), 15_000);
+  });
+});