Copy only the named schemas in data copy (#124)

changelog.d/124-data-copy-schema.md

@@ -0,0 +1,3 @@
+### Added
+
+- `cloudfall data copy` takes a repeatable `--source-schema NAME` (and the `data_copy` MCP tool a `source_schema` list): the dump and restore copy only the named schemas, the empty-target check and the per-table row-count check read exactly those schemas, and the plan without `--yes` names them; a name that is empty, longer than 63 bytes, or holds a control character or a brace is refused as a usage error. Without it the dump is the whole database as in 0.7.0, and the checks now cover every non-system schema of the source instead of `public` only, so a row-count mismatch outside `public` now fails the copy. The `DataMigrationReceipt` gains `spec.schemas` (sorted) and a `schema` on each `spec.tables[]` entry beside `table` and `rows` (#124)

docs/neon-migration-guide.md

@@ -174,8 +174,14 @@ The guided migration dumps from Neon on the target host, restores over
 the peer-authenticated socket as the declared owner, verifies per-table
 row counts match the source exactly, removes the staged URL file and the
 dump, and writes a `DataMigrationReceipt` with per-table counts. It
-refuses non-empty target databases, so a re-run requires deliberately
-dropping what was restored.
+refuses a target database that already has tables in the copied schemas,
+so a re-run requires deliberately dropping what was restored.
+
+By default the copy takes every non-system schema of the source, and the
+row-count check and the receipt cover all of them. To take only some,
+name each with a repeatable `--source-schema`, such as
+`--source-schema public`; the plan names the schemas it will copy, and the
+receipt lists them under `spec.schemas`, each table with its `schema`.
 
 The row-count check is the verify step and the receipt is the record's
 entry for the migration. The restore is not done because `pg_restore`

docs/render-migration-guide.md

@@ -147,11 +147,13 @@ uv run cloudfall migrate --build acme-api=main --data acme=~/.render/acme-db.url
 ```
 
 Without `--yes` the command shows the plan. The engine dumps on the target
-host, restores over the peer-authenticated socket, refuses a non-empty
-target database, verifies per-table row counts, and writes a
-`DataMigrationReceipt`. The row-count check is the verify step: the
-migration is not done because the restore exited zero, it is done because
-the counts match.
+host, restores over the peer-authenticated socket, refuses a target
+database that already has tables in the copied schemas, verifies per-table
+row counts, and writes a `DataMigrationReceipt`. The row-count check is the
+verify step: the migration is not done because the restore exited zero, it
+is done because the counts match. The copy takes every non-system schema of
+the source; `data copy --source-schema NAME` (repeatable) takes only the
+named ones.
 
 ## Step 7: cut over DNS
 

engine/ansible/playbooks/data.yml

@@ -14,6 +14,8 @@
         cloudfall_data_migration_database: "{{ cloudfall_data_database }}"
         cloudfall_data_migration_source_url_file: >-
           {{ cloudfall_data_source_url_file }}
+        cloudfall_data_migration_schemas: >-
+          {{ cloudfall_data_schemas | default([]) }}
         cloudfall_data_migration_receipt_directory: >-
           {{ cloudfall_data_receipt_directory | default("") }}
       loop: >-

engine/ansible/roles/cloudfall_data_migration/defaults/main.yml

@@ -2,14 +2,35 @@
 cloudfall_data_migration_receipt_directory: ""
 cloudfall_data_migration_url_path: /root/.cloudfall-data-source-url
 cloudfall_data_migration_dump_path: /var/tmp/cloudfall-data-migration.dump
-# Exact per-table row counts in one deterministic query, so source and
-# target evidence can be compared verbatim.
+# Schemas to copy; empty copies every non-system schema of the source,
+# which is what a pg_dump without --schema dumps.
+cloudfall_data_migration_schemas: []
+# Every non-system schema of the source, as one JSON array line: the
+# schemas a pg_dump without --schema dumps.
+cloudfall_data_migration_source_schemas_query: >-
+  SELECT coalesce(json_agg(nspname::text ORDER BY nspname), '[]')
+  FROM pg_catalog.pg_namespace
+  WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
+    AND nspname !~ '^pg_(toast_)?temp_'
+# Tables of the selected schemas on the target. The schema names arrive as
+# the psql variable cloudfall_data_schemas, a JSON array that psql quotes as
+# a literal, so no name is ever spliced into the SQL text.
+cloudfall_data_migration_target_tables_query: >-
+  SELECT count(*) FROM information_schema.tables
+  WHERE table_schema IN
+    (SELECT jsonb_array_elements_text(:'cloudfall_data_schemas'::jsonb))
+# Exact per-table row counts of the selected schemas in one deterministic
+# query, one schema,table,rows line per base table, so source and target
+# evidence can be compared verbatim. The schema names arrive as the psql
+# variable cloudfall_data_schemas, like the target tables query.
 cloudfall_data_migration_count_query: >-
-  SELECT table_name,
+  SELECT table_schema, table_name,
          (xpath('/row/c/text()',
                 query_to_xml(format('SELECT count(*) AS c FROM %I.%I',
                                     table_schema, table_name),
                              false, true, '')))[1]::text::bigint AS rows
   FROM information_schema.tables
-  WHERE table_schema = 'public' AND table_type = 'BASE TABLE'
-  ORDER BY table_name
+  WHERE table_schema IN
+    (SELECT jsonb_array_elements_text(:'cloudfall_data_schemas'::jsonb))
+    AND table_type = 'BASE TABLE'
+  ORDER BY table_schema, table_name

engine/ansible/roles/cloudfall_data_migration/meta/argument_specs.yml

@@ -17,6 +17,14 @@ argument_specs:
         description: >-
           Controller-side file whose only content is the source database
           connection URL; the URL itself never crosses a command line
+      cloudfall_data_migration_schemas:
+        type: list
+        elements: str
+        required: false
+        default: []
+        description: >-
+          Source schemas to copy; empty copies every non-system schema of the
+          source, as a pg_dump without --schema does
       cloudfall_data_migration_receipt_directory:
         type: str
         required: false
@@ -33,4 +41,19 @@ argument_specs:
       cloudfall_data_migration_count_query:
         type: str
         required: false
-        description: Deterministic per-table row-count query used on both sides
+        description: >-
+          Deterministic per-table row-count query used on both sides; it reads
+          the selected schemas from the psql variable cloudfall_data_schemas
+          and returns one schema,table,rows line per base table
+      cloudfall_data_migration_source_schemas_query:
+        type: str
+        required: false
+        description: >-
+          Query returning every non-system schema of the source as one JSON
+          array, used when no schema is named
+      cloudfall_data_migration_target_tables_query:
+        type: str
+        required: false
+        description: >-
+          Query counting the target's tables in the selected schemas, read
+          from the psql variable cloudfall_data_schemas

engine/ansible/roles/cloudfall_data_migration/tasks/main.yml

@@ -39,18 +39,50 @@
         mode: "0600"
       no_log: true
 
-    - name: Count existing tables in the target database
+    - name: Read every non-system schema of the source
+      ansible.builtin.shell:
+        cmd: >-
+          psql
+          --dbname "$(cat {{ cloudfall_data_migration_url_path | quote }})"
+          --no-psqlrc --tuples-only --no-align
+          --set ON_ERROR_STOP=1
+        stdin: "{{ cloudfall_data_migration_source_schemas_query }};"
+      register: cloudfall_data_migration_source_schemas
+      changed_when: false
+      when: cloudfall_data_migration_schemas | length == 0
+
+    - name: Select the schemas to copy
+      ansible.builtin.set_fact:
+        cloudfall_data_migration_selected_schemas: >-
+          {{
+            (
+              cloudfall_data_migration_schemas
+              if cloudfall_data_migration_schemas | length > 0
+              else cloudfall_data_migration_source_schemas.stdout | from_json
+            ) | unique | sort
+          }}
+
+    - name: Pass the selected schemas to psql as one JSON variable
+      ansible.builtin.set_fact:
+        cloudfall_data_migration_schemas_variable: >-
+          cloudfall_data_schemas={{
+            cloudfall_data_migration_selected_schemas | to_json
+          }}
+
+    - name: Count existing tables of the selected schemas in the target
       ansible.builtin.command:
         argv:
           - /usr/bin/psql
           - --dbname
           - "{{ cloudfall_data_migration_database }}"
+          - --no-psqlrc
           - --tuples-only
           - --no-align
-          - --command
-          - >-
-            SELECT count(*) FROM information_schema.tables
-            WHERE table_schema = 'public'
+          - --set
+          - ON_ERROR_STOP=1
+          - --set
+          - "{{ cloudfall_data_migration_schemas_variable }}"
+        stdin: "{{ cloudfall_data_migration_target_tables_query }};"
       become: true
       become_user: postgres
       register: cloudfall_data_migration_target_tables
@@ -71,7 +103,11 @@
           pg_dump
           --dbname "$(cat {{ cloudfall_data_migration_url_path | quote }})"
           --no-owner --no-privileges --format custom
-          --file {{ cloudfall_data_migration_dump_path | quote }}
+          --file {{ cloudfall_data_migration_dump_path | quote }}{%
+            for schema in cloudfall_data_migration_schemas | unique | sort
+          %} --schema {{
+            ('"' ~ (schema | replace('"', '""')) ~ '"') | quote
+          }}{% endfor %}
       register: cloudfall_data_migration_dump
       retries: 3
       delay: 10
@@ -90,8 +126,10 @@
         cmd: >-
           psql
           --dbname "$(cat {{ cloudfall_data_migration_url_path | quote }})"
-          --csv --tuples-only
-          --command {{ cloudfall_data_migration_count_query | quote }}
+          --no-psqlrc --csv --tuples-only
+          --set ON_ERROR_STOP=1
+          --set {{ cloudfall_data_migration_schemas_variable | quote }}
+        stdin: "{{ cloudfall_data_migration_count_query }};"
       register: cloudfall_data_migration_source_counts
       changed_when: false
 
@@ -100,8 +138,9 @@
         that:
           - cloudfall_data_migration_source_counts.stdout_lines | length > 0
         fail_msg: >-
-          The source database contains no tables in schema public; there is
-          nothing to migrate
+          The source database contains no tables in schemas
+          {{ cloudfall_data_migration_selected_schemas | join(', ') }}; there
+          is nothing to migrate
 
     - name: Restore the dump into the declared database
       ansible.builtin.command:
@@ -124,10 +163,14 @@
           - /usr/bin/psql
           - --dbname
           - "{{ cloudfall_data_migration_database }}"
+          - --no-psqlrc
           - --csv
           - --tuples-only
-          - --command
-          - "{{ cloudfall_data_migration_count_query }}"
+          - --set
+          - ON_ERROR_STOP=1
+          - --set
+          - "{{ cloudfall_data_migration_schemas_variable }}"
+        stdin: "{{ cloudfall_data_migration_count_query }};"
       become: true
       become_user: postgres
       register: cloudfall_data_migration_restored_counts
@@ -144,6 +187,31 @@
           {{ cloudfall_data_migration_source_counts.stdout_lines }}
           versus restored
           {{ cloudfall_data_migration_restored_counts.stdout_lines }}
+
+    - name: Record the verified tables for the receipt
+      ansible.builtin.set_fact:
+        # One psql CSV line per table: schema,table,rows, where a name that
+        # holds a comma or a double quote arrives double-quoted.
+        cloudfall_data_migration_tables: >-
+          {%- set result = namespace(tables=[]) -%}
+          {%- for line in cloudfall_data_migration_source_counts.stdout_lines -%}
+          {%- set fields = (line | regex_findall(
+                '^("(?:[^"]|"")*"|[^,"]*),("(?:[^"]|"")*"|[^,"]*),([0-9]+)$'
+              ))[0] -%}
+          {%- set names = namespace(parts=[]) -%}
+          {%- for field in fields[:2] -%}
+          {%- set names.parts = names.parts + [
+                field[1:-1] | replace('""', '"')
+                if field.startswith('"') else field
+              ] -%}
+          {%- endfor -%}
+          {%- set result.tables = result.tables + [{
+                'schema': names.parts[0],
+                'table': names.parts[1],
+                'rows': fields[2] | int,
+              }] -%}
+          {%- endfor -%}
+          {{ result.tables }}
   always:
     - name: Remove the staged source connection URL
       ansible.builtin.file:
@@ -183,10 +251,8 @@
             'owner': cloudfall_data_migration_owner,
             'server': cloudfall_server_id,
             'migratedAt': ansible_facts['date_time']['iso8601'],
-            'tables': cloudfall_data_migration_source_counts.stdout_lines
-              | map('regex_replace', '^(.*),([0-9]+)$',
-                    '{"table": "\1", "rows": \2}')
-              | map('from_json') | list,
+            'schemas': cloudfall_data_migration_selected_schemas,
+            'tables': cloudfall_data_migration_tables,
             'verification': 'per-table row counts matched',
           },
         } | to_nice_json