From 0e695318f1a07f3b97cfd1e6e5cc8196743dc698 Mon Sep 17 00:00:00 2001 From: "shipmill-romamo[bot]" <338557544+shipmill-romamo[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:17:06 +0200 Subject: [PATCH 1/4] Copy only the named schemas in data copy cloudfall data copy dumped every schema but its empty-target and row-count checks read only public, so the receipt vouched for one schema of many. A repeatable --source-schema now selects what pg_dump takes, and both checks read exactly the selected schemas; without it the dump command is unchanged and the checks cover every non-system schema of the source. The receipt records spec.schemas and a schema per table. Schema names reach SQL only as one psql variable that psql quotes as a literal, and reach pg_dump as shell-quoted, double-quoted patterns. The spec names the flag --schema; treaty reserves that name for printing a command's schema, so the flag is --source-schema. Fixes #124 Co-Authored-By: Claude Opus 5.5 --- changelog.d/124-data-copy-schema.md | 3 + docs/neon-migration-guide.md | 10 +- docs/render-migration-guide.md | 12 +- engine/ansible/playbooks/data.yml | 2 + .../defaults/main.yml | 31 +- .../meta/argument_specs.yml | 25 +- .../cloudfall_data_migration/tasks/main.yml | 98 +++++- engine/tests/test_data_migration_role.py | 298 ++++++++++++++++++ sdk/src/cloudfall/app.py | 23 +- sdk/src/cloudfall/lifecycle.py | 51 ++- sdk/tests/test_data_migration.py | 53 ++++ sdk/tests/test_lifecycle.py | 33 ++ 12 files changed, 601 insertions(+), 38 deletions(-) create mode 100644 changelog.d/124-data-copy-schema.md create mode 100644 engine/tests/test_data_migration_role.py diff --git a/changelog.d/124-data-copy-schema.md b/changelog.d/124-data-copy-schema.md new file mode 100644 index 0000000..f951f55 --- /dev/null +++ b/changelog.d/124-data-copy-schema.md @@ -0,0 +1,3 @@ +### Added + +- `cloudfall data copy` takes a repeatable `--source-schema NAME` (and the `data_copy` MCP tool a `source_schema` list): the dump and restore copy only the named schemas, the empty-target check and the per-table row-count check read exactly those schemas, and the plan without `--yes` names them. Without it the dump is the whole database as in 0.7.0, and the checks now cover every non-system schema of the source instead of `public` only, so a row-count mismatch outside `public` now fails the copy. The `DataMigrationReceipt` gains `spec.schemas` (sorted) and a `schema` on each `spec.tables[]` entry beside `table` and `rows` (#124) diff --git a/docs/neon-migration-guide.md b/docs/neon-migration-guide.md index bc0ee81..dc94f24 100644 --- a/docs/neon-migration-guide.md +++ b/docs/neon-migration-guide.md @@ -174,8 +174,14 @@ The guided migration dumps from Neon on the target host, restores over the peer-authenticated socket as the declared owner, verifies per-table row counts match the source exactly, removes the staged URL file and the dump, and writes a `DataMigrationReceipt` with per-table counts. It -refuses non-empty target databases, so a re-run requires deliberately -dropping what was restored. +refuses a target database that already has tables in the copied schemas, +so a re-run requires deliberately dropping what was restored. + +By default the copy takes every non-system schema of the source, and the +row-count check and the receipt cover all of them. To take only some, +name each with a repeatable `--source-schema`, such as +`--source-schema public`; the plan names the schemas it will copy, and the +receipt lists them under `spec.schemas`, each table with its `schema`. The row-count check is the verify step and the receipt is the record's entry for the migration. The restore is not done because `pg_restore` diff --git a/docs/render-migration-guide.md b/docs/render-migration-guide.md index ff2edb2..77e7f67 100644 --- a/docs/render-migration-guide.md +++ b/docs/render-migration-guide.md @@ -147,11 +147,13 @@ uv run cloudfall migrate --build acme-api=main --data acme=~/.render/acme-db.url ``` Without `--yes` the command shows the plan. The engine dumps on the target -host, restores over the peer-authenticated socket, refuses a non-empty -target database, verifies per-table row counts, and writes a -`DataMigrationReceipt`. The row-count check is the verify step: the -migration is not done because the restore exited zero, it is done because -the counts match. +host, restores over the peer-authenticated socket, refuses a target +database that already has tables in the copied schemas, verifies per-table +row counts, and writes a `DataMigrationReceipt`. The row-count check is the +verify step: the migration is not done because the restore exited zero, it +is done because the counts match. The copy takes every non-system schema of +the source; `data copy --source-schema NAME` (repeatable) takes only the +named ones. ## Step 7: cut over DNS diff --git a/engine/ansible/playbooks/data.yml b/engine/ansible/playbooks/data.yml index 020ab50..b95ed87 100644 --- a/engine/ansible/playbooks/data.yml +++ b/engine/ansible/playbooks/data.yml @@ -14,6 +14,8 @@ cloudfall_data_migration_database: "{{ cloudfall_data_database }}" cloudfall_data_migration_source_url_file: >- {{ cloudfall_data_source_url_file }} + cloudfall_data_migration_schemas: >- + {{ cloudfall_data_schemas | default([]) }} cloudfall_data_migration_receipt_directory: >- {{ cloudfall_data_receipt_directory | default("") }} loop: >- diff --git a/engine/ansible/roles/cloudfall_data_migration/defaults/main.yml b/engine/ansible/roles/cloudfall_data_migration/defaults/main.yml index ea8f2b3..8ba838b 100644 --- a/engine/ansible/roles/cloudfall_data_migration/defaults/main.yml +++ b/engine/ansible/roles/cloudfall_data_migration/defaults/main.yml @@ -2,14 +2,35 @@ cloudfall_data_migration_receipt_directory: "" cloudfall_data_migration_url_path: /root/.cloudfall-data-source-url cloudfall_data_migration_dump_path: /var/tmp/cloudfall-data-migration.dump -# Exact per-table row counts in one deterministic query, so source and -# target evidence can be compared verbatim. +# Schemas to copy; empty copies every non-system schema of the source, +# which is what a pg_dump without --schema dumps. +cloudfall_data_migration_schemas: [] +# Every non-system schema of the source, as one JSON array line: the +# schemas a pg_dump without --schema dumps. +cloudfall_data_migration_source_schemas_query: >- + SELECT coalesce(json_agg(nspname::text ORDER BY nspname), '[]') + FROM pg_catalog.pg_namespace + WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast') + AND nspname !~ '^pg_(toast_)?temp_' +# Tables of the selected schemas on the target. The schema names arrive as +# the psql variable cloudfall_data_schemas, a JSON array that psql quotes as +# a literal, so no name is ever spliced into the SQL text. +cloudfall_data_migration_target_tables_query: >- + SELECT count(*) FROM information_schema.tables + WHERE table_schema IN + (SELECT jsonb_array_elements_text(:'cloudfall_data_schemas'::jsonb)) +# Exact per-table row counts of the selected schemas in one deterministic +# query, one schema,table,rows line per base table, so source and target +# evidence can be compared verbatim. The schema names arrive as the psql +# variable cloudfall_data_schemas, like the target tables query. cloudfall_data_migration_count_query: >- - SELECT table_name, + SELECT table_schema, table_name, (xpath('/row/c/text()', query_to_xml(format('SELECT count(*) AS c FROM %I.%I', table_schema, table_name), false, true, '')))[1]::text::bigint AS rows FROM information_schema.tables - WHERE table_schema = 'public' AND table_type = 'BASE TABLE' - ORDER BY table_name + WHERE table_schema IN + (SELECT jsonb_array_elements_text(:'cloudfall_data_schemas'::jsonb)) + AND table_type = 'BASE TABLE' + ORDER BY table_schema, table_name diff --git a/engine/ansible/roles/cloudfall_data_migration/meta/argument_specs.yml b/engine/ansible/roles/cloudfall_data_migration/meta/argument_specs.yml index 775f8a0..a55b2b1 100644 --- a/engine/ansible/roles/cloudfall_data_migration/meta/argument_specs.yml +++ b/engine/ansible/roles/cloudfall_data_migration/meta/argument_specs.yml @@ -17,6 +17,14 @@ argument_specs: description: >- Controller-side file whose only content is the source database connection URL; the URL itself never crosses a command line + cloudfall_data_migration_schemas: + type: list + elements: str + required: false + default: [] + description: >- + Source schemas to copy; empty copies every non-system schema of the + source, as a pg_dump without --schema does cloudfall_data_migration_receipt_directory: type: str required: false @@ -33,4 +41,19 @@ argument_specs: cloudfall_data_migration_count_query: type: str required: false - description: Deterministic per-table row-count query used on both sides + description: >- + Deterministic per-table row-count query used on both sides; it reads + the selected schemas from the psql variable cloudfall_data_schemas + and returns one schema,table,rows line per base table + cloudfall_data_migration_source_schemas_query: + type: str + required: false + description: >- + Query returning every non-system schema of the source as one JSON + array, used when no schema is named + cloudfall_data_migration_target_tables_query: + type: str + required: false + description: >- + Query counting the target's tables in the selected schemas, read + from the psql variable cloudfall_data_schemas diff --git a/engine/ansible/roles/cloudfall_data_migration/tasks/main.yml b/engine/ansible/roles/cloudfall_data_migration/tasks/main.yml index ed0a1ad..6a019c7 100644 --- a/engine/ansible/roles/cloudfall_data_migration/tasks/main.yml +++ b/engine/ansible/roles/cloudfall_data_migration/tasks/main.yml @@ -39,18 +39,50 @@ mode: "0600" no_log: true - - name: Count existing tables in the target database + - name: Read every non-system schema of the source + ansible.builtin.shell: + cmd: >- + psql + --dbname "$(cat {{ cloudfall_data_migration_url_path | quote }})" + --no-psqlrc --tuples-only --no-align + --set ON_ERROR_STOP=1 + stdin: "{{ cloudfall_data_migration_source_schemas_query }};" + register: cloudfall_data_migration_source_schemas + changed_when: false + when: cloudfall_data_migration_schemas | length == 0 + + - name: Select the schemas to copy + ansible.builtin.set_fact: + cloudfall_data_migration_selected_schemas: >- + {{ + ( + cloudfall_data_migration_schemas + if cloudfall_data_migration_schemas | length > 0 + else cloudfall_data_migration_source_schemas.stdout | from_json + ) | unique | sort + }} + + - name: Pass the selected schemas to psql as one JSON variable + ansible.builtin.set_fact: + cloudfall_data_migration_schemas_variable: >- + cloudfall_data_schemas={{ + cloudfall_data_migration_selected_schemas | to_json + }} + + - name: Count existing tables of the selected schemas in the target ansible.builtin.command: argv: - /usr/bin/psql - --dbname - "{{ cloudfall_data_migration_database }}" + - --no-psqlrc - --tuples-only - --no-align - - --command - - >- - SELECT count(*) FROM information_schema.tables - WHERE table_schema = 'public' + - --set + - ON_ERROR_STOP=1 + - --set + - "{{ cloudfall_data_migration_schemas_variable }}" + stdin: "{{ cloudfall_data_migration_target_tables_query }};" become: true become_user: postgres register: cloudfall_data_migration_target_tables @@ -71,7 +103,11 @@ pg_dump --dbname "$(cat {{ cloudfall_data_migration_url_path | quote }})" --no-owner --no-privileges --format custom - --file {{ cloudfall_data_migration_dump_path | quote }} + --file {{ cloudfall_data_migration_dump_path | quote }}{% + for schema in cloudfall_data_migration_schemas | unique | sort + %} --schema {{ + ('"' ~ (schema | replace('"', '""')) ~ '"') | quote + }}{% endfor %} register: cloudfall_data_migration_dump retries: 3 delay: 10 @@ -90,8 +126,10 @@ cmd: >- psql --dbname "$(cat {{ cloudfall_data_migration_url_path | quote }})" - --csv --tuples-only - --command {{ cloudfall_data_migration_count_query | quote }} + --no-psqlrc --csv --tuples-only + --set ON_ERROR_STOP=1 + --set {{ cloudfall_data_migration_schemas_variable | quote }} + stdin: "{{ cloudfall_data_migration_count_query }};" register: cloudfall_data_migration_source_counts changed_when: false @@ -100,8 +138,9 @@ that: - cloudfall_data_migration_source_counts.stdout_lines | length > 0 fail_msg: >- - The source database contains no tables in schema public; there is - nothing to migrate + The source database contains no tables in schemas + {{ cloudfall_data_migration_selected_schemas | join(', ') }}; there + is nothing to migrate - name: Restore the dump into the declared database ansible.builtin.command: @@ -124,10 +163,14 @@ - /usr/bin/psql - --dbname - "{{ cloudfall_data_migration_database }}" + - --no-psqlrc - --csv - --tuples-only - - --command - - "{{ cloudfall_data_migration_count_query }}" + - --set + - ON_ERROR_STOP=1 + - --set + - "{{ cloudfall_data_migration_schemas_variable }}" + stdin: "{{ cloudfall_data_migration_count_query }};" become: true become_user: postgres register: cloudfall_data_migration_restored_counts @@ -144,6 +187,31 @@ {{ cloudfall_data_migration_source_counts.stdout_lines }} versus restored {{ cloudfall_data_migration_restored_counts.stdout_lines }} + + - name: Record the verified tables for the receipt + ansible.builtin.set_fact: + # One psql CSV line per table: schema,table,rows, where a name that + # holds a comma or a double quote arrives double-quoted. + cloudfall_data_migration_tables: >- + {%- set result = namespace(tables=[]) -%} + {%- for line in cloudfall_data_migration_source_counts.stdout_lines -%} + {%- set fields = (line | regex_findall( + '^("(?:[^"]|"")*"|[^,"]*),("(?:[^"]|"")*"|[^,"]*),([0-9]+)$' + ))[0] -%} + {%- set names = namespace(parts=[]) -%} + {%- for field in fields[:2] -%} + {%- set names.parts = names.parts + [ + field[1:-1] | replace('""', '"') + if field.startswith('"') else field + ] -%} + {%- endfor -%} + {%- set result.tables = result.tables + [{ + 'schema': names.parts[0], + 'table': names.parts[1], + 'rows': fields[2] | int, + }] -%} + {%- endfor -%} + {{ result.tables }} always: - name: Remove the staged source connection URL ansible.builtin.file: @@ -183,10 +251,8 @@ 'owner': cloudfall_data_migration_owner, 'server': cloudfall_server_id, 'migratedAt': ansible_facts['date_time']['iso8601'], - 'tables': cloudfall_data_migration_source_counts.stdout_lines - | map('regex_replace', '^(.*),([0-9]+)$', - '{"table": "\1", "rows": \2}') - | map('from_json') | list, + 'schemas': cloudfall_data_migration_selected_schemas, + 'tables': cloudfall_data_migration_tables, 'verification': 'per-table row counts matched', }, } | to_nice_json diff --git a/engine/tests/test_data_migration_role.py b/engine/tests/test_data_migration_role.py new file mode 100644 index 0000000..33ded94 --- /dev/null +++ b/engine/tests/test_data_migration_role.py @@ -0,0 +1,298 @@ +"""The data-migration role's schema selection, rendered with Ansible's templating. + +The role needs a live source and a target PostgreSQL, so these tests replay +what it renders: the dump command, the psql variable that carries the +selected schemas into each query, the empty-target assertion, and the +receipt built from the row-count lines psql returns. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar, trust_as_template + +ROOT = Path(__file__).parents[2] +ROLE = ROOT / "engine" / "ansible" / "roles" / "cloudfall_data_migration" +PLAYBOOK = ROOT / "engine" / "ansible" / "playbooks" / "data.yml" +URL_PATH = "/root/.cloudfall-data-source-url" +DUMP_PATH = "/var/tmp/cloudfall-data-migration.dump" # noqa: S108 - role default. +# The dump command 0.7.0 ran, which a copy without --schema keeps. +WHOLE_DATABASE_DUMP = ( + f'pg_dump --dbname "$(cat {URL_PATH})" ' + "--no-owner --no-privileges --format custom " + f"--file {DUMP_PATH}" +) + + +def _defaults() -> dict[str, object]: + defaults = DataLoader().load_from_file(str(ROLE / "defaults" / "main.yml")) + assert isinstance(defaults, dict) + return defaults + + +def _tasks() -> list[dict[str, object]]: + tasks = DataLoader().load_from_file(str(ROLE / "tasks" / "main.yml")) + assert isinstance(tasks, list) + flat: list[dict[str, object]] = [] + for task in tasks: + flat.append(task) + for section in ("block", "always"): + nested = task.get(section, []) + assert isinstance(nested, list) + flat.extend(nested) + return flat + + +def _task(name: str) -> dict[str, object]: + return next(task for task in _tasks() if task.get("name") == name) + + +def _module(name: str, module: str) -> dict[str, object]: + arguments = _task(name)[module] + assert isinstance(arguments, dict) + return arguments + + +def _trusted(value: object) -> object: + if isinstance(value, str): + return trust_as_template(value) + if isinstance(value, list): + return [_trusted(item) for item in value] + return value + + +def _render(value: object, variables: dict[str, object]) -> object: + templar = Templar(loader=DataLoader(), variables={**_defaults(), **variables}) + return templar.template(_trusted(value)) + + +def _holds(expression: object, variables: dict[str, object]) -> bool: + templar = Templar(loader=DataLoader(), variables={**_defaults(), **variables}) + return bool(templar.evaluate_conditional(trust_as_template(str(expression)))) + + +def _selected(named: list[str], source_schemas: list[str]) -> list[str]: + """Replay how the role picks the schemas from --schema or the source.""" + fact = _module("Select the schemas to copy", "ansible.builtin.set_fact") + rendered = _render( + fact["cloudfall_data_migration_selected_schemas"], + { + "cloudfall_data_migration_schemas": named, + # A named selection skips the source read, as the role does. + "cloudfall_data_migration_source_schemas": ( + {"skipped": True} + if named + else {"stdout": json.dumps(source_schemas)} + ), + }, + ) + assert isinstance(rendered, list) + return [str(schema) for schema in rendered] + + +def _schemas_variable(selected: list[str]) -> str: + fact = _module( + "Pass the selected schemas to psql as one JSON variable", + "ansible.builtin.set_fact", + ) + return str( + _render( + fact["cloudfall_data_migration_schemas_variable"], + {"cloudfall_data_migration_selected_schemas": selected}, + ) + ) + + +def _dump_command(named: list[str]) -> str: + shell = _module( + "Dump the source database on the target host", "ansible.builtin.shell" + ) + return str(_render(shell["cmd"], {"cloudfall_data_migration_schemas": named})) + + +def _psql_substitute(sql: str, variable: str) -> str: + """Do what psql does to ``:'cloudfall_data_schemas'``: quote it as a literal.""" + name, value = variable.split("=", 1) + literal = "'" + value.replace("'", "''") + "'" + return sql.replace(f":'{name}'", literal) + + +SOURCE_SCHEMAS = ["app", "audit", "public"] +SOURCE_LINES = [ + "app,accounts,3", + "app,orders,12", + "public,notes,5", + 'public,"odd, ""name""",1', +] + + +def _receipt(selected: list[str], lines: list[str]) -> dict[str, object]: + tables_fact = _module( + "Record the verified tables for the receipt", "ansible.builtin.set_fact" + ) + counts = {"stdout_lines": lines} + tables = _render( + tables_fact["cloudfall_data_migration_tables"], + {"cloudfall_data_migration_source_counts": counts}, + ) + copy = _module( + "Write the data-migration receipt on the controller", "ansible.builtin.copy" + ) + content = _render( + copy["content"], + { + "cloudfall_data_migration_service": {"id": "postgresql-main"}, + "cloudfall_data_migration_database": "crm", + "cloudfall_data_migration_owner": "crm", + "cloudfall_server_id": "h1", + "ansible_facts": {"date_time": {"iso8601": "2026-10-27T00:00:00Z"}}, + "cloudfall_data_migration_selected_schemas": selected, + "cloudfall_data_migration_tables": tables, + }, + ) + receipt = json.loads(str(content)) + assert isinstance(receipt, dict) + return receipt + + +def test_s007_1_named_schemas_reach_pg_dump_and_the_receipt() -> None: + selected = _selected(["public", "app"], SOURCE_SCHEMAS) + command = _dump_command(["public", "app"]) + receipt = _receipt(selected, SOURCE_LINES[:3]) + + assert selected == ["app", "public"] + assert command == f"{WHOLE_DATABASE_DUMP} --schema '\"app\"' --schema '\"public\"'" + spec = receipt["spec"] + assert isinstance(spec, dict) + assert spec["schemas"] == ["app", "public"] + + +def test_s007_1_a_schema_name_stays_one_quoted_pg_dump_argument() -> None: + """A hostile name is one shell word and one literal pg_dump pattern.""" + command = _dump_command(['Bad;Name"$(touch x)']) + + assert command == ( + f"{WHOLE_DATABASE_DUMP} --schema '\"Bad;Name\"\"$(touch x)\"'" + ) + + +def test_s007_2_without_schema_the_dump_is_the_whole_database_as_in_0_7_0() -> None: + selected = _selected([], SOURCE_SCHEMAS) + receipt = _receipt(selected, SOURCE_LINES) + + assert _dump_command([]) == WHOLE_DATABASE_DUMP + assert selected == ["app", "audit", "public"] + spec = receipt["spec"] + assert isinstance(spec, dict) + assert spec["schemas"] == ["app", "audit", "public"] + + +def test_s007_2_source_schemas_query_skips_only_the_system_schemas() -> None: + query = str(_defaults()["cloudfall_data_migration_source_schemas_query"]) + read = _task("Read every non-system schema of the source") + + assert "'pg_catalog', 'information_schema', 'pg_toast'" in query + assert "'^pg_(toast_)?temp_'" in query + assert "ORDER BY nspname" in query + assert _holds(read["when"], {"cloudfall_data_migration_schemas": []}) + assert not _holds(read["when"], {"cloudfall_data_migration_schemas": ["app"]}) + + +def test_s007_3_receipt_tables_carry_schema_table_and_rows() -> None: + receipt = _receipt(["app", "public"], SOURCE_LINES) + + spec = receipt["spec"] + assert isinstance(spec, dict) + assert spec["tables"] == [ + {"schema": "app", "table": "accounts", "rows": 3}, + {"schema": "app", "table": "orders", "rows": 12}, + {"schema": "public", "table": "notes", "rows": 5}, + {"schema": "public", "table": 'odd, "name"', "rows": 1}, + ] + assert spec["verification"] == "per-table row counts matched" + + +def test_s007_4_row_count_query_reads_only_the_selected_schemas() -> None: + variable = _schemas_variable(["app", "public"]) + query = _psql_substitute( + str(_defaults()["cloudfall_data_migration_count_query"]), variable + ) + + assert variable == 'cloudfall_data_schemas=["app", "public"]' + assert "SELECT table_schema, table_name," in query + assert ( + "WHERE table_schema IN\n" + " (SELECT jsonb_array_elements_text('[\"app\", \"public\"]'::jsonb))\n" + " AND table_type = 'BASE TABLE'" + ) in query + assert query.rstrip().endswith("ORDER BY table_schema, table_name") + assert "'public'" not in query.replace('"public"', "") + + +def test_s007_4_both_sides_pass_the_schemas_to_the_row_count_query() -> None: + variables: dict[str, object] = { + "cloudfall_data_migration_schemas_variable": _schemas_variable( + ["app", "public"] + ), + "cloudfall_data_migration_database": "crm", + } + source = _module("Read source table row counts", "ansible.builtin.shell") + restored = _module("Read restored table row counts", "ansible.builtin.command") + query = str(_defaults()["cloudfall_data_migration_count_query"]) + + source_cmd = str(_render(source["cmd"], variables)) + restored_argv = _render(restored["argv"], variables) + assert isinstance(restored_argv, list) + assert "--set ON_ERROR_STOP=1" in source_cmd + assert "--set 'cloudfall_data_schemas=[\"app\", \"public\"]'" in source_cmd + assert restored_argv[-2:] == ["--set", 'cloudfall_data_schemas=["app", "public"]'] + assert _render(source["stdin"], variables) == f"{query};" + assert _render(restored["stdin"], variables) == f"{query};" + + +def test_s007_5_empty_target_check_counts_only_the_selected_schemas() -> None: + count = _module( + "Count existing tables of the selected schemas in the target", + "ansible.builtin.command", + ) + refuse = _module( + "Refuse to restore into a non-empty database", "ansible.builtin.assert" + ) + variable = _schemas_variable(["public"]) + variables: dict[str, object] = { + "cloudfall_data_migration_schemas_variable": variable, + "cloudfall_data_migration_database": "crm", + } + query = _psql_substitute( + str(_defaults()["cloudfall_data_migration_target_tables_query"]), variable + ) + argv = _render(count["argv"], variables) + conditions = refuse["that"] + assert isinstance(conditions, list) + (condition,) = conditions + + assert isinstance(argv, list) + assert argv[-1] == 'cloudfall_data_schemas=["public"]' + assert "jsonb_array_elements_text('[\"public\"]'::jsonb)" in query + assert "'public'" not in query + target = "cloudfall_data_migration_target_tables" + assert _holds(condition, {target: {"stdout": "0\n"}}) + assert not _holds(condition, {target: {"stdout": "3"}}) + message = _render( + refuse["fail_msg"], + {target: {"stdout": "3"}, "cloudfall_data_migration_database": "crm"}, + ) + assert "already contains 3 tables" in str(message) + + +def test_the_data_playbook_passes_the_named_schemas_to_the_role() -> None: + plays = DataLoader().load_from_file(str(PLAYBOOK)) + assert isinstance(plays, list) + include = plays[0]["tasks"][0] + value = include["vars"]["cloudfall_data_migration_schemas"] + + assert _render(value, {"cloudfall_data_schemas": ["app"]}) == ["app"] + assert _render(value, {}) == [] diff --git a/sdk/src/cloudfall/app.py b/sdk/src/cloudfall/app.py index 08adc78..2aabc4a 100644 --- a/sdk/src/cloudfall/app.py +++ b/sdk/src/cloudfall/app.py @@ -165,6 +165,7 @@ ) from cloudfall.lifecycle import ( ERROR_EXECUTION_FAILED, + DataCopySchemas, DeployOptions, DomainApply, DomainObserve, @@ -3835,6 +3836,16 @@ class DataMigrateArgs(EngineArgs): # The flag names a file; the URL with its password never reaches argv. secret=False, ) + # S-007 names this flag --schema, which treaty reserves for printing a + # command's schema (REQ-F-079); --source-schema pairs with + # --source-url-file instead. + source_schema: tuple[str, ...] = Flag( + default=(), + description=( + "Source schema to copy (repeatable); without it every non-system " + "schema of the source is copied" + ), + ) data_migrations: Path = Flag( default=Path("tmp/data-migrations"), description="Migration receipt directory (default: tmp/data-migrations)", @@ -4031,6 +4042,12 @@ def restart_command(args: ComponentArgs, ctx: Ctx, fleet: Fleet) -> RestartPaylo "cloudfall data copy postgresql-main --database crm " "--source-url-file source.url", ), + ( + "Copy only two schemas", + "cloudfall data copy postgresql-main --database crm " + "--source-url-file source.url --source-schema public " + "--source-schema app --yes", + ), ], ) def data_copy( @@ -4039,11 +4056,14 @@ def data_copy( """Restore the source database into the service, or show the plan.""" context = args.context(fleet, ctx) source = fleet.path(args.source_url_file) + schemas = DataCopySchemas(args.source_schema) try: if not args.yes: return DataMigratePayload( _plan( - preview_data_migration(context, args.service, args.database, source) + preview_data_migration( + context, args.service, args.database, source, schemas + ) ) ) body = migrate_data( @@ -4052,6 +4072,7 @@ def data_copy( args.database, source, fleet.path(args.data_migrations), + schemas, ) except LifecycleError as error: raise _lifecycle_failed(error) from error diff --git a/sdk/src/cloudfall/lifecycle.py b/sdk/src/cloudfall/lifecycle.py index 1f53dc3..65fae9b 100644 --- a/sdk/src/cloudfall/lifecycle.py +++ b/sdk/src/cloudfall/lifecycle.py @@ -257,6 +257,33 @@ def as_dict(self) -> dict[str, object]: } +@dataclass(frozen=True, slots=True) +class DataCopySchemas: + """The source schemas a data copy takes, in sorted order. + + Empty means every non-system schema of the source, the whole database a + ``pg_dump`` without ``--schema`` dumps. + """ + + names: tuple[str, ...] = () + + def __post_init__(self) -> None: + """Keep the names sorted so plans and receipts compare verbatim.""" + object.__setattr__(self, "names", tuple(sorted(set(self.names)))) + + @property + def every_schema(self) -> bool: + """Whether no schema was named, so the whole database is copied.""" + return not self.names + + def describe(self) -> str: + """Name the selection for a plan.""" + if self.every_schema: + return "every non-system schema" + noun = "schema" if len(self.names) == 1 else "schemas" + return f"{noun} {', '.join(self.names)}" + + class PreviewSubject(StrEnum): """Kind of declared resource a previewed action targets.""" @@ -1234,24 +1261,28 @@ def health( ) -def plan_data_migration( +def plan_data_migration( # noqa: PLR0913 - the copy's inputs plus its schemas. context: EngineContext, service_id: ResourceId, database: str, source_url_file: Path, receipt_directory: Path | None, + schemas: DataCopySchemas | None = None, ) -> ExecutionPlan: """Compose the subprocess steps for one guided data migration. The plan reuses ``ExecutionPlan`` with the service identifier in the ``component_id`` slot; the source URL never crosses a process argument, - only the controller-side file path does. + only the controller-side file path does. Without named schemas the + extra vars are the whole-database copy's, unchanged. """ extra_vars: dict[str, object] = { "cloudfall_data_service_id": service_id.value, "cloudfall_data_database": database, "cloudfall_data_source_url_file": str(source_url_file.resolve()), } + if schemas is not None and not schemas.every_schema: + extra_vars["cloudfall_data_schemas"] = list(schemas.names) if receipt_directory is not None: extra_vars["cloudfall_data_receipt_directory"] = str( receipt_directory.resolve() @@ -1267,17 +1298,18 @@ def plan_data_migration( ) -def migrate_data( +def migrate_data( # noqa: PLR0913 - as plan_data_migration. context: EngineContext, service_id: ResourceId, database: str, source_url_file: Path, receipt_directory: Path | None = None, + schemas: DataCopySchemas | None = None, ) -> dict[str, object]: """Dump one external database and restore it into a declared service.""" service = _checked_data_migration(context, service_id, database, source_url_file) plan = plan_data_migration( - context, service_id, database, source_url_file, receipt_directory + context, service_id, database, source_url_file, receipt_directory, schemas ) execute_plan(plan, context.run) result: dict[str, object] = { @@ -1300,9 +1332,11 @@ def preview_data_migration( service_id: ResourceId, database: str, source_url_file: Path, + schemas: DataCopySchemas | None = None, ) -> LifecyclePreview: """Validate a data migration without touching any server.""" service = _checked_data_migration(context, service_id, database, source_url_file) + selection = schemas if schemas is not None else DataCopySchemas() return LifecyclePreview( action="data-migration", subject=PreviewSubject.SERVICE, @@ -1310,10 +1344,11 @@ def preview_data_migration( release=None, servers=(service.server_id,), effect=( - f"dump the database whose URL is in {source_url_file} on " - f"{service.server_id} and restore it into declared database " - f"{database} of service {service_id}, refusing a non-empty target " - "and verifying per-table row counts" + f"dump {selection.describe()} of the database whose URL is in " + f"{source_url_file} on {service.server_id} and restore it into " + f"declared database {database} of service {service_id}, refusing " + "a target with tables in those schemas and verifying per-table " + "row counts" ), ) diff --git a/sdk/tests/test_data_migration.py b/sdk/tests/test_data_migration.py index 1b05d29..126b2f1 100644 --- a/sdk/tests/test_data_migration.py +++ b/sdk/tests/test_data_migration.py @@ -10,7 +10,9 @@ from cloudfall.agent_tools import AgentConfig from cloudfall.domain import ResourceId from cloudfall.lifecycle import ( + DataCopySchemas, EngineContext, + ExecutionPlan, LifecycleError, migrate_data, plan_data_migration, @@ -61,6 +63,57 @@ def test_data_migration_plan_composes_engine_contract_steps( assert Path(extra_vars["cloudfall_data_receipt_directory"]).is_absolute() +def _extra_vars(plan: ExecutionPlan) -> dict[str, object]: + playbook = plan.steps[-1] + extra_vars = json.loads(playbook.argv[playbook.argv.index("--extra-vars") + 1]) + assert isinstance(extra_vars, dict) + return extra_vars + + +def test_s007_1_data_migration_plan_passes_the_named_schemas_sorted( + tmp_path: Path, +) -> None: + plan = plan_data_migration( + _context(tmp_path), + ResourceId("postgresql-main"), + "crm", + _url_file(tmp_path), + tmp_path / "receipts", + DataCopySchemas(("public", "app")), + ) + + assert _extra_vars(plan)["cloudfall_data_schemas"] == ["app", "public"] + + +def test_s007_2_data_migration_plan_without_schemas_is_unchanged( + tmp_path: Path, +) -> None: + """No --schema keeps the extra vars a 0.7.0 whole-database copy sent.""" + arguments = ( + _context(tmp_path), + ResourceId("postgresql-main"), + "crm", + _url_file(tmp_path), + tmp_path / "receipts", + ) + + assert _extra_vars(plan_data_migration(*arguments)) == _extra_vars( + plan_data_migration(*arguments, DataCopySchemas()) + ) + assert set(_extra_vars(plan_data_migration(*arguments))) == { + "cloudfall_data_service_id", + "cloudfall_data_database", + "cloudfall_data_source_url_file", + "cloudfall_data_receipt_directory", + } + + +def test_data_copy_schemas_describe_the_selection() -> None: + assert DataCopySchemas().describe() == "every non-system schema" + assert DataCopySchemas(("public",)).describe() == "schema public" + assert DataCopySchemas(("public", "app")).describe() == "schemas app, public" + + def test_data_migration_rejects_an_undeclared_service(tmp_path: Path) -> None: with pytest.raises(LifecycleError) as error: migrate_data( diff --git a/sdk/tests/test_lifecycle.py b/sdk/tests/test_lifecycle.py index bc35a2b..0d9a886 100644 --- a/sdk/tests/test_lifecycle.py +++ b/sdk/tests/test_lifecycle.py @@ -335,6 +335,39 @@ def test_cli_data_migration_plan_checks_the_source_url_file( assert "u:p@" not in json.dumps(payload) +def test_s007_14_data_copy_plan_names_the_selected_schemas( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + source_url_file = tmp_path / "source.url" + source_url_file.write_text("postgresql://u:p@db.example.test/crm\n") + argv = [ + "data", + "copy", + "postgresql-main", + "--database", + "crm", + "--source-url-file", + str(source_url_file), + ] + + exit_code, payload = _cli_lifecycle( + [*argv, "--source-schema", "public", "--source-schema", "app"], + tmp_path, + capsys, + ) + assert exit_code == 0 + assert payload["meta"]["dry_run"] is True + assert "dump schemas app, public of the database" in str( + payload["data"]["would_run"] + ) + + exit_code, payload = _cli_lifecycle(argv, tmp_path, capsys) + assert exit_code == 0 + assert "dump every non-system schema of the database" in str( + payload["data"]["would_run"] + ) + + def _git(*arguments: str) -> None: subprocess.run( # noqa: S603 - fixture repository setup. [ From 46b57b1c20ad46c8e54b3272be1f2a20cd1de5e0 Mon Sep 17 00:00:00 2001 From: "shipmill-romamo[bot]" <338557544+shipmill-romamo[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:13:58 +0200 Subject: [PATCH 2/4] Refuse a source schema name Ansible would template A --source-schema name reaches the engine as an Ansible extra var, and Ansible templates extra vars: a name such as {{ lookup('pipe', 'id') }} ran a command on the management host. Names are now a PostgresSchemaName: non-empty, at most 63 bytes, no control character and no brace, refused as a usage error before any plan. Co-Authored-By: Claude Opus 5.5 --- sdk/src/cloudfall/app.py | 14 ++++++++++++-- sdk/src/cloudfall/domain.py | 33 ++++++++++++++++++++++++++++++++ sdk/src/cloudfall/lifecycle.py | 15 +++++++++++---- sdk/tests/test_data_migration.py | 12 ++++++++---- sdk/tests/test_lifecycle.py | 31 ++++++++++++++++++++++++++++++ 5 files changed, 95 insertions(+), 10 deletions(-) diff --git a/sdk/src/cloudfall/app.py b/sdk/src/cloudfall/app.py index 2aabc4a..da63d41 100644 --- a/sdk/src/cloudfall/app.py +++ b/sdk/src/cloudfall/app.py @@ -129,6 +129,7 @@ ConnectionAddress, Hostname, LinuxUser, + PostgresSchemaName, ReleaseId, ResourceId, ResourceKind, @@ -3860,9 +3861,18 @@ class DataMigrateArgs(EngineArgs): ) def __post_init__(self) -> None: - """Keep relative paths inside the project.""" + """Keep relative paths inside the project; check the schema names.""" EngineArgs.__post_init__(self) _inside_project(self.data_migrations, "data-migrations") + self.source_schemas() + + def source_schemas(self) -> DataCopySchemas: + """Return the named source schemas, refusing a malformed name.""" + try: + names = tuple(PostgresSchemaName(name) for name in self.source_schema) + except ValueError as error: + raise ParseError(str(error), context={"flag": "source-schema"}) from error + return DataCopySchemas(names) class DeployPayload(Payload): @@ -4056,7 +4066,7 @@ def data_copy( """Restore the source database into the service, or show the plan.""" context = args.context(fleet, ctx) source = fleet.path(args.source_url_file) - schemas = DataCopySchemas(args.source_schema) + schemas = args.source_schemas() try: if not args.yes: return DataMigratePayload( diff --git a/sdk/src/cloudfall/domain.py b/sdk/src/cloudfall/domain.py index dffa315..10d30eb 100644 --- a/sdk/src/cloudfall/domain.py +++ b/sdk/src/cloudfall/domain.py @@ -34,6 +34,7 @@ _FILE_MODE_PATTERN = re.compile(r"^0[0-7]{3}$") _SHA256_PATTERN = re.compile(r"^[a-f0-9]{64}$") _MAX_HOSTNAME_LENGTH = 253 +_POSTGRES_NAME_MAX_BYTES = 63 _MAX_TCP_PORT = 65535 _MIN_HTTP_STATUS = 100 _MAX_HTTP_STATUS = 599 @@ -337,6 +338,38 @@ def from_boundary(cls, value: object) -> LinuxUser: return cls(_required_string(value, "Linux user")) +@dataclass(frozen=True, slots=True) +class PostgresSchemaName: + """A PostgreSQL schema name a data copy may select. + + The name travels to the engine as an Ansible extra var, which Ansible + templates, so braces are refused; PostgreSQL truncates a name past 63 + bytes, and a control character breaks the row-count lines. + """ + + value: str + + def __post_init__(self) -> None: + """Refuse a name PostgreSQL or Ansible would read differently.""" + if not self.value: + message = "schema name must not be empty" + raise ValueError(message) + if len(self.value.encode()) > _POSTGRES_NAME_MAX_BYTES: + message = f"schema name exceeds 63 bytes: {self.value!r}" + raise ValueError(message) + if _CONTROL_CHARACTER_PATTERN.search(self.value) or any( + brace in self.value for brace in "{}" + ): + message = ( + f"schema name holds a control character or a brace: {self.value!r}" + ) + raise ValueError(message) + + def __str__(self) -> str: + """Return the name as PostgreSQL stores it.""" + return self.value + + @dataclass(frozen=True, slots=True) class TcpPort: """Validated TCP port.""" diff --git a/sdk/src/cloudfall/lifecycle.py b/sdk/src/cloudfall/lifecycle.py index 65fae9b..126a5a3 100644 --- a/sdk/src/cloudfall/lifecycle.py +++ b/sdk/src/cloudfall/lifecycle.py @@ -49,6 +49,7 @@ from cloudfall.domain import ( CommandArgv, + PostgresSchemaName, ReleaseCommandTimeout, ReleaseId, ResourceId, @@ -265,23 +266,29 @@ class DataCopySchemas: ``pg_dump`` without ``--schema`` dumps. """ - names: tuple[str, ...] = () + names: tuple[PostgresSchemaName, ...] = () def __post_init__(self) -> None: """Keep the names sorted so plans and receipts compare verbatim.""" - object.__setattr__(self, "names", tuple(sorted(set(self.names)))) + unique = sorted(set(self.names), key=lambda name: name.value) + object.__setattr__(self, "names", tuple(unique)) @property def every_schema(self) -> bool: """Whether no schema was named, so the whole database is copied.""" return not self.names + @property + def values(self) -> list[str]: + """The names as the engine takes them, sorted.""" + return [name.value for name in self.names] + def describe(self) -> str: """Name the selection for a plan.""" if self.every_schema: return "every non-system schema" noun = "schema" if len(self.names) == 1 else "schemas" - return f"{noun} {', '.join(self.names)}" + return f"{noun} {', '.join(self.values)}" class PreviewSubject(StrEnum): @@ -1282,7 +1289,7 @@ def plan_data_migration( # noqa: PLR0913 - the copy's inputs plus its schemas. "cloudfall_data_source_url_file": str(source_url_file.resolve()), } if schemas is not None and not schemas.every_schema: - extra_vars["cloudfall_data_schemas"] = list(schemas.names) + extra_vars["cloudfall_data_schemas"] = schemas.values if receipt_directory is not None: extra_vars["cloudfall_data_receipt_directory"] = str( receipt_directory.resolve() diff --git a/sdk/tests/test_data_migration.py b/sdk/tests/test_data_migration.py index 126b2f1..a017180 100644 --- a/sdk/tests/test_data_migration.py +++ b/sdk/tests/test_data_migration.py @@ -8,7 +8,7 @@ import pytest from cloudfall.agent_tools import AgentConfig -from cloudfall.domain import ResourceId +from cloudfall.domain import PostgresSchemaName, ResourceId from cloudfall.lifecycle import ( DataCopySchemas, EngineContext, @@ -63,6 +63,10 @@ def test_data_migration_plan_composes_engine_contract_steps( assert Path(extra_vars["cloudfall_data_receipt_directory"]).is_absolute() +def _schemas(*names: str) -> DataCopySchemas: + return DataCopySchemas(tuple(PostgresSchemaName(name) for name in names)) + + def _extra_vars(plan: ExecutionPlan) -> dict[str, object]: playbook = plan.steps[-1] extra_vars = json.loads(playbook.argv[playbook.argv.index("--extra-vars") + 1]) @@ -79,7 +83,7 @@ def test_s007_1_data_migration_plan_passes_the_named_schemas_sorted( "crm", _url_file(tmp_path), tmp_path / "receipts", - DataCopySchemas(("public", "app")), + _schemas("public", "app"), ) assert _extra_vars(plan)["cloudfall_data_schemas"] == ["app", "public"] @@ -110,8 +114,8 @@ def test_s007_2_data_migration_plan_without_schemas_is_unchanged( def test_data_copy_schemas_describe_the_selection() -> None: assert DataCopySchemas().describe() == "every non-system schema" - assert DataCopySchemas(("public",)).describe() == "schema public" - assert DataCopySchemas(("public", "app")).describe() == "schemas app, public" + assert _schemas("public").describe() == "schema public" + assert _schemas("public", "app", "public").describe() == "schemas app, public" def test_data_migration_rejects_an_undeclared_service(tmp_path: Path) -> None: diff --git a/sdk/tests/test_lifecycle.py b/sdk/tests/test_lifecycle.py index 0d9a886..bc1b3c3 100644 --- a/sdk/tests/test_lifecycle.py +++ b/sdk/tests/test_lifecycle.py @@ -368,6 +368,37 @@ def test_s007_14_data_copy_plan_names_the_selected_schemas( ) +@pytest.mark.parametrize( + "schema", + ["", "{{ lookup('pipe', 'id') }}", "{% raw %}", "a\nb", "x" * 64], +) +def test_data_copy_refuses_a_schema_name_ansible_or_postgres_would_misread( + schema: str, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A name reaches the engine as an extra var, which Ansible templates.""" + source_url_file = tmp_path / "source.url" + source_url_file.write_text("postgresql://u:p@db.example.test/crm\n") + + exit_code, payload = _cli_lifecycle( + [ + "data", + "copy", + "postgresql-main", + "--database", + "crm", + "--source-url-file", + str(source_url_file), + "--source-schema", + schema, + ], + tmp_path, + capsys, + ) + + assert exit_code == 2 + assert payload["ok"] is False + + def _git(*arguments: str) -> None: subprocess.run( # noqa: S603 - fixture repository setup. [ From c6a0cbc0dde8c0ef34f77830766958891b532d8a Mon Sep 17 00:00:00 2001 From: "shipmill-romamo[bot]" <338557544+shipmill-romamo[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:14:42 +0200 Subject: [PATCH 3/4] Say which source schema names data copy refuses Co-Authored-By: Claude Opus 5.5 --- changelog.d/124-data-copy-schema.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/124-data-copy-schema.md b/changelog.d/124-data-copy-schema.md index f951f55..389e976 100644 --- a/changelog.d/124-data-copy-schema.md +++ b/changelog.d/124-data-copy-schema.md @@ -1,3 +1,3 @@ ### Added -- `cloudfall data copy` takes a repeatable `--source-schema NAME` (and the `data_copy` MCP tool a `source_schema` list): the dump and restore copy only the named schemas, the empty-target check and the per-table row-count check read exactly those schemas, and the plan without `--yes` names them. Without it the dump is the whole database as in 0.7.0, and the checks now cover every non-system schema of the source instead of `public` only, so a row-count mismatch outside `public` now fails the copy. The `DataMigrationReceipt` gains `spec.schemas` (sorted) and a `schema` on each `spec.tables[]` entry beside `table` and `rows` (#124) +- `cloudfall data copy` takes a repeatable `--source-schema NAME` (and the `data_copy` MCP tool a `source_schema` list): the dump and restore copy only the named schemas, the empty-target check and the per-table row-count check read exactly those schemas, and the plan without `--yes` names them; a name that is empty, longer than 63 bytes, or holds a control character or a brace is refused as a usage error. Without it the dump is the whole database as in 0.7.0, and the checks now cover every non-system schema of the source instead of `public` only, so a row-count mismatch outside `public` now fails the copy. The `DataMigrationReceipt` gains `spec.schemas` (sorted) and a `schema` on each `spec.tables[]` entry beside `table` and `rows` (#124) From f9dfff2aa2b91fa056f9a789bd5cb5be2b48293f Mon Sep 17 00:00:00 2001 From: "shipmill-romamo[bot]" <338557544+shipmill-romamo[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:25:03 +0200 Subject: [PATCH 4/4] Replay the S-006-13 receipt with the schema,table,rows lines The data-copy receipt now reads the selected schemas and the parsed tables, so the replay sets both and runs the role's table-recording task. Co-Authored-By: Claude Opus 5.5 --- engine/tests/test_postgresql_owner_role.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/engine/tests/test_postgresql_owner_role.py b/engine/tests/test_postgresql_owner_role.py index f1046b8..5ecb0f0 100644 --- a/engine/tests/test_postgresql_owner_role.py +++ b/engine/tests/test_postgresql_owner_role.py @@ -309,11 +309,12 @@ def _data_copy_receipt(tmp_path: Path) -> dict[str, object]: (DATA_MIGRATION / "tasks" / "main.yml").read_text(encoding="utf-8") ) named = {task["name"]: task for task in tasks} - restore = next( - task + block = { + task["name"]: task for task in named["Migrate and verify the declared database"]["block"] - if task["name"] == "Restore the dump into the declared database" - ) + } + restore = block["Restore the dump into the declared database"] + tables = block["Record the verified tables for the receipt"] validate, select, require, record, directory, write = ( named[name] for name in DATA_TASKS ) @@ -327,11 +328,12 @@ def _data_copy_receipt(tmp_path: Path) -> dict[str, object]: }, } counts = { - "name": "Stand in for the source row counts", + "name": "Stand in for the selected schemas and the source row counts", "ansible.builtin.set_fact": { + "cloudfall_data_migration_selected_schemas": ["public"], "cloudfall_data_migration_source_counts": { - "stdout_lines": ["public.invoices,3"] - } + "stdout_lines": ["public,invoices,3"] + }, }, } play = [ @@ -354,6 +356,7 @@ def _data_copy_receipt(tmp_path: Path) -> dict[str, object]: record, restored_as, counts, + tables, directory, write, ], @@ -396,6 +399,7 @@ def test_s006_13_data_copy_restores_as_the_owner_and_receipts_it( assert spec["owner"] == "billing" assert spec["database"] == "billing" assert "application" not in spec + assert spec["tables"] == [{"schema": "public", "table": "invoices", "rows": 3}] assert PASSWORD not in json.dumps(receipt)