diff --git a/changelog.d/122-owned-postgresql-converge.md b/changelog.d/122-owned-postgresql-converge.md new file mode 100644 index 0000000..f6cef51 --- /dev/null +++ b/changelog.d/122-owned-postgresql-converge.md @@ -0,0 +1,3 @@ +### Added + +- `services.yml` converges a PostgreSQL database owned by a declared role: it creates the `owner` login role, sets its password from `passwordRef` as SCRAM-SHA-256 on every converge, and gives it the database; backups, the restore drill, and `data copy` (restoring as the owner, its receipt's `spec.owner`) cover it like any other database. Every path that runs `services.yml` (the `converge_services` MCP tool, the `services` step of `cloudfall migrate`, `operator approve` and `operator run`, and `cloudfall-engine playbook run services.yml`) reads the password from a `--secrets-dir` flag (default `secrets`) and stages it in a 0600 file in a private temp dir outside the project, removed after the run; the value is in no inventory, command line, Ansible output, answer, or receipt, and the answer's `owned_roles` names the reference, the key, and the value's SHA-256; `operator approve` records them as `ownedRoles` in the proposal receipt's `outcome` (an optional property) and `operator run` in each executed proposal's entry. A missing secrets file or key fails with `PRECONDITION` naming the expected path and key before any playbook runs (#122) diff --git a/config/schemas/v1/operator-proposal.schema.json b/config/schemas/v1/operator-proposal.schema.json index 1d9d1a3..e0bbc9c 100644 --- a/config/schemas/v1/operator-proposal.schema.json +++ b/config/schemas/v1/operator-proposal.schema.json @@ -247,6 +247,62 @@ "type": "string", "minLength": 1, "maxLength": 1000 + }, + "ownedRoles": { + "description": "Owner role passwords the run set: the reference, the key, and the value's SHA-256, never the value", + "type": "array", + "items": { + "type": "object", + "properties": { + "service": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(-[a-z0-9]+)*$" + }, + "owner": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]{0,62}$" + }, + "passwordRef": { + "type": "object", + "properties": { + "environment": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(-[a-z0-9]+)*$" + }, + "path": { + "type": "string", + "pattern": "^/" + }, + "key": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]{0,254}$" + } + }, + "required": [ + "environment", + "path", + "key" + ], + "additionalProperties": false + }, + "key": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]{0,254}$" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + }, + "required": [ + "service", + "owner", + "passwordRef", + "key", + "sha256" + ], + "additionalProperties": false + } } }, "required": [ diff --git a/docs/secrets-guide.md b/docs/secrets-guide.md index ee09a25..e6bf0cc 100644 --- a/docs/secrets-guide.md +++ b/docs/secrets-guide.md @@ -107,6 +107,38 @@ with `EnvironmentFile=`. A missing source fails fast with the exact expected path; a malformed line fails with its line number; an unreadable value never half-renders a file. +## Database owner passwords + +A PostgreSQL database owned by a declared role names its password with a +`passwordRef`, one key of one source in the same layout: + +```yaml +# services/postgresql-billing.yaml (excerpt) +databases: + - name: billing + owner: billing + passwordRef: + environment: production + path: /billing/postgresql + key: BILLING_PASSWORD +``` + +Every path that runs `services.yml` (the `converge_services` MCP tool, the +`services` step of `cloudfall migrate`, `cloudfall operator approve` and +`operator run`, and `cloudfall-engine playbook run services.yml`) reads +`secrets/production/billing/postgresql.env` from its `--secrets-dir` (default +`secrets`) before any playbook starts. A missing file or key stops it with +`PRECONDITION`, naming the expected path and key. + +The value is written to a `0600` file in a private temp directory outside the +project, and only that file's path reaches Ansible. The role copies the file +to the host as `0600`, `psql` reads it there, PostgreSQL stores the password as +SCRAM-SHA-256, and both files are removed when the run ends. Every task that +sees it runs with `no_log`. The answer's `owned_roles` names the reference, +the key, and the value's SHA-256 (an operator proposal keeps them as +`ownedRoles` in its receipt's `outcome`); the password is set on every converge, so a +rotation is a commit in the private repository and a services converge. + ## Drift detection Rendering writes an `EnvironmentReceipt` (component, content hash, key diff --git a/engine/ansible/playbooks/services.yml b/engine/ansible/playbooks/services.yml index 0365a33..7724bfa 100644 --- a/engine/ansible/playbooks/services.yml +++ b/engine/ansible/playbooks/services.yml @@ -11,6 +11,13 @@ name: cloudfall_postgresql vars: cloudfall_postgresql_service: "{{ cloudfall_service_item }}" + # Controller paths of the staged owner passwords cloudfall passes + # as cloudfall_services_password_files; never the values. + cloudfall_postgresql_password_files: >- + {{ + (cloudfall_services_password_files | default({})) + [cloudfall_service_item.id] | default({}) + }} loop: >- {{ cloudfall_services | default([]) diff --git a/engine/ansible/roles/cloudfall_postgresql/defaults/main.yml b/engine/ansible/roles/cloudfall_postgresql/defaults/main.yml index c814fb7..7be4923 100644 --- a/engine/ansible/roles/cloudfall_postgresql/defaults/main.yml +++ b/engine/ansible/roles/cloudfall_postgresql/defaults/main.yml @@ -4,3 +4,5 @@ cloudfall_postgresql_repository_key_url: >- https://www.postgresql.org/media/keys/ACCC4CF8.asc cloudfall_postgresql_repository_key_path: /etc/apt/keyrings/postgresql.asc cloudfall_postgresql_repository_url: https://apt.postgresql.org/pub/repos/apt +cloudfall_postgresql_password_files: {} +cloudfall_postgresql_password_path: /var/lib/postgresql/.cloudfall-owner-password diff --git a/engine/ansible/roles/cloudfall_postgresql/meta/argument_specs.yml b/engine/ansible/roles/cloudfall_postgresql/meta/argument_specs.yml index ed51ddd..d02923e 100644 --- a/engine/ansible/roles/cloudfall_postgresql/meta/argument_specs.yml +++ b/engine/ansible/roles/cloudfall_postgresql/meta/argument_specs.yml @@ -52,10 +52,21 @@ argument_specs: required: true application: type: str - required: true owner: type: str required: true + passwordRef: + type: dict + options: + environment: + type: str + required: true + path: + type: str + required: true + key: + type: str + required: true backup: type: dict required: true @@ -77,6 +88,16 @@ argument_specs: enabled: type: bool required: true + cloudfall_postgresql_password_files: + type: dict + default: {} + description: >- + Controller path of each owner role's staged password file, by + role; the values never enter a variable + cloudfall_postgresql_password_path: + type: path + default: /var/lib/postgresql/.cloudfall-owner-password + description: Host path an owner role's password is staged at, 0600 cloudfall_postgresql_metrics_role: type: str default: alloy diff --git a/engine/ansible/roles/cloudfall_postgresql/tasks/main.yml b/engine/ansible/roles/cloudfall_postgresql/tasks/main.yml index 6b1ec45..769446e 100644 --- a/engine/ansible/roles/cloudfall_postgresql/tasks/main.yml +++ b/engine/ansible/roles/cloudfall_postgresql/tasks/main.yml @@ -12,6 +12,22 @@ PostgreSQL deployment requires its declared Debian server and a loopback-only bind address +- name: Require a staged password for every owner role + ansible.builtin.assert: + that: + - >- + cloudfall_postgresql_service.postgresql.databases + | selectattr('passwordRef', 'defined') + | map(attribute='owner') + | reject('in', cloudfall_postgresql_password_files) + | list + | length == 0 + fail_msg: >- + An owner role of service {{ cloudfall_postgresql_service.id }} has no + staged password; run services.yml through cloudfall (converge_services, + cloudfall migrate, operator approve, or cloudfall-engine playbook run + services.yml), which reads each passwordRef from --secrets-dir + - name: Validate the declared backup calendar expression ansible.builtin.command: argv: @@ -132,7 +148,7 @@ changed_when: false when: not ansible_check_mode -- name: Create missing application login roles +- name: Create missing database owner login roles ansible.builtin.command: argv: - /usr/bin/psql @@ -154,6 +170,20 @@ - item not in cloudfall_postgresql_existing_roles.stdout_lines changed_when: true +- name: Set each owner role password from its staged file + ansible.builtin.include_tasks: owner_password.yml + loop: >- + {{ + cloudfall_postgresql_service.postgresql.databases + | selectattr('passwordRef', 'defined') + | map(attribute='owner') + | unique + | list + }} + loop_control: + loop_var: cloudfall_postgresql_owner + when: not ansible_check_mode + - name: Create the metrics monitoring role ansible.builtin.command: argv: @@ -221,6 +251,51 @@ - item.name not in cloudfall_postgresql_existing_databases.stdout_lines changed_when: true +- name: Read the owners of existing PostgreSQL databases + ansible.builtin.command: + argv: + - /usr/bin/psql + - --port + - "{{ cloudfall_postgresql_service.bind.port }}" + - --tuples-only + - --no-align + - --command + - SELECT datname || '=' || pg_get_userbyid(datdba) FROM pg_database + become: true + become_user: postgres + register: cloudfall_postgresql_database_owners + changed_when: false + when: + - not ansible_check_mode + - >- + cloudfall_postgresql_service.postgresql.databases + | selectattr('passwordRef', 'defined') | list | length > 0 + +- name: Give each owned database to its declared owner role + ansible.builtin.command: + argv: + - /usr/bin/psql + - --port + - "{{ cloudfall_postgresql_service.bind.port }}" + - --command + - ALTER DATABASE "{{ item.name }}" OWNER TO "{{ item.owner }}" + become: true + become_user: postgres + loop: >- + {{ + cloudfall_postgresql_service.postgresql.databases + | selectattr('passwordRef', 'defined') + | list + }} + loop_control: + label: "{{ item.name }}" + when: + - not ansible_check_mode + - >- + item.name ~ '=' ~ item.owner + not in cloudfall_postgresql_database_owners.stdout_lines + changed_when: true + - name: Create the backup directory ansible.builtin.file: path: "{{ cloudfall_postgresql_service.backup.directory }}" diff --git a/engine/ansible/roles/cloudfall_postgresql/tasks/owner_password.yml b/engine/ansible/roles/cloudfall_postgresql/tasks/owner_password.yml new file mode 100644 index 0000000..4bec582 --- /dev/null +++ b/engine/ansible/roles/cloudfall_postgresql/tasks/owner_password.yml @@ -0,0 +1,40 @@ +--- +# Sets one owner role's password on every converge, so a rotation is a commit +# in the secrets repository and a services converge. The value reaches the +# host in a 0600 file removed after the run; psql reads the file itself, so +# the value is in no command line, no task argument, and no Ansible output. +- name: Set the password of owner role {{ cloudfall_postgresql_owner }} + block: + - name: Stage the owner role password on the host + ansible.builtin.copy: + src: "{{ cloudfall_postgresql_password_files[cloudfall_postgresql_owner] }}" + dest: "{{ cloudfall_postgresql_password_path }}" + owner: postgres + group: postgres + mode: "0600" + no_log: true + + - name: Store the owner role password as SCRAM-SHA-256 + ansible.builtin.command: + argv: + - /usr/bin/psql + - --port + - "{{ cloudfall_postgresql_service.bind.port }}" + - --no-psqlrc + - --quiet + - --set + - ON_ERROR_STOP=1 + stdin: | + SET password_encryption = 'scram-sha-256'; + \set cloudfall_password `cat {{ cloudfall_postgresql_password_path | quote }}` + ALTER ROLE "{{ cloudfall_postgresql_owner }}" WITH LOGIN PASSWORD :'cloudfall_password'; + become: true + become_user: postgres + changed_when: true + no_log: true + always: + - name: Remove the staged owner role password + ansible.builtin.file: + path: "{{ cloudfall_postgresql_password_path }}" + state: absent + no_log: true diff --git a/engine/src/cloudfall_engine/cli.py b/engine/src/cloudfall_engine/cli.py index 6a75734..0e02358 100644 --- a/engine/src/cloudfall_engine/cli.py +++ b/engine/src/cloudfall_engine/cli.py @@ -6,22 +6,43 @@ streams to stderr as plain text. """ +import dataclasses import json import os +import shutil from dataclasses import dataclass from functools import partial from importlib.metadata import version from pathlib import Path -from typing import TYPE_CHECKING, Self +from typing import TYPE_CHECKING, Self, cast -from cloudfall.domain import ResourceId -from cloudfall.inventory import PlatformInventory +from cloudfall.commands import snake_case_keys +from cloudfall.domain import ( + AbsolutePath, + PostgresDatabaseName, + PostgresRoleName, + ResourceId, + SecretKeyName, +) +from cloudfall.inventory import ( + OwnedPostgresDatabase, + PlatformInventory, + SecretValueReference, +) from cloudfall.project import ( PROJECT_DIRECTORY_VARIABLE, ProjectError, resolve_project_directory, ) from cloudfall.resources import default_engine_directory, default_schema_directory +from cloudfall.secrets import ( + ERROR_DECRYPT_FAILED, + ERROR_SOPS_MISSING, + SecretsError, + SopsSecretProvider, + owned_role_passwords, + staged_password_files, +) from cloudfall.validation import ConfigValidationError, validate_config from treaty import ( App, @@ -54,7 +75,9 @@ ) if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Callable, Mapping, Sequence + + from cloudfall.secrets import OwnedRolePassword app = App( "cloudfall-engine", @@ -367,19 +390,29 @@ class RunArgs(EngineArgs): syntax_check: bool = Flag( default=False, description="Only check the playbook syntax" ) + secrets_dir: Path = Flag( + default=Path("secrets"), + description=( + "sops-encrypted secrets directory services.yml reads owner role " + "passwords from (default: secrets)" + ), + secret=False, + ) def __post_init__(self) -> None: """Report a missing playbook, inventory, or role directory as input.""" _playbook_run(self) -def _playbook_run(args: RunArgs) -> PlaybookRun: +def _playbook_run( + args: RunArgs, staged_vars: tuple[str, ...] = () +) -> PlaybookRun: try: return PlaybookRun( playbook=resolve_playbook(args.playbook, args.engine), inventory_file=args.inventory, role_directories=args.roles, - extra_vars=args.extra_vars, + extra_vars=(*args.extra_vars, *staged_vars), tags=args.tags, limit=args.limit, check=args.dry_run, @@ -397,6 +430,124 @@ class Played: effect: str playbook: Path inventory: Path + owned_roles: list[dict[str, object]] = Out( + default_factory=list, ordered=True, high_entropy=False + ) + """Each owner role password services.yml set: its passwordRef, its key, + and the value's SHA-256, never the value""" + + +SERVICES_PLAYBOOK = "services.yml" + + +def _sops_decrypt(ctx: Ctx) -> Callable[[Path], str]: + """Decrypt one sops file through ``ctx.run``.""" + + def decrypt(source: Path) -> str: + binary = shutil.which("sops") + if binary is None: + message = ( + "the sops binary is not installed; install sops and age, and " + "set SOPS_AGE_KEY_FILE to your age key" + ) + raise SecretsError(ERROR_SOPS_MISSING, message) + done = ctx.run([binary, "--decrypt", str(source)], check=False) + if done.returncode != 0: + message = f"sops could not decrypt {source}: {done.stderr.strip()[:300]}" + raise SecretsError(ERROR_DECRYPT_FAILED, message) + return done.stdout + + return decrypt + + +def _rendered_owned_databases( + inventory_file: Path, +) -> tuple[tuple[ResourceId, OwnedPostgresDatabase], ...]: + """Read every owned database, with its service, from a rendered inventory. + + An inventory that isn't a rendered Cloudfall one (YAML, INI, a directory, + or JSON of another shape) declares none, so the run stays as it was. + Only an entry that names a ``passwordRef`` must be well formed. + """ + if not inventory_file.is_file(): + return () + try: + document = cast( + "object", json.loads(inventory_file.read_text(encoding="utf-8")) + ) + except (UnicodeDecodeError, json.JSONDecodeError): + return () + hosts = _nested(document, "all", "children", "cloudfall_servers", "hosts") + owned: list[tuple[ResourceId, OwnedPostgresDatabase]] = [] + for host in hosts.values(): + for service in _mappings(_nested(host).get("cloudfall_services")): + if service.get("serviceKind") != "postgresql": + continue + databases = _nested(service, "postgresql").get("databases") + owned.extend( + (ResourceId.from_boundary(service.get("id")), _owned_database(database)) + for database in _mappings(databases) + if "passwordRef" in database + ) + return tuple(owned) + + +def _nested(value: object, *keys: str) -> Mapping[str, object]: + """Follow ``keys`` through mappings; anything else reads as empty.""" + current = value + for key in keys: + if not isinstance(current, dict): + return {} + current = current.get(key) + return cast("Mapping[str, object]", current) if isinstance(current, dict) else {} + + +def _mappings(value: object) -> list[Mapping[str, object]]: + if not isinstance(value, list): + return [] + return [ + cast("Mapping[str, object]", item) for item in value if isinstance(item, dict) + ] + + +def _owned_database(database: Mapping[str, object]) -> OwnedPostgresDatabase: + reference = database["passwordRef"] + if not isinstance(reference, dict): + message = "an owned database's passwordRef is not a mapping" + raise TypeError(message) + return OwnedPostgresDatabase( + name=PostgresDatabaseName.from_boundary(database.get("name")), + owner=PostgresRoleName.from_boundary(database.get("owner")), + password_ref=SecretValueReference( + environment=ResourceId.from_boundary(reference.get("environment")), + path=AbsolutePath.from_boundary(reference.get("path")), + key=SecretKeyName.from_boundary(reference.get("key")), + ), + ) + + +def _services_passwords( + args: RunArgs, run: PlaybookRun, ctx: Ctx +) -> tuple[OwnedRolePassword, ...]: + """Decrypt the owner passwords the bundled services.yml sets, before it starts.""" + if args.syntax_check or run.playbook != resolve_playbook( + SERVICES_PLAYBOOK, args.engine + ): + return () + try: + databases = _rendered_owned_databases(args.inventory) + except (TypeError, ValueError) as error: + message = f"the inventory declares a malformed owned database: {error}" + raise ParseError(message, context={"flag": "inventory"}) from error + if not databases: + return () + provider = SopsSecretProvider( + secrets_directory=args.secrets_dir, decrypt=_sops_decrypt(ctx) + ) + try: + return owned_role_passwords(databases, provider) + except SecretsError as error: + raise Exit.PRECONDITION(error.message, context={"code": error.code}) from error @playbook.command( @@ -426,8 +577,27 @@ class Played: ], ) def run_playbook(args: RunArgs, ctx: Ctx) -> Played: - """Run ansible-playbook in the foreground and report how it ended.""" - run = _playbook_run(args) + """Run ansible-playbook in the foreground and report how it ended. + + The bundled services.yml gets each owner role password staged in a 0600 + file outside the project, removed when the run ends; only the file paths + cross the command line. + """ + passwords = _services_passwords(args, _playbook_run(args), ctx) + with staged_password_files(passwords) as staged: + played = _run_ansible( + args, ctx, _playbook_run(args, (json.dumps(staged),) if staged else ()) + ) + return dataclasses.replace( + played, + owned_roles=[ + cast("dict[str, object]", snake_case_keys(password.as_dict())) + for password in passwords + ], + ) + + +def _run_ansible(args: RunArgs, ctx: Ctx, run: PlaybookRun) -> Played: try: command = playbook_command(run, args.engine) except PlaybookError as error: diff --git a/engine/tests/golden/examples/ansible-inventory.json b/engine/tests/golden/examples/ansible-inventory.json new file mode 100644 index 0000000..c89883a --- /dev/null +++ b/engine/tests/golden/examples/ansible-inventory.json @@ -0,0 +1 @@ +{"all": {"children": {"cloudfall_applications": {"children": {"application_crm": {"hosts": {"h1": {}, "h2": {}}}}}, "cloudfall_components": {"children": {"component_crm_backend": {"hosts": {"h1": {}, "h2": {}}}}}, "cloudfall_domain_origins": {"hosts": {"h1": {}}}, "cloudfall_environments": {"children": {"environment_production": {"hosts": {"h1": {}, "h2": {}}}}}, "cloudfall_logging_backends": {"hosts": {"h1": {}}}, "cloudfall_logging_collectors": {"hosts": {"h1": {}, "h2": {}}}, "cloudfall_release_servers": {"hosts": {}}, "cloudfall_servers": {"hosts": {"h1": {"ansible_host": "h1.example.internal", "ansible_port": 22, "ansible_user": "cloudfall", "cloudfall_alert_rules": [{"environment": "production", "expr": "pg_up{service=\"postgresql-main\"} == 0", "for": "1m", "id": "postgresql-down", "severity": "critical", "summary": "PostgreSQL postgresql-main is not answering its exporter"}], "cloudfall_applications": [{"approval": "manual", "id": "crm", "user": "crm"}], "cloudfall_components": [{"application": "crm", "application_user": "crm", "healthCheck": {"attempts": 5, "expectedStatuses": [200], "path": "/health", "port": 8100, "scheme": "http", "timeoutSeconds": 5, "type": "http"}, "id": "crm-backend", "install_root": "/srv/apps/crm/backend", "repository": {"url": "https://github.com/example/crm-backend.git"}, "retainUntilCleanup": true, "runtime": {"packageManager": "uv", "type": "python", "version": "3.14"}, "service": {"command": [".venv/bin/uvicorn", "crm.asgi:application", "--host", "127.0.0.1", "--port", "8100"], "manager": "systemd", "name": "crm-backend"}}], "cloudfall_domains": [], "cloudfall_environment": "production", "cloudfall_firewall": {"allowedInbound": [{"description": "ssh", "port": 22, "protocol": "tcp"}, {"description": "http", "port": 80, "protocol": "tcp"}, {"description": "https", "port": 443, "protocol": "tcp"}], "policy": "default-deny"}, "cloudfall_hostname": "h1.example.internal", "cloudfall_labels": {"region": "eu-central", "workload": "application"}, "cloudfall_lifecycle": "active", "cloudfall_logging_backend": {"alerting": {"alertmanager": {"listenAddress": "127.0.0.1", "port": 9093}, "receivers": [{"id": "operations-webhook", "webhook": {"url": "https://hooks.example.internal/cloudfall"}}, {"email": {"authPasswordFile": "/etc/cloudfall/logging/smtp-password", "authUsername": "alerts@example.internal", "from": "alerts@example.internal", "smarthost": "smtp.example.internal:587", "to": "oncall@example.internal"}, "id": "operations-email"}]}, "backend": {"listenAddress": "127.0.0.1", "port": 3100, "retentionHours": 720, "server": "h1", "storage": {"path": "/var/lib/loki-cloudfall", "type": "filesystem"}}, "collectors": {"clientTls": {"caPath": "/etc/cloudfall/logging/ca.crt", "certificatePath": "/etc/cloudfall/logging/client.crt", "keyPath": "/etc/cloudfall/logging/client.key"}, "files": [{"application": "crm", "component": "crm-backend", "id": "crm-backend", "path": "/var/log/crm/backend/*.log", "servers": ["h1", "h2"]}], "journal": {"enabled": true, "maxAgeHours": 24}, "servers": ["h1", "h2"]}, "environment": "production", "gateway": {"listenAddress": "0.0.0.0", "port": 3101, "serverName": "logs.example.internal", "tls": {"certificatePath": "/etc/cloudfall/logging/server.crt", "clientCaPath": "/etc/cloudfall/logging/client-ca.crt", "keyPath": "/etc/cloudfall/logging/server.key"}}, "grafana": {"adminPasswordPath": "/etc/cloudfall/logging/grafana-admin-password", "listenAddress": "127.0.0.1", "port": 3000}, "id": "operations", "metrics": {"backend": {"listenAddress": "127.0.0.1", "port": 9090, "retentionHours": 360, "storagePath": "/var/lib/prometheus-cloudfall"}, "collection": {"intervalSeconds": 60}}, "migration": {"mode": "parallel", "preserveLegacyAgents": true}, "software": {"alloyPackageVersion": "1.17.1-1", "grafanaPackageVersion": "13.1.0", "lokiPackageVersion": "3.7.2"}}, "cloudfall_logging_collector": {"alerting": {"alertmanager": {"listenAddress": "127.0.0.1", "port": 9093}, "receivers": [{"id": "operations-webhook", "webhook": {"url": "https://hooks.example.internal/cloudfall"}}, {"email": {"authPasswordFile": "/etc/cloudfall/logging/smtp-password", "authUsername": "alerts@example.internal", "from": "alerts@example.internal", "smarthost": "smtp.example.internal:587", "to": "oncall@example.internal"}, "id": "operations-email"}]}, "backend": {"listenAddress": "127.0.0.1", "port": 3100, "retentionHours": 720, "server": "h1", "storage": {"path": "/var/lib/loki-cloudfall", "type": "filesystem"}}, "collectors": {"clientTls": {"caPath": "/etc/cloudfall/logging/ca.crt", "certificatePath": "/etc/cloudfall/logging/client.crt", "keyPath": "/etc/cloudfall/logging/client.key"}, "files": [{"application": "crm", "component": "crm-backend", "id": "crm-backend", "path": "/var/log/crm/backend/*.log", "servers": ["h1", "h2"]}], "journal": {"enabled": true, "maxAgeHours": 24}, "servers": ["h1", "h2"]}, "environment": "production", "gateway": {"listenAddress": "0.0.0.0", "port": 3101, "serverName": "logs.example.internal", "tls": {"certificatePath": "/etc/cloudfall/logging/server.crt", "clientCaPath": "/etc/cloudfall/logging/client-ca.crt", "keyPath": "/etc/cloudfall/logging/server.key"}}, "grafana": {"adminPasswordPath": "/etc/cloudfall/logging/grafana-admin-password", "listenAddress": "127.0.0.1", "port": 3000}, "id": "operations", "metrics": {"backend": {"listenAddress": "127.0.0.1", "port": 9090, "retentionHours": 360, "storagePath": "/var/lib/prometheus-cloudfall"}, "collection": {"intervalSeconds": 60}}, "migration": {"mode": "parallel", "preserveLegacyAgents": true}, "software": {"alloyPackageVersion": "1.17.1-1", "grafanaPackageVersion": "13.1.0", "lokiPackageVersion": "3.7.2"}}, "cloudfall_origin_domains": [{"aliases": ["www.crm.example.test"], "edge": {"mode": "dns-only", "provider": "cloudflare"}, "healthCheck": {"expectedStatuses": [200], "path": "/health", "scheme": "https", "timeoutSeconds": 5}, "id": "crm-site", "origin": {"configurationPath": "/etc/nginx/sites-available/crm-origin.conf", "port": 8100, "scheme": "http", "server": "h1", "serverName": "crm.example.test", "service": "nginx.service"}, "primaryName": "crm.example.test", "proxy": {"configurationPath": "/etc/nginx/sites-available/crm.example.test.conf", "server": "h2", "service": "nginx.service", "upstream": {"address": "127.0.0.1", "port": 8100}}, "tls": {"mode": "required"}}], "cloudfall_server_id": "h1", "cloudfall_server_type": {"configuration": {"files": [{"capture": "hash", "group": "root", "mode": "0644", "owner": "root", "path": "/etc/ssh/sshd_config"}]}, "firewall": {"allowedInbound": [{"description": "http", "port": 80, "protocol": "tcp"}, {"description": "https", "port": 443, "protocol": "tcp"}], "policy": "default-deny"}, "id": "debian-application", "os": {"distribution": "Debian", "serviceManager": "systemd", "versions": ["12", "13"]}, "packages": {"forbidden": ["telnetd"], "required": [{"name": "acl"}, {"name": "ca-certificates"}, {"name": "curl"}, {"name": "git"}, {"name": "rsync"}, {"name": "unzip"}]}, "services": {"required": [{"name": "ssh.service", "state": "running", "status": "enabled"}, {"name": "apt-daily.timer", "state": "running", "status": "enabled"}]}, "storage": {"mounts": [{"filesystem": "ext4", "minimumBytes": 850000000000, "path": "/"}], "softwareRaid": {"level": "raid1", "minimumActiveDevices": 2, "minimumUsableBytes": 900000000000}}}, "cloudfall_services": [{"backup": {"directory": "/var/backups/cloudfall/postgresql-main", "onCalendar": "*-*-* 02:00:00 UTC", "restoreCheckOnCalendar": "*-*-* 05:00:00 UTC", "retentionDays": 14}, "bind": {"address": "127.0.0.1", "port": 5432}, "environment": "production", "id": "postgresql-main", "metrics": {"enabled": true}, "postgresql": {"databases": [{"application": "crm", "name": "crm", "owner": "crm"}], "majorVersion": "17"}, "server": "h1", "serviceKind": "postgresql"}, {"backup": {"directory": "/var/backups/cloudfall/redis-cache", "onCalendar": "*-*-* 03:00:00 UTC", "restoreCheckOnCalendar": "*-*-* 05:30:00 UTC", "retentionDays": 7}, "bind": {"address": "127.0.0.1", "port": 6379}, "environment": "production", "id": "redis-cache", "metrics": {"enabled": true}, "redis": {"appendOnly": true, "maxmemoryMb": 256}, "server": "h1", "serviceKind": "redis"}], "cloudfall_ssh_public_keys": [{"algorithm": "ssh-rsa", "environment": "production", "id": "example-admin", "lifecycle": "active", "owner": "example-admin", "publicKey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCqmOqPPapyG6hL5FlR/KOK2ENEvi/2qlhoRtrgK7JfafHXb1i4WJwf5FMGrPE4DBRfDQDwokABdvUPITZOymFMgiGMmFKGnp0xq+/B0wTTOtdnr8GC3sCmSRwuFDrroVWwfjIqyC/wlpoPXpSvyc1VKBuXOe/gLv1tTzDH3Jc+3fUYaUCj+uIVO65n0N2ajNg9ibBsCjiUp9CrWEoprqxR4lz3TaiPuRtMxWQlQ2NKPSTdUZWqg9KVXm3TS8xgf4Ox2dFwedVMcucC8UBBAwetgylXNZMv1yAMYD3QDokLi0AC+XnNR+UhtFXr9Y3VveQfH4YKnv0Fhr2opCyN99Lr example"}]}, "h2": {"ansible_host": "h2.example.internal", "ansible_port": 22, "ansible_user": "cloudfall", "cloudfall_applications": [{"approval": "manual", "id": "crm", "user": "crm"}], "cloudfall_components": [{"application": "crm", "application_user": "crm", "healthCheck": {"attempts": 5, "expectedStatuses": [200], "path": "/health", "port": 8100, "scheme": "http", "timeoutSeconds": 5, "type": "http"}, "id": "crm-backend", "install_root": "/srv/apps/crm/backend", "repository": {"url": "https://github.com/example/crm-backend.git"}, "retainUntilCleanup": true, "runtime": {"packageManager": "uv", "type": "python", "version": "3.14"}, "service": {"command": [".venv/bin/uvicorn", "crm.asgi:application", "--host", "127.0.0.1", "--port", "8100"], "manager": "systemd", "name": "crm-backend"}}], "cloudfall_domains": [{"aliases": ["www.crm.example.test"], "edge": {"mode": "dns-only", "provider": "cloudflare"}, "healthCheck": {"expectedStatuses": [200], "path": "/health", "scheme": "https", "timeoutSeconds": 5}, "id": "crm-site", "origin": {"configurationPath": "/etc/nginx/sites-available/crm-origin.conf", "port": 8100, "scheme": "http", "server": "h1", "serverName": "crm.example.test", "service": "nginx.service"}, "primaryName": "crm.example.test", "proxy": {"configurationPath": "/etc/nginx/sites-available/crm.example.test.conf", "server": "h2", "service": "nginx.service", "upstream": {"address": "127.0.0.1", "port": 8100}}, "tls": {"mode": "required"}}], "cloudfall_environment": "production", "cloudfall_firewall": {"allowedInbound": [{"description": "ssh", "port": 22, "protocol": "tcp"}, {"description": "http", "port": 80, "protocol": "tcp"}, {"description": "https", "port": 443, "protocol": "tcp"}], "policy": "default-deny"}, "cloudfall_hostname": "h2.example.internal", "cloudfall_labels": {"region": "eu-central", "workload": "application"}, "cloudfall_lifecycle": "active", "cloudfall_logging_collector": {"alerting": {"alertmanager": {"listenAddress": "127.0.0.1", "port": 9093}, "receivers": [{"id": "operations-webhook", "webhook": {"url": "https://hooks.example.internal/cloudfall"}}, {"email": {"authPasswordFile": "/etc/cloudfall/logging/smtp-password", "authUsername": "alerts@example.internal", "from": "alerts@example.internal", "smarthost": "smtp.example.internal:587", "to": "oncall@example.internal"}, "id": "operations-email"}]}, "backend": {"listenAddress": "127.0.0.1", "port": 3100, "retentionHours": 720, "server": "h1", "storage": {"path": "/var/lib/loki-cloudfall", "type": "filesystem"}}, "collectors": {"clientTls": {"caPath": "/etc/cloudfall/logging/ca.crt", "certificatePath": "/etc/cloudfall/logging/client.crt", "keyPath": "/etc/cloudfall/logging/client.key"}, "files": [{"application": "crm", "component": "crm-backend", "id": "crm-backend", "path": "/var/log/crm/backend/*.log", "servers": ["h1", "h2"]}], "journal": {"enabled": true, "maxAgeHours": 24}, "servers": ["h1", "h2"]}, "environment": "production", "gateway": {"listenAddress": "0.0.0.0", "port": 3101, "serverName": "logs.example.internal", "tls": {"certificatePath": "/etc/cloudfall/logging/server.crt", "clientCaPath": "/etc/cloudfall/logging/client-ca.crt", "keyPath": "/etc/cloudfall/logging/server.key"}}, "grafana": {"adminPasswordPath": "/etc/cloudfall/logging/grafana-admin-password", "listenAddress": "127.0.0.1", "port": 3000}, "id": "operations", "metrics": {"backend": {"listenAddress": "127.0.0.1", "port": 9090, "retentionHours": 360, "storagePath": "/var/lib/prometheus-cloudfall"}, "collection": {"intervalSeconds": 60}}, "migration": {"mode": "parallel", "preserveLegacyAgents": true}, "software": {"alloyPackageVersion": "1.17.1-1", "grafanaPackageVersion": "13.1.0", "lokiPackageVersion": "3.7.2"}}, "cloudfall_origin_domains": [], "cloudfall_server_id": "h2", "cloudfall_server_type": {"configuration": {"files": [{"capture": "hash", "group": "root", "mode": "0644", "owner": "root", "path": "/etc/ssh/sshd_config"}]}, "firewall": {"allowedInbound": [{"description": "http", "port": 80, "protocol": "tcp"}, {"description": "https", "port": 443, "protocol": "tcp"}], "policy": "default-deny"}, "id": "debian-application", "os": {"distribution": "Debian", "serviceManager": "systemd", "versions": ["12", "13"]}, "packages": {"forbidden": ["telnetd"], "required": [{"name": "acl"}, {"name": "ca-certificates"}, {"name": "curl"}, {"name": "git"}, {"name": "rsync"}, {"name": "unzip"}]}, "services": {"required": [{"name": "ssh.service", "state": "running", "status": "enabled"}, {"name": "apt-daily.timer", "state": "running", "status": "enabled"}]}, "storage": {"mounts": [{"filesystem": "ext4", "minimumBytes": 850000000000, "path": "/"}], "softwareRaid": {"level": "raid1", "minimumActiveDevices": 2, "minimumUsableBytes": 900000000000}}}, "cloudfall_services": [], "cloudfall_ssh_public_keys": [{"algorithm": "ssh-rsa", "environment": "production", "id": "example-admin", "lifecycle": "active", "owner": "example-admin", "publicKey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCqmOqPPapyG6hL5FlR/KOK2ENEvi/2qlhoRtrgK7JfafHXb1i4WJwf5FMGrPE4DBRfDQDwokABdvUPITZOymFMgiGMmFKGnp0xq+/B0wTTOtdnr8GC3sCmSRwuFDrroVWwfjIqyC/wlpoPXpSvyc1VKBuXOe/gLv1tTzDH3Jc+3fUYaUCj+uIVO65n0N2ajNg9ibBsCjiUp9CrWEoprqxR4lz3TaiPuRtMxWQlQ2NKPSTdUZWqg9KVXm3TS8xgf4Ox2dFwedVMcucC8UBBAwetgylXNZMv1yAMYD3QDokLi0AC+XnNR+UhtFXr9Y3VveQfH4YKnv0Fhr2opCyN99Lr example"}]}}}, "ungrouped": {"hosts": {}}}}} diff --git a/engine/tests/golden/examples/h1/nftables.conf b/engine/tests/golden/examples/h1/nftables.conf new file mode 100644 index 0000000..f2e6881 --- /dev/null +++ b/engine/tests/golden/examples/h1/nftables.conf @@ -0,0 +1,27 @@ +#!/usr/sbin/nft -f +# Managed by Cloudfall. Local changes will be overwritten. + +flush ruleset + +table inet cloudfall { + chain input { + type filter hook input priority 0; policy drop; + + iif "lo" accept + ct state established,related accept + ct state invalid drop + ip protocol icmp accept + meta l4proto ipv6-icmp accept + tcp dport 22 accept comment "ssh" + tcp dport 80 accept comment "http" + tcp dport 443 accept comment "https" + } + + chain forward { + type filter hook forward priority 0; policy drop; + } + + chain output { + type filter hook output priority 0; policy accept; + } +} diff --git a/engine/tests/golden/examples/h1/postgresql-main/backup.sh b/engine/tests/golden/examples/h1/postgresql-main/backup.sh new file mode 100644 index 0000000..317b25c --- /dev/null +++ b/engine/tests/golden/examples/h1/postgresql-main/backup.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Managed by Cloudfall. Local changes will be overwritten. +set -euo pipefail + +backup_directory="/var/backups/cloudfall/postgresql-main" +port="5432" +retention_days="14" +timestamp="$(date --utc +%Y%m%dT%H%M%SZ)" + +pg_dump --port "${port}" --format=custom \ + --file "${backup_directory}/crm-${timestamp}.dump" \ + "crm" + +find "${backup_directory}" -name '*.dump' -type f \ + -mtime "+${retention_days}" -delete diff --git a/engine/tests/golden/examples/h1/postgresql-main/cloudfall.conf b/engine/tests/golden/examples/h1/postgresql-main/cloudfall.conf new file mode 100644 index 0000000..667b43f --- /dev/null +++ b/engine/tests/golden/examples/h1/postgresql-main/cloudfall.conf @@ -0,0 +1,3 @@ +# Managed by Cloudfall. Local changes will be overwritten. +listen_addresses = '127.0.0.1' +port = 5432 diff --git a/engine/tests/golden/examples/h1/postgresql-main/restore-check.sh b/engine/tests/golden/examples/h1/postgresql-main/restore-check.sh new file mode 100644 index 0000000..515cb6e --- /dev/null +++ b/engine/tests/golden/examples/h1/postgresql-main/restore-check.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Managed by Cloudfall. Local changes will be overwritten. +# Restores the newest dump of every declared database into a scratch +# database, counts catalog rows, and drops the scratch database again. +set -euo pipefail + +backup_directory="/var/backups/cloudfall/postgresql-main" +port="5432" + +latest="$(find "${backup_directory}" -name 'crm-*.dump' \ + -type f | sort | tail -n 1)" +if [ -z "${latest}" ]; then + echo "no backup found for crm" >&2 + exit 1 +fi +scratch="crm_restore_check" +dropdb --port "${port}" --if-exists "${scratch}" +createdb --port "${port}" "${scratch}" +pg_restore --port "${port}" --dbname "${scratch}" --no-owner \ + --exit-on-error "${latest}" +relations="$(psql --port "${port}" --tuples-only --no-align \ + --command 'SELECT count(*) FROM pg_class' "${scratch}")" +dropdb --port "${port}" "${scratch}" +echo "restore check passed for crm:" \ + "${latest} (${relations} catalog relations)" diff --git a/engine/tests/golden/examples/h2/nftables.conf b/engine/tests/golden/examples/h2/nftables.conf new file mode 100644 index 0000000..f2e6881 --- /dev/null +++ b/engine/tests/golden/examples/h2/nftables.conf @@ -0,0 +1,27 @@ +#!/usr/sbin/nft -f +# Managed by Cloudfall. Local changes will be overwritten. + +flush ruleset + +table inet cloudfall { + chain input { + type filter hook input priority 0; policy drop; + + iif "lo" accept + ct state established,related accept + ct state invalid drop + ip protocol icmp accept + meta l4proto ipv6-icmp accept + tcp dport 22 accept comment "ssh" + tcp dport 80 accept comment "http" + tcp dport 443 accept comment "https" + } + + chain forward { + type filter hook forward priority 0; policy drop; + } + + chain output { + type filter hook output priority 0; policy accept; + } +} diff --git a/engine/tests/test_examples_render_unchanged.py b/engine/tests/test_examples_render_unchanged.py new file mode 100644 index 0000000..0c63a55 --- /dev/null +++ b/engine/tests/test_examples_render_unchanged.py @@ -0,0 +1,103 @@ +"""The example project renders byte-identical to the committed golden files. + +The golden files under ``golden/examples`` were captured from main before +S-006 changed the PostgreSQL role, so any drift in what a project without an +owned database renders shows up here as a diff. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import TYPE_CHECKING, cast + +from cloudfall_engine.cli import main +from jinja2 import Environment, FileSystemLoader, StrictUndefined + +if TYPE_CHECKING: + import pytest + +ROOT = Path(__file__).parents[2] +EXAMPLES = ROOT / "config" / "examples" +SCHEMAS = ROOT / "config" / "schemas" / "v1" +ROLES = ROOT / "engine" / "ansible" / "roles" +GOLDEN = Path(__file__).parent / "golden" / "examples" +POSTGRESQL_TEMPLATES = { + "cloudfall.conf": "cloudfall.conf.j2", + "backup.sh": "backup.sh.j2", + "restore-check.sh": "restore-check.sh.j2", +} + + +def _environment(role: str) -> Environment: + """Jinja with the whitespace rules Ansible's template module applies.""" + return Environment( + loader=FileSystemLoader(ROLES / role / "templates"), + undefined=StrictUndefined, + autoescape=False, # noqa: S701 - configuration files are not HTML. + keep_trailing_newline=True, + trim_blocks=True, + ) + + +def render_examples(output: Path) -> dict[str, str]: + """Render the example inventory and the files S-006-12 names, by name.""" + inventory_path = output / "ansible-inventory.json" + exit_code = main( + [ + "inventory", + "render", + "--project", + str(EXAMPLES), + "--schemas", + str(SCHEMAS), + "--output", + str(inventory_path), + ] + ) + assert exit_code == 0 + inventory_text = inventory_path.read_text(encoding="utf-8") + rendered = {"ansible-inventory.json": inventory_text} + document = cast("dict[str, dict[str, object]]", json.loads(inventory_text)) + servers = cast( + "dict[str, dict[str, dict[str, dict[str, object]]]]", + document["all"]["children"], + )["cloudfall_servers"]["hosts"] + firewall = _environment("cloudfall_firewall").get_template("nftables.conf.j2") + postgresql = _environment("cloudfall_postgresql") + for host, variables in sorted(servers.items()): + if "cloudfall_firewall" in variables: + rendered[f"{host}/nftables.conf"] = firewall.render( + cloudfall_firewall_ruleset=variables["cloudfall_firewall"] + ) + services = cast("list[dict[str, object]]", variables["cloudfall_services"]) + for service in services: + if service["serviceKind"] != "postgresql": + continue + for name, template in POSTGRESQL_TEMPLATES.items(): + rendered[f"{host}/{service['id']}/{name}"] = postgresql.get_template( + template + ).render(cloudfall_postgresql_service=service) + return rendered + + +def _golden() -> dict[str, str]: + return { + path.relative_to(GOLDEN).as_posix(): path.read_text(encoding="utf-8") + for path in sorted(GOLDEN.rglob("*")) + if path.is_file() + } + + +# proves: S-006-12 +def test_s006_12_examples_render_byte_identical( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + rendered = render_examples(tmp_path) + capsys.readouterr() + + golden = _golden() + assert sorted(rendered) == sorted(golden) + assert {"h1/nftables.conf", "h1/postgresql-main/cloudfall.conf"} <= set(golden) + for name, text in rendered.items(): + assert text == golden[name], name diff --git a/engine/tests/test_postgresql_owner_role.py b/engine/tests/test_postgresql_owner_role.py new file mode 100644 index 0000000..f942f88 --- /dev/null +++ b/engine/tests/test_postgresql_owner_role.py @@ -0,0 +1,413 @@ +"""The PostgreSQL role's owner roles, and what an owned database goes through. + +The role needs a Debian host with PostgreSQL, so its decisions are evaluated +with Ansible's own templating, and the password tasks and the data-copy +receipt tasks run for real on localhost with a stand-in for ``psql``. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path +from typing import cast + +import yaml +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar, trust_as_template +from jinja2 import Environment, FileSystemLoader, StrictUndefined + +ROOT = Path(__file__).parents[2] +ROLES = ROOT / "engine" / "ansible" / "roles" +POSTGRESQL = ROLES / "cloudfall_postgresql" +DATA_MIGRATION = ROLES / "cloudfall_data_migration" +PASSWORD = "s3cr3t-billing-value" # noqa: S105 - the value every output is grepped for +BILLING: dict[str, object] = { + "name": "billing", + "owner": "billing", + "passwordRef": { + "environment": "production", + "path": "/billing/postgresql", + "key": "BILLING_PASSWORD", + }, +} +CRM: dict[str, object] = {"name": "crm", "application": "crm", "owner": "crm"} + + +def _service(*databases: dict[str, object]) -> dict[str, object]: + return { + "id": "postgresql-billing", + "serviceKind": "postgresql", + "environment": "production", + "server": "h1", + "bind": {"address": "127.0.0.1", "port": 5432}, + "postgresql": {"majorVersion": "17", "databases": list(databases)}, + "backup": { + "directory": "/var/backups/cloudfall/postgresql-billing", + "onCalendar": "*-*-* 02:00:00 UTC", + "restoreCheckOnCalendar": "*-*-* 05:00:00 UTC", + "retentionDays": 14, + }, + } + + +def _tasks(role: Path, name: str = "main.yml") -> list[dict[str, object]]: + tasks = DataLoader().load_from_file(str(role / "tasks" / name)) + assert isinstance(tasks, list) + return tasks + + +def _task(name: str, tasks: list[dict[str, object]]) -> dict[str, object]: + return next(task for task in tasks if task["name"] == name) + + +def _templar(variables: dict[str, object]) -> Templar: + return Templar(loader=DataLoader(), variables=variables) + + +def _holds(expression: object, variables: dict[str, object]) -> bool: + return bool( + _templar(variables).evaluate_conditional(trust_as_template(str(expression))) + ) + + +def _loop(task: dict[str, object], variables: dict[str, object]) -> object: + return _templar(variables).template(trust_as_template(str(task["loop"]))) + + +def _conditions(task: dict[str, object]) -> list[object]: + value = task.get("when", []) + return value if isinstance(value, list) else [value] + + +def _variables(**extra: object) -> dict[str, object]: + return { + "ansible_check_mode": False, + "cloudfall_postgresql_service": _service(BILLING, CRM), + "cloudfall_postgresql_password_files": {"billing": "/tmp/staged/billing"}, # noqa: S108 + **extra, + } + + +# proves: S-006-5 +def test_s006_5_the_owner_gets_a_login_role_and_its_password() -> None: + tasks = _tasks(POSTGRESQL) + variables = _variables() + + create = _task("Create missing database owner login roles", tasks) + include = _task("Set each owner role password from its staged file", tasks) + + roles = _loop(create, variables) + owners = _loop(include, variables) + + assert roles == ["billing", "crm"] + assert owners == ["billing"] + + +# proves: S-006-5 +def test_s006_5_the_owned_database_is_created_and_kept_owned_by_its_role() -> None: + tasks = _tasks(POSTGRESQL) + create = _task("Create missing application databases", tasks) + give = _task("Give each owned database to its declared owner role", tasks) + variables = _variables( + cloudfall_postgresql_database_owners={ + "stdout_lines": ["billing=postgres", "crm=crm"] + }, + ) + + assert cast("dict[str, list[str]]", create["ansible.builtin.command"])["argv"][ + -1 + ] == 'CREATE DATABASE "{{ item.name }}" OWNER "{{ item.owner }}"' + assert _loop(give, variables) == [BILLING] + assert all( + _holds(condition, {**variables, "item": BILLING}) + for condition in _conditions(give) + ) + owned = {**variables, "item": BILLING} + owned["cloudfall_postgresql_database_owners"] = { + "stdout_lines": ["billing=billing"] + } + assert not all(_holds(condition, owned) for condition in _conditions(give)) + + +# proves: S-006-5 +def test_s006_5_the_password_is_stored_as_scram_from_the_staged_file() -> None: + (block,) = _tasks(POSTGRESQL, "owner_password.yml") + store = _task( + "Store the owner role password as SCRAM-SHA-256", + cast("list[dict[str, object]]", block["block"]), + ) + command = cast("dict[str, str]", store["ansible.builtin.command"]) + stdin = str( + _templar( + { + "cloudfall_postgresql_owner": "billing", + "cloudfall_postgresql_password_path": "/var/lib/postgresql/.pw", + } + ).template(trust_as_template(command["stdin"])) + ) + + assert stdin.splitlines() == [ + "SET password_encryption = 'scram-sha-256';", + r"\set cloudfall_password `cat /var/lib/postgresql/.pw`", + "ALTER ROLE \"billing\" WITH LOGIN PASSWORD :'cloudfall_password';", + ] + assert store["become_user"] == "postgres" + + +def test_a_converge_without_a_staged_password_stops_before_any_change() -> None: + (check, *_) = ( + task + for task in _tasks(POSTGRESQL) + if task["name"] == "Require a staged password for every owner role" + ) + that = cast("dict[str, list[str]]", check["ansible.builtin.assert"])["that"] + + assert all(_holds(condition, _variables()) for condition in that) + unstaged = _variables(cloudfall_postgresql_password_files={}) + assert not all(_holds(condition, unstaged) for condition in that) + application_only = _variables( + cloudfall_postgresql_service=_service(CRM), + cloudfall_postgresql_password_files={}, + ) + assert all(_holds(condition, application_only) for condition in that) + + +# proves: S-006-6 +def test_s006_6_every_task_that_sees_the_password_is_no_log() -> None: + (block,) = _tasks(POSTGRESQL, "owner_password.yml") + tasks = [ + *cast("list[dict[str, object]]", block["block"]), + *cast("list[dict[str, object]]", block["always"]), + ] + + assert len(tasks) == 3 + assert all(task.get("no_log") is True for task in tasks) + + +FAKE_PSQL = """\ +#!{python} +import os, pathlib, re, sys +log = pathlib.Path({log!r}) +script = sys.stdin.read() +staged = re.search(r"`cat (.*)`", script).group(1) +log.joinpath("stdin").write_text(script) +log.joinpath("argv").write_text("\\n".join(sys.argv[1:])) +log.joinpath("value").write_text(pathlib.Path(staged).read_text()) +log.joinpath("mode").write_text(oct(os.stat(staged).st_mode & 0o777)) +""" + + +def _password_play(tmp_path: Path) -> tuple[Path, Path, Path]: + """Write the owner password tasks as a local play with a stand-in psql.""" + log = tmp_path / "psql-log" + log.mkdir() + psql = tmp_path / "psql" + psql.write_text(FAKE_PSQL.format(python=sys.executable, log=str(log)), "utf-8") + psql.chmod(0o700) + staged = tmp_path / "controller" / "postgresql-billing-billing" + staged.parent.mkdir() + staged.write_text(PASSWORD, encoding="utf-8") + host_path = tmp_path / "host" / ".cloudfall-owner-password" + host_path.parent.mkdir() + # Plain YAML, not Ansible's loader: the tasks are written out again. + (block,) = yaml.safe_load( + (POSTGRESQL / "tasks" / "owner_password.yml").read_text(encoding="utf-8") + ) + stage, store = block["block"] + del stage["ansible.builtin.copy"]["owner"], stage["ansible.builtin.copy"]["group"] + store["ansible.builtin.command"]["argv"][0] = str(psql) + del store["become"], store["become_user"] + play = [ + { + "name": "Replay the owner password tasks of cloudfall_postgresql", + "hosts": "localhost", + "connection": "local", + "gather_facts": False, + "vars": { + "cloudfall_postgresql_owner": "billing", + "cloudfall_postgresql_password_files": {"billing": str(staged)}, + "cloudfall_postgresql_password_path": str(host_path), + "cloudfall_postgresql_service": _service(BILLING), + }, + "tasks": [block], + } + ] + playbook = tmp_path / "owner-password.yml" + playbook.write_text(yaml.safe_dump(play), encoding="utf-8") + return playbook, log, host_path + + +# proves: S-006-6 +def test_s006_6_the_password_reaches_psql_through_a_removed_0600_file( + tmp_path: Path, +) -> None: + playbook, log, host_path = _password_play(tmp_path) + binary = shutil.which("ansible-playbook") + assert binary is not None + + done = subprocess.run( # noqa: S603 - fixed binary and arguments. + [binary, "-vvv", "--inventory", "localhost,", str(playbook)], + check=False, + capture_output=True, + text=True, + timeout=300, + ) + + assert done.returncode == 0, done.stdout + done.stderr + assert PASSWORD not in done.stdout + done.stderr + assert (log / "value").read_text(encoding="utf-8") == PASSWORD + assert (log / "mode").read_text(encoding="utf-8") == "0o600" + assert PASSWORD not in (log / "stdin").read_text(encoding="utf-8") + assert PASSWORD not in (log / "argv").read_text(encoding="utf-8") + assert not host_path.exists() + + +def _render(name: str, service: dict[str, object]) -> str: + environment = Environment( + loader=FileSystemLoader(POSTGRESQL / "templates"), + undefined=StrictUndefined, + autoescape=False, # noqa: S701 - configuration files are not HTML. + keep_trailing_newline=True, + trim_blocks=True, + ) + return environment.get_template(name).render(cloudfall_postgresql_service=service) + + +# proves: S-006-13 +def test_s006_13_the_backup_and_the_drill_cover_the_owned_database() -> None: + service = _service(BILLING) + + backup = _render("backup.sh.j2", service) + drill = _render("restore-check.sh.j2", service) + + assert '--file "${backup_directory}/billing-${timestamp}.dump"' in backup + assert "-name 'billing-*.dump'" in drill + assert 'scratch="billing_restore_check"' in drill + assert "restore check passed for billing:" in drill + + +DATA_TASKS = ( + "Validate the migration target service", + "Select the declared target database", + "Require exactly one declared database entry", + "Record the declared database owner", + "Ensure the controller receipt directory exists", + "Write the data-migration receipt on the controller", +) + + +def _data_copy_receipt(tmp_path: Path) -> dict[str, object]: + """Replay the data-copy selection and receipt tasks for the owned database.""" + tasks = yaml.safe_load( + (DATA_MIGRATION / "tasks" / "main.yml").read_text(encoding="utf-8") + ) + named = {task["name"]: task for task in tasks} + restore = next( + task + for task in named["Migrate and verify the declared database"]["block"] + if task["name"] == "Restore the dump into the declared database" + ) + validate, select, require, record, directory, write = ( + named[name] for name in DATA_TASKS + ) + receipts = tmp_path / "data-migrations" + restored_as = { + "name": "Record the role the restore runs as", + "ansible.builtin.copy": { + "content": "{{ " + _role_argument(restore) + " }}", + "dest": str(tmp_path / "restore-role"), + "mode": "0600", + }, + } + counts = { + "name": "Stand in for the source row counts", + "ansible.builtin.set_fact": { + "cloudfall_data_migration_source_counts": { + "stdout_lines": ["public.invoices,3"] + } + }, + } + play = [ + { + "name": "Replay the data-copy receipt of cloudfall_data_migration", + "hosts": "localhost", + "connection": "local", + "gather_facts": True, + "gather_subset": ["min"], + "vars": { + "cloudfall_server_id": "h1", + "cloudfall_data_migration_service": _service(BILLING), + "cloudfall_data_migration_database": "billing", + "cloudfall_data_migration_receipt_directory": str(receipts), + }, + "tasks": [ + validate, + select, + require, + record, + restored_as, + counts, + directory, + write, + ], + } + ] + playbook = tmp_path / "data-copy.yml" + playbook.write_text(yaml.safe_dump(play), encoding="utf-8") + binary = shutil.which("ansible-playbook") + assert binary is not None + done = subprocess.run( # noqa: S603 - fixed binary and arguments. + [binary, "--inventory", "localhost,", str(playbook)], + check=False, + capture_output=True, + text=True, + timeout=300, + ) + assert done.returncode == 0, done.stdout + done.stderr + receipt = json.loads( + (receipts / "postgresql-billing-billing.json").read_text(encoding="utf-8") + ) + assert isinstance(receipt, dict) + return receipt + + +def _role_argument(restore: dict[str, object]) -> str: + argv = cast("dict[str, list[str]]", restore["ansible.builtin.command"])["argv"] + role = argv[argv.index("--role") + 1] + return role.removeprefix("{{").removesuffix("}}").strip() + + +# proves: S-006-13 +def test_s006_13_data_copy_restores_as_the_owner_and_receipts_it( + tmp_path: Path, +) -> None: + receipt = _data_copy_receipt(tmp_path) + + assert (tmp_path / "restore-role").read_text(encoding="utf-8").strip() == "billing" + spec = receipt["spec"] + assert isinstance(spec, dict) + assert spec["owner"] == "billing" + assert spec["database"] == "billing" + assert "application" not in spec + assert PASSWORD not in json.dumps(receipt) + + +def test_the_role_argument_spec_takes_an_owned_database() -> None: + specs = yaml.safe_load( + (POSTGRESQL / "meta" / "argument_specs.yml").read_text(encoding="utf-8") + ) + options = specs["argument_specs"]["main"]["options"] + database = options["cloudfall_postgresql_service"]["options"]["postgresql"][ + "options" + ]["databases"]["options"] + + assert "required" not in database["application"] + assert database["passwordRef"]["options"]["key"]["required"] is True + # Remote access is #123's: until it converges, the spec refuses it. + assert "remoteAccess" not in options["cloudfall_postgresql_service"]["options"][ + "postgresql" + ]["options"] + assert options["cloudfall_postgresql_password_files"]["type"] == "dict" diff --git a/engine/tests/test_services_passwords.py b/engine/tests/test_services_passwords.py new file mode 100644 index 0000000..016ce7d --- /dev/null +++ b/engine/tests/test_services_passwords.py @@ -0,0 +1,279 @@ +"""``cloudfall-engine playbook run services.yml`` with an owned database (S-006).""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any + +import pytest +import yaml +from cloudfall.inventory import PlatformInventory +from cloudfall.validation import validate_config +from cloudfall_engine.ansible_inventory import render_ansible_inventory +from cloudfall_engine.cli import main + +ROOT = Path(__file__).parents[2] +EXAMPLES = ROOT / "config" / "examples" +SCHEMAS = ROOT / "config" / "schemas" / "v1" +ENGINE = ROOT / "engine" +PASSWORD = "s3cr3t-billing-value" # noqa: S105 - the value every output is grepped for +PASSWORD_REF = { + "environment": "production", + "path": "/billing/postgresql", + "key": "BILLING_PASSWORD", +} + + +def _inventory(tmp_path: Path) -> Path: + """Render the owned service of S-006-1, its host reached locally.""" + project = tmp_path / "project" + for kind in ("servers", "server-types", "services"): + (project / kind).mkdir(parents=True) + shutil.copy(EXAMPLES / "servers" / "h1.yaml", project / "servers" / "h1.yaml") + shutil.copy( + EXAMPLES / "server-types" / "debian-application.yaml", + project / "server-types" / "debian-application.yaml", + ) + service = { + "apiVersion": "cloudfall/v1", + "kind": "Service", + "metadata": {"id": "postgresql-billing"}, + "spec": { + "serviceKind": "postgresql", + "environment": "production", + "server": "h1", + "bind": {"address": "127.0.0.1", "port": 5432}, + "postgresql": { + "majorVersion": "17", + "databases": [ + {"name": "billing", "owner": "billing", "passwordRef": PASSWORD_REF} + ], + }, + "backup": { + "directory": "/var/backups/cloudfall/postgresql-billing", + "onCalendar": "*-*-* 02:00:00 UTC", + "retentionDays": 14, + }, + }, + } + (project / "services" / "postgresql-billing.yaml").write_text( + yaml.safe_dump(service), encoding="utf-8" + ) + rendered: dict[str, Any] = render_ansible_inventory( + PlatformInventory.from_state(validate_config(project, SCHEMAS)) + ) + host = rendered["all"]["children"]["cloudfall_servers"]["hosts"]["h1"] + host.update( + { + "ansible_connection": "local", + "ansible_become": False, + "ansible_host": "localhost", + "ansible_python_interpreter": sys.executable, + } + ) + path = tmp_path / "ansible-inventory.json" + path.write_text(json.dumps(rendered), encoding="utf-8") + return path + + +def _run(argv: list[str], capsys: pytest.CaptureFixture[str]) -> tuple[int, Any]: + exit_code = main(argv) + return exit_code, json.loads(capsys.readouterr().out) + + +# proves: S-006-7 +def test_s006_7_a_missing_secret_file_stops_before_ansible_starts( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + inventory = _inventory(tmp_path) + secrets = tmp_path / "secrets" + + exit_code, payload = _run( + [ + "playbook", + "run", + "services", + "--engine", + str(ENGINE), + "--inventory", + str(inventory), + "--secrets-dir", + str(secrets), + ], + capsys, + ) + + assert exit_code != 0 + assert payload["error"]["code"] == "PRECONDITION" + message = payload["error"]["message"] + assert str(secrets / "production" / "billing" / "postgresql.env") in message + assert "BILLING_PASSWORD" in message + + +# proves: S-006-7 +def test_s006_7_a_missing_key_stops_before_ansible_starts(tmp_path: Path) -> None: + inventory = _inventory(tmp_path) + secrets = tmp_path / "secrets" + source = secrets / "production" / "billing" / "postgresql.env" + source.parent.mkdir(parents=True) + source.write_text("OTHER=x\n", encoding="utf-8") + fake = _fake_sops(tmp_path) + + done = _engine(inventory, secrets, fake) + + payload = json.loads(done.stdout) + assert payload["error"]["code"] == "PRECONDITION", done.stdout + assert "BILLING_PASSWORD is missing from" in payload["error"]["message"] + assert str(source) in payload["error"]["message"] + assert "PLAY [" not in done.stderr + + +def test_other_playbooks_and_syntax_checks_read_no_secrets( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + inventory = _inventory(tmp_path) + + exit_code, payload = _run( + [ + "playbook", + "run", + "services", + "--engine", + str(ENGINE), + "--inventory", + str(inventory), + "--secrets-dir", + str(tmp_path / "absent"), + "--syntax-check", + ], + capsys, + ) + + assert exit_code == 0, payload + assert payload["data"]["owned_roles"] == [] + + +@pytest.mark.parametrize( + ("name", "content"), + [ + ("hosts.ini", "[web]\nlocalhost ansible_connection=local\n"), + ("hosts.json", '{"all": {"hosts": {"localhost": {}}}}\n'), + ], + ids=["ini", "json-of-another-shape"], +) +def test_a_plain_inventory_runs_services_as_before_without_secrets( + tmp_path: Path, capsys: pytest.CaptureFixture[str], name: str, content: str +) -> None: + inventory = tmp_path / name + inventory.write_text(content, encoding="utf-8") + + exit_code, payload = _run( + [ + "playbook", + "run", + "services.yml", + "--engine", + str(ENGINE), + "--inventory", + str(inventory), + "--secrets-dir", + str(tmp_path / "absent"), + ], + capsys, + ) + + # No cloudfall_servers host: Ansible matches none and exits 0, as on main. + assert exit_code == 0, payload + assert payload["data"]["effect"] == "updated" + assert payload["data"]["owned_roles"] == [] + + +FAKE_SOPS = """\ +#!{python} +import pathlib, sys +assert sys.argv[1] == "--decrypt" +sys.stdout.write(pathlib.Path(sys.argv[2]).read_text()) +""" + + +def _fake_sops(tmp_path: Path) -> Path: + """Write a ``sops`` stand-in for the engine child's PATH: the files are plain.""" + directory = tmp_path / "bin" + directory.mkdir() + sops = directory / "sops" + sops.write_text(FAKE_SOPS.format(python=sys.executable), encoding="utf-8") + sops.chmod(0o700) + return directory + + +def _engine( + inventory: Path, secrets: Path, fake: Path, *extra: str +) -> subprocess.CompletedProcess[str]: + environment = { + **os.environ, + "PATH": os.pathsep.join((str(fake), os.environ["PATH"])), + } + return subprocess.run( # noqa: S603 - fixed interpreter and arguments. + [ + sys.executable, + "-m", + "cloudfall_engine", + "playbook", + "run", + "services", + "--engine", + str(ENGINE), + "--inventory", + str(inventory), + "--secrets-dir", + str(secrets), + *extra, + ], + env=environment, + capture_output=True, + text=True, + timeout=300, + check=False, + ) + + +# proves: S-006-6 +def test_s006_6_the_answer_names_the_reference_key_and_hash_only( + tmp_path: Path, +) -> None: + inventory = _inventory(tmp_path) + secrets = tmp_path / "secrets" + source = secrets / "production" / "billing" / "postgresql.env" + source.parent.mkdir(parents=True) + source.write_text(f"BILLING_PASSWORD={PASSWORD}\n", encoding="utf-8") + fake = _fake_sops(tmp_path) + + # A tag no task carries: Ansible gathers facts and changes nothing. + done = _engine(inventory, secrets, fake, "--tags", "s006-no-task", "-vvv") + + assert done.returncode == 0, done.stdout + done.stderr + payload = json.loads(done.stdout) + assert payload["data"]["owned_roles"] == [ + { + "service": "postgresql-billing", + "owner": "billing", + "password_ref": PASSWORD_REF, + "key": "BILLING_PASSWORD", + "sha256": hashlib.sha256(PASSWORD.encode()).hexdigest(), + } + ] + staged = re.search( + r"cloudfall-passwords-[^/]+/postgresql-billing-billing", done.stderr + ) + assert staged is not None, done.stderr + assert not any(Path(tempfile.gettempdir()).glob(staged.group(0))) + assert PASSWORD not in done.stdout + done.stderr + assert PASSWORD not in inventory.read_text(encoding="utf-8") diff --git a/sdk/src/cloudfall/agent_tools.py b/sdk/src/cloudfall/agent_tools.py index 4c85cbb..8307fbd 100644 --- a/sdk/src/cloudfall/agent_tools.py +++ b/sdk/src/cloudfall/agent_tools.py @@ -8,12 +8,17 @@ from dataclasses import dataclass from pathlib import Path +from typing import TYPE_CHECKING from cloudfall.lifecycle import ( EngineContext, StepRunner, subprocess_step, ) +from cloudfall.secrets import SopsSecretProvider + +if TYPE_CHECKING: + from collections.abc import Callable @dataclass(frozen=True, slots=True) @@ -40,6 +45,14 @@ class AgentConfig: gateway_key_path: Path | None = None run: StepRunner = subprocess_step """Starts each engine step: ``subprocess`` here, ``ctx.run`` under treaty.""" + decrypt: Callable[[Path], str] | None = None + """Decrypts one sops file: ``sops`` here, through ``ctx.run`` under treaty.""" + + def secret_provider(self) -> SopsSecretProvider: + """Return the reader of the secrets directory owner passwords come from.""" + return SopsSecretProvider( + secrets_directory=self.secrets_directory, decrypt=self.decrypt + ) def context(self) -> EngineContext: """Return the engine execution context shared by mutating tools.""" diff --git a/sdk/src/cloudfall/app.py b/sdk/src/cloudfall/app.py index d9c1213..9d8f92a 100644 --- a/sdk/src/cloudfall/app.py +++ b/sdk/src/cloudfall/app.py @@ -171,6 +171,7 @@ backup_service, build_release_artifact, converge_domains, + converge_services, deploy, health, migrate_data, @@ -202,6 +203,7 @@ AlertFeed, ApproveOptions, DriftCheck, + Executor, OperatorError, PassKind, ProposalStatus, @@ -2690,12 +2692,33 @@ class OperatorEngineArgs(ProposalArgs): default=None, description="Alerts endpoint (default: derived from the declared gateway)", ) + secrets_dir: Path = Flag( + default=Path("secrets"), + description=( + "sops-encrypted secrets directory owner role passwords are read " + "from (default: secrets)" + ), + secret=False, + ) def __post_init__(self) -> None: """Keep relative paths inside the project.""" ProposalArgs.__post_init__(self) _inside_project(self.inventory_file, "inventory-file") _inside_project(self.observed, "observed") + _inside_project(self.secrets_dir, "secrets-dir") + + def executor( + self, fleet: Fleet, ctx: Ctx + ) -> Executor: + """Return the runner of a proposal's operation, reading owner passwords.""" + return engine_executor( + self.context(fleet, ctx), + SopsSecretProvider( + secrets_directory=fleet.path(self.secrets_dir), + decrypt=_sops_decrypt(ctx), + ), + ) def context(self, fleet: Fleet, ctx: Ctx) -> EngineContext: """Return the engine contract, with steps run through ``ctx.run``.""" @@ -2857,7 +2880,7 @@ def operator_approve( proposal = approve_proposal( store, args.proposal, - engine_executor(args.context(fleet, ctx)), + args.executor(fleet, ctx), verifier, ApproveOptions(verify_timeout_seconds=args.verify_timeout), via=ApprovalChannel(attestation.channel), @@ -2972,7 +2995,7 @@ def operator_run( autonomous_pass, store, fleet.inventory, - engine_executor(args.context(fleet, ctx)), + args.executor(fleet, ctx), args.verifier_for(fleet, ctx, lambda: feed), ) drift = None @@ -3067,6 +3090,14 @@ class MigrateArgs(EngineArgs): default=False, description="Discard the persisted plan and start again", ) + secrets_dir: Path = Flag( + default=Path("secrets"), + description=( + "sops-encrypted secrets directory owner role passwords are read " + "from (default: secrets)" + ), + secret=False, + ) def __post_init__(self) -> None: """Refuse a malformed pair, and keep relative paths in the project.""" @@ -3078,6 +3109,7 @@ def __post_init__(self) -> None: "releases", "artifacts", "plan_file", + "secrets_dir", ): _inside_project(getattr(self, flag), flag.replace("_", "-")) for option, entries in ( @@ -3121,7 +3153,9 @@ def config(self, fleet: Fleet, ctx: Ctx) -> AgentConfig: deployments_directory=fleet.path(self.deployments), releases_directory=fleet.path(self.releases), artifacts_directory=fleet.path(self.artifacts), + secrets_directory=fleet.path(self.secrets_dir), run=_step_runner(ctx, fleet.directory), + decrypt=_sops_decrypt(ctx), ) @@ -4143,10 +4177,63 @@ class Converged: playbook: str +@dataclass(frozen=True, slots=True) +class ServicesConverged: + """``converge_services``: the playbook run and each owner password it set. + + ``owned_roles`` names each role's ``passwordRef``, its key, and the + value's SHA-256; never the value. + """ + + status: str + action: str + playbook: str + owned_roles: list[dict[str, object]] = Out( + default_factory=list, ordered=True, high_entropy=False + ) + + _ENGINE_EXIT_CODES = ("PRECONDITION", "ENGINE_STEP_FAILED") _COMPONENT_PATTERN = r"^[a-z][a-z0-9]*(-[a-z0-9]+)*$" +def _converge_services_handler( + args: McpServeArgs, root: ServerRoot, context: Callable[[Ctx], EngineContext] +) -> Callable[[Mapping[str, object], Ctx], ServicesConverged]: + """Return ``converge_services``: owner passwords from ``--secrets-dir`` first.""" + + def converge_declared_services( + arguments: Mapping[str, object], call: Ctx + ) -> ServicesConverged: + del arguments # the tool takes none + provider = SopsSecretProvider( + secrets_directory=root.path(args.secrets_dir), + decrypt=_sops_decrypt(call), + ) + try: + passwords = converge_services(context(call), provider) + except LifecycleError as error: + raise _lifecycle_failed(error) from error + except ConfigValidationError as error: + raise _config_invalid(error) from error + return ServicesConverged( + status="ok", + action="converge-services", + playbook="services.yml", + owned_roles=[ + cast("dict[str, object]", snake_case_keys(password.as_dict())) + for password in passwords + ], + ) + + converge_declared_services.__annotations__ = { + "arguments": Mapping[str, object], + "call": Ctx, + "return": ServicesConverged, + } + return converge_declared_services + + def _engine_tools(args: McpServeArgs, root: ServerRoot, ctx: Ctx) -> list[McpTool]: """Return the engine work no ``cloudfall`` command covers, as tools.""" del ctx # each call brings its own @@ -4254,9 +4341,9 @@ def converge_domain_routes( "every declared infrastructure service" ), input_schema=no_inputs, - handler=converge("converge-services", "services.yml", lambda _: {}), + handler=_converge_services_handler(args, root, context), danger_level="destructive", - exit_codes=_ENGINE_EXIT_CODES, + exit_codes=(*_ENGINE_EXIT_CODES, "CONFIG_INVALID"), ), McpTool( name="converge_domains", diff --git a/sdk/src/cloudfall/lifecycle.py b/sdk/src/cloudfall/lifecycle.py index 76dddbb..7a3c039 100644 --- a/sdk/src/cloudfall/lifecycle.py +++ b/sdk/src/cloudfall/lifecycle.py @@ -21,6 +21,12 @@ from jsonschema.exceptions import ValidationError from cloudfall.inventory import PlatformInventory +from cloudfall.secrets import ( + SecretsError, + owned_databases, + owned_role_passwords, + staged_password_files, +) from cloudfall.validation import SchemaCatalog, validate_config if TYPE_CHECKING: @@ -33,6 +39,7 @@ DomainInventory, ServiceInventory, ) + from cloudfall.secrets import OwnedRolePassword, SopsSecretProvider _ARTIFACT_SCHEMA = "artifact.schema.json" _OUTPUT_TAIL_CHARACTERS = 2000 @@ -448,6 +455,47 @@ def converge_domains(context: EngineContext, run: DomainsRun) -> None: run_engine_playbook(context, "domains.yml", extra_vars, limit=proxy_server) +ERROR_SERVICES_SECRET = "lifecycle_services_secret_unavailable" # noqa: S105 - a code +"""An owner role's password could not be read; no playbook ran.""" + + +def services_passwords( + context: EngineContext, provider: SopsSecretProvider +) -> tuple[OwnedRolePassword, ...]: + """Decrypt every owner role's password before ``services.yml`` may run. + + A missing file or key fails here, naming the expected path and key, so + the converge stops before any playbook starts. + """ + state = validate_config(context.project_directory, context.schema_directory) + inventory = PlatformInventory.from_state(state) + try: + return owned_role_passwords(owned_databases(inventory), provider) + except SecretsError as error: + raise LifecycleError(ERROR_SERVICES_SECRET, error.message) from error + + +def run_services_playbook( + context: EngineContext, passwords: tuple[OwnedRolePassword, ...] +) -> None: + """Run ``services.yml`` with each owner password staged in a 0600 file. + + The files live in a private temp dir outside the project, removed when + the run ends, whether it passed or failed. + """ + with staged_password_files(passwords) as extra_vars: + run_engine_playbook(context, "services.yml", extra_vars) + + +def converge_services( + context: EngineContext, provider: SopsSecretProvider +) -> tuple[OwnedRolePassword, ...]: + """Converge every declared service; return the passwords it set, by hash.""" + passwords = services_passwords(context, provider) + run_services_playbook(context, passwords) + return passwords + + def build_release_artifact( context: EngineContext, component_id: ResourceId, diff --git a/sdk/src/cloudfall/migrate.py b/sdk/src/cloudfall/migrate.py index 665fb9d..cc42e5e 100644 --- a/sdk/src/cloudfall/migrate.py +++ b/sdk/src/cloudfall/migrate.py @@ -34,6 +34,8 @@ deploy, migrate_data, run_engine_playbook, + run_services_playbook, + services_passwords, ) from cloudfall.observation import load_observations from cloudfall.operations import ( @@ -58,6 +60,7 @@ from cloudfall.agent_tools import AgentConfig from cloudfall.inventory import DomainInventory, ServerInventory + from cloudfall.secrets import OwnedRolePassword Runner = Callable[[], dict[str, object]] @@ -154,10 +157,22 @@ def execute_migration( return _envelope("plan", steps) releases = _recover_releases(steps, options) + # An owner password that can't be read stops the run before any step, + # so no playbook starts (S-006-7). + passwords = ( + _services_passwords(config) + if any( + step.step_id == "services" and step.status is StepStatus.PENDING + for step in steps + ) + else () + ) resolved_runners = ( runners if runners is not None - else _build_runners(config, options, inventory, releases, steps) + else _build_runners( + config, options, inventory, releases, steps, passwords + ) ) for step in steps: if step.status is StepStatus.COMPLETED: @@ -405,17 +420,18 @@ def _recover_releases( return releases -def _build_runners( +def _build_runners( # noqa: PLR0913 - the run's whole contract. config: AgentConfig, options: MigrateOptions, inventory: PlatformInventory, releases: dict[str, str], steps: list[PlanStep], + passwords: tuple[OwnedRolePassword, ...], ) -> dict[str, Runner]: context = config.context() runners: dict[str, Runner] = { "baseline": lambda: _playbook(config, "baseline.yml", {}), - "services": lambda: _playbook(config, "services.yml", {}), + "services": lambda: _services(config, passwords), "ttl-lower": lambda: _lower_ttl(inventory), "parallel-run": lambda: _parallel_run(inventory), "rollback-window": lambda: _rollback_window(steps), @@ -536,6 +552,23 @@ def _playbook( return {"playbook": playbook_name} +def _services_passwords(config: AgentConfig) -> tuple[OwnedRolePassword, ...]: + try: + return services_passwords(config.context(), config.secret_provider()) + except LifecycleError as error: + raise MigrateError(error.code, error.detail) from error + + +def _services( + config: AgentConfig, passwords: tuple[OwnedRolePassword, ...] +) -> dict[str, object]: + run_services_playbook(config.context(), passwords) + detail: dict[str, object] = {"playbook": "services.yml"} + if passwords: + detail["ownedRoles"] = [password.as_dict() for password in passwords] + return detail + + def _domain_vars(config: AgentConfig) -> dict[str, object]: return { "cloudfall_domains_receipt_directory": str( diff --git a/sdk/src/cloudfall/operator.py b/sdk/src/cloudfall/operator.py index a5c60da..2b3128d 100644 --- a/sdk/src/cloudfall/operator.py +++ b/sdk/src/cloudfall/operator.py @@ -23,8 +23,13 @@ from cloudfall.audit import AuditStatus, audit_inventory from cloudfall.decision import ApprovalChannel from cloudfall.domain import AlertSeverity, ResourceId -from cloudfall.lifecycle import LifecycleError, run_engine_playbook +from cloudfall.lifecycle import ( + LifecycleError, + converge_services, + run_engine_playbook, +) from cloudfall.observation import load_observations +from cloudfall.secrets import OwnedRoleRecord if TYPE_CHECKING: from collections.abc import Callable, Iterator, Mapping @@ -33,8 +38,12 @@ from cloudfall.audit import AuditReport from cloudfall.inventory import OperatorPolicyInventory, PlatformInventory from cloudfall.lifecycle import EngineContext + from cloudfall.secrets import SopsSecretProvider from cloudfall.validation import SchemaCatalog +type Executor = Callable[[OperatorProposal], tuple[OwnedRoleRecord, ...] | None] +"""Runs one proposal's operation; returns the owner passwords it set, if any.""" + _PROPOSAL_SCHEMA = "operator-proposal.schema.json" _FINGERPRINT_LENGTH = 16 ERROR_FEED_UNREACHABLE = "operator_feed_unreachable" @@ -184,15 +193,20 @@ class ProposalOutcome: verified_at: str | None result: str detail: str + owned_roles: tuple[OwnedRoleRecord, ...] = () + """Owner role passwords the run set: reference, key and hash, never a value.""" def as_dict(self) -> dict[str, object]: """Serialize the outcome for the proposal receipt.""" - return { + result: dict[str, object] = { "executedAt": self.executed_at, "verifiedAt": self.verified_at, "result": self.result, "detail": self.detail, } + if self.owned_roles: + result["ownedRoles"] = [role.as_dict() for role in self.owned_roles] + return result @dataclass(frozen=True, slots=True) @@ -609,6 +623,10 @@ def _proposal_from_document(document: Mapping[str, object]) -> OperatorProposal: verified_at=cast("str | None", raw_outcome["verifiedAt"]), result=cast("str", raw_outcome["result"]), detail=cast("str", raw_outcome["detail"]), + owned_roles=tuple( + OwnedRoleRecord.from_dict(cast("Mapping[str, object]", role)) + for role in cast("list[object]", raw_outcome.get("ownedRoles", [])) + ), ) raw_service = operation.get("service") return OperatorProposal( @@ -738,10 +756,15 @@ def _open_count(store: ProposalStore) -> int: def engine_executor( context: EngineContext, -) -> Callable[[OperatorProposal], None]: - """Executor running the proposal's engine entry point.""" + secrets: SopsSecretProvider, +) -> Executor: + """Executor running the proposal's engine entry point. - def _execute(proposal: OperatorProposal) -> None: + ``converge-services`` reads every owner role's password from ``secrets`` + first, so a missing one fails before any playbook runs. + """ + + def _execute(proposal: OperatorProposal) -> tuple[OwnedRoleRecord, ...]: playbook = _OPERATION_PLAYBOOKS.get(proposal.operation_kind) if playbook is None: message = ( @@ -749,7 +772,11 @@ def _execute(proposal: OperatorProposal) -> None: f"{proposal.operation_kind.value}" ) raise OperatorError(_ERROR_OPERATION_UNSUPPORTED, message) + if proposal.operation_kind is OperationKind.CONVERGE_SERVICES: + passwords = converge_services(context, secrets) + return tuple(password.record() for password in passwords) run_engine_playbook(context, playbook, {}) + return () return _execute @@ -832,7 +859,7 @@ class ApproveOptions: def approve( # noqa: PLR0913 - the run's contract plus how the approval came. store: ProposalStore, proposal_id: ResourceId, - executor: Callable[[OperatorProposal], None], + executor: Executor, verifier: Callable[[OperatorProposal], bool], options: ApproveOptions | None = None, *, @@ -883,7 +910,7 @@ def _execution_failure_detail(error: OperatorError | LifecycleError) -> str: def _execute_and_verify( # noqa: PLR0913 - internal execution contract. store: ProposalStore, proposal: OperatorProposal, - executor: Callable[[OperatorProposal], None], + executor: Executor, verifier: Callable[[OperatorProposal], bool], approval: ApprovalRecord, options: ApproveOptions | None = None, @@ -891,7 +918,7 @@ def _execute_and_verify( # noqa: PLR0913 - internal execution contract. resolved_options = options if options is not None else ApproveOptions() executed_at = _utc_now() try: - executor(proposal) + owned_roles = executor(proposal) or () except (OperatorError, LifecycleError) as error: failed = replace( proposal, @@ -915,6 +942,7 @@ def _execute_and_verify( # noqa: PLR0913 - internal execution contract. verified_at=None, result="failed", detail="executed, verification pending", + owned_roles=owned_roles, ), ) store.update(executed) @@ -932,6 +960,7 @@ def _execute_and_verify( # noqa: PLR0913 - internal execution contract. "trigger evidence resolved after convergence; " "observed outcome matches the declared state" ), + owned_roles=owned_roles, ), ) store.update(verified) @@ -949,6 +978,7 @@ def _execute_and_verify( # noqa: PLR0913 - internal execution contract. f"{int(resolved_options.verify_timeout_seconds)}s; " "remediation did not restore the declared state" ), + owned_roles=owned_roles, ), ) store.update(failed) @@ -1049,12 +1079,16 @@ class AutonomyReport: executed: tuple[tuple[str, str], ...] withheld: tuple[tuple[str, str], ...] + owned_roles: Mapping[str, tuple[OwnedRoleRecord, ...]] = field( + default_factory=dict + ) + """Owner role passwords each executed proposal set, by proposal id.""" def as_dict(self) -> dict[str, object]: """Serialize for structured run output.""" return { "executed": [ - {"proposal": proposal_id, "status": status} + _executed_entry(proposal_id, status, self.owned_roles.get(proposal_id)) for proposal_id, status in self.executed ], "withheld": [ @@ -1067,7 +1101,7 @@ def as_dict(self) -> dict[str, object]: def autonomous_pass( # noqa: PLR0913 - boundary mirrors approve(). store: ProposalStore, inventory: PlatformInventory, - executor: Callable[[OperatorProposal], None], + executor: Executor, verifier_for: Callable[ [OperatorProposal], Callable[[OperatorProposal], bool] ], @@ -1084,6 +1118,7 @@ def autonomous_pass( # noqa: PLR0913 - boundary mirrors approve(). } executed: list[tuple[str, str]] = [] withheld: list[tuple[str, str]] = [] + owned_roles: dict[str, tuple[OwnedRoleRecord, ...]] = {} for proposal in store.list(): if proposal.status is not ProposalStatus.PROPOSED: continue @@ -1114,7 +1149,20 @@ def autonomous_pass( # noqa: PLR0913 - boundary mirrors approve(). options, ) executed.append((result.resource_id.value, result.status.value)) - return AutonomyReport(executed=tuple(executed), withheld=tuple(withheld)) + if result.outcome is not None and result.outcome.owned_roles: + owned_roles[result.resource_id.value] = result.outcome.owned_roles + return AutonomyReport( + executed=tuple(executed), withheld=tuple(withheld), owned_roles=owned_roles + ) + + +def _executed_entry( + proposal_id: str, status: str, owned_roles: tuple[OwnedRoleRecord, ...] | None +) -> dict[str, object]: + entry: dict[str, object] = {"proposal": proposal_id, "status": status} + if owned_roles: + entry["ownedRoles"] = [role.as_dict() for role in owned_roles] + return entry def gateway_feed( diff --git a/sdk/src/cloudfall/secrets.py b/sdk/src/cloudfall/secrets.py index e37ca62..5f93d0e 100644 --- a/sdk/src/cloudfall/secrets.py +++ b/sdk/src/cloudfall/secrets.py @@ -15,22 +15,35 @@ import re import shutil import subprocess -from dataclasses import dataclass +import tempfile +from contextlib import contextmanager +from dataclasses import dataclass, field from datetime import UTC, datetime +from pathlib import Path from typing import TYPE_CHECKING, cast -from cloudfall.inventory import PlatformInventory +from cloudfall.domain import ( + AbsolutePath, + PostgresRoleName, + ResourceId, + SecretKeyName, +) +from cloudfall.inventory import ( + OwnedPostgresDatabase, + PlatformInventory, + SecretValueReference, +) from cloudfall.validation import SchemaCatalog, validate_config if TYPE_CHECKING: - from collections.abc import Callable - from pathlib import Path + from collections.abc import Callable, Iterable, Iterator, Mapping - from cloudfall.domain import ResourceId from cloudfall.inventory import ComponentInventory, SecretReference from cloudfall.lifecycle import EngineContext ERROR_SOPS_MISSING = "secrets_sops_missing" +ERROR_KEY_MISSING = "secrets_key_missing" +_ERROR_OWNER_CONFLICT = "secrets_owner_conflict" _ERROR_SOURCE_MISSING = "secrets_source_missing" _ERROR_SOURCE_INVALID = "secrets_source_invalid" _ERROR_COMPONENT_UNKNOWN = "secrets_component_unknown" @@ -89,7 +102,9 @@ def fetch( ) return _parse_dotenv(text, source) - def source_path(self, reference: SecretReference) -> Path: + def source_path( + self, reference: SecretReference | SecretValueReference + ) -> Path: """Return the encrypted file backing one reference.""" relative = reference.path.value.lstrip("/") return ( @@ -98,6 +113,193 @@ def source_path(self, reference: SecretReference) -> Path: / f"{relative}.env" ) + def fetch_value(self, reference: SecretValueReference) -> str: + """Decrypt one reference's fragment and return its one key's value. + + A missing file, a missing key, or an empty value fails naming the + expected path and key, never a value. + """ + source = self.source_path(reference) + key = reference.key.value + if not source.is_file(): + message = ( + f"secret source does not exist: expected {source} holding key " + f"{key}" + ) + raise SecretsError(_ERROR_SOURCE_MISSING, message) + text = ( + self.decrypt(source) + if self.decrypt is not None + else _sops_decrypt(source) + ) + values = [value for name, value in _parse_dotenv(text, source) if name == key] + if not values: + message = f"secret key {key} is missing from {source}" + raise SecretsError(ERROR_KEY_MISSING, message) + if len(values) > 1: + message = f"secret key {key} appears more than once in {source}" + raise SecretsError(_ERROR_SOURCE_INVALID, message) + if not values[0]: + message = f"secret key {key} in {source} holds an empty value" + raise SecretsError(ERROR_KEY_MISSING, message) + return values[0] + + +@dataclass(frozen=True, slots=True) +class OwnedRolePassword: + """The decrypted password of one owner role, held only in memory. + + ``value`` stays out of ``repr`` and out of ``as_dict``: what leaves this + object is the reference and the value's SHA-256. + """ + + service: ResourceId + owner: PostgresRoleName + reference: SecretValueReference + value: str = field(repr=False) + + @property + def sha256(self) -> str: + """Return the hex SHA-256 of the value, for receipts and answers.""" + return hashlib.sha256(self.value.encode("utf-8")).hexdigest() + + def record(self) -> OwnedRoleRecord: + """Return what may be recorded of this password: never the value.""" + return OwnedRoleRecord( + service=self.service, + owner=self.owner, + reference=self.reference, + sha256=self.sha256, + ) + + def as_dict(self) -> dict[str, object]: + """Serialize the reference, the key, and the hash; never the value.""" + return self.record().as_dict() + + +_SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") + + +@dataclass(frozen=True, slots=True) +class OwnedRoleRecord: + """One owner role password a converge set, as answers and receipts name it.""" + + service: ResourceId + owner: PostgresRoleName + reference: SecretValueReference + sha256: str + + def __post_init__(self) -> None: + """Hold a hex SHA-256 digest only.""" + if not _SHA256_PATTERN.fullmatch(self.sha256): + message = f"not a hex SHA-256 digest: {self.sha256!r}" + raise ValueError(message) + + def as_dict(self) -> dict[str, object]: + """Serialize the reference, the key, and the hash.""" + return { + "service": self.service.value, + "owner": self.owner.value, + "passwordRef": self.reference.as_dict(), + "key": self.reference.key.value, + "sha256": self.sha256, + } + + @classmethod + def from_dict(cls, content: Mapping[str, object]) -> OwnedRoleRecord: + """Read one record back from a schema-validated receipt.""" + reference = content.get("passwordRef") + if not isinstance(reference, dict): + message = "owned role record has no passwordRef mapping" + raise TypeError(message) + return cls( + service=ResourceId.from_boundary(content.get("service")), + owner=PostgresRoleName.from_boundary(content.get("owner")), + reference=SecretValueReference( + environment=ResourceId.from_boundary(reference.get("environment")), + path=AbsolutePath.from_boundary(reference.get("path")), + key=SecretKeyName.from_boundary(reference.get("key")), + ), + sha256=str(content.get("sha256")), + ) + + +def owned_databases( + inventory: PlatformInventory, +) -> tuple[tuple[ResourceId, OwnedPostgresDatabase], ...]: + """Return every database owned by a declared role, with its service.""" + return tuple( + (service.resource_id, database) + for service in inventory.services + if service.postgresql is not None + for database in service.postgresql.databases + if isinstance(database, OwnedPostgresDatabase) + ) + + +def owned_role_passwords( + databases: Iterable[tuple[ResourceId, OwnedPostgresDatabase]], + provider: SopsSecretProvider, +) -> tuple[OwnedRolePassword, ...]: + """Decrypt the password of every owner role, once per service and role. + + One role named by two databases of a service must name one reference. + """ + passwords: dict[tuple[ResourceId, PostgresRoleName], OwnedRolePassword] = {} + for service_id, database in databases: + slot = (service_id, database.owner) + known = passwords.get(slot) + if known is not None: + if known.reference != database.password_ref: + message = ( + f"role {database.owner.value} of service {service_id.value} " + "names two passwordRefs; every database it owns must name " + "the same one" + ) + raise SecretsError(_ERROR_OWNER_CONFLICT, message) + continue + passwords[slot] = OwnedRolePassword( + service=service_id, + owner=database.owner, + reference=database.password_ref, + value=provider.fetch_value(database.password_ref), + ) + return tuple(passwords.values()) + + +PASSWORD_FILES_VARIABLE = "cloudfall_services_password_files" # noqa: S105 - a name +"""The ``services.yml`` variable mapping service, then owner, to a staged file.""" + + +@contextmanager +def staged_password_files( + passwords: tuple[OwnedRolePassword, ...], +) -> Iterator[dict[str, object]]: + """Write each password to a 0600 file in a private temp dir, then remove it. + + Yields the extra vars ``services.yml`` reads: the controller paths only, + so the values never cross a command line, the inventory, or the project. + No passwords yields no variable, leaving the run as it was. + """ + if not passwords: + yield {} + return + with tempfile.TemporaryDirectory(prefix="cloudfall-passwords-") as directory: + root = Path(directory) + root.chmod(0o700) + files: dict[str, dict[str, str]] = {} + for password in passwords: + path = root / f"{password.service.value}-{password.owner.value}" + descriptor = os.open( + path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, _FILE_MODE + ) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(password.value) + files.setdefault(password.service.value, {})[password.owner.value] = str( + path + ) + yield {PASSWORD_FILES_VARIABLE: files} + def render_environment( context: EngineContext, diff --git a/sdk/tests/test_owned_postgresql_converge.py b/sdk/tests/test_owned_postgresql_converge.py new file mode 100644 index 0000000..51e09b0 --- /dev/null +++ b/sdk/tests/test_owned_postgresql_converge.py @@ -0,0 +1,537 @@ +"""Converging a PostgreSQL database owned by a declared role (S-006). + +``services.yml`` needs real hosts, so these tests run every path that starts +it (the lifecycle call the MCP tool, the operator, and ``cloudfall migrate`` +share) with a recording step runner: it reads the staged password file the +way the role's copy task does, and the tests grep every output for the value. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import shutil +import stat +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import cast + +import pytest +import yaml +from cloudfall.agent_tools import AgentConfig +from cloudfall.cli import main +from cloudfall.decision import ApprovalChannel +from cloudfall.domain import ResourceId +from cloudfall.inventory import PlatformInventory +from cloudfall.lifecycle import ( + ERROR_SERVICES_SECRET, + EngineContext, + ExecutionStep, + LifecycleError, + converge_services, +) +from cloudfall.migrate import MigrateError, MigrateOptions, execute_migration +from cloudfall.operator import ( + AutonomyReport, + OperationKind, + ProposalStatus, + ProposalStore, + approve, + engine_executor, + propose_for_drift, +) +from cloudfall.secrets import ( + PASSWORD_FILES_VARIABLE, + OwnedRoleRecord, + SopsSecretProvider, +) +from cloudfall.validation import ( + ConfigValidationError, + SchemaCatalog, + validate_config, +) +from cloudfall_engine.ansible_inventory import render_ansible_inventory +from mcp import ClientSession, StdioServerParameters, stdio_client +from mcp.types import TextContent + +ROOT = Path(__file__).parents[2] +EXAMPLES = ROOT / "config" / "examples" +SCHEMAS = ROOT / "config" / "schemas" / "v1" +ENGINE = ROOT / "engine" +PASSWORD = "s3cr3t-billing-value" # noqa: S105 - the value every output is grepped for +PASSWORD_SHA256 = hashlib.sha256(PASSWORD.encode()).hexdigest() +PASSWORD_REF = { + "environment": "production", + "path": "/billing/postgresql", + "key": "BILLING_PASSWORD", +} +OWNED_ROLE: dict[str, object] = { + "service": "postgresql-billing", + "owner": "billing", + "passwordRef": PASSWORD_REF, + "key": "BILLING_PASSWORD", + "sha256": PASSWORD_SHA256, +} + + +def _project(tmp_path: Path) -> Path: + """One Server, its server type, and the owned PostgreSQL service of S-006-1.""" + project = tmp_path / "project" + for kind in ("servers", "server-types", "services"): + (project / kind).mkdir(parents=True) + shutil.copy(EXAMPLES / "servers" / "h1.yaml", project / "servers" / "h1.yaml") + shutil.copy( + EXAMPLES / "server-types" / "debian-application.yaml", + project / "server-types" / "debian-application.yaml", + ) + service = { + "apiVersion": "cloudfall/v1", + "kind": "Service", + "metadata": {"id": "postgresql-billing"}, + "spec": { + "serviceKind": "postgresql", + "environment": "production", + "server": "h1", + "bind": {"address": "127.0.0.1", "port": 5432}, + "postgresql": { + "majorVersion": "17", + "databases": [ + {"name": "billing", "owner": "billing", "passwordRef": PASSWORD_REF} + ], + }, + "backup": { + "directory": "/var/backups/cloudfall/postgresql-billing", + "onCalendar": "*-*-* 02:00:00 UTC", + "retentionDays": 14, + }, + }, + } + (project / "services" / "postgresql-billing.yaml").write_text( + yaml.safe_dump(service, sort_keys=False), encoding="utf-8" + ) + return project + + +def _secrets(project: Path, content: str) -> Path: + secrets = project / "secrets" + source = secrets / "production" / "billing" / "postgresql.env" + source.parent.mkdir(parents=True) + source.write_text(content, encoding="utf-8") + return secrets + + +def _plaintext(source: Path) -> str: + """Stand in for sops: the test's secret files are not encrypted.""" + return source.read_text(encoding="utf-8") + + +@dataclass +class Recorder: + """Steps a converge started, and what its staged files held mid-run.""" + + steps: list[ExecutionStep] = field(default_factory=list) + staged: dict[str, tuple[str, int, int]] = field(default_factory=dict) + paths: list[Path] = field(default_factory=list) + + def __call__(self, step: ExecutionStep) -> Finished: + """Record the step and read every password file it was handed.""" + self.steps.append(step) + if "--extra-vars" in step.argv: + extra = json.loads(step.argv[step.argv.index("--extra-vars") + 1]) + files = cast( + "dict[str, dict[str, str]]", extra.get(PASSWORD_FILES_VARIABLE, {}) + ) + for service, owners in files.items(): + for owner, raw in owners.items(): + path = Path(raw) + self.paths.append(path) + self.staged[f"{service}/{owner}"] = ( + path.read_text(encoding="utf-8"), + stat.S_IMODE(path.stat().st_mode), + stat.S_IMODE(path.parent.stat().st_mode), + ) + return Finished() + + +@dataclass(frozen=True) +class Finished: + """A step that exited 0 and printed nothing.""" + + returncode: int = 0 + stdout: str = "" + stderr: str = "" + + +def _context(project: Path, recorder: Recorder) -> EngineContext: + return EngineContext( + project_directory=project, + schema_directory=SCHEMAS, + engine_directory=ENGINE, + inventory_file=project / "tmp" / "ansible-inventory.json", + run=recorder, + ) + + +def _provider(secrets: Path) -> SopsSecretProvider: + return SopsSecretProvider(secrets_directory=secrets, decrypt=_plaintext) + + +def _everything_written(project: Path) -> str: + return "\n".join( + path.read_text(encoding="utf-8", errors="replace") + for path in project.rglob("*") + if path.is_file() and "secrets" not in path.parts + ) + + +# proves: S-006-6 +def test_s006_6_converge_stages_the_password_and_answers_its_hash( + tmp_path: Path, +) -> None: + project = _project(tmp_path) + secrets = _secrets(project, f"OTHER=x\nBILLING_PASSWORD={PASSWORD}\n") + recorder = Recorder() + + passwords = converge_services(_context(project, recorder), _provider(secrets)) + + render, playbook = recorder.steps + assert playbook.argv[-1].endswith("services.yml") + assert recorder.staged == { + "postgresql-billing/billing": (PASSWORD, 0o600, 0o700) + } + (staged,) = recorder.paths + assert not staged.exists() + assert not staged.parent.exists() + assert project not in staged.parents + (answer,) = [password.as_dict() for password in passwords] + assert answer == { + "service": "postgresql-billing", + "owner": "billing", + "passwordRef": PASSWORD_REF, + "key": "BILLING_PASSWORD", + "sha256": PASSWORD_SHA256, + } + assert PASSWORD not in repr(passwords) + for step in (render, playbook): + assert PASSWORD not in " ".join(step.argv) + assert PASSWORD not in json.dumps(dict(step.environment)) + assert PASSWORD not in _everything_written(project) + + +# proves: S-006-6 +def test_s006_6_the_rendered_inventory_carries_the_reference_only( + tmp_path: Path, +) -> None: + project = _project(tmp_path) + _secrets(project, f"BILLING_PASSWORD={PASSWORD}\n") + + inventory = render_ansible_inventory( + PlatformInventory.from_state(validate_config(project, SCHEMAS)) + ) + + rendered = json.dumps(inventory) + assert PASSWORD not in rendered + assert '"passwordRef": {"environment": "production"' in rendered + + +# proves: S-006-7 +@pytest.mark.parametrize( + ("content", "expected"), + [ + (None, "production/billing/postgresql.env"), + ("OTHER=x\n", "BILLING_PASSWORD is missing from"), + ], + ids=["missing-file", "missing-key"], +) +def test_s006_7_a_missing_file_or_key_stops_before_any_playbook( + tmp_path: Path, content: str | None, expected: str +) -> None: + project = _project(tmp_path) + secrets = ( + project / "secrets" if content is None else _secrets(project, content) + ) + recorder = Recorder() + + with pytest.raises(LifecycleError) as error: + converge_services(_context(project, recorder), _provider(secrets)) + + assert error.value.code == ERROR_SERVICES_SECRET + assert expected in error.value.detail + assert "BILLING_PASSWORD" in error.value.detail + assert str(secrets / "production" / "billing" / "postgresql.env") in ( + error.value.detail + ) + assert recorder.steps == [] + + +def test_an_empty_password_is_refused_like_a_missing_key(tmp_path: Path) -> None: + project = _project(tmp_path) + secrets = _secrets(project, "BILLING_PASSWORD=\n") + recorder = Recorder() + + with pytest.raises(LifecycleError, match="holds an empty value"): + converge_services(_context(project, recorder), _provider(secrets)) + + assert recorder.steps == [] + + +def test_a_project_without_an_owned_database_converges_as_before( + tmp_path: Path, +) -> None: + recorder = Recorder() + context = EngineContext( + project_directory=EXAMPLES, + schema_directory=SCHEMAS, + engine_directory=ENGINE, + inventory_file=tmp_path / "inventory.json", + run=recorder, + ) + + passwords = converge_services(context, _provider(tmp_path / "no-secrets")) + + assert passwords == () + _, playbook = recorder.steps + assert json.loads(playbook.argv[playbook.argv.index("--extra-vars") + 1]) == {} + + +# proves: S-006-7 +def test_s006_7_the_operator_marks_the_proposal_failed_and_runs_nothing( + tmp_path: Path, +) -> None: + project = _project(tmp_path) + recorder = Recorder() + execute = engine_executor( + _context(project, recorder), _provider(project / "secrets") + ) + proposal = propose_for_drift( + ResourceId("h1"), + ("services.bind[postgresql-billing]",), + OperationKind.CONVERGE_SERVICES, + "2026-10-27T10:00:00Z", + ) + + with pytest.raises(LifecycleError) as error: + execute(proposal) + + assert error.value.code == ERROR_SERVICES_SECRET + assert "BILLING_PASSWORD" in error.value.detail + assert recorder.steps == [] + assert proposal.status is ProposalStatus.PROPOSED + + +# proves: S-006-6 +def test_s006_6_operator_approve_answers_and_receipts_the_hash_only( + tmp_path: Path, +) -> None: + project = _project(tmp_path) + secrets = _secrets(project, f"BILLING_PASSWORD={PASSWORD}\n") + recorder = Recorder() + store = ProposalStore( + directory=project / "tmp" / "operator" / "proposals", + catalog=SchemaCatalog(SCHEMAS), + ) + proposal = propose_for_drift( + ResourceId("h1"), + ("services.bind[postgresql-billing]",), + OperationKind.CONVERGE_SERVICES, + "2026-10-27T10:00:00Z", + ) + store.save(proposal) + + approved = approve( + store, + proposal.resource_id, + engine_executor(_context(project, recorder), _provider(secrets)), + lambda _: True, + via=ApprovalChannel.TERMINAL, + ) + + assert recorder.staged["postgresql-billing/billing"][0] == PASSWORD + answer = approved.as_document() + outcome = cast("dict[str, dict[str, object]]", answer["spec"])["outcome"] + assert outcome["ownedRoles"] == [OWNED_ROLE] + receipt = (store.directory / f"{proposal.resource_id.value}.json").read_text( + encoding="utf-8" + ) + assert json.loads(receipt)["spec"]["outcome"]["ownedRoles"] == [OWNED_ROLE] + assert store.load(proposal.resource_id) == approved + assert PASSWORD not in json.dumps(answer) + assert PASSWORD not in receipt + assert all(PASSWORD not in " ".join(step.argv) for step in recorder.steps) + assert PASSWORD not in _everything_written(project) + + +# proves: S-006-6 +def test_s006_6_operator_run_names_the_hash_of_each_executed_proposal() -> None: + record = OwnedRoleRecord.from_dict(OWNED_ROLE) + + report = AutonomyReport( + executed=(("op-a", "verified"), ("op-b", "verified")), + withheld=(), + owned_roles={"op-a": (record,)}, + ) + + assert report.as_dict()["executed"] == [ + {"proposal": "op-a", "status": "verified", "ownedRoles": [OWNED_ROLE]}, + {"proposal": "op-b", "status": "verified"}, + ] + + +def _agent_config(project: Path, recorder: Recorder, secrets: Path) -> AgentConfig: + return AgentConfig( + project_directory=project, + schema_directory=SCHEMAS, + engine_directory=ENGINE, + inventory_file=project / "tmp" / "inventory.json", + observed_directory=project / "tmp" / "observed", + service_observed_directory=project / "tmp" / "observed-services", + deployments_directory=project / "tmp" / "deployments", + releases_directory=project / "tmp" / "releases", + artifacts_directory=project / "tmp" / "artifacts", + secrets_directory=secrets, + run=recorder, + decrypt=_plaintext, + ) + + +def _migrate_options(project: Path) -> MigrateOptions: + return MigrateOptions( + plan_file=project / "tmp" / "plan.json", + builds={}, + releases={}, + environment_files={}, + execute=True, + ) + + +# proves: S-006-7 +def test_s006_7_migrate_stops_before_the_baseline_playbook(tmp_path: Path) -> None: + project = _project(tmp_path) + recorder = Recorder() + config = _agent_config(project, recorder, project / "secrets") + + with pytest.raises(MigrateError) as error: + execute_migration(config, _migrate_options(project)) + + assert error.value.code == ERROR_SERVICES_SECRET + assert "BILLING_PASSWORD" in error.value.detail + assert recorder.steps == [] + + +# proves: S-006-6 +def test_s006_6_migrate_records_the_hash_in_its_services_step(tmp_path: Path) -> None: + project = _project(tmp_path) + secrets = _secrets(project, f"BILLING_PASSWORD={PASSWORD}\n") + recorder = Recorder() + + # Every playbook "passes"; the run stops at the audit, which reads + # snapshots no stand-in wrote, after the services step was saved. + with pytest.raises(ConfigValidationError, match="observation directory"): + execute_migration( + _agent_config(project, recorder, secrets), _migrate_options(project) + ) + + plan = (project / "tmp" / "plan.json").read_text(encoding="utf-8") + services = next( + step + for step in cast("list[dict[str, object]]", json.loads(plan)["steps"]) + if step["id"] == "services" + ) + assert services["status"] == "completed" + assert services["detail"] == { + "playbook": "services.yml", + "ownedRoles": [ + { + "service": "postgresql-billing", + "owner": "billing", + "passwordRef": PASSWORD_REF, + "key": "BILLING_PASSWORD", + "sha256": PASSWORD_SHA256, + } + ], + } + assert recorder.staged["postgresql-billing/billing"][0] == PASSWORD + assert PASSWORD not in plan + assert all(PASSWORD not in " ".join(step.argv) for step in recorder.steps) + + +# proves: S-006-7 +def test_s006_7_cli_migrate_answers_precondition_naming_path_and_key( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + project = _project(tmp_path) + + exit_code = main( + [ + "migrate", + "--project", + str(project), + "--schemas", + str(SCHEMAS), + "--engine", + str(ENGINE), + "--secrets-dir", + "private", + "--yes", + ] + ) + + payload = json.loads(capsys.readouterr().out) + assert payload["error"]["code"] == "PRECONDITION", payload + assert exit_code != 0 + message = payload["error"]["message"] + assert str(project / "private" / "production" / "billing" / "postgresql.env") in ( + message + ) + assert "BILLING_PASSWORD" in message + assert not (project / "tmp" / "plan.json").exists() + + +async def _converge_over_mcp(project: Path) -> dict[str, object]: + server = StdioServerParameters( + command=sys.executable, + args=[ + "-c", + "from cloudfall.cli import run; run()", + "mcp", + "serve", + "--project", + str(project), + "--secrets-dir", + "private", + ], + cwd=str(project), + ) + async with ( + stdio_client(server) as (read, write), + ClientSession(read, write) as session, + ): + await session.initialize() + result = await session.call_tool( + "converge_services", {"confirm_destructive": True} + ) + content = result.content[0] + assert isinstance(content, TextContent) + answer = json.loads(content.text) + assert isinstance(answer, dict) + return answer + + +# proves: S-006-7 +def test_s006_7_the_converge_services_tool_answers_precondition( + tmp_path: Path, +) -> None: + project = _project(tmp_path) + + answer = asyncio.run(_converge_over_mcp(project)) + + error = cast("dict[str, object]", answer["error"]) + assert error["code"] == "PRECONDITION", answer + message = str(error["message"]) + assert str(project / "private" / "production" / "billing" / "postgresql.env") in ( + message + ) + assert "BILLING_PASSWORD" in message + assert not (project / "tmp" / "ansible-inventory.json").exists()