From 22dbc3e9fe4eb61a214c3666edd6ff8785ed2fa8 Mon Sep 17 00:00:00 2001 From: "shipmill-romamo[bot]" <338557544+shipmill-romamo[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:19:56 +0200 Subject: [PATCH] Gate a deploy on the command health check and receipt it (S-009-2..9, 11, 12) A component's healthCheck of type: command now runs in the deploy role's health gate, after the switch and the restart and inside the block whose rescue restores the previous release. It runs in a transient systemd unit cloudfall-health- as the application user, in /current, with the unit's EnvironmentFile=- line and RuntimeMaxSec= set to timeoutSeconds, up to attempts times 3 seconds apart. The run task is no_log and the unit has no --pipe, so the command's output stays in the server's journal; only the exit code and systemd-run's timeout line are read. A failed check rolls back (or stops a first release), writes the release receipt with healthCheckPassed: false and the check's result, and fails the deploy naming the check and its exit code or timeout. The receipt tasks move to release_receipt.yml so the pass path and the rescue write the same receipt. The receipt schema gains healthCheck (no other key) and accepts healthCheckPassed: false only with a failed command check; receipts written before still validate. http and none gates and receipts are unchanged. restart, rollback and health keep refusing a command check until #166. Fixes #164 Co-Authored-By: Claude Opus 5.5 --- changelog.d/164-command-health-gate.md | 3 + config/README.md | 4 +- config/schemas/v1/release-receipt.schema.json | 105 +- docs/ci-deploy-guide.md | 10 +- engine/README.md | 21 +- .../roles/cloudfall_deploy/defaults/main.yml | 3 +- .../tasks/command_health_check.yml | 116 +++ .../roles/cloudfall_deploy/tasks/health.yml | 4 +- .../roles/cloudfall_deploy/tasks/main.yml | 106 +- .../tasks/release_receipt.yml | 62 ++ .../roles/cloudfall_deploy/tasks/restart.yml | 4 +- .../roles/cloudfall_deploy/tasks/rollback.yml | 4 +- engine/tests/test_command_health_gate.py | 910 ++++++++++++++++++ engine/tests/test_deploy_role.py | 12 +- engine/tests/test_release_command.py | 5 +- 15 files changed, 1285 insertions(+), 84 deletions(-) create mode 100644 changelog.d/164-command-health-gate.md create mode 100644 engine/ansible/roles/cloudfall_deploy/tasks/command_health_check.yml create mode 100644 engine/ansible/roles/cloudfall_deploy/tasks/release_receipt.yml create mode 100644 engine/tests/test_command_health_gate.py diff --git a/changelog.d/164-command-health-gate.md b/changelog.d/164-command-health-gate.md new file mode 100644 index 0000000..f86286e --- /dev/null +++ b/changelog.d/164-command-health-gate.md @@ -0,0 +1,3 @@ +### Added + +- Deploy: a component's `healthCheck` of `type: command` now gates `deploy --yes`. After the switch and the restart it runs on each server in a transient systemd unit `cloudfall-health-`, as the application's user, in `/current`, with the service's env file and `timeoutSeconds` as its runtime limit, up to `attempts` times 3 seconds apart. When every attempt fails or times out, the previous release is restored (a first release is stopped), the deploy fails with `ENGINE_STEP_FAILED` naming the check and its exit code or timeout, and the release receipt is written with `healthCheckPassed: false`. Receipts of a command check carry `healthCheck` (command, attempts, exit code, timed out), never the command's output, which stays in the server's journal. The release receipt schema accepts `healthCheckPassed: false` only with a failed command `healthCheck`, and every receipt written before. `restart`, `rollback` and `health` still refuse a command check until #166 (#164) diff --git a/config/README.md b/config/README.md index 4b604fc..585b7a0 100644 --- a/config/README.md +++ b/config/README.md @@ -44,7 +44,9 @@ and public HTTP evidence. `deployment-receipt.schema.json` validates receipts written only after an Ansible proxy run has applied and validated its final configuration. `artifact.schema.json` validates build metadata for hashed release tarballs, and `release-receipt.schema.json` validates receipts written -only after a component release has passed its health gate. Together with server observations, these inputs derive planned, +after a component release has passed its health gate, or failed a command +health check (`healthCheckPassed: false`, with the check's attempts, exit code +and timeout under `healthCheck`, never its output). Together with server observations, these inputs derive planned, ready-to-deploy, deployed, configured, and healthy service states. `examples/` contains a complete valid desired-state set. `tests/invalid/` diff --git a/config/schemas/v1/release-receipt.schema.json b/config/schemas/v1/release-receipt.schema.json index 308150c..44d08d3 100644 --- a/config/schemas/v1/release-receipt.schema.json +++ b/config/schemas/v1/release-receipt.schema.json @@ -41,7 +41,68 @@ "pattern": "^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{7,40}$" }, "healthCheckPassed": { - "const": true + "type": "boolean" + }, + "healthCheck": { + "type": "object", + "properties": { + "type": { + "const": "command" + }, + "command": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1 + }, + "attempts": { + "type": "integer", + "minimum": 1, + "maximum": 20 + }, + "exitCode": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "maximum": 255 + }, + "timedOut": { + "type": "boolean" + } + }, + "required": [ + "type", + "command", + "attempts", + "exitCode", + "timedOut" + ], + "additionalProperties": false, + "if": { + "properties": { + "timedOut": { + "const": true + } + } + }, + "then": { + "properties": { + "exitCode": { + "type": "null" + } + } + }, + "else": { + "properties": { + "exitCode": { + "type": "integer" + } + } + } }, "releaseCommand": { "type": "object", @@ -91,7 +152,47 @@ "previousRelease", "healthCheckPassed" ], - "additionalProperties": false + "additionalProperties": false, + "if": { + "properties": { + "healthCheckPassed": { + "const": false + } + }, + "required": [ + "healthCheckPassed" + ] + }, + "then": { + "properties": { + "healthCheck": { + "properties": { + "exitCode": { + "not": { + "const": 0 + } + } + } + } + }, + "required": [ + "healthCheck" + ] + }, + "else": { + "properties": { + "healthCheck": { + "properties": { + "exitCode": { + "const": 0 + }, + "timedOut": { + "const": false + } + } + } + } + } } }, "required": [ diff --git a/docs/ci-deploy-guide.md b/docs/ci-deploy-guide.md index 329c141..ec619a7 100644 --- a/docs/ci-deploy-guide.md +++ b/docs/ci-deploy-guide.md @@ -264,7 +264,11 @@ What each step does: - **Show the plan.** `deploy` without `--yes` validates the release and prints which servers it would change, and changes nothing - **Deploy.** Ships the release behind the component's health check, and - writes the release receipt `tmp/releases/.json` + writes the release receipt `tmp/releases/.json`. A smoke check + declared as a `type: command` health check runs on each server after its + restart, so it gates the deploy and needs no CI step of its own: when it + fails, the server goes back to the previous release, the deploy fails, and + the receipt records the failure with `healthCheckPassed: false` - **Remove the decrypted files.** GitHub-hosted runners are thrown away after the job; on a self-hosted runner this step is what keeps the values off its disk @@ -346,7 +350,9 @@ step keeps, for 90 days: - `tmp/artifacts//.json`: the artifact metadata, with the commit and the archive's SHA-256 -- `tmp/releases/.json`: the release receipt from `deploy` +- `tmp/releases/.json`: the release receipt from `deploy`; after + a failed command health check its `previousRelease` names the release that + serves again (`null` when a first release was stopped) - `tmp/env-receipts/.json`: the key names and hash of the rendered environment file, never the values diff --git a/engine/README.md b/engine/README.md index 2946d50..4963a06 100644 --- a/engine/README.md +++ b/engine/README.md @@ -244,7 +244,26 @@ running. The kept copy stays after a successful deploy and is never overwritten; a later hand-made unit that differs from it refuses the deploy. Check mode only reports the unit it would keep. Successful runs write a schema-valid release receipt on the -controller. The first slice supports Python components managed by uv on +controller. + +A `healthCheck` of `type: command` (a smoke check, say) is the gate's last +step: after the switch and the restart, the command runs in a transient +systemd unit, `cloudfall-health-`, as the application's user, in +`/current` (a relative first element of `command` resolves +there), with the service unit's `EnvironmentFile=-` line, and is stopped +after `timeoutSeconds`. It runs up to `attempts` times, 3 seconds apart, and +passes at the first exit 0. Its stdout and stderr stay in the server's +journal (`journalctl -u cloudfall-health-`) and never reach the +receipt or the Ansible output. When every attempt fails, the rescue above +restores the previous release (or stops a first one), writes the receipt with +`healthCheckPassed: false`, the last exit code or the timeout, and +`previousRelease` naming the release that serves again, and fails the run. +The receipt of a passing command check records it under `healthCheck` too; +`http` and `none` checks keep their receipts and their gate as before. +`restart`, `rollback` and `health` still refuse a command check until +cloudfall-dev/cloudfall#166 runs it there. + +The first slice supports Python components managed by uv on x86_64 Debian hosts. ## Public domain routes diff --git a/engine/ansible/roles/cloudfall_deploy/defaults/main.yml b/engine/ansible/roles/cloudfall_deploy/defaults/main.yml index 6e070a2..289eba9 100644 --- a/engine/ansible/roles/cloudfall_deploy/defaults/main.yml +++ b/engine/ansible/roles/cloudfall_deploy/defaults/main.yml @@ -4,7 +4,8 @@ cloudfall_deploy_receipt_directory: "" # A component without an HTTP health check passes its gate when its unit is # active right after the restart and still active this many seconds later. cloudfall_deploy_unit_settle_seconds: 10 -# A release command runs in a transient unit started by this binary. +# A release command and a command health check run in a transient unit +# started by this binary. cloudfall_deploy_systemd_run: /usr/bin/systemd-run # The receipt keeps this much of the release command's output; a failure # message keeps less, so it stays inside the end of the Ansible log the diff --git a/engine/ansible/roles/cloudfall_deploy/tasks/command_health_check.yml b/engine/ansible/roles/cloudfall_deploy/tasks/command_health_check.yml new file mode 100644 index 0000000..0534c47 --- /dev/null +++ b/engine/ansible/roles/cloudfall_deploy/tasks/command_health_check.yml @@ -0,0 +1,116 @@ +--- +# Runs the component's command health check against the release current +# points at, in a transient systemd unit with the service's user, working +# directory and EnvironmentFile=, up to `attempts` times 3 seconds apart. +# Paths come from install_root, so every entry point can include this file. +# +# The command's stdout and stderr stay in the server's journal under the +# transient unit's name (no --pipe), and the run task is no_log: only its +# exit code and systemd-run's own timeout line are read. The result lands in +# cloudfall_deploy_command_health_result, then a failed check fails the task +# that follows, so a block's rescue can still read what happened. +- name: Compute the command health check invocation + ansible.builtin.set_fact: + cloudfall_deploy_command_health_current: >- + {{ cloudfall_deploy_component.install_root }}/current + cloudfall_deploy_command_health_executable: >- + {{ + cloudfall_deploy_component.healthCheck.command[0] + if cloudfall_deploy_component.healthCheck.command[0].startswith('/') + else cloudfall_deploy_component.install_root ~ '/current/' + ~ cloudfall_deploy_component.healthCheck.command[0] + }} + cloudfall_deploy_command_health_attempts: >- + {{ cloudfall_deploy_component.healthCheck.attempts | int }} + +- name: Run the command health check + ansible.builtin.command: + argv: >- + {{ + [ + cloudfall_deploy_systemd_run, + '--unit=cloudfall-health-' ~ cloudfall_deploy_component.id, + '--wait', + '--collect', + '--service-type=exec', + '--uid=' ~ cloudfall_deploy_component.application_user, + '--gid=' ~ cloudfall_deploy_component.application_user, + '--working-directory=' ~ cloudfall_deploy_command_health_current, + '--property=EnvironmentFile=-' + ~ cloudfall_deploy_component.install_root ~ '/shared/env/' + ~ cloudfall_deploy_component.id ~ '.env', + '--property=RuntimeMaxSec=' + ~ cloudfall_deploy_component.healthCheck.timeoutSeconds, + '--property=TimeoutStopSec=10', + '--property=NoNewPrivileges=true', + '--property=PrivateTmp=true', + '--', + cloudfall_deploy_command_health_executable, + ] + + cloudfall_deploy_component.healthCheck.command[1:] + }} + register: cloudfall_deploy_command_health_run + changed_when: false + failed_when: false + # The last attempt ends the loop itself, so running out of attempts is not + # a task failure and the result below is still recorded. + retries: "{{ cloudfall_deploy_command_health_attempts | int - 1 }}" + delay: 3 + until: >- + ( + cloudfall_deploy_command_health_run.rc == 0 + and 'Finished with result: timeout' + not in cloudfall_deploy_command_health_run.stderr + ) + or cloudfall_deploy_command_health_run.attempts | default(1) | int + >= cloudfall_deploy_command_health_attempts | int + no_log: true + +- name: Record the command health check result + ansible.builtin.set_fact: + cloudfall_deploy_command_health_result: >- + {{ + { + 'type': 'command', + 'command': cloudfall_deploy_component.healthCheck.command, + 'attempts': cloudfall_deploy_command_health_run.attempts + | default(1) | int, + 'exitCode': ( + none + if cloudfall_deploy_command_health_timed_out + else cloudfall_deploy_command_health_run.rc | int + ), + 'timedOut': cloudfall_deploy_command_health_timed_out, + } + }} + vars: + cloudfall_deploy_command_health_timed_out: >- + {{ + 'Finished with result: timeout' + in cloudfall_deploy_command_health_run.stderr + }} + +- name: Fail the command health check + ansible.builtin.fail: + msg: >- + The command health check of {{ cloudfall_deploy_component.id }} failed + on {{ cloudfall_server_id }}: {{ + cloudfall_deploy_command_health_result.command | join(' ') + }} {{ + 'timed out after ' + ~ cloudfall_deploy_component.healthCheck.timeoutSeconds + ~ ' seconds and was stopped' + if cloudfall_deploy_command_health_result.timedOut + else 'exited with code ' + ~ cloudfall_deploy_command_health_result.exitCode + }} {{ + 'on its only attempt' + if cloudfall_deploy_command_health_result.attempts == 1 + else 'on the last of ' + ~ cloudfall_deploy_command_health_result.attempts ~ ' attempts' + }}; its output is in journalctl -u cloudfall-health-{{ + cloudfall_deploy_component.id + }} on that server + when: >- + cloudfall_deploy_command_health_result.timedOut + or cloudfall_deploy_command_health_result.exitCode != 0 diff --git a/engine/ansible/roles/cloudfall_deploy/tasks/health.yml b/engine/ansible/roles/cloudfall_deploy/tasks/health.yml index 8c15bf8..c00f408 100644 --- a/engine/ansible/roles/cloudfall_deploy/tasks/health.yml +++ b/engine/ansible/roles/cloudfall_deploy/tasks/health.yml @@ -1,12 +1,12 @@ --- -# Until cloudfall-dev/cloudfall#164 runs it, a command health check fails here +# Until cloudfall-dev/cloudfall#166 runs it, a command health check fails here # rather than reporting healthy without having run. - name: Refuse a command health check this engine does not run yet ansible.builtin.fail: msg: >- Component {{ cloudfall_deploy_component.id }} declares a command health check, which this engine does not run yet - (cloudfall-dev/cloudfall#164); nothing was changed + (cloudfall-dev/cloudfall#166); nothing was changed when: cloudfall_deploy_component.healthCheck.type == "command" - name: Probe the component's declared health check diff --git a/engine/ansible/roles/cloudfall_deploy/tasks/main.yml b/engine/ansible/roles/cloudfall_deploy/tasks/main.yml index 1efdfc9..503b97f 100644 --- a/engine/ansible/roles/cloudfall_deploy/tasks/main.yml +++ b/engine/ansible/roles/cloudfall_deploy/tasks/main.yml @@ -1,14 +1,4 @@ --- -# Until cloudfall-dev/cloudfall#164 runs it, a command health check fails here, -# before anything changes, rather than letting the deploy pass an unrun gate. -- name: Refuse a command health check this engine does not run yet - ansible.builtin.fail: - msg: >- - Component {{ cloudfall_deploy_component.id }} declares a command health - check, which this engine does not run yet - (cloudfall-dev/cloudfall#164); nothing was changed - when: cloudfall_deploy_component.healthCheck.type == "command" - - name: Validate deployment target and declaration ansible.builtin.assert: that: @@ -345,6 +335,12 @@ - not ansible_check_mode - cloudfall_deploy_component.healthCheck.type == "http" + - name: Gate the release on its declared command health check + ansible.builtin.include_tasks: command_health_check.yml + when: + - not ansible_check_mode + - cloudfall_deploy_component.healthCheck.type == "command" + - name: Gate the release on an active service unit ansible.builtin.command: argv: @@ -493,11 +489,40 @@ - cloudfall_deploy_unit_hand_made | bool - cloudfall_deploy_unit_was_active.rc == 0 + - name: Receipt the release that failed its command health check + ansible.builtin.include_tasks: release_receipt.yml + vars: + cloudfall_deploy_receipt_passed: false + when: + - not ansible_check_mode + - cloudfall_deploy_command_health_result is defined + - cloudfall_deploy_command_health_result.timedOut + or cloudfall_deploy_command_health_result.exitCode != 0 + - name: Fail the deployment after rolling back ansible.builtin.fail: msg: >- - Release {{ cloudfall_deploy_release }} failed its health check and - was {{ + Release {{ cloudfall_deploy_release }} failed {{ + 'the command health check of ' ~ cloudfall_deploy_component.id + ~ ' on ' ~ cloudfall_server_id ~ ' (' + ~ ( + 'timed out after ' + ~ cloudfall_deploy_component.healthCheck.timeoutSeconds + ~ ' seconds' + if cloudfall_deploy_command_health_result.timedOut + else 'exit code ' + ~ cloudfall_deploy_command_health_result.exitCode + ) + ~ ( + ' on its only attempt' + if cloudfall_deploy_command_health_result.attempts == 1 + else ' on the last of ' + ~ cloudfall_deploy_command_health_result.attempts ~ ' attempts' + ) + ~ ')' + if cloudfall_deploy_command_health_result is defined + else 'its health check' + }} and was {{ 'rolled back to ' ~ (cloudfall_deploy_previous_directory | basename) if cloudfall_deploy_previous_directory | length > 0 @@ -516,56 +541,7 @@ else '' }} -- name: Ensure the controller receipt directory exists - ansible.builtin.file: - path: "{{ cloudfall_deploy_receipt_directory }}" - state: directory - mode: "0755" - delegate_to: localhost - become: false - when: cloudfall_deploy_receipt_directory | length > 0 - -- name: Write the release receipt on the controller - ansible.builtin.copy: - content: >- - {{ - { - 'apiVersion': 'cloudfall/v1', - 'kind': 'ReleaseReceipt', - 'metadata': { - 'id': cloudfall_deploy_component.id, - 'description': 'Release receipt written by cloudfall_deploy', - }, - 'spec': { - 'component': cloudfall_deploy_component.id, - 'server': cloudfall_server_id, - 'release': cloudfall_deploy_release, - 'artifactSha256': cloudfall_deploy_artifact_sha256, - 'deployedAt': ansible_facts['date_time']['iso8601'], - 'previousRelease': ( - (cloudfall_deploy_previous_directory | basename) - if cloudfall_deploy_previous_directory | length > 0 - else none - ), - 'healthCheckPassed': true, - } | combine( - { - 'releaseCommand': hostvars[ - cloudfall_deploy_component.release.server - ].cloudfall_deploy_release_command_result, - } - if cloudfall_deploy_component.release is defined - else {} - ), - } | to_nice_json - }} - dest: >- - {{ cloudfall_deploy_receipt_directory }}/{{ - cloudfall_deploy_component.id - }}.json - mode: "0644" - delegate_to: localhost - become: false - when: - - not ansible_check_mode - - cloudfall_deploy_receipt_directory | length > 0 +- name: Receipt the release that passed its health gate + ansible.builtin.include_tasks: release_receipt.yml + vars: + cloudfall_deploy_receipt_passed: true diff --git a/engine/ansible/roles/cloudfall_deploy/tasks/release_receipt.yml b/engine/ansible/roles/cloudfall_deploy/tasks/release_receipt.yml new file mode 100644 index 0000000..6913772 --- /dev/null +++ b/engine/ansible/roles/cloudfall_deploy/tasks/release_receipt.yml @@ -0,0 +1,62 @@ +--- +# Writes the release receipt on the controller: after the health gate passed, +# and from the gate's rescue after a failed command health check, with +# cloudfall_deploy_receipt_passed saying which. A component with a command +# health check records its result under healthCheck; http and none do not. +- name: Ensure the controller receipt directory exists + ansible.builtin.file: + path: "{{ cloudfall_deploy_receipt_directory }}" + state: directory + mode: "0755" + delegate_to: localhost + become: false + when: cloudfall_deploy_receipt_directory | length > 0 + +- name: Write the release receipt on the controller + ansible.builtin.copy: + content: >- + {{ + { + 'apiVersion': 'cloudfall/v1', + 'kind': 'ReleaseReceipt', + 'metadata': { + 'id': cloudfall_deploy_component.id, + 'description': 'Release receipt written by cloudfall_deploy', + }, + 'spec': { + 'component': cloudfall_deploy_component.id, + 'server': cloudfall_server_id, + 'release': cloudfall_deploy_release, + 'artifactSha256': cloudfall_deploy_artifact_sha256, + 'deployedAt': ansible_facts['date_time']['iso8601'], + 'previousRelease': ( + (cloudfall_deploy_previous_directory | basename) + if cloudfall_deploy_previous_directory | length > 0 + else none + ), + 'healthCheckPassed': cloudfall_deploy_receipt_passed | bool, + } | combine( + { + 'releaseCommand': hostvars[ + cloudfall_deploy_component.release.server + ].cloudfall_deploy_release_command_result, + } + if cloudfall_deploy_component.release is defined + else {} + ) | combine( + {'healthCheck': cloudfall_deploy_command_health_result} + if cloudfall_deploy_component.healthCheck.type == 'command' + else {} + ), + } | to_nice_json + }} + dest: >- + {{ cloudfall_deploy_receipt_directory }}/{{ + cloudfall_deploy_component.id + }}.json + mode: "0644" + delegate_to: localhost + become: false + when: + - not ansible_check_mode + - cloudfall_deploy_receipt_directory | length > 0 diff --git a/engine/ansible/roles/cloudfall_deploy/tasks/restart.yml b/engine/ansible/roles/cloudfall_deploy/tasks/restart.yml index f72ae81..84b5f41 100644 --- a/engine/ansible/roles/cloudfall_deploy/tasks/restart.yml +++ b/engine/ansible/roles/cloudfall_deploy/tasks/restart.yml @@ -1,12 +1,12 @@ --- -# Until cloudfall-dev/cloudfall#164 runs it, a command health check fails here, +# Until cloudfall-dev/cloudfall#166 runs it, a command health check fails here, # before anything changes, rather than letting the restart pass an unrun gate. - name: Refuse a command health check this engine does not run yet ansible.builtin.fail: msg: >- Component {{ cloudfall_deploy_component.id }} declares a command health check, which this engine does not run yet - (cloudfall-dev/cloudfall#164); nothing was changed + (cloudfall-dev/cloudfall#166); nothing was changed when: cloudfall_deploy_component.healthCheck.type == "command" - name: Validate restart target and declaration diff --git a/engine/ansible/roles/cloudfall_deploy/tasks/rollback.yml b/engine/ansible/roles/cloudfall_deploy/tasks/rollback.yml index d17bbff..3313498 100644 --- a/engine/ansible/roles/cloudfall_deploy/tasks/rollback.yml +++ b/engine/ansible/roles/cloudfall_deploy/tasks/rollback.yml @@ -1,12 +1,12 @@ --- -# Until cloudfall-dev/cloudfall#164 runs it, a command health check fails here, +# Until cloudfall-dev/cloudfall#166 runs it, a command health check fails here, # before anything changes, rather than letting the rollback pass an unrun gate. - name: Refuse a command health check this engine does not run yet ansible.builtin.fail: msg: >- Component {{ cloudfall_deploy_component.id }} declares a command health check, which this engine does not run yet - (cloudfall-dev/cloudfall#164); nothing was changed + (cloudfall-dev/cloudfall#166); nothing was changed when: cloudfall_deploy_component.healthCheck.type == "command" - name: Validate rollback target and declaration diff --git a/engine/tests/test_command_health_gate.py b/engine/tests/test_command_health_gate.py new file mode 100644 index 0000000..ae35a3c --- /dev/null +++ b/engine/tests/test_command_health_gate.py @@ -0,0 +1,910 @@ +"""A component's command health check gating a deploy (S-009-2..9, 11, 12). + +The role needs a Debian host with systemd, so these tests take the routes +``test_release_command.py`` takes. ``command_health_check.yml`` and +``release_receipt.yml`` run for real under ``ansible-playbook`` on the +controller, with ``systemd-run`` swapped through the role's +``cloudfall_deploy_systemd_run`` variable for a stand-in that honours +``--working-directory``, runs the argv after ``--`` and, unlike the real one, +passes the command's output on: so a marker the command prints reaching the +output would show here. The gate block's order, its rescue and the deploy's +failure message are replayed with Ansible's own templating. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from dataclasses import dataclass +from itertools import pairwise +from pathlib import Path +from typing import cast + +import pytest +import yaml +from ansible.parsing.dataloader import DataLoader +from ansible.template import Templar, trust_as_template +from cloudfall.validation import SchemaCatalog +from cloudfall_engine.cli import main +from cloudfall_engine.playbook import PlaybookRun, playbook_command +from jsonschema.exceptions import ValidationError + +ROOT = Path(__file__).parents[2] +SCHEMAS = ROOT / "config" / "schemas" / "v1" +ENGINE = ROOT / "engine" +ROLE = ENGINE / "ansible" / "roles" / "cloudfall_deploy" +ACTIVATE = "Activate the release behind the health gate" +SWITCH = "Switch the current release symlink" +RESTART = "Restart the component service" +HTTP_GATE = "Gate the release on its declared health check" +COMMAND_GATE = "Gate the release on its declared command health check" +UNIT_GATE = [ + "Gate the release on an active service unit", + "Read the service unit restart count before settling", + "Let the service unit settle", + "Gate the release on a service unit still active after settling", + "Read the service unit restart count after settling", + "Gate the release on a service unit that did not restart while settling", +] +RUN = "Run the command health check" +RESTORE = "Restore the previous release symlink" +RESTART_RESTORED = "Restart the restored release" +STOP_FIRST = "Stop the failed first release" +FAILED_RECEIPT = "Receipt the release that failed its command health check" +FAIL = "Fail the deployment after rolling back" +PASSED_RECEIPT = "Receipt the release that passed its health gate" +PREVIOUS = "20261027T000000Z-abcdef1" +RELEASE = "20261028T000000Z-abcdef2" +MARKER = "cloudfall-smoke-secret-7f3a" +TIMEOUT_MARKER = "cloudfall-test-timeout" +COMPONENT_ID = "crm-backend" +HTTP_CHECK: dict[str, object] = { + "type": "http", + "scheme": "http", + "port": 8100, + "path": "/healthz", + "expectedStatuses": [200], + "timeoutSeconds": 5, + "attempts": 10, +} +NO_CHECK: dict[str, object] = {"type": "none"} + +FAKE_SYSTEMD_RUN = f"""#!/bin/sh +# Stand-in for systemd-run --wait: chdir like --working-directory, then run +# the argv after "--" and pass on its exit code and, unlike systemd-run +# without --pipe, its output. +printf '%s\\n' "$@" > "$0.argv" +directory= +while [ "$1" != "--" ]; do + case "$1" in + --working-directory=*) directory="${{1#--working-directory=}}" ;; + esac + shift +done +shift +cd "$directory" || exit 200 +"$@" +status=$? +if [ -f "$directory/{TIMEOUT_MARKER}" ]; then + echo "Finished with result: timeout" >&2 + exit 1 +fi +exit $status +""" + +SMOKE = f"""#!/bin/sh +# smoke : counts its runs in the working +# directory, prints a marker on both streams, then exits 0 from the passing +# attempt on (0: never) and with before it. +count=$(( $(cat "$PWD/attempts" 2>/dev/null || echo 0) + 1 )) +echo "$count" > "$PWD/attempts" +date +%s >> "$PWD/attempt-times" +echo "$PWD" > "$PWD/where" +echo "{MARKER} on stdout" +echo "{MARKER} on stderr" >&2 +if [ "$2" -gt 0 ] && [ "$count" -ge "$2" ]; then + exit 0 +fi +exit "$1" +""" + + +@dataclass(frozen=True, slots=True) +class _Run: + """One run of the check under ``ansible-playbook`` and what it left.""" + + tmp_path: Path + exit_code: int + output: str + + @property + def install_root(self) -> Path: + return self.tmp_path / "app" + + @property + def release_directory(self) -> Path: + return self.install_root / "releases" / RELEASE + + @property + def argv(self) -> list[str]: + path = self.tmp_path / "systemd-run.argv" + return path.read_text(encoding="utf-8").splitlines() + + @property + def attempts(self) -> int: + path = self.release_directory / "attempts" + return int(path.read_text(encoding="utf-8")) + + @property + def attempt_times(self) -> list[int]: + path = self.release_directory / "attempt-times" + text = path.read_text(encoding="utf-8") + return [int(line) for line in text.splitlines()] + + @property + def receipt_text(self) -> str: + path = self.tmp_path / "receipts" / f"{COMPONENT_ID}.json" + return path.read_text(encoding="utf-8") + + @property + def receipt(self) -> dict[str, object]: + receipt = cast("dict[str, object]", json.loads(self.receipt_text)) + _validate(receipt) + return receipt + + @property + def spec(self) -> dict[str, object]: + spec = self.receipt["spec"] + assert isinstance(spec, dict) + return spec + + @property + def health_check(self) -> dict[str, object]: + health_check = self.spec["healthCheck"] + assert isinstance(health_check, dict) + return health_check + + +def _validate(receipt: dict[str, object]) -> None: + SchemaCatalog(SCHEMAS).validate_named("release-receipt.schema.json", receipt) + + +def _executable(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + path.chmod(0o755) + + +def _local_inventory(tmp_path: Path) -> Path: + inventory = tmp_path / "inventory.json" + host = {"ansible_connection": "local", "ansible_python_interpreter": sys.executable} + inventory.write_text( + json.dumps({"all": {"hosts": {"localhost": host}}}), encoding="utf-8" + ) + return inventory + + +def _component( + health_check: dict[str, object], install_root: Path +) -> dict[str, object]: + return { + "id": COMPONENT_ID, + "application_user": "crm", + "install_root": str(install_root), + "service": {"name": COMPONENT_ID, "command": [".venv/bin/app"]}, + "healthCheck": health_check, + } + + +def _command_check( + command: list[str], *, attempts: int = 1, timeout: int = 60 +) -> dict[str, object]: + return { + "type": "command", + "command": command, + "timeoutSeconds": timeout, + "attempts": attempts, + } + + +def _check( # noqa: PLR0913 - one knob per criterion. + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + *, + exit_code: int = 0, + passing_attempt: int = 0, + attempts: int = 1, + first_release: bool = False, + timed_out: bool = False, + absolute: bool = False, + verbose: bool = False, +) -> _Run: + """Run the role's command check, receipting it as the gate and rescue do. + + The block's rescue here stands in for the role's: it writes the failed + receipt through the role's own ``release_receipt.yml`` and fails the play. + """ + install_root = tmp_path / "app" + release_directory = install_root / "releases" / RELEASE + release_directory.mkdir(parents=True) + if not first_release: + (install_root / "releases" / PREVIOUS).mkdir() + (install_root / "current").symlink_to(release_directory) + if absolute: + executable = tmp_path / "checks" / "smoke" + _executable(executable, SMOKE) + first = str(executable) + else: + _executable(release_directory / "bin" / "smoke", SMOKE) + first = "bin/smoke" + if timed_out: + (release_directory / TIMEOUT_MARKER).write_text("", encoding="utf-8") + systemd_run = tmp_path / "systemd-run" + _executable(systemd_run, FAKE_SYSTEMD_RUN) + health_check = _command_check( + [first, str(exit_code), str(passing_attempt)], attempts=attempts + ) + variables = { + "cloudfall_server_id": "h1", + "cloudfall_deploy_release": RELEASE, + "cloudfall_deploy_artifact_sha256": "a" * 64, + "cloudfall_deploy_previous_directory": ( + "" if first_release else str(install_root / "releases" / PREVIOUS) + ), + "cloudfall_deploy_receipt_directory": str(tmp_path / "receipts"), + "cloudfall_deploy_systemd_run": str(systemd_run), + "cloudfall_deploy_component": _component(health_check, install_root), + } + receipt = { + "ansible.builtin.include_role": { + "name": "cloudfall_deploy", + "tasks_from": "release_receipt", + }, + } + playbook = tmp_path / "check.yml" + playbook.write_text( + yaml.safe_dump( + [ + { + "name": "Gate a release on its command health check", + "hosts": "localhost", + "gather_facts": True, + "vars": variables, + "tasks": [ + { + "name": "Gate as the role does", + "block": [ + { + "name": "Run the role's command check tasks", + "ansible.builtin.include_role": { + "name": "cloudfall_deploy", + "tasks_from": "command_health_check", + }, + } + ], + "rescue": [ + { + "name": "Receipt the failed check", + **receipt, + "vars": {"cloudfall_deploy_receipt_passed": False}, + }, + { + "name": "Fail after receipting", + "ansible.builtin.fail": {"msg": "gate failed"}, + }, + ], + }, + { + "name": "Receipt the passed check", + **receipt, + "vars": {"cloudfall_deploy_receipt_passed": True}, + }, + ], + } + ] + ), + encoding="utf-8", + ) + inventory = _local_inventory(tmp_path) + if verbose: + # As verbose as ansible-playbook gets: every task result is printed + # unless the task is no_log. + command = playbook_command(PlaybookRun(playbook, inventory), ENGINE) + done = subprocess.run( # noqa: S603 - the engine's own argv + [*command.argv[:1], "-vvvv", *command.argv[1:]], + env={**os.environ, **command.environment}, + capture_output=True, + text=True, + check=False, + ) + return _Run(tmp_path, done.returncode, f"{done.stdout}\n{done.stderr}") + code = main( + [ + "playbook", + "run", + str(playbook), + "--engine", + str(ENGINE), + "--inventory", + str(inventory), + ] + ) + captured = capsys.readouterr() + return _Run(tmp_path, code, f"{captured.out}\n{captured.err}") + + +# The gate's place in the deploy and its transient unit. + + +def _main() -> list[dict[str, object]]: + tasks = DataLoader().load_from_file(str(ROLE / "tasks" / "main.yml")) + assert isinstance(tasks, list) + return tasks + + +def _activation() -> dict[str, object]: + return next(t for t in _main() if t["name"] == ACTIVATE) + + +def _section(key: str) -> list[dict[str, object]]: + section = _activation()[key] + assert isinstance(section, list) + return section + + +def _task(name: str, tasks: list[dict[str, object]]) -> dict[str, object]: + return next(t for t in tasks if t["name"] == name) + + +def _check_tasks() -> list[dict[str, object]]: + path = ROLE / "tasks" / "command_health_check.yml" + tasks = DataLoader().load_from_file(str(path)) + assert isinstance(tasks, list) + return tasks + + +def _templar(variables: dict[str, object]) -> Templar: + return Templar(loader=DataLoader(), variables=variables) + + +def _holds(expression: object, variables: dict[str, object]) -> bool: + return bool( + _templar(variables).evaluate_conditional(trust_as_template(str(expression))) + ) + + +def _conditions(task: dict[str, object]) -> list[object]: + value = task.get("when", []) + return value if isinstance(value, list) else [value] + + +def _runs(task: dict[str, object], variables: dict[str, object]) -> bool: + return all(_holds(c, variables) for c in _conditions(task)) + + +def test_s009_2_the_check_runs_after_the_restart_inside_the_rescued_block() -> None: + names = [str(task["name"]) for task in _section("block")] + gate = _task(COMMAND_GATE, _section("block")) + + assert names.index(SWITCH) < names.index(RESTART) < names.index(COMMAND_GATE) + assert gate["ansible.builtin.include_tasks"] == "command_health_check.yml" + assert RESTORE in [str(task["name"]) for task in _section("rescue")] + variables: dict[str, object] = { + "ansible_check_mode": False, + "cloudfall_deploy_component": {"healthCheck": _command_check(["bin/smoke"])}, + } + assert _runs(gate, variables) + assert not _runs(gate, {**variables, "ansible_check_mode": True}) + + +def test_s009_2_the_check_runs_as_the_user_in_current_with_the_unit_environment( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + run = _check(tmp_path, capsys) + + assert run.exit_code == 0, run.output + argv = run.argv + options = argv[: argv.index("--")] + assert "--unit=cloudfall-health-crm-backend" in options + assert {"--wait", "--collect", "--service-type=exec"} <= set(options) + assert "--uid=crm" in options + assert "--gid=crm" in options + assert f"--working-directory={run.install_root}/current" in options + assert "--property=RuntimeMaxSec=60" in options + assert "--property=TimeoutStopSec=10" in options + assert "--property=NoNewPrivileges=true" in options + assert "--property=PrivateTmp=true" in options + unit = _templar( + {"cloudfall_deploy_component": _component(NO_CHECK, run.install_root)} + ).template( + trust_as_template( + (ROLE / "templates" / "component.service.j2").read_text(encoding="utf-8") + ) + ) + unit_line = next( + line for line in str(unit).splitlines() if line.startswith("EnvironmentFile=") + ) + assert unit_line.startswith("EnvironmentFile=-") + assert f"--property={unit_line}" in options + where = (run.release_directory / "where").read_text(encoding="utf-8").strip() + assert where == f"{run.install_root}/current" + + +def test_s009_3_a_relative_first_element_runs_from_current( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + run = _check(tmp_path, capsys) + + argv = run.argv + assert argv[argv.index("--") + 1 :] == [ + f"{run.install_root}/current/bin/smoke", + "0", + "0", + ] + + +def test_s009_3_an_absolute_first_element_runs_unchanged( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + run = _check(tmp_path, capsys, absolute=True) + + assert run.exit_code == 0, run.output + argv = run.argv + executable = str(tmp_path / "checks" / "smoke") + assert argv[argv.index("--") + 1 :] == [executable, "0", "0"] + + +# Pass, failure and retries. + + +def test_s009_4_a_passing_check_receipts_its_run( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + run = _check(tmp_path, capsys) + + assert run.exit_code == 0, run.output + spec = run.spec + assert spec["healthCheckPassed"] is True + assert spec["release"] == RELEASE + assert spec["previousRelease"] == PREVIOUS + assert spec["healthCheck"] == { + "type": "command", + "command": ["bin/smoke", "0", "0"], + "attempts": 1, + "exitCode": 0, + "timedOut": False, + } + + +def test_s009_5_a_check_exiting_3_fails_and_receipts_the_failed_release( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + run = _check(tmp_path, capsys, exit_code=3, attempts=2) + + assert run.exit_code == 82 + assert ( + "The command health check of crm-backend failed on h1: bin/smoke 3 0" + " exited with code 3 on the last of 2 attempts" + ) in run.output + spec = run.spec + assert spec["healthCheckPassed"] is False + assert spec["release"] == RELEASE + assert spec["previousRelease"] == PREVIOUS + assert spec["healthCheck"] == { + "type": "command", + "command": ["bin/smoke", "3", "0"], + "attempts": 2, + "exitCode": 3, + "timedOut": False, + } + + +def test_s009_6_a_failed_first_release_receipts_no_previous_release( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + run = _check(tmp_path, capsys, exit_code=3, first_release=True) + + assert run.exit_code == 82 + spec = run.spec + assert spec["healthCheckPassed"] is False + assert spec["previousRelease"] is None + assert run.health_check["exitCode"] == 3 + + +def test_s009_7_a_timed_out_check_fails_naming_the_timeout( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + run = _check(tmp_path, capsys, timed_out=True) + + assert run.exit_code == 82 + assert "timed out after 60 seconds and was stopped" in run.output + assert run.spec["healthCheckPassed"] is False + assert run.health_check["timedOut"] is True + assert run.health_check["exitCode"] is None + + +def test_s009_7_a_timed_out_run_counts_as_a_failed_attempt( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """The command itself exits 0, but the unit was stopped at its limit.""" + run = _check(tmp_path, capsys, timed_out=True, attempts=2) + + assert run.exit_code == 82 + assert run.attempts == 2 + + +def test_s009_8_a_check_failing_twice_then_passing_records_three_attempts( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + run = _check(tmp_path, capsys, exit_code=1, passing_attempt=3, attempts=3) + + assert run.exit_code == 0, run.output + assert run.attempts == 3 + assert run.spec["healthCheckPassed"] is True + assert run.health_check["attempts"] == 3 + assert run.health_check["exitCode"] == 0 + + +def test_s009_8_a_check_always_failing_runs_exactly_attempts_times_3s_apart( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + run = _check(tmp_path, capsys, exit_code=1, attempts=3) + + assert run.exit_code == 82 + assert run.attempts == 3 + times = run.attempt_times + assert len(times) == 3 + assert all(3 <= later - earlier < 10 for earlier, later in pairwise(times)) + assert run.health_check["attempts"] == 3 + + +def test_s009_8_a_passing_first_attempt_stops_there( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + run = _check(tmp_path, capsys, attempts=3) + + assert run.exit_code == 0, run.output + assert run.attempts == 1 + + +# The command's output stays on the server. + + +def test_s009_9_the_output_reaches_neither_the_receipt_nor_the_ansible_output( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + run = _check(tmp_path, capsys, exit_code=1, verbose=True) + + assert run.exit_code == 2, run.output + assert "The command health check of crm-backend failed" in run.output + assert "cloudfall-health-crm-backend" in run.output + assert MARKER not in run.output + assert MARKER not in run.receipt_text + assert MARKER not in _deploy_failure(_failed_check(timed_out=False)) + + +def test_s009_9_the_run_is_no_log_and_pipes_nothing() -> None: + run = _task(RUN, _check_tasks()) + command = run["ansible.builtin.command"] + assert isinstance(command, dict) + + assert run["no_log"] is True + assert "--pipe" not in str(command["argv"]) + + +@pytest.mark.parametrize("key", ["output", "outputTail", "stdout", "stderr"]) +def test_s009_9_the_receipt_schema_refuses_any_other_health_check_key( + key: str, +) -> None: + receipt = _receipt_document( + passed=False, + health_check={**FAILED_CHECK, key: "secret"}, + ) + + with pytest.raises(ValidationError, match="Additional properties"): + _validate(receipt) + + +# The receipt schema. + +FAILED_CHECK: dict[str, object] = { + "type": "command", + "command": ["bin/smoke"], + "attempts": 3, + "exitCode": 3, + "timedOut": False, +} + + +def _receipt_document( + *, + passed: bool, + health_check: dict[str, object] | None, + release_command: bool = False, + previous: str | None = PREVIOUS, +) -> dict[str, object]: + spec: dict[str, object] = { + "component": COMPONENT_ID, + "server": "h1", + "release": RELEASE, + "artifactSha256": "a" * 64, + "deployedAt": "2026-10-28T00:00:00Z", + "previousRelease": previous, + "healthCheckPassed": passed, + } + if release_command: + spec["releaseCommand"] = { + "server": "h1", + "command": ["bin/migrate"], + "workingDirectory": None, + "exitCode": 0, + "outputTail": "migrated", + } + if health_check is not None: + spec["healthCheck"] = health_check + return { + "apiVersion": "cloudfall/v1", + "kind": "ReleaseReceipt", + "metadata": { + "id": COMPONENT_ID, + "description": "Release receipt written by cloudfall_deploy", + }, + "spec": spec, + } + + +@pytest.mark.parametrize( + "health_check", + [None, {"type": "http"}, {**FAILED_CHECK, "exitCode": 0}], + ids=["no-health-check", "not-a-command-check", "exit-0"], +) +def test_s009_11_a_failed_receipt_needs_a_failed_command_check( + health_check: dict[str, object] | None, +) -> None: + with pytest.raises(ValidationError): + _validate(_receipt_document(passed=False, health_check=health_check)) + + +@pytest.mark.parametrize( + "health_check", + [ + {**FAILED_CHECK, "exitCode": None}, + {**FAILED_CHECK, "timedOut": True}, + {**FAILED_CHECK, "attempts": 21}, + ], + ids=["null-exit-without-timeout", "timeout-with-exit-code", "attempts-21"], +) +def test_s009_11_a_command_check_record_must_be_coherent( + health_check: dict[str, object], +) -> None: + with pytest.raises(ValidationError): + _validate(_receipt_document(passed=False, health_check=health_check)) + + +def test_s009_11_a_passed_receipt_refuses_a_failed_command_check() -> None: + with pytest.raises(ValidationError): + _validate(_receipt_document(passed=True, health_check=FAILED_CHECK)) + + +@pytest.mark.parametrize("release_command", [False, True]) +@pytest.mark.parametrize("previous", [PREVIOUS, None]) +def test_s009_11_every_receipt_written_today_still_validates( + release_command: bool, # noqa: FBT001 - a pytest parameter + previous: str | None, +) -> None: + """The shape main wrote before #164: passed, no ``healthCheck``.""" + _validate( + _receipt_document( + passed=True, + health_check=None, + release_command=release_command, + previous=previous, + ) + ) + + +def test_s009_11_a_timed_out_failure_validates() -> None: + _validate( + _receipt_document( + passed=False, + health_check={**FAILED_CHECK, "exitCode": None, "timedOut": True}, + ) + ) + + +# The rescue and the deploy's failure message. + + +def _failed_check(*, timed_out: bool) -> dict[str, object]: + return { + **FAILED_CHECK, + "exitCode": None if timed_out else 3, + "timedOut": timed_out, + } + + +def _rescue_variables( + previous_directory: str, result: dict[str, object] | None +) -> dict[str, object]: + variables: dict[str, object] = { + "ansible_check_mode": False, + "cloudfall_server_id": "h1", + "cloudfall_deploy_previous_directory": previous_directory, + "cloudfall_deploy_unit_hand_made": False, + "cloudfall_deploy_unit_was_active": {"rc": 0}, + "cloudfall_deploy_unit_kept_path": ( + "/etc/systemd/system/crm-backend.service.pre-cloudfall" + ), + "cloudfall_deploy_release": RELEASE, + "cloudfall_deploy_component": { + "id": COMPONENT_ID, + "service": {"name": COMPONENT_ID}, + "healthCheck": _command_check(["bin/smoke"], attempts=3), + }, + } + if result is not None: + variables["cloudfall_deploy_command_health_result"] = result + return variables + + +def _rescue(previous_directory: str, result: dict[str, object] | None) -> list[str]: + variables = _rescue_variables(previous_directory, result) + return [ + str(task["name"]) for task in _section("rescue") if _runs(task, variables) + ] + + +def _deploy_failure( + result: dict[str, object], previous_directory: str = f"/srv/releases/{PREVIOUS}" +) -> str: + fail = _task(FAIL, _section("rescue"))["ansible.builtin.fail"] + assert isinstance(fail, dict) + variables = _rescue_variables(previous_directory, result) + return str(_templar(variables).template(trust_as_template(fail["msg"]))) + + +def test_s009_5_a_failed_check_rolls_back_receipts_then_fails() -> None: + assert _rescue(f"/srv/releases/{PREVIOUS}", _failed_check(timed_out=False)) == [ + RESTORE, + RESTART_RESTORED, + FAILED_RECEIPT, + FAIL, + ] + receipt = _task(FAILED_RECEIPT, _section("rescue")) + assert receipt["ansible.builtin.include_tasks"] == "release_receipt.yml" + assert receipt["vars"] == {"cloudfall_deploy_receipt_passed": False} + + +def test_s009_5_the_deploy_error_names_the_check_its_exit_and_the_rollback() -> None: + assert _deploy_failure(_failed_check(timed_out=False)) == ( + f"Release {RELEASE} failed the command health check of crm-backend on h1" + f" (exit code 3 on the last of 3 attempts) and was rolled back to" + f" {PREVIOUS}" + ) + + +def test_s009_6_a_failed_first_release_is_stopped_and_receipted() -> None: + assert _rescue("", _failed_check(timed_out=False)) == [ + STOP_FIRST, + FAILED_RECEIPT, + FAIL, + ] + assert _deploy_failure(_failed_check(timed_out=False), "").endswith( + "and was stopped because no previous release exists" + ) + + +def test_s009_7_the_deploy_error_names_the_timeout() -> None: + assert _deploy_failure(_failed_check(timed_out=True)) == ( + f"Release {RELEASE} failed the command health check of crm-backend on h1" + f" (timed out after 60 seconds on the last of 3 attempts) and was rolled" + f" back to {PREVIOUS}" + ) + single = {**_failed_check(timed_out=True), "attempts": 1} + assert "(timed out after 60 seconds on its only attempt)" in _deploy_failure(single) + + +# http and none keep today's gate and receipt. + + +def _gate(health_check: dict[str, object]) -> list[str]: + variables: dict[str, object] = { + "ansible_check_mode": False, + "cloudfall_deploy_component": { + "service": {"name": COMPONENT_ID}, + "healthCheck": health_check, + }, + } + return [ + str(task["name"]) + for task in _section("block") + if task["name"] in {HTTP_GATE, COMMAND_GATE, *UNIT_GATE} + and _runs(task, variables) + ] + + +def test_s009_12_http_and_none_run_the_same_gate_tasks() -> None: + assert _gate(HTTP_CHECK) == [HTTP_GATE] + assert _gate(NO_CHECK) == UNIT_GATE + assert _gate(_command_check(["bin/smoke"])) == [COMMAND_GATE] + + +def test_s009_12_a_failed_http_or_none_gate_writes_no_receipt() -> None: + """Their gate never sets the command result, so the rescue skips the receipt.""" + for previous in (f"/srv/releases/{PREVIOUS}", ""): + assert FAILED_RECEIPT not in _rescue(previous, None) + variables = _rescue_variables("", None) + fail = _task(FAIL, _section("rescue"))["ansible.builtin.fail"] + assert isinstance(fail, dict) + message = str(_templar(variables).template(trust_as_template(fail["msg"]))) + assert message == ( + f"Release {RELEASE} failed its health check and was stopped because no" + " previous release exists" + ) + + +@pytest.mark.parametrize("health_check", [HTTP_CHECK, NO_CHECK], ids=["http", "none"]) +def test_s009_12_http_and_none_receipts_have_no_health_check( + health_check: dict[str, object], tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + variables = { + "cloudfall_server_id": "h1", + "cloudfall_deploy_release": RELEASE, + "cloudfall_deploy_artifact_sha256": "a" * 64, + "cloudfall_deploy_previous_directory": "", + "cloudfall_deploy_receipt_directory": str(tmp_path / "receipts"), + "cloudfall_deploy_component": _component(health_check, tmp_path / "app"), + } + playbook = tmp_path / "receipt.yml" + playbook.write_text( + yaml.safe_dump( + [ + { + "name": "Receipt a passed gate", + "hosts": "localhost", + "gather_facts": True, + "vars": variables, + "tasks": [ + { + "name": "Receipt as the role does", + "ansible.builtin.include_role": { + "name": "cloudfall_deploy", + "tasks_from": "release_receipt", + }, + "vars": {"cloudfall_deploy_receipt_passed": True}, + } + ], + } + ] + ), + encoding="utf-8", + ) + + exit_code = main( + [ + "playbook", + "run", + str(playbook), + "--engine", + str(ENGINE), + "--inventory", + str(_local_inventory(tmp_path)), + ] + ) + + assert exit_code == 0, capsys.readouterr() + run = _Run(tmp_path, exit_code, "") + assert run.spec["healthCheckPassed"] is True + assert "healthCheck" not in run.spec + + +def test_s009_12_the_pass_receipt_is_written_where_it_was() -> None: + names = [str(task["name"]) for task in _main()] + receipt = _task(PASSED_RECEIPT, _main()) + + assert names.index(ACTIVATE) < names.index(PASSED_RECEIPT) + assert receipt["ansible.builtin.include_tasks"] == "release_receipt.yml" + assert receipt["vars"] == {"cloudfall_deploy_receipt_passed": True} + assert "when" not in receipt diff --git a/engine/tests/test_deploy_role.py b/engine/tests/test_deploy_role.py index 242da59..722684a 100644 --- a/engine/tests/test_deploy_role.py +++ b/engine/tests/test_deploy_role.py @@ -491,13 +491,15 @@ def test_the_failure_of_a_first_release_without_a_kept_unit_is_unchanged() -> No REFUSE_COMMAND = "Refuse a command health check this engine does not run yet" -@pytest.mark.parametrize( - "entry_point", ["main.yml", "restart.yml", "rollback.yml", "health.yml"] -) +def test_s009_2_a_deploy_no_longer_refuses_a_command_check() -> None: + assert REFUSE_COMMAND not in [str(task["name"]) for task in _main()] + + +@pytest.mark.parametrize("entry_point", ["restart.yml", "rollback.yml", "health.yml"]) def test_the_engine_refuses_a_command_check_it_does_not_run_yet( entry_point: str, ) -> None: - """Until #164 runs it, a command check fails first, never passes unrun.""" + """Until #166 runs it there, a command check fails first, never passes unrun.""" tasks = _load(ROLE / "tasks" / entry_point) assert isinstance(tasks, list) refusal = tasks[0] @@ -522,7 +524,7 @@ def variables(health_check: dict[str, object]) -> dict[str, object]: assert message.startswith( "Component crm-backend declares a command health check" ) - assert "cloudfall-dev/cloudfall#164" in message + assert "cloudfall-dev/cloudfall#166" in message @pytest.mark.parametrize("health_check", [HTTP_CHECK, NO_CHECK, COMMAND_CHECK]) diff --git a/engine/tests/test_release_command.py b/engine/tests/test_release_command.py index c75904a..f35e085 100644 --- a/engine/tests/test_release_command.py +++ b/engine/tests/test_release_command.py @@ -449,10 +449,13 @@ def passes(hostvars: dict[str, object]) -> bool: def _receipt( component: dict[str, object], server: str, hostvars: dict[str, object] ) -> dict[str, object]: - task = next(t for t in _main_tasks() if t["name"] == RECEIPT) + tasks = DataLoader().load_from_file(str(ROLE / "tasks" / "release_receipt.yml")) + assert isinstance(tasks, list) + task = next(t for t in tasks if t["name"] == RECEIPT) copy = task["ansible.builtin.copy"] assert isinstance(copy, dict) variables: dict[str, object] = { + "cloudfall_deploy_receipt_passed": True, "hostvars": hostvars, "cloudfall_server_id": server, "cloudfall_deploy_component": component,