Run a component's release command once per deploy (#119)

changelog.d/119-release-command.md

@@ -0,0 +1,3 @@
+### Added
+
+- Components: a `release.command` (with optional `workingDirectory` and `timeoutSeconds`, 1 to 1800, default 300) runs once per deploy on the first server of `deployment.servers`, which now deploys first, after the release's dependencies are materialized and before the unit file changes or `current` moves; it runs as the application's user with the component's env file loaded under systemd's `EnvironmentFile=` rules. A non-zero exit or a timeout fails the deploy with `ENGINE_STEP_FAILED` before any server switches, and a passing run is recorded as `releaseCommand` (server, command, exit code, last 2000 characters of output) in every release receipt of that deploy (#119)

config/schemas/v1/component.schema.json

@@ -124,6 +124,34 @@
           ],
           "additionalProperties": false
         },
+        "release": {
+          "type": "object",
+          "properties": {
+            "command": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "minLength": 1
+              },
+              "minItems": 1
+            },
+            "workingDirectory": {
+              "type": "string",
+              "minLength": 1,
+              "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$))[^\\u0000-\\u001f\\u007f]+$"
+            },
+            "timeoutSeconds": {
+              "type": "integer",
+              "minimum": 1,
+              "maximum": 1800,
+              "default": 300
+            }
+          },
+          "required": [
+            "command"
+          ],
+          "additionalProperties": false
+        },
         "healthCheck": {
           "oneOf": [
             {

config/schemas/v1/release-receipt.schema.json

@@ -42,6 +42,44 @@
         },
         "healthCheckPassed": {
           "const": true
+        },
+        "releaseCommand": {
+          "type": "object",
+          "properties": {
+            "server": {
+              "$ref": "common.schema.json#/$defs/resourceId"
+            },
+            "command": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "minLength": 1
+              },
+              "minItems": 1
+            },
+            "workingDirectory": {
+              "type": [
+                "string",
+                "null"
+              ],
+              "minLength": 1
+            },
+            "exitCode": {
+              "const": 0
+            },
+            "outputTail": {
+              "type": "string",
+              "maxLength": 2000
+            }
+          },
+          "required": [
+            "server",
+            "command",
+            "workingDirectory",
+            "exitCode",
+            "outputTail"
+          ],
+          "additionalProperties": false
         }
       },
       "required": [

engine/README.md

@@ -182,7 +182,17 @@ task health COMPONENT=crm-backend
 
 These tasks call the SDK's lifecycle commands, which verify the artifact and
 then execute the engine's `deploy.yml`, `rollback.yml`, `restart.yml`, and
-`health.yml` playbook contracts. Rollback activates an already-retained
+`health.yml` playbook contracts.
+
+A component's `release.command` (database migrations, say) runs once per
+deploy, on the first server of `deployment.servers`, which deploys before the
+others. It runs in a transient systemd unit as the application's user, in the
+new release directory (or its `workingDirectory`), with the component's
+`shared/env/<component>.env` loaded by the unit's own `EnvironmentFile=`
+rules, and is stopped after `timeoutSeconds`. A failure or a timeout fails the
+deploy before any server's `current` moves; a success is recorded under
+`releaseCommand` in every release receipt of that deploy. The command must be
+safe while the old release still serves traffic. Rollback activates an already-retained
 release without re-transferring an artifact and requires the declared health
 check to pass; it deliberately does not auto-restore on failure, because a
 failed rollback is an incident, not a deployment.
@@ -191,7 +201,8 @@ The deploy role transfers the artifact and refuses it if the digest does not
 match the build metadata, unpacks it under `releases/<release-id>/`,
 materializes locked dependencies with a pinned `uv` (`uv sync --frozen
 --no-dev`, with the managed Python toolchain and cache under the component's
-`shared/` directory), optionally installs an environment file, renders the
+`shared/` directory), optionally installs an environment file, runs the
+component's `release.command` when it declares one, renders the
 systemd unit from the declared service command, and then activates behind a
 health gate: symlink switch, restart, and the declared HTTP health check. If
 the health check fails, the previous release is restored and restarted

engine/ansible/playbooks/deploy.yml

@@ -1,11 +1,14 @@
 ---
-- name: Deploy one Cloudfall component release
-  hosts: cloudfall_servers
-  become: true
-  gather_facts: true
-  any_errors_fatal: true
-  serial: 1
-  pre_tasks:
+# A component that declares a release command runs it once, on its release
+# server (the first entry of deployment.servers). That server deploys first,
+# in its own play, so the command has run before any server's current release
+# moves, and a failed command stops the playbook before the other plays start.
+- name: Select the release server of the deployed component
+  hosts: localhost
+  connection: local
+  become: false
+  gather_facts: false
+  tasks:
     - name: Require an explicit component and release selection
       ansible.builtin.assert:
         that:
@@ -17,7 +20,54 @@
           Deployment requires cloudfall_deploy_component_id,
           cloudfall_deploy_release, cloudfall_deploy_artifact_archive, and
           cloudfall_deploy_artifact_sha256 extra variables
-      run_once: true
+
+    - name: Deploy the release server first
+      ansible.builtin.add_host:
+        name: "{{ cloudfall_release_server_item }}"
+        groups: cloudfall_release_servers
+      loop: >-
+        {{
+          groups['cloudfall_servers'] | default([])
+          | map('extract', hostvars, 'cloudfall_components')
+          | flatten
+          | selectattr('id', 'equalto', cloudfall_deploy_component_id)
+          | selectattr('release', 'defined')
+          | map(attribute='release')
+          | map(attribute='server')
+          | unique
+          | list
+        }}
+      loop_control:
+        loop_var: cloudfall_release_server_item
+
+- name: Deploy one Cloudfall component release on its release server
+  hosts: cloudfall_release_servers
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  serial: 1
+  tasks:
+    - name: Deploy the selected component and run its release command
+      ansible.builtin.include_role:
+        name: cloudfall_deploy
+      vars:
+        cloudfall_deploy_component: "{{ cloudfall_component_item }}"
+      loop: >-
+        {{
+          cloudfall_components | default([])
+          | selectattr('id', 'equalto', cloudfall_deploy_component_id)
+          | list
+        }}
+      loop_control:
+        loop_var: cloudfall_component_item
+        label: "{{ cloudfall_component_item.id }}"
+
+- name: Deploy one Cloudfall component release
+  hosts: cloudfall_servers:!cloudfall_release_servers
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  serial: 1
   tasks:
     - name: Deploy the selected component on its declared servers
       ansible.builtin.include_role:

engine/ansible/roles/cloudfall_deploy/defaults/main.yml

@@ -4,6 +4,13 @@ cloudfall_deploy_receipt_directory: ""
 # A component without an HTTP health check passes its gate when its unit is
 # active right after the restart and still active this many seconds later.
 cloudfall_deploy_unit_settle_seconds: 10
+# A release command runs in a transient unit started by this binary.
+cloudfall_deploy_systemd_run: /usr/bin/systemd-run
+# The receipt keeps this much of the release command's output; a failure
+# message keeps less, so it stays inside the end of the Ansible log the
+# controller reports.
+cloudfall_deploy_release_output_characters: 2000
+cloudfall_deploy_release_failure_characters: 500
 cloudfall_deploy_uv_version: "0.12.10"
 cloudfall_deploy_uv_sha256: >-
   173d95a0c32d18c896c46ba6fafbf3cf9c14ab74b033f81b76c883ef492a976b

engine/ansible/roles/cloudfall_deploy/meta/argument_specs.yml

@@ -80,6 +80,21 @@ argument_specs:
           retainUntilCleanup:
             type: bool
             required: true
+          release:
+            type: dict
+            options:
+              command:
+                type: list
+                elements: str
+                required: true
+              workingDirectory:
+                type: str
+              timeoutSeconds:
+                type: int
+                required: true
+              server:
+                type: str
+                required: true
       cloudfall_deploy_release:
         type: str
         required: true
@@ -98,6 +113,15 @@ argument_specs:
       cloudfall_deploy_unit_settle_seconds:
         type: int
         default: 10
+      cloudfall_deploy_systemd_run:
+        type: str
+        default: /usr/bin/systemd-run
+      cloudfall_deploy_release_output_characters:
+        type: int
+        default: 2000
+      cloudfall_deploy_release_failure_characters:
+        type: int
+        default: 500
       cloudfall_deploy_uv_version:
         type: str
         default: "0.12.10"

engine/ansible/roles/cloudfall_deploy/tasks/main.yml

@@ -143,6 +143,33 @@
     UV_PYTHON_INSTALL_DIR: "{{ cloudfall_deploy_shared_directory }}/uv/python"
   changed_when: true
 
+- name: Run the release command on the release server
+  ansible.builtin.include_tasks: release_command.yml
+  when:
+    - not ansible_check_mode
+    - cloudfall_deploy_component.release is defined
+    - cloudfall_deploy_component.release.server == cloudfall_server_id
+
+- name: Require the release command to have passed before switching
+  ansible.builtin.assert:
+    that:
+      - cloudfall_deploy_release_evidence is mapping
+      - cloudfall_deploy_release_evidence.exitCode == 0
+    fail_msg: >-
+      The release command of {{ cloudfall_deploy_component.id }} has not run
+      on its release server {{ cloudfall_deploy_component.release.server }}
+      in this deploy, so {{ cloudfall_server_id }} does not switch
+  vars:
+    cloudfall_deploy_release_evidence: >-
+      {{
+        hostvars[cloudfall_deploy_component.release.server]
+        .cloudfall_deploy_release_command_result
+        | default(none)
+      }}
+  when:
+    - not ansible_check_mode
+    - cloudfall_deploy_component.release is defined
+
 - name: Inspect the currently active release
   ansible.builtin.stat:
     path: "{{ cloudfall_deploy_current_link }}"
@@ -349,7 +376,15 @@
               else none
             ),
             'healthCheckPassed': true,
-          },
+          } | combine(
+            {
+              'releaseCommand': hostvars[
+                cloudfall_deploy_component.release.server
+              ].cloudfall_deploy_release_command_result,
+            }
+            if cloudfall_deploy_component.release is defined
+            else {}
+          ),
         } | to_nice_json
       }}
     dest: >-

engine/ansible/roles/cloudfall_deploy/tasks/release_command.yml

@@ -0,0 +1,88 @@
+---
+# Runs the component's release command once against the unpacked release,
+# before the unit file changes and before current moves. A transient systemd
+# unit runs it, so the application user, the working directory, and the
+# EnvironmentFile= parsing are the ones the service unit gets.
+- name: Compute the release command invocation
+  ansible.builtin.set_fact:
+    cloudfall_deploy_release_working_directory: >-
+      {{ cloudfall_deploy_release_directory }}{{
+        '/' ~ cloudfall_deploy_component.release.workingDirectory
+        if cloudfall_deploy_component.release.workingDirectory is defined
+        else ''
+      }}
+    cloudfall_deploy_release_executable: >-
+      {{
+        cloudfall_deploy_component.release.command[0]
+        if cloudfall_deploy_component.release.command[0].startswith('/')
+        else cloudfall_deploy_release_directory ~ '/'
+        ~ cloudfall_deploy_component.release.command[0]
+      }}
+    cloudfall_deploy_release_timeout: >-
+      {{ cloudfall_deploy_component.release.timeoutSeconds }}
+
+- name: Run the release command against the new release
+  ansible.builtin.command:
+    argv: >-
+      {{
+        [
+          cloudfall_deploy_systemd_run,
+          '--unit=cloudfall-release-' ~ cloudfall_deploy_component.id,
+          '--wait',
+          '--pipe',
+          '--collect',
+          '--service-type=exec',
+          '--uid=' ~ cloudfall_deploy_component.application_user,
+          '--gid=' ~ cloudfall_deploy_component.application_user,
+          '--working-directory=' ~ cloudfall_deploy_release_working_directory,
+          '--property=EnvironmentFile=-' ~ cloudfall_deploy_shared_directory
+          ~ '/env/' ~ cloudfall_deploy_component.id ~ '.env',
+          '--property=RuntimeMaxSec=' ~ cloudfall_deploy_release_timeout,
+          '--property=TimeoutStopSec=10',
+          '--property=NoNewPrivileges=true',
+          '--property=PrivateTmp=true',
+          '--',
+          '/bin/sh',
+          '-c',
+          'exec "$@" 2>&1',
+          'cloudfall-release',
+          cloudfall_deploy_release_executable,
+        ]
+        + cloudfall_deploy_component.release.command[1:]
+      }}
+  register: cloudfall_deploy_release_run
+  changed_when: true
+  failed_when: false
+
+- name: Record the release command run
+  ansible.builtin.set_fact:
+    cloudfall_deploy_release_command_result:
+      server: "{{ cloudfall_server_id }}"
+      command: "{{ cloudfall_deploy_component.release.command }}"
+      workingDirectory: >-
+        {{ cloudfall_deploy_component.release.workingDirectory | default(none) }}
+      exitCode: "{{ cloudfall_deploy_release_run.rc }}"
+      outputTail: >-
+        {{ cloudfall_deploy_release_run.stdout[-cloudfall_deploy_release_output_characters:] }}
+    cloudfall_deploy_release_timed_out: >-
+      {{ 'Finished with result: timeout' in cloudfall_deploy_release_run.stderr }}
+
+- name: Fail the deployment when the release command failed
+  ansible.builtin.fail:
+    msg: >-
+      Release command {{ cloudfall_deploy_component.release.command | join(' ') }}
+      {{
+        'timed out after ' ~ cloudfall_deploy_release_timeout
+        ~ ' seconds and was stopped'
+        if cloudfall_deploy_release_timed_out
+        else 'exited with code ' ~ cloudfall_deploy_release_run.rc
+      }}
+      on {{ cloudfall_server_id }}; no server switched to release
+      {{ cloudfall_deploy_release }}. Output ends: {{
+        cloudfall_deploy_release_run.stdout[-cloudfall_deploy_release_failure_characters:]
+      }} systemd-run reported: {{
+        cloudfall_deploy_release_run.stderr[-cloudfall_deploy_release_failure_characters:]
+      }}
+  when: >-
+    cloudfall_deploy_release_timed_out
+    or cloudfall_deploy_release_run.rc != 0