Keep a hand-made unit and restore it when a first release fails (#151)

changelog.d/151-keep-hand-made-unit.md

@@ -0,0 +1,3 @@
+### Fixed
+
+- A first deploy over a hand-made systemd unit of the same name no longer loses it when the release fails its health gate: the deploy role keeps a unit without the `# Managed by Cloudfall` header at `/etc/systemd/system/<name>.service.pre-cloudfall` before installing the managed unit, and on a failed first release copies it back, restores its enabled state and starts it again when it was active; check mode reports the unit it would keep, and a hand-made unit that differs from an earlier kept copy refuses the deploy (#151)

engine/README.md

@@ -234,7 +234,16 @@ systemd unit from the declared service command, and then activates behind a
 health gate: symlink switch, restart, and the declared HTTP health check. If
 the health check fails, the previous release is restored and restarted
 automatically and the run fails; a first release with no predecessor is
-stopped instead. Successful runs write a schema-valid release receipt on the
+stopped instead. A unit of the same name that Cloudfall did not render (no
+`# Managed by Cloudfall` header, such as a hand-made unit adopted on a
+brownfield move) is kept at `/etc/systemd/system/<name>.service.pre-cloudfall`
+before the managed unit replaces it, together with whether it was enabled and
+active; when that first release fails its health check, the kept unit is
+copied back, its enabled state restored, and it is started again if it was
+running. The kept copy stays after a successful deploy and is never
+overwritten; a later hand-made unit that differs from it refuses the deploy.
+Check mode only reports the unit it would keep. Successful runs write a
+schema-valid release receipt on the
 controller. The first slice supports Python components managed by uv on
 x86_64 Debian hosts.
 

engine/ansible/roles/cloudfall_deploy/tasks/main.yml

@@ -187,11 +187,112 @@
         else ""
       }}
 
+- name: Compute service unit paths
+  ansible.builtin.set_fact:
+    cloudfall_deploy_unit_path: >-
+      /etc/systemd/system/{{ cloudfall_deploy_component.service.name }}.service
+    cloudfall_deploy_unit_kept_path: >-
+      /etc/systemd/system/{{
+        cloudfall_deploy_component.service.name
+      }}.service.pre-cloudfall
+
+- name: Inspect the existing service unit
+  ansible.builtin.stat:
+    path: "{{ cloudfall_deploy_unit_path }}"
+    follow: false
+    checksum_algorithm: sha256
+  register: cloudfall_deploy_unit_existing
+
+- name: Read the existing service unit
+  ansible.builtin.slurp:
+    src: "{{ cloudfall_deploy_unit_path }}"
+  register: cloudfall_deploy_unit_existing_content
+  when: cloudfall_deploy_unit_existing.stat.isreg | default(false)
+
+- name: Recognize a hand-made service unit
+  ansible.builtin.set_fact:
+    cloudfall_deploy_unit_hand_made: >-
+      {{
+        (cloudfall_deploy_unit_existing.stat.isreg | default(false))
+        and not (
+          cloudfall_deploy_unit_existing_content.content | b64decode
+        ).startswith('# Managed by Cloudfall')
+      }}
+
+- name: Inspect the kept pre-Cloudfall service unit
+  ansible.builtin.stat:
+    path: "{{ cloudfall_deploy_unit_kept_path }}"
+    follow: false
+    checksum_algorithm: sha256
+  register: cloudfall_deploy_unit_kept_before
+  when: cloudfall_deploy_unit_hand_made | bool
+
+- name: Refuse to replace a hand-made unit that differs from the one already kept
+  ansible.builtin.assert:
+    that:
+      - not cloudfall_deploy_unit_kept_before.stat.exists
+        or cloudfall_deploy_unit_kept_before.stat.checksum
+        == cloudfall_deploy_unit_existing.stat.checksum
+    fail_msg: >-
+      {{ cloudfall_deploy_unit_path }} is not managed by Cloudfall and differs
+      from the unit already kept at {{ cloudfall_deploy_unit_kept_path }};
+      move one of them away so Cloudfall keeps exactly the unit it replaces
+  when: cloudfall_deploy_unit_hand_made | bool
+
+- name: Record whether the hand-made service unit is enabled
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/systemctl
+      - is-enabled
+      - "{{ cloudfall_deploy_component.service.name }}.service"
+  register: cloudfall_deploy_unit_was_enabled
+  changed_when: false
+  failed_when: false
+  when:
+    - not ansible_check_mode
+    - cloudfall_deploy_unit_hand_made | bool
+
+- name: Record whether the hand-made service unit is active
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/systemctl
+      - is-active
+      - --quiet
+      - "{{ cloudfall_deploy_component.service.name }}.service"
+  register: cloudfall_deploy_unit_was_active
+  changed_when: false
+  failed_when: false
+  when:
+    - not ansible_check_mode
+    - cloudfall_deploy_unit_hand_made | bool
+
+- name: Report the hand-made service unit a deploy would replace
+  ansible.builtin.debug:
+    msg: >-
+      {{ cloudfall_deploy_unit_path }} is not managed by Cloudfall; a deploy
+      keeps it at {{ cloudfall_deploy_unit_kept_path }} and replaces it with
+      the managed unit
+  when:
+    - ansible_check_mode
+    - cloudfall_deploy_unit_hand_made | bool
+
+- name: Keep the hand-made service unit before replacing it
+  ansible.builtin.copy:
+    src: "{{ cloudfall_deploy_unit_path }}"
+    dest: "{{ cloudfall_deploy_unit_kept_path }}"
+    remote_src: true
+    force: false
+    owner: root
+    group: root
+    mode: "0644"
+  when:
+    - not ansible_check_mode
+    - cloudfall_deploy_unit_hand_made | bool
+
 - name: Install the managed service unit
   ansible.builtin.template:
     src: component.service.j2
-    dest: >-
-      /etc/systemd/system/{{ cloudfall_deploy_component.service.name }}.service
+    dest: "{{ cloudfall_deploy_unit_path }}"
     owner: root
     group: root
     mode: "0644"
@@ -333,6 +434,55 @@
         state: stopped
       when: cloudfall_deploy_previous_directory | length == 0
 
+    - name: Restore the pre-Cloudfall service unit
+      ansible.builtin.copy:
+        src: "{{ cloudfall_deploy_unit_kept_path }}"
+        dest: "{{ cloudfall_deploy_unit_path }}"
+        remote_src: true
+        force: true
+        owner: root
+        group: root
+        mode: "0644"
+      when:
+        - cloudfall_deploy_previous_directory | length == 0
+        - not ansible_check_mode
+        - cloudfall_deploy_unit_hand_made | bool
+
+    - name: Reload systemd for the restored pre-Cloudfall unit
+      ansible.builtin.systemd_service:
+        daemon_reload: true
+      when:
+        - cloudfall_deploy_previous_directory | length == 0
+        - not ansible_check_mode
+        - cloudfall_deploy_unit_hand_made | bool
+
+    - name: Restore the enabled state of the pre-Cloudfall unit
+      ansible.builtin.command:
+        argv:
+          - /usr/bin/systemctl
+          - >-
+            {{
+              'reenable'
+              if cloudfall_deploy_unit_was_enabled.stdout == 'enabled'
+              else 'disable'
+            }}
+          - "{{ cloudfall_deploy_component.service.name }}.service"
+      changed_when: true
+      when:
+        - cloudfall_deploy_previous_directory | length == 0
+        - not ansible_check_mode
+        - cloudfall_deploy_unit_hand_made | bool
+
+    - name: Start the pre-Cloudfall unit that was active before the deploy
+      ansible.builtin.systemd_service:
+        name: "{{ cloudfall_deploy_component.service.name }}.service"
+        state: started
+      when:
+        - cloudfall_deploy_previous_directory | length == 0
+        - not ansible_check_mode
+        - cloudfall_deploy_unit_hand_made | bool
+        - cloudfall_deploy_unit_was_active.rc == 0
+
     - name: Fail the deployment after rolling back
       ansible.builtin.fail:
         msg: >-
@@ -342,6 +492,18 @@
             ~ (cloudfall_deploy_previous_directory | basename)
             if cloudfall_deploy_previous_directory | length > 0
             else 'stopped because no previous release exists'
+          }}{{
+            '; the pre-Cloudfall unit was restored from '
+            ~ cloudfall_deploy_unit_kept_path
+            ~ (
+              ' and started'
+              if cloudfall_deploy_unit_was_active.rc == 0
+              else ' and left stopped as it was before the deploy'
+            )
+            if cloudfall_deploy_previous_directory | length == 0
+            and not ansible_check_mode
+            and cloudfall_deploy_unit_hand_made | bool
+            else ''
           }}
 
 - name: Ensure the controller receipt directory exists