diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index b7022918..488346e8 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -46,7 +46,9 @@ # # keel, inithooks and confconsole are not published yet (Keel-Linux/apt: # the signing subkey is being rotated), so they are built here from the -# commits pinned below: keel 0.12.0 (Keel-Linux/keel#57), inithooks +# commits pinned below: keel 0.21.0 (the merge of Keel-Linux/keel#87, +# which keel-overlay-vip 0.2.0 and keel-overlay-etcd 0.3.0 depend on), +# inithooks # 2.3.6+keel14 and confconsole 2.2.3+keel8, the versions # keel-overlay-installer depends on; and keel 0.14.0 for the Web overlays, # whose spec side the suite reads. Their own tests are their CI's. Their @@ -84,7 +86,7 @@ jobs: download: overlay-debs download-dir: dist run: | - KEEL_COMMIT=befaaea21e9c983fbb8b2ad4e47f9625866ae857 + KEEL_COMMIT=ebede2e08ac74332c36a81fde0a7575a7e1ffd60 INITHOOKS_COMMIT=2543d506590a17a762a4288e2ee99b0a3c160386 CONFCONSOLE_COMMIT=71828736ca7e99fff1d99ce5e9e9002c24dbd8b2 apt-get update -qq @@ -124,7 +126,8 @@ jobs: ./dist/keel-overlay-vip_*.deb \ ntpsec # unit states and manifests on the machine - KEEL_OVERLAY_INSTALL_TEST=1 bats tests/overlay-install.bats + KEEL_OVERLAY_INSTALL_TEST=1 KEEL_DEB="$(ls /tmp/deps/keel_*.deb)" \ + bats tests/overlay-install.bats # the units a container cannot run, under conf/turnkey.d/container-units; # last, because it masks networkd on this machine KEEL_OVERLAY_INSTALL_TEST=1 bats tests/container-units-live.bats diff --git a/packages/README.md b/packages/README.md index 7336c3ff..79a17336 100644 --- a/packages/README.md +++ b/packages/README.md @@ -11,8 +11,8 @@ directory each, with its own changelog and version, released on its own | --- | --- | --- | --- | | `installer/` | `keel-overlay-installer` | Keel's `inithooks` (>= 2.3.6+keel14), `confconsole` (>= 2.2.3+keel8), `keel` (>= 0.12.0) | none; its manifest names the first boot hooks the three ship | | `wireguard/` | `keel-overlay-wireguard` | trixie's `wireguard-tools` 1.0.20210914 | none; the interface is the instance spec's | -| `etcd/` | `keel-overlay-etcd` | trixie's `etcd-server` 3.5.16, and `keel-overlay-wireguard`, which its manifest `requires` | `etcd.service` | -| `vip/` | `keel-overlay-vip` | `keel` (>= 0.20.0), whose `keel vip` the units run, and `keel-overlay-wireguard`, which its manifest `requires` | `keel-vip.service`, the VIP's root helper with etcd, which starts its unprivileged controller as the transient `keel-vip-control`; and `keel-vip-check.timer`, enabled in every mode | +| `etcd/` | `keel-overlay-etcd` | trixie's `etcd-server` 3.5.16, `keel` (>= 0.21.0), whose gate its drop-in runs, and `keel-overlay-wireguard`, which its manifest `requires` | `etcd.service`, and its drop-in `keel-overlay-etcd.conf`: one member restarts at a time | +| `vip/` | `keel-overlay-vip` | `keel` (>= 0.21.0), whose `keel vip` the units run, and `keel-overlay-wireguard`, which its manifest `requires` | `keel-vip.service`, the VIP's root helper with etcd, which starts its unprivileged controller as the transient `keel-vip-control`; and `keel-vip-check.timer`, part of it | | `crowdsec/` | `keel-overlay-crowdsec` | trixie's `crowdsec` 1.4.6-10 and `crowdsec-firewall-bouncer` 0.0.25 | `crowdsec.service`, `crowdsec-firewall-bouncer.service` | | `nginx/` | `keel-overlay-nginx` | trixie's `nginx` 1.26.3 and `libnginx-mod-stream` | `nginx.service`, enabled in every mode | | `coraza/` | `keel-overlay-coraza` | Keel's `libnginx-mod-http-coraza` 0.21.0 and `coreruleset` 4.25.1 (step 5), and `keel-overlay-nginx` | none: an Nginx module; `/usr/lib/keel/overlays/coraza/state` turns it on and off | @@ -138,6 +138,35 @@ none of the units is masked. spec enabled them; `keel diff` reports that as drift and `keel apply` converges it. +## Upgrades where the VIP is active + +The maintainer's requirement of 2026-10-10: upgrading packages never +breaks the service on a machine where the VIP is active, beyond a short +blip that keel's CI measures (keel's `docs/vip.md`, "Upgrading a pair +without downtime"). + +- **keel-overlay-vip** is built with `--no-stop-on-upgrade`. Its preinst + stops nothing, and its postinst try-restarts `keel-vip.service` where it + runs, and starts it nowhere it does not. 0.1.1's `--no-start` alone made + the preinst stop the controller and its check on every upgrade and + start neither again. A file trigger on keel's Python files + (`/usr/lib/python3/dist-packages/keel`) try-restarts it on an upgrade of + keel too, so the helper runs the new code. The restart keeps the VIP: + keel (>= 0.21.0) carries the address with a lifetime the kernel ends at + the release time, keeps it in ExecStopPost while the unit restarts, and + renews the same lease from the new controller. +- **keel-overlay-etcd** ships a drop-in of `etcd.service`. Its ExecStop + runs `keel mesh etcd gate stop`, and its ExecStartPost runs `keel mesh + etcd gate started`. etcd-server's postinst restarts etcd on every + upgrade. The gate makes that restart wait until every other member is + healthy and none restarts, so two members upgraded at once never lose + the majority. A stop is never refused for good. +- `tests/overlay-install.bats` checks four cases on a booted trixie + container, upgrading to a built package with a bumped version: a + running controller restarted, a stopped one left stopped, an upgrade of + keel restarting it through the trigger, and etcd restarting through its + gate. + ## CrowdSec's identity: work for step 4 (tracker#47) trixie's `crowdsec` registers the machine with its local API (keyed on diff --git a/packages/etcd/debian/changelog b/packages/etcd/debian/changelog index 2c205e09..9551e0df 100644 --- a/packages/etcd/debian/changelog +++ b/packages/etcd/debian/changelog @@ -1,3 +1,21 @@ +keel-overlay-etcd (0.3.0) trixie; urgency=medium + + * etcd restarted one member at a time (the maintainer's requirement of + 2026-10-10: an upgrade never breaks the service where the VIP is + active). etcd-server's postinst restarts etcd on every upgrade and + knows nothing of the other members, so apt run on two of three + members at once lost the majority while both restarted. A drop-in of + etcd.service, keel-overlay-etcd.conf, runs keel mesh etcd gate stop + as its ExecStop: it waits until every other voter is healthy and no + other member restarts, then takes the mesh's restart lock; and keel + mesh etcd gate started as its ExecStartPost, which releases it once + this member is back in the majority. A stop is never refused for + good (300 s, then it goes on; none at shutdown), and a node in no + etcd cluster passes at once. + * Depends on keel (>= 0.21.0), which has the gate. + + -- KeelLinux maintainers Thu, 08 Oct 2026 12:00:00 +0000 + keel-overlay-etcd (0.2.0) trixie; urgency=medium * Monit's etcd-health check asks /health on etcd's plain metrics diff --git a/packages/etcd/debian/control b/packages/etcd/debian/control index 274d6d5d..f79f19c4 100644 --- a/packages/etcd/debian/control +++ b/packages/etcd/debian/control @@ -14,7 +14,7 @@ Package: keel-overlay-etcd Architecture: all Depends: etcd-server (>= 3.5.16), - keel (>= 0.12.0), + keel (>= 0.21.0), keel-overlay-wireguard, ${misc:Depends}, Description: Keel overlay: etcd, the mesh registry @@ -29,3 +29,8 @@ Description: Keel overlay: etcd, the mesh registry spec says so. On a running system where etcd was already enabled or running before this package, it is left as it was; an image build always ends with it disabled. + . + A drop-in of etcd.service restarts etcd one member at a time: before it + stops, keel mesh etcd gate waits until every other member is healthy and + none restarts, so an upgrade of etcd-server on several members at once + never loses the cluster's majority. diff --git a/packages/etcd/debian/postinst b/packages/etcd/debian/postinst index 8effb757..35b6d331 100755 --- a/packages/etcd/debian/postinst +++ b/packages/etcd/debian/postinst @@ -54,6 +54,12 @@ if [ "$1" = configure ] && [ -z "$2" ]; then rmdir "${KEPT%/*}" 2>/dev/null || : fi +# the drop-in of etcd.service (etcd restarted one member at a time) read +# by systemd, on every configuration; it starts nothing +if [ "$1" = configure ] && [ -d /run/systemd/system ]; then + systemctl --system daemon-reload >/dev/null || true +fi + #DEBHELPER# exit 0 diff --git a/packages/etcd/debian/postrm b/packages/etcd/debian/postrm index 0dad80a7..1d85d56d 100755 --- a/packages/etcd/debian/postrm +++ b/packages/etcd/debian/postrm @@ -10,6 +10,11 @@ set -e KEPT=/var/lib/keel-overlay-etcd/kept-units +# the drop-in of etcd.service gone: systemd forgets it +if [ "$1" = remove ] && [ -d /run/systemd/system ]; then + systemctl --system daemon-reload >/dev/null || true +fi + if [ "$1" = purge ] || [ "$1" = abort-install ]; then rm -f "$KEPT" /var/lib/keel-overlay-etcd/no-member-before \ /var/lib/keel-overlay-etcd/package-member diff --git a/packages/etcd/debian/rules b/packages/etcd/debian/rules index 7ed0c63d..d05adaec 100755 --- a/packages/etcd/debian/rules +++ b/packages/etcd/debian/rules @@ -1,7 +1,8 @@ #!/usr/bin/make -f # The overlay manifest and the systemd preset sit beside debian/ (docs/ # manifest-v1.md, "Kinds, and where each file lives") and are installed -# under the names keel and systemd look them up by. +# under the names keel and systemd look them up by. keel-overlay-etcd.conf +# is a drop-in of etcd.service: etcd restarted one member at a time. %: dh $@ @@ -11,3 +12,5 @@ execute_after_dh_auto_install: debian/keel-overlay-etcd/usr/share/keel/overlays/etcd.yaml install -D -m 0644 keel-overlay-etcd.preset \ debian/keel-overlay-etcd/usr/lib/systemd/system-preset/20-keel-overlay-etcd.preset + install -D -m 0644 keel-overlay-etcd.conf \ + debian/keel-overlay-etcd/usr/lib/systemd/system/etcd.service.d/keel-overlay-etcd.conf diff --git a/packages/etcd/keel-overlay-etcd.conf b/packages/etcd/keel-overlay-etcd.conf new file mode 100644 index 00000000..e8ef4bc3 --- /dev/null +++ b/packages/etcd/keel-overlay-etcd.conf @@ -0,0 +1,18 @@ +# keel-overlay-etcd: etcd restarted one member at a time (keel's +# docs/mesh.md, "keel mesh etcd gate, and upgrades"). etcd-server's +# postinst restarts etcd on every upgrade and knows nothing of the other +# members: apt run on two members of three at once would lose the +# majority while both restart. Before etcd is sent SIGTERM, the gate +# waits until every other voter is healthy and no other member is +# restarting, and takes the mesh's restart lock (a key on a lease of +# 120 s); once etcd started, it waits until this member is back in the +# majority and releases the lock. A stop is never refused for good: +# after 300 s the gate lets it go on, and at shutdown it does not wait. +# On a node in no etcd cluster the gate passes at once. Both run as root +# (+), which reads keel's etcd client certificate; etcd itself still runs +# as the etcd user. A failed start gate fails nothing (-). +[Service] +ExecStop=+/usr/bin/keel mesh etcd gate stop +ExecStartPost=-+/usr/bin/keel mesh etcd gate started +TimeoutStopSec=330 +TimeoutStartSec=180 diff --git a/packages/vip/debian/changelog b/packages/vip/debian/changelog index 18184818..d7da933d 100644 --- a/packages/vip/debian/changelog +++ b/packages/vip/debian/changelog @@ -1,3 +1,22 @@ +keel-overlay-vip (0.2.0) trixie; urgency=medium + + * An upgrade keeps the VIP (the maintainer's requirement of + 2026-10-10). 0.1.1's dh_installsystemd --no-start made the preinst + stop keel-vip.service and its check on every upgrade and nothing start + them again: the holder dropped the VIP and the replica had no + controller to claim it, until a reboot. With --no-stop-on-upgrade the + preinst stops nothing, and the postinst try-restarts keel-vip.service + where it runs, never starting it where it does not. + * A trigger on keel's Python files try-restarts it when keel is + upgraded, so the helper runs the new code. + * Depends on keel (>= 0.21.0), whose controller keeps the address over + a restart: a lifetime the kernel ends by the release time, kept by + ExecStopPost while the unit restarts, the same lease renewed by the + next controller. RestartSec=2, so a crashed helper is back well within + the address's lifetime. + + -- KeelLinux maintainers Thu, 08 Oct 2026 12:00:00 +0000 + keel-overlay-vip (0.1.1) trixie; urgency=medium * Every unit is off on the first installation, like the other overlays diff --git a/packages/vip/debian/control b/packages/vip/debian/control index e77eed1c..cbe30ba1 100644 --- a/packages/vip/debian/control +++ b/packages/vip/debian/control @@ -14,7 +14,7 @@ Package: keel-overlay-vip Architecture: all Depends: iproute2, - keel (>= 0.20.0), + keel (>= 0.21.0), keel-overlay-wireguard, wireguard-tools, ${misc:Depends}, @@ -27,5 +27,6 @@ Description: Keel overlay: the service VIP of a replicated pair keel-vip.service is the VIP's root helper with etcd, which starts the unprivileged controller as its own transient unit; installed disabled and stopped: keel spec apply enables it with the overlay. keel-vip-check.timer - runs keel vip check at boot and every minute in every mode; a node that - knows no VIP skips it. + runs keel vip check at boot and every minute while the controller runs. + An upgrade of this package or of keel try-restarts the controller where + it runs, which keeps the VIP on the holder. diff --git a/packages/vip/debian/postinst b/packages/vip/debian/postinst new file mode 100755 index 00000000..26bebce8 --- /dev/null +++ b/packages/vip/debian/postinst @@ -0,0 +1,35 @@ +#!/bin/sh +# keel-overlay-vip: keel-vip.service restarted where it runs, never +# stopped first and never started where it was not running, on an upgrade +# of this package and when keel's code changes (debian/triggers: an +# upgrade of keel). The restart keeps the VIP on the holder: the address +# carries a lifetime the kernel ends by the release time of the last +# renewal the majority confirmed, ExecStopPost keeps it while the unit +# restarts, and the next controller renews the same lease (keel's +# docs/vip.md, "Upgrading a pair without downtime"). keel-vip-check.timer +# is PartOf keel-vip.service and restarts with it. A first installation +# changes nothing: both units stay disabled and stopped (decision 0041). +set -e + +restart() { + if [ -d /run/systemd/system ]; then + systemctl --system daemon-reload >/dev/null || true + # deb-systemd-invoke honours policy-rc.d, as an image build needs + deb-systemd-invoke try-restart keel-vip.service || true + fi +} + +case "$1" in + configure) + if [ -n "$2" ]; then + restart + fi + ;; + triggered) + restart + ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/packages/vip/debian/rules b/packages/vip/debian/rules index 85f15446..9ba2478c 100755 --- a/packages/vip/debian/rules +++ b/packages/vip/debian/rules @@ -22,7 +22,13 @@ execute_after_dh_auto_install: $(PKG)/usr/lib/systemd/system-preset/20-keel-overlay-vip.preset # The controller is installed disabled and stopped, the state of a simple -# installation: keel spec apply enables it with the overlay. The check's -# timer runs in every mode. +# installation: keel spec apply enables it with the overlay. +# --no-stop-on-upgrade: without it, --no-start makes the preinst stop both +# units on every upgrade and nothing start them again, which dropped the +# VIP on the holder and left the replica without a controller. The +# postinst try-restarts keel-vip.service instead, on an upgrade of this +# package and of keel (debian/triggers), which keeps the VIP (keel's +# docs/vip.md, "Upgrading a pair without downtime"). override_dh_installsystemd: - dh_installsystemd --no-enable --no-start keel-vip.service keel-vip-check.timer + dh_installsystemd --no-enable --no-start --no-stop-on-upgrade \ + keel-vip.service keel-vip-check.timer diff --git a/packages/vip/debian/triggers b/packages/vip/debian/triggers new file mode 100644 index 00000000..65936c9e --- /dev/null +++ b/packages/vip/debian/triggers @@ -0,0 +1,4 @@ +# keel's code, which keel-vip.service runs: an upgrade of keel try-restarts +# it (debian/postinst), so the helper and its controller run the new code +# rather than old code that imports new modules as it goes +interest-noawait /usr/lib/python3/dist-packages/keel diff --git a/packages/vip/keel-vip.service b/packages/vip/keel-vip.service index 65409ca7..da698910 100644 --- a/packages/vip/keel-vip.service +++ b/packages/vip/keel-vip.service @@ -12,9 +12,14 @@ # unit's MainPID); it sends it the etcd client certificate's key as a # memfd. The holder renews its etcd lease (TTL 20 s) every 2 s and # carries the VIP only while the last renewal the majority confirmed is -# under 10 s old. ExecStopPost drops every VIP this node carries, however -# the helper stopped, and the unit is never left stopped for good: a -# crash loop is retried. +# under 10 s old, and gives the address the rest of those 10 s as its +# lifetime, so the kernel removes it by then whatever becomes of the +# controller. ExecStopPost drops every VIP this node carries, except +# while the unit restarts (an upgrade's try-restart, or a crash that +# Restart= starts again 2 s later): then the address stays for its +# lifetime and the next controller renews the same lease, so an upgrade +# keeps the VIP. The unit is never left stopped for good: a crash loop +# is retried. [Unit] Description=keel vip: the VIP's root helper with etcd Documentation=https://github.com/Keel-Linux/keel/blob/main/docs/vip.md @@ -27,7 +32,7 @@ StartLimitIntervalSec=0 ExecStart=/usr/bin/keel vip tend ExecStopPost=/usr/bin/keel vip tend --stopped Restart=always -RestartSec=5 +RestartSec=2 CapabilityBoundingSet=CAP_NET_ADMIN NoNewPrivileges=yes PrivateTmp=yes diff --git a/tests/overlay-install.bats b/tests/overlay-install.bats index 8e29cc38..fa79e74c 100644 --- a/tests/overlay-install.bats +++ b/tests/overlay-install.bats @@ -49,6 +49,14 @@ teardown() { # a test that died while it played an image build gives systemd back # its marker before anything else asks whether systemd runs restore_systemd_marker + # a test that let the maintainer scripts act on units puts fab's + # policy-rc.d back, and stops the VIP controller's stand-in + restore_policy + if [ -e "$VIP_STAND_IN" ]; then + systemctl stop keel-vip.service >/dev/null 2>&1 || true + rm -rf "${VIP_STAND_IN%/*}" + systemctl daemon-reload + fi # a test that enabled a unit to play keel apply leaves it as it found it if [ -n "${ENABLED_BY_TEST:-}" ]; then systemctl disable $ENABLED_BY_TEST >/dev/null 2>&1 || true @@ -174,6 +182,44 @@ assert_enabled_state() { [ "$output" = "$expected" ] } +# fab's policy-rc.d, which the CI job writes and which refuses every +# action of deb-systemd-invoke, moved aside for a test that needs the +# maintainer scripts to act on units as on a running machine; teardown +# puts it back +POLICY=/usr/sbin/policy-rc.d +POLICY_HIDDEN=/usr/sbin/policy-rc.d.keel-test + +allow_unit_actions() { + if [ -e "$POLICY" ]; then + mv "$POLICY" "$POLICY_HIDDEN" + fi +} + +restore_policy() { + if [ -e "$POLICY_HIDDEN" ]; then + mv "$POLICY_HIDDEN" "$POLICY" + fi +} + +# keel-vip.service with its ExecStart replaced by a sleep, and its +# condition on etcd's cluster lifted: what the maintainer scripts do to +# the unit is under test here, not the VIP, which keel's CI measures end +# to end (keel's tests/test_vip_upgrade_netns.py). A runtime drop-in, +# which teardown removes. +VIP_STAND_IN=/run/systemd/system/keel-vip.service.d/keel-test.conf + +vip_stand_in() { + mkdir -p "${VIP_STAND_IN%/*}" + printf '%s\n' "[Unit]" "ConditionPathExists=" "[Service]" \ + "ExecStart=" "ExecStart=/bin/sleep infinity" "ExecStopPost=" \ + > "$VIP_STAND_IN" + systemctl daemon-reload +} + +invocation() { + systemctl show --property=InvocationID --value "$1" +} + assert_active_state() { local unit="$1" expected="$2" systemd_running || return 0 @@ -431,6 +477,80 @@ assert_simple_state() { assert_enabled_state crowdsec-firewall-bouncer.service disabled } +@test "an upgrade of keel-overlay-vip restarts a running controller, never stops it for good" { + systemd_running || skip "systemd does not run here" + vip_stand_in + allow_unit_actions + systemctl start keel-vip.service + local before + before="$(invocation keel-vip.service)" + + dpkg -i "$(with_version "$(overlay_deb vip)" 0.2.0+keeltest1)" + + # 0.1.1's preinst stopped it and nothing started it again: the holder + # dropped the VIP until a reboot. Now it is try-restarted: running, + # as a new invocation, its check's timer with it + assert_active_state keel-vip.service active + [ "$(invocation keel-vip.service)" != "$before" ] + assert_active_state keel-vip-check.timer active + # and back to the built package, a configuration over a newer version + dpkg -i "$(overlay_deb vip)" + assert_active_state keel-vip.service active +} + +@test "an upgrade of keel-overlay-vip starts no controller that was stopped" { + systemd_running || skip "systemd does not run here" + vip_stand_in + allow_unit_actions + systemctl stop keel-vip.service + + dpkg -i "$(with_version "$(overlay_deb vip)" 0.2.0+keeltest2)" + + assert_active_state keel-vip.service inactive + assert_active_state keel-vip-check.timer inactive + assert_enabled_state keel-vip.service disabled + dpkg -i "$(overlay_deb vip)" +} + +@test "an upgrade of keel restarts the VIP controller, so it runs the new code" { + systemd_running || skip "systemd does not run here" + [ -n "${KEEL_DEB:-}" ] || skip "KEEL_DEB names no keel package to upgrade to" + vip_stand_in + allow_unit_actions + systemctl start keel-vip.service + local before + before="$(invocation keel-vip.service)" + + # keel's files under /usr/lib/python3/dist-packages/keel activate + # keel-overlay-vip's trigger, whose postinst try-restarts the unit + dpkg -i "$(with_version "$KEEL_DEB" "$(dpkg-deb -f "$KEEL_DEB" Version)+keeltest1")" + + assert_active_state keel-vip.service active + [ "$(invocation keel-vip.service)" != "$before" ] + dpkg -i "$KEEL_DEB" +} + +@test "etcd restarts through keel's gate, which a node in no cluster passes at once" { + run systemctl cat etcd.service + [ "$status" -eq 0 ] + grep -qx 'ExecStop=+/usr/bin/keel mesh etcd gate stop' <<<"$output" + grep -qx 'ExecStartPost=-+/usr/bin/keel mesh etcd gate started' <<<"$output" + grep -qx 'TimeoutStopSec=330' <<<"$output" + # the gate itself, as root, on this machine in no etcd cluster + run keel mesh etcd gate stop --wait 0 + [ "$status" -eq 0 ] + run keel mesh etcd gate started --wait 0 + [ "$status" -eq 0 ] + systemd_running || return 0 + enable_unit etcd.service + local started + started="$(date +%s)" + systemctl restart etcd.service + assert_active_state etcd.service active + # nothing to wait for: well under the gate's own wait + [ $(( $(date +%s) - started )) -lt 60 ] +} + @test "keel apply can still enable the units: none of them is masked" { local unit for unit in "${DISABLED_UNITS[@]}"; do