diff --git a/.github/workflows/mesh-integration.yml b/.github/workflows/mesh-integration.yml index 2da686fa..8b49078e 100644 --- a/.github/workflows/mesh-integration.yml +++ b/.github/workflows/mesh-integration.yml @@ -1,6 +1,6 @@ name: Mesh integration -# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. rejoin removes the bootstrap Docker server's etcd member and checks the server idles on its tombstone, refuses to start a second cluster from its moved-aside datastore with no server to join, joins the existing cluster from a join URL in its role directory with the same cluster ID, and idles as a server whose role file names the agent. role_controller adds a standby agent, runs the role controller's probe and controller from the image built from this checkout, makes a voter slow with tc netem and pauses another, and checks every probe reports the link medium the cluster role labelled its node with, the controller reports the voter unfit with the swap named, reports that it would not act with a voter paused, and changes nothing. role_actuator acts as the role controller by hand on three Docker servers and an eligible native standby: the native host's role actuator promotes it to a voter with writes succeeding throughout, steps it down to etcd-only with no pod's shim left, and once its member is removed through k3s's annotation finishes it as an agent with the same node password and Tailscale address and its datastore moved aside, the two k3s units never running together; then a Docker server's role-actuator service demotes it the same way, its agent keeping the server's node password and Tailscale address as one Node with the datastore moved aside, and promotes it again into the same etcd cluster. Only on changes that can affect how nodes join the mesh. +# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. rejoin removes the bootstrap Docker server's etcd member and checks the server idles on its tombstone, refuses to start a second cluster from its moved-aside datastore with no server to join, joins the existing cluster from a join URL in its role directory with the same cluster ID, and idles as a server whose role file names the agent. role_controller adds a standby agent, runs the role controller's probe and controller from the image built from this checkout, makes a voter slow with tc netem and pauses another, and checks every probe reports the link medium the cluster role labelled its node with, the controller reports the voter unfit with the swap named, reports that it would not act with a voter paused, and changes nothing. role_actuator acts as the role controller by hand on three Docker servers and an eligible native standby: the native host's role actuator promotes it to a voter with writes succeeding throughout, steps it down to etcd-only with no pod's shim left, and once its member is removed through k3s's annotation finishes it as an agent with the same node password and Tailscale address and its datastore moved aside, the two k3s units never running together; then a Docker server's role-actuator service demotes it the same way, its agent keeping the server's node password and Tailscale address as one Node with the datastore moved aside, and promotes it again into the same etcd cluster. role_swap, role_swap_paused, role_swap_blocked_peer and role_swap_refused run the role controller's own voter swap by command, with the controller's image built from this checkout: a native standby in and the Docker bootstrap host out with writes throughout and the bootstrap host back in a second swap; the outgoing voter paused (the removal still completes, and the tombstone keeps it out on its return); the candidate's peer port dropped (the promotion is rolled back); and a swap refused while a voter is paused. role_controller_swap runs the controller at the ARC role's default period, makes a voter slow, and turns the dry run off, checking the controller swaps the slow voter for the Docker standby on its own decision; it is a scenario of its own because filling a window of samples at that period takes as long as the controller says. Only on changes that can affect how nodes join the mesh. # # Runs on GitHub-hosted ubuntu-latest, whose Docker daemon runs the privileged containers the test needs (k3s, and Tailscale inside it). The work directory sits under runner.temp, which the job always has write access to. @@ -32,7 +32,8 @@ jobs: mesh: name: Mesh (${{ matrix.scenario }}, Compose ${{ matrix.compose }}${{ matrix.pin && ', context pinned' || '' }}) runs-on: ubuntu-latest - timeout-minutes: 45 + # role_controller_swap first fills 100 samples at the default 15-second period (25 minutes), waits up to 15 minutes for the induced unfitness and then allows the controller's own swap budget: seven 5-minute operations, a 10-minute runner wait and a 5-minute learner stall (50 minutes). Those bounds total 90 minutes before cluster setup, so CI allows the 15-minute setup and teardown budget as well. + timeout-minutes: 105 strategy: fail-fast: false matrix: @@ -44,8 +45,13 @@ jobs: - {scenario: regrade, compose: v5.5.1} - {scenario: native_server, compose: v5.5.1} - {scenario: role_controller, compose: v5.5.1} + - {scenario: role_controller_swap, compose: v5.5.1} - {scenario: rejoin, compose: v5.5.1} - {scenario: role_actuator, compose: v5.5.1} + - {scenario: role_swap, compose: v5.5.1} + - {scenario: role_swap_paused, compose: v5.5.1} + - {scenario: role_swap_blocked_peer, compose: v5.5.1} + - {scenario: role_swap_refused, compose: v5.5.1} - {scenario: hosted, compose: v2.39.0} - {scenario: compose_faulty, compose: v2.38.2} - {scenario: in_cluster, compose: v5.5.1, pin: true} diff --git a/README.md b/README.md index 2e5ee814..4a552707 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ Its image, `github_runner_arc_node_recovery_image`, is pulled without a pull Sec ### Role controller -`fleet-role-probe` and `fleet-role` (an optional DaemonSet and Deployment in the platform namespace, see `roles/github_runner_arc/templates/fleet-role.yaml.j2`, `scripts/fleet-role-probe.sh` and `scripts/fleet-role.sh`; `github_runner_cluster_role_controller`, off by default, which `github_runner_arc_role_controller_enabled` takes as its default) measure which hosts are fit to hold an etcd vote and report the swap of an unfit voter for a fit standby that the controller would make. They only report: the pods' ClusterRoles can `get` and `list` nodes and nothing else, and the only objects they write are their own two ConfigMaps. The controller refuses to start with any setting other than `FLEET_ROLE_DRY_RUN=true`, and runs unchanged when no host is marked eligible, when it reports the voters' fitness alone. +`fleet-role-probe` and `fleet-role` (an optional DaemonSet and Deployment in the platform namespace, see `roles/github_runner_arc/templates/fleet-role.yaml.j2`, `scripts/fleet-role-probe.sh` and `scripts/fleet-role.sh`; `github_runner_cluster_role_controller`, off by default, which `github_runner_arc_role_controller_enabled` takes as its default) measure which hosts are fit to hold an etcd vote, report the swap of an unfit voter for a fit standby that the controller would make and, when asked or when `github_runner_arc_role_controller_dry_run` is false, carry the swap out ([Voter swap](#voter-swap)). The probe only reports: its ClusterRole can `get` and `list` nodes and nothing else, and the only object it writes is its own ConfigMap. The controller runs unchanged when no host is marked eligible, when it reports the voters' fitness alone. The probe runs on every Linux node that is an etcd voter (k3s's `node-role.kubernetes.io/etcd` label) or is marked eligible and has a kubelet (a server regraded to etcd-only has none, and its side is read from its peers), on the node's own network, through a node affinity on those two labels, so a node that is neither (a Unraid host, an image-flashed agent) has no pod and nothing is asked of its kernel. A pod stopped on a node that has lost both labels, or is gone, removes the node's entry as it ends, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the control plane and not with the fleet. Every `github_runner_arc_role_controller_sample_seconds` (15) it appends one sample to its node's key of the `fleet-role-probe` ConfigMap, which keeps the last window of them so that a replaced pod loses nothing. A sample holds the TCP connect time to every voter's etcd peer port (a connect that does not finish within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what its etcd metrics (the `127.0.0.1:2381` listener) gained since the last sample: the WAL fsync and backend commit histograms, the round trip to each peer, leader changes, and whether it has a leader, is a learner and knows as many members as there are voters (a learner or a member that has not started shows as an extra known member); on a node that is not a voter, an fdatasync reading from fio with the write size etcd's own disk check uses, once per window; the memory pressure (`/proc/pressure/memory` of the pod's own procfs, which serves the whole node's, so nothing is mounted) and `MemAvailable` the regrade core reads, with the core's own functions, where a node on a kernel without pressure stall information (built or booted without it, which has no `/proc/pressure` or, on older kernels, has the files and refuses to read them) gives `MemAvailable` and `MemTotal` and no pressure figure; and the medium of the node's link, `wired`, `wifi` or `unknown`: the value of the node's `github-runner.exadev.com/link` label (`github_runner_arc_role_controller_link_label`), which the cluster role reads on the host itself (see the cluster role's README) since a node in a container or a VM sees only its own virtual interface, and otherwise the medium of the interface the pod's default route leaves by (an interface that is virtual, such as the k3s container's veth, hides the link beneath it), each sample saying which (`link_source`, `label` or `interface`). The pod's own reading is taken from the interface's `/sys/class/net` entry and never by walking `/sys`, whose symlinks loop. `scripts/fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of these readings alone (the link as the pod's own interface gives it), with no cluster access, to compare with a reading taken by hand on the host. @@ -213,6 +213,24 @@ The controller reads the Nodes and the probes' samples every sample period and p A slow link makes both of its ends unfit, since a voter is unfit when its round trip to any other voter is, so every unfit verdict is attributed to the node at fault for it (`attributed_to` on each host, and `attribution`, the hosts each node accounts for). A host's own disk, memory, readiness or restarts are its own fault. Its round trip is the link to its worst peer, and the end at fault for a slow link is the one whose links to the other voters are slower too, compared by the best of them, since a node on a poor path is slow to every peer and a node on a good one is fast to its others; when the two ends' other links are as fast as each other's, or one end has none, the link is attributed to both. The voter to replace is the unfit voter the most hosts' unfit verdicts are attributed to, the worst-ranked among those with as many, and the log names the attribution beside each unfit verdict. A swap is reported only when that voter is eligible and not pinned (when it is not, replacing a voter that shares a slow link with it would fix nothing, so nothing is swapped) and a fit eligible standby ranks above it. A candidate is judged against the voters that would remain, since a slow link to the very voter it would replace says nothing against it. A voter is a node that carries k3s's etcd label and whose `EtcdIsVoter` condition is True: k3s labels a server the moment it joins etcd as a learner and never removes the label, so a labelled node without the condition is a member that does not vote: it is not counted as a voter and holds a swap back (`member but not a voter` in the status). It is held back, with the reason in the status and the log, while the voter count is not `github_runner_arc_role_controller_voters`, any voter is below `full`, is not Ready, has no leader, is a learner or knows a member that is not a voter, is not connected to every other voter, fewer hosts are eligible than the target, or the cooldown after a change of the voter set has not passed (k3s's five minutes to remove a stalled learner plus one window). The target must be odd, at least 3, and at most `github_runner_cluster_max_servers` minus one, which leaves a swap one place of headroom; the role refuses anything else. A host is eligible, and a voter pinned, by a Node label with the value `true` (`github_runner_arc_role_controller_eligible_label` and `_pinned_label`), which the cluster role publishes from `github_runner_cluster_voter_eligible` and `github_runner_cluster_voter_pinned` per host (and the target from `github_runner_cluster_voters`, refused unless odd, at least 3 and below `github_runner_cluster_max_servers`; see the cluster role's README); a voter not marked eligible is never swapped out, since nothing has prepared it to come back. +#### Voter swap + +A swap brings an eligible standby in as a voter and takes an unfit voter out, as one operation under the `fleet-regrade` Lease, which the controller holds as `fleet-role` and renews on every pass for as long as the swap runs. A regrade step, a run of the cluster role and node recovery all leave the nodes of a swap alone while it does. The controller takes at most one action in a pass and reads the step a swap is at from the Nodes, the hosts' role actuators' reports and the Lease, so a controller pod that is rescheduled resumes where the swap is. The state machine is `scripts/fleet-role-swap.jq`, a pure function over that state, and `scripts/fleet-role.sh` carries out the one action it names. + +A swap starts in one of two ways. `kubectl -n github-runner-platform exec deploy/fleet-role -- /app/scripts/fleet-role.sh swap OUT IN` asks for it by command: the command exits 0 when every gate holds and the request is recorded for the next pass, 1 when it is refused (each reason is printed) and 2 for a node that is unknown, not marked eligible or not the voter and the standby it is taken for. A request is carried out whatever the verdicts say, and inside the cooldown, but never takes out a pinned voter and honours every gate. The controller starts the swap it decides on only when `FLEET_ROLE_DRY_RUN` is `false` (`github_runner_arc_role_controller_dry_run`, true by default, which reports such a swap and carries out requests alone). A swap already under way is finished or rolled back whatever the setting, so an even count of voters is never left behind. `fleet-role.sh status` prints the swap and its step. + +The gates are those of the epic: every voter at `full`, Ready, with a fresh probe report, etcd with a leader, no learner and no member that has not started, the voter count the target, and the regrade Lease free. The steps: + +1. A snapshot is asked of a remaining voter through its Node (`snapshot-request`) and waited for until k3s publishes an `ETCDSnapshotFile` of that name that reports `readyToUse`. Nothing has changed yet, so a snapshot that is not ready within the operation timeout abandons the swap. +2. The candidate is cordoned and the controller waits for its runner pods to finish (a planned change never cuts a job); past the runner wait the swap is abandoned, the candidate uncordoned and the Lease released, with no cooldown. Otherwise it is drained and asked to promote: `desired-role=server`, with `role-swap`, `role-step=promote` and `role-join-node` naming a remaining voter. Its actuator writes the role file, a join URL and the unit changes, and starts the server, which k3s joins as a learner that the etcd leader promotes once it has caught up. The step ends when the candidate is a voter (`EtcdIsVoter`) and Ready on a kubelet heartbeat after the server started, and the candidate is uncordoned. +3. The gates are held again, now with one more voter than the target. If the outgoing voter is down they are held to the other voters alone. +4. The outgoing voter is tainted and cordoned, its runner pods waited for (past the runner wait it is drained anyway, as regrading drains a server) and drained, labelled `regrade-level=etcd-only` and asked to step down (`desired-role=agent`, `role-step=step-down`). Its actuator brings its server to etcd-only and writes the role file for the agent. When it reports done, or when the node is down and has not reported within the operation timeout, the controller annotates `etcd.k3s.cattle.io/remove=true` while the remaining voters are healthy, and again if k3s clears the annotation, and waits for `removed-node-name`. A step-down the actuator reports failed on a node that stays Ready is held for the owner. The member counts as removed once k3s records it, or once every remaining voter's probe reports etcd listing as many members as there are voters, which holds when a server still running on the host has put its etcd label back; the controller then marks the Node with `role-member-gone` naming the swap, the proof its actuator finishes on, since k3s's own record can name another member than the host's datastore. The actuator then finishes the demotion (the server stopped, its datastore moved aside, the agent started) and the controller removes the `control-plane`, `etcd` and `master` labels, the `etcd.k3s.cattle.io/*` annotations, the level label and the taint, waits for the node to be Ready as an agent and uncordons it. A host that has not come back within the operation timeout is left to node recovery and the swap records `outgoing_pending`. +5. The swap annotations are cleared and the Lease is released. The cooldown starts from the membership change, as for a decision. + +A candidate that is not a voter by the deadline (k3s's learner stall time plus the operation timeout), or whose actuator reports the promotion failed, is rolled back: the controller annotates the removal of its learner if it joined, asks its actuator to put the host back to an agent (`role-step=rollback`), waits until every voter's probe lists exactly the target number of members, then cleans the Node up and uncordons it. A refused promotion, which changed nothing on the host, abandons the swap. Every wait is derived from the operation timeout, the runner wait and the learner stall time, which the role takes from the cluster role's own settings (`github_runner_arc_role_controller_operation_timeout_seconds`, `github_runner_arc_role_controller_runner_wait_seconds` and `github_runner_arc_role_controller_lease_seconds`). + +`fleet-role-status` carries `swap` (`id`, `out`, `in`, `step`, `started_at`, `step_since`, `deadline`, `manual`, `snapshot`), `waiting` (why the swap is not acting this pass), `last_swap` (`outcome`, `completed`, `rolled-back` or `abandoned`, with its reasons), the `request` pending and the real `dry_run`. A manual request is kept in the status ConfigMap under `request.json`, a key of its own, so that the command and the controller's pass never overwrite each other. The controller's ClusterRole adds to the probe's `patch` on nodes, `get` and `list` on pods, `create` on `pods/eviction`, `get` on daemon sets and `get` and `list` on k3s's `etcdsnapshotfiles`; its Roles let it `create` Leases and `get` and `update` the regrade Lease in `kube-system`, `get` the kubelets' Leases and `get` the actuators' ConfigMap. What stays manual is removing a voter that is down and not coming back, restoring after quorum loss and changing the target count. + The controller and the probe run one image, `github_runner_arc_role_controller_image`, pulled without a pull Secret, so the published package must be public. ## Conventions diff --git a/fleet-role/Dockerfile b/fleet-role/Dockerfile index c4624945..5cb5ed80 100644 --- a/fleet-role/Dockerfile +++ b/fleet-role/Dockerfile @@ -1,4 +1,4 @@ -# Image for the role controller (see roles/github_runner_arc/templates/fleet-role.yaml.j2): the `fleet-role` Deployment runs scripts/fleet-role.sh through loop.sh, and the `fleet-role-probe` DaemonSet runs scripts/fleet-role-probe.sh from the same image. bash, curl, jq, kubectl and fio, all from Alpine's apk (fio is the disk check etcd's own documentation uses), so it is small and builds for either architecture. Build context is the repository root so it can COPY the scripts, the regrade core, whose own functions the probe reads memory with, and the cluster role's link reader, whose own functions it reads the link with. The Deployment and the DaemonSet pull it without a pull Secret, so the published package must be public. +# Image for the role controller (see roles/github_runner_arc/templates/fleet-role.yaml.j2): the `fleet-role` Deployment runs scripts/fleet-role.sh through loop.sh (and an owner runs its `swap` command in the same pod), and the `fleet-role-probe` DaemonSet runs scripts/fleet-role-probe.sh from the same image. bash, curl, jq, kubectl and fio, all from Alpine's apk (fio is the disk check etcd's own documentation uses), so it is small and builds for either architecture. Build context is the repository root so it can COPY the scripts, the regrade core, whose own functions the probe reads memory with and the controller takes, renews and releases the Lease with, and the cluster role's link reader, whose own functions it reads the link with. The Deployment and the DaemonSet pull it without a pull Secret, so the published package must be public. FROM alpine:3.20 RUN apk add --no-cache bash curl jq kubectl fio @@ -6,7 +6,7 @@ RUN apk add --no-cache bash curl jq kubectl fio COPY fleet-role/loop.sh /app/loop.sh COPY scripts/fleet-role.sh scripts/fleet-role-probe.sh /app/scripts/ # The jq programs the scripts run from their own directory, and the module they share. -COPY scripts/fleet-role.jq scripts/fleet-role-stats.jq /app/scripts/ +COPY scripts/fleet-role.jq scripts/fleet-role-swap.jq scripts/fleet-role-stats.jq /app/scripts/ COPY roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh /app/fleet-regrade-core.sh COPY roles/github_runner_cluster/files/node-link.sh /app/node-link.sh RUN chmod +x /app/loop.sh /app/scripts/fleet-role.sh /app/scripts/fleet-role-probe.sh diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 272c8a1e..678e7321 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -56,7 +56,7 @@ Helm 4 and server-side apply. The role reads `helm version --short` and, on Helm - `github_runner_arc_heartbeat_gist_id`: install the fleet-health platform from this host, refreshing this gist. `github_runner_arc_heartbeat_bootstrap_gist`, `github_runner_arc_heartbeat_gist_description` and `github_runner_arc_heartbeat_gist_consumer` control the bootstrap described above. - `github_runner_arc_autoscaler_usable_budget_gi`, `github_runner_arc_autoscaler_floor`: required when a profile sets `autoscale: true` and the platform is installed. The floor must equal the autoscaled profile's `maxRunners`, which every Helm upgrade reverts to. The autoscaler works its ceiling out from the nodes on every poll (see the project README's Autoscaler section); `github_runner_arc_autoscaler_max_ceiling`, optional, is an upper bound on it and must be no lower than the floor. The other `github_runner_arc_autoscaler_*` and `github_runner_arc_heartbeat_*` settings have defaults; see `defaults/main.yml`. - `github_runner_arc_node_recovery_enabled` (default `true`), `github_runner_arc_node_recovery_after_seconds` (default 120), `github_runner_arc_node_recovery_poll_seconds` (default 15), `github_runner_arc_node_recovery_prune_after_seconds` (default empty, which leaves pruning off), `github_runner_arc_node_recovery_prune_label` (default `github-runner.exadev.io/removable`), `github_runner_arc_node_recovery_level_label` (default the cluster role's `github_runner_cluster_regrade_level_label`, else `github-runner.exadev.com/regrade-level`), `github_runner_arc_node_recovery_image`: the node recovery watcher (see [Node recovery](#node-recovery)). -- `github_runner_arc_role_controller_enabled` (default the cluster role's `github_runner_cluster_role_controller`, itself `false`), `github_runner_arc_role_controller_image`, `github_runner_arc_role_controller_sample_seconds` (default 15), `github_runner_arc_role_controller_voters` (default 3), `github_runner_arc_role_controller_max_servers`, `github_runner_arc_role_controller_election_timeout_ms`, `github_runner_arc_role_controller_pressure_threshold`, `github_runner_arc_role_controller_available_floor_percent`, `github_runner_arc_role_controller_eligible_label`, `github_runner_arc_role_controller_pinned_label`, `github_runner_arc_role_controller_level_label`, `github_runner_arc_role_controller_link_label`, `github_runner_arc_role_controller_probe_configmap`, `github_runner_arc_role_controller_status_configmap`, `github_runner_arc_role_controller_data_dir`: the role controller, which reports which hosts are fit to hold an etcd vote (see [Role controller](#role-controller)). Those that mirror the cluster role (`voters`, `max_servers`, `election_timeout_ms`, `pressure_threshold`, `available_floor_percent`, `eligible_label`, `pinned_label`, `level_label`, `link_label`) take the cluster role's own variable when it is defined. +- `github_runner_arc_role_controller_enabled` (default the cluster role's `github_runner_cluster_role_controller`, itself `false`), `github_runner_arc_role_controller_image`, `github_runner_arc_role_controller_sample_seconds` (default 15), `github_runner_arc_role_controller_voters` (default 3), `github_runner_arc_role_controller_max_servers`, `github_runner_arc_role_controller_election_timeout_ms`, `github_runner_arc_role_controller_pressure_threshold`, `github_runner_arc_role_controller_available_floor_percent`, `github_runner_arc_role_controller_eligible_label`, `github_runner_arc_role_controller_pinned_label`, `github_runner_arc_role_controller_level_label`, `github_runner_arc_role_controller_link_label`, `github_runner_arc_role_controller_probe_configmap`, `github_runner_arc_role_controller_status_configmap`, `github_runner_arc_role_controller_data_dir`, `github_runner_arc_role_controller_dry_run` (default true), `github_runner_arc_role_controller_operation_timeout_seconds`, `github_runner_arc_role_controller_runner_wait_seconds`, `github_runner_arc_role_controller_lease_seconds`, `github_runner_arc_role_controller_actuator_configmap`, `github_runner_arc_role_controller_lease_namespace` and `github_runner_arc_role_controller_lease_name`: the role controller, which reports which hosts are fit to hold an etcd vote and carries out a voter swap (see [Role controller](#role-controller)). Those that mirror the cluster role (`voters`, `max_servers`, `election_timeout_ms`, `pressure_threshold`, `available_floor_percent`, `eligible_label`, `pinned_label`, `level_label`, `link_label`, `operation_timeout_seconds`, `runner_wait_seconds`, `lease_seconds`, `actuator_configmap`, `lease_namespace`, `lease_name`) take the cluster role's own variable when it is defined. - `github_runner_arc_runner_reaper_enabled` (default `false`), `github_runner_arc_runner_reaper_max_job_seconds` (default 432000, five days), `github_runner_arc_runner_reaper_completed_grace_seconds` (default 300), `github_runner_arc_runner_reaper_schedule` (default every 5 minutes), `github_runner_arc_runner_reaper_dry_run` (default `false`), `github_runner_arc_runner_reaper_image` (default the renewal's image): the runner reaper (see [Runner reaper](#runner-reaper)). - `github_runner_arc_runner_active_deadline_seconds` (default unset): `activeDeadlineSeconds` on every runner pod (see [Runner reaper](#runner-reaper)). - `github_runner_arc_app_setup_*`, `github_runner_arc_app_manifest_code`, `github_runner_arc_app_private_key_path`: inputs to `playbooks/github_app_setup.yml` (see below), including `github_runner_arc_app_setup_write_secret`, `github_runner_arc_app_setup_secret_namespaces`, `github_runner_arc_app_setup_secret_name`, `github_runner_arc_app_setup_keep_private_key` and `github_runner_arc_app_setup_command`. @@ -485,7 +485,7 @@ A pruned node returns only when its k3s agent process restarts. A running k3s ag ## Role controller -With `github_runner_cluster_role_controller: true` (or `github_runner_arc_role_controller_enabled: true`, for a host that does not run the cluster role) the role installs `fleet-role-probe`, a DaemonSet on every Linux node that is an etcd voter or is marked eligible (`github_runner_arc_role_controller_eligible_label`) and has a kubelet, and `fleet-role`, a Deployment, both in `github_runner_arc_platform_namespace` and both from `github_runner_arc_role_controller_image`. Together they report, for every etcd voter and every host marked eligible, whether it is fit to hold an etcd vote and why, and the swap of an unfit voter for a fit standby that the controller would make. They change nothing in the cluster: each has a ClusterRole that can `get` and `list` nodes and nothing else, and a Role that reaches only the two ConfigMaps it uses by name (`get` and `patch`, never `create`; the role creates both empty and keeps them). `false` removes the workloads, their RBAC and both ConfigMaps. The repository README's Role controller section explains each signal, how it is read, the thresholds and when a swap is held back. +With `github_runner_cluster_role_controller: true` (or `github_runner_arc_role_controller_enabled: true`, for a host that does not run the cluster role) the role installs `fleet-role-probe`, a DaemonSet on every Linux node that is an etcd voter or is marked eligible (`github_runner_arc_role_controller_eligible_label`) and has a kubelet, and `fleet-role`, a Deployment, both in `github_runner_arc_platform_namespace` and both from `github_runner_arc_role_controller_image`. Together they report, for every etcd voter and every host marked eligible, whether it is fit to hold an etcd vote and why, and the swap of an unfit voter for a fit standby that the controller would make. The probe changes nothing in the cluster: its ClusterRole can `get` and `list` nodes and nothing else, and its Role reaches only its own ConfigMap by name (`get` and `patch`, never `create`; the role creates both ConfigMaps empty and keeps them). The controller carries out a voter swap when it is asked to (`fleet-role.sh swap OUT IN`, run with `kubectl exec deploy/fleet-role`) or, with `github_runner_arc_role_controller_dry_run` false, when it decides on one, and always finishes a swap already under way. It therefore also has `patch` on nodes, `get` and `list` on pods in every namespace, `create` on `pods/eviction`, `get` on daemon sets and `get` and `list` on k3s's `etcdsnapshotfiles` in its ClusterRole, `get` on the actuators' ConfigMap, and two Roles of its own outside the platform namespace: in the regrade Lease's namespace (`create` on Leases, and `get` and `update` on the regrade Lease by name) and in `kube-node-lease` (`get`, to tell a restarted node's kubelet is running). The dry run is on by default and the role never turns it off by itself. `false` removes the workloads, their RBAC and both ConfigMaps. The repository README's Role controller section explains each signal, how it is read, the thresholds and when a swap is held back. The probe runs with the node's network and as root with every capability dropped, because the directory it measures the disk in (`github_runner_arc_role_controller_data_dir`, on the filesystem k3s keeps its data on) is created by the kubelet and owned by root. It mounts nothing for memory pressure: `/proc/pressure/memory` of its own procfs is the node's, as `/proc/meminfo` is, and a `hostPath` mount of `/proc/pressure` would stop the pod starting on a kernel without pressure stall information, since the container runtime then tries to create the directory. The controller runs as an unprivileged user with a read-only root filesystem. The heartbeat, when it is installed, reads the controller's status ConfigMap too and republishes it in its gist as `fleet-role-status.json`. diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index ba6a5684..7b897aa1 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -191,7 +191,7 @@ github_runner_arc_node_recovery_level_label: "{{ github_runner_cluster_regrade_l # Image the watcher runs. Pulled without a pull Secret, so it must be public. github_runner_arc_node_recovery_image: "ghcr.io/exadev/github-runner-node-recovery:2.38.0" -# The role controller (see tasks/install_fleet_role.yml, scripts/fleet-role.sh and scripts/fleet-role-probe.sh): a `fleet-role-probe` DaemonSet that measures, on every node with a kubelet, what decides whether a host is fit to hold an etcd vote, and a `fleet-role` Deployment that reads the measurements and reports which eligible hosts are fit and which swap of an unfit voter for a fit standby it would make. It reports and changes nothing: its only writes are its own two ConfigMaps. Off by default, so a consumer that upgrades changes nothing. +# The role controller (see tasks/install_fleet_role.yml, scripts/fleet-role.sh and scripts/fleet-role-probe.sh): a `fleet-role-probe` DaemonSet that measures, on every node with a kubelet, what decides whether a host is fit to hold an etcd vote, and a `fleet-role` Deployment that reads the measurements, reports which eligible hosts are fit and which swap of an unfit voter for a fit standby it would make, and carries a swap out (an eligible standby becomes a voter and the unfit voter an agent) when the owner asks for one by command (`fleet-role.sh swap OUT IN`) or, with github_runner_arc_role_controller_dry_run false, when it decides on one. Off by default, so a consumer that upgrades changes nothing. # The cluster role's github_runner_cluster_role_controller when it is defined, so one inventory setting turns it on; set this one to turn it on from a host that does not run the cluster role (playbooks/arc.yml against a kubeconfig). github_runner_arc_role_controller_enabled: "{{ github_runner_cluster_role_controller | default(false) }}" # Image the controller and the probe run, one image whose entrypoint is the controller. Pulled without a pull Secret, so it must be public. @@ -213,6 +213,18 @@ github_runner_arc_role_controller_pinned_label: "{{ github_runner_cluster_voter_ github_runner_arc_role_controller_level_label: "{{ github_runner_cluster_regrade_level_label | default('github-runner.exadev.com/regrade-level') }}" # The Node label the cluster role publishes the medium of each host's link under (wired, wifi or unknown), which the probe reports for its node in place of what it can see itself: the cluster role's github_runner_cluster_link_label when that is defined. github_runner_arc_role_controller_link_label: "{{ github_runner_cluster_link_label | default('github-runner.exadev.com/link') }}" +# true (the default) carries out only the swaps an owner asks for with `kubectl -n exec deploy/fleet-role -- /app/scripts/fleet-role.sh swap OUT IN` and reports the ones the controller would decide on; false carries out its own decisions too. A swap already under way is always finished or rolled back. Turn it off only after a swap by command has been watched to its end on the fleet. +github_runner_arc_role_controller_dry_run: true +# How long a drain, the wait for a snapshot and each step of a host's role actuator may take, in seconds: the cluster role's github_runner_cluster_regrade_operation_timeout_seconds when it is defined, which the actuators bound their own steps by, else that variable's default (tests/unit/test_fleet_role_arc.py holds the two together). +github_runner_arc_role_controller_operation_timeout_seconds: "{{ github_runner_cluster_regrade_operation_timeout_seconds | default(300) }}" +# How long a planned change waits for the runner pods on a node to finish before it is abandoned (the candidate) or goes on (the outgoing voter): the cluster role's github_runner_cluster_native_runner_wait_timeout when it is defined, which the role's own role switch waits, else that variable's default. +github_runner_arc_role_controller_runner_wait_seconds: "{{ github_runner_cluster_native_runner_wait_timeout | default(600) }}" +# How long the regrade Lease a swap holds lasts if the controller stops renewing it, which the controller renews on every pass: the cluster role's github_runner_cluster_regrade_lease_duration_seconds when it is defined, else that variable's default, three operation timeouts. +github_runner_arc_role_controller_lease_seconds: "{{ github_runner_cluster_regrade_lease_duration_seconds | default(3 * (github_runner_arc_role_controller_operation_timeout_seconds | int)) }}" +# The ConfigMap the role actuators on the hosts report their steps in (the cluster role's github_runner_cluster_role_actuator_configmap when it is defined), which the controller reads, and the namespace and name of the regrade Lease a swap holds (the cluster role's github_runner_cluster_regrade_lease_namespace and github_runner_cluster_regrade_lease_name when defined). +github_runner_arc_role_controller_actuator_configmap: "{{ github_runner_cluster_role_actuator_configmap | default('fleet-role-actuators') }}" +github_runner_arc_role_controller_lease_namespace: "{{ github_runner_cluster_regrade_lease_namespace | default('kube-system') }}" +github_runner_arc_role_controller_lease_name: "{{ github_runner_cluster_regrade_lease_name | default('fleet-regrade') }}" # The ConfigMaps in the platform namespace that the probes publish to (a key per node) and that the controller publishes its status to, which the heartbeat republishes beside the autoscaler's. github_runner_arc_role_controller_probe_configmap: fleet-role-probe github_runner_arc_role_controller_status_configmap: fleet-role-status diff --git a/roles/github_runner_arc/meta/argument_specs.yml b/roles/github_runner_arc/meta/argument_specs.yml index aef1a1dc..1a2f75a3 100644 --- a/roles/github_runner_arc/meta/argument_specs.yml +++ b/roles/github_runner_arc/meta/argument_specs.yml @@ -381,8 +381,8 @@ argument_specs: type: bool description: - Whether to install the role controller. Defaults to the cluster role's github_runner_cluster_role_controller when that is defined, else false. - - Install the role controller, a `fleet-role-probe` DaemonSet that measures on every node with a kubelet what decides whether a host is fit to hold an etcd vote, and a `fleet-role` Deployment that reports which eligible hosts are fit and which swap of an unfit voter for a fit standby it would make. - - It only reports. Its writes are its own two ConfigMaps; its ClusterRoles can get and list nodes and nothing else. false removes both, their RBAC and their ConfigMaps. + - Install the role controller, a `fleet-role-probe` DaemonSet that measures on every node with a kubelet what decides whether a host is fit to hold an etcd vote, and a `fleet-role` Deployment that reports which eligible hosts are fit and which swap of an unfit voter for a fit standby it would make, and carries a swap out when asked or, with the dry run off, when it decides on one. + - The probe's ClusterRole can get and list nodes and nothing else. The controller's can also patch nodes, get and list pods, create evictions, read daemon sets and read k3s's snapshot files, and its Roles let it create Leases and replace the regrade Lease and read the kubelets' Leases and the actuators' reports. false removes both, their RBAC and their ConfigMaps. github_runner_arc_role_controller_image: type: str description: Image the controller and the probe run, pulled without a pull Secret. @@ -390,6 +390,28 @@ argument_specs: type: int default: 15 description: Seconds between probe samples and between controller passes. A figure is read over a window of 100 samples. + github_runner_arc_role_controller_dry_run: + type: bool + default: true + description: true carries out only the swaps an owner asks for by command and reports those the controller would decide on; false carries out its own decisions too. A swap under way is always finished or rolled back. + github_runner_arc_role_controller_operation_timeout_seconds: + type: int + description: How long a drain, the wait for a snapshot and each step of a host's role actuator may take, from github_runner_cluster_regrade_operation_timeout_seconds when that is defined, else 300. + github_runner_arc_role_controller_runner_wait_seconds: + type: int + description: How long a planned change waits for the runner pods on a node to finish, from github_runner_cluster_native_runner_wait_timeout when that is defined, else 600. + github_runner_arc_role_controller_lease_seconds: + type: int + description: How long the regrade Lease a swap holds lasts if the controller stops renewing it, from github_runner_cluster_regrade_lease_duration_seconds when that is defined, else three operation timeouts. + github_runner_arc_role_controller_actuator_configmap: + type: str + description: The ConfigMap the hosts' role actuators report their steps in, from github_runner_cluster_role_actuator_configmap when that is defined, else fleet-role-actuators. + github_runner_arc_role_controller_lease_namespace: + type: str + description: The namespace of the regrade Lease a swap holds, from github_runner_cluster_regrade_lease_namespace when that is defined, else kube-system. + github_runner_arc_role_controller_lease_name: + type: str + description: The name of the regrade Lease a swap holds, from github_runner_cluster_regrade_lease_name when that is defined, else fleet-regrade. github_runner_arc_role_controller_voters: type: int description: The number of etcd voters the cluster is held at, from github_runner_cluster_voters when that is defined, else 3. Odd, at least 3, and at most github_runner_arc_role_controller_max_servers minus one. diff --git a/roles/github_runner_arc/templates/fleet-role.yaml.j2 b/roles/github_runner_arc/templates/fleet-role.yaml.j2 index 53823d3a..1b3a2a30 100644 --- a/roles/github_runner_arc/templates/fleet-role.yaml.j2 +++ b/roles/github_runner_arc/templates/fleet-role.yaml.j2 @@ -1,4 +1,4 @@ -{# The role controller (see tasks/install_fleet_role.yml, scripts/fleet-role.sh and scripts/fleet-role-probe.sh): the controller's ServiceAccount, ClusterRole, binding, ConfigMap Role and Deployment, and the probe's own, with its DaemonSet. The namespace is ensured, and the two ConfigMaps are created, separately, since applying an empty ConfigMap again would wipe what the controller and the probes have published. #} +{# The role controller (see tasks/install_fleet_role.yml, scripts/fleet-role.sh and scripts/fleet-role-probe.sh): the controller's ServiceAccount, ClusterRole, binding, ConfigMap Role, the Roles for the Leases and the Deployment, and the probe's own, with its DaemonSet. The namespace is ensured, and the two ConfigMaps are created, separately, since applying an empty ConfigMap again would wipe what the controller and the probes have published. #} {% set controller = 'fleet-role' %} {% set probe = 'fleet-role-probe' %} {% set labels = {'app.kubernetes.io/part-of': 'github-runner'} %} @@ -7,8 +7,19 @@ {% set status_configmap = github_runner_arc_role_controller_status_configmap %} {# The label k3s puts, with the value true, on every node that is an etcd member: the controller (scripts/fleet-role.sh) and the probe (scripts/fleet-role-probe.sh) each name it, and tests/unit/test_fleet_role_arc.py holds the three together. #} {% set voter_label = 'node-role.kubernetes.io/etcd' %} -{# A ClusterRole can only name nodes cluster-wide; the controller and the probe both list them to find the voters and the standbys, and neither can write one. #} -{% set node_rules = [{'apiGroups': [''], 'resources': ['nodes'], 'verbs': ['get', 'list']}] %} +{% set actuator_configmap = github_runner_arc_role_controller_actuator_configmap %} +{% set lease_namespace = github_runner_arc_role_controller_lease_namespace %} +{% set lease_name = github_runner_arc_role_controller_lease_name %} +{# A ClusterRole can only name nodes cluster-wide; the controller and the probe both list them to find the voters and the standbys, and only the controller writes one (the cordon, the labels and annotations of a swap, the taint). The probe's rules stay read-only. #} +{% set probe_node_rules = [{'apiGroups': [''], 'resources': ['nodes'], 'verbs': ['get', 'list']}] %} +{# What a swap does on the API, as the issue lists it: draining the nodes it empties (the pods of every namespace are listed, read while the drain waits for each to go, and evicted, and a drain reads the daemon sets that own pods it leaves), and reading the snapshot files k3s publishes. Creating a Lease and reading kubelet leases are in the namespaces' own Roles below. #} +{% set controller_node_rules = [ + {'apiGroups': [''], 'resources': ['nodes'], 'verbs': ['get', 'list', 'patch']}, + {'apiGroups': [''], 'resources': ['pods'], 'verbs': ['get', 'list']}, + {'apiGroups': [''], 'resources': ['pods/eviction'], 'verbs': ['create']}, + {'apiGroups': ['apps'], 'resources': ['daemonsets'], 'verbs': ['get']}, + {'apiGroups': ['k3s.cattle.io'], 'resources': ['etcdsnapshotfiles'], 'verbs': ['get', 'list']}, +] %} {% for name in [controller, probe] %} {% set own_labels = labels | combine({'app.kubernetes.io/name': name}) %} apiVersion: v1 @@ -23,7 +34,7 @@ kind: ClusterRole metadata: name: {{ ('github-runner-' ~ name) | to_json }} labels: {{ own_labels | to_json }} -rules: {{ node_rules | to_json }} +rules: {{ (controller_node_rules if name == controller else probe_node_rules) | to_json }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -56,6 +67,10 @@ rules: resources: ["configmaps"] resourceNames: {{ [status_configmap] | to_json }} verbs: ["get", "patch"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: {{ [actuator_configmap] | to_json }} + verbs: ["get"] {% else %} - apiGroups: [""] resources: ["configmaps"] @@ -79,6 +94,40 @@ roleRef: name: {{ name | to_json }} --- {% endfor %} +{# The regrade Lease a swap holds lives in another namespace than the controller, and the kubelets' own Leases in a third, so each gets a Role there. Create cannot be limited to a name, so the controller can make a Lease, and can read and replace only the regrade Lease; the kubelet Leases are read to tell that a restarted node's kubelet is running. #} +{% for scope in [ + {'namespace': lease_namespace, 'name': 'github-runner-fleet-role-swap-lease', 'rules': [ + {'apiGroups': ['coordination.k8s.io'], 'resources': ['leases'], 'verbs': ['create']}, + {'apiGroups': ['coordination.k8s.io'], 'resources': ['leases'], 'resourceNames': [lease_name], 'verbs': ['get', 'update']}, + ]}, + {'namespace': 'kube-node-lease', 'name': 'github-runner-fleet-role-node-lease', 'rules': [ + {'apiGroups': ['coordination.k8s.io'], 'resources': ['leases'], 'verbs': ['get']}, + ]}, +] %} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ scope.name | to_json }} + namespace: {{ scope.namespace | to_json }} + labels: {{ (labels | combine({'app.kubernetes.io/name': controller})) | to_json }} +rules: {{ scope.rules | to_json }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ scope.name | to_json }} + namespace: {{ scope.namespace | to_json }} + labels: {{ (labels | combine({'app.kubernetes.io/name': controller})) | to_json }} +subjects: + - kind: ServiceAccount + name: {{ controller | to_json }} + namespace: {{ namespace | to_json }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ scope.name | to_json }} +--- +{% endfor %} apiVersion: apps/v1 kind: Deployment metadata: @@ -105,9 +154,9 @@ spec: - name: {{ controller | to_json }} image: {{ github_runner_arc_role_controller_image | to_json }} env: - # The controller reports and changes nothing; the script refuses any other value. + # true carries out a swap an owner asks for by command (scripts/fleet-role.sh swap) and only reports the ones it would decide on; false carries out those too. A swap already under way is finished either way. - name: FLEET_ROLE_DRY_RUN - value: "true" + value: {{ github_runner_arc_role_controller_dry_run | bool | lower | to_json }} - name: FLEET_ROLE_VOTERS value: {{ github_runner_arc_role_controller_voters | string | to_json }} - name: FLEET_ROLE_MAX_SERVERS @@ -126,6 +175,14 @@ spec: value: {{ github_runner_arc_role_controller_pinned_label | to_json }} - name: FLEET_ROLE_LEVEL_LABEL value: {{ github_runner_arc_role_controller_level_label | to_json }} + - name: FLEET_ROLE_OPERATION_TIMEOUT_SECONDS + value: {{ github_runner_arc_role_controller_operation_timeout_seconds | string | to_json }} + - name: FLEET_ROLE_RUNNER_WAIT_SECONDS + value: {{ github_runner_arc_role_controller_runner_wait_seconds | string | to_json }} + - name: FLEET_ROLE_LEASE_SECONDS + value: {{ github_runner_arc_role_controller_lease_seconds | string | to_json }} + - name: FLEET_ROLE_ACTUATOR_CONFIGMAP + value: {{ actuator_configmap | to_json }} - name: FLEET_ROLE_STATE_NAMESPACE value: {{ namespace | to_json }} - name: FLEET_ROLE_PROBE_CONFIGMAP diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index e659949a..72a723bc 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -311,7 +311,7 @@ The decision logic lives in `files/regrade/fleet-regrade-core.sh` and calls only ## Role controller -`github_runner_cluster_role_controller: true` turns on the ARC role's role controller, which measures which hosts are fit to hold an etcd vote and reports the swap of an unfit voter for a fit standby that it would make (see the ARC role's README, Role controller). It reports and changes nothing. On a native server or eligible native host the role also runs the role actuator that a swap switches the host through ([Role actuator](#role-actuator)). It is off by default. This role holds the controller's inventory: +`github_runner_cluster_role_controller: true` turns on the ARC role's role controller, which measures which hosts are fit to hold an etcd vote, reports the swap of an unfit voter for a fit standby that it would make and carries a swap out when asked (see the ARC role's README, Role controller, and the repository README's Voter swap). Its dry run is on unless the ARC role's `github_runner_arc_role_controller_dry_run` says otherwise. While it holds the `fleet-regrade` Lease for a swap, every run of this role refuses to change a node. On a native server or eligible native host the role also runs the role actuator that a swap switches the host through ([Role actuator](#role-actuator)). It is off by default. This role holds the controller's inventory: - `github_runner_cluster_voter_eligible` (per host, default false) marks a host the controller may promote to a voter or demote from one. The role prepares no host for promotion by marking it, so mark only a host that is, or has been staged as, a server. - `github_runner_cluster_voter_pinned` (per host, default false) marks a voter the controller must never demote. Set each pin in the inventory with a comment saying why. @@ -343,7 +343,7 @@ On a native host the controller may switch or take a snapshot on, a server or a **What it reports**, in the ConfigMap `fleet-role-actuators` in the platform namespace (the ARC role's `github_runner_arc_platform_namespace`), under the node's name, as JSON: `{"swap", "step", "phase", "role", "started_at", "at", "message", "snapshot"}`, `phase` being `running`, `done`, `failed` or `refused`, `role` what the role file names afterwards and `started_at` the host's time just before a unit was started, which the controller waits for a later heartbeat than. A report the API does not take is made again on the next pass, since every step can be taken again: a step whose report is not done is run again, and a finish of a demotion not reported done is run again while the Node still carries the swap's `step-down`, even when nothing of the server is left, when it only reports (on a Docker host the node container that carries kubectl is the agent just started, which has no route to the API for a while, so that report is lost more often than a native host's). The finish's own report replaces the step-down's, and the watch counts a step-down followed by a finish for the same swap as taken. -**Authority.** `promote`, `step-down`, `rollback` and `snapshot` are taken only while the regrade Lease (`kube-system/fleet-regrade`) is held, unexpired, by `fleet-role` with the swap's id in its `github-runner.exadev.com/swap-id` annotation (see [Runs during a voter swap](#runs-during-a-voter-swap)). Finishing a demotion needs no Lease: it runs once the role file names the agent and the member is gone (a tombstone in the datastore, `etcd.k3s.cattle.io/removed-node-name` naming the datastore's member, or no etcd label on the Node), which can never cost quorum, so a host that was down while its member was removed finishes when it returns. +**Authority.** `promote`, `step-down`, `rollback` and `snapshot` are taken only while the regrade Lease (`kube-system/fleet-regrade`) is held, unexpired, by `fleet-role` with the swap's id in its `github-runner.exadev.com/swap-id` annotation (see [Runs during a voter swap](#runs-during-a-voter-swap)). Finishing a demotion needs no Lease: it runs once the role file names the agent (or names the server while the datastore holds k3s's tombstone, or while the Node asks for a step-down, whose failure on a node that is not Ready the controller follows with the removal) and the member is gone (a tombstone in the datastore, `etcd.k3s.cattle.io/removed-node-name` naming the datastore's member, `github-runner.exadev.com/role-member-gone` naming the swap the Node is in, which the controller sets once every remaining voter reports that etcd lists only the voters, or no etcd label on the Node), which can never cost quorum, so a host that was down while its member was removed finishes when it returns. **The steps.** @@ -498,7 +498,7 @@ The role controller holds the regrade Lease (`kube-system/fleet-regrade`, the Le - `github_runner_cluster_native_tailscale_leave_other_tailnet` (default `false`): whether the role logs a client out of another tailnet so that k3s can join the cluster's; `false` stops the run instead. - `github_runner_cluster_role_actuator_dir`, `github_runner_cluster_role_actuator_service`, `github_runner_cluster_role_actuator_name`, `github_runner_cluster_role_actuator_configmap`, `github_runner_cluster_role_actuator_kubeconfig`, `github_runner_cluster_role_actuator_namespace` and `github_runner_cluster_role_actuator_commands` are derived: where the role actuator's scripts and settings go (on a native host `/usr/local/lib/github-runner-k3s/role`, on a Docker host the Compose project's `role` directory), its unit, its ServiceAccount, its report ConfigMap, its kubeconfig, the platform namespace it reports in and the commands its watcher needs with the package of each (see [Role actuator](#role-actuator)); `github_runner_cluster_api_candidates` is derived too, the other servers and eligible hosts whose API the regrade service and the role actuator reach the cluster through. - `github_runner_cluster_docker_role_file`, `github_runner_cluster_role_lease_holder` and `github_runner_cluster_role_lease_swap_id_annotation` are derived: the file that records an eligible Docker host's role, and the role controller's holder name on the regrade Lease and the annotation naming its swap (see [A Docker host's recorded role](#a-docker-hosts-recorded-role) and [Runs during a voter swap](#runs-during-a-voter-swap)). -- `github_runner_cluster_native_agent_service`, `github_runner_cluster_native_server_service`, `github_runner_cluster_native_service`, `github_runner_cluster_native_join_url`, `github_runner_cluster_native_join_host`, `github_runner_cluster_native_role_file` and `github_runner_cluster_native_restart_marker` are derived: the units' names (`k3s-agent` and `k3s`), the unit of the host's role, the URL the node joins and the host in it, which the Tailscale wrapper waits for, the role file both units' conditions read, and the prefix of the per-unit file that says a restart is waiting; `github_runner_cluster_native_regrade_level_file`, `github_runner_cluster_native_regrade_started_file`, `github_runner_cluster_native_regrade_dir`, `github_runner_cluster_native_regrade_service`, `github_runner_cluster_native_regrade_commands`, `github_runner_cluster_native_agent_runtime_dir` and `github_runner_cluster_native_server_runtime_dir` are derived too: a native server's requested and started regrading levels, where its regrade scripts and settings go, the unit that runs them, the commands regrading needs with the package of each, and each unit's own runtime directory under `/run` (one per unit, since systemd removes a unit's runtime directory when its condition skips a start, which on a shared one would take the running unit's resolver file). `github_runner_cluster_native_drain_for` is derived as well: what `tasks/native/drain.yml` says was not done when it cannot empty the node, a role switch's by default, which a Colima VM resize passes as its own. `github_runner_cluster_native_bin_dir`, `github_runner_cluster_native_lib_dir`, `github_runner_cluster_native_config_dir`, `github_runner_cluster_native_unit_dir`, `github_runner_cluster_native_data_dir`, `github_runner_cluster_native_tailscale_cli`, `github_runner_cluster_native_apt_keyring`, `github_runner_cluster_native_apt_source`, `github_runner_cluster_native_run_dir` and `github_runner_cluster_native_psi_file` are where the native node's files live, which are not settings, and are variables only so the tests can point them at a stand-in. +- `github_runner_cluster_native_agent_service`, `github_runner_cluster_native_server_service`, `github_runner_cluster_native_service`, `github_runner_cluster_native_join_url`, `github_runner_cluster_agent_server_url`, `github_runner_cluster_native_join_host`, `github_runner_cluster_native_role_file` and `github_runner_cluster_native_restart_marker` are derived: the units' names (`k3s-agent` and `k3s`), the unit of the host's role, the URL the node joins and the host in it, which the Tailscale wrapper waits for, the role file both units' conditions read, and the prefix of the per-unit file that says a restart is waiting; `github_runner_cluster_native_regrade_level_file`, `github_runner_cluster_native_regrade_started_file`, `github_runner_cluster_native_regrade_dir`, `github_runner_cluster_native_regrade_service`, `github_runner_cluster_native_regrade_commands`, `github_runner_cluster_native_agent_runtime_dir` and `github_runner_cluster_native_server_runtime_dir` are derived too: a native server's requested and started regrading levels, where its regrade scripts and settings go, the unit that runs them, the commands regrading needs with the package of each, and each unit's own runtime directory under `/run` (one per unit, since systemd removes a unit's runtime directory when its condition skips a start, which on a shared one would take the running unit's resolver file). `github_runner_cluster_native_drain_for` is derived as well: what `tasks/native/drain.yml` says was not done when it cannot empty the node, a role switch's by default, which a Colima VM resize passes as its own. `github_runner_cluster_native_bin_dir`, `github_runner_cluster_native_lib_dir`, `github_runner_cluster_native_config_dir`, `github_runner_cluster_native_unit_dir`, `github_runner_cluster_native_data_dir`, `github_runner_cluster_native_tailscale_cli`, `github_runner_cluster_native_apt_keyring`, `github_runner_cluster_native_apt_source`, `github_runner_cluster_native_run_dir` and `github_runner_cluster_native_psi_file` are where the native node's files live, which are not settings, and are variables only so the tests can point them at a stand-in. ## Examples diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index 0da104d5..4f7070ba 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -331,8 +331,10 @@ github_runner_cluster_native_tailscale_leave_other_tailnet: false github_runner_cluster_native_agent_service: k3s-agent github_runner_cluster_native_server_service: k3s github_runner_cluster_native_service: "{{ github_runner_cluster_native_server_service if github_runner_cluster_native_role == 'server' else github_runner_cluster_native_agent_service }}" -# Derived, never set directly: the server this host's k3s joins, as https://
:. A node that joins the bootstrap host dials it; the bootstrap host itself, when it runs as an agent or rejoins a cluster that already exists, dials the first other server of the inventory. Empty for a host with nothing to dial. -github_runner_cluster_native_join_url: "{{ github_runner_cluster_effective_server_url if github_runner_cluster_effective_server_url | length > 0 else (github_runner_cluster_topology.hosts[inventory_hostname].peer_server_urls | first | default('')) }}" +# Derived, never set directly: the server an agent of this host joins, as https://
:, on a Docker host (the agent service's K3S_AGENT_SERVER_URL) and a native one alike. A node that joins the bootstrap host dials it; the bootstrap host itself, once it runs as an agent (a role switch) or rejoins a cluster that already exists, dials the first other server of the inventory, since it has no bootstrap host to dial. Empty for a host with nothing to dial. +github_runner_cluster_agent_server_url: "{{ github_runner_cluster_effective_server_url if github_runner_cluster_effective_server_url | length > 0 else (github_runner_cluster_topology.hosts[inventory_hostname].peer_server_urls | first | default('')) }}" +# Derived, never set directly: the server this native host's k3s joins, which is the one above unless native/role.yml found a running server to join instead. +github_runner_cluster_native_join_url: "{{ github_runner_cluster_agent_server_url }}" # Derived, never set directly: the host name in that URL, which the Tailscale wrapper waits for. github_runner_cluster_native_join_host: "{{ github_runner_cluster_native_join_url | urlsplit('hostname') if github_runner_cluster_native_join_url | length > 0 else '' }}" # Derived, never set directly: the file that names the role the host runs, server or agent, which each unit's ExecCondition reads (files/native/fleet-role-is.sh), so the host's role outlives a restart, a reboot and a run of the role. diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh index 0e789277..fdf86bc5 100644 --- a/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh @@ -181,15 +181,18 @@ micro_time() { jq -nr --argjson t "$1" '$t | todate | sub("Z$"; ".000000Z")' } -# The Lease as the API server stores it: held by $1, acquired at epoch $2, last renewed at epoch $3, for LEASE_DURATION_SECONDS. A fourth argument is the resourceVersion of the Lease being replaced, which makes the replacement fail if anyone else has changed it since it was read. +# The Lease as the API server stores it: held by $1, acquired at epoch $2, last renewed at epoch $3, for LEASE_DURATION_SECONDS. A fourth argument is the resourceVersion of the Lease being replaced, which makes the replacement fail if anyone else has changed it since it was read, and a fifth the annotations the Lease carries, as a JSON object (none when omitted). build_lease() { + local annotations=${5:-} jq -n --arg name "$LEASE_NAME" --arg namespace "$LEASE_NAMESPACE" --arg holder "$1" \ --arg acquired "$(micro_time "$2")" --arg renewed "$(micro_time "$3")" \ - --argjson duration "$LEASE_DURATION_SECONDS" --arg version "${4:-}" ' + --argjson duration "$LEASE_DURATION_SECONDS" --arg version "${4:-}" --argjson annotations "${annotations:-null}" ' { apiVersion: "coordination.k8s.io/v1", kind: "Lease", - metadata: ({ name: $name, namespace: $namespace } + (if $version == "" then {} else { resourceVersion: $version } end)), + metadata: ({ name: $name, namespace: $namespace } + + (if $version == "" then {} else { resourceVersion: $version } end) + + (if $annotations == null then {} else { annotations: $annotations } end)), spec: { holderIdentity: $holder, leaseDurationSeconds: $duration, acquireTime: $acquired, renewTime: $renewed } }' } @@ -207,15 +210,15 @@ lease_state() { else "held" end' <<< "$1" } -# Takes the Lease, or fails. Created when there is none, replaced when it is free, expired or already this node's, refused when another holds it, and refused when the API server cannot be asked, since a step must not start on a guess. Both writes fail if someone else got there first. +# Takes the Lease for the holder $1 (this node by default), or fails; the annotations $2, a JSON object, are written with it (none by default). Created when there is none, replaced when it is free, expired or already the holder's, refused when another holds it, and refused when the API server cannot be asked, since a step must not start on a guess. Both writes fail if someone else got there first. acquire_lease() { : "${LEASE_DURATION_SECONDS:?LEASE_DURATION_SECONDS must be set}" - local current now state holder version + local holder=${1:-$NODE_NAME} annotations=${2:-} current now state other version now=$(now_epoch) if ! current=$(kubectl_ get lease "$LEASE_NAME" --namespace "$LEASE_NAMESPACE" --output json 2>&1); then case $current in *"(NotFound)"*) - if build_lease "$NODE_NAME" "$now" "$now" | kubectl_ create -f - > /dev/null; then + if build_lease "$holder" "$now" "$now" "" "$annotations" | kubectl_ create -f - > /dev/null; then return 0 fi log "cannot step: another node created the lease first" @@ -227,26 +230,26 @@ acquire_lease() { ;; esac fi - state=$(lease_state "$current" "$NODE_NAME" "$now") + state=$(lease_state "$current" "$holder" "$now") if [ "$state" = held ]; then - holder=$(jq -r '.spec.holderIdentity' <<< "$current") - log "cannot step: the lease is held by $holder" + other=$(jq -r '.spec.holderIdentity' <<< "$current") + log "cannot step: the lease is held by $other" return 1 fi version=$(jq -r '.metadata.resourceVersion' <<< "$current") - if build_lease "$NODE_NAME" "$now" "$now" "$version" | kubectl_ replace -f - > /dev/null; then + if build_lease "$holder" "$now" "$now" "$version" "$annotations" | kubectl_ replace -f - > /dev/null; then return 0 fi log "cannot step: the lease changed while it was being taken" return 1 } -# Gives the Lease back, if it is still this node's: the holder is cleared, so the next node does not wait for it to expire. If it has meanwhile passed to another node it is left alone. +# Gives the Lease back, if it is still the holder's ($1, this node by default): the holder and the annotations are cleared, so the next node does not wait for it to expire. If it has meanwhile passed to another node it is left alone. release_lease() { - local current + local holder=${1:-$NODE_NAME} current current=$(kubectl_ get lease "$LEASE_NAME" --namespace "$LEASE_NAMESPACE" --output json) || return 1 - [ "$(jq -r '.spec.holderIdentity // ""' <<< "$current")" = "$NODE_NAME" ] || return 0 - jq '.spec.holderIdentity = ""' <<< "$current" | kubectl_ replace -f - > /dev/null + [ "$(jq -r '.spec.holderIdentity // ""' <<< "$current")" = "$holder" ] || return 0 + jq '.spec.holderIdentity = "" | del(.metadata.annotations)' <<< "$current" | kubectl_ replace -f - > /dev/null } # Extends the Lease its holder $1 already has, or fails. The holder, acquire time, duration and annotations are kept and the renew time becomes now, and the replacement fails if anyone has changed the Lease since it was read, so a Lease that has passed to another holder, been released or never existed is refused and left as it was. A holder whose Lease has run out but that nobody has taken since may still renew it. Used by a holder that works for longer than one duration, such as the role controller; a step of the regrade core keeps its sized duration and never renews. diff --git a/roles/github_runner_cluster/files/role/fleet-role-actuator-docker.sh b/roles/github_runner_cluster/files/role/fleet-role-actuator-docker.sh index 8998efde..0bfb47a8 100644 --- a/roles/github_runner_cluster/files/role/fleet-role-actuator-docker.sh +++ b/roles/github_runner_cluster/files/role/fleet-role-actuator-docker.sh @@ -125,8 +125,12 @@ copy_identity() { } # A server whose container did not come up has not joined anything, so the agent is brought back at once, with the role file naming it again and the join URL gone: unlike a native host, whose own Tailscale client keeps the API in reach, a Docker host with neither node container up cannot read the Lease that a rollback asks for, and would be left with nothing running. +# A container already running was started by an earlier take of the step, whose report may not have reached the API, so its start is reported again: the controller waits for a heartbeat later than STARTED_AT, and an empty one never comes. start_server_unit() { - if server_active; then return 0; fi + if server_active; then + STARTED_AT=$(last_start) + return 0 + fi compose agent stop "$(service_of agent)" || return 1 if copy_identity agent server && record_start && compose server up --detach --no-deps "$(service_of server)"; then return 0 @@ -143,8 +147,12 @@ stop_server_unit() { compose server stop "$(service_of server)" } +# As start_server_unit, for the agent: a finish taken again finds the agent it started running. start_agent_unit() { - if agent_active; then return 0; fi + if agent_active; then + STARTED_AT=$(last_start) + return 0 + fi copy_identity server agent || return 1 record_start || return 1 compose agent up --detach --no-deps "$(service_of agent)" diff --git a/roles/github_runner_cluster/files/role/fleet-role-actuator.sh b/roles/github_runner_cluster/files/role/fleet-role-actuator.sh index 5271569c..83be11cc 100644 --- a/roles/github_runner_cluster/files/role/fleet-role-actuator.sh +++ b/roles/github_runner_cluster/files/role/fleet-role-actuator.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash # fleet-role-actuator: switches this host between a k3s server and a k3s agent, for the role controller's voter swap (scripts/fleet-role.sh), as its Node asks. It does only what has to happen on the host: the controller does everything on the API (the cordon, the wait for runner pods, the drain, the taint, the regrading level label, the removal of the etcd member, the clean-up of the Node's labels and annotations, the uncordon), and the actuator's own credential can read Nodes and the regrade Lease and patch its report, nothing more. # -# THE NODE. Only the controller sets these, on this host's Node; all are under github-runner.exadev.com/. The label desired-role (server or agent) is the role the host is to run. The annotation role-swap names the swap (--), role-step the step this host is to take in it (promote, step-down or rollback), role-join-node the node of the voter a promotion joins, and snapshot-request a swap whose snapshot this host, a voter that stays, is to take. +# THE NODE. Only the controller sets these, on this host's Node; all are under github-runner.exadev.com/. The label desired-role (server or agent) is the role the host is to run. The annotation role-swap names the swap (--), role-step the step this host is to take in it (promote, step-down or rollback), role-join-node the node of the voter a promotion joins, snapshot-request a swap whose snapshot this host, a voter that stays, is to take, and role-member-gone the swap under which every remaining voter reports that etcd no longer lists this host's member. # # THE REPORT. Each step's outcome goes to the ConfigMap ACTUATOR_CONFIGMAP in ACTUATOR_NAMESPACE, under this node's name, as JSON: {swap, step, phase, role, started_at, at, message, snapshot}, phase being running, done, failed or refused, role what the role file names afterwards, started_at the host's time just before a unit was started (what the controller waits for a later heartbeat than), at the time of the report and snapshot the names of the files a snapshot created. A report the API does not take is made again by the watch, which runs a step again until its outcome is reported; every step can be run again. # -# AUTHORITY. Every step that could add a member (promote), and every other step a swap asks for (step-down, rollback, snapshot), is taken only while the regrade Lease (LEASE_NAMESPACE/LEASE_NAME) is held, unexpired, by ROLE_LEASE_HOLDER with the swap's id in its SWAP_ID_ANNOTATION: the controller holds it for the whole swap, and a regrade step or a run of the role cannot hold it meanwhile. finish-demote needs no Lease: it runs only once the role file names the agent and the server's etcd member is gone (a tombstone in the datastore, the Node's etcd.k3s.cattle.io/removed-node-name naming the datastore's member, or no etcd label on the Node), and completing the demotion of a member that is no longer one can never cost quorum, so a host that was down while it was removed finishes on its return. +# AUTHORITY. Every step that could add a member (promote), and every other step a swap asks for (step-down, rollback, snapshot), is taken only while the regrade Lease (LEASE_NAMESPACE/LEASE_NAME) is held, unexpired, by ROLE_LEASE_HOLDER with the swap's id in its SWAP_ID_ANNOTATION: the controller holds it for the whole swap, and a regrade step or a run of the role cannot hold it meanwhile. finish-demote needs no Lease: it runs only once the role file names the agent (or, naming the server, the datastore holds k3s's tombstone, which a host down during its step-down and removal has in place of the role file, or the Node asks for a step-down that failed before the role file was written, after which the controller removes a host that is not Ready) and the server's etcd member is gone (a tombstone in the datastore, the Node's etcd.k3s.cattle.io/removed-node-name naming the datastore's member, the Node's role-member-gone naming the swap it is in, or no etcd label on the Node), and completing the demotion of a member that is no longer one can never cost quorum, so a host that was down while it was removed finishes on its return. # # COMMANDS. watch runs for ever, reading this host's Node every POLL_SECONDS and taking the step it asks for, a snapshot it asks for, and the finish of a demotion whose member is gone. status prints the host's role, units and datastore and its last report. promote ID, step-down ID, rollback ID, snapshot ID and finish-demote take one step by hand and report it as the watch does. promote-local [JOIN_URL] and demote-local are the host-local part of a switch with no Lease and no report, for the role's own promotion and demotion (tasks/native/server_turn.yml and demote.yml), so that Ansible and the actuator switch a host the same way. # @@ -48,6 +48,7 @@ SWAP_ANNOTATION=github-runner.exadev.com/role-swap STEP_ANNOTATION=github-runner.exadev.com/role-step JOIN_NODE_ANNOTATION=github-runner.exadev.com/role-join-node SNAPSHOT_ANNOTATION=github-runner.exadev.com/snapshot-request +MEMBER_GONE_ANNOTATION=github-runner.exadev.com/role-member-gone # What k3s writes: the etcd role label it puts on a member's Node, and the annotation its member controller records once etcd has removed the member named in it (pkg/etcd/member_controller.go). ETCD_LABEL=node-role.kubernetes.io/etcd REMOVED_ANNOTATION=etcd.k3s.cattle.io/removed-node-name @@ -248,6 +249,11 @@ member_gone() { MESSAGE="k3s recorded the removal of member $member" return 0 fi + # The controller's word, for the swap the Node is in, that etcd lists none of this host's members, from every remaining voter's own report: k3s's record names only the member it last removed and is cleared whenever the Node's etcd name annotation changes (pkg/etcd/member_controller.go), so it can disagree with the datastore's member while etcd lists neither. + if [ -n "$(node_annotation "$SWAP_ANNOTATION")" ] && [ "$(node_annotation "$MEMBER_GONE_ANNOTATION")" = "$(node_annotation "$SWAP_ANNOTATION")" ]; then + MESSAGE="the controller reports that etcd no longer lists this host's member, for swap $(node_annotation "$SWAP_ANNOTATION")" + return 0 + fi if [ -z "$(node_label "$ETCD_LABEL")" ]; then MESSAGE="the Node carries no etcd label" return 0 @@ -256,10 +262,25 @@ member_gone() { return 1 } +# Succeeds when this host's Node, as last read, asks it to step down to an agent: a swap's step-down, with the agent as the desired role. The controller removes the member once that step-down is done or, while the Node is not Ready, once it has failed or run past its time, so a server can be removed with its role file still naming the server. +asked_to_step_down() { + [ -n "$(node_annotation "$SWAP_ANNOTATION")" ] && [ "$(node_annotation "$STEP_ANNOTATION")" = step-down ] && [ "$(node_label "$DESIRED_ROLE_LABEL")" = agent ] +} + +# Succeeds when a host whose role file names the server is to be finished as an agent once its member is gone: its datastore holds k3s's tombstone (a host that was down while it was stepped down and removed never wrote the agent's role, and its units idle on the tombstone), or its Node asks it to step down (a step-down that failed leaves the server's role file, and the server unit's restarts would join etcd again as a new member, under a new name). member_gone still decides whether the member is gone, and finishing the demotion of a member that is gone can never cost quorum. $1 is "read" for the Node to be read first, anything else when the caller has just read it. +server_to_finish() { + datastore_tombstoned && return 0 + if [ "$1" = read ]; then { select_api_server && read_node; } || return 1; fi + asked_to_step_down +} + cmd_finish_demote() { local role role=$(read_role) - [ "$role" = agent ] || { MESSAGE="the role file names '${role}', not agent, so no demotion of this host is under way"; return "$EXIT_REFUSED"; } + if [ "$role" != agent ] && ! { [ "$role" = server ] && server_to_finish read; }; then + MESSAGE="the role file names '${role}', not agent, the datastore holds no tombstone and the Node asks for no step-down, so no demotion of this host is under way" + return "$EXIT_REFUSED" + fi member_gone || return "$EXIT_NOT_YET" report running locked demote_local @@ -342,10 +363,11 @@ step_taken() { [ "$step" = step-down ] && settled "$entry" "$swap" finish-demote running "done" failed } -# Succeeds when the watch is to take (or retake) the finish of the demotion whose swap is $2, given this node's report $1: the role file names the agent, the finish is not reported done, and something of the server is left, or the Node still carries this swap's step-down (so the controller is waiting for the finish and has not seen it reported). A finish whose report the API did not take leaves nothing of the server behind, so only the report is made again; the finish is idempotent, and member_gone still decides whether the member is gone. +# Succeeds when the watch is to take (or retake) the finish of the demotion whose swap is $2, given this node's report $1 and its Node as the pass read it: the role file names the agent, or names the server and server_to_finish holds, the finish is not reported done, and something of the server is left, or the Node still carries this swap's step-down (so the controller is waiting for the finish and has not seen it reported). A finish whose report the API did not take leaves nothing of the server behind, so only the report is made again; the finish is idempotent, and member_gone still decides whether the member is gone. finish_due() { - local entry=$1 swap=$2 - [ "$(read_role)" = agent ] || return 1 + local entry=$1 swap=$2 role + role=$(read_role) + [ "$role" = agent ] || { [ "$role" = server ] && server_to_finish read-already; } || return 1 settled "$entry" "$swap" finish-demote "done" && return 1 if server_active || datastore_present; then return 0; fi [ -n "$swap" ] && [ "$(node_annotation "$STEP_ANNOTATION")" = step-down ] diff --git a/roles/github_runner_cluster/templates/env.j2 b/roles/github_runner_cluster/templates/env.j2 index b16495c0..55b0eefa 100644 --- a/roles/github_runner_cluster/templates/env.j2 +++ b/roles/github_runner_cluster/templates/env.j2 @@ -15,7 +15,7 @@ K3S_VPN_AUTH_JOIN_KEY={{ github_runner_cluster_mesh_join_key }} K3S_VPN_AUTH_CONTROL_SERVER_URL={{ github_runner_cluster_mesh_control_server_url }} {% endif %} K3S_JOIN_SERVER_URL={{ github_runner_cluster_effective_server_url }} -K3S_AGENT_SERVER_URL={{ github_runner_cluster_effective_server_url }} +K3S_AGENT_SERVER_URL={{ github_runner_cluster_agent_server_url }} {% if github_runner_cluster_datastore == 'sqlite' %} K3S_DATASTORE=sqlite {% endif %} diff --git a/scripts/fleet-role-stats.jq b/scripts/fleet-role-stats.jq index 8ab28111..5a376981 100644 --- a/scripts/fleet-role-stats.jq +++ b/scripts/fleet-role-stats.jq @@ -1,4 +1,10 @@ -# Statistics and etcd metrics parsing shared by the role controller's probe (scripts/fleet-role-probe.sh) and the controller itself (scripts/fleet-role.jq), included with `include "fleet-role-stats";` and jq -L naming this directory. +# Statistics, etcd metrics parsing and the facts about k3s that are shared by the role controller's probe (scripts/fleet-role-probe.sh) and the controller itself (scripts/fleet-role.jq), included with `include "fleet-role-stats";` and jq -L naming this directory. + +# k3s's learnerMaxStallTime (pkg/etcd/etcd.go): a learner that makes no progress for this long is removed, so a membership change has settled this long after it, and a candidate that has not become a voter by then has failed. +def learner_stall_seconds: 300; + +# The annotation k3s's member controller sets on a Node once etcd has removed the member named in it (pkg/etcd/member_controller.go). The Node keeps its etcd label and its stale EtcdIsVoter condition until a demotion cleans them up, so a Node carrying this is not a voter whatever else it says. +def removed_annotation: "etcd.k3s.cattle.io/removed-node-name"; # The tail the controller judges a host by, as a whole percentile. def tail_percentile: 99; diff --git a/scripts/fleet-role-swap.jq b/scripts/fleet-role-swap.jq new file mode 100644 index 00000000..50c64ec1 --- /dev/null +++ b/scripts/fleet-role-swap.jq @@ -0,0 +1,365 @@ +# The role controller's voter swap (see scripts/fleet-role.sh, which gathers the inputs and carries out the action this program names): what the next step of a swap is, from what the cluster shows, and whether a swap may start. Pure: nothing here reads the cluster or the clock, and the controller holds no state of its own beyond the status it publishes, since the step a swap is at is read from the Nodes, the actuators' reports and the regrade Lease, so a controller pod that is rescheduled resumes where the swap is. +# +# Included with `include "fleet-role-swap";` and jq -L naming this directory. `context($s)` and `next_action($s)` take one object: +# - config: `now` (epoch seconds), `target`, `holder` (the Lease holder the controller is), `dry_run`, `operation_timeout` and `runner_wait` (seconds), `eligible_label`, `pinned_label` and `level_label` (the Node labels the controller reads). +# - nodes: the Nodes, as `kubectl get nodes -o json` lists them (the `items`). +# - lease: the regrade Lease as the controller read it, as {state, holder, swap_id, swap_nodes}: state being free, expired, mine or held as fleet-regrade-core.sh's lease_state says for the controller as the holder (after any renewal), swap_nodes the annotation's comma-separated "out,in". +# - actuators: the actuators' reports by node (the ConfigMap's keys, parsed), {} when there are none. +# - node_leases: the renew time of each swap node's kube-node-lease Lease, by node, which a running kubelet renews. +# - snapshots: the ETCDSnapshotFile objects (`items`), or null when they could not be listed. +# - runner_pods: how many unfinished runner pods run on each swap node, by node. +# - decision: the decision program's output (scripts/fleet-role.jq) for this pass, given the `context` below. +# - previous: the status the controller published on the last pass, or null; request: the manual request pending, {out, in, at}, or null. +# +# Every wait derives from the settings: the operation timeout, the runner wait and k3s's learner stall time (fleet-role-stats.jq). + +include "fleet-role-stats"; + +# The names the controller and the actuators share, fixed by the actuator (roles/github_runner_cluster/files/role/fleet-role-actuator.sh) and by k3s; tests/unit/test_fleet_role_swap.py holds the two together, and fleet-role.sh reads the names it writes from here. +def domain: "github-runner.exadev.com/"; +def interface: { + desired_role_label: (domain + "desired-role"), + swap_annotation: (domain + "role-swap"), + step_annotation: (domain + "role-step"), + join_node_annotation: (domain + "role-join-node"), + snapshot_annotation: (domain + "snapshot-request"), + member_gone_annotation: (domain + "role-member-gone"), + lease_swap_id_annotation: (domain + "swap-id"), + lease_swap_nodes_annotation: (domain + "swap-nodes"), + etcd_label: "node-role.kubernetes.io/etcd", + control_plane_label: "node-role.kubernetes.io/control-plane", + master_label: "node-role.kubernetes.io/master", + etcd_annotation_prefix: "etcd.k3s.cattle.io/", + remove_annotation: "etcd.k3s.cattle.io/remove", + removed_annotation: removed_annotation, + control_plane_taint: "node-role.kubernetes.io/control-plane", + lease_holder: "fleet-role" +}; + +def voter_condition: "EtcdIsVoter"; + +# The seconds a step may take before the swap gives up on it or the controller says it is overdue, by step. +def budget($c): + { + snapshot: $c.operation_timeout, + drain: ($c.runner_wait + $c.operation_timeout), + evict: ($c.runner_wait + $c.operation_timeout), + promote: (learner_stall_seconds + $c.operation_timeout), + gates: $c.operation_timeout, + "step-down": $c.operation_timeout, + remove: $c.operation_timeout, + finish: $c.operation_timeout, + cleanup: $c.operation_timeout, + rollback: $c.operation_timeout + }; + +# What the decision reads off a Node. +def facts($c): + interface as $i + | (.metadata.labels // {}) as $labels + | (.metadata.annotations // {}) as $annotations + | ((.status.conditions // [])) as $conditions + | { + name: .metadata.name, + ready: (($conditions | map(select(.type == "Ready")) | .[0].status) == "True"), + labelled: ($labels[$i.etcd_label] == "true"), + voter: ($labels[$i.etcd_label] == "true" and (($conditions | map(select(.type == voter_condition)) | .[0].status) == "True") and (($annotations[$i.removed_annotation] // "") == "")), + removed: (($annotations[$i.removed_annotation] // "") != ""), + remove_requested: (($annotations[$i.remove_annotation] // "") != ""), + cordoned: (.spec.unschedulable // false), + tainted: ((.spec.taints // []) | any(.key == $i.control_plane_taint)), + level: $labels[$c.level_label], + eligible: ($labels[$c.eligible_label] == "true"), + pinned: ($labels[$c.pinned_label] == "true"), + desired: ($labels[$i.desired_role_label] // ""), + swap: ($annotations[$i.swap_annotation] // ""), + step: ($annotations[$i.step_annotation] // ""), + snapshot_request: ($annotations[$i.snapshot_annotation] // ""), + member_gone: ($annotations[$i.member_gone_annotation] // ""), + etcd_annotations: ($annotations | keys | map(select(startswith($i.etcd_annotation_prefix)))), + server_labels: ([$i.etcd_label, $i.control_plane_label, $i.master_label] | map(select($labels[.] != null))) + }; + +# The Nodes by name, with their facts. +def by_name($s): $s.nodes | map(facts($s.config)) | map({key: .name, value: .}) | from_entries; + +def swap_id($out; $in; $now): ($now | strftime("%Y%m%dT%H%M%SZ")) + "-" + $out + "-" + $in; + +# The epoch time a swap id names (--). +def id_epoch: .[0:16] | strptime("%Y%m%dT%H%M%SZ") | mktime; + +# The swap in progress, as {id, out, in, source}, or null: from the Lease while the controller holds it with a swap's annotations, else from the status it last published, else from the swap annotations on the Nodes (the one id they carry, split at the pair of nodes whose names it ends with, since a node name may hold a hyphen). +def identity($s): + $s.lease as $lease + | ($s.previous.swap // null) as $previous + | if $lease.holder == $s.config.holder and ($lease.swap_id // "") != "" and (($lease.swap_nodes // "") | split(",") | length) == 2 then + ($lease.swap_nodes | split(",")) as $nodes + | {id: $lease.swap_id, out: $nodes[0], in: $nodes[1], source: "lease"} + elif $previous != null then + {id: $previous.id, out: $previous.out, in: $previous.in, source: "status"} + else + ([$s.nodes[] | (.metadata.annotations // {})[interface.swap_annotation] // empty | select(. != "")] | unique) as $ids + | ($s.nodes | map(.metadata.name)) as $names + | if ($ids | length) != 1 then null + else $ids[0] as $id + | ([$names[] as $out | $names[] as $in | select($out != $in and ($id | endswith("-" + $out + "-" + $in)) and ($id | id_epoch | . != null)) | {id: $id, out: $out, in: $in, source: "nodes"}] | first) + end + end; + +# What the decision program must be told about the swap in progress: the voters to expect (one more than the target while the new voter and the outgoing one both vote) and the voter the swap is taking out, which the gates do not hold to its own health. +def context($s): + by_name($s) as $nodes + | identity($s) as $id + | $s.config.target as $target + | if $id == null then {swap: null, expected_voters: $target, ignored_voters: []} + else ($nodes[$id.out]) as $out | ($nodes[$id.in]) as $in + | (($in != null and $in.voter) and ($out != null) and ($out.removed | not)) as $both + | {swap: $id, + expected_voters: (if $both then $target + 1 else $target end), + ignored_voters: (if $both and (($out.ready | not) or $out.level != null or $out.desired == "agent") then [$id.out] else [] end)} + end; + +# The reasons a swap of $out for $in may not start, as {status, reasons}: status 2 for a name that is no node, a node that is not marked eligible or not the voter or standby it is taken for (the request is wrong), 1 for a cluster that is not ready for it (every gate), 0 for none. Fitness and the cooldown are not among them: a request is carried out whatever the verdicts say. +def check($s; $out; $in): + by_name($s) as $nodes + | $s.decision.gates as $gates + | $s.lease as $lease + | ($nodes[$out]) as $o + | ($nodes[$in]) as $i + | [ + (if $out == $in then "a swap needs two nodes, not \($out) twice" else empty end), + (if $o == null then "no node is named \($out)" else empty end), + (if $i == null then "no node is named \($in)" else empty end), + (if $o != null and ($o.voter | not) then "\($out) is not a voter" else empty end), + (if $i != null and ($i.labelled or $i.voter) then "\($in) is already a member of etcd" else empty end), + (if $o != null and ($o.eligible | not) then "\($out) is not marked eligible" else empty end), + (if $i != null and ($i.eligible | not) then "\($in) is not marked eligible" else empty end) + ] as $usage + | [ + (if $o != null and $o.pinned then "voter \($out) is pinned, so it is never taken out" else empty end), + (if $i != null and ($i.ready | not) then "\($in) is not Ready" else empty end), + (if $lease.state == "held" then "the regrade Lease is held by \($lease.holder)" else empty end), + (if $lease.state == "mine" and ($lease.swap_id // "") != "" then "a swap is in progress (\($lease.swap_id))" else empty end), + ($gates[]) + ] as $refusals + | if ($usage | length) > 0 then {status: 2, reasons: $usage} + elif ($refusals | length) > 0 then {status: 1, reasons: $refusals} + else {status: 0, reasons: []} end; + +def step_record($s; $id; $step; $extra): + ($s.previous.swap // null) as $previous + | (if $previous != null and $previous.id == $id.id and $previous.step == $step then $previous.step_since else $s.config.now end) as $since + | {id: $id.id, out: $id.out, in: $id.in, step: $step, + started_at: ($id.id | id_epoch), + step_since: $since, + deadline: ($since + budget($s.config)[$step]), + manual: ($previous.manual // false), + snapshot: ($previous.snapshot // null), + outgoing_pending: ($previous.outgoing_pending // false)} + $extra; + +# The actuator's last report on $node for the step $step of swap $id, or null. +def report($s; $node; $id; $step): + ($s.actuators[$node] // null) as $r + | if $r != null and $r.swap == $id and $r.step == $step then $r else null end; + +def ended($outcome; $reasons; $s; $id): + {outcome: $outcome, id: $id.id, out: $id.out, in: $id.in, at: $s.config.now, reasons: $reasons}; + +# An RFC 3339 time cut to whole seconds, which compares as text. +def seconds: .[0:19]; + +# Succeeds when $node's kubelet has renewed its node Lease at or after $since (a time as RFC 3339). +def renewed_since($s; $node; $since): + ($s.node_leases[$node] // "") as $renewed + | $since != null and $since != "" and $renewed != "" and ($renewed | seconds) >= ($since | seconds); + +# Succeeds when etcd no longer lists the outgoing node $o's member: it is not a voter, and every voter reports, fresh, that etcd lists as many members as there are voters, so the members are the voters and nothing else (a learner or a member that has not started counts among the members a voter reports). +def gone_from_etcd($s; $o): + ($s.decision.membership.voters) as $voters + | ($o.voter | not) and ($voters | length) > 0 + and ($voters | all(. as $v | $s.decision.membership.known_members[$v] == ($voters | length))); + +# The swap, step by step. Returns {swap, action, wait, consume_request, last_swap}: `swap` the record the status holds, `action` the one thing to do this pass (or null), `wait` why nothing is done, `last_swap` the record of a swap or a request that ends this pass if the action succeeds. +def in_progress($s; $id): + by_name($s) as $nodes + | ($nodes[$id.out]) as $o + | ($nodes[$id.in]) as $i + | $s.config as $c + | $s.decision.gates as $gates + | $s.lease as $lease + | ($s.snapshots) as $snapshots + | def wait($step; $reasons; $extra): {swap: step_record($s; $id; $step; $extra), action: null, wait: $reasons, consume_request: false, last_swap: null}; + def wait($step; $reasons): wait($step; $reasons; {}); + def do($step; $action; $extra): {swap: step_record($s; $id; $step; $extra), action: $action, wait: [], consume_request: false, last_swap: null}; + def do($step; $action): do($step; $action; {}); + def finalise($step; $outcome; $reasons; $uncordon; $pending): + {swap: step_record($s; $id; $step; {}), + action: {kind: "finalise", outcome: $outcome, uncordon: $uncordon}, + wait: [], consume_request: false, + last_swap: (ended($outcome; $reasons; $s; $id) + (if $pending then {outgoing_pending: true} else {} end))}; + def settled: ($gates | length) == 0; + def cleaned($n): ($n.labelled | not) and $n.level == null and ($n.etcd_annotations | length) == 0 and ($n.tainted | not); + + if $lease.state == "held" then + wait($s.previous.swap.step // "snapshot"; ["the regrade Lease is held by \($lease.holder): nothing is done to the swap of \($id.out) for \($id.in) until it is released or has expired"]) + elif $lease.state != "mine" then + do($s.previous.swap.step // "snapshot"; {kind: "take_lease", id: $id.id, out: $id.out, in: $id.in, manual: ($s.previous.swap.manual // false)}) + elif $i == null then + finalise("gates"; "abandoned"; ["the candidate node \($id.in) is gone"]; []; false) + elif $o == null then + if $i.voter then finalise("cleanup"; "completed"; ["the outgoing node \($id.out) is gone"]; []; false) + else finalise("gates"; "abandoned"; ["the outgoing node \($id.out) is gone before the candidate voted"]; [$id.in]; false) end + + # A candidate that did not become a voter is being rolled back. + elif $i.swap == $id.id and $i.step == "rollback" then + report($s; $id.in; $id.id; "rollback") as $r + | if $r == null or $r.phase == "running" then wait("rollback"; ["waiting for \($id.in)'s actuator to roll the promotion back"]) + elif $r.phase == "failed" or $r.phase == "refused" then wait("rollback"; ["\($id.in)'s actuator reports the rollback \($r.phase): \($r.message)"]) + elif ($s.decision.membership.known_members | to_entries | map(.value) | all(. == $c.target)) | not then + wait("rollback"; ["waiting for every voter to list \($c.target) members, which is etcd without the candidate"]) + elif ($i.labelled or $i.cordoned or $i.tainted or ($i.etcd_annotations | length) > 0) then + do("rollback"; {kind: "clean_node", node: $id.in, uncordon: true}) + else finalise("rollback"; "rolled-back"; ["\($id.in) did not become a voter and is an agent again"]; []; false) end + + # The outgoing voter has been asked to step down. + elif $o.swap == $id.id and $o.step == "step-down" then + report($s; $id.out; $id.id; "step-down") as $down + | report($s; $id.out; $id.id; "finish-demote") as $finish + | if cleaned($o) then + if ($o.cordoned | not) then finalise("cleanup"; "completed"; ["\($id.in) votes and \($id.out) is an agent"]; []; ($s.previous.swap.outgoing_pending // false)) + elif ($s.previous.swap.outgoing_pending // false) then + finalise("cleanup"; "completed"; ["\($id.in) votes; \($id.out) did not come back as an agent in time and is left to node recovery"]; [$id.out]; true) + elif $o.ready and renewed_since($s; $id.out; $finish.started_at) then do("cleanup"; {kind: "uncordon", node: $id.out}) + else + step_record($s; $id; "cleanup"; {}) as $record + | if $c.now >= $record.deadline then + {swap: ($record + {outgoing_pending: true}), action: null, wait: ["\($id.out) did not come back as an agent within \(budget($c).cleanup) s; it is left to node recovery"], consume_request: false, last_swap: null} + else wait("cleanup"; ["waiting for \($id.out) to be Ready as an agent"]) end + end + # Removed, as k3s records it on the Node or as etcd's own membership shows it, which is the proof that holds when the Node's labels and annotations have been set again by a server still running on the host, or name a member other than the one its datastore holds. + elif $o.removed or gone_from_etcd($s; $o) then + if $finish != null and $finish.phase == "done" then do("cleanup"; {kind: "clean_node", node: $id.out, uncordon: false}) + # The actuator cannot read etcd's membership, and k3s's record on the Node can name another member than the datastore's, so the controller tells it, under this swap, what every remaining voter reports. + elif gone_from_etcd($s; $o) and $o.member_gone != $id.id then do("finish"; {kind: "mark_member_gone", node: $id.out, id: $id.id}) + else + step_record($s; $id; "finish"; {}) as $record + | if $c.now >= $record.deadline and ($s.previous.swap.outgoing_pending // false) then + finalise("finish"; "completed"; ["\($id.in) votes and etcd no longer lists \($id.out), whose actuator did not finish its demotion within \(budget($c).finish) s; it is left to node recovery"]; []; true) + elif $c.now >= $record.deadline then + {swap: ($record + {outgoing_pending: true}), action: {kind: "clean_node", node: $id.out, uncordon: false}, wait: [], consume_request: false, last_swap: null} + else wait("finish"; ["waiting for \($id.out)'s actuator to finish its demotion"]) end + end + elif $o.remove_requested then wait("remove"; ["waiting for k3s to remove \($id.out)'s member"]) + else + step_record($s; $id; "step-down"; {}) as $record + | (($down != null and $down.phase == "done") + or ($o.ready | not) and ($down != null and ($down.phase == "failed" or $down.phase == "refused") or $c.now >= $record.deadline)) as $removable + | if $removable then + if settled then do("remove"; {kind: "annotate_remove", node: $id.out}) + else wait("remove"; ["not removing \($id.out) while the remaining voters are not all healthy"] + $gates) end + elif $down != null and ($down.phase == "failed" or $down.phase == "refused") then + wait("step-down"; ["\($id.out)'s actuator reports the step-down \($down.phase) and the node is Ready, so it is held for the owner: \($down.message)"]) + else wait("step-down"; ["waiting for \($id.out)'s actuator to step it down to etcd-only"]) end + end + + # The candidate has been asked to become a voter. + elif $i.swap == $id.id and $i.step == "promote" then + report($s; $id.in; $id.id; "promote") as $r + | step_record($s; $id; "promote"; {}) as $record + | ($r != null and $r.phase == "done" and $i.voter and $i.ready and renewed_since($s; $id.in; $r.started_at)) as $voting + | if $voting and $i.cordoned then do("promote"; {kind: "uncordon", node: $id.in}) + elif $voting then + if settled then + if ($o.ready | not) then do("step-down"; {kind: "step_down_out", node: $id.out, id: $id.id, drain: false}) + elif ($o.cordoned | not) or ($o.tainted | not) then do("evict"; {kind: "cordon_out", node: $id.out}) + else + step_record($s; $id; "evict"; {}) as $evict + | if ($s.runner_pods[$id.out] // 0) > 0 and $c.now < $evict.step_since + $c.runner_wait then + wait("evict"; ["waiting for \($s.runner_pods[$id.out]) runner pod(s) on \($id.out) to finish before it is drained"]) + else do("step-down"; {kind: "step_down_out", node: $id.out, id: $id.id, drain: true}) end + end + else wait("gates"; ["not taking \($id.out) out while the cluster is not healthy with \($id.in) in"] + $gates) end + elif $r != null and $r.phase == "refused" then + finalise("promote"; "abandoned"; ["\($id.in)'s actuator refused the promotion: \($r.message)"]; [$id.in]; false) + elif ($r != null and $r.phase == "failed") then + do("rollback"; {kind: "begin_rollback", node: $id.in, id: $id.id, remove: (($i.etcd_annotations | length) > 0)}) + elif $c.now >= $record.deadline then + do("rollback"; {kind: "begin_rollback", node: $id.in, id: $id.id, remove: (($i.etcd_annotations | length) > 0)}) + else wait("promote"; ["waiting for \($id.in) to be a voter and Ready on a heartbeat after its server started"]) end + + # The candidate votes and no swap annotation is left on it or on the outgoing voter: a swap that ended with the Lease still held, which is ended and never started again. + elif $i.voter and $i.swap == "" then + finalise("cleanup"; "completed"; ["\($id.in) votes and the swap's annotations are cleared"]; []; ($s.previous.swap.outgoing_pending // false)) + + # Before anything has been done to the candidate: the snapshot, then emptying it. + else + step_record($s; $id; "snapshot"; {}) as $record + | ($s.decision.membership.voters | map(select(. != $id.out))) as $keepers + | ($keepers | map(select(. as $n | $nodes[$n].snapshot_request == $id.id))) as $asked + | (if ($asked | length) > 0 then report($s; $asked[-1]; $id.id; "snapshot") else null end) as $taken + | (if $taken != null and $taken.phase == "done" and $snapshots != null + then ($taken.snapshot | split(",") | map(select(. != ""))) as $names + | ($names | length) > 0 and ($names | all(. as $name | $snapshots | map(select(.spec.snapshotName == $name and .status.readyToUse == true)) | length > 0)) + else false end) as $ready + | if $ready then + step_record($s; $id; "drain"; {snapshot: $taken.snapshot}) as $drain + | if ($i.cordoned | not) then + if settled then do("drain"; {kind: "cordon", node: $id.in}; {snapshot: $taken.snapshot}) + elif $c.now >= $record.deadline then finalise("snapshot"; "abandoned"; ["the cluster was not healthy enough to start within \(budget($c).snapshot) s"] + $gates; []; false) + else wait("snapshot"; ["not emptying \($id.in) while the cluster is not healthy"] + $gates) end + elif ($s.runner_pods[$id.in] // 0) > 0 then + if $c.now >= $drain.step_since + $c.runner_wait then finalise("drain"; "abandoned"; ["runner pods on \($id.in) did not finish within \($c.runner_wait) s"]; [$id.in]; false) + else wait("drain"; ["waiting for \($s.runner_pods[$id.in]) runner pod(s) on \($id.in) to finish"]; {snapshot: $taken.snapshot}) end + elif $c.now >= $drain.deadline then finalise("drain"; "abandoned"; ["\($id.in) could not be drained within \(budget($c).drain) s"]; [$id.in]; false) + elif settled then do("drain"; {kind: "promote_in", node: $id.in, id: $id.id, join_node: $keepers[0]}; {snapshot: $taken.snapshot}) + else wait("drain"; ["not promoting \($id.in) while the cluster is not healthy"] + $gates; {snapshot: $taken.snapshot}) end + elif $taken != null and ($taken.phase == "failed" or $taken.phase == "refused") then + ($keepers | map(select(. as $n | $nodes[$n].snapshot_request != $id.id))) as $untried + | if ($untried | length) > 0 then do("snapshot"; {kind: "request_snapshot", node: $untried[0], id: $id.id}) + else finalise("snapshot"; "abandoned"; ["no voter could take a snapshot: \($taken.message)"]; []; false) end + elif $c.now >= $record.deadline then + finalise("snapshot"; "abandoned"; ["no snapshot was ready within \(budget($c).snapshot) s"]; []; false) + elif ($asked | length) > 0 then wait("snapshot"; ["waiting for \($asked[-1])'s snapshot to be ready"]) + elif ($gates | length) > 0 then wait("snapshot"; ["not starting while the cluster is not healthy"] + $gates) + elif ($keepers | length) == 0 then finalise("snapshot"; "abandoned"; ["there is no voter to take a snapshot"]; []; false) + else do("snapshot"; {kind: "request_snapshot", node: $keepers[0], id: $id.id}) end + end; + +# Whether a swap starts now, and which: a manual request that passes its checks, or, with the dry run off, the decision's swap. {swap, action, wait, consume_request, last_swap, check}. +def starting($s): + $s.request as $request + | $s.config as $c + | $s.lease as $lease + | (if $request == null then null else check($s; $request.out; $request.in) end) as $check + | if $request != null then + if $check.status == 0 then + swap_id($request.out; $request.in; $c.now) as $id + | {swap: {id: $id, out: $request.out, in: $request.in, step: "snapshot", started_at: $c.now, step_since: $c.now, deadline: ($c.now + budget($c).snapshot), manual: true, snapshot: null, outgoing_pending: false}, + action: {kind: "take_lease", id: $id, out: $request.out, in: $request.in, manual: true}, + wait: [], consume_request: true, last_swap: null, check: $check} + else + {swap: null, action: null, wait: ["the request to swap \($request.out) for \($request.in) is refused: \($check.reasons | join("; "))"], consume_request: true, + last_swap: {outcome: "abandoned", id: null, out: $request.out, in: $request.in, at: $c.now, reasons: $check.reasons}, check: $check} + end + elif $s.decision.decision.action == "swap" then + $s.decision.decision.swap as $swap + | if $c.dry_run then + {swap: null, action: null, wait: ["dry run: would swap out \($swap.out) for \($swap.in)"], consume_request: false, last_swap: null, check: null} + elif $lease.state == "held" then + {swap: null, action: null, wait: ["not swapping \($swap.out) for \($swap.in) while the regrade Lease is held by \($lease.holder)"], consume_request: false, last_swap: null, check: null} + else + swap_id($swap.out; $swap.in; $c.now) as $id + | {swap: {id: $id, out: $swap.out, in: $swap.in, step: "snapshot", started_at: $c.now, step_since: $c.now, deadline: ($c.now + budget($c).snapshot), manual: false, snapshot: null, outgoing_pending: false}, + action: {kind: "take_lease", id: $id, out: $swap.out, in: $swap.in, manual: false}, + wait: [], consume_request: false, last_swap: null, check: null} + end + else + {swap: null, action: null, wait: [], consume_request: false, last_swap: null, check: null} + end; + +def next_action($s): + identity($s) as $id + | if $id == null then starting($s) + else in_progress($s; $id) + {check: (if $s.request == null then null else {status: 1, reasons: ["a swap is in progress (\($id.id))"]} end)} + | .consume_request = ($s.request != null) + end; diff --git a/scripts/fleet-role.jq b/scripts/fleet-role.jq index 411febbd..7b1db03b 100644 --- a/scripts/fleet-role.jq +++ b/scripts/fleet-role.jq @@ -1,7 +1,7 @@ # The role controller's decision (see scripts/fleet-role.sh, which gathers the inputs and publishes the result): which hosts are fit to hold an etcd vote, and the swap the controller would make. Pure: nothing here reads the cluster or the clock. # # Inputs, as jq named arguments: -# - $config: the controller's settings (see scripts/fleet-role.sh for each), with `now` the time of the pass in epoch seconds and `expected_voters` the number of voters the cluster should hold at this moment: the target, or one more while a swap holds a new voter beside the old. +# - $config: the controller's settings (see scripts/fleet-role.sh for each), with `now` the time of the pass in epoch seconds, `expected_voters` the number of voters the cluster should hold at this moment (the target, or one more while a swap holds a new voter beside the old) and `ignored_voters`, optional, the voters a swap in progress is taking out, which the gates do not hold to their health (a voter stepped down to etcd-only, or one that is down), and whose connections the others may lack. # - $nodes_file: the output of `kubectl get nodes -o json`, as --slurpfile gives it (a one-element array). # - $probes_file: each node's probe entry (scripts/fleet-role-probe.sh), by node name, likewise. # - $previous_file: the status this program published on the last pass, or null on the first, likewise. @@ -20,9 +20,6 @@ def fit_rtt_divisor: 20; def wal_fsync_limit_ms: 10; def backend_commit_limit_ms: 25; -# k3s's learnerMaxStallTime (pkg/etcd/etcd.go): a learner that makes no progress for this long is removed, so a membership change has settled this long after it. -def learner_stall_seconds: 300; - # A probe report older than this many samples counts as no report, as the autoscaler treats a node pressure report older than three of its polls. def stale_after_samples: 3; @@ -70,9 +67,10 @@ $config as $c ready: ($ready.status == "True"), ready_since: ($ready.lastTransitionTime | fromdateiso8601? // null), labelled: $labelled, - voter: ($labelled and condition_true(voter_condition)), + voter: ($labelled and condition_true(voter_condition) and ((.metadata.annotations // {})[removed_annotation] // "") == ""), eligible: ($labels[$c.eligible_label] == "true"), pinned: ($labels[$c.pinned_label] == "true"), + removed: (((.metadata.annotations // {})[removed_annotation] // "") != ""), regrade_level: $labels[$c.level_label]}] as $all | ($all | map(select(.voter)) | sort_by(.name)) as $voters | ($voters | map(.name)) as $voter_names @@ -237,22 +235,38 @@ $config as $c | ([$hosts[] | select(.eligible)] | length) as $eligible_count | ($all | map(select(.voter))) as $voter_nodes -# Why no swap may start now whatever the hosts' fitness: the epic's conditions on the cluster as a whole. +# Why no swap may start now whatever the hosts' fitness: the epic's conditions on the cluster as a whole, as `gates`, which hold for a swap by command as well as by decision, and the cooldown after a membership change, which only a decision waits out. A voter a swap in progress is taking out (`ignored_voters`) is held to none of the gates on its own health, and its peers may lack their connection to it. It is still an etcd member until k3s removes it, whether or not its Node still says EtcdIsVoter (a server stepped down to etcd-only runs no kubelet), so the members etcd lists include it. +# How many members each voter's fresh probe report says etcd lists, by voter, null for a voter with no fresh report or no reading: a rollback is settled once every voter lists the target and no more. +| ($voter_names | map({key: ., value: (if ($probes[.].samples[-1].t // 0) >= ($now - $stale_seconds) then $probes[.].samples[-1].etcd.known_members else null end)}) | from_entries) as $known_members +| ($c.ignored_voters // []) as $ignored +| ($voter_names | map(select(. as $v | $ignored | index($v) | not))) as $judged +# A voter a swap is taking out that no longer says EtcdIsVoter has left etcd once every voter it does not ignore reports, fresh, that etcd lists as many members as there are voters: each voter is a member, so those members are the voters and nothing else. Its Node's etcd label and the absence of k3s's removed-node-name annotation say nothing then, since a server that is still running sets the label again and k3s clears the annotation when a server rejoins under a new name (pkg/etcd/metadata_controller.go and member_controller.go); counting it there would expect a member etcd does not have, for ever. +| (($judged | length) > 0 and ($judged | all(. as $v | $known_members[$v] == ($voters | length)))) as $etcd_lists_the_voters +| ([$all[] | select(.name as $n | $ignored | index($n)) | select(.labelled and (.removed | not))]) as $ignored_labelled +| (if $etcd_lists_the_voters then $ignored_labelled | map(select(.voter | not) | .name) else [] end) as $departed +| ($ignored_labelled | map(select(.name as $n | $departed | index($n) | not))) as $ignored_members +| ($ignored_members | map(select(.voter | not)) | length) as $held_members +| (($voters | length) + $held_members) as $members_expected +# The voters expected, as the controller was told, less a departed voter, which it counted as one of the swap's two. +| ($c.expected_voters - ($departed | length)) as $expected_voters | ([ - (if ($voters | length) != $c.expected_voters then "there are \($voters | length) voters, not the \(if $c.expected_voters == $c.target then "target \($c.target)" else "expected \($c.expected_voters) (the target \($c.target) and the new voter a swap holds)" end)" else empty end), - ($all[] | select(.labelled and (.voter | not)) | "node \(.name) is a member but not a voter: it carries the etcd label without \(voter_condition)=True"), - ($voter_nodes[] | select(.regrade_level != null) | "voter \(.name) is at regrade level \(.regrade_level), below full"), - ($voter_nodes[] | select(.ready | not) | "voter \(.name) is not Ready"), - ($voter_names[] as $v | $probes[$v].samples[-1] as $last | $last.etcd as $e + (if $members_expected != $expected_voters then "there are \($members_expected) voters, not the \(if $expected_voters == $c.target then "target \($c.target)" else "expected \($expected_voters) (the target \($c.target) and the new voter a swap holds)" end)" else empty end), + ($all[] | select(.name as $n | $ignored | index($n) | not) | select(.labelled and (.voter | not)) | if .removed then "node \(.name) is a member k3s has removed: it still carries the etcd label" else "node \(.name) is a member but not a voter: it carries the etcd label without \(voter_condition)=True" end), + ($voter_nodes[] | select(.name as $n | $ignored | index($n) | not) | select(.regrade_level != null) | "voter \(.name) is at regrade level \(.regrade_level), below full"), + ($voter_nodes[] | select(.name as $n | $ignored | index($n) | not) | select(.ready | not) | "voter \(.name) is not Ready"), + ($judged[] as $v | $probes[$v].samples[-1] as $last | $last.etcd as $e | if $last == null or $last.t < ($now - $stale_seconds) then "voter \($v) has no probe report in the last \($stale_seconds) s" elif $e == null then "etcd's state on voter \($v) is unknown" elif ($e.has_leader | not) then "voter \($v) reports no leader" elif $e.is_learner then "voter \($v) is a learner" - elif $e.known_members != ($voters | length) then "voter \($v) knows \($e.known_members) members, not the \($voters | length) voters: a learner or a member that has not started" - elif $e.active_peers != (($voters | length) - 1) then "voter \($v) is connected to \($e.active_peers) of its \(($voters | length) - 1) peers" - else empty end), + elif $e.known_members != $members_expected then "voter \($v) knows \($e.known_members) members, not the \($members_expected) voters: a learner or a member that has not started" + elif $e.active_peers > ($members_expected - 1) or $e.active_peers < ($members_expected - 1 - ($ignored_members | length)) then "voter \($v) is connected to \($e.active_peers) of its \($members_expected - 1) peers" + else empty end) + ]) as $gates +| ([ (if $cooldown_until != null and $now < $cooldown_until then "the membership changed at \($changed_at), and the cooldown lasts until \($cooldown_until)" else empty end) - ]) as $blocks + ]) as $cooldown_blocks +| ($gates + $cooldown_blocks) as $blocks # Which hosts' unfit verdicts each node accounts for, by node. | ([$hosts[] | .name as $h | .attributed_to[] | {node: ., host: $h}] | group_by(.node) | map({key: .[0].node, value: map(.host)}) | from_entries) as $attribution @@ -285,10 +299,11 @@ $config as $c unfit_rtt_ms: $unfit_rtt_ms, fit_rtt_ms: $fit_rtt_ms, wal_fsync_limit_ms: wal_fsync_limit_ms, backend_commit_limit_ms: backend_commit_limit_ms, window_samples: window_samples, sample_seconds: $c.sample_seconds, window_seconds: $window_seconds, cooldown_seconds: $cooldown_seconds, quota_bytes: $quota_bytes}, - membership: {voters: $voter_names, changed_at: $changed_at, cooldown_until: $cooldown_until}, + membership: {voters: $voter_names, known_members: $known_members, departed: $departed, changed_at: $changed_at, cooldown_until: $cooldown_until}, hosts: ($hosts | map({key: .name, value: del(.name, .rank, .m)}) | from_entries), attribution: $attribution, ranking: ($hosts | map(select(.verdict == "fit")) | sort_by(.rank) | map(.name)), + gates: $gates, blocks: $blocks, decision: $decision } diff --git a/scripts/fleet-role.sh b/scripts/fleet-role.sh index 2fa9af06..e5e72d6f 100755 --- a/scripts/fleet-role.sh +++ b/scripts/fleet-role.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash -# One pass of the role controller (see roles/github_runner_arc/templates/fleet-role.yaml.j2 and fleet-role/loop.sh, which runs this on an interval). It decides which of the hosts the owner marks eligible are fit to hold an etcd vote, from what scripts/fleet-role-probe.sh publishes, and which swap of an unfit voter for a fit standby it would make. It changes nothing in the cluster: its only write is its own status ConfigMap, which the heartbeat republishes. The decision is scripts/fleet-role.jq; this script gathers its inputs and publishes the result. +# The role controller (see roles/github_runner_arc/templates/fleet-role.yaml.j2 and fleet-role/loop.sh, which runs a pass of this on an interval). It decides which of the hosts the owner marks eligible are fit to hold an etcd vote, from what scripts/fleet-role-probe.sh publishes, and carries out a voter swap: an eligible standby becomes a voter and an unfit voter is removed and becomes an agent, as one swap under the regrade Lease. The decision is scripts/fleet-role.jq, the swap's state machine is scripts/fleet-role-swap.jq (which reads what step a swap is at from the Nodes, the actuators' reports and the Lease), and this script gathers their inputs, carries out the one action a pass takes, and publishes the result to its status ConfigMap, which the heartbeat republishes. The host-side work of a swap (the units, the role file, the datastore) is the role actuator's (roles/github_runner_cluster/files/role/fleet-role-actuator.sh), driven through the Node's desired-role label and swap annotations; everything on the API (the snapshot request, the cordon, the runner wait, the drain, the taint, the regrading level label, the removal of the member, the clean-up of the Node) is done here. # -# It runs unchanged with no host marked eligible, and then only reports the voters' fitness. +# With no arguments it runs one pass: it renews the regrade Lease if a swap holds it, takes at most one action and publishes the status. `fleet-role.sh swap OUT IN` asks for OUT, a voter, to be swapped for IN, an eligible standby, as a manual request that the next pass carries out; its exit status is 0 when the request is accepted (every gate holds now), 1 when it is refused (each reason is printed) and 2 for a usage error or a node that is unknown, not marked eligible or not the voter and the standby they are taken for. `fleet-role.sh status` prints the swap in progress and its step, the last swap and a pending request, from the published status. Run it as `kubectl -n github-runner-platform exec deploy/fleet-role -- /app/scripts/fleet-role.sh swap OUT IN`. +# +# A manual request honours every gate (the voters at full, Ready, etcd with a leader and no learner, the regrade Lease free) but not the fitness verdict or the cooldown, and never takes out a pinned voter or brings in a host that is not marked eligible. With FLEET_ROLE_DRY_RUN=true only a manual request is carried out and automatic decisions are reported; false carries out automatic decisions as well. A swap already in progress is driven to its end, or rolled back, whatever the setting, so an even count of voters is never left behind. The one thing a failed pass leaves is a swap at the step it was at: every action is repeated safely by the next pass. # # Environment (provided by the Deployment, see the template): -# - FLEET_ROLE_DRY_RUN: must be "true", the default. The controller has no way to change a role; the setting exists so that a Deployment that names it keeps working when it can. +# - FLEET_ROLE_DRY_RUN: true (the default) or false, as above. # - FLEET_ROLE_VOTERS: the number of voters the cluster is held at. Odd, at least 3, and at most FLEET_ROLE_MAX_SERVERS minus one, which leaves a swap one place of headroom (the new voter joins before the old one leaves). Required. # - FLEET_ROLE_MAX_SERVERS: the most servers the cluster may have (github_runner_cluster_max_servers). Required. # - FLEET_ROLE_ELECTION_TIMEOUT_MS: the cluster's etcd election timeout in milliseconds, k3s's own default of 1000 where the cluster configures none (the role works that out). Required. @@ -13,13 +15,23 @@ # - FLEET_ROLE_ETCD_QUOTA_BYTES: etcd's storage quota, used for a host's memory floor when no voter reports one; etcd's own default when unset. # - FLEET_ROLE_ELIGIBLE_LABEL, FLEET_ROLE_PINNED_LABEL: the Node labels, with the value true, that mark a host the controller may promote or demote and one it must never demote. Required. # - FLEET_ROLE_LEVEL_LABEL: the label regrading puts on a server below full, with the level as its value. Required. -# - FLEET_ROLE_STATE_NAMESPACE, FLEET_ROLE_PROBE_CONFIGMAP, FLEET_ROLE_STATUS_CONFIGMAP: where the probes publish and this controller keeps its status; both ConfigMaps are created by the role. +# - FLEET_ROLE_OPERATION_TIMEOUT_SECONDS: how long a drain, the wait for a snapshot and each step of an actuator may take (github_runner_cluster_regrade_operation_timeout_seconds, which the actuators bound their own steps by). Required. +# - FLEET_ROLE_RUNNER_WAIT_SECONDS: how long a planned change waits for the runner pods on a node to finish before it is abandoned or goes on (github_runner_cluster_native_runner_wait_timeout, which the role's own switch waits). Required. +# - FLEET_ROLE_LEASE_SECONDS: how long the regrade Lease lasts if the controller stops renewing it (github_runner_cluster_regrade_lease_duration_seconds). Required. +# - FLEET_ROLE_STATE_NAMESPACE, FLEET_ROLE_PROBE_CONFIGMAP, FLEET_ROLE_STATUS_CONFIGMAP, FLEET_ROLE_ACTUATOR_CONFIGMAP: where the probes publish, this controller keeps its status and the actuators report; the first three are created by the role, the actuators' by the cluster role. FLEET_ROLE_ACTUATOR_CONFIGMAP is required. +# - FLEET_REGRADE_CORE: the regrade core (default /app/fleet-regrade-core.sh), whose Lease functions this uses. set -euo pipefail # etcd's default storage quota, 2 GiB, in bytes (--quota-backend-bytes). readonly etcd_default_quota_bytes=2147483648 # etcd needs a majority, so fewer than three voters tolerate no failure. readonly min_voters=3 +# The label ARC puts on every ephemeral runner pod (plugins/filter/cluster_guard.py's RUNNER_POD_LABEL). +readonly runner_pod_selector=actions-ephemeral-runner=True +# The taint the regrade core puts on a server it has stepped down (github_runner_cluster_regrade_taint), applied to the outgoing voter first so nothing new lands on it. +readonly control_plane_taint=node-role.kubernetes.io/control-plane=true:NoSchedule +# The ConfigMap key a manual request is kept under. The controller's pass owns status.json and the request command owns this key, so the two never overwrite one another. +readonly request_key=request.json dry_run="${FLEET_ROLE_DRY_RUN:-true}" target="${FLEET_ROLE_VOTERS:?FLEET_ROLE_VOTERS must be set}" @@ -32,16 +44,20 @@ default_quota_bytes="${FLEET_ROLE_ETCD_QUOTA_BYTES:-$etcd_default_quota_bytes}" eligible_label="${FLEET_ROLE_ELIGIBLE_LABEL:?FLEET_ROLE_ELIGIBLE_LABEL must be set}" pinned_label="${FLEET_ROLE_PINNED_LABEL:?FLEET_ROLE_PINNED_LABEL must be set}" level_label="${FLEET_ROLE_LEVEL_LABEL:?FLEET_ROLE_LEVEL_LABEL must be set}" +operation_timeout="${FLEET_ROLE_OPERATION_TIMEOUT_SECONDS:?FLEET_ROLE_OPERATION_TIMEOUT_SECONDS must be set}" +runner_wait="${FLEET_ROLE_RUNNER_WAIT_SECONDS:?FLEET_ROLE_RUNNER_WAIT_SECONDS must be set}" +lease_seconds="${FLEET_ROLE_LEASE_SECONDS:?FLEET_ROLE_LEASE_SECONDS must be set}" namespace="${FLEET_ROLE_STATE_NAMESPACE:-github-runner-platform}" probe_configmap="${FLEET_ROLE_PROBE_CONFIGMAP:-fleet-role-probe}" status_configmap="${FLEET_ROLE_STATUS_CONFIGMAP:-fleet-role-status}" +actuator_configmap="${FLEET_ROLE_ACTUATOR_CONFIGMAP:?FLEET_ROLE_ACTUATOR_CONFIGMAP must be set}" modules="$(dirname "$0")" log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*"; } fail() { echo "$*" >&2; exit 1; } -[ "$dry_run" = true ] || fail "FLEET_ROLE_DRY_RUN must be true: this controller reports the swap it would make and has no way to make one (got '${dry_run}')" -for setting in target max_servers election_timeout_ms sample_seconds default_quota_bytes; do +[ "$dry_run" = true ] || [ "$dry_run" = false ] || fail "FLEET_ROLE_DRY_RUN must be true or false, not '${dry_run}'" +for setting in target max_servers election_timeout_ms sample_seconds default_quota_bytes operation_timeout runner_wait lease_seconds; do [[ "${!setting}" =~ ^[1-9][0-9]*$ ]] || fail "${setting} must be a whole number, at least 1, not '${!setting}'" done for setting in pressure_threshold available_floor_percent; do @@ -51,37 +67,365 @@ done [ "$target" -ge "$min_voters" ] || fail "FLEET_ROLE_VOTERS must be at least ${min_voters}, not ${target}" [ "$target" -le $((max_servers - 1)) ] || fail "FLEET_ROLE_VOTERS (${target}) must leave a swap one place of headroom below FLEET_ROLE_MAX_SERVERS (${max_servers})" -now="$(date +%s)" -# The inputs go to jq in files, never as arguments: a probe entry holds a window of samples, the entries together can fill a ConfigMap, and one argument is limited to a fraction of the argument space (128 KiB on Linux), which a fleet's nodes and probes pass. +# The regrade core's Lease functions, run as the controller: its kubectl_ is the pod's own kubectl (KUBECONFIG_PATH empty leaves the in-cluster configuration to kubectl), and the holder is named in each call. +export KUBECTL=kubectl KUBECONFIG_PATH='' LEASE_DURATION_SECONDS=$lease_seconds NODE_NAME=fleet-role FLEET_REGRADE_ENV=/nonexistent +# shellcheck source=/dev/null +. "${FLEET_REGRADE_CORE:-/app/fleet-regrade-core.sh}" + +# The names the controller and the actuators share, from the one place that defines them. +read -r desired_role_label swap_annotation step_annotation join_node_annotation snapshot_annotation member_gone_annotation lease_swap_id_annotation lease_swap_nodes_annotation \ + etcd_label control_plane_label master_label etcd_annotation_prefix remove_annotation lease_holder < <(jq -nr -L "$modules" \ + 'include "fleet-role-swap"; interface | [.desired_role_label, .swap_annotation, .step_annotation, .join_node_annotation, .snapshot_annotation, .member_gone_annotation, .lease_swap_id_annotation, .lease_swap_nodes_annotation, .etcd_label, .control_plane_label, .master_label, .etcd_annotation_prefix, .remove_annotation, .lease_holder] | join(" ")') + work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT -kubectl get nodes -o json > "$work/nodes.json" -# Each node's entry is a JSON document in its own key. -kubectl get configmap "$probe_configmap" -n "$namespace" -o json | jq -c '(.data // {}) | map_values(fromjson)' > "$work/probes.json" -kubectl get configmap "$status_configmap" -n "$namespace" -o json | jq -c '(.data["status.json"] // "") | if . == "" then null else fromjson end' > "$work/previous.json" - -# A pass holds the cluster at the target: this controller changes no membership, so no swap holds a new voter beside the old and the expected count is the target. -config="$(jq -nc \ - --argjson now "$now" --argjson target "$target" --argjson max_servers "$max_servers" \ - --argjson election_timeout_ms "$election_timeout_ms" --argjson sample_seconds "$sample_seconds" \ - --argjson pressure_threshold "$pressure_threshold" --argjson available_floor_percent "$available_floor_percent" \ - --argjson default_quota_bytes "$default_quota_bytes" \ - --arg eligible_label "$eligible_label" --arg pinned_label "$pinned_label" --arg level_label "$level_label" \ - '{now: $now, target: $target, expected_voters: $target, max_servers: $max_servers, election_timeout_ms: $election_timeout_ms, sample_seconds: $sample_seconds, - pressure_threshold: $pressure_threshold, available_floor_percent: $available_floor_percent, default_quota_bytes: $default_quota_bytes, - voter_label: "node-role.kubernetes.io/etcd", eligible_label: $eligible_label, pinned_label: $pinned_label, level_label: $level_label}')" - -result="$(jq -n -L "$modules" --argjson config "$config" --slurpfile nodes_file "$work/nodes.json" --slurpfile probes_file "$work/probes.json" --slurpfile previous_file "$work/previous.json" -f "$modules/fleet-role.jq")" -status="$(jq -c --arg last_run "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson dry_run true '. + {last_run: $last_run, dry_run: $dry_run}' <<< "$result")" - -# The patch goes in a file: it is read back as a whole, and an argument has a size limit. -jq -n --arg status "$status" '{data: {"status.json": $status}}' > "$work/patch.json" -kubectl patch configmap "$status_configmap" -n "$namespace" --type merge --patch-file "$work/patch.json" > /dev/null - -# What each host's verdict rests on, then the decision, in the log. -while IFS= read -r line; do log "$line"; done < <(jq -r ' - .hosts | to_entries[] - | "\(.key) (\(.value.role)\(if .value.pinned then ", pinned" else "" end)\(if .value.eligible then "" else ", not eligible" end), link \(.value.link)): \(.value.verdict), \(.value.reasons | if length == 0 then "all readings within their limits" else join("; ") end)\(if (.value.attributed_to | length) > 0 then " (attributed to \(.value.attributed_to | join(" and ")))" else "" end)\(.value.unknown_signals | map("; \(.) unknown") | join(""))"' <<< "$result") -while IFS= read -r line; do log "$line"; done < <(jq -r ' - if .decision.action == "swap" then "DRY RUN: would swap out \(.decision.swap.out) for \(.decision.swap.in): \(.decision.because | join("; "))" - else "DRY RUN: would not act: \(.decision.because | join("; "))\(if .decision.wanted == null and (.blocks | length) > 0 then " (also: \(.blocks | join("; ")))" else "" end)" end' <<< "$result") + +# One kubectl call whose output is not wanted. The actions chain these with `|| return 1`, since errexit is off inside a function that is called as the condition of an `if`. +kube() { kubectl_ "$@" > /dev/null; } + +# --- Reading the cluster. The inputs go to jq in files, never as arguments: a probe entry holds a window of samples, the entries together can fill a ConfigMap, and one argument is limited to a fraction of the argument space (128 KiB on Linux), which a fleet's nodes and probes pass. --- + +# The ConfigMap's data key $2 as JSON, null when the ConfigMap or the key has none. +read_status_key() { + kubectl_ get configmap "$status_configmap" --namespace "$namespace" --output json | jq -c --arg key "$1" '(.data[$key] // "") | if . == "" then null else fromjson end' +} + +# The regrade Lease as the API holds it, or null when there is none; fails for any other reason it cannot be read. +read_lease() { + local output + if output=$(kubectl_ get lease "$LEASE_NAME" --namespace "$LEASE_NAMESPACE" --output json 2>&1); then + echo "$output" + else + case $output in + *"(NotFound)"*) echo null ;; + *) echo "the regrade Lease cannot be read: ${output}" >&2; return 1 ;; + esac + fi +} + +# The Lease as the swap program reads it: {state, holder, swap_id, swap_nodes}, state as lease_state says for the controller. +lease_view() { + local lease=$1 state=free + if [ "$lease" != null ]; then state=$(lease_state "$lease" "$lease_holder" "$now"); fi + jq -c --arg state "$state" --arg id_key "$lease_swap_id_annotation" --arg nodes_key "$lease_swap_nodes_annotation" \ + '{state: $state, holder: (.spec.holderIdentity // ""), swap_id: (.metadata.annotations[$id_key] // ""), swap_nodes: (.metadata.annotations[$nodes_key] // "")}' <<< "$lease" +} + +# The settings the programs read, as one JSON object. +make_config() { + jq -nc \ + --argjson now "$now" --argjson target "$target" --argjson max_servers "$max_servers" \ + --argjson election_timeout_ms "$election_timeout_ms" --argjson sample_seconds "$sample_seconds" \ + --argjson pressure_threshold "$pressure_threshold" --argjson available_floor_percent "$available_floor_percent" \ + --argjson default_quota_bytes "$default_quota_bytes" --argjson operation_timeout "$operation_timeout" --argjson runner_wait "$runner_wait" \ + --argjson dry_run "$dry_run" --arg holder "$lease_holder" \ + --arg eligible_label "$eligible_label" --arg pinned_label "$pinned_label" --arg level_label "$level_label" \ + '{now: $now, target: $target, max_servers: $max_servers, election_timeout_ms: $election_timeout_ms, sample_seconds: $sample_seconds, + pressure_threshold: $pressure_threshold, available_floor_percent: $available_floor_percent, default_quota_bytes: $default_quota_bytes, + operation_timeout: $operation_timeout, runner_wait: $runner_wait, dry_run: $dry_run, holder: $holder, + voter_label: "node-role.kubernetes.io/etcd", eligible_label: $eligible_label, pinned_label: $pinned_label, level_label: $level_label}' +} + +# Runs a program of the swap module over the state object in the file $1. +swap_program() { + jq -c -n -L "$modules" --slurpfile s "$1" "include \"fleet-role-swap\"; $2" +} + +# Reads everything a pass or a request needs into $work: the Nodes, the probes, the status and any request, the Lease, then, with a swap known, what its steps read. Sets now, lease_json, swap_json, config, result and the files the swap program reads, and runs the decision. +read_state() { + now="$(date +%s)" + kubectl_ get nodes --output json > "$work/nodes.json" + # Each node's entry is a JSON document in its own key. + kubectl_ get configmap "$probe_configmap" --namespace "$namespace" --output json | jq -c '(.data // {}) | map_values(fromjson)' > "$work/probes.json" + read_status_key status.json > "$work/previous.json" + read_status_key "$request_key" > "$work/request.json" + lease_json=$(read_lease) + lease_view "$lease_json" > "$work/lease.json" + config=$(make_config) + jq -c --argjson config "$config" --slurpfile nodes "$work/nodes.json" --slurpfile lease "$work/lease.json" --slurpfile previous "$work/previous.json" \ + '{config: $config, nodes: $nodes[0].items, lease: $lease[0], previous: $previous[0]}' <<< null > "$work/context-state.json" + # shellcheck disable=SC2016 # The jq program's own variable. + swap_json=$(swap_program "$work/context-state.json" 'context($s[0])') + decide + gather_swap_inputs +} + +# The decision program over the Nodes and probes, told how many voters to expect and which voter a swap is taking out. +decide() { + local expected ignored + expected=$(jq -r '.expected_voters' <<< "$swap_json") + ignored=$(jq -c '.ignored_voters' <<< "$swap_json") + decision_config=$(jq -c --argjson expected "$expected" --argjson ignored "$ignored" '. + {expected_voters: $expected, ignored_voters: $ignored}' <<< "$config") + result="$(jq -n -L "$modules" --argjson config "$decision_config" --slurpfile nodes_file "$work/nodes.json" --slurpfile probes_file "$work/probes.json" --slurpfile previous_file "$work/previous.json" -f "$modules/fleet-role.jq")" + echo "$result" > "$work/result.json" +} + +# With a swap known: the actuators' reports, the snapshot files, each swap node's kubelet Lease and the runner pods on it, and the state file the plan reads. +gather_swap_inputs() { + local out in + out=$(jq -r '.swap.out // ""' <<< "$swap_json") + in=$(jq -r '.swap.in // ""' <<< "$swap_json") + echo '{}' > "$work/actuators.json" + echo null > "$work/snapshots.json" + echo '{}' > "$work/node-leases.json" + echo '{}' > "$work/runner-pods.json" + if [ -n "$in" ]; then + read_actuators > "$work/actuators.json" + # A cluster that does not publish snapshot files leaves this null, which the plan says when it waits for a snapshot. + if ! kubectl_ get etcdsnapshotfiles.k3s.cattle.io --output json | jq -c '.items' > "$work/snapshots.json" 2> /dev/null; then echo null > "$work/snapshots.json"; fi + local node renewed pods + for node in "$out" "$in"; do + renewed=$(kubectl_ get lease "$node" --namespace kube-node-lease --output 'jsonpath={.spec.renewTime}' 2> /dev/null || true) + jq -c --arg node "$node" --arg renewed "$renewed" '. + {($node): $renewed}' "$work/node-leases.json" > "$work/node-leases.new" && mv "$work/node-leases.new" "$work/node-leases.json" + pods=$(kubectl_ get pods --all-namespaces --selector "$runner_pod_selector" --field-selector "spec.nodeName=${node},status.phase!=Succeeded,status.phase!=Failed" --output json | jq '.items | length') + jq -c --arg node "$node" --argjson pods "$pods" '. + {($node): $pods}' "$work/runner-pods.json" > "$work/runner-pods.new" && mv "$work/runner-pods.new" "$work/runner-pods.json" + done + fi +} + +# The actuators' reports by node, parsed, {} when their ConfigMap does not exist yet. +read_actuators() { + local output + if output=$(kubectl_ get configmap "$actuator_configmap" --namespace "$namespace" --output json 2>&1); then + jq -c '(.data // {}) | map_values(fromjson)' <<< "$output" + else + case $output in + *"(NotFound)"*) echo '{}' ;; + *) echo "the actuators' reports cannot be read: ${output}" >&2; return 1 ;; + esac + fi +} + +# The plan: what the swap program names for this pass, over the state read. $1 is a request as JSON, or null for the one pending. +plan_for() { + local request=${1:-} + if [ -z "$request" ]; then request=$(cat "$work/request.json"); fi + jq -c --argjson request "$request" --slurpfile result "$work/result.json" --slurpfile context "$work/context-state.json" \ + --slurpfile actuators "$work/actuators.json" --slurpfile snapshots "$work/snapshots.json" --slurpfile node_leases "$work/node-leases.json" --slurpfile runner_pods "$work/runner-pods.json" \ + '$context[0] + {actuators: $actuators[0], snapshots: $snapshots[0], node_leases: $node_leases[0], runner_pods: $runner_pods[0], decision: $result[0], request: $request}' <<< null > "$work/plan-state.json" + # shellcheck disable=SC2016 # The jq program's own variable. + swap_program "$work/plan-state.json" 'next_action($s[0])' +} + +# --- Acting. One function per kind of action the plan names; each is repeated safely, so a pass that is cut short is finished by the next. --- + +# Takes the Lease for the swap, with the swap's id and nodes on it, where the actuators and node recovery read them. +act_take_lease() { + local id=$1 out=$2 in=$3 annotations + annotations=$(jq -nc --arg id_key "$lease_swap_id_annotation" --arg id "$id" --arg nodes_key "$lease_swap_nodes_annotation" --arg nodes "${out},${in}" '{($id_key): $id, ($nodes_key): $nodes}') || return 1 + acquire_lease "$lease_holder" "$annotations" +} + +act_request_snapshot() { + kube annotate --overwrite node "$1" "${snapshot_annotation}=$2" +} + +act_cordon() { + kube cordon "$1" +} + +act_uncordon() { + kube uncordon "$1" +} + +# Drains the candidate, then asks its actuator to promote it, joining the voter named: the label, then the annotations, since the actuator acts on the annotations and refuses a label that disagrees. +act_promote_in() { + local node=$1 id=$2 join_node=$3 + kube drain "$node" --ignore-daemonsets --delete-emptydir-data --timeout="${operation_timeout}s" || return 1 + kube label --overwrite node "$node" "${desired_role_label}=server" || return 1 + kube annotate --overwrite node "$node" "${swap_annotation}=${id}" "${step_annotation}=promote" "${join_node_annotation}=${join_node}" +} + +# Asks the actuator of a candidate that did not become a voter to switch the host back, after asking k3s to remove the learner that is still listed (which never touches quorum). +act_begin_rollback() { + local node=$1 id=$2 remove=$3 + if [ "$remove" = true ]; then kube annotate --overwrite node "$node" "${remove_annotation}=true" || return 1; fi + kube label --overwrite node "$node" "${desired_role_label}=agent" || return 1 + kube annotate --overwrite node "$node" "${swap_annotation}=${id}" "${step_annotation}=rollback" +} + +# Taints and cordons the outgoing voter, so nothing new lands on it while its runner pods finish. +act_cordon_out() { + kube taint nodes "$1" "$control_plane_taint" --overwrite || return 1 + kube cordon "$1" +} + +# Empties the outgoing voter (a node that is down cannot be drained), marks it for regrading's etcd-only level, which its actuator enters, and asks the actuator to step it down. +act_step_down_out() { + local node=$1 id=$2 drain=$3 + act_cordon_out "$node" || return 1 + if [ "$drain" = true ]; then kube drain "$node" --ignore-daemonsets --delete-emptydir-data --force --timeout="${operation_timeout}s" || return 1; fi + kube label --overwrite node "$node" "${level_label}=etcd-only" "${desired_role_label}=agent" || return 1 + kube annotate --overwrite node "$node" "${swap_annotation}=${id}" "${step_annotation}=step-down" +} + +# Asks k3s's member controller, on another server, to remove the outgoing voter's etcd member. +act_annotate_remove() { + kube annotate --overwrite node "$1" "${remove_annotation}=true" +} + +# Tells the outgoing voter's actuator, under the swap $2, that etcd no longer lists its member, as every remaining voter reports. +act_mark_member_gone() { + kube annotate --overwrite node "$1" "${member_gone_annotation}=$2" +} + +# Removes what k3s and regrading left on a node that is no longer a server (the role labels, the etcd annotations, the level label and the taint), whichever of them it carries, and uncordons it when asked. The desired-role label stays, as the record of the role the host runs, and the swap annotations stay until the swap ends. +act_clean_node() { + local node=$1 uncordon=$2 json labels annotations taint + json=$(kubectl_ get node "$node" --output json) || return 1 + labels=$(jq -r --arg etcd "$etcd_label" --arg control_plane "$control_plane_label" --arg master "$master_label" --arg level "$level_label" \ + '(.metadata.labels // {}) as $present | [$etcd, $control_plane, $master, $level][] | select($present[.] != null)' <<< "$json") || return 1 + annotations=$(jq -r --arg prefix "$etcd_annotation_prefix" '(.metadata.annotations // {}) | keys[] | select(startswith($prefix))' <<< "$json") || return 1 + taint=$(jq -r --arg key "${control_plane_taint%%=*}" '[(.spec.taints // [])[] | select(.key == $key)] | length' <<< "$json") || return 1 + local label_args=() annotation_args=() key + while IFS= read -r key; do if [ -n "$key" ]; then label_args+=("${key}-"); fi; done <<< "$labels" + while IFS= read -r key; do if [ -n "$key" ]; then annotation_args+=("${key}-"); fi; done <<< "$annotations" + if [ "${#label_args[@]}" -gt 0 ]; then kube label node "$node" "${label_args[@]}" || return 1; fi + if [ "${#annotation_args[@]}" -gt 0 ]; then kube annotate node "$node" "${annotation_args[@]}" || return 1; fi + if [ "$taint" -gt 0 ]; then kube taint nodes "$node" "${control_plane_taint%%=*}:NoSchedule-" || return 1; fi + if [ "$uncordon" = true ]; then act_uncordon "$node"; fi +} + +# Ends the swap $1: the nodes $2 (a JSON array) are uncordoned, the swap annotations come off every node that carries them, and the Lease is given back last, so a pass cut short still holds it and finishes the job. The outcome is recorded by the caller. +act_finalise() { + local id=$1 uncordon=$2 node nodes + for node in $(jq -r '.[]' <<< "$uncordon"); do act_uncordon "$node" || return 1; done + nodes=$(jq -r --arg id "$id" --arg swap "$swap_annotation" --arg snapshot "$snapshot_annotation" \ + '.items[] | select((.metadata.annotations[$swap] == $id) or (.metadata.annotations[$snapshot] == $id)) | .metadata.name' "$work/nodes.json") || return 1 + for node in $nodes; do + kube annotate node "$node" "${swap_annotation}-" "${step_annotation}-" "${join_node_annotation}-" "${snapshot_annotation}-" "${member_gone_annotation}-" || return 1 + done + release_lease "$lease_holder" +} + +# Carries out the action $1 of the plan $2, and returns its status. +carry_out() { + local action=$1 plan=$2 kind + kind=$(jq -r '.kind' <<< "$action") + case $kind in + take_lease) act_take_lease "$(jq -r '.id' <<< "$action")" "$(jq -r '.out' <<< "$action")" "$(jq -r '.in' <<< "$action")" ;; + request_snapshot) act_request_snapshot "$(jq -r '.node' <<< "$action")" "$(jq -r '.id' <<< "$action")" ;; + cordon) act_cordon "$(jq -r '.node' <<< "$action")" ;; + uncordon) act_uncordon "$(jq -r '.node' <<< "$action")" ;; + promote_in) act_promote_in "$(jq -r '.node' <<< "$action")" "$(jq -r '.id' <<< "$action")" "$(jq -r '.join_node' <<< "$action")" ;; + begin_rollback) act_begin_rollback "$(jq -r '.node' <<< "$action")" "$(jq -r '.id' <<< "$action")" "$(jq -r '.remove' <<< "$action")" ;; + cordon_out) act_cordon_out "$(jq -r '.node' <<< "$action")" ;; + step_down_out) act_step_down_out "$(jq -r '.node' <<< "$action")" "$(jq -r '.id' <<< "$action")" "$(jq -r '.drain' <<< "$action")" ;; + annotate_remove) act_annotate_remove "$(jq -r '.node' <<< "$action")" ;; + mark_member_gone) act_mark_member_gone "$(jq -r '.node' <<< "$action")" "$(jq -r '.id' <<< "$action")" ;; + clean_node) act_clean_node "$(jq -r '.node' <<< "$action")" "$(jq -r '.uncordon' <<< "$action")" ;; + finalise) act_finalise "$(jq -r '.swap.id' <<< "$plan")" "$(jq -c '.uncordon' <<< "$action")" ;; + *) fail "the plan names an action this controller does not know: ${kind}" ;; + esac +} + +# --- The commands. --- + +# One pass. +run_pass() { + local plan action action_status=0 swap last_swap request had_swap line + # A swap holds the Lease for as long as it runs and each pass renews it, so it lasts only as long as a controller that is working. A renewal that fails is not fatal: the Lease is read again below, as the state it is really in. + lease_json=$(read_lease) + if [ "$lease_json" != null ] && [ "$(jq -r '.spec.holderIdentity // ""' <<< "$lease_json")" = "$lease_holder" ]; then + renew_lease "$lease_holder" > /dev/null || log "the regrade Lease could not be renewed; it is read again" + fi + read_state + plan=$(plan_for) + had_swap=$(jq -r '.swap != null' <<< "$swap_json") + action=$(jq -c '.action' <<< "$plan") + swap=$(jq -c '.swap' <<< "$plan") + last_swap=$(jq -c '.last_swap' <<< "$plan") + request=$(cat "$work/request.json") + + # What each host's verdict rests on, then the decision, in the log. + while IFS= read -r line; do log "$line"; done < <(jq -r ' + .hosts | to_entries[] + | "\(.key) (\(.value.role)\(if .value.pinned then ", pinned" else "" end)\(if .value.eligible then "" else ", not eligible" end), link \(.value.link)): \(.value.verdict), \(.value.reasons | if length == 0 then "all readings within their limits" else join("; ") end)\(if (.value.attributed_to | length) > 0 then " (attributed to \(.value.attributed_to | join(" and ")))" else "" end)\(.value.unknown_signals | map("; \(.) unknown") | join(""))"' <<< "$result") + if [ "$had_swap" = false ]; then + if [ "$dry_run" = true ]; then + while IFS= read -r line; do log "$line"; done < <(jq -r ' + if .decision.action == "swap" then "DRY RUN: would swap out \(.decision.swap.out) for \(.decision.swap.in): \(.decision.because | join("; "))" + else "DRY RUN: would not act: \(.decision.because | join("; "))\(if .decision.wanted == null and (.blocks | length) > 0 then " (also: \(.blocks | join("; ")))" else "" end)" end' <<< "$result") + else + while IFS= read -r line; do log "$line"; done < <(jq -r ' + if .decision.action == "swap" then "decision: swap out \(.decision.swap.out) for \(.decision.swap.in): \(.decision.because | join("; "))" + else "decision: no swap: \(.decision.because | join("; "))\(if .decision.wanted == null and (.blocks | length) > 0 then " (also: \(.blocks | join("; ")))" else "" end)" end' <<< "$result") + fi + fi + while IFS= read -r line; do log "$line"; done < <(jq -r '.wait[] | "swap: \(.)"' <<< "$plan") + + if [ "$action" != null ]; then + log "swap$(jq -r 'if .swap != null then " \(.swap.id) (\(.swap.step))" else "" end' <<< "$plan"): $(jq -r '.kind + " " + (del(.kind) | to_entries | map("\(.key)=\(.value)") | join(" "))' <<< "$action")" + if ! carry_out "$action" "$plan"; then + action_status=1 + log "the action failed; the next pass repeats it" + # A swap that was not yet under way is not begun by an action that failed. + if [ "$had_swap" = false ]; then swap=null; fi + last_swap=null + elif [ "$(jq -r '.kind' <<< "$action")" = finalise ]; then + swap=null + log "swap ended: $(jq -c '.last_swap' <<< "$plan")" + fi + fi + if [ "$(jq -r '.consume_request' <<< "$plan")" = true ] && [ "$action_status" -eq 0 ]; then + jq -n --arg key "$request_key" '{data: {($key): null}}' > "$work/patch.json" + kubectl_ patch configmap "$status_configmap" --namespace "$namespace" --type merge --patch-file "$work/patch.json" > /dev/null + request=null + fi + + publish "$swap" "$last_swap" "$request" "$(jq -c '.wait' <<< "$plan")" + return "$action_status" +} + +# Publishes the status: the decision program's result, with the swap in progress and what it waits for, the last swap that ended (the one before it while none has since) and the pending request. The patch goes in a file: it is read back as a whole, and an argument has a size limit. +publish() { + local swap=$1 last_swap=$2 request=$3 waiting=$4 status + status="$(jq -c --arg last_run "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson dry_run "$dry_run" --argjson swap "$swap" --argjson last_swap "$last_swap" --argjson request "$request" --argjson waiting "$waiting" --slurpfile previous "$work/previous.json" \ + '. + {last_run: $last_run, dry_run: $dry_run, swap: $swap, waiting: $waiting, last_swap: ($last_swap // $previous[0].last_swap // null), request: $request}' <<< "$result")" + jq -n --arg status "$status" '{data: {"status.json": $status}}' > "$work/status-patch.json" + kubectl_ patch configmap "$status_configmap" --namespace "$namespace" --type merge --patch-file "$work/status-patch.json" > /dev/null +} + +# Asks for the swap of $1 for $2. +request_swap() { + local out=$1 in=$2 plan check status + read_state + plan=$(plan_for "$(jq -nc --arg out "$out" --arg in "$in" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" '{out: $out, in: $in, at: $at}')") + check=$(jq -c '.check' <<< "$plan") + status=$(jq -r '.status' <<< "$check") + if [ "$status" -eq 0 ]; then + jq -n --arg key "$request_key" --arg request "$(jq -nc --arg out "$out" --arg in "$in" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" '{out: $out, in: $in, at: $at}')" '{data: {($key): $request}}' > "$work/patch.json" + kubectl_ patch configmap "$status_configmap" --namespace "$namespace" --type merge --patch-file "$work/patch.json" > /dev/null + echo "accepted: ${out} will be swapped out for ${in} by the next pass" + return 0 + fi + echo "refused:" >&2 + jq -r '.reasons[] | " \(.)"' <<< "$check" >&2 + return "$status" +} + +# Prints the swap in progress and its step, the last swap and the request pending. +print_status() { + local status + status=$(read_status_key status.json) + if [ "$status" = null ]; then echo "no status has been published yet"; return 0; fi + jq -r ' + (if .swap == null then "no swap in progress" + else "swap \(.swap.id): \(.swap.out) out, \(.swap.in) in, step \(.swap.step) since \(.swap.step_since | todate), deadline \(.swap.deadline | todate)\(if .swap.manual then ", by request" else "" end)\(if .swap.outgoing_pending then ", the outgoing host has not come back as an agent" else "" end)" end), + (if .last_swap == null then "no swap has ended yet" else "last swap: \(.last_swap.outcome) (\(.last_swap.reasons | join("; ")))" end), + (if .request == null then "no request pending" else "request pending: \(.request.out) out, \(.request.in) in" end), + "dry run: \(.dry_run)"' <<< "$status" +} + +case ${1:-} in + "") [ $# -eq 0 ] || fail "usage: fleet-role.sh [swap OUT IN | status]"; run_pass ;; + swap) + [ $# -eq 3 ] || { echo "usage: fleet-role.sh swap OUT IN" >&2; exit 2; } + request_swap "$2" "$3" + ;; + status) + [ $# -eq 1 ] || { echo "usage: fleet-role.sh status" >&2; exit 2; } + print_status + ;; + *) echo "usage: fleet-role.sh [swap OUT IN | status]" >&2; exit 2 ;; +esac diff --git a/tests/fleet_role/image-smoke.sh b/tests/fleet_role/image-smoke.sh index bfc425f5..2bc26f7c 100755 --- a/tests/fleet_role/image-smoke.sh +++ b/tests/fleet_role/image-smoke.sh @@ -14,7 +14,7 @@ docker run --rm --entrypoint bash "$image" -c ' set -euo pipefail for tool in jq kubectl curl fio; do command -v "$tool" > /dev/null; done test -x /app/loop.sh -a -x /app/scripts/fleet-role.sh -a -x /app/scripts/fleet-role-probe.sh - test -f /app/fleet-regrade-core.sh -a -f /app/node-link.sh -a -f /app/scripts/fleet-role.jq -a -f /app/scripts/fleet-role-stats.jq' + test -f /app/fleet-regrade-core.sh -a -f /app/node-link.sh -a -f /app/scripts/fleet-role.jq -a -f /app/scripts/fleet-role-swap.jq -a -f /app/scripts/fleet-role-stats.jq' docker run --rm --entrypoint jq "$image" -en -L /app/scripts 'include "fleet-role-stats"; {window: window_samples, tail: tail_percentile} | select(.window == 100 and .tail == 99)' > /dev/null @@ -24,6 +24,9 @@ chmod 644 "$work/meminfo" "$work/pressure" docker run --rm -v "$work/meminfo:/meminfo:ro" -v "$work/pressure:/pressure:ro" -e MEMINFO_FILE=/meminfo -e PSI_FILE=/pressure \ --entrypoint /app/scripts/fleet-role-probe.sh "$image" check memory | jq -e '.psi_avg300 == 1.5 and .mem_available_kb == 400' > /dev/null +# The swap module compiles under the image's jq and holds the Lease under the name the actuators expect. +docker run --rm --entrypoint jq "$image" -en -L /app/scripts 'include "fleet-role-swap"; interface | select(.lease_holder == "fleet-role" and .desired_role_label == "github-runner.exadev.com/desired-role")' > /dev/null + # With no file named, the probe reads the pod's own /proc/pressure/memory, which is the node's, so the DaemonSet mounts nothing for it: the container sees pressure exactly when this host's kernel has it (a mounted /proc/pressure would stop a pod on a kernel without it from starting) and the reading is then absent, never an error. host_has_psi=false [ -r /proc/pressure/memory ] && host_has_psi=true @@ -33,13 +36,15 @@ docker run --rm --entrypoint /app/scripts/fleet-role-probe.sh "$image" check mem # The link reader the image carries reads the container's own interface: a veth, whose link it cannot see. docker run --rm --entrypoint /app/scripts/fleet-role-probe.sh "$image" check link | jq -e '.link == "unknown" and (.interface | length > 0)' > /dev/null -# A kubectl that answers an empty cluster and keeps the patch it is given, in place of the image's own on the PATH. +# A kubectl that answers an empty cluster (no Lease, no actuator reports) and keeps the patch it is given, in place of the image's own on the PATH. The controller reaches it through the regrade core's kubectl_, which passes --kubeconfig first. mkdir "$work/out" chmod 777 "$work/out" cat > "$work/kubectl" << 'STUB' #!/bin/sh +[ "$1" = --kubeconfig ] && shift 2 case "$1 $2" in "get nodes") echo '{"items":[]}' ;; + "get lease") echo 'Error from server (NotFound): leases.coordination.k8s.io "fleet-regrade" not found' >&2; exit 1 ;; "get configmap") case "$3" in fleet-role-probe) echo '{"data":{}}' ;; *) echo '{"data":{"status.json":""}}' ;; esac ;; "patch configmap") previous='' @@ -55,6 +60,7 @@ output="$(docker run --rm -v "$work/kubectl:/usr/local/bin/kubectl:ro" -v "$work -e FLEET_ROLE_DRY_RUN=true -e FLEET_ROLE_VOTERS=3 -e FLEET_ROLE_MAX_SERVERS=5 -e FLEET_ROLE_ELECTION_TIMEOUT_MS=1000 -e FLEET_ROLE_SAMPLE_SECONDS=15 \ -e FLEET_ROLE_PRESSURE_THRESHOLD=10 -e FLEET_ROLE_AVAILABLE_FLOOR_PERCENT=15 -e FLEET_ROLE_ELIGIBLE_LABEL=example.com/eligible \ -e FLEET_ROLE_PINNED_LABEL=example.com/pinned -e FLEET_ROLE_LEVEL_LABEL=example.com/level \ + -e FLEET_ROLE_OPERATION_TIMEOUT_SECONDS=300 -e FLEET_ROLE_RUNNER_WAIT_SECONDS=600 -e FLEET_ROLE_LEASE_SECONDS=900 -e FLEET_ROLE_ACTUATOR_CONFIGMAP=fleet-role-actuators \ --entrypoint /app/scripts/fleet-role.sh "$image")" grep -q 'no host is marked eligible' <<< "$output" jq -e '.data["status.json"] | fromjson | .dry_run == true and .decision.action == "none"' "$work/out/patch.json" > /dev/null diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 46c11c3d..d9b9c571 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). native_server three Docker servers and a native host (a privileged systemd container, tests/mesh/native/Dockerfile, managed over Ansible's docker connection) that joins as an agent, is then made a server by the inventory and promoted in place: it must join etcd as a learner, be promoted to a voter with its mesh address as its peer URL, and keep the cluster writable throughout, and a rerun must not restart it. role_controller four hosts (three servers and a standby agent, with etcd's data on memory so the machine's disk does not decide): the role controller's probe and controller run in the cluster from the image built from this checkout, with the manifests the ARC role renders; every host is reported fit from a full window of real samples, with the link the cluster role labelled its node with, a voter made slow with tc netem on its mesh interface is reported unfit with the swap named and every unfit verdict attributed to it, a paused voter makes the controller report that it would not act, and the controller changes nothing (not in the default set; needs jq, and takes a window of 100 samples before it can judge). role_actuator three Docker servers and a native standby, every host eligible and the role controller on, with the test acting as the controller (holding the regrade Lease and writing the Node's desired-role label and swap annotations by hand): the native host's role actuator promotes it (a learner, then a voter, with writes succeeding throughout), steps it down (its etcd serving at etcd-only with no pod's shim left), and, once its member is removed through k3s's annotation, finishes it as an agent with the same node password and Tailscale address and its datastore moved aside, the two k3s units never running together. rejoin three Docker servers whose hosts the inventory marks eligible, so the role mounts each one's role directory: the bootstrap server's etcd member is removed through another server, and restarted on its tombstone it idles instead of joining again; with its datastore moved aside and no server to join it refuses to start rather than bootstrap a second cluster; given a join URL naming another server in its role directory it joins the existing cluster as a learner and is promoted, with the same cluster ID before and after; and a role file naming the agent leaves its server container idle with no k3s. compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). host_network each node runs in a Docker daemon of its own (a docker:dind container standing in for a VM with an address on the LAN, on the shared test network) with github_runner_cluster_network_mode host, joined through a Headscale server the test runs (headscale_existing); the nodes' Tailscale clients must then reach each other directly at those daemons' addresses, and the role must not publish any port; then the role must refuse to move two running nodes to the bridge network in one run, and move one there and back through its guard, the node keeping its mesh address and etcd membership. Linux only, since the control node reaches each daemon at its address on the test network. The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. +# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). native_server three Docker servers and a native host (a privileged systemd container, tests/mesh/native/Dockerfile, managed over Ansible's docker connection) that joins as an agent, is then made a server by the inventory and promoted in place: it must join etcd as a learner, be promoted to a voter with its mesh address as its peer URL, and keep the cluster writable throughout, and a rerun must not restart it. role_controller four hosts (three servers and a standby agent, with etcd's data on memory so the machine's disk does not decide): the role controller's probe and controller run in the cluster from the image built from this checkout, with the manifests the ARC role renders; every host is reported fit from a full window of real samples, with the link the cluster role labelled its node with, a voter made slow with tc netem on its mesh interface is reported unfit with the swap named and every unfit verdict attributed to it, a paused voter makes the controller report that it would not act, and the controller changes nothing (not in the default set; needs jq, and takes a window of 100 samples before it can judge). role_actuator three Docker servers and a native standby, every host eligible and the role controller on, with the test acting as the controller (holding the regrade Lease and writing the Node's desired-role label and swap annotations by hand): the native host's role actuator promotes it (a learner, then a voter, with writes succeeding throughout), steps it down (its etcd serving at etcd-only with no pod's shim left), and, once its member is removed through k3s's annotation, finishes it as an agent with the same node password and Tailscale address and its datastore moved aside, the two k3s units never running together. role_swap three Docker servers and an eligible native standby, with the role controller's probe and controller running from this checkout's image and none pinned: the controller's own `swap` command takes the Docker bootstrap host out and the native host in as one swap under the regrade Lease, with writes succeeding throughout, three voters at the end, the demoted node's labels and annotations gone, its datastore moved aside and its identity kept, and a second swap brings the former bootstrap host back into the same etcd cluster. role_swap_paused the outgoing voter's k3s is paused once the new voter votes: the removal still completes with three voters, and on unpause the tombstone keeps the removed server out of etcd for a learner stall time while its actuator finishes it as an agent. role_swap_blocked_peer the candidate's etcd peer port is dropped, so its learner never catches up: the controller rolls the promotion back and leaves the members, the voters and the host as they were. role_swap_refused a voter is paused before the swap is asked for: the command exits 1 naming it and nothing is set. The role_controller scenario also turns the dry run off after the slow voter is reported unfit, and the controller swaps it for the Docker standby. rejoin three Docker servers whose hosts the inventory marks eligible, so the role mounts each one's role directory: the bootstrap server's etcd member is removed through another server, and restarted on its tombstone it idles instead of joining again; with its datastore moved aside and no server to join it refuses to start rather than bootstrap a second cluster; given a join URL naming another server in its role directory it joins the existing cluster as a learner and is promoted, with the same cluster ID before and after; and a role file naming the agent leaves its server container idle with no k3s. compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). host_network each node runs in a Docker daemon of its own (a docker:dind container standing in for a VM with an address on the LAN, on the shared test network) with github_runner_cluster_network_mode host, joined through a Headscale server the test runs (headscale_existing); the nodes' Tailscale clients must then reach each other directly at those daemons' addresses, and the role must not publish any port; then the role must refuse to move two running nodes to the bridge network in one run, and move one there and back through its guard, the node keeping its mesh address and etcd membership. Linux only, since the control node reaches each daemon at its address on the test network. The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. # # With GRTEST_PIN_CONTEXT=1, the cluster scenarios run the role, and the in_cluster recovery playbook, with github_runner_cluster_docker_context pinned while the current Docker context they see points at nothing (tests/lib/docker_context.sh), so any docker call that ignored the pin fails the run; afterwards the current context must be unchanged. context_checks the role refuses a pinned context that does not exist, and a pinned context alongside DOCKER_HOST, before writing anything (not in the default set). # @@ -32,10 +32,16 @@ native_role="" native_regrade=false # The Docker hosts the inventory lists, by index; all of them unless a scenario leaves one out (native_server's second run, where node 3 keeps running as a voter outside the inventory). inventory_nodes="" +# The node whose API the swap scenarios read and write through: a node that keeps its server through the scenario (the swap scenarios take node 1, the bootstrap host, out of the cluster). Node 1 unless a scenario says otherwise. +api_node=1 +# Whether the inventory pins the first host (role_controller=1 turns the controller on for every host and pins node 1, which a swap of node 1 must not be). +pin_first=1 # Where tc comes from for the role_controller scenario: a throwaway container in a node's network namespace, since the node's own image has none. tc_image=alpine:3.20 -# The probe's period in the role_controller scenario: long enough that a sample's own work (a few kubectl calls) is a small part of it, since a probe that has not reported for three periods counts as silent. +# The probe's period in the role_controller scenario while it fills a window of samples: long enough that a sample's own work (a few kubectl calls) is a small part of it, since a probe that has not reported for three periods counts as silent. role_controller_sample_seconds=5 +# The probe's period in the role_controller_swap scenario and the role_swap ones: the ARC role's own default. A swap holds its steps while any voter's probe is silent, and a probe on a runner with this many k3s nodes under way misses three periods of 5 seconds, which only stalls the swap; the default keeps a silent probe a fault and not a flutter. +swap_sample_seconds=15 # Tries per pod pair, 5 seconds apart plus each try's own timeout: about two minutes for the mesh to converge after a restart. reach_attempts=8 # etcd's timing for every server, in milliseconds: a heartbeat interval and an election timeout of ten times it, above k3s's own defaults and so told apart from them, and above etcd's required five times. @@ -75,6 +81,13 @@ diagnose() { docker_on "$index" inspect --format 'restarts={{.RestartCount}} started={{.State.StartedAt}} status={{.State.Status}} exit={{.State.ExitCode}}' "grtest-node${index}-role-actuator-1" || true docker_on "$index" logs --tail 60 "grtest-node${index}-role-actuator-1" 2>&1 | cut -c1-400 || true fi + # The agent container a server's host runs once it has been demoted, which the node's own container above is not: whether it runs, whether it reached the mesh, and why it is not Ready. + if [ "$index" -lt "$agent_from" ] && docker_on "$index" inspect "grtest-node${index}-agent" >/dev/null 2>&1; then + echo "----- grtest-node${index}-agent: state, tailscale and last log lines" + docker_on "$index" inspect --format 'restarts={{.RestartCount}} started={{.State.StartedAt}} status={{.State.Status}} exit={{.State.ExitCode}}' "grtest-node${index}-agent" || true + docker_on "$index" exec "grtest-node${index}-agent" tailscale status &1 | head -n 20 || true + docker_on "$index" logs --tail 80 "grtest-node${index}-agent" 2>&1 | cut -c1-400 || true + fi done # The reports every actuator made (JSON of swap, step, phase, role, times and message, no secret), through the first node that answers. if [ -n "${platform_namespace:-}" ]; then @@ -86,11 +99,19 @@ diagnose() { break fi done + echo "----- the role controller's status and log" + for index in $(seq 1 "$node_count"); do + docker_on "$index" inspect "$(container "$index")" >/dev/null 2>&1 || continue + if kubectl_on "$index" --namespace "$platform_namespace" get configmap fleet-role-status -o 'jsonpath={.data.status\.json}' /dev/null | jq -c '{swap, waiting, last_swap, request, gates, blocks, decision}' 2>/dev/null | cut -c1-3000; then + kubectl_on "$index" --namespace "$platform_namespace" logs deployment/fleet-role --tail 80 &1 | cut -c1-400 || true + break + fi + done fi if docker inspect "$native_host" >/dev/null 2>&1; then echo "----- ${native_host}: units, role file, Tailscale and the last journal lines" docker exec "$native_host" sh -c 'systemctl is-active k3s k3s-agent tailscaled; cat /var/lib/rancher/k3s/fleet-role; tailscale status | head -n 20' &1 || true - docker exec "$native_host" journalctl -u k3s -u k3s-agent -u tailscaled --no-pager -n 80 &1 | cut -c1-400 || true + docker exec "$native_host" journalctl -u k3s -u k3s-agent -u tailscaled -u fleet-role-actuator --no-pager -n 120 &1 | cut -c1-400 || true fi if docker inspect grtest-headscale >/dev/null 2>&1; then echo "----- grtest-headscale: nodes and routes" @@ -146,7 +167,10 @@ ansible_run() { (cd "${repo_root}/ansible" && env "${config[@]}" ANSIBLE_COLLECTIONS_PATH="${repo_root}/playbooks/collections:${ANSIBLE_COLLECTIONS_PATH:-}" "$ansible_playbook" "$@") } +# The background loop that loads the role controller's image into new node containers, once a scenario has started one. +image_loader_pid= cleanup() { + if [ -n "$image_loader_pid" ]; then kill "$image_loader_pid" 2>/dev/null || true; fi if [ "${GRTEST_KEEP:-}" = 1 ]; then log "Keeping the test environment in ${work} (GRTEST_KEEP=1)" return @@ -180,11 +204,6 @@ prepare_nodes() { else echo " ports: !reset []" fi - # etcd's data on memory when asked: the role_controller scenario judges the voters' and the standby's disk against etcd's own limits (a WAL fsync under 10 ms), which a laptop's or a runner's virtual disk does not reliably meet (a laptop's measured 15 to 30 ms), and what it tests is the controller's pipeline, not the disk of the machine it runs on. - if [ "${etcd_on_memory:-0}" = 1 ]; then - echo " tmpfs:" - echo " - /var/lib/rancher/k3s/server/db" - fi echo " networks:" echo " ${network}:" echo " ipv4_address: $(node_ip "$index")" @@ -200,6 +219,12 @@ prepare_nodes() { echo "networks:" echo " ${network}:" echo " external: true" + # etcd's data on memory when asked: the role_controller scenario judges the voters' and the standby's disk against etcd's own limits (a WAL fsync under 10 ms), which a laptop's or a runner's virtual disk does not reliably meet (a laptop's measured 15 to 30 ms), and what it tests is the controller's pipeline, not the disk of the machine it runs on. The server's whole data volume is on memory, rather than a tmpfs mount over server/db in the k3s service alone: the volume is what the role actuator mounts too, so it sees the datastore the server runs on (its member name, k3s's tombstone) and moves that one aside, and it outlives a restart of the server's container while the actuator holds it, as a disk does. A tmpfs mount is emptied at every restart, so a server restarted into etcd-only, or by its restart policy after its member was removed, came back with no datastore and no tombstone and joined etcd again as a new member. + if [ "${etcd_on_memory:-0}" = 1 ]; then + echo "volumes:" + echo " k3s-data:" + echo " driver_opts: {type: tmpfs, device: tmpfs}" + fi } > "${dir}/compose.grtest.yml" done } @@ -265,7 +290,7 @@ EOF if [ "${role_controller:-0}" = 1 ] || [ "${voters_eligible:-0}" = 1 ]; then echo " github_runner_cluster_voter_eligible: true" fi - if [ "${role_controller:-0}" = 1 ] && [ "$index" = 1 ]; then echo " github_runner_cluster_voter_pinned: true"; fi + if [ "${role_controller:-0}" = 1 ] && [ "$index" = 1 ] && [ "$pin_first" = 1 ]; then echo " github_runner_cluster_voter_pinned: true"; fi done if [ -n "$native_role" ]; then echo " ${native_host}:" @@ -1025,13 +1050,13 @@ docker_servers_serving() { } # node_status names a Docker node by its index; the native host's node is named for the host. -native_node_ready() { [ "$(kubectl_on 1 get node "$native_host" --no-headers 2>/dev/null | awk '{print $2}')" = Ready ]; } +native_node_ready() { [ "$(kubectl_on "$api_node" get node "$native_host" --no-headers 2>/dev/null | awk '{print $2}')" = Ready ]; } native_unit_is() { [ "$(native_exec systemctl is-active "$1" /dev/null)" = "$2" ]; } -# The etcd member list, as assert_cluster reads it, through node 1 or the node given. +# The etcd member list, as assert_cluster reads it, through the node given or the api node. etcd_members() { docker rm -f grtest-etcdctl >/dev/null 2>&1 || true - docker run --rm --name grtest-etcdctl --network "container:$(container "${1:-1}")" --volumes-from "$(container "${1:-1}")" --entrypoint etcdctl "$etcd_image" \ + docker run --rm --name grtest-etcdctl --network "container:$(container "${1:-$api_node}")" --volumes-from "$(container "${1:-$api_node}")" --entrypoint etcdctl "$etcd_image" \ --endpoints https://127.0.0.1:2379 --cacert /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \ --cert /var/lib/rancher/k3s/server/tls/etcd/client.crt --key /var/lib/rancher/k3s/server/tls/etcd/client.key \ member list -w simple /dev/null)" = True ] + [ "$(kubectl_on "$api_node" get node "${1:-$native_host}" -o 'jsonpath={.status.conditions[?(@.type=="EtcdIsVoter")].status}' 2>/dev/null)" = True ] } # Writes to the cluster through node 1 once a second until the stop file appears, recording each write as ok, or as failed with the time and what kubectl said. Each write is a ConfigMap named by its sequence number, so no two writes share a name: write_marker's random suffix repeats often enough over a promotion's worth of writes that a refusal for a name already taken would read as a lost write. write_throughout() { local log="$1" stop="$2" sequence=0 error : > "$log" + : > "${log}.times" until [ -e "$stop" ]; do sequence=$((sequence + 1)) - if error=$(kubectl_on 1 create configmap "grtest-native-write-${sequence}" --from-literal=a=1 2>&1 >/dev/null &1 >/dev/null > "$log" + echo "$(date +%s) ok" >> "${log}.times" else echo "failed $(date -u +%H:%M:%S) ${error}" >> "$log" + echo "$(date +%s) failed" >> "${log}.times" fi sleep 1 done } # Watches the member list until the stop file appears, recording every line that names the native host's member, so a learner seen at any point is on record. +# Fails unless every write that failed in the log $1 (kept with its time by write_throughout) was followed by a write that succeeded within the time a leader election takes: etcd's election timeout, plus the up to one more election timeout of randomised delay a candidate waits before it stands, since stepping down the member that happens to lead makes the others elect another and a write in flight when it goes is answered "request timed out, possibly due to previous leader failure". Quorum lost is writes failing and not recovering; one election is not. +assert_writes_recovered() { + local window=$(((etcd_election_ms * 2 + 999) / 1000)) bad + # A run of failures is judged from its first failure to the success that ends it. + bad=$(awk -v window="$window" ' + $2 == "failed" { if (start == "") start = $1; last = $1; next } + $2 == "ok" { if (start != "" && $1 - start > window) print "writes failed from " start " until " $1; start = "" } + END { if (start != "") print "writes were still failing at " last " after failing from " start }' "${1}.times") + [ -z "$bad" ] || { echo "$bad"; grep '^failed' "$1"; return 1; } +} + watch_native_member() { local log="$1" stop="$2" : > "$log" @@ -1154,11 +1193,11 @@ role_controller_delay_ms=$((etcd_election_ms / 10 + 100)) role_controller_image=grtest/fleet-role:test platform_namespace=github-runner-platform -role_status() { kubectl_on 1 --namespace "$platform_namespace" get configmap fleet-role-status -o 'jsonpath={.data.status\.json}' 2>/dev/null; } +role_status() { kubectl_on "$api_node" --namespace "$platform_namespace" get configmap fleet-role-status -o 'jsonpath={.data.status\.json}' 2>/dev/null; } # Whether the controller's last status satisfies a jq test, as an exit status. role_status_is() { role_status | jq -e "$1" >/dev/null 2>&1; } # The node a pod of the app runs on. -role_pod_node() { kubectl_on 1 --namespace "$platform_namespace" get pods -l "app.kubernetes.io/name=$1" -o 'jsonpath={.items[0].spec.nodeName}'; } +role_pod_node() { kubectl_on "$api_node" --namespace "$platform_namespace" get pods -l "app.kubernetes.io/name=$1" -o 'jsonpath={.items[0].spec.nodeName}'; } # Puts a delay on a node's mesh interface, or takes it off, from a throwaway container in the node's network namespace (the node's own image has no tc). mesh_delay() { local index="$1" action="$2" ms="${3:-0}" command @@ -1166,9 +1205,69 @@ mesh_delay() { docker run --rm --network "container:$(container "$index")" --cap-add NET_ADMIN --entrypoint sh "$tc_image" -c "$command" /dev/null | grep -F "$role_controller_image" >/dev/null; then continue; fi + docker exec -i "$container" /bin/ctr --address /run/k3s/containerd/containerd.sock --namespace k8s.io images import - < "$1" >/dev/null 2>&1 || true + done + sleep 3 + done +} + +# Runs the role controller's probe and controller in the cluster, the probes at $1 seconds (role_controller_sample_seconds by default): the image built from this checkout is loaded into every node (the native host's containerd too, when the scenario has one), the ARC role renders its manifests, and they are applied and rolled out. +deploy_role_controller() { + local index image_file manifests sample_seconds=${1:-$role_controller_sample_seconds} + log "Building the role controller image from this checkout and loading it into every node" + docker build --quiet --file "${repo_root}/fleet-role/Dockerfile" --tag "$role_controller_image" "$repo_root" >/dev/null + image_file="${work}/fleet-role-image.tar" + docker save --output "$image_file" "$role_controller_image" + for index in $(seq 1 "$node_count"); do + docker exec -i "$(container "$index")" /bin/ctr --address /run/k3s/containerd/containerd.sock --namespace k8s.io images import - < "$image_file" >/dev/null + done + if [ -n "$native_role" ]; then native_exec k3s ctr --namespace k8s.io images import - < "$image_file" >/dev/null; fi + # A host that changes role starts a container whose containerd has no copy of the image (the server and agent services keep separate data volumes, and a real host pulls the image from its registry), so a loop loads it into each node container as one comes up, which keeps the new voter's probe running. + keep_role_image_loaded "$image_file" & + image_loader_pid=$! + + log "Rendering the ARC role's manifests for the controller and the probe, and applying them" + manifests="${work}/fleet-role.yaml" + ANSIBLE_COLLECTIONS_PATH="${repo_root}/playbooks/collections" ANSIBLE_LOCALHOST_WARNING=false ANSIBLE_INVENTORY_UNPARSED_WARNING=false \ + "$ansible_playbook" "${repo_root}/tests/fleet_role/render.yml" -e "fleet_role_output=${manifests}" \ + -e "github_runner_arc_role_controller_image=${role_controller_image}" -e "github_runner_arc_role_controller_sample_seconds=${sample_seconds}" \ + -e "github_runner_arc_role_controller_election_timeout_ms=${etcd_election_ms}" >/dev/null + # Applied rather than created: the cluster role makes the namespace too, for the role actuators' credential, which it sets up while the controller is on. + kubectl_on "$api_node" apply -f - >/dev/null </dev/null + kubectl_on "$api_node" --namespace "$platform_namespace" rollout status daemonset/fleet-role-probe --timeout=300s + kubectl_on "$api_node" --namespace "$platform_namespace" rollout status deployment/fleet-role --timeout=300s + kubectl_on "$api_node" --namespace "$platform_namespace" get pods -o wide +} + +# Brings up three servers and a standby agent with the role controller on and waits for every node to be Ready: the start of both role_controller scenarios. Sets `ids`, the etcd members at the start, in the caller's variable of that name. +role_controller_cluster_up() { + local deadline node_count=4 agent_from=4 etcd_on_memory=1 @@ -1187,6 +1286,12 @@ scenario_role_controller() { [ "$(kubectl_on 1 get nodes -l node-role.kubernetes.io/etcd=true --no-headers | wc -l | tr -d ' ')" = 3 ] || fail "expected three etcd nodes" if kubectl_on 1 get node "$(node_name 4)" -o 'jsonpath={.metadata.labels}' | grep -F 'node-role.kubernetes.io/etcd' >/dev/null; then fail "the fourth node is a server"; fi ids=$(etcd_member_ids 1) +} + +# Brings up three servers and a standby agent, runs the role controller's probe and controller in the cluster (the pods from the image built from this checkout, the manifests the ARC role renders), and checks that the controller reports what it sees on real k3s and real etcd: all hosts fit, then one voter made slow with tc netem on its mesh interface reported unfit with the swap named, then a voter paused and the controller reporting that it would not act, and throughout that it has changed nothing. +scenario_role_controller() { + local index ids image_file manifests slow paused paused_name controller_node deadline account verb expected link + role_controller_cluster_up log "A check run on hosts that lack the Compose files of the role controller, as a host that ran the role before the controller was turned on does, passes and writes nothing" local before_started file @@ -1210,43 +1315,7 @@ scenario_role_controller() { done [ "$before_started" = "$(started_at)" ] || fail "the real run restarted a node whose configuration it left as it was" - log "Building the role controller image from this checkout and loading it into every node" - docker build --quiet --file "${repo_root}/fleet-role/Dockerfile" --tag "$role_controller_image" "$repo_root" >/dev/null - image_file="${work}/fleet-role-image.tar" - docker save --output "$image_file" "$role_controller_image" - for index in $(seq 1 "$node_count"); do - docker exec -i "$(container "$index")" /bin/ctr --address /run/k3s/containerd/containerd.sock --namespace k8s.io images import - < "$image_file" >/dev/null - done - rm -f "$image_file" - - log "Rendering the ARC role's manifests for the controller and the probe, and applying them" - manifests="${work}/fleet-role.yaml" - ANSIBLE_COLLECTIONS_PATH="${repo_root}/playbooks/collections" ANSIBLE_LOCALHOST_WARNING=false ANSIBLE_INVENTORY_UNPARSED_WARNING=false \ - "$ansible_playbook" "${repo_root}/tests/fleet_role/render.yml" -e "fleet_role_output=${manifests}" \ - -e "github_runner_arc_role_controller_image=${role_controller_image}" -e "github_runner_arc_role_controller_sample_seconds=${role_controller_sample_seconds}" \ - -e "github_runner_arc_role_controller_election_timeout_ms=${etcd_election_ms}" >/dev/null - # Applied rather than created: the cluster role makes the namespace too, for the role actuators' credential, which it sets up while the controller is on. - kubectl_on 1 apply -f - >/dev/null </dev/null + deploy_role_controller # Every host of this scenario is this machine, which the role reads the link of with its own reader. link="$("${repo_root}/roles/github_runner_cluster/files/node-link.sh" | jq -r .link)" log "The role published the labels from the inventory, eligible on every host and pinned on the first alone, and the medium of this machine's link (${link}) on every host" @@ -1257,19 +1326,22 @@ MANIFESTS if [ "$index" = 1 ]; then expected=true; fi [ "$(kubectl_on 1 get node "$(node_name "$index")" -o 'jsonpath={.metadata.labels.github-runner\.exadev\.com/voter-pinned}')" = "$expected" ] || fail "$(node_name "$index") does not carry voter-pinned=${expected}" done - kubectl_on 1 --namespace "$platform_namespace" rollout status daemonset/fleet-role-probe --timeout=300s - kubectl_on 1 --namespace "$platform_namespace" rollout status deployment/fleet-role --timeout=300s - kubectl_on 1 --namespace "$platform_namespace" get pods -o wide - log "The controller and the probe can read nodes and nothing more, and the controller cannot write the probes' ConfigMap" + log "The probe can read nodes and nothing more; the controller can also patch them, as a swap needs, but not update, delete or create one, and neither can create or patch a ConfigMap it does not own" for account in fleet-role fleet-role-probe; do [ "$(kubectl_on 1 auth can-i list nodes --as "system:serviceaccount:${platform_namespace}:${account}")" = yes ] || fail "${account} cannot list nodes" - for verb in patch update delete create; do + for verb in update delete create; do [ "$(kubectl_on 1 auth can-i "$verb" nodes --as "system:serviceaccount:${platform_namespace}:${account}" || true)" = no ] || fail "${account} can ${verb} nodes" done + expected=no + if [ "$account" = fleet-role ]; then expected=yes; fi + [ "$(kubectl_on 1 auth can-i patch nodes --as "system:serviceaccount:${platform_namespace}:${account}" || true)" = "$expected" ] || fail "${account} does not get patch on nodes as ${expected}" [ "$(kubectl_on 1 auth can-i create configmaps --namespace "$platform_namespace" --as "system:serviceaccount:${platform_namespace}:${account}" || true)" = no ] || fail "${account} can create configmaps" done [ "$(kubectl_on 1 auth can-i patch configmaps/fleet-role-probe --namespace "$platform_namespace" --as "system:serviceaccount:${platform_namespace}:fleet-role" || true)" = no ] || fail "the controller can patch the probes' ConfigMap" + [ "$(kubectl_on 1 auth can-i update lease/fleet-regrade --namespace kube-system --as "system:serviceaccount:${platform_namespace}:fleet-role" || true)" = yes ] || fail "the controller cannot replace the regrade Lease" + [ "$(kubectl_on 1 auth can-i update lease/another --namespace kube-system --as "system:serviceaccount:${platform_namespace}:fleet-role" || true)" = no ] || fail "the controller can replace a Lease that is not the regrade Lease" + [ "$(kubectl_on 1 auth can-i update lease/fleet-regrade --namespace kube-system --as "system:serviceaccount:${platform_namespace}:fleet-role-probe" || true)" = no ] || fail "the probe can replace the regrade Lease" log "Every host is fit, from a window of real samples, and the controller would not act" wait_for "every host to be reported fit and no swap wanted" 1800 role_status_is '(.hosts | length == 4) and ([.hosts[].verdict] | all(. == "fit")) and .decision.action == "none" and (.blocks | length == 0)' @@ -1278,17 +1350,7 @@ MANIFESTS # A node in a container sees only its veth, so what each probe reports is the label the role published. role_status | jq -e --arg link "$link" '[.hosts[] | .link == $link and .link_source == "label"] | all' >/dev/null || fail "the probes did not report the link the role labelled the nodes with (${link}): $(role_status | jq -c '[.hosts[] | {link, link_source}]')" - log "A voter is made slow with tc netem on its mesh interface (${role_controller_delay_ms} ms each way): the controller reports it unfit and names the swap" - slow=3 - mesh_delay "$slow" add "$role_controller_delay_ms" - wait_for "$(node_name "$slow") to be reported unfit and the swap named" 900 role_status_is \ - ".hosts[\"$(node_name "$slow")\"].verdict == \"unfit\" and .decision.action == \"swap\" and .decision.swap == {out: \"$(node_name "$slow")\", in: \"$(node_name 4)\"}" - role_status | jq . - role_status | jq -r '.decision.because[]' - role_status | jq -e ".hosts[\"$(node_name "$slow")\"].reasons | any(test(\"round trip p99\"))" >/dev/null || fail "the reason does not name the round trip" - role_status | jq -e --arg slow "$(node_name "$slow")" '([.hosts[] | select(.verdict == "unfit") | .attributed_to | index($slow) != null] | all) and (.attribution[$slow] | index($slow) != null)' >/dev/null \ - || fail "the unfit verdicts are not attributed to $(node_name "$slow"): $(role_status | jq -c '.attribution')" - kubectl_on 1 --namespace "$platform_namespace" logs deployment/fleet-role --tail 20 | grep -F "DRY RUN: would swap out $(node_name "$slow") for $(node_name 4)" >/dev/null || fail "the controller did not log the swap it would make" + role_controller_slow_voter_named log "A voter is paused: the controller reports that it would not act" controller_node=$(role_pod_node fleet-role) @@ -1301,13 +1363,61 @@ MANIFESTS role_status | jq '{decision, blocks}' kubectl_on 1 --namespace "$platform_namespace" logs deployment/fleet-role --tail 5 | grep -F 'DRY RUN: would not act' >/dev/null || fail "the controller did not log that it would not act" docker unpause "$(container "$paused")" >/dev/null - mesh_delay "$slow" del log "The controller changed nothing: the same etcd members, the same servers, no node touched" [ "$(etcd_member_ids 1)" = "$ids" ] || fail "the etcd members changed: $(etcd_member_ids 1) (was ${ids})" [ "$(kubectl_on 1 get nodes -l node-role.kubernetes.io/etcd=true --no-headers | wc -l | tr -d ' ')" = 3 ] || fail "the set of etcd nodes changed" kubectl_on 1 get nodes -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{.spec.unschedulable}{" "}{.spec.taints}{"\n"}{end}' if kubectl_on 1 get nodes -o json | jq -e '[.items[] | select(.metadata.annotations["etcd.k3s.cattle.io/remove"] != null)] | length > 0' >/dev/null; then fail "a node was annotated for removal"; fi + [ -z "$(kubectl_on 1 get lease fleet-regrade --namespace kube-system -o 'jsonpath={.metadata.name}' 2>/dev/null /dev/null || fail "the reason does not name the round trip" + role_status | jq -e --arg slow "$(node_name "$slow")" '([.hosts[] | select(.verdict == "unfit") | .attributed_to | index($slow) != null] | all) and (.attribution[$slow] | index($slow) != null)' >/dev/null \ + || fail "the unfit verdicts are not attributed to $(node_name "$slow"): $(role_status | jq -c '.attribution')" + kubectl_on 1 --namespace "$platform_namespace" logs deployment/fleet-role --tail 20 | grep -F "DRY RUN: would swap out $(node_name "$slow") for $(node_name 4)" >/dev/null || fail "the controller did not log the swap it would make" +} + +# The window of samples the controller reads each figure over, in seconds, at the probes' period $1: the count is the controller's own, read from the module it takes it from. +role_controller_window_seconds() { echo $(($1 * $(jq -nr -L "${repo_root}/scripts" 'include "fleet-role-stats"; window_samples'))); } + +# The role controller at the ARC role's default period carrying out the swap it reported: a voter is made slow, the controller (in dry run) names the swap, and with the dry run turned off it makes it. It is a scenario of its own because a window of samples at the default period takes as long to fill as the controller says (role_controller_window_seconds), and a swap has its own bounds (swap_wait_seconds), which together do not fit beside role_controller's other checks in one job. +scenario_role_controller_swap() { + local ids slow window_seconds + window_seconds=$(role_controller_window_seconds "$swap_sample_seconds") + role_controller_cluster_up + deploy_role_controller "$swap_sample_seconds" + + log "Every host is fit, from a window of real samples at the default period, and the controller would not act" + wait_for "every host to be reported fit and no swap wanted" $((window_seconds + 900)) role_status_is '(.hosts | length == 4) and ([.hosts[].verdict] | all(. == "fit")) and .decision.action == "none" and (.blocks | length == 0)' + + role_controller_slow_voter_named + + log "With the dry run turned off, the controller carries out the swap it reported, bringing the standby agent, a Docker host that was never a server, in for the slow voter, which stays slow" + kubectl_on 1 --namespace "$platform_namespace" set env deployment/fleet-role FLEET_ROLE_DRY_RUN=false >/dev/null + kubectl_on 1 --namespace "$platform_namespace" rollout status deployment/fleet-role --timeout=300s + wait_for_swap "$(node_name "$slow")" "$(node_name 4)" completed + role_status | jq -e '.dry_run == false and .request == null' >/dev/null || fail "the swap was not the controller's own decision: $(role_status | jq -c '{dry_run, request, last_swap}')" + # The slow voter stays up throughout, so its actuator finishes the demotion inside the swap; a swap that leaves it to node recovery has lost the finish, whatever the outcome says. + role_status | jq -e '.last_swap.outgoing_pending != true' >/dev/null || fail "the swap left $(node_name "$slow") to node recovery rather than finishing it as an agent: $(role_status | jq -c .last_swap)" + [ "$(node_is_voter "$(node_name 4)" && echo yes)" = yes ] || fail "$(node_name 4) is not a voter after the swap" + [ "$(voter_count)" = 3 ] || fail "expected three voters after the swap, got $(voter_count)" + node_is_clean "$(node_name "$slow")" || fail "$(node_name "$slow")'s labels and annotations were not cleaned up" + [ "$(docker_state grtest-node3-agent)" = running ] || fail "$(node_name "$slow") does not run its agent" + [ "$(docker_state grtest-node3-k3s)" = exited ] || fail "$(node_name "$slow") still runs its server" + [ "$(docker_state grtest-node4-k3s)" = running ] || fail "$(node_name 4) does not run a server container" + [ "$(started_member_count)" = 3 ] || { etcd_members; fail "expected three started etcd members after the swap"; } + [ "$(etcd_member_ids 1)" != "$ids" ] || fail "the etcd members are the ones the swap started from" + lease_released || fail "the regrade Lease was not released" } # --- Memory-pressure regrading of the native server (native_server's last part) --- @@ -1461,6 +1571,8 @@ etcd_cluster_id() { --cert /var/lib/rancher/k3s/server/tls/etcd/client.crt --key /var/lib/rancher/k3s/server/tls/etcd/client.key \ endpoint status -w fields &1 | grep -F "$2" >/dev/null; } removed_member_recorded() { [ -n "$(kubectl_on 2 get node "$(node_name 1)" -o 'jsonpath={.metadata.annotations.etcd\.k3s\.cattle\.io/removed-node-name}' 2>/dev/null)" ]; } @@ -1542,6 +1654,7 @@ scenario_rejoin() { done [ "$promoted" = 1 ] || fail "no other server's k3s logged promoting $(node_name 1)'s learner" [ "$(started_members)" = 3 ] || { etcd_members 2; fail "expected three started etcd members after the rejoin"; } + wait_for "$(node_name 1)'s etcd to answer" 300 etcd_answers 1 [ "$(etcd_cluster_id 1)" = "$cluster_id" ] || fail "$(node_name 1) joined a cluster with another ID: $(etcd_cluster_id 1) (was ${cluster_id})" [ "$(node_password_hash 1)" = "$password" ] || fail "$(node_name 1)'s node password changed" docker run --rm -v "$(node_name 1)_k3s-data:/data" "$busybox_image" test -d /data/server/db.demoted || fail "the moved-aside datastore is gone" @@ -1558,6 +1671,87 @@ scenario_rejoin() { [ "$(docker inspect --format '{{.State.StartedAt}} {{.RestartCount}}' "$(container 1)")" = "$started_before" ] || fail "the idle container restarted" } +# --- The role controller carrying out a voter swap (the role_swap scenarios) --- + +# The controller's own waits, as the cluster role's defaults give them: github_runner_cluster_regrade_operation_timeout_seconds (the regrade pressure window), github_runner_cluster_native_runner_wait_timeout (its server ready timeout) and k3s's learnerMaxStallTime. +operation_timeout_seconds=300 +runner_wait_seconds=600 +learner_stall_seconds=300 +# The most seconds a swap is waited for: the sum of every step's own bound (scripts/fleet-role-swap.jq's budget): the snapshot, the candidate's drain (the runner wait and an operation), the promotion (the learner stall time and an operation), the step-down, the removal, the finish and the clean-up of the outgoing host, an operation each. +swap_wait_seconds=$((7 * operation_timeout_seconds + runner_wait_seconds + learner_stall_seconds)) + +# Waits until the controller's status says no swap is under way and the last one that ended, from node $1 for node $2, ended as $3 (completed by default), saying the step the swap is at and what it waits for as they change. +wait_for_swap() { + local out=$1 in=$2 outcome=${3:-completed} deadline=$((SECONDS + swap_wait_seconds)) seen='' now + until role_status_is ".swap == null and .last_swap.outcome == \"${outcome}\" and .last_swap.out == \"${out}\" and .last_swap.in == \"${in}\""; do + [ "$SECONDS" -lt "$deadline" ] || { role_status | jq '{swap, waiting, last_swap, blocks}' || true; fail "the swap of ${out} for ${in} did not end as ${outcome}"; } + now=$(role_status | jq -r '(.swap.step // "none") + ": " + ((.waiting // []) | join("; "))' 2>/dev/null || true) + if [ "$now" != "$seen" ]; then log "swap ${out} for ${in}: ${now}"; seen=$now; fi + sleep 5 + done + role_status | jq '{swap, last_swap}' +} + +# Asks the controller, in its own pod as an owner does, for the swap of node $1 for node $2, once every gate holds: the command refuses while one does not, and the pod may be rescheduled meanwhile. +swap_command() { kubectl_on "$api_node" --namespace "$platform_namespace" exec deploy/fleet-role -- /app/scripts/fleet-role.sh swap "$1" "$2" /dev/null +} + +# Whether node $1 carries no swap annotation, regrade level or cordon: a voter, or an agent, that no swap has touched. +node_untouched() { + kubectl_on "$api_node" get node "$1" -o json /dev/null +} + +# Whether the regrade Lease is free or gone: nobody holds it. +lease_released() { [ -z "$(kubectl_on "$api_node" get lease fleet-regrade --namespace kube-system -o 'jsonpath={.spec.holderIdentity}' 2>/dev/null /dev/null /dev/null /dev/null)" = "$1 $2" ]; } # Whether the Node $2 (the native host's by default) reports the Ready condition True and its kubelet has renewed the node's Lease (kube-node-lease/) at or after the time $1 (RFC 3339, UTC), as tasks/native/server_ready.yml judges a restart: the API keeps a stopped kubelet's Ready condition until the node controller's grace period runs out, so the condition alone could be the server's from before its step-down. Read from the condition, not the STATUS column, which says Ready,SchedulingDisabled while the node is cordoned. Both times are cut to whole seconds before they are compared, as strings, since RFC 3339 in UTC sorts as it reads. node_ready_since() { local renewed node=${2:-$native_host} - [ "$(kubectl_on 1 get node "$node" -o 'jsonpath={.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null /dev/null /dev/null /dev/null /dev/null /dev/null /dev/null | grep -c ', started, ' || true; } # Holds the regrade Lease as the role controller does for the whole of the swap $1 of the nodes $2. @@ -1795,6 +1989,7 @@ role_actuator_docker() { [ "$(docker_state "$agent")" = exited ] || fail "${node}'s agent container is $(docker_state "$agent"), not stopped" [ "$(docker exec "$server" sha256sum /etc/rancher/node/password /dev/null @@ -1802,6 +1997,158 @@ role_actuator_docker() { [ "$(kubectl_on 1 get nodes --no-headers | grep -c "^${node} ")" = 1 ] || fail "${node} is not exactly one Node" } +# A manual swap by the controller's own command: the native standby is promoted and the Docker bootstrap host demoted in one swap, with writes succeeding throughout, then the bootstrap host is promoted again in a second swap, joining the cluster it left. +scenario_role_swap() { + local password address native_password cluster_id writes stop writer units unit_watcher ok failed node + setup_swap_cluster + node=$(node_name 1) + password=$(node_password_hash 1) + address=$(docker exec "$(container 1)" tailscale ip -4 /dev/null + log "${out}'s server is paused, after ${native_host} voted" + wait_for_swap "$out" "$native_host" + role_status | jq -e '.last_swap.outgoing_pending == true' >/dev/null || log "the swap did not record the outgoing host as pending: $(role_status | jq -c .last_swap)" + [ "$(voter_count)" = 3 ] || { etcd_members 1; fail "expected three voters with ${out} paused, got $(voter_count)"; } + [ "$(etcd_members 1 | grep -c ", ${out}-" || true)" = 0 ] || { etcd_members 1; fail "${out}'s member is still listed"; } + [ "$(started_member_count)" -ge 3 ] || { etcd_members 1; fail "expected three started etcd members with ${out} paused"; } + lease_released || fail "the regrade Lease was not released" + + log "Unpaused, ${out} must not rejoin etcd: the member count stays three for a learner stall time, and its actuator finishes it as an agent" + docker unpause "$(container "$out_node")" >/dev/null + deadline=$((SECONDS + learner_stall_seconds + 30)) + while [ "$SECONDS" -lt "$deadline" ]; do + [ "$(etcd_members 1 | wc -l | tr -d ' ')" = 3 ] || { etcd_members 1; fail "${out} is a member of etcd again"; } + sleep 10 + done + wait_for "${out}'s agent to run" 600 container_is "grtest-node${out_node}-agent" running + in_volume "${out}_k3s-data" 'test -d /volume/server/db.demoted && test ! -e /volume/server/db' || fail "${out}'s datastore was not moved aside" + [ "$(cat "${work}/${out}/fleet-role.d/role")" = agent ] || fail "${out}'s role file does not name the agent" + [ "$(docker_state "$(container "$out_node")")" = exited ] || fail "${out}'s server container is $(docker_state "$(container "$out_node")"), not stopped" + [ "$(etcd_members 1 | wc -l | tr -d ' ')" = 3 ] || { etcd_members 1; fail "the member count changed after ${out} was finished"; } +} + +# The candidate's peer port is dropped in its network namespace, so its learner never catches up: k3s removes it after a learner stall time, the controller's deadline passes and it rolls the promotion back, leaving the members and the voters as they were and the host an agent again. +scenario_role_swap_blocked_peer() { + local out members_before out_index + setup_swap_cluster + api_node=1 + out_index=3 + out=$(node_name "$out_index") + members_before=$(etcd_member_ids 1) + docker run --rm --network "container:${native_host}" --cap-add NET_ADMIN --entrypoint sh "$tc_image" -c 'apk add --quiet iptables && iptables -A INPUT -p tcp --dport 2380 -j DROP && iptables -A OUTPUT -p tcp --dport 2380 -j DROP' /dev/null + wait_for "the controller to report a gate failing for $(node_name "$paused")" 600 role_status_is ".gates | any(contains(\"$(node_name "$paused")\"))" + local output status=0 + output=$(swap_command "$out" "$native_host" 2>&1) || status=$? + echo "$output" + [ "$status" = 1 ] || fail "the swap command exited ${status}, not 1" + grep -F "$(node_name "$paused")" <<< "$output" >/dev/null || fail "the refusal does not name $(node_name "$paused")" + docker unpause "$(container "$paused")" >/dev/null + if kubectl_on "$api_node" get node "$native_host" -o json 0)' >/dev/null; then fail "a label or annotation was set on ${native_host}"; fi + node_untouched "$out" || fail "${out} was changed by a refused request" + lease_released || fail "the regrade Lease was taken by a refused request" + sleep 30 + role_status | jq -e '.swap == null and .request == null and .last_swap == null' >/dev/null || fail "a refused request left a swap or a request: $(role_status | jq -c '{swap, request, last_swap}')" +} + scenarios=("$@") [ "${#scenarios[@]}" -gt 0 ] || scenarios=(existing hosted in_cluster) for scenario in "${scenarios[@]}"; do @@ -1812,8 +2159,13 @@ for scenario in "${scenarios[@]}"; do regrade) scenario_regrade ;; native_server) scenario_native_server ;; role_controller) scenario_role_controller ;; + role_controller_swap) scenario_role_controller_swap ;; rejoin) scenario_rejoin ;; role_actuator) scenario_role_actuator ;; + role_swap) scenario_role_swap ;; + role_swap_paused) scenario_role_swap_paused ;; + role_swap_blocked_peer) scenario_role_swap_blocked_peer ;; + role_swap_refused) scenario_role_swap_refused ;; host_network) scenario_host_network ;; compose_faulty) scenario_compose_faulty ;; context_checks) scenario_context_checks ;; diff --git a/tests/regrade/regrade-lease.sh b/tests/regrade/regrade-lease.sh index b53b0d9d..52f99152 100644 --- a/tests/regrade/regrade-lease.sh +++ b/tests/regrade/regrade-lease.sh @@ -104,6 +104,28 @@ write_lease other-node "$((NOW - 10))" release_lease check "release leaves a lease that has passed to another node alone" test "$(holder)" = other-node +# A holder other than the node, with annotations, as the role controller takes the Lease for a swap. +reset_cluster +acquire_lease fleet-role '{"example.test/id":"swap-1","example.test/nodes":"a,b"}' > /dev/null +check "a named holder takes the lease in its own name" test "$(holder)" = fleet-role +check "the annotations are written with the lease" test "$(jq -c '.metadata.annotations' "$lease")" = '{"example.test/id":"swap-1","example.test/nodes":"a,b"}' +acquires_for() { acquire_lease "$1" > /dev/null 2>&1; } +check_not "another holder is refused while the named holder has it" acquires_for testnode +check "the named holder takes its own lease again" acquires_for fleet-role +check "taking it again with no annotations leaves the lease as that call wrote it" test "$(jq -c '.metadata.annotations' "$lease")" = null + +reset_cluster +acquire_lease fleet-role '{"example.test/id":"swap-1"}' > /dev/null +release_lease fleet-role +check "the named holder's release clears the holder" test "$(holder)" = "" +check "and the annotations it wrote" test "$(jq -c '.metadata.annotations' "$lease")" = null + +reset_cluster +acquire_lease fleet-role '{"example.test/id":"swap-1"}' > /dev/null +release_lease +check "a release for this node leaves another holder's lease alone" test "$(holder)" = fleet-role +check "and its annotations" test "$(jq -c '.metadata.annotations' "$lease")" = '{"example.test/id":"swap-1"}' + # Renewing: the holder keeps the Lease past its duration by writing a later renew time, and nobody else may. renews() { renew_lease "$1" > /dev/null 2>&1; } # Writes a Lease held by $1, acquired at epoch $2, last renewed at epoch $3, carrying an annotation. diff --git a/tests/unit/fleet_role_support.py b/tests/unit/fleet_role_support.py index 43fc9dd2..5471304e 100644 --- a/tests/unit/fleet_role_support.py +++ b/tests/unit/fleet_role_support.py @@ -20,6 +20,11 @@ NAMESPACE = "github-runner-platform" PROBE_CONFIGMAP = "fleet-role-probe" STATUS_CONFIGMAP = "fleet-role-status" +ACTUATOR_CONFIGMAP = "fleet-role-actuators" +# The swap's settings, distinct so that a test that names one is told apart from another by its figure; the controller derives every wait from them. +OPERATION_TIMEOUT_SECONDS = 300 +RUNNER_WAIT_SECONDS = 600 +LEASE_SECONDS = 900 ELIGIBLE_LABEL = "github-runner.exadev.com/voter-eligible" PINNED_LABEL = "github-runner.exadev.com/voter-pinned" LINK_LABEL = "github-runner.exadev.com/link" @@ -50,19 +55,64 @@ TOTAL_KB = 16 * 1024 * 1024 AVAILABLE_KB = 8 * 1024 * 1024 -# Answers the kubectl calls the probe and the controller make from a JSON file (FAKE_STORE) holding {"nodes": [...], "configmaps": {"namespace/name": {"data": {...}}}}, merges a merge patch's data into a ConfigMap, and records every call in FAKE_CALLS. A ConfigMap the store does not hold is NotFound, as the cluster answers. +# Answers the kubectl calls the probe and the controller make from a JSON file (FAKE_STORE) holding {"nodes": [...], "configmaps": {"namespace/name": {"data": {...}}}, "leases": {"namespace/name": {...}}, "snapshots": [...], "runner_pods": {"node": count}}, keeps what a write changes (a merge patch of a ConfigMap, the labels, annotations, taints and cordon of a Node, a Lease created or replaced) in the file, and records every call in FAKE_CALLS. A ConfigMap, Node or Lease the store does not hold is NotFound, as the cluster answers; FAKE_DRAIN_FAILS (a node name, or 1 for every node) makes a drain fail as one that cannot evict does, and FAKE_FAIL_CALLS (a comma-separated list of the first two words of a call) makes those calls fail. FAKE_KUBECTL = textwrap.dedent( """\ #!/usr/bin/env python3 import json, os, sys - args = sys.argv[1:] + raw = sys.argv[1:] + args = [] + skip = False + for i, a in enumerate(raw): + if skip: + skip = False + elif a == "--kubeconfig": + skip = True + elif a.startswith("--request-timeout"): + pass + else: + args.append(a) with open(os.environ["FAKE_CALLS"], "a") as log: log.write(json.dumps(args) + "\\n") store = json.load(open(os.environ["FAKE_STORE"])) - def flag(name): - return args[args.index(name) + 1] if name in args else None + store.setdefault("leases", {}) + store.setdefault("snapshots", []) + store.setdefault("runner_pods", {}) + def save(): + json.dump(store, open(os.environ["FAKE_STORE"], "w")) + def flag(*names): + for name in names: + if name in args: + return args[args.index(name) + 1] + return None + def not_found(kind, name): + sys.stderr.write('Error from server (NotFound): %s "%s" not found\\n' % (kind, name)) + sys.exit(1) + def positional(): + out, skip = [], False + for a in args: + if skip: + skip = False + elif a in ("-n", "--namespace", "-o", "--output", "-l", "--selector", "--field-selector", "--patch-file", "--type", "--timeout", "-f"): + skip = True + elif not a.startswith("-"): + out.append(a) + return out + def node_named(name): + match = [n for n in store["nodes"] if n["metadata"]["name"] == name] + if not match: + not_found("nodes", name) + return match[0] + def fail_if_asked(): + asked = os.environ.get("FAKE_FAIL_CALLS", "").split(",") + if " ".join(args[:2]) in asked: + sys.stderr.write("Error from server: the call was made to fail\\n") + sys.exit(1) + fail_if_asked() + verb = args[0] + pos = positional() if args[:2] == ["get", "nodes"]: - selector = flag("-l") + selector = flag("-l", "--selector") items = store["nodes"] if selector is not None: key, _, value = selector.partition("=") @@ -70,19 +120,35 @@ def flag(name): print(json.dumps({"items": items})) sys.exit(0) if args[:2] == ["get", "node"]: - match = [n for n in store["nodes"] if n["metadata"]["name"] == args[2]] - if not match: - sys.stderr.write('Error from server (NotFound): nodes "%s" not found\\n' % args[2]) - sys.exit(1) - print(json.dumps(match[0])) + print(json.dumps(node_named(args[2]))) sys.exit(0) if args[:2] == ["get", "configmap"]: - key = flag("-n") + "/" + args[2] + key = flag("-n", "--namespace") + "/" + args[2] if key not in store["configmaps"]: - sys.stderr.write('Error from server (NotFound): configmaps "%s" not found\\n' % args[2]) - sys.exit(1) + not_found("configmaps", args[2]) print(json.dumps({"metadata": {"name": args[2]}, "data": store["configmaps"][key]["data"]})) sys.exit(0) + if args[:2] == ["get", "lease"]: + key = flag("-n", "--namespace") + "/" + args[2] + if key not in store["leases"]: + not_found("leases.coordination.k8s.io", args[2]) + lease = store["leases"][key] + output = flag("-o", "--output") + if output and output.startswith("jsonpath="): + print(lease.get("spec", {}).get("renewTime", ""), end="") + else: + print(json.dumps(lease)) + sys.exit(0) + if args[:2] == ["get", "etcdsnapshotfiles.k3s.cattle.io"]: + if os.environ.get("FAKE_NO_SNAPSHOTS"): + sys.stderr.write("error: the server doesn't have a resource type etcdsnapshotfiles\\n") + sys.exit(1) + print(json.dumps({"items": store["snapshots"]})) + sys.exit(0) + if args[:2] == ["get", "pods"]: + node = [p for p in flag("--field-selector").split(",") if p.startswith("spec.nodeName=")][0].split("=", 1)[1] + print(json.dumps({"items": [{} for _ in range(store["runner_pods"].get(node, 0))]})) + sys.exit(0) if args[:2] == ["patch", "configmap"] and os.environ.get("FAKE_FAIL_PATCHES"): # Refuses the next patches as an API server on a poor path does: the count is kept in the file the variable names. remaining = int(open(os.environ["FAKE_FAIL_PATCHES"]).read() or "0") @@ -91,7 +157,7 @@ def flag(name): sys.stderr.write("Unable to connect to the server: net/http: TLS handshake timeout\\n") sys.exit(1) if args[:2] == ["patch", "configmap"]: - key = flag("-n") + "/" + args[2] + key = flag("-n", "--namespace") + "/" + args[2] patch = json.load(open(flag("--patch-file"))) data = store["configmaps"][key]["data"] for name, value in patch["data"].items(): @@ -99,7 +165,67 @@ def flag(name): data.pop(name, None) else: data[name] = value - json.dump(store, open(os.environ["FAKE_STORE"], "w")) + save() + sys.exit(0) + if verb in ("create", "replace") and args[1:3] == ["-f", "-"]: + lease = json.load(sys.stdin) + key = lease["metadata"]["namespace"] + "/" + lease["metadata"]["name"] + current = store["leases"].get(key) + if verb == "create": + if current is not None: + sys.stderr.write('Error from server (AlreadyExists): leases "%s" already exists\\n' % lease["metadata"]["name"]) + sys.exit(1) + lease["metadata"]["resourceVersion"] = "1" + else: + if current is None or lease["metadata"].get("resourceVersion") != current["metadata"]["resourceVersion"]: + sys.stderr.write('Error from server (Conflict): the object has been modified\\n') + sys.exit(1) + lease["metadata"]["resourceVersion"] = str(int(current["metadata"]["resourceVersion"]) + 1) + store["leases"][key] = lease + save() + sys.exit(0) + if verb in ("annotate", "label"): + node = node_named(pos[2]) + section = "annotations" if verb == "annotate" else "labels" + overwrite = "--overwrite" in args + target = node["metadata"].setdefault(section, {}) + for item in pos[3:]: + if item.endswith("-") and "=" not in item: + target.pop(item[:-1], None) + else: + key, _, value = item.partition("=") + if key in target and not overwrite: + sys.stderr.write("error: '%s' already has a value (%s), and --overwrite is false\\n" % (key, target[key])) + sys.exit(1) + target[key] = value + save() + sys.exit(0) + if verb in ("cordon", "uncordon"): + node_named(pos[1])["spec"] = {**node_named(pos[1]).get("spec", {}), "unschedulable": verb == "cordon"} + save() + sys.exit(0) + if verb == "taint": + node = node_named(pos[2]) + spec = node.setdefault("spec", {}) + taints = spec.setdefault("taints", []) + item = pos[3] + if item.endswith("-"): + key = item[:-1].split(":")[0].split("=")[0] + spec["taints"] = [t for t in taints if t["key"] != key] + else: + key, _, rest = item.partition("=") + value, _, effect = rest.partition(":") + spec["taints"] = [t for t in taints if t["key"] != key] + [{"key": key, "value": value, "effect": effect}] + save() + sys.exit(0) + if verb == "drain": + failing = os.environ.get("FAKE_DRAIN_FAILS", "") + if failing in ("1", pos[1]): + sys.stderr.write("error: unable to drain node: cannot evict pod\\n") + sys.exit(1) + node = node_named(pos[1]) + node["spec"] = {**node.get("spec", {}), "unschedulable": True} + save() sys.exit(0) sys.stderr.write("unexpected kubectl call: %s\\n" % args) sys.exit(2) @@ -145,13 +271,17 @@ def __init__(self) -> None: self.configmaps: dict[str, dict[str, Any]] = { f"{NAMESPACE}/{PROBE_CONFIGMAP}": {"data": {}}, f"{NAMESPACE}/{STATUS_CONFIGMAP}": {"data": {"status.json": ""}}, + f"{NAMESPACE}/{ACTUATOR_CONFIGMAP}": {"data": {}}, } + self.leases: dict[str, Any] = {} + self.snapshots: list[dict[str, Any]] = [] + self.runner_pods: dict[str, int] = {} def cleanup(self) -> None: self.tmp.cleanup() def write_store(self) -> None: - self.store_path.write_text(json.dumps({"nodes": self.nodes, "configmaps": self.configmaps})) + self.store_path.write_text(json.dumps({"nodes": self.nodes, "configmaps": self.configmaps, "leases": self.leases, "snapshots": self.snapshots, "runner_pods": self.runner_pods})) def read_store(self) -> dict[str, Any]: return json.loads(self.store_path.read_text()) @@ -174,13 +304,14 @@ def env(self, **extra: str) -> dict[str, str]: env.update(extra) return env - def run(self, script: str, **env: str) -> subprocess.CompletedProcess[str]: + def run(self, script: str, *args: str, **env: str) -> subprocess.CompletedProcess[str]: self.write_store() - completed = subprocess.run([str(SCRIPTS / script)], env=self.env(**env), capture_output=True, text=True, stdin=subprocess.DEVNULL, check=False) + completed = subprocess.run([str(SCRIPTS / script), *args], env=self.env(**env), capture_output=True, text=True, stdin=subprocess.DEVNULL, check=False) if self.store_path.exists(): store = self.read_store() self.nodes = store["nodes"] self.configmaps = store["configmaps"] + self.leases = store["leases"] return completed @@ -195,6 +326,11 @@ def run(self, script: str, **env: str) -> subprocess.CompletedProcess[str]: "FLEET_ROLE_PINNED_LABEL": PINNED_LABEL, "FLEET_ROLE_LEVEL_LABEL": LEVEL_LABEL, "FLEET_ROLE_STATE_NAMESPACE": NAMESPACE, + "FLEET_ROLE_OPERATION_TIMEOUT_SECONDS": str(OPERATION_TIMEOUT_SECONDS), + "FLEET_ROLE_RUNNER_WAIT_SECONDS": str(RUNNER_WAIT_SECONDS), + "FLEET_ROLE_LEASE_SECONDS": str(LEASE_SECONDS), + "FLEET_ROLE_ACTUATOR_CONFIGMAP": ACTUATOR_CONFIGMAP, + "FLEET_REGRADE_CORE": str(REGRADE_CORE), } diff --git a/tests/unit/test_docker_role.py b/tests/unit/test_docker_role.py index 2de31bdf..0cd2e0a6 100644 --- a/tests/unit/test_docker_role.py +++ b/tests/unit/test_docker_role.py @@ -273,6 +273,15 @@ def test_there_is_one_template_and_it_is_the_same_for_either_role(self) -> None: for line in ("K3S_JOIN_SERVER_URL=https://node1.example.internal:6443", "K3S_AGENT_SERVER_URL=https://node1.example.internal:6443", "K3S_CONTAINER_NAME=github-runner-k3s", "K3S_AGENT_CONTAINER_NAME=github-runner-k3s-agent"): self.assertIn(line, lines) + def test_the_bootstrap_hosts_agent_joins_the_first_other_server_since_it_has_no_bootstrap_host_to_dial(self) -> None: + # A bootstrap host has no server URL of its own, and a role switch makes it an agent that must join the cluster it left. + peers = ["https://node2.example.internal:6443", "https://node3.example.internal:6443"] + extra = ["-e", json.dumps({"github_runner_cluster_effective_server_url": "", "github_runner_cluster_topology": {"hosts": {"localhost": {"peer_server_urls": peers}}}})] + with tempfile.TemporaryDirectory() as work: + lines = self.render(Path(work), *extra).splitlines() + self.assertIn("K3S_JOIN_SERVER_URL=", lines, "the bootstrap server itself still joins nothing") + self.assertIn(f"K3S_AGENT_SERVER_URL={peers[0]}", lines) + @unittest.skipUnless(has_compose(), "docker compose is not installed") def test_each_service_renders_as_it_did_from_the_env_its_role_got_before(self) -> None: # Each service's configuration hash, which Compose compares with the running container's to decide whether to recreate it, is the same from the one .env as from the .env a host of its role got before there was one: a server's held every variable but the agent's own, and an agent's held its own (its server URL and node name, from the same settings as the server's) beside the shared ones and the variables Compose requires of every service whichever profile is active. So an upgrade recreates no running node. diff --git a/tests/unit/test_fleet_role.py b/tests/unit/test_fleet_role.py index 5034352b..da13f3c0 100644 --- a/tests/unit/test_fleet_role.py +++ b/tests/unit/test_fleet_role.py @@ -655,7 +655,7 @@ def test_inputs_the_size_of_a_full_configmap_are_read_whole(self) -> None: class MembershipTest(ControllerCase): def test_the_first_pass_records_the_voters_and_no_change(self) -> None: status = self.pass_(healthy()) - self.assertEqual(status["membership"], {"voters": ["a", "b", "c"], "changed_at": None, "cooldown_until": None}) + self.assertEqual(status["membership"], {"voters": ["a", "b", "c"], "known_members": {"a": 3, "b": 3, "c": 3}, "departed": [], "changed_at": None, "cooldown_until": None}) def test_a_pass_that_sees_the_same_voters_keeps_the_time_of_the_last_change(self) -> None: self.cluster.set_status({"membership": {"voters": ["a", "b", "c"], "changed_at": 1700000000, "cooldown_until": 1700000000 + COOLDOWN_SECONDS}}) @@ -697,7 +697,7 @@ def test_where_each_hosts_link_was_read_is_reported(self) -> None: self.assertEqual({name: host["link_source"] for name, host in status["hosts"].items()}, {"a": "label", "b": "interface", "c": "interface", "d": "interface"}) self.assertIn("a (voter, link wifi): fit", self.log) - def test_a_pass_writes_only_its_own_status_and_changes_nothing_in_the_cluster(self) -> None: + def test_a_dry_run_pass_with_no_swap_writes_only_its_own_status_and_changes_nothing_in_the_cluster(self) -> None: self.pass_(healthy(a={"tcp": UNFIT_RTT_MS + 50})) calls = [json.loads(line) for line in self.cluster.calls.read_text().splitlines()] writes = [call for call in calls if call[0] not in ("get",)] @@ -727,8 +727,16 @@ def test_the_largest_target_that_leaves_headroom_is_accepted(self) -> None: status = self.pass_(healthy(), FLEET_ROLE_VOTERS="5", FLEET_ROLE_MAX_SERVERS="6") self.assertEqual(status["config"]["target"], 5) - def test_a_pass_that_is_not_a_dry_run_is_refused(self) -> None: - self.refuses("FLEET_ROLE_DRY_RUN must be true", FLEET_ROLE_DRY_RUN="false") + def test_a_dry_run_setting_that_is_not_true_or_false_is_refused(self) -> None: + self.refuses("FLEET_ROLE_DRY_RUN must be true or false", FLEET_ROLE_DRY_RUN="maybe") + + def test_the_swap_settings_are_required_so_that_every_wait_derives_from_the_roles_own(self) -> None: + for name in ("FLEET_ROLE_OPERATION_TIMEOUT_SECONDS", "FLEET_ROLE_RUNNER_WAIT_SECONDS", "FLEET_ROLE_LEASE_SECONDS", "FLEET_ROLE_ACTUATOR_CONFIGMAP"): + with self.subTest(name=name): + populate(self.cluster, healthy()) + completed = self.cluster.run("fleet-role.sh", **{k: v for k, v in CONTROLLER_ENV.items() if k != name}) + self.assertNotEqual(completed.returncode, 0) + self.assertIn(f"{name} must be set", completed.stderr) def test_the_election_timeout_is_required_so_that_the_controller_and_the_probe_cannot_disagree_on_a_default(self) -> None: populate(self.cluster, healthy()) diff --git a/tests/unit/test_fleet_role_arc.py b/tests/unit/test_fleet_role_arc.py index b30718b0..0974defe 100644 --- a/tests/unit/test_fleet_role_arc.py +++ b/tests/unit/test_fleet_role_arc.py @@ -70,17 +70,57 @@ def test_each_of_the_controller_and_the_probe_has_its_own_account_and_the_worklo [("ServiceAccount", n) for n in ("fleet-role", "fleet-role-probe")] + [("ClusterRole", n) for n in ("github-runner-fleet-role", "github-runner-fleet-role-probe")] + [("ClusterRoleBinding", n) for n in ("github-runner-fleet-role", "github-runner-fleet-role-probe")] - + [("Role", n) for n in ("fleet-role", "fleet-role-probe")] - + [("RoleBinding", n) for n in ("fleet-role", "fleet-role-probe")] + + [("Role", n) for n in ("fleet-role", "fleet-role-probe", "github-runner-fleet-role-swap-lease", "github-runner-fleet-role-node-lease")] + + [("RoleBinding", n) for n in ("fleet-role", "fleet-role-probe", "github-runner-fleet-role-swap-lease", "github-runner-fleet-role-node-lease")] + [("Deployment", "fleet-role"), ("DaemonSet", "fleet-role-probe")] ), ) - def test_neither_can_write_a_node_and_both_can_only_read_them(self) -> None: - for role in of_kind(self.manifests, "ClusterRole"): - self.assertEqual(role["rules"], [{"apiGroups": [""], "resources": ["nodes"], "verbs": ["get", "list"]}], role["metadata"]["name"]) + def test_the_probe_can_only_read_nodes_and_the_controller_writes_only_what_a_swap_needs(self) -> None: + probe = of_kind(self.manifests, "ClusterRole", "github-runner-fleet-role-probe")[0] + self.assertEqual(probe["rules"], [{"apiGroups": [""], "resources": ["nodes"], "verbs": ["get", "list"]}]) + controller = of_kind(self.manifests, "ClusterRole", "github-runner-fleet-role")[0] + self.assertEqual( + controller["rules"], + [ + {"apiGroups": [""], "resources": ["nodes"], "verbs": ["get", "list", "patch"]}, + {"apiGroups": [""], "resources": ["pods"], "verbs": ["get", "list"]}, + {"apiGroups": [""], "resources": ["pods/eviction"], "verbs": ["create"]}, + {"apiGroups": ["apps"], "resources": ["daemonsets"], "verbs": ["get"]}, + {"apiGroups": ["k3s.cattle.io"], "resources": ["etcdsnapshotfiles"], "verbs": ["get", "list"]}, + ], + ) + for rule in controller["rules"]: + self.assertNotIn("delete", rule["verbs"]) + self.assertNotIn("update", rule["verbs"]) - def test_their_only_namespaced_permissions_are_on_their_two_configmaps_by_name(self) -> None: + def test_the_controller_can_create_a_lease_and_replace_only_the_regrade_lease_and_read_the_kubelets_leases(self) -> None: + swap = of_kind(self.manifests, "Role", "github-runner-fleet-role-swap-lease")[0] + self.assertEqual(swap["metadata"]["namespace"], "kube-system") + self.assertEqual( + swap["rules"], + [ + {"apiGroups": ["coordination.k8s.io"], "resources": ["leases"], "verbs": ["create"]}, + {"apiGroups": ["coordination.k8s.io"], "resources": ["leases"], "resourceNames": ["fleet-regrade"], "verbs": ["get", "update"]}, + ], + ) + kubelet = of_kind(self.manifests, "Role", "github-runner-fleet-role-node-lease")[0] + self.assertEqual(kubelet["metadata"]["namespace"], "kube-node-lease") + self.assertEqual(kubelet["rules"], [{"apiGroups": ["coordination.k8s.io"], "resources": ["leases"], "verbs": ["get"]}]) + for name in ("github-runner-fleet-role-swap-lease", "github-runner-fleet-role-node-lease"): + [binding] = of_kind(self.manifests, "RoleBinding", name) + self.assertEqual(binding["subjects"], [{"kind": "ServiceAccount", "name": "fleet-role", "namespace": "github-runner-platform"}]) + self.assertEqual(binding["roleRef"]["name"], name) + for probe_rule in [r for m in self.manifests if m["kind"] in ("Role", "ClusterRole") and "probe" in m["metadata"]["name"] for r in m["rules"]]: + self.assertNotIn("leases", probe_rule["resources"]) + + def test_the_regrade_lease_the_controller_may_replace_is_the_one_the_cluster_role_names(self) -> None: + cluster = yaml.safe_load((CLUSTER_ROLE / "defaults" / "main.yml").read_text()) + swap = of_kind(self.manifests, "Role", "github-runner-fleet-role-swap-lease")[0] + self.assertEqual(swap["metadata"]["namespace"], cluster["github_runner_cluster_regrade_lease_namespace"]) + self.assertEqual(swap["rules"][1]["resourceNames"], [cluster["github_runner_cluster_regrade_lease_name"]]) + + def test_their_namespaced_permissions_in_the_platform_namespace_are_on_configmaps_by_name(self) -> None: controller = of_kind(self.manifests, "Role", "fleet-role")[0]["rules"] probe = of_kind(self.manifests, "Role", "fleet-role-probe")[0]["rules"] self.assertEqual( @@ -88,6 +128,7 @@ def test_their_only_namespaced_permissions_are_on_their_two_configmaps_by_name(s [ {"apiGroups": [""], "resources": ["configmaps"], "resourceNames": ["fleet-role-probe"], "verbs": ["get"]}, {"apiGroups": [""], "resources": ["configmaps"], "resourceNames": ["fleet-role-status"], "verbs": ["get", "patch"]}, + {"apiGroups": [""], "resources": ["configmaps"], "resourceNames": ["fleet-role-actuators"], "verbs": ["get"]}, ], ) self.assertEqual(probe, [{"apiGroups": [""], "resources": ["configmaps"], "resourceNames": ["fleet-role-probe"], "verbs": ["get", "patch"]}]) @@ -102,10 +143,10 @@ def test_each_binding_grants_its_role_to_its_own_account(self) -> None: [subject] = binding["subjects"] self.assertEqual(subject["kind"], "ServiceAccount") self.assertIn(subject["name"], ("fleet-role", "fleet-role-probe")) - self.assertTrue(binding["roleRef"]["name"].endswith(subject["name"]), binding) + self.assertTrue(binding["roleRef"]["name"].endswith(subject["name"]) or "fleet-role-" in binding["roleRef"]["name"] and subject["name"] == "fleet-role", binding) self.assertEqual(subject["namespace"], "github-runner-platform") - def test_the_controller_runs_as_an_unprivileged_user_with_a_read_only_root_and_is_a_dry_run(self) -> None: + def test_the_controller_runs_as_an_unprivileged_user_with_a_read_only_root_and_defaults_to_a_dry_run(self) -> None: deployment = of_kind(self.manifests, "Deployment")[0] pod = deployment["spec"]["template"]["spec"] self.assertTrue(pod["securityContext"]["runAsNonRoot"]) @@ -143,9 +184,29 @@ def test_the_default_settings_reach_the_controller_as_strings(self) -> None: "FLEET_ROLE_LEVEL_LABEL": "github-runner.exadev.com/regrade-level", "FLEET_ROLE_PROBE_CONFIGMAP": "fleet-role-probe", "FLEET_ROLE_STATUS_CONFIGMAP": "fleet-role-status", + "FLEET_ROLE_ACTUATOR_CONFIGMAP": "fleet-role-actuators", + "FLEET_ROLE_OPERATION_TIMEOUT_SECONDS": "300", + "FLEET_ROLE_RUNNER_WAIT_SECONDS": "600", + "FLEET_ROLE_LEASE_SECONDS": "900", }, ) + def test_the_dry_run_is_off_only_when_the_owner_says_so(self) -> None: + self.assertEqual(env_of(of_kind(render(github_runner_arc_role_controller_dry_run=False), "Deployment")[0])["FLEET_ROLE_DRY_RUN"], "false") + self.assertEqual(env_of(of_kind(render(github_runner_arc_role_controller_dry_run=True), "Deployment")[0])["FLEET_ROLE_DRY_RUN"], "true") + + def test_the_cluster_roles_own_waits_reach_the_controller_when_they_are_defined(self) -> None: + env = env_of(of_kind(render( + github_runner_cluster_regrade_operation_timeout_seconds=120, + github_runner_cluster_native_runner_wait_timeout=240, + github_runner_cluster_regrade_lease_duration_seconds=360, + github_runner_cluster_role_actuator_configmap="example-actuators", + ), "Deployment")[0]) + self.assertEqual( + (env["FLEET_ROLE_OPERATION_TIMEOUT_SECONDS"], env["FLEET_ROLE_RUNNER_WAIT_SECONDS"], env["FLEET_ROLE_LEASE_SECONDS"], env["FLEET_ROLE_ACTUATOR_CONFIGMAP"]), + ("120", "240", "360", "example-actuators"), + ) + def test_the_cluster_roles_own_settings_reach_the_controller_when_they_are_defined(self) -> None: manifests = render( github_runner_cluster_max_servers=7, @@ -272,12 +333,12 @@ def test_every_script_the_manifests_run_is_copied_into_the_image_and_made_execut def test_every_jq_module_the_scripts_include_is_copied_beside_them(self) -> None: dockerfile = (ROOT / "fleet-role" / "Dockerfile").read_text() - for program in ("fleet-role.jq", "fleet-role-stats.jq"): + for program in ("fleet-role.jq", "fleet-role-swap.jq", "fleet-role-stats.jq"): for module in re.findall(r'^include "([A-Za-z0-9._-]+)";', (ROOT / "scripts" / program).read_text(), re.MULTILINE): with self.subTest(program=program, module=module): self.assertTrue((ROOT / "scripts" / f"{module}.jq").is_file()) self.assertRegex(dockerfile, rf"COPY [^\n]*scripts/{re.escape(module)}\.jq[^\n]* /app/scripts/") - for program in ("fleet-role.jq", "fleet-role-stats.jq"): + for program in ("fleet-role.jq", "fleet-role-swap.jq", "fleet-role-stats.jq"): self.assertRegex(dockerfile, rf"COPY [^\n]*scripts/{re.escape(program)}[^\n]* /app/scripts/") def test_the_probe_reads_the_link_reader_the_image_carries(self) -> None: @@ -358,6 +419,15 @@ def test_the_voter_target_and_the_two_voter_labels(self) -> None: self.assertEqual(self.fallback("eligible_label"), self.cluster["github_runner_cluster_voter_eligible_label"]) self.assertEqual(self.fallback("pinned_label"), self.cluster["github_runner_cluster_voter_pinned_label"]) + def test_the_swap_waits_and_the_actuators_configmap(self) -> None: + self.assertEqual(int(self.fallback("operation_timeout_seconds")), int(self.cluster["github_runner_cluster_regrade_pressure_window_seconds"])) + self.assertEqual(int(self.fallback("runner_wait_seconds")), self.cluster["github_runner_cluster_server_ready_timeout"]) + self.assertEqual(self.fallback("actuator_configmap"), self.cluster["github_runner_cluster_role_actuator_configmap"]) + self.assertEqual(self.fallback("lease_namespace"), self.cluster["github_runner_cluster_regrade_lease_namespace"]) + self.assertEqual(self.fallback("lease_name"), self.cluster["github_runner_cluster_regrade_lease_name"]) + self.assertIn("default(3 * (github_runner_arc_role_controller_operation_timeout_seconds | int))", self.arc) + self.assertEqual(self.cluster["github_runner_cluster_regrade_lease_duration_seconds"], "{{ 3 * (github_runner_cluster_regrade_operation_timeout_seconds | int) }}") + def test_the_link_label(self) -> None: self.assertEqual(self.fallback("link_label"), self.cluster["github_runner_cluster_link_label"]) diff --git a/tests/unit/test_fleet_role_swap.py b/tests/unit/test_fleet_role_swap.py new file mode 100644 index 00000000..3e9db384 --- /dev/null +++ b/tests/unit/test_fleet_role_swap.py @@ -0,0 +1,1023 @@ +"""Tests for the voter swap of scripts/fleet-role.sh and scripts/fleet-role-swap.jq: the controller's state machine, run against a scripted stand-in for kubectl that keeps Nodes, ConfigMaps and Leases and carries out what a pass writes, with the real bash and jq. Each test sets the cluster at one step of a swap (the Nodes' labels and annotations, the regrade Lease, the actuators' reports, the probes) and runs a pass, since the step a swap is at is read from those and nothing else.""" + +from __future__ import annotations + +import json +import re +import subprocess +import time +import unittest +from typing import Any + +from fleet_role_support import ( + ACTUATOR_CONFIGMAP, + CONTROLLER_ENV, + COOLDOWN_SECONDS, + LEASE_SECONDS, + LEVEL_LABEL, + NAMESPACE, + OPERATION_TIMEOUT_SECONDS, + ROOT, + RUNNER_WAIT_SECONDS, + SAMPLE_SECONDS, + SCRIPTS, + STATUS_CONFIGMAP, + TARGET, + UNFIT_RTT_MS, + VOTER_LABEL, + transition_time, +) +from test_fleet_role import ControllerCase, Host, healthy, populate + +DOMAIN = "github-runner.exadev.com/" +DESIRED = DOMAIN + "desired-role" +SWAP_ANNOTATION = DOMAIN + "role-swap" +STEP_ANNOTATION = DOMAIN + "role-step" +JOIN_ANNOTATION = DOMAIN + "role-join-node" +SNAPSHOT_ANNOTATION = DOMAIN + "snapshot-request" +MEMBER_GONE = DOMAIN + "role-member-gone" +LEASE_ID = DOMAIN + "swap-id" +LEASE_NODES = DOMAIN + "swap-nodes" +REMOVE = "etcd.k3s.cattle.io/remove" +REMOVED = "etcd.k3s.cattle.io/removed-node-name" +CONTROL_PLANE = "node-role.kubernetes.io/control-plane" +MASTER = "node-role.kubernetes.io/master" +LEASE_KEY = "kube-system/fleet-regrade" +LEARNER_STALL_SECONDS = 300 +OUT = "c" +IN = "d" +SNAPSHOT_FILE = "fleet-role-snapshot-a-1" + + +def iso(seconds_ago: float = 0) -> str: + return time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(time.time() - seconds_ago)) + ".000000Z" + + +def swap_id(out: str = OUT, in_: str = IN, seconds_ago: float = 0) -> str: + return time.strftime("%Y%m%dT%H%M%SZ", time.gmtime(time.time() - seconds_ago)) + f"-{out}-{in_}" + + +def lease(swap: str | None, *, holder: str = "fleet-role", renewed_ago: float = 0, nodes: str = f"{OUT},{IN}") -> dict[str, Any]: + annotations = {} if swap is None else {LEASE_ID: swap, LEASE_NODES: nodes} + return { + "apiVersion": "coordination.k8s.io/v1", + "kind": "Lease", + "metadata": {"name": "fleet-regrade", "namespace": "kube-system", "resourceVersion": "7", "annotations": annotations}, + "spec": {"holderIdentity": holder, "leaseDurationSeconds": LEASE_SECONDS, "acquireTime": iso(renewed_ago + 5), "renewTime": iso(renewed_ago)}, + } + + +def report(swap: str, step: str, phase: str = "done", **fields: Any) -> str: + return json.dumps({"swap": swap, "step": step, "phase": phase, "role": "agent", "started_at": "", "at": iso(), "message": "", "snapshot": "", **fields}) + + +class SwapCase(ControllerCase): + """A cluster of voters a, b and c and an eligible standby d, healthy; helpers that put it at a step of the swap of c for d.""" + + def setUp(self) -> None: + super().setUp() + self.hosts: list[Host] = healthy() + self.id = swap_id() + self.edits: dict[str, dict[str, Any]] = {} + self.completed: subprocess.CompletedProcess[str] | None = None + + # --- Building the cluster --- + + def voters(self, *names: str) -> None: + for host in self.hosts: + host.voter = host.name in names + + def edit(self, name: str, *, labels: dict[str, str | None] | None = None, annotations: dict[str, str | None] | None = None, cordoned: bool | None = None, taint: bool | None = None) -> None: + change = self.edits.setdefault(name, {"labels": {}, "annotations": {}}) + change["labels"].update(labels or {}) + change["annotations"].update(annotations or {}) + if cordoned is not None: + change["cordoned"] = cordoned + if taint is not None: + change["taint"] = taint + + def apply_edits(self) -> None: + for item in self.cluster.nodes: + change = self.edits.get(item["metadata"]["name"]) + if change is None: + continue + for section in ("labels", "annotations"): + target = item["metadata"].setdefault(section, {}) + for key, value in change[section].items(): + if value is None: + target.pop(key, None) + else: + target[key] = value + if "cordoned" in change: + item.setdefault("spec", {})["unschedulable"] = change["cordoned"] + if "taint" in change: + item.setdefault("spec", {})["taints"] = [{"key": CONTROL_PLANE, "value": "true", "effect": "NoSchedule"}] if change["taint"] else [] + + def hold(self, swap: str | None = None, **changes: Any) -> None: + self.cluster.leases[LEASE_KEY] = lease(self.id if swap is None else swap, **changes) + + def report(self, node: str, swap: str, step: str, phase: str = "done", **fields: Any) -> None: + self.cluster.configmaps[f"{NAMESPACE}/{ACTUATOR_CONFIGMAP}"]["data"][node] = report(swap, step, phase, **fields) + + def kubelet_lease(self, node: str, seconds_ago: float = 0) -> None: + self.cluster.leases[f"kube-node-lease/{node}"] = {"spec": {"renewTime": iso(seconds_ago)}} + + def previous(self, step: str, since_ago: float = 0, **fields: Any) -> None: + """The status the last pass published: the swap at this step since this long ago.""" + now = int(time.time()) + record = {"id": self.id, "out": OUT, "in": IN, "step": step, "started_at": now - 60, "step_since": now - int(since_ago), "deadline": now, "manual": True, "snapshot": None, "outgoing_pending": False, **fields} + self.cluster.set_status({"swap": record, "membership": {"voters": [h.name for h in self.hosts if h.voter], "changed_at": None, "cooldown_until": None}}) + + # --- Running --- + + def run_pass(self, *, repopulate: bool = True, **env: str) -> subprocess.CompletedProcess[str]: + if repopulate: + populate(self.cluster, self.hosts) + self.apply_edits() + self.completed = self.cluster.run("fleet-role.sh", **{**CONTROLLER_ENV, **env}) + return self.completed + + def ok_pass(self, **kwargs: Any) -> subprocess.CompletedProcess[str]: + completed = self.run_pass(**kwargs) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + return completed + + def command(self, *args: str, **env: str) -> subprocess.CompletedProcess[str]: + populate(self.cluster, self.hosts) + self.apply_edits() + return self.cluster.run("fleet-role.sh", *args, **{**CONTROLLER_ENV, **env}) + + # --- Reading the result --- + + def node(self, name: str) -> dict[str, Any]: + [found] = [n for n in self.cluster.nodes if n["metadata"]["name"] == name] + return found + + def annotations(self, name: str) -> dict[str, str]: + return self.node(name)["metadata"].get("annotations", {}) + + def labels(self, name: str) -> dict[str, str]: + return self.node(name)["metadata"].get("labels", {}) + + def cordoned(self, name: str) -> bool: + return bool(self.node(name).get("spec", {}).get("unschedulable")) + + def taints(self, name: str) -> list[dict[str, str]]: + return self.node(name).get("spec", {}).get("taints", []) + + def writes(self) -> list[list[str]]: + return [call for call in map(json.loads, self.cluster.calls.read_text().splitlines()) if call[0] not in ("get",)] + + def changes(self) -> list[list[str]]: + """What a pass changed in the cluster: every write but the renewal of the Lease it holds and the publishing of its own status.""" + return [w for w in self.writes() if w[0] != "replace" and w[:3] != ["patch", "configmap", "fleet-role-status"]] + + def held_by(self) -> str: + return self.cluster.leases[LEASE_KEY]["spec"]["holderIdentity"] + + def swap(self) -> dict[str, Any] | None: + return self.cluster.status()["swap"] + + def last_swap(self) -> dict[str, Any] | None: + return self.cluster.status()["last_swap"] + + def request_data(self) -> str | None: + return self.cluster.configmaps[f"{NAMESPACE}/{STATUS_CONFIGMAP}"]["data"].get("request.json") + + # --- Standard positions in a swap --- + + def at_snapshot_taken(self) -> None: + """The Lease is held, a snapshot was asked of a and is ready.""" + self.hold() + self.edit("a", annotations={SNAPSHOT_ANNOTATION: self.id}) + self.report("a", self.id, "snapshot", snapshot=SNAPSHOT_FILE) + self.cluster.snapshots = [{"spec": {"snapshotName": SNAPSHOT_FILE}, "status": {"readyToUse": True}}] + self.previous("drain") + + def at_promotion_asked(self, **fields: Any) -> None: + """The candidate is cordoned and has been asked to promote.""" + self.at_snapshot_taken() + self.edit(IN, labels={DESIRED: "server"}, annotations={SWAP_ANNOTATION: self.id, STEP_ANNOTATION: "promote", JOIN_ANNOTATION: "a"}, cordoned=True) + self.previous("promote", **fields) + + def at_candidate_voting(self) -> None: + """The candidate votes and is Ready on a heartbeat after its server started; it is still cordoned.""" + self.at_promotion_asked() + self.voters("a", "b", "c", "d") + started = iso(120) + self.report(IN, self.id, "promote", started_at=started) + self.kubelet_lease(IN) + + def at_candidate_uncordoned(self) -> None: + self.at_candidate_voting() + self.edit(IN, cordoned=False) + self.previous("gates") + + def at_step_down_asked(self) -> None: + self.at_candidate_uncordoned() + self.edit(OUT, labels={LEVEL_LABEL: "etcd-only", DESIRED: "agent"}, annotations={SWAP_ANNOTATION: self.id, STEP_ANNOTATION: "step-down"}, cordoned=True, taint=True) + self.previous("step-down") + + def at_removal_asked(self) -> None: + self.at_step_down_asked() + self.report(OUT, self.id, "step-down") + self.edit(OUT, annotations={REMOVE: "true"}) + self.previous("remove") + + def at_member_removed(self) -> None: + self.at_removal_asked() + self.edit(OUT, annotations={REMOVED: "c-1234", REMOVE: None}) + self.previous("finish") + + def at_demotion_finished(self) -> None: + self.at_member_removed() + self.report(OUT, self.id, "finish-demote", started_at=iso(60)) + + +class StartTest(SwapCase): + def test_a_manual_request_is_accepted_when_every_gate_holds_and_recorded_for_the_next_pass(self) -> None: + completed = self.command("swap", OUT, IN) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(json.loads(self.request_data() or "null")["out"], OUT) + self.assertEqual(json.loads(self.request_data() or "null")["in"], IN) + self.assertEqual(self.writes()[-1][:3], ["patch", "configmap", "fleet-role-status"]) + self.assertNotIn("fleet-regrade", json.dumps(self.cluster.leases), "accepting a request takes no Lease") + + def test_the_next_pass_takes_the_lease_for_the_swap_naming_its_nodes_and_clears_the_request(self) -> None: + self.command("swap", OUT, IN) + self.ok_pass(repopulate=False) + taken = self.cluster.leases[LEASE_KEY] + self.assertEqual(taken["spec"]["holderIdentity"], "fleet-role") + self.assertEqual(taken["spec"]["leaseDurationSeconds"], LEASE_SECONDS) + self.assertRegex(taken["metadata"]["annotations"][LEASE_ID], rf"^\d{{8}}T\d{{6}}Z-{OUT}-{IN}$") + self.assertEqual(taken["metadata"]["annotations"][LEASE_NODES], f"{OUT},{IN}") + self.assertEqual((self.swap() or {})["step"], "snapshot") + self.assertTrue((self.swap() or {})["manual"]) + self.assertIsNone(self.request_data()) + + def test_dry_run_carries_out_a_manual_request_and_reports_an_automatic_decision_only(self) -> None: + self.hosts = healthy(c={"tcp": UNFIT_RTT_MS + 50}) + self.ok_pass(FLEET_ROLE_DRY_RUN="true") + self.assertNotIn(LEASE_KEY, self.cluster.leases) + self.assertIn("DRY RUN: would swap out c for d", self.completed.stdout if self.completed else "") + self.assertEqual([w for w in self.writes() if w[0] != "patch"], []) + self.command("swap", OUT, IN, FLEET_ROLE_DRY_RUN="true") + self.ok_pass(repopulate=False, FLEET_ROLE_DRY_RUN="true") + self.assertIn(LEASE_KEY, self.cluster.leases) + + def test_with_the_dry_run_off_the_controller_starts_the_swap_it_decides_on(self) -> None: + self.hosts = healthy(c={"tcp": UNFIT_RTT_MS + 50}) + self.ok_pass(FLEET_ROLE_DRY_RUN="false") + self.assertEqual(self.held_by(), "fleet-role") + self.assertRegex(self.cluster.leases[LEASE_KEY]["metadata"]["annotations"][LEASE_ID], rf"-{OUT}-{IN}$") + self.assertFalse((self.swap() or {})["manual"]) + self.assertFalse(self.cluster.status()["dry_run"]) + + def test_no_second_swap_is_decided_inside_the_cooldown_after_a_membership_change(self) -> None: + self.hosts = healthy(c={"tcp": UNFIT_RTT_MS + 50}) + changed_at = int(time.time()) - (COOLDOWN_SECONDS - 60) + self.cluster.set_status({"membership": {"voters": ["a", "b", "c"], "changed_at": changed_at, "cooldown_until": changed_at + COOLDOWN_SECONDS}}) + self.ok_pass(FLEET_ROLE_DRY_RUN="false") + self.assertNotIn(LEASE_KEY, self.cluster.leases) + self.assertIsNone(self.swap()) + + def test_a_manual_request_is_carried_out_inside_the_cooldown_and_whatever_the_verdicts_say(self) -> None: + changed_at = int(time.time()) - 60 + self.cluster.set_status({"membership": {"voters": ["a", "b", "c"], "changed_at": changed_at, "cooldown_until": changed_at + COOLDOWN_SECONDS}}) + self.assertEqual(self.command("swap", OUT, IN).returncode, 0) + + def test_a_manual_request_is_refused_for_a_pinned_voter_naming_it(self) -> None: + self.hosts = healthy(c={"pinned": True}) + completed = self.command("swap", OUT, IN) + self.assertEqual(completed.returncode, 1) + self.assertIn("voter c is pinned", completed.stderr) + self.assertIsNone(self.request_data()) + + def test_a_manual_request_is_refused_for_nodes_that_are_unknown_not_eligible_or_the_wrong_kind(self) -> None: + self.hosts = healthy(d={"eligible": False}) + cases = { + ("c", "d"): "d is not marked eligible", + ("c", "zz"): "no node is named zz", + ("zz", "d"): "no node is named zz", + ("d", "c"): "d is not a voter", + ("a", "b"): "b is already a member of etcd", + ("c", "c"): "a swap needs two nodes", + } + for (out, in_), message in cases.items(): + with self.subTest(out=out, in_=in_): + completed = self.command("swap", out, in_) + self.assertEqual(completed.returncode, 2, completed.stderr) + self.assertIn(message, completed.stderr) + self.assertEqual(self.command("swap", OUT).returncode, 2) + + def test_a_manual_request_is_refused_with_each_reason_while_a_voter_is_paused(self) -> None: + self.hosts = healthy(b={"stale_by": 10 * SAMPLE_SECONDS}) + completed = self.command("swap", OUT, IN) + self.assertEqual(completed.returncode, 1) + self.assertIn("voter b has no probe report", completed.stderr) + self.assertIsNone(self.request_data()) + self.assertNotIn(LEASE_KEY, self.cluster.leases) + self.assertEqual(self.labels(IN).get(DESIRED), None) + + def test_a_manual_request_is_refused_while_the_regrade_lease_is_held_by_another_and_while_a_swap_runs(self) -> None: + self.cluster.leases[LEASE_KEY] = lease(None, holder="other-node") + completed = self.command("swap", OUT, IN) + self.assertEqual(completed.returncode, 1) + self.assertIn("the regrade Lease is held by other-node", completed.stderr) + self.hold() + completed = self.command("swap", OUT, IN) + self.assertEqual(completed.returncode, 1) + self.assertIn(f"a swap is in progress ({self.id})", completed.stderr) + + def test_a_request_that_stopped_being_valid_before_the_pass_is_dropped_and_recorded(self) -> None: + self.command("swap", OUT, IN) + self.hosts = healthy(b={"stale_by": 10 * SAMPLE_SECONDS}) + self.ok_pass() + self.assertIsNone(self.request_data()) + self.assertNotIn(LEASE_KEY, self.cluster.leases) + self.assertEqual((self.last_swap() or {})["outcome"], "abandoned") + self.assertIn("voter b has no probe report", " ".join((self.last_swap() or {})["reasons"])) + + def test_an_expired_lease_of_another_holder_is_taken_and_a_released_one_too(self) -> None: + for existing in (lease(None, holder="regrade-node", renewed_ago=LEASE_SECONDS + 5), lease(None, holder="", renewed_ago=10)): + with self.subTest(holder=existing["spec"]["holderIdentity"]): + self.cluster.leases[LEASE_KEY] = existing + self.command("swap", OUT, IN) + self.ok_pass(repopulate=False) + self.assertEqual(self.held_by(), "fleet-role") + self.cluster.leases.clear() + + +class SnapshotTest(SwapCase): + def test_a_snapshot_is_asked_of_a_remaining_voter_and_never_of_the_outgoing_one(self) -> None: + self.hold() + self.previous("snapshot") + self.ok_pass() + asked = [name for name in "abcd" if SNAPSHOT_ANNOTATION in self.annotations(name)] + self.assertEqual(asked, ["a"]) + self.assertEqual(self.annotations("a")[SNAPSHOT_ANNOTATION], self.id) + self.assertEqual(self.swap()["step"], "snapshot") + + def test_a_snapshot_is_never_asked_of_the_outgoing_voter_even_when_it_sorts_first(self) -> None: + self.id = swap_id("a", IN) + self.hold(nodes=f"a,{IN}") + self.previous("snapshot", out="a") + self.ok_pass() + self.assertEqual([name for name in "abcd" if SNAPSHOT_ANNOTATION in self.annotations(name)], ["b"]) + + def test_the_swap_waits_while_the_snapshot_is_not_ready_even_when_the_actuator_says_done(self) -> None: + self.at_snapshot_taken() + self.cluster.snapshots = [{"spec": {"snapshotName": SNAPSHOT_FILE}, "status": {"readyToUse": False}}] + self.previous("snapshot") + self.ok_pass() + self.assertEqual(self.changes(), [], "nothing but the status is written") + self.assertFalse(self.cordoned(IN)) + + def test_a_ready_snapshot_lets_the_candidate_be_cordoned(self) -> None: + self.at_snapshot_taken() + self.previous("snapshot") + self.ok_pass() + self.assertTrue(self.cordoned(IN)) + self.assertEqual(self.swap()["snapshot"], SNAPSHOT_FILE) + + def test_a_snapshot_that_is_not_ready_in_the_operation_timeout_abandons_the_swap_and_releases_the_lease(self) -> None: + self.hold() + self.edit("a", annotations={SNAPSHOT_ANNOTATION: self.id}) + self.previous("snapshot", since_ago=OPERATION_TIMEOUT_SECONDS + 5) + self.ok_pass() + self.assertEqual(self.held_by(), "") + self.assertEqual(self.last_swap()["outcome"], "abandoned") + self.assertIn(f"no snapshot was ready within {OPERATION_TIMEOUT_SECONDS} s", self.last_swap()["reasons"][0]) + self.assertNotIn(SNAPSHOT_ANNOTATION, self.annotations("a")) + self.assertIsNone(self.swap()) + + def test_one_second_inside_the_operation_timeout_the_swap_still_waits(self) -> None: + self.hold() + self.edit("a", annotations={SNAPSHOT_ANNOTATION: self.id}) + self.previous("snapshot", since_ago=OPERATION_TIMEOUT_SECONDS - 10) + self.ok_pass() + self.assertEqual(self.held_by(), "fleet-role") + self.assertEqual(self.swap()["step"], "snapshot") + + def test_a_snapshot_that_fails_on_one_voter_is_asked_of_the_next_and_when_none_can_the_swap_is_abandoned(self) -> None: + self.hold() + self.edit("a", annotations={SNAPSHOT_ANNOTATION: self.id}) + self.report("a", self.id, "snapshot", "failed", message="k3s etcd-snapshot save failed") + self.previous("snapshot") + self.ok_pass() + self.assertEqual(self.annotations("b")[SNAPSHOT_ANNOTATION], self.id) + self.report("b", self.id, "snapshot", "refused", message="not a running server") + self.ok_pass(repopulate=False) + self.assertEqual(self.last_swap()["outcome"], "abandoned") + self.assertIn("no voter could take a snapshot", self.last_swap()["reasons"][0]) + self.assertEqual(self.held_by(), "") + + def test_the_swap_does_not_start_to_empty_the_candidate_while_a_gate_fails(self) -> None: + self.at_snapshot_taken() + self.hosts = healthy(b={"stale_by": 10 * SAMPLE_SECONDS}) + self.previous("snapshot") + self.ok_pass() + self.assertFalse(self.cordoned(IN)) + self.assertEqual(self.held_by(), "fleet-role") + + +class CandidateTest(SwapCase): + def test_the_candidate_is_cordoned_and_while_runner_pods_run_on_it_the_swap_waits(self) -> None: + self.at_snapshot_taken() + self.edit(IN, cordoned=True) + self.cluster.runner_pods[IN] = 2 + self.previous("drain", since_ago=RUNNER_WAIT_SECONDS - 60) + self.ok_pass() + self.assertEqual(self.changes(), []) + self.assertNotIn(DESIRED, self.labels(IN)) + self.assertEqual(self.swap()["step"], "drain") + + def test_past_the_runner_wait_the_swap_is_abandoned_with_the_candidate_uncordoned_the_lease_released_and_no_cooldown(self) -> None: + self.at_snapshot_taken() + self.edit(IN, cordoned=True) + self.cluster.runner_pods[IN] = 2 + self.previous("drain", since_ago=RUNNER_WAIT_SECONDS + 5) + self.ok_pass() + self.assertFalse(self.cordoned(IN)) + self.assertEqual(self.held_by(), "") + self.assertEqual(self.last_swap()["outcome"], "abandoned") + self.assertIn(f"did not finish within {RUNNER_WAIT_SECONDS} s", self.last_swap()["reasons"][0]) + self.assertEqual(self.cluster.status()["membership"]["voters"], ["a", "b", "c"]) + self.assertNotIn(SNAPSHOT_ANNOTATION, self.annotations("a")) + + def test_with_no_runner_pods_left_the_candidate_is_drained_and_asked_to_promote_joining_a_current_voter(self) -> None: + self.at_snapshot_taken() + self.edit(IN, cordoned=True) + self.previous("drain") + self.ok_pass() + drains = [w for w in self.writes() if w[0] == "drain"] + self.assertEqual(len(drains), 1) + self.assertIn(f"--timeout={OPERATION_TIMEOUT_SECONDS}s", drains[0]) + self.assertEqual(self.labels(IN)[DESIRED], "server") + self.assertEqual(self.annotations(IN)[SWAP_ANNOTATION], self.id) + self.assertEqual(self.annotations(IN)[STEP_ANNOTATION], "promote") + self.assertEqual(self.annotations(IN)[JOIN_ANNOTATION], "a") + + def test_a_drain_that_fails_asks_for_nothing_and_exits_non_zero_so_the_next_pass_repeats_it(self) -> None: + self.at_snapshot_taken() + self.edit(IN, cordoned=True) + self.previous("drain") + completed = self.run_pass(FAKE_DRAIN_FAILS=IN) + self.assertNotEqual(completed.returncode, 0) + self.assertNotIn(DESIRED, self.labels(IN)) + self.assertEqual(self.swap()["id"], self.id, "the swap stays at its step") + self.assertEqual(self.held_by(), "fleet-role") + + def test_a_drain_that_never_succeeds_abandons_the_swap_after_the_runner_wait_and_the_operation_timeout(self) -> None: + self.at_snapshot_taken() + self.edit(IN, cordoned=True) + self.previous("drain", since_ago=RUNNER_WAIT_SECONDS + OPERATION_TIMEOUT_SECONDS + 5) + self.ok_pass() + self.assertEqual(self.last_swap()["outcome"], "abandoned") + self.assertIn("could not be drained", self.last_swap()["reasons"][0]) + self.assertFalse(self.cordoned(IN)) + + def test_the_promotion_waits_for_a_voter_that_is_ready_on_a_heartbeat_after_the_server_started(self) -> None: + self.at_candidate_voting() + self.kubelet_lease(IN, seconds_ago=3600) + self.previous("promote") + self.ok_pass() + self.assertTrue(self.cordoned(IN)) + self.assertEqual(self.changes(), []) + + def test_a_voting_ready_candidate_with_a_fresh_heartbeat_is_uncordoned(self) -> None: + self.at_candidate_voting() + self.previous("promote") + self.ok_pass() + self.assertFalse(self.cordoned(IN)) + + def test_a_candidate_that_is_not_a_voter_by_the_deadline_is_rolled_back_asking_k3s_to_remove_its_learner_when_it_joined(self) -> None: + self.at_promotion_asked() + self.edit(IN, labels={VOTER_LABEL: "true"}, annotations={"etcd.k3s.cattle.io/node-name": "d-1"}) + self.previous("promote", since_ago=LEARNER_STALL_SECONDS + OPERATION_TIMEOUT_SECONDS + 5) + self.ok_pass() + self.assertEqual(self.annotations(IN)[REMOVE], "true") + self.assertEqual(self.labels(IN)[DESIRED], "agent") + self.assertEqual(self.annotations(IN)[STEP_ANNOTATION], "rollback") + self.assertEqual(self.annotations(IN)[SWAP_ANNOTATION], self.id) + + def test_a_candidate_that_never_joined_is_rolled_back_without_asking_for_a_removal(self) -> None: + self.at_promotion_asked() + self.previous("promote", since_ago=LEARNER_STALL_SECONDS + OPERATION_TIMEOUT_SECONDS + 5) + self.ok_pass() + self.assertNotIn(REMOVE, self.annotations(IN)) + self.assertEqual(self.annotations(IN)[STEP_ANNOTATION], "rollback") + + def test_one_second_inside_the_deadline_the_promotion_is_still_waited_for(self) -> None: + self.at_promotion_asked() + self.previous("promote", since_ago=LEARNER_STALL_SECONDS + OPERATION_TIMEOUT_SECONDS - 20) + self.ok_pass() + self.assertEqual(self.annotations(IN)[STEP_ANNOTATION], "promote") + + def test_an_actuator_that_reports_the_promotion_failed_starts_the_rollback_at_once(self) -> None: + self.at_promotion_asked() + self.report(IN, self.id, "promote", "failed", message="the server unit did not start") + self.ok_pass() + self.assertEqual(self.annotations(IN)[STEP_ANNOTATION], "rollback") + + def test_an_actuator_that_refuses_the_promotion_abandons_the_swap_since_nothing_changed_on_the_host(self) -> None: + self.at_promotion_asked() + self.report(IN, self.id, "promote", "refused", message="no server configuration") + self.ok_pass() + self.assertEqual(self.last_swap()["outcome"], "abandoned") + self.assertIn("no server configuration", self.last_swap()["reasons"][0]) + self.assertFalse(self.cordoned(IN)) + self.assertEqual(self.held_by(), "") + self.assertNotIn(SWAP_ANNOTATION, self.annotations(IN)) + + +class RollbackTest(SwapCase): + def at_rollback_done(self, **edits: Any) -> None: + self.at_promotion_asked() + self.edit(IN, labels={VOTER_LABEL: "true", CONTROL_PLANE: "true", MASTER: "true", DESIRED: "agent"}, + annotations={STEP_ANNOTATION: "rollback", "etcd.k3s.cattle.io/node-name": "d-1", "etcd.k3s.cattle.io/node-address": "100.64.0.4", REMOVE: "true"}, taint=True) + self.report(IN, self.id, "rollback") + self.previous("rollback") + + def test_a_rollback_that_the_actuator_has_not_finished_is_waited_for(self) -> None: + self.at_rollback_done() + self.report(IN, self.id, "rollback", "running") + self.ok_pass() + self.assertEqual(self.changes(), []) + + def test_a_finished_rollback_waits_until_every_voter_lists_the_target_members_and_no_more(self) -> None: + self.at_rollback_done() + for host in self.hosts: + host.etcd = {"known_members": TARGET + 1} + self.ok_pass() + self.assertEqual(self.changes(), []) + self.assertIn("waiting for every voter to list 3 members", " ".join(self.cluster.status()["waiting"])) + + def test_a_finished_rollback_with_membership_as_it_was_clears_the_servers_labels_annotations_and_taint_and_uncordons(self) -> None: + self.at_rollback_done() + self.voters("a", "b", "c") + self.ok_pass() + self.assertFalse(self.cordoned(IN)) + for key in (VOTER_LABEL, CONTROL_PLANE, MASTER): + self.assertNotIn(key, self.labels(IN)) + self.assertEqual([k for k in self.annotations(IN) if k.startswith("etcd.k3s.cattle.io/")], []) + self.assertEqual(self.taints(IN), []) + self.assertEqual(self.held_by(), "fleet-role", "the lease is held until the swap ends") + + def test_after_the_clean_up_the_rollback_ends_with_the_lease_released_and_the_swap_annotations_cleared(self) -> None: + self.at_rollback_done() + self.voters("a", "b", "c") + self.ok_pass() + self.ok_pass(repopulate=False) + self.assertEqual(self.last_swap()["outcome"], "rolled-back") + self.assertEqual(self.held_by(), "") + self.assertEqual([k for k in self.annotations(IN) if k.startswith(DOMAIN)], []) + self.assertEqual(self.labels(IN)[DESIRED], "agent") + self.assertIsNone(self.swap()) + + def test_a_rollback_the_actuator_reports_failed_is_held_for_the_owner_and_says_why(self) -> None: + self.at_rollback_done() + self.report(IN, self.id, "rollback", "failed", message="the datastore could not be moved aside") + self.ok_pass() + self.assertEqual(self.held_by(), "fleet-role") + self.assertIn("the datastore could not be moved aside", self.completed.stdout if self.completed else "") + + +class OutgoingTest(SwapCase): + def test_with_the_candidate_voting_and_the_gates_holding_the_outgoing_voter_is_taken_out_starting_with_its_cordon_and_taint(self) -> None: + self.at_candidate_uncordoned() + self.cluster.runner_pods[OUT] = 1 + self.ok_pass() + self.assertTrue(self.cordoned(OUT)) + self.assertEqual([t["key"] for t in self.taints(OUT)], [CONTROL_PLANE]) + self.assertNotIn(STEP_ANNOTATION, self.annotations(OUT), "the runner pod is waited for before the drain") + + def test_past_the_runner_wait_the_outgoing_voter_is_drained_labelled_etcd_only_and_asked_to_step_down(self) -> None: + self.at_candidate_uncordoned() + self.edit(OUT, cordoned=True, taint=True) + self.cluster.runner_pods[OUT] = 1 + self.previous("evict", since_ago=RUNNER_WAIT_SECONDS + 5) + self.ok_pass() + self.assertEqual([w[1] for w in self.writes() if w[0] == "drain"], [OUT]) + self.assertEqual(self.labels(OUT)[LEVEL_LABEL], "etcd-only") + self.assertEqual(self.labels(OUT)[DESIRED], "agent") + self.assertEqual(self.annotations(OUT)[STEP_ANNOTATION], "step-down") + self.assertEqual(self.annotations(OUT)[SWAP_ANNOTATION], self.id) + + def test_the_gates_are_held_with_four_voters_and_the_outgoing_voter_is_not_touched_while_one_fails(self) -> None: + self.at_candidate_uncordoned() + self.hosts[0].etcd = {"active_peers": 1} + self.ok_pass() + self.assertFalse(self.cordoned(OUT)) + self.assertEqual(self.changes(), []) + + def test_a_down_outgoing_voter_is_cordoned_and_marked_without_a_drain_and_the_gates_ignore_its_health(self) -> None: + self.at_candidate_uncordoned() + self.hosts[2].ready = False + self.hosts[2].stale_by = 10 * SAMPLE_SECONDS + for host in self.hosts[:2] + self.hosts[3:]: + host.etcd = {"active_peers": 2} + self.ok_pass() + self.assertEqual([w for w in self.writes() if w[0] == "drain"], []) + self.assertEqual(self.annotations(OUT)[STEP_ANNOTATION], "step-down") + self.assertEqual(self.labels(OUT)[LEVEL_LABEL], "etcd-only") + + def test_the_removal_is_asked_once_the_actuator_reports_the_step_down_done(self) -> None: + self.at_step_down_asked() + self.report(OUT, self.id, "step-down") + self.hosts[2].level = "etcd-only" + self.ok_pass() + self.assertEqual(self.annotations(OUT)[REMOVE], "true") + + def test_a_stepped_down_member_whose_node_no_longer_says_it_votes_is_still_counted_as_a_member_and_removed(self) -> None: + # A server at etcd-only runs no kubelet, so its Node stops saying EtcdIsVoter while etcd still lists the member: the gates count it, ignore its health and let the removal go on. + self.at_step_down_asked() + self.report(OUT, self.id, "step-down") + self.hosts[2].voter = False + self.hosts[2].learner = True + self.hosts[2].level = "etcd-only" + self.hosts[2].ready = False + self.hosts[2].stale_by = 10 * SAMPLE_SECONDS + for host in self.hosts[:2] + self.hosts[3:]: + host.etcd = {"known_members": TARGET + 1, "active_peers": TARGET - 1} + self.ok_pass() + self.assertEqual(self.annotations(OUT)[REMOVE], "true") + self.assertEqual(self.cluster.status()["gates"], []) + + def test_the_removal_waits_for_the_step_down_while_the_outgoing_voter_is_ready_and_has_not_reported(self) -> None: + self.at_step_down_asked() + self.hosts[2].level = "etcd-only" + self.previous("step-down", since_ago=OPERATION_TIMEOUT_SECONDS + 5) + self.ok_pass() + self.assertNotIn(REMOVE, self.annotations(OUT)) + + def test_a_step_down_reported_failed_on_a_node_that_stays_ready_is_held_for_the_owner(self) -> None: + self.at_step_down_asked() + self.hosts[2].level = "etcd-only" + self.report(OUT, self.id, "step-down", "failed", message="the server did not come back at etcd-only") + self.ok_pass() + self.assertNotIn(REMOVE, self.annotations(OUT)) + self.assertIn("held for the owner", self.completed.stdout if self.completed else "") + + def test_an_outgoing_voter_that_is_down_and_has_not_reported_in_the_operation_timeout_is_removed_while_the_others_are_healthy(self) -> None: + self.at_step_down_asked() + self.hosts[2].ready = False + self.hosts[2].stale_by = 10 * SAMPLE_SECONDS + self.hosts[2].level = "etcd-only" + for host in self.hosts[:2] + self.hosts[3:]: + host.etcd = {"active_peers": 2} + self.previous("step-down", since_ago=OPERATION_TIMEOUT_SECONDS + 5) + self.ok_pass() + self.assertEqual(self.annotations(OUT)[REMOVE], "true") + + def test_a_down_outgoing_voter_is_not_removed_before_the_operation_timeout_has_passed(self) -> None: + self.at_step_down_asked() + self.hosts[2].ready = False + self.hosts[2].stale_by = 10 * SAMPLE_SECONDS + for host in self.hosts[:2] + self.hosts[3:]: + host.etcd = {"active_peers": 2} + self.previous("step-down", since_ago=OPERATION_TIMEOUT_SECONDS - 30) + self.ok_pass() + self.assertNotIn(REMOVE, self.annotations(OUT)) + + def test_the_removal_is_held_while_a_remaining_voter_is_unhealthy(self) -> None: + self.at_step_down_asked() + self.report(OUT, self.id, "step-down") + self.hosts[2].level = "etcd-only" + self.hosts[1].stale_by = 10 * SAMPLE_SECONDS + self.ok_pass() + self.assertNotIn(REMOVE, self.annotations(OUT)) + self.assertIn("not removing c while the remaining voters are not all healthy", self.completed.stdout if self.completed else "") + + def test_a_removal_k3s_rejected_and_cleared_is_asked_for_again_while_the_others_are_healthy_and_not_otherwise(self) -> None: + self.at_step_down_asked() + self.report(OUT, self.id, "step-down") + self.hosts[2].level = "etcd-only" + self.previous("remove", since_ago=OPERATION_TIMEOUT_SECONDS + 5) + self.hosts[1].stale_by = 10 * SAMPLE_SECONDS + self.ok_pass() + self.assertNotIn(REMOVE, self.annotations(OUT)) + self.hosts[1].stale_by = 0 + self.ok_pass() + self.assertEqual(self.annotations(OUT)[REMOVE], "true") + + def test_a_removal_that_is_asked_for_and_not_yet_done_is_waited_for_without_another_write(self) -> None: + self.at_removal_asked() + self.hosts[2].level = "etcd-only" + self.ok_pass() + self.assertEqual(self.changes(), []) + + def test_once_k3s_has_recorded_the_removal_the_controller_waits_for_the_actuators_finish(self) -> None: + self.at_member_removed() + self.ok_pass() + self.assertEqual(self.changes(), []) + self.assertTrue(self.labels(OUT).get(VOTER_LABEL)) + + def test_a_demotion_the_actuator_finished_is_cleaned_up_the_role_labels_etcd_annotations_level_label_and_taint(self) -> None: + self.at_demotion_finished() + self.edit(OUT, labels={MASTER: "true", CONTROL_PLANE: "true"}, annotations={"etcd.k3s.cattle.io/node-name": "c-1", "etcd.k3s.cattle.io/node-address": "100.64.0.3"}) + self.ok_pass() + for key in (VOTER_LABEL, MASTER, CONTROL_PLANE, LEVEL_LABEL): + self.assertNotIn(key, self.labels(OUT)) + self.assertEqual([k for k in self.annotations(OUT) if k.startswith("etcd.k3s.cattle.io/")], []) + self.assertEqual(self.taints(OUT), []) + self.assertTrue(self.cordoned(OUT), "it is uncordoned only once it is Ready as an agent") + self.assertEqual(self.labels(OUT)[DESIRED], "agent", "the record of the role the host runs stays") + self.assertEqual(self.annotations(OUT)[SWAP_ANNOTATION], self.id) + + def test_the_outgoing_node_is_uncordoned_once_ready_on_a_heartbeat_after_the_agent_started(self) -> None: + self.at_demotion_finished() + self.edit(OUT, labels={VOTER_LABEL: None, MASTER: None, CONTROL_PLANE: None, LEVEL_LABEL: None}, annotations={REMOVED: None}, taint=False) + self.cluster.leases.pop("kube-system/fleet-regrade", None) + self.hold() + self.kubelet_lease(OUT, seconds_ago=3600) + self.hosts[2].voter = False + self.previous("cleanup") + self.ok_pass() + self.assertTrue(self.cordoned(OUT)) + self.kubelet_lease(OUT) + self.ok_pass(repopulate=False) + self.assertFalse(self.cordoned(OUT)) + + def test_the_swap_ends_completed_the_annotations_cleared_the_lease_released(self) -> None: + self.at_demotion_finished() + self.edit(OUT, labels={VOTER_LABEL: None, MASTER: None, CONTROL_PLANE: None, LEVEL_LABEL: None}, annotations={REMOVED: None, MEMBER_GONE: self.id}, cordoned=False, taint=False) + self.hosts[2].voter = False + self.previous("cleanup") + self.ok_pass() + self.assertEqual(self.last_swap()["outcome"], "completed") + self.assertEqual(self.held_by(), "") + for name in "abcd": + self.assertEqual([k for k in self.annotations(name) if k.startswith(DOMAIN)], [], name) + self.assertIsNone(self.swap()) + self.assertNotIn(LEVEL_LABEL, self.labels(OUT)) + + def test_a_host_that_does_not_come_back_in_the_operation_timeout_is_left_to_node_recovery_with_the_swap_recorded_pending(self) -> None: + self.at_demotion_finished() + self.edit(OUT, labels={VOTER_LABEL: None, MASTER: None, CONTROL_PLANE: None, LEVEL_LABEL: None}, annotations={REMOVED: None}, taint=False) + self.hosts[2].voter = False + self.hosts[2].ready = False + self.previous("cleanup", since_ago=OPERATION_TIMEOUT_SECONDS + 5) + self.ok_pass() + self.assertTrue(self.swap()["outgoing_pending"]) + self.ok_pass(repopulate=False) + self.assertEqual(self.last_swap()["outcome"], "completed") + self.assertTrue(self.last_swap()["outgoing_pending"]) + self.assertEqual(self.held_by(), "") + self.assertFalse(self.cordoned(OUT), "a node that is down is not left cordoned for when it returns") + + def test_an_actuator_that_does_not_finish_the_demotion_in_the_operation_timeout_does_not_hold_the_swap_for_ever(self) -> None: + self.at_member_removed() + self.previous("finish", since_ago=OPERATION_TIMEOUT_SECONDS + 5) + self.ok_pass() + self.assertNotIn(VOTER_LABEL, self.labels(OUT)) + self.assertTrue(self.swap()["outgoing_pending"]) + + def at_removed_with_its_server_still_running(self) -> None: + """The outgoing voter's step-down failed while it was not Ready and k3s removed its member, but the server its role file still names runs on: it has set the etcd label again, and the removed-node-name annotation is gone (k3s clears it when the server joins again under a new name, and the controller's clean-up takes it off at the finish's deadline). The voters that remain each report that etcd lists three members.""" + self.at_member_removed() + self.report(OUT, self.id, "step-down", "failed", message="the server did not come back at etcd-only") + self.edit(OUT, annotations={REMOVED: None}) + self.hosts[2].voter = False + self.hosts[2].learner = True + self.hosts[2].level = "etcd-only" + self.hosts[2].ready = False + self.hosts[2].stale_by = 10 * SAMPLE_SECONDS + + def test_a_member_etcd_no_longer_lists_is_finished_not_removed_again_though_its_node_says_it_is_one(self) -> None: + self.at_removed_with_its_server_still_running() + self.ok_pass() + self.assertNotIn(REMOVE, self.annotations(OUT)) + status = self.cluster.status() + self.assertEqual(status["gates"], []) + self.assertEqual(status["membership"]["departed"], [OUT]) + self.assertEqual(self.swap()["step"], "finish") + # The actuator is told, under this swap, that etcd lists none of its members, and the next pass waits for its finish with nothing more written. + self.assertEqual(self.annotations(OUT)[MEMBER_GONE], self.id) + self.ok_pass(repopulate=False) + self.assertEqual(len([w for w in self.changes() if f"{MEMBER_GONE}={self.id}" in w]), 1) + self.assertIn(f"waiting for {OUT}'s actuator to finish its demotion", self.completed.stdout if self.completed else "") + + def test_a_member_k3s_recorded_removed_is_marked_gone_once_every_remaining_voter_lists_only_the_voters(self) -> None: + # k3s's record on the Node can name another member than the datastore's, so the actuator is told what etcd itself shows. + self.at_member_removed() + self.hosts[2].voter = False + self.hosts[0].etcd = {"known_members": TARGET + 1} + self.ok_pass() + self.assertNotIn(MEMBER_GONE, self.annotations(OUT), "a remaining voter that lists the outgoing member is no proof") + for host in self.hosts[:2] + self.hosts[3:]: + host.etcd = {"known_members": TARGET, "active_peers": TARGET - 1} + self.ok_pass() + self.assertEqual(self.annotations(OUT)[MEMBER_GONE], self.id) + + def assert_still_counted_as_a_member(self) -> None: + self.ok_pass() + status = self.cluster.status() + self.assertEqual(status["membership"]["departed"], []) + self.assertNotEqual(status["gates"], []) + self.assertNotIn(REMOVE, self.annotations(OUT), "nothing is removed while a remaining voter is not healthy") + self.assertNotEqual(self.swap()["step"], "finish") + + def test_a_member_is_counted_as_one_while_a_remaining_voter_still_lists_it(self) -> None: + self.at_removed_with_its_server_still_running() + self.hosts[0].etcd = {"known_members": TARGET + 1} + self.assert_still_counted_as_a_member() + + def test_a_member_is_counted_as_one_while_a_remaining_voter_has_not_reported(self) -> None: + self.at_removed_with_its_server_still_running() + self.hosts[0].stale_by = 10 * SAMPLE_SECONDS + self.assert_still_counted_as_a_member() + + def test_a_departed_member_whose_server_never_finishes_ends_the_swap_completed_and_left_to_node_recovery(self) -> None: + self.at_removed_with_its_server_still_running() + self.edit(OUT, annotations={MEMBER_GONE: self.id}) + self.previous("finish", since_ago=OPERATION_TIMEOUT_SECONDS + 5) + self.ok_pass() + self.assertTrue(self.swap()["outgoing_pending"]) + # The server sets its etcd label again after the clean-up, so the Node is never clean; the swap ends rather than cleaning it for ever. + self.previous("finish", since_ago=OPERATION_TIMEOUT_SECONDS + 5, outgoing_pending=True) + self.ok_pass() + self.assertEqual(self.last_swap()["outcome"], "completed") + self.assertTrue(self.last_swap()["outgoing_pending"]) + self.assertEqual(self.held_by(), "") + self.assertIsNone(self.swap()) + + def test_the_removed_member_is_no_voter_so_the_membership_the_controller_publishes_has_three_voters(self) -> None: + self.at_member_removed() + self.ok_pass() + self.assertEqual(self.cluster.status()["membership"]["voters"], ["a", "b", "d"]) + + +class ResumeTest(SwapCase): + """A controller pod that is rescheduled has the Lease, the Nodes and the reports and no status of its own, and takes the step the swap is at.""" + + positions = ( + ("at_snapshot_taken", "the candidate is cordoned", lambda self: self.assertTrue(self.cordoned(IN))), + ("at_promotion_asked", None, None), + ("at_candidate_voting", "the candidate is uncordoned", lambda self: self.assertFalse(self.cordoned(IN))), + ("at_candidate_uncordoned", "the outgoing voter is cordoned", lambda self: self.assertTrue(self.cordoned(OUT))), + ("at_step_down_asked", None, None), + ("at_removal_asked", None, None), + ("at_demotion_finished", "the outgoing node is cleaned", lambda self: self.assertNotIn(VOTER_LABEL, self.labels(OUT))), + ) + + def test_a_pass_with_no_status_takes_the_step_the_nodes_and_reports_show_at_every_step(self) -> None: + for builder, description, check in self.positions: + with self.subTest(step=builder): + self.setUp() + getattr(self, builder)() + self.cluster.set_status(None) + self.ok_pass() + self.assertEqual(self.held_by(), "fleet-role") + self.assertEqual(self.cluster.status()["swap"]["id"], self.id) + if check is not None: + check(self) + + def test_the_swap_is_known_from_its_annotations_alone_when_the_lease_and_the_status_are_both_gone(self) -> None: + self.at_step_down_asked() + self.cluster.set_status(None) + self.cluster.leases.clear() + self.ok_pass() + self.assertEqual(self.held_by(), "fleet-role") + self.assertEqual(self.cluster.leases[LEASE_KEY]["metadata"]["annotations"][LEASE_ID], self.id) + self.assertEqual(self.cluster.leases[LEASE_KEY]["metadata"]["annotations"][LEASE_NODES], f"{OUT},{IN}") + + def test_the_lease_is_taken_again_with_the_swaps_annotations_when_it_expired_and_nobody_took_it_the_pass_renews_it(self) -> None: + self.at_step_down_asked() + self.hold(renewed_ago=LEASE_SECONDS + 100) + self.ok_pass() + renewed = self.cluster.leases[LEASE_KEY] + self.assertEqual(renewed["spec"]["holderIdentity"], "fleet-role") + self.assertEqual(renewed["metadata"]["annotations"][LEASE_ID], self.id) + self.assertLess(time.time() - time.mktime(time.strptime(renewed["spec"]["renewTime"][:19], "%Y-%m-%dT%H:%M:%S")) + time.timezone, 30) + + def test_a_swap_that_ended_with_the_annotations_cleared_before_the_lease_was_released_is_ended_not_started_again(self) -> None: + self.at_demotion_finished() + self.edit(OUT, labels={VOTER_LABEL: None, MASTER: None, CONTROL_PLANE: None, LEVEL_LABEL: None}, annotations={REMOVED: None, SWAP_ANNOTATION: None, STEP_ANNOTATION: None}, cordoned=False, taint=False) + self.edit(IN, annotations={SWAP_ANNOTATION: None, STEP_ANNOTATION: None, JOIN_ANNOTATION: None}) + self.hosts[2].voter = False + self.cluster.set_status(None) + self.ok_pass() + self.assertEqual(self.held_by(), "") + self.assertEqual(self.cluster.status()["last_swap"]["outcome"], "completed") + self.assertEqual([w for w in self.writes() if f"{SNAPSHOT_ANNOTATION}=" in " ".join(w)], [], "no snapshot is asked for again") + + +class LeaseTest(SwapCase): + def test_every_pass_renews_the_lease_the_swap_holds_and_keeps_its_annotations(self) -> None: + self.at_step_down_asked() + self.hold(renewed_ago=LEASE_SECONDS // 2) + before = self.cluster.leases[LEASE_KEY]["spec"]["renewTime"] + self.ok_pass() + after = self.cluster.leases[LEASE_KEY] + self.assertGreater(after["spec"]["renewTime"], before) + self.assertEqual(after["metadata"]["annotations"][LEASE_ID], self.id) + self.assertEqual(after["spec"]["acquireTime"], self.cluster.leases[LEASE_KEY]["spec"]["acquireTime"]) + + def test_a_pass_with_no_swap_renews_and_takes_nothing(self) -> None: + self.ok_pass() + self.assertNotIn(LEASE_KEY, self.cluster.leases) + self.assertEqual([w for w in self.writes() if w[0] in ("create", "replace")], []) + + def test_a_lease_that_regrade_took_mid_swap_is_never_acted_under_and_never_renewed(self) -> None: + self.at_step_down_asked() + self.cluster.leases[LEASE_KEY] = lease(None, holder="some-server") + self.cluster.leases[LEASE_KEY]["metadata"]["annotations"] = {} + self.hosts[2].level = "etcd-only" + self.report(OUT, self.id, "step-down") + self.ok_pass() + self.assertEqual(self.held_by(), "some-server") + self.assertEqual(self.changes(), [], "only the status is written") + self.assertNotIn(REMOVE, self.annotations(OUT)) + self.assertEqual(self.cluster.status()["swap"]["id"], self.id) + self.assertIn("the regrade Lease is held by some-server", self.completed.stdout if self.completed else "") + + def test_when_the_lease_that_regrade_took_has_expired_the_swap_takes_it_again_and_goes_on(self) -> None: + self.at_removal_asked() + self.cluster.leases[LEASE_KEY] = lease(None, holder="some-server", renewed_ago=LEASE_SECONDS + 10) + self.hosts[2].level = "etcd-only" + self.ok_pass() + self.assertEqual(self.held_by(), "fleet-role") + self.assertEqual(self.cluster.leases[LEASE_KEY]["metadata"]["annotations"][LEASE_ID], self.id) + + def test_an_even_count_is_finished_before_any_new_decision_whatever_the_dry_run_says(self) -> None: + self.at_candidate_uncordoned() + self.hosts[0].tcp = UNFIT_RTT_MS + 50 + self.ok_pass(FLEET_ROLE_DRY_RUN="true") + self.assertTrue(self.cordoned(OUT), "the swap in progress goes on, with four voters, in a dry run") + self.assertEqual(self.cluster.status()["swap"]["id"], self.id) + self.assertNotIn("DRY RUN", self.completed.stdout if self.completed else "") + + def test_an_action_that_fails_is_repeated_and_leaves_the_swap_where_it_was(self) -> None: + self.at_candidate_uncordoned() + completed = self.run_pass(FAKE_FAIL_CALLS=f"cordon {OUT}") + self.assertNotEqual(completed.returncode, 0) + self.assertFalse(self.cordoned(OUT)) + self.assertEqual(self.held_by(), "fleet-role") + self.assertEqual(self.cluster.status()["swap"]["id"], self.id) + self.ok_pass(repopulate=False) + self.assertTrue(self.cordoned(OUT)) + + +class StatusTest(SwapCase): + def test_the_status_command_says_what_the_swap_is_doing(self) -> None: + self.at_step_down_asked() + self.ok_pass() + completed = self.cluster.run("fleet-role.sh", "status", **CONTROLLER_ENV) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertIn(f"swap {self.id}: c out, d in, step ", completed.stdout) + self.assertIn("no request pending", completed.stdout) + + def test_the_status_carries_the_swap_the_last_swap_the_request_and_the_real_dry_run(self) -> None: + self.ok_pass(FLEET_ROLE_DRY_RUN="false") + status = self.cluster.status() + for key in ("swap", "last_swap", "request", "dry_run"): + self.assertIn(key, status) + self.assertFalse(status["dry_run"]) + + def test_the_swap_record_names_its_step_start_and_deadline_from_the_settings(self) -> None: + self.at_step_down_asked() + self.previous("step-down", since_ago=30) + self.ok_pass() + record = self.swap() + self.assertEqual(record["step"], "step-down") + self.assertEqual(record["deadline"] - record["step_since"], OPERATION_TIMEOUT_SECONDS) + self.assertEqual(record["started_at"], int(time.mktime(time.strptime(self.id[:16], "%Y%m%dT%H%M%SZ"))) - time.timezone) + self.assertEqual(set(record) >= {"id", "out", "in", "step", "started_at", "deadline", "manual", "snapshot"}, True) + + def test_the_step_since_is_kept_while_the_step_is_unchanged_and_reset_when_it_changes(self) -> None: + self.at_step_down_asked() + self.previous("step-down", since_ago=100) + before = json.loads(self.cluster.configmaps[f"{NAMESPACE}/{STATUS_CONFIGMAP}"]["data"]["status.json"])["swap"]["step_since"] + self.ok_pass() + self.assertEqual(self.swap()["step_since"], before) + self.previous("remove", since_ago=100) + self.ok_pass(repopulate=False) + self.assertGreater(self.swap()["step_since"], before) + + +class InterfaceTest(unittest.TestCase): + def test_the_names_the_controller_writes_are_the_ones_the_actuator_reads(self) -> None: + interface = json.loads(subprocess.run(["jq", "-n", "-c", "-L", str(SCRIPTS), 'include "fleet-role-swap"; interface'], capture_output=True, text=True, check=True).stdout) + actuator = (ROOT / "roles" / "github_runner_cluster" / "files" / "role" / "fleet-role-actuator.sh").read_text() + constants = dict(re.findall(r"^([A-Z_]+_(?:LABEL|ANNOTATION))=(\S+)$", actuator, re.MULTILINE)) + self.assertEqual(interface["desired_role_label"], constants["DESIRED_ROLE_LABEL"]) + self.assertEqual(interface["swap_annotation"], constants["SWAP_ANNOTATION"]) + self.assertEqual(interface["step_annotation"], constants["STEP_ANNOTATION"]) + self.assertEqual(interface["join_node_annotation"], constants["JOIN_NODE_ANNOTATION"]) + self.assertEqual(interface["snapshot_annotation"], constants["SNAPSHOT_ANNOTATION"]) + self.assertEqual(interface["member_gone_annotation"], constants["MEMBER_GONE_ANNOTATION"]) + self.assertEqual(interface["etcd_label"], constants["ETCD_LABEL"]) + self.assertEqual(interface["removed_annotation"], re.search(r"^REMOVED_ANNOTATION=(\S+)$", actuator, re.MULTILINE).group(1)) # type: ignore[union-attr] + cluster = (ROOT / "roles" / "github_runner_cluster" / "defaults" / "main.yml").read_text() + self.assertIn(f"github_runner_cluster_role_lease_holder: {interface['lease_holder']}", cluster) + self.assertIn(f"github_runner_cluster_role_lease_swap_id_annotation: {interface['lease_swap_id_annotation']}", cluster) + + def test_the_actuators_valid_swap_id_shape_is_the_one_the_controller_makes(self) -> None: + result = subprocess.run(["jq", "-n", "-r", "-L", str(SCRIPTS), 'include "fleet-role-swap"; swap_id("a-b"; "c"; 1767225600)'], capture_output=True, text=True, check=True).stdout.strip() + self.assertEqual(result, "20260101T000000Z-a-b-c") + self.assertRegex(result, r"^[0-9]{8}T[0-9]{6}Z-[a-z0-9]([-a-z0-9.]*[a-z0-9])?$") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_role_actuator.py b/tests/unit/test_role_actuator.py index 83ec886f..d4e7f03f 100644 --- a/tests/unit/test_role_actuator.py +++ b/tests/unit/test_role_actuator.py @@ -374,12 +374,60 @@ def test_finishing_keeps_only_the_newest_datastore_copy(self) -> None: self.assertEqual(older.read_text(), f"{NODE}-0a1b2c3d") self.assertEqual(sorted(path.name for path in (self.data / "server").iterdir()), ["db.demoted"]) - def test_finishing_a_host_whose_role_file_names_the_server_is_refused(self) -> None: + def test_finishing_a_host_whose_role_file_names_the_server_and_whose_datastore_has_no_tombstone_is_refused(self) -> None: self.running_server() - (self.data / "server" / "db" / "etcd" / "tombstone").write_text("") result = self.actuator("finish-demote") self.assert_status(result, REFUSED) + self.assertIn("holds no tombstone", result.stdout) self.assertTrue((self.data / "server" / "db").is_dir()) + self.assertTrue(self.active("k3s")) + + def asked_to_step_down(self, annotations: dict[str, str]) -> None: + self.set_node({"github-runner.exadev.com/desired-role": "agent", "node-role.kubernetes.io/etcd": "true"}, {"github-runner.exadev.com/role-swap": SWAP, "github-runner.exadev.com/role-step": "step-down", **annotations}) + + def test_a_server_whose_step_down_failed_is_finished_as_an_agent_once_k3s_records_the_members_removal(self) -> None: + # The step-down failed before the role file named the agent, and the controller removed the member of a host that was not Ready; without the finish, the server unit's restarts would join etcd again as a new member. + self.running_server(level="etcd-only") + self.asked_to_step_down({"etcd.k3s.cattle.io/removed-node-name": f"{NODE}-0a1b2c3d"}) + self.assert_status(self.actuator("finish-demote"), DONE) + self.assertEqual(self.role_file.read_text(), "agent") + self.assertFalse(self.active("k3s")) + self.assertTrue(self.active("k3s-agent")) + self.assertTrue((self.data / "server" / "db.demoted").is_dir()) + self.assert_never_both_running() + + def test_a_demotion_is_finished_on_the_controllers_word_for_this_swap_that_etcd_lists_no_member_of_the_host(self) -> None: + # k3s's record names another member than the datastore's, as it does once it has been cleared and rewritten. + self.stepped_down() + self.asked_to_step_down({"etcd.k3s.cattle.io/removed-node-name": "another-member", "github-runner.exadev.com/role-member-gone": SWAP}) + self.assert_status(self.actuator("finish-demote"), DONE) + self.assert_finished() + + def test_the_controllers_word_for_another_swap_is_no_proof_that_the_member_is_gone(self) -> None: + self.stepped_down() + self.asked_to_step_down({"github-runner.exadev.com/role-member-gone": "20260101T000000Z-other-node"}) + self.assert_status(self.actuator("finish-demote"), NOT_YET) + self.assertTrue(self.active("k3s")) + self.assertTrue((self.data / "server" / "db").is_dir()) + + def test_a_server_asked_to_step_down_is_not_finished_while_its_member_stays(self) -> None: + self.running_server(level="etcd-only") + self.asked_to_step_down({"etcd.k3s.cattle.io/removed-node-name": "another-member"}) + self.assert_status(self.actuator("finish-demote"), NOT_YET) + self.assertEqual(self.role_file.read_text(), "server") + self.assertTrue(self.active("k3s")) + self.assertTrue((self.data / "server" / "db").is_dir()) + + def test_a_server_host_that_was_down_while_removed_and_holds_the_tombstone_is_finished_as_an_agent_on_its_return(self) -> None: + self.running_server() + (self.data / "server" / "db" / "etcd" / "tombstone").write_text("") + self.assert_status(self.actuator("finish-demote"), DONE) + self.assertEqual(self.role_file.read_text(), "agent") + self.assertFalse(self.active("k3s")) + self.assertTrue(self.active("k3s-agent")) + self.assertTrue((self.data / "server" / "db.demoted").is_dir()) + self.assertFalse((self.data / "server" / "db").exists()) + self.assert_never_both_running() class RollbackAndSnapshotTest(ActuatorHarness): @@ -569,6 +617,34 @@ def test_a_report_the_api_refused_to_take_is_made_again_once_it_takes_it_and_the self.wait_for_report("finish-demote", "done") self.stay_at("finish-demote", "done") + def test_the_watcher_finishes_a_server_whose_role_file_still_names_the_server_once_its_datastore_holds_the_tombstone(self) -> None: + # A host that was down while it was stepped down and removed: its role file was never written, no swap is on its Node any more and k3s's tombstone is all it has. + self.running_server() + self.watch() + time.sleep(3 * WATCH_POLL_SECONDS) + self.assertTrue(self.active("k3s"), "a running server with no tombstone is left alone") + (self.data / "server" / "db" / "etcd" / "tombstone").write_text("") + self.wait_for_report("finish-demote", "done") + self.assertEqual(self.role_file.read_text(), "agent") + self.assertTrue(self.active("k3s-agent")) + self.assertFalse(self.active("k3s")) + self.assertTrue((self.data / "server" / "db.demoted").is_dir()) + + def test_the_watcher_finishes_a_server_whose_step_down_failed_once_its_member_is_removed(self) -> None: + self.running_server(level="etcd-only") + step_down = {"github-runner.exadev.com/role-swap": SWAP, "github-runner.exadev.com/role-step": "step-down"} + self.set_node({"github-runner.exadev.com/desired-role": "agent", "node-role.kubernetes.io/etcd": "true"}, step_down) + self.seed_report(step="step-down", phase="failed", role="server") + self.watch() + time.sleep(3 * WATCH_POLL_SECONDS) + self.assertTrue(self.active("k3s"), "a server whose member stays is left alone") + self.set_node({"github-runner.exadev.com/desired-role": "agent", "node-role.kubernetes.io/etcd": "true"}, {**step_down, "etcd.k3s.cattle.io/removed-node-name": f"{NODE}-0a1b2c3d"}) + self.wait_for_report("finish-demote", "done") + self.assertEqual(self.role_file.read_text(), "agent") + self.assertTrue(self.active("k3s-agent")) + self.assertFalse(self.active("k3s")) + self.assert_never_both_running() + def test_a_step_that_is_not_a_demotion_gets_no_finish_from_the_watcher(self) -> None: self.running_agent() self.set_node({"github-runner.exadev.com/desired-role": "agent"}, {"github-runner.exadev.com/role-swap": SWAP, "github-runner.exadev.com/role-step": "rollback"}) @@ -817,6 +893,19 @@ def test_a_demotion_is_finished_by_moving_the_identity_back_and_starting_the_age self.assert_clean_switch(result) self.assertEqual({key: self.report()[key] for key in ("step", "phase", "role")}, {"step": "finish-demote", "phase": "done", "role": "agent"}) + def test_a_finish_taken_again_reports_the_start_of_the_agent_it_finds_running(self) -> None: + # A Docker host's first report of the finish is often lost, since its agent has no Tailscale yet, and the watch takes the finish again: the agent already runs, and the report must carry the time it was started, which the controller waits for a later heartbeat than before it uncordons the node. + self.running_as("server", level="etcd-only") + self.role_file.write_text("agent") + (self.data / "server" / "db" / "etcd" / "tombstone").write_text("") + (self.project / "fleet-role.d" / "join-url").write_text(JOIN_URL + "\n") + self.assert_status(self.actuator("finish-demote"), DONE) + first = self.report()["started_at"] + self.assertRegex(first, r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") + self.assert_status(self.actuator("finish-demote"), DONE) + self.assertEqual(self.container_state(AGENT_CONTAINER), "running") + self.assertEqual(self.report()["started_at"], first) + def test_a_demotion_is_not_finished_while_the_member_stays(self) -> None: self.running_as("server", level="etcd-only") self.role_file.write_text("agent")