From 55cc719f2cbe3d2cfa5118ed87473d086a30c55e Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 5 Nov 2026 14:38:54 +0000 Subject: [PATCH 1/5] test: compare published recovery and renewal images --- .github/workflows/platform-image-canary.yml | 66 +++++++++++++++++++++ tests/node_recovery/run.sh | 11 ++-- tests/pull_secret_renewal/run.sh | 10 ++-- 3 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/platform-image-canary.yml diff --git a/.github/workflows/platform-image-canary.yml b/.github/workflows/platform-image-canary.yml new file mode 100644 index 00000000..8bec45c1 --- /dev/null +++ b/.github/workflows/platform-image-canary.yml @@ -0,0 +1,66 @@ +name: Platform image canary + +on: + workflow_dispatch: + inputs: + component: + description: Component to compare + required: true + type: choice + options: + - node-recovery + - pull-secret-renewer + old_image: + description: Previously published image pinned by digest + required: true + type: string + new_image: + description: Candidate published image pinned by digest + required: true + type: string + +permissions: + contents: read + +jobs: + canary: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Validate image references + env: + COMPONENT: ${{ inputs.component }} + OLD_IMAGE: ${{ inputs.old_image }} + NEW_IMAGE: ${{ inputs.new_image }} + run: | + set -euo pipefail + for image in "$OLD_IMAGE" "$NEW_IMAGE"; do + [[ "$image" =~ ^ghcr\.io/exadev/github-runner-${COMPONENT}@sha256:[0-9a-f]{64}$ ]] || { echo "Invalid image reference: $image" >&2; exit 1; } + done + [[ "$OLD_IMAGE" != "$NEW_IMAGE" ]] || { echo 'Images must differ' >&2; exit 1; } + - name: Install Ansible + run: | + set -euo pipefail + python3 -m venv "${RUNNER_TEMP}/venv" + "${RUNNER_TEMP}/venv/bin/pip" install --quiet ansible-core + echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" + - name: Compare published images + env: + COMPONENT: ${{ inputs.component }} + OLD_IMAGE: ${{ inputs.old_image }} + NEW_IMAGE: ${{ inputs.new_image }} + run: | + set -euo pipefail + for image in "$OLD_IMAGE" "$NEW_IMAGE"; do + for run in 1 2; do + echo "Testing ${image}, run ${run}" + case "$COMPONENT" in + node-recovery) GRTEST_NODE_RECOVERY_IMAGE="$image" tests/node_recovery/run.sh ;; + pull-secret-renewer) GRTEST_RENEWER_IMAGE="$image" tests/pull_secret_renewal/run.sh ;; + *) exit 1 ;; + esac + done + done diff --git a/tests/node_recovery/run.sh b/tests/node_recovery/run.sh index cd8af528..09001de3 100755 --- a/tests/node_recovery/run.sh +++ b/tests/node_recovery/run.sh @@ -11,7 +11,7 @@ ansible_playbook="${ANSIBLE_PLAYBOOK:-ansible-playbook}" cluster=grtest-recovery platform_namespace=github-runner-platform namespace=grtest-listeners -image=grtest/node-recovery:test +image="${GRTEST_NODE_RECOVERY_IMAGE:-grtest/node-recovery:test}" pod_image=busybox:1.36 watcher_node="${cluster}-worker" doomed_node="${cluster}-worker2" @@ -153,9 +153,12 @@ server_version="$(kubectl version -o json | jq -r '.serverVersion.gitVersion')" log "Server version ${server_version}" doomed_uid="$(kubectl get node "$doomed_node" -o jsonpath='{.metadata.uid}')" -log "Building and loading the watcher image and the stand-in's image" -docker build -q -f "$repo_root/node-recovery/Dockerfile" -t "$image" "$repo_root" >/dev/null -kind load docker-image "$image" --name "$cluster" >/dev/null +if [ -z "${GRTEST_NODE_RECOVERY_IMAGE:-}" ]; then + log "Building and loading the watcher image" + docker build -q -f "$repo_root/node-recovery/Dockerfile" -t "$image" "$repo_root" >/dev/null + kind load docker-image "$image" --name "$cluster" >/dev/null +fi +log "Loading the stand-in's image" # The stand-in's image is pulled by each node's own runtime: a host Docker on the containerd image store pulls a multi-platform index holding only the host platform's blobs, and kind's import of it with all platforms fails on the missing ones. for node in $(kind get nodes --name "$cluster"); do docker exec "$node" crictl pull "$pod_image" >/dev/null diff --git a/tests/pull_secret_renewal/run.sh b/tests/pull_secret_renewal/run.sh index c5615152..a55245db 100755 --- a/tests/pull_secret_renewal/run.sh +++ b/tests/pull_secret_renewal/run.sh @@ -8,7 +8,7 @@ repo_root="$(cd "$(dirname "$0")/../.." && pwd)" ansible_playbook="${ANSIBLE_PLAYBOOK:-ansible-playbook}" cluster=grtest-renewal namespace=grtest-runners -image=grtest/pull-secret-renewer:test +image="${GRTEST_RENEWER_IMAGE:-grtest/pull-secret-renewer:test}" stub_image=python:3.13-alpine pull_secret=ghcr-pull renewer="${pull_secret}-renewer" @@ -63,9 +63,11 @@ run_job() { log "Creating kind cluster $cluster" kind create cluster --name "$cluster" --kubeconfig "$KUBECONFIG" --wait 120s >/dev/null -log "Building and loading the renewer image" -docker build -q -f "$repo_root/pull-secret-renewer/Dockerfile" -t "$image" "$repo_root" >/dev/null -kind load docker-image "$image" --name "$cluster" >/dev/null +if [ -z "${GRTEST_RENEWER_IMAGE:-}" ]; then + log "Building and loading the renewer image" + docker build -q -f "$repo_root/pull-secret-renewer/Dockerfile" -t "$image" "$repo_root" >/dev/null + kind load docker-image "$image" --name "$cluster" >/dev/null +fi log "Starting the GitHub stand-in" kubectl create namespace "$namespace" >/dev/null From 7e1ff4f54a86ae5c64d424114a3b1e5a2d9b3d0c Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 5 Nov 2026 14:40:28 +0000 Subject: [PATCH 2/5] test: exercise published autoscaler in kind --- .github/workflows/platform-image-canary.yml | 13 ++++++++++++- tests/autoscaler_ceiling/run.sh | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/platform-image-canary.yml b/.github/workflows/platform-image-canary.yml index 8bec45c1..2b160888 100644 --- a/.github/workflows/platform-image-canary.yml +++ b/.github/workflows/platform-image-canary.yml @@ -8,6 +8,7 @@ on: required: true type: choice options: + - autoscaler - node-recovery - pull-secret-renewer old_image: @@ -45,8 +46,17 @@ jobs: run: | set -euo pipefail python3 -m venv "${RUNNER_TEMP}/venv" - "${RUNNER_TEMP}/venv/bin/pip" install --quiet ansible-core + "${RUNNER_TEMP}/venv/bin/pip" install --quiet ansible-core pyyaml jinja2 echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" + - name: Install Helm + if: inputs.component == 'autoscaler' + run: | + set -euo pipefail + archive=helm-v4.3.0-linux-amd64.tar.gz + curl -fsSLo "${RUNNER_TEMP}/${archive}" "https://get.helm.sh/${archive}" + echo "86584a54def73570558f66f5111cc53dfed56689637ae32c1201205d494f54fb ${RUNNER_TEMP}/${archive}" | sha256sum -c - + tar -xzf "${RUNNER_TEMP}/${archive}" -C "${RUNNER_TEMP}" + echo "${RUNNER_TEMP}/linux-amd64" >> "$GITHUB_PATH" - name: Compare published images env: COMPONENT: ${{ inputs.component }} @@ -58,6 +68,7 @@ jobs: for run in 1 2; do echo "Testing ${image}, run ${run}" case "$COMPONENT" in + autoscaler) GRTEST_AUTOSCALER_IMAGE="$image" PYTHON="${RUNNER_TEMP}/venv/bin/python" tests/autoscaler_ceiling/run.sh ;; node-recovery) GRTEST_NODE_RECOVERY_IMAGE="$image" tests/node_recovery/run.sh ;; pull-secret-renewer) GRTEST_RENEWER_IMAGE="$image" tests/pull_secret_renewal/run.sh ;; *) exit 1 ;; diff --git a/tests/autoscaler_ceiling/run.sh b/tests/autoscaler_ceiling/run.sh index 10cc0673..1c6fb256 100755 --- a/tests/autoscaler_ceiling/run.sh +++ b/tests/autoscaler_ceiling/run.sh @@ -166,6 +166,20 @@ for namespace in "${pooled_namespaces[@]}"; do targets="$targets $namespace/grte # One poll, for real (not dry run), its output kept for the checks and for a failure's report. poll() { + if [ -n "${GRTEST_AUTOSCALER_IMAGE:-}" ]; then + kubectl -n "$platform_namespace" delete pod autoscaler-poll --ignore-not-found --wait=true >/dev/null + kubectl -n "$platform_namespace" run autoscaler-poll --image="$GRTEST_AUTOSCALER_IMAGE" --restart=Never \ + --overrides "$(jq -n --arg targets "${targets# }" --arg ns "$platform_namespace" --arg selector "${eligibility_label}=true" '{spec:{serviceAccountName:"autoscaler",containers:[{name:"autoscaler-poll",image:env.GRTEST_AUTOSCALER_IMAGE,command:["/app/scripts/autoscaler.sh"],env:[{name:"AUTOSCALER_TARGETS",value:$targets},{name:"AUTOSCALER_DRY_RUN",value:"false"},{name:"AUTOSCALER_USABLE_BUDGET_GI",value:"1024"},{name:"AUTOSCALER_FLOOR",value:"1"},{name:"AUTOSCALER_NODE_SELECTOR",value:$selector},{name:"AUTOSCALER_STATE_NAMESPACE",value:$ns},{name:"AUTOSCALER_STATUS_CONFIGMAP",value:"autoscaler-status"}]}]}}')" >/dev/null + local phase + for _ in {1..120}; do + phase="$(kubectl -n "$platform_namespace" get pod autoscaler-poll -o jsonpath='{.status.phase}')" + case "$phase" in Succeeded|Failed) break ;; esac + sleep 2 + done + kubectl -n "$platform_namespace" logs autoscaler-poll > "$work/poll.log" || fail "could not read poll logs" + [ "$phase" = Succeeded ] || fail "the poll pod ended in $phase" + return + fi PATH="$work/bin:$PATH" \ AUTOSCALER_TARGETS="${targets# }" \ AUTOSCALER_DRY_RUN=false \ From 2a30f287fe040a3e215ae82bfb2b1c8cfbfe3406 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 5 Nov 2026 14:43:44 +0000 Subject: [PATCH 3/5] feat: allow heartbeat API endpoint injection --- scripts/heartbeat.sh | 5 +++-- tests/unit/test_heartbeat.py | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/heartbeat.sh b/scripts/heartbeat.sh index b8dee073..559d35c2 100755 --- a/scripts/heartbeat.sh +++ b/scripts/heartbeat.sh @@ -21,6 +21,7 @@ set -euo pipefail HEARTBEAT_GH_TOKEN="${HEARTBEAT_GH_TOKEN:?HEARTBEAT_GH_TOKEN must be set (a PAT with gist scope)}" HEARTBEAT_GIST_ID="${HEARTBEAT_GIST_ID:?HEARTBEAT_GIST_ID must be set}" +HEARTBEAT_GITHUB_API_URL="${HEARTBEAT_GITHUB_API_URL:-https://api.github.com}" GIST_FILE="${GIST_FILE:-arc-healthy-until}" HEALTH_GIST_FILE="${HEALTH_GIST_FILE:-fleet-health.json}" JOB_PEAK_SUMMARY_GIST_FILE="${HEARTBEAT_JOB_PEAK_SUMMARY_GIST_FILE:-job-peak-summary.json}" @@ -101,7 +102,7 @@ reasons_json="$(printf '%s\n' "${reasons[@]}" | jq -R . | jq -sc 'map(select(len # When each problem began, so a reader (scripts/fleet-alert.sh) can tell an outage that has lasted from a blip without a clock of its own: a problem still present keeps the onset the last tick published, and a new one starts now. The last tick's document is read back from the gist; if that read fails every onset restarts at now, which delays an alert and never raises a false one. at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" previous_health="$(curl -fsS -H "Authorization: Bearer ${HEARTBEAT_GH_TOKEN}" -H "Accept: application/vnd.github+json" \ - "https://api.github.com/gists/${HEARTBEAT_GIST_ID}" 2>/dev/null \ + "${HEARTBEAT_GITHUB_API_URL}/gists/${HEARTBEAT_GIST_ID}" 2>/dev/null \ | jq -c --arg file "$HEALTH_GIST_FILE" '.files[$file].content // empty | fromjson' 2>/dev/null || true)" # A server node is one carrying the control-plane role label, which k3s sets on every server. A node that is not Ready is listed whether or not it is a server, but only a server's absence endangers etcd quorum and the API, which is why the flag is published. A server that regrading has stepped down carries the level label and is not Ready on purpose (at etcd-only it runs no kubelet), so its level is published beside it, for a reader to tell it from a server that has failed. @@ -157,7 +158,7 @@ curl -fsS -X PATCH \ -H "Authorization: Bearer ${HEARTBEAT_GH_TOKEN}" \ -H "Accept: application/vnd.github+json" \ -d "$body" \ - "https://api.github.com/gists/${HEARTBEAT_GIST_ID}" >/dev/null + "${HEARTBEAT_GITHUB_API_URL}/gists/${HEARTBEAT_GIST_ID}" >/dev/null if [ "$healthy" != "true" ]; then echo "Unhealthy (${reasons[*]}) - heartbeat gist reason updated, timestamp not refreshed" >&2 diff --git a/tests/unit/test_heartbeat.py b/tests/unit/test_heartbeat.py index 36258b2e..90c9cea8 100644 --- a/tests/unit/test_heartbeat.py +++ b/tests/unit/test_heartbeat.py @@ -57,6 +57,8 @@ #!/usr/bin/env python3 import json, os, sys args = sys.argv[1:] + with open(os.environ["FAKE_CURL_BODY"] + ".urls", "a") as urls: + urls.write(args[-1] + "\\n") if "-d" in args: open(os.environ["FAKE_CURL_BODY"], "w").write(args[args.index("-d") + 1]) sys.exit(0) @@ -138,6 +140,11 @@ def assertWithheld(self, files: dict[str, str]) -> None: self.assertFalse(health["healthy"]) self.assertTrue(health["reasons"]) + def test_gist_api_endpoint_is_configurable(self) -> None: + result, _ = self.tick(env={"HEARTBEAT_GITHUB_API_URL": "http://example.test:8080"}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((self.dir / "curl-body.json.urls").read_text().splitlines(), ["http://example.test:8080/gists/test-gist"] * 2) + def test_a_healthy_fleet_refreshes_the_gist(self) -> None: result, files = self.tick() self.assertEqual(result.returncode, 0, result.stderr) From adf4549e8630177c59938f75e051fe24121f68b4 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 5 Nov 2026 14:44:08 +0000 Subject: [PATCH 4/5] test: check autoscaler pod security parity --- tests/autoscaler_ceiling/run.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/autoscaler_ceiling/run.sh b/tests/autoscaler_ceiling/run.sh index 1c6fb256..e0a754b1 100755 --- a/tests/autoscaler_ceiling/run.sh +++ b/tests/autoscaler_ceiling/run.sh @@ -176,6 +176,18 @@ poll() { case "$phase" in Succeeded|Failed) break ;; esac sleep 2 done + kubectl -n "$platform_namespace" get pod autoscaler-poll -o json > "$work/autoscaler-pod.json" + "$python" - "$repo_root" "$work/autoscaler-pod.json" <<'PY' +import json, pathlib, sys, yaml +root = pathlib.Path(sys.argv[1]) +tasks = yaml.safe_load((root / 'roles/github_runner_arc/tasks/install_platform.yml').read_text()) +task = next(t for t in tasks if t.get('name') == 'Platform: install/upgrade the autoscaler Deployment') +expected = task['kubernetes.core.k8s']['definition']['spec']['template']['spec'] +actual = json.loads(pathlib.Path(sys.argv[2]).read_text())['spec'] +assert actual['serviceAccountName'] == expected['serviceAccountName'] +assert actual.get('securityContext', {}) == expected.get('securityContext', {}) +assert actual['containers'][0].get('securityContext', {}) == expected['containers'][0].get('securityContext', {}) +PY kubectl -n "$platform_namespace" logs autoscaler-poll > "$work/poll.log" || fail "could not read poll logs" [ "$phase" = Succeeded ] || fail "the poll pod ended in $phase" return From 232b9b9ebb74b239b1dca394560996b8a4d368f7 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 5 Nov 2026 14:44:47 +0000 Subject: [PATCH 5/5] Revert "feat: allow heartbeat API endpoint injection" This reverts commit 2a30f287fe040a3e215ae82bfb2b1c8cfbfe3406. --- scripts/heartbeat.sh | 5 ++--- tests/unit/test_heartbeat.py | 7 ------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/scripts/heartbeat.sh b/scripts/heartbeat.sh index 559d35c2..b8dee073 100755 --- a/scripts/heartbeat.sh +++ b/scripts/heartbeat.sh @@ -21,7 +21,6 @@ set -euo pipefail HEARTBEAT_GH_TOKEN="${HEARTBEAT_GH_TOKEN:?HEARTBEAT_GH_TOKEN must be set (a PAT with gist scope)}" HEARTBEAT_GIST_ID="${HEARTBEAT_GIST_ID:?HEARTBEAT_GIST_ID must be set}" -HEARTBEAT_GITHUB_API_URL="${HEARTBEAT_GITHUB_API_URL:-https://api.github.com}" GIST_FILE="${GIST_FILE:-arc-healthy-until}" HEALTH_GIST_FILE="${HEALTH_GIST_FILE:-fleet-health.json}" JOB_PEAK_SUMMARY_GIST_FILE="${HEARTBEAT_JOB_PEAK_SUMMARY_GIST_FILE:-job-peak-summary.json}" @@ -102,7 +101,7 @@ reasons_json="$(printf '%s\n' "${reasons[@]}" | jq -R . | jq -sc 'map(select(len # When each problem began, so a reader (scripts/fleet-alert.sh) can tell an outage that has lasted from a blip without a clock of its own: a problem still present keeps the onset the last tick published, and a new one starts now. The last tick's document is read back from the gist; if that read fails every onset restarts at now, which delays an alert and never raises a false one. at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" previous_health="$(curl -fsS -H "Authorization: Bearer ${HEARTBEAT_GH_TOKEN}" -H "Accept: application/vnd.github+json" \ - "${HEARTBEAT_GITHUB_API_URL}/gists/${HEARTBEAT_GIST_ID}" 2>/dev/null \ + "https://api.github.com/gists/${HEARTBEAT_GIST_ID}" 2>/dev/null \ | jq -c --arg file "$HEALTH_GIST_FILE" '.files[$file].content // empty | fromjson' 2>/dev/null || true)" # A server node is one carrying the control-plane role label, which k3s sets on every server. A node that is not Ready is listed whether or not it is a server, but only a server's absence endangers etcd quorum and the API, which is why the flag is published. A server that regrading has stepped down carries the level label and is not Ready on purpose (at etcd-only it runs no kubelet), so its level is published beside it, for a reader to tell it from a server that has failed. @@ -158,7 +157,7 @@ curl -fsS -X PATCH \ -H "Authorization: Bearer ${HEARTBEAT_GH_TOKEN}" \ -H "Accept: application/vnd.github+json" \ -d "$body" \ - "${HEARTBEAT_GITHUB_API_URL}/gists/${HEARTBEAT_GIST_ID}" >/dev/null + "https://api.github.com/gists/${HEARTBEAT_GIST_ID}" >/dev/null if [ "$healthy" != "true" ]; then echo "Unhealthy (${reasons[*]}) - heartbeat gist reason updated, timestamp not refreshed" >&2 diff --git a/tests/unit/test_heartbeat.py b/tests/unit/test_heartbeat.py index 90c9cea8..36258b2e 100644 --- a/tests/unit/test_heartbeat.py +++ b/tests/unit/test_heartbeat.py @@ -57,8 +57,6 @@ #!/usr/bin/env python3 import json, os, sys args = sys.argv[1:] - with open(os.environ["FAKE_CURL_BODY"] + ".urls", "a") as urls: - urls.write(args[-1] + "\\n") if "-d" in args: open(os.environ["FAKE_CURL_BODY"], "w").write(args[args.index("-d") + 1]) sys.exit(0) @@ -140,11 +138,6 @@ def assertWithheld(self, files: dict[str, str]) -> None: self.assertFalse(health["healthy"]) self.assertTrue(health["reasons"]) - def test_gist_api_endpoint_is_configurable(self) -> None: - result, _ = self.tick(env={"HEARTBEAT_GITHUB_API_URL": "http://example.test:8080"}) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual((self.dir / "curl-body.json.urls").read_text().splitlines(), ["http://example.test:8080/gists/test-gist"] * 2) - def test_a_healthy_fleet_refreshes_the_gist(self) -> None: result, files = self.tick() self.assertEqual(result.returncode, 0, result.stderr)