diff --git a/.github/workflows/platform-image-canary.yml b/.github/workflows/platform-image-canary.yml new file mode 100644 index 00000000..2b160888 --- /dev/null +++ b/.github/workflows/platform-image-canary.yml @@ -0,0 +1,77 @@ +name: Platform image canary + +on: + workflow_dispatch: + inputs: + component: + description: Component to compare + required: true + type: choice + options: + - autoscaler + - node-recovery + - pull-secret-renewer + old_image: + description: Previously published image pinned by digest + required: true + type: string + new_image: + description: Candidate published image pinned by digest + required: true + type: string + +permissions: + contents: read + +jobs: + canary: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Validate image references + env: + COMPONENT: ${{ inputs.component }} + OLD_IMAGE: ${{ inputs.old_image }} + NEW_IMAGE: ${{ inputs.new_image }} + run: | + set -euo pipefail + for image in "$OLD_IMAGE" "$NEW_IMAGE"; do + [[ "$image" =~ ^ghcr\.io/exadev/github-runner-${COMPONENT}@sha256:[0-9a-f]{64}$ ]] || { echo "Invalid image reference: $image" >&2; exit 1; } + done + [[ "$OLD_IMAGE" != "$NEW_IMAGE" ]] || { echo 'Images must differ' >&2; exit 1; } + - name: Install Ansible + run: | + set -euo pipefail + python3 -m venv "${RUNNER_TEMP}/venv" + "${RUNNER_TEMP}/venv/bin/pip" install --quiet ansible-core pyyaml jinja2 + echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" + - name: Install Helm + if: inputs.component == 'autoscaler' + run: | + set -euo pipefail + archive=helm-v4.3.0-linux-amd64.tar.gz + curl -fsSLo "${RUNNER_TEMP}/${archive}" "https://get.helm.sh/${archive}" + echo "86584a54def73570558f66f5111cc53dfed56689637ae32c1201205d494f54fb ${RUNNER_TEMP}/${archive}" | sha256sum -c - + tar -xzf "${RUNNER_TEMP}/${archive}" -C "${RUNNER_TEMP}" + echo "${RUNNER_TEMP}/linux-amd64" >> "$GITHUB_PATH" + - name: Compare published images + env: + COMPONENT: ${{ inputs.component }} + OLD_IMAGE: ${{ inputs.old_image }} + NEW_IMAGE: ${{ inputs.new_image }} + run: | + set -euo pipefail + for image in "$OLD_IMAGE" "$NEW_IMAGE"; do + for run in 1 2; do + echo "Testing ${image}, run ${run}" + case "$COMPONENT" in + autoscaler) GRTEST_AUTOSCALER_IMAGE="$image" PYTHON="${RUNNER_TEMP}/venv/bin/python" tests/autoscaler_ceiling/run.sh ;; + node-recovery) GRTEST_NODE_RECOVERY_IMAGE="$image" tests/node_recovery/run.sh ;; + pull-secret-renewer) GRTEST_RENEWER_IMAGE="$image" tests/pull_secret_renewal/run.sh ;; + *) exit 1 ;; + esac + done + done diff --git a/tests/autoscaler_ceiling/run.sh b/tests/autoscaler_ceiling/run.sh index 10cc0673..e0a754b1 100755 --- a/tests/autoscaler_ceiling/run.sh +++ b/tests/autoscaler_ceiling/run.sh @@ -166,6 +166,32 @@ for namespace in "${pooled_namespaces[@]}"; do targets="$targets $namespace/grte # One poll, for real (not dry run), its output kept for the checks and for a failure's report. poll() { + if [ -n "${GRTEST_AUTOSCALER_IMAGE:-}" ]; then + kubectl -n "$platform_namespace" delete pod autoscaler-poll --ignore-not-found --wait=true >/dev/null + kubectl -n "$platform_namespace" run autoscaler-poll --image="$GRTEST_AUTOSCALER_IMAGE" --restart=Never \ + --overrides "$(jq -n --arg targets "${targets# }" --arg ns "$platform_namespace" --arg selector "${eligibility_label}=true" '{spec:{serviceAccountName:"autoscaler",containers:[{name:"autoscaler-poll",image:env.GRTEST_AUTOSCALER_IMAGE,command:["/app/scripts/autoscaler.sh"],env:[{name:"AUTOSCALER_TARGETS",value:$targets},{name:"AUTOSCALER_DRY_RUN",value:"false"},{name:"AUTOSCALER_USABLE_BUDGET_GI",value:"1024"},{name:"AUTOSCALER_FLOOR",value:"1"},{name:"AUTOSCALER_NODE_SELECTOR",value:$selector},{name:"AUTOSCALER_STATE_NAMESPACE",value:$ns},{name:"AUTOSCALER_STATUS_CONFIGMAP",value:"autoscaler-status"}]}]}}')" >/dev/null + local phase + for _ in {1..120}; do + phase="$(kubectl -n "$platform_namespace" get pod autoscaler-poll -o jsonpath='{.status.phase}')" + case "$phase" in Succeeded|Failed) break ;; esac + sleep 2 + done + kubectl -n "$platform_namespace" get pod autoscaler-poll -o json > "$work/autoscaler-pod.json" + "$python" - "$repo_root" "$work/autoscaler-pod.json" <<'PY' +import json, pathlib, sys, yaml +root = pathlib.Path(sys.argv[1]) +tasks = yaml.safe_load((root / 'roles/github_runner_arc/tasks/install_platform.yml').read_text()) +task = next(t for t in tasks if t.get('name') == 'Platform: install/upgrade the autoscaler Deployment') +expected = task['kubernetes.core.k8s']['definition']['spec']['template']['spec'] +actual = json.loads(pathlib.Path(sys.argv[2]).read_text())['spec'] +assert actual['serviceAccountName'] == expected['serviceAccountName'] +assert actual.get('securityContext', {}) == expected.get('securityContext', {}) +assert actual['containers'][0].get('securityContext', {}) == expected['containers'][0].get('securityContext', {}) +PY + kubectl -n "$platform_namespace" logs autoscaler-poll > "$work/poll.log" || fail "could not read poll logs" + [ "$phase" = Succeeded ] || fail "the poll pod ended in $phase" + return + fi PATH="$work/bin:$PATH" \ AUTOSCALER_TARGETS="${targets# }" \ AUTOSCALER_DRY_RUN=false \ diff --git a/tests/node_recovery/run.sh b/tests/node_recovery/run.sh index cd8af528..09001de3 100755 --- a/tests/node_recovery/run.sh +++ b/tests/node_recovery/run.sh @@ -11,7 +11,7 @@ ansible_playbook="${ANSIBLE_PLAYBOOK:-ansible-playbook}" cluster=grtest-recovery platform_namespace=github-runner-platform namespace=grtest-listeners -image=grtest/node-recovery:test +image="${GRTEST_NODE_RECOVERY_IMAGE:-grtest/node-recovery:test}" pod_image=busybox:1.36 watcher_node="${cluster}-worker" doomed_node="${cluster}-worker2" @@ -153,9 +153,12 @@ server_version="$(kubectl version -o json | jq -r '.serverVersion.gitVersion')" log "Server version ${server_version}" doomed_uid="$(kubectl get node "$doomed_node" -o jsonpath='{.metadata.uid}')" -log "Building and loading the watcher image and the stand-in's image" -docker build -q -f "$repo_root/node-recovery/Dockerfile" -t "$image" "$repo_root" >/dev/null -kind load docker-image "$image" --name "$cluster" >/dev/null +if [ -z "${GRTEST_NODE_RECOVERY_IMAGE:-}" ]; then + log "Building and loading the watcher image" + docker build -q -f "$repo_root/node-recovery/Dockerfile" -t "$image" "$repo_root" >/dev/null + kind load docker-image "$image" --name "$cluster" >/dev/null +fi +log "Loading the stand-in's image" # The stand-in's image is pulled by each node's own runtime: a host Docker on the containerd image store pulls a multi-platform index holding only the host platform's blobs, and kind's import of it with all platforms fails on the missing ones. for node in $(kind get nodes --name "$cluster"); do docker exec "$node" crictl pull "$pod_image" >/dev/null diff --git a/tests/pull_secret_renewal/run.sh b/tests/pull_secret_renewal/run.sh index c5615152..a55245db 100755 --- a/tests/pull_secret_renewal/run.sh +++ b/tests/pull_secret_renewal/run.sh @@ -8,7 +8,7 @@ repo_root="$(cd "$(dirname "$0")/../.." && pwd)" ansible_playbook="${ANSIBLE_PLAYBOOK:-ansible-playbook}" cluster=grtest-renewal namespace=grtest-runners -image=grtest/pull-secret-renewer:test +image="${GRTEST_RENEWER_IMAGE:-grtest/pull-secret-renewer:test}" stub_image=python:3.13-alpine pull_secret=ghcr-pull renewer="${pull_secret}-renewer" @@ -63,9 +63,11 @@ run_job() { log "Creating kind cluster $cluster" kind create cluster --name "$cluster" --kubeconfig "$KUBECONFIG" --wait 120s >/dev/null -log "Building and loading the renewer image" -docker build -q -f "$repo_root/pull-secret-renewer/Dockerfile" -t "$image" "$repo_root" >/dev/null -kind load docker-image "$image" --name "$cluster" >/dev/null +if [ -z "${GRTEST_RENEWER_IMAGE:-}" ]; then + log "Building and loading the renewer image" + docker build -q -f "$repo_root/pull-secret-renewer/Dockerfile" -t "$image" "$repo_root" >/dev/null + kind load docker-image "$image" --name "$cluster" >/dev/null +fi log "Starting the GitHub stand-in" kubectl create namespace "$namespace" >/dev/null