feat: compare published role images on one runner

.github/workflows/published-image-canary.yml

@@ -7,6 +7,10 @@ on:
         description: Public fleet-role image pinned by sha256 digest
         required: true
         type: string
+      comparison_image:
+        description: Optional second public fleet-role image pinned by sha256 digest
+        required: false
+        type: string
       scenario:
         description: Mesh role-controller scenario
         required: true
@@ -26,7 +30,7 @@ jobs:
   canary:
     name: Published image mesh (${{ inputs.scenario }})
     runs-on: ubuntu-latest
-    timeout-minutes: 105
+    timeout-minutes: 360
     steps:
       - uses: actions/checkout@v7
         with:
@@ -36,7 +40,13 @@ jobs:
         env:
           IMAGE: ${{ inputs.image }}
           SCENARIO: ${{ inputs.scenario }}
-        run: bash tests/mesh/validate-canary.sh "$IMAGE" "$SCENARIO"
+          COMPARISON_IMAGE: ${{ inputs.comparison_image }}
+        run: |
+          bash tests/mesh/validate-canary.sh "$IMAGE" "$SCENARIO"
+          if [ -n "$COMPARISON_IMAGE" ]; then
+            bash tests/mesh/validate-canary.sh "$COMPARISON_IMAGE" "$SCENARIO"
+            [ "$IMAGE" != "$COMPARISON_IMAGE" ] || { echo 'Images must differ' >&2; exit 1; }
+          fi
 
       - name: Install Docker Compose
         env:
@@ -68,9 +78,28 @@ jobs:
           echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH"
 
       - name: Run published image in the mesh
+        if: inputs.comparison_image == ''
         env:
           ANSIBLE_COLLECTIONS_PATH: ${{ runner.temp }}/collections
           GRTEST_WORK: ${{ runner.temp }}/grtest-mesh
           GRTEST_ROLE_IMAGE: ${{ inputs.image }}
           SCENARIO: ${{ inputs.scenario }}
         run: tests/mesh/run.sh "$SCENARIO"
+
+      - name: Compare published images in the mesh
+        if: inputs.comparison_image != ''
+        env:
+          ANSIBLE_COLLECTIONS_PATH: ${{ runner.temp }}/collections
+          IMAGE: ${{ inputs.image }}
+          COMPARISON_IMAGE: ${{ inputs.comparison_image }}
+          SCENARIO: ${{ inputs.scenario }}
+          RESULTS: ${{ runner.temp }}/image-comparison
+        run: bash tests/mesh/compare-images.sh "$IMAGE" "$COMPARISON_IMAGE" "$SCENARIO" "$RESULTS"
+
+      - name: Upload comparison results
+        if: always() && inputs.comparison_image != ''
+        uses: actions/upload-artifact@v7
+        with:
+          name: published-image-comparison
+          path: ${{ runner.temp }}/image-comparison/
+          if-no-files-found: warn

README.md

@@ -72,6 +72,7 @@ Each Docker-in-Docker runner pod starts with an empty Docker daemon, so every jo
 - **Test capabilities:** `tests/capabilities/run.sh` builds the reference Node.js tool image and libpq system-library image, renders a profile with capabilities through the ARC role and the scale-set chart, and runs a pod from the chart's runner template in a throwaway kind cluster to check that the tool reaches the tool cache and `PATH` and the library reaches the sysroot and its search paths; `.github/workflows/capabilities.yml` runs it on pull requests. `tests/system_libraries/run.sh [openmpi|highs|libpq|glpk|libmariadb]...` proves each reference system-library image works relocated: it copies the payload to `/opt/sysroot` as the role's init container does, applies the role's job-started hook, and runs the payload's programs in the stock `actions-runner` image and a program compiled against it in the reference build runner image (`tool-images/build`, the stock image with `build-essential` and `pkg-config`), including a two-process `mpirun` for Open MPI; `.github/workflows/system-libraries.yml` runs it on amd64 and arm64 on pull requests. The images under `tool-images/` build with `docker buildx build --platform linux/arm64,linux/amd64 tool-images/<name>`; `.github/workflows/publish-tool-images.yml` publishes them (see Releases), and the release workflow does not.
 - **Test the build runner image's ids:** `tests/runner_build/run.sh` builds `tool-images/build` with its default runner ids and with `RUNNER_UID` and `RUNNER_GID` set to another, and checks the runner user in each; `.github/workflows/runner-build.yml` runs it on amd64 and arm64 for changes to the image.
 - **Canary a published fleet-role image:** `gh workflow run published-image-canary.yml --ref main -f image='ghcr.io/exadev/github-runner-fleet-role@sha256:<digest>' -f scenario=role_controller` runs the chosen role-controller scenario in the disposable mesh on a GitHub-hosted Linux amd64 runner. The image must be publicly pullable and digest-pinned. The scenario choices are `role_controller`, `role_controller_swap`, `role_swap`, `role_swap_paused`, `role_swap_blocked_peer` and `role_swap_refused`; each tests only its own behaviour and the image selected for that runner's architecture, not other platform images.
+- **Compare two published fleet-role images:** `gh workflow run published-image-canary.yml --ref main -f image='ghcr.io/exadev/github-runner-fleet-role@sha256:<first-digest>' -f comparison_image='ghcr.io/exadev/github-runner-fleet-role@sha256:<second-digest>' -f scenario=role_controller` runs the selected scenario in ABBA order on one GitHub-hosted runner, rebuilding the disposable mesh each time. The `published-image-comparison` artifact contains each run's log and a TSV of image, scenario, outcome and elapsed seconds. The run fails if any scenario fails; timings include mesh setup and teardown, may reflect caching and runner contention, and are not a fleet benchmark. The job's GitHub-hosted limit can end a comparison before all runs finish.
 - **Test the registry cache:** `tests/registry_cache/run.sh` installs the cache as the ARC role renders it in a throwaway kind cluster, with Docker Hub, ghcr.io and a private registry behind basic auth, runs a pod from the chart's runner template as the role renders a Docker-in-Docker profile with the cache on, and checks in the cache's access logs that `docker pull` and a `docker-container` buildx builder pulled through it from all three without logging in, then scales the cache to nothing and checks that pulls and builds still work directly; `.github/workflows/registry-cache.yml` runs it on pull requests.
 - **Lint:** `.github/workflows/ci.yml` runs Shellcheck, actionlint, yamllint, hadolint, an Ansible syntax check, ansible-lint (at its `production` profile), and a collection build check on every push and pull request, plus commitlint (on every commit in a pull request, and on the pull request's own title) on pull requests specifically - all gated by one `required-checks` job so a repository ruleset only ever has to require that one check regardless of how many lint jobs exist. hadolint's warning-level pinning advice is visible but non-blocking (`--failure-threshold error`): this repo deliberately floats several images/packages on `latest` rather than pinning. yamllint's config lives in `.yamllint`, relaxing the three default rules that conflict with this project's own conventions (long single-line comments, no leading `---`, GitHub Actions' `on:` key).
 - **Commit messages:** `.husky/commit-msg` runs commitlint locally (after `npm ci`), the same check CI's commitlint job applies, so a message semantic-release would mishandle is rejected before it is pushed. The machine-wide hooks dispatcher hands off to it; without one, `git config core.hooksPath .husky` enables it.

tests/mesh/compare-images.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Run each published image twice in an ABBA order on one disposable runner.
+set -uo pipefail
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+image_a="${1:?first image required}"
+image_b="${2:?second image required}"
+scenario="${3:?scenario required}"
+results="${4:?results directory required}"
+bash "${root}/tests/mesh/validate-canary.sh" "$image_a" "$scenario" || exit 1
+bash "${root}/tests/mesh/validate-canary.sh" "$image_b" "$scenario" || exit 1
+if [ "$image_a" = "$image_b" ]; then
+  printf '%s\n' 'Images must be different digest references' >&2
+  exit 1
+fi
+mkdir -p "$results"
+printf 'run\timage\tscenario\tresult\telapsed_seconds\n' > "$results/results.tsv"
+failed=0
+for run in 1 2 3 4; do
+  case "$run" in 1 | 4) image="$image_a" ;; *) image="$image_b" ;; esac
+  printf 'Running %s: %s (%s)\n' "$run" "$image" "$scenario"
+  start=$SECONDS
+  if GRTEST_ROLE_IMAGE="$image" GRTEST_WORK="${results}/mesh-${run}" "${root}/tests/mesh/run.sh" "$scenario" > "$results/run-${run}.log" 2>&1; then
+    result=pass
+  else
+    result=fail
+    failed=1
+  fi
+  printf '%s\t%s\t%s\t%s\t%s\n' "$run" "$image" "$scenario" "$result" "$((SECONDS - start))" | tee -a "$results/results.tsv"
+done
+exit "$failed"

tests/unit/test_mesh_role_image.py

@@ -2,7 +2,9 @@
 
 import os
 import subprocess
+import tempfile
 import unittest
+import shutil
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parents[2]
@@ -55,5 +57,26 @@ def test_rejects_mutable_and_malformed_references(self) -> None:
                 self.assertNotEqual(self.validate(image, "role_controller").returncode, 0)
 
 
+class ComparisonTest(unittest.TestCase):
+    def test_abba_order_and_failures_are_recorded(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            mesh = root / "tests/mesh"
+            mesh.mkdir(parents=True)
+            for name in ("compare-images.sh", "validate-canary.sh"):
+                shutil.copy(ROOT / "tests/mesh" / name, mesh / name)
+            (mesh / "run.sh").write_text('#!/bin/bash\nprintf "%s\\n" "$GRTEST_ROLE_IMAGE"\n[[ "$GRTEST_WORK" != *mesh-2 ]]\n')
+            (mesh / "run.sh").chmod(0o755)
+            image_a = "ghcr.io/example/role@sha256:" + "a" * 64
+            image_b = "ghcr.io/example/role@sha256:" + "b" * 64
+            results = root / "results"
+            result = subprocess.run(["bash", str(mesh / "compare-images.sh"), image_a, image_b, "role_controller", str(results)], capture_output=True, text=True, check=False)
+            self.assertEqual(result.returncode, 1, result.stderr)
+            rows = [line.split("\t") for line in (results / "results.tsv").read_text().splitlines()[1:]]
+            self.assertEqual([row[1] for row in rows], [image_a, image_b, image_b, image_a])
+            self.assertEqual([row[3] for row in rows], ["pass", "fail", "pass", "pass"])
+            self.assertEqual((results / "run-2.log").read_text().strip(), image_b)
+
+
 if __name__ == "__main__":
     unittest.main()