diff --git a/.github/workflows/published-image-canary.yml b/.github/workflows/published-image-canary.yml index f4440a0c..79ce246f 100644 --- a/.github/workflows/published-image-canary.yml +++ b/.github/workflows/published-image-canary.yml @@ -7,6 +7,10 @@ on: description: Public fleet-role image pinned by sha256 digest required: true type: string + comparison_image: + description: Optional second public fleet-role image pinned by sha256 digest + required: false + type: string scenario: description: Mesh role-controller scenario required: true @@ -26,7 +30,7 @@ jobs: canary: name: Published image mesh (${{ inputs.scenario }}) runs-on: ubuntu-latest - timeout-minutes: 105 + timeout-minutes: 360 steps: - uses: actions/checkout@v7 with: @@ -36,7 +40,13 @@ jobs: env: IMAGE: ${{ inputs.image }} SCENARIO: ${{ inputs.scenario }} - run: bash tests/mesh/validate-canary.sh "$IMAGE" "$SCENARIO" + COMPARISON_IMAGE: ${{ inputs.comparison_image }} + run: | + bash tests/mesh/validate-canary.sh "$IMAGE" "$SCENARIO" + if [ -n "$COMPARISON_IMAGE" ]; then + bash tests/mesh/validate-canary.sh "$COMPARISON_IMAGE" "$SCENARIO" + [ "$IMAGE" != "$COMPARISON_IMAGE" ] || { echo 'Images must differ' >&2; exit 1; } + fi - name: Install Docker Compose env: @@ -68,9 +78,28 @@ jobs: echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" - name: Run published image in the mesh + if: inputs.comparison_image == '' env: ANSIBLE_COLLECTIONS_PATH: ${{ runner.temp }}/collections GRTEST_WORK: ${{ runner.temp }}/grtest-mesh GRTEST_ROLE_IMAGE: ${{ inputs.image }} SCENARIO: ${{ inputs.scenario }} run: tests/mesh/run.sh "$SCENARIO" + + - name: Compare published images in the mesh + if: inputs.comparison_image != '' + env: + ANSIBLE_COLLECTIONS_PATH: ${{ runner.temp }}/collections + IMAGE: ${{ inputs.image }} + COMPARISON_IMAGE: ${{ inputs.comparison_image }} + SCENARIO: ${{ inputs.scenario }} + RESULTS: ${{ runner.temp }}/image-comparison + run: bash tests/mesh/compare-images.sh "$IMAGE" "$COMPARISON_IMAGE" "$SCENARIO" "$RESULTS" + + - name: Upload comparison results + if: always() && inputs.comparison_image != '' + uses: actions/upload-artifact@v7 + with: + name: published-image-comparison + path: ${{ runner.temp }}/image-comparison/ + if-no-files-found: warn diff --git a/README.md b/README.md index 1af30c4a..8ccfaf45 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ Each Docker-in-Docker runner pod starts with an empty Docker daemon, so every jo - **Test capabilities:** `tests/capabilities/run.sh` builds the reference Node.js tool image and libpq system-library image, renders a profile with capabilities through the ARC role and the scale-set chart, and runs a pod from the chart's runner template in a throwaway kind cluster to check that the tool reaches the tool cache and `PATH` and the library reaches the sysroot and its search paths; `.github/workflows/capabilities.yml` runs it on pull requests. `tests/system_libraries/run.sh [openmpi|highs|libpq|glpk|libmariadb]...` proves each reference system-library image works relocated: it copies the payload to `/opt/sysroot` as the role's init container does, applies the role's job-started hook, and runs the payload's programs in the stock `actions-runner` image and a program compiled against it in the reference build runner image (`tool-images/build`, the stock image with `build-essential` and `pkg-config`), including a two-process `mpirun` for Open MPI; `.github/workflows/system-libraries.yml` runs it on amd64 and arm64 on pull requests. The images under `tool-images/` build with `docker buildx build --platform linux/arm64,linux/amd64 tool-images/`; `.github/workflows/publish-tool-images.yml` publishes them (see Releases), and the release workflow does not. - **Test the build runner image's ids:** `tests/runner_build/run.sh` builds `tool-images/build` with its default runner ids and with `RUNNER_UID` and `RUNNER_GID` set to another, and checks the runner user in each; `.github/workflows/runner-build.yml` runs it on amd64 and arm64 for changes to the image. - **Canary a published fleet-role image:** `gh workflow run published-image-canary.yml --ref main -f image='ghcr.io/exadev/github-runner-fleet-role@sha256:' -f scenario=role_controller` runs the chosen role-controller scenario in the disposable mesh on a GitHub-hosted Linux amd64 runner. The image must be publicly pullable and digest-pinned. The scenario choices are `role_controller`, `role_controller_swap`, `role_swap`, `role_swap_paused`, `role_swap_blocked_peer` and `role_swap_refused`; each tests only its own behaviour and the image selected for that runner's architecture, not other platform images. +- **Compare two published fleet-role images:** `gh workflow run published-image-canary.yml --ref main -f image='ghcr.io/exadev/github-runner-fleet-role@sha256:' -f comparison_image='ghcr.io/exadev/github-runner-fleet-role@sha256:' -f scenario=role_controller` runs the selected scenario in ABBA order on one GitHub-hosted runner, rebuilding the disposable mesh each time. The `published-image-comparison` artifact contains each run's log and a TSV of image, scenario, outcome and elapsed seconds. The run fails if any scenario fails; timings include mesh setup and teardown, may reflect caching and runner contention, and are not a fleet benchmark. The job's GitHub-hosted limit can end a comparison before all runs finish. - **Test the registry cache:** `tests/registry_cache/run.sh` installs the cache as the ARC role renders it in a throwaway kind cluster, with Docker Hub, ghcr.io and a private registry behind basic auth, runs a pod from the chart's runner template as the role renders a Docker-in-Docker profile with the cache on, and checks in the cache's access logs that `docker pull` and a `docker-container` buildx builder pulled through it from all three without logging in, then scales the cache to nothing and checks that pulls and builds still work directly; `.github/workflows/registry-cache.yml` runs it on pull requests. - **Lint:** `.github/workflows/ci.yml` runs Shellcheck, actionlint, yamllint, hadolint, an Ansible syntax check, ansible-lint (at its `production` profile), and a collection build check on every push and pull request, plus commitlint (on every commit in a pull request, and on the pull request's own title) on pull requests specifically - all gated by one `required-checks` job so a repository ruleset only ever has to require that one check regardless of how many lint jobs exist. hadolint's warning-level pinning advice is visible but non-blocking (`--failure-threshold error`): this repo deliberately floats several images/packages on `latest` rather than pinning. yamllint's config lives in `.yamllint`, relaxing the three default rules that conflict with this project's own conventions (long single-line comments, no leading `---`, GitHub Actions' `on:` key). - **Commit messages:** `.husky/commit-msg` runs commitlint locally (after `npm ci`), the same check CI's commitlint job applies, so a message semantic-release would mishandle is rejected before it is pushed. The machine-wide hooks dispatcher hands off to it; without one, `git config core.hooksPath .husky` enables it. diff --git a/tests/mesh/compare-images.sh b/tests/mesh/compare-images.sh new file mode 100644 index 00000000..6d946e04 --- /dev/null +++ b/tests/mesh/compare-images.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Run each published image twice in an ABBA order on one disposable runner. +set -uo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +image_a="${1:?first image required}" +image_b="${2:?second image required}" +scenario="${3:?scenario required}" +results="${4:?results directory required}" +bash "${root}/tests/mesh/validate-canary.sh" "$image_a" "$scenario" || exit 1 +bash "${root}/tests/mesh/validate-canary.sh" "$image_b" "$scenario" || exit 1 +if [ "$image_a" = "$image_b" ]; then + printf '%s\n' 'Images must be different digest references' >&2 + exit 1 +fi +mkdir -p "$results" +printf 'run\timage\tscenario\tresult\telapsed_seconds\n' > "$results/results.tsv" +failed=0 +for run in 1 2 3 4; do + case "$run" in 1 | 4) image="$image_a" ;; *) image="$image_b" ;; esac + printf 'Running %s: %s (%s)\n' "$run" "$image" "$scenario" + start=$SECONDS + if GRTEST_ROLE_IMAGE="$image" GRTEST_WORK="${results}/mesh-${run}" "${root}/tests/mesh/run.sh" "$scenario" > "$results/run-${run}.log" 2>&1; then + result=pass + else + result=fail + failed=1 + fi + printf '%s\t%s\t%s\t%s\t%s\n' "$run" "$image" "$scenario" "$result" "$((SECONDS - start))" | tee -a "$results/results.tsv" +done +exit "$failed" diff --git a/tests/unit/test_mesh_role_image.py b/tests/unit/test_mesh_role_image.py index 8b865a0b..a76df346 100644 --- a/tests/unit/test_mesh_role_image.py +++ b/tests/unit/test_mesh_role_image.py @@ -2,7 +2,9 @@ import os import subprocess +import tempfile import unittest +import shutil from pathlib import Path ROOT = Path(__file__).resolve().parents[2] @@ -55,5 +57,26 @@ def test_rejects_mutable_and_malformed_references(self) -> None: self.assertNotEqual(self.validate(image, "role_controller").returncode, 0) +class ComparisonTest(unittest.TestCase): + def test_abba_order_and_failures_are_recorded(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + mesh = root / "tests/mesh" + mesh.mkdir(parents=True) + for name in ("compare-images.sh", "validate-canary.sh"): + shutil.copy(ROOT / "tests/mesh" / name, mesh / name) + (mesh / "run.sh").write_text('#!/bin/bash\nprintf "%s\\n" "$GRTEST_ROLE_IMAGE"\n[[ "$GRTEST_WORK" != *mesh-2 ]]\n') + (mesh / "run.sh").chmod(0o755) + image_a = "ghcr.io/example/role@sha256:" + "a" * 64 + image_b = "ghcr.io/example/role@sha256:" + "b" * 64 + results = root / "results" + result = subprocess.run(["bash", str(mesh / "compare-images.sh"), image_a, image_b, "role_controller", str(results)], capture_output=True, text=True, check=False) + self.assertEqual(result.returncode, 1, result.stderr) + rows = [line.split("\t") for line in (results / "results.tsv").read_text().splitlines()[1:]] + self.assertEqual([row[1] for row in rows], [image_a, image_b, image_b, image_a]) + self.assertEqual([row[3] for row in rows], ["pass", "fail", "pass", "pass"]) + self.assertEqual((results / "run-2.log").read_text().strip(), image_b) + + if __name__ == "__main__": unittest.main()