diff --git a/.github/workflows/ci-local.yml b/.github/workflows/ci-local.yml index bc43f462..6447ecc9 100644 --- a/.github/workflows/ci-local.yml +++ b/.github/workflows/ci-local.yml @@ -1,8 +1,9 @@ name: CI (ThinFloe) # ThinFloe's own CI jobs, called from ci.yml (docs/release-ci.md, "CI -# split"). `unit` gives a result within minutes; `image` builds the rootfs -# once and runs the integration tests, the image, the nine QEMU smoke tests +# split"). `unit` runs the unit tests and gives a result within minutes; +# `image` builds the rootfs once and runs the integration tests (not the unit +# tests again), the image, the nine QEMU smoke tests # and the runtime tests on it. scripts/ci-runner-setup.sh prepares the # runner (docs/release-ci.md, "Runner setup for the project's workflows"). on: @@ -48,8 +49,8 @@ jobs: - name: Prepare the runner run: scripts/ci-runner-setup.sh all - - name: make test - run: make test MEMCAP= + - name: make test-integration + run: make test-integration MEMCAP= - name: make image run: make image MEMCAP= diff --git a/CLAUDE.md b/CLAUDE.md index 83443a21..73507553 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,8 @@ Boot in short: syslinux loads the kernel with `ramroot=` (a path on the stick's make image # all stages, bootable files in artifacts/ make rootfs # Ubuntu container; kernel, initrd, ramroot.sqsh to artifacts/rootfs/ make ipxe # iPXE loaders to artifacts/rootfs/ -make test # test-unit, then the integration tests against the built image +make test # test-unit, then test-integration +make test-integration # the integration tests alone against the built image (CI's image job) make test-unit # Python unit tests in test/unit/ and the menu's cargo test make test-runtime # labwc and the menu headless make qemu-smoke # boot the USB image in QEMU (also: -addons, -write, -upgrade, diff --git a/Makefile b/Makefile index 831b1d63..1bd7ae2e 100644 --- a/Makefile +++ b/Makefile @@ -117,7 +117,7 @@ QEMU_EXTRA_ARGS ?= QEMU_XRES ?= 1920 QEMU_YRES ?= 1080 -.PHONY: all clean rootfs image ipxe qemu-test qemu-smoke qemu-smoke-netboot qemu-smoke-timezone qemu-smoke-timezone-netboot qemu-smoke-upgrade qemu-smoke-addons qemu-smoke-two-sticks qemu-smoke-write qemu-smoke-grow release-assets test-unit test test-runtime help +.PHONY: all clean rootfs image ipxe qemu-test qemu-smoke qemu-smoke-netboot qemu-smoke-timezone qemu-smoke-timezone-netboot qemu-smoke-upgrade qemu-smoke-addons qemu-smoke-two-sticks qemu-smoke-write qemu-smoke-grow release-assets test-unit test-integration test test-runtime help all: image @echo "✓ Build complete!" @@ -142,7 +142,8 @@ help: @echo " qemu-smoke-write - Headless write of a USB stick from a boot server, then boot it" @echo " qemu-smoke-grow - Two headless boots of an 8 GB stick: the ESP grows once" @echo " test-unit - Run offline unit tests (no rootfs needed)" - @echo " test - Run integration guard tests against rootfs" + @echo " test-integration - Run integration guard tests against rootfs" + @echo " test - test-unit, then test-integration" @echo " test-runtime - Run runtime session tests (needs network, compositor)" @echo " clean - Remove artifacts (FORCE=1 if artifacts is a symlink)" @echo " help - Show this help" @@ -598,7 +599,9 @@ test-unit: @$(TEST_ENV) python3 -m unittest discover -s test/unit -v cd $(MENU_DIR) && $(TEST_ENV) CARGO_BUILD_JOBS=$(BUILD_JOBS) $(MEMCAP) cargo test --locked -j $(BUILD_JOBS) -- --test-threads=$(TEST_THREADS) -test: test-unit rootfs +test: test-unit test-integration + +test-integration: rootfs @mkdir -p '$(TEST_TMPDIR)' @$(TEST_ENV) test/integration/test-firmware.sh @$(TEST_ENV) test/integration/test-wifi-stack.sh @@ -637,6 +640,7 @@ test: test-unit rootfs @$(TEST_ENV) test/integration/test-make-mounts.sh @$(TEST_ENV) TF_UBUNTU_IMAGE=$(UBUNTU_IMAGE) test/integration/test-addon-deps.sh @$(TEST_ENV) TF_UBUNTU_IMAGE=$(UBUNTU_IMAGE) test/integration/test-network-netplan.sh + @$(TEST_ENV) test/integration/test-make-targets.sh # Runtime session tests. Needs network (installs python3-xlib into a # test-only image) and runs a compositor, so it is deliberately not part of diff --git a/docs/release-ci.md b/docs/release-ci.md index 2cfbaf89..6fc22528 100644 --- a/docs/release-ci.md +++ b/docs/release-ci.md @@ -94,7 +94,8 @@ The split changes as follows (decision 17, revised 2026-11-12): - `ci.yml` (every pull request, every push to `main`, the release branch) runs two jobs through `ci-local.yml`: - `unit`: `make test-unit MEMCAP=`, for a result within minutes. - - `image`: `make test MEMCAP=`, `make image`, the nine smoke targets, and + - `image`: `make test-integration MEMCAP=` (the unit tests run only in + `unit`), `make image`, the nine smoke targets, and `make test-runtime`, in one job so the rootfs is built once. About 25 minutes. - `release-build.yml` stays as decision 17 describes it: `make image` with diff --git a/docs/superpowers/plans/2026-11-21-dev-pace.md b/docs/superpowers/plans/2026-11-21-dev-pace.md new file mode 100644 index 00000000..01840ec5 --- /dev/null +++ b/docs/superpowers/plans/2026-11-21-dev-pace.md @@ -0,0 +1,160 @@ +# Dev pace Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Shorter feature rounds: one CI run per round, a shorter `local / image` job, faster local tests, fixtures that survive a new struct field, and a review loop with a fixed end. + +**Architecture:** Two pull requests. PR 1 (branch `devpace-ci-dedup`, Task 1) removes the duplicate unit run from the image job and goes to `main` alone, so its runs give the baseline. PR 2 (branch `devpace-round`, made from `main` after PR 1 is merged, Tasks 2-8) adds a `PREBUILT=1` make mode, splits the smoke tests into parallel CI jobs, runs the Python unit tests in parallel, adds `make test-menu`, `..Default::default()` fixtures and the workflow rules in CLAUDE.md. + +**Tech Stack:** GNU make, bash, Python 3 (stdlib only), GitHub Actions (reusable workflow `ci-local.yml`), Rust/cargo for the menu crate. + +**Spec:** `docs/superpowers/specs/2026-11-21-dev-pace-design.md` + +## Global Constraints + +- At most 4 cores locally: `BUILD_JOBS`/`TEST_THREADS` default 4; cargo under `flock /scratch/oetiker/claude-tmp/thinfloe-cargo.lock systemd-run --user --scope -q -p MemoryMax=6G --`; image targets under `flock /scratch/oetiker/claude-tmp/dbrrg-build.lock make VERSION=devpace`. +- No test is removed and no test checks less. Every smoke test and `test-runtime` still runs on every PR. +- `ci.yml` is not changed (repo-infra standard; `ci-passed` keeps `needs: [local, release-pr-current, release-verdict]`). `release-build-local.yml` is not changed. +- No CHANGES.md entry for any part. Both PRs need the `no-changelog` label (owner sets it). +- A smoke job never builds the `thinfloe-ubuntu` container and never exports the rootfs. +- Every GitHub-side action (push, PR, label, re-run, merge, branch delete) is owner-confirmed. Merges are rebase merges. +- Comments and docs in English, in the house style (repo-infra:writing-style). + +## Review Focus + +1. **A smoke job that silently rebuilds or runs on other bytes.** A smoke job must fail if any downloaded file changed or a `thinfloe-ubuntu` image appears. Test: Task 3, `scripts/ci-check-prebuilt.sh` and its test. +2. **`PREBUILT=1` with a file missing** must stop with make's "No rule to make target", not build it. Test: Task 2, `test-make-targets.sh` case "missing kernel". +3. **A smoke job's failure hidden from `ci-passed`** (a called workflow's job with `needs:`, or a matrix leg). Test: Task 4, the negative run. +4. **An empty or missing `test/unit`** must not pass the parallel runner vacuously (exit 2). Test: Task 5, `test_no_modules_is_an_error`. +5. **Two Python test modules sharing a fixed path** fail only when run at the same time. Test: Task 5, the suite run three times in a row with `-j 4`. + +--- + +## PR 1: branch `devpace-ci-dedup` (worktree `/scratch/oetiker/claude-worktrees/thinfloe-devpace`, exists, spec committed) + +### Task 1: `make test-integration`, used by the image job + +**Files:** +- Modify: `Makefile` (the `test:` rule, ~line 601) +- Modify: `.github/workflows/ci-local.yml` (step "make test" of job `image`; the header comment) +- Create: `test/integration/test-make-targets.sh` (and add it to the integration recipe) +- Modify: `CLAUDE.md` (Commands block), `docs/release-ci.md` (CI split, the `image` bullet) + +**Interfaces:** +- Produces: make target `test-integration` (prerequisite `rootfs`, the current `test` recipe); `test: test-unit test-integration` with no recipe. `test-make-targets.sh` with helpers `ok`, `bad`, `dry [VAR=val...]` (prints `make -n` output in the scratch tree), which Tasks 2 and 6 extend. + +- [ ] **Step 1: Write the failing test** `test/integration/test-make-targets.sh`. Build the scratch tree and podman stub as `test-usb-image-deps.sh` does (`tf_guard_tree`, stub answering `image inspect`). Cases: + - `dry test-integration` contains `test-firmware.sh` and contains neither `unittest` nor `cargo test` → "test-integration runs no unit tests". + - `dry test` contains `unittest` (or `run-unit-tests.py` after Task 5), `cargo test` and `test-firmware.sh` → "make test still runs the unit and the integration tests". +- [ ] **Step 2: Run it, expect FAIL** — `test/integration/test-make-targets.sh` → `FAIL - test-integration runs no unit tests` (no such target). +- [ ] **Step 3: Implement.** Rename the `test:` rule to `test-integration: rootfs` with its recipe unchanged; add `test: test-unit test-integration`; add `@$(TEST_ENV) test/integration/test-make-targets.sh` to the recipe. In `ci-local.yml` the step becomes `name: make test-integration`, `run: make test-integration MEMCAP=`; reword the header comment. +- [ ] **Step 4: Run, expect PASS** — only `ok` lines, exit 0; `shellcheck test/integration/test-make-targets.sh` clean. +- [ ] **Step 5: Full local gate** — `make test VERSION=devpace` under the build lock; `ok` count ~1403 (aea01da) plus the new lines; no FAIL. +- [ ] **Step 6: Docs.** CLAUDE.md Commands: `make test-integration # the integration tests alone (CI's image job); make test = test-unit + this`. `docs/release-ci.md` CI split: `image` runs `make test-integration`; only `unit` runs the unit tests. +- [ ] **Step 7: Commit** — `ci: the image job runs the integration tests without the unit tests` (body: saves ~4:45, run 44822629521). +- [ ] **Step 8: PR** (owner OK: push, PR, `no-changelog` label). After the merge, record the `local` time (first job start to last job end, `gh run view --json jobs`) of two runs on `main` as the baseline in `/scratch/oetiker/claude-tmp/devpace/baseline.txt`. + +--- + +## PR 2: branch `devpace-round` from `main` after PR 1 (new worktree `/scratch/oetiker/claude-worktrees/thinfloe-devround`) + +### Task 2: `PREBUILT=1`: smoke targets on files that are already there + +**Files:** +- Modify: `Makefile` (the input-hash `$(shell ...)` and the three `drop_stale_stamp` calls, ~line 227-230; the rules for `$(KERNEL) $(INITRD) $(SQUASHFS)`, `$(USB_IMAGE)`, `$(USB_IMAGE_COMPRESSED)`, `$(USB_CHECKSUM)`) +- Modify: `test/integration/test-make-targets.sh` + +**Interfaces:** +- Produces: `make PREBUILT=1` needs only `artifacts/rootfs/{vmlinuz,initrd.img,ramroot.sqsh}` and `artifacts/images/thinfloe-usb.img.zst{,.sha256}`; `$(USB_IMAGE)` is made by `zstd -d -f` from the `.zst`. + +- [ ] **Step 1: Failing tests** in `test-make-targets.sh`, scratch tree with the five files above (empty files, a real tiny `.zst` made with `zstd`), no container stamps: + - for each of the nine `qemu-smoke*` goals: `dry PREBUILT=1` contains no `podman build`, no `export-rootfs.sh`, no `make-bootable-image.sh` → "PREBUILT builds nothing". + - `dry qemu-smoke-upgrade PREBUILT=1` contains `zstd -d` → "PREBUILT makes the raw image from the .zst". + - with `vmlinuz` removed: `make -n qemu-smoke PREBUILT=1` exits non-zero and prints `No rule to make target` → "PREBUILT with a missing kernel stops". + - without `PREBUILT`: `dry qemu-smoke` still contains `export-rootfs.sh` (no stamps) → "without PREBUILT the build rules stand". +- [ ] **Step 2: Run, expect FAIL** on the first case. +- [ ] **Step 3: Implement** with `ifeq ($(PREBUILT),1)` / `else`: the PREBUILT branch skips the stamp `$(shell)` calls and defines only `$(USB_IMAGE): $(USB_IMAGE_COMPRESSED)` → `zstd -d -f $< -o $@`; the `else` branch holds the existing rules unchanged. A comment above names the CI smoke jobs as the user and the guard script. +- [ ] **Step 4: Run, expect PASS**; `test-usb-image-deps.sh`, `test-artifact-source.sh`, `test-container-stamp.sh` still pass. +- [ ] **Step 5: Commit** — `build: PREBUILT=1 runs the smoke targets on files already in artifacts/`. + +### Task 3: Parallel smoke jobs in `ci-local.yml` + +**Files:** +- Modify: `.github/workflows/ci-local.yml` +- Create: `scripts/ci-check-prebuilt.sh` +- Create: `test/integration/test-ci-check-prebuilt.sh`; add it to the `test-integration` recipe + +**Interfaces:** +- Consumes: `PREBUILT=1` (Task 2). +- Produces: `scripts/ci-check-prebuilt.sh record|verify`. `record` writes `artifacts/prebuilt.sha256` (sha256 of the five files, paths relative to the checkout). `verify` exits 1 with `::error::` if the manifest is missing, if `sha256sum -c` fails, or if `${CONTAINER_RUNTIME:-podman} images -q --filter reference='thinfloe-ubuntu*'` prints anything. + +- [ ] **Step 1: Failing test** `test-ci-check-prebuilt.sh` (scratch dir, `CONTAINER_RUNTIME` stub): record then verify → exit 0; a changed `ramroot.sqsh` → exit 1; no manifest → exit 1; a stub that lists an image → exit 1. +- [ ] **Step 2: Run, expect FAIL** (script missing). +- [ ] **Step 3: Implement the script**, then the workflow: + - `image`: setup `all`, `make test-integration MEMCAP=`, `make image MEMCAP=`, `scripts/ci-check-prebuilt.sh record`, `actions/upload-artifact@v7` (name `prebuilt`, the five files and the manifest, `retention-days: 1`, `compression-level: 0`, `if-no-files-found: error`), then `make test-runtime MEMCAP=`. No smoke log upload here. + - `smoke`: `needs: image`, `strategy.fail-fast: false`, matrix `include` of three groups — `upgrade`: `qemu-smoke-upgrade`; `sticks`: `qemu-smoke-two-sticks qemu-smoke-write qemu-smoke-grow`; `boot`: `qemu-smoke qemu-smoke-netboot qemu-smoke-addons qemu-smoke-timezone qemu-smoke-timezone-netboot`. Steps: checkout, setup `all` (podman unshare, KVM, user namespaces), download `prebuilt` into the checkout, `make PREBUILT=1 MEMCAP= ${{ matrix.targets }}`, then `scripts/ci-check-prebuilt.sh verify` with `if: success() || failure()`, then the log upload (`ci-logs-${{ matrix.group }}`, `if: failure()`). +- [ ] **Step 4: Run** the new test, `shellcheck`, and `actionlint .github/workflows/ci-local.yml` if installed; expect clean. +- [ ] **Step 5: Commit** — `ci: the nine smoke tests run in three parallel jobs on the image job's files`. + +### Task 4: Measure, prove the failure path, decide + +**Files:** Modify `docs/release-ci.md` (CI split, Measurements), `ci-local.yml` header comment. + +- [ ] **Step 1:** Push PR 2 (owner OK). Take two green runs; `local` time per run from `gh run view --json jobs` (first `startedAt` to last `completedAt` of the `local / *` jobs). A group longer than 6 min: move one target to another group and run once more. +- [ ] **Step 2: Negative run.** A throwaway branch `devpace-ci-negative` with one extra commit (`exit 1` at the top of `scripts/run-qemu-grow-smoke.sh`), as a draft PR (owner OK). Expected: `local / smoke (sticks)` red, `local` failure, `ci-passed` red. Close the PR and delete the branch (owner OK). +- [ ] **Step 3: Decide.** Mean of the two PR 2 runs vs. the two baseline runs. A drop of ≥4 min: keep. Else: revert Tasks 2-3 (`git revert`) and keep only the numbers. +- [ ] **Step 4: Docs.** `docs/release-ci.md`: the new split (jobs, groups, `PREBUILT=1`, the guard) and a table of the four runs; if `local` still takes > 20 min, the part-6 recommendation from the spec. Commit — `docs(ci): the parallel smoke jobs and their times`. + +### Task 5: Parallel Python unit tests + +**Files:** +- Create: `scripts/run-unit-tests.py` +- Create: `test/unit/test_unit_runner.py` +- Modify: `Makefile` `test-unit` (the `unittest discover` line) + +**Interfaces:** +- Produces: `python3 scripts/run-unit-tests.py [-j N] DIR`. Runs `python3 -m unittest -v ` for each `DIR/test_*.py` (top level only, sorted), cwd `DIR`, environment passed through, at most N at once (`concurrent.futures`, default 4). As each module ends it prints `=== : ok|FAIL ( s) ===` and then that module's whole output (stdout and stderr merged). Last line `unit: modules, failed[: ]`. Exit 0 if all pass, 1 if any failed, 2 if no module was found or DIR is not a directory. + +- [ ] **Step 1: Failing tests** in `test_unit_runner.py` (fixtures written into `tempfile.mkdtemp()` at run time, the runner called with `subprocess`): + - `test_all_pass_exits_zero`: two passing modules → rc 0, last line `unit: 2 modules, 0 failed`. + - `test_one_failure_exits_one`: one passing, one with `self.fail()` → rc 1, last line ends `1 failed: test_b.py`, the traceback appears after the `=== test_b.py: FAIL` header. + - `test_import_error_counts_as_failure`: a module with a syntax error → rc 1. + - `test_no_modules_is_an_error`: empty dir → rc 2; missing dir → rc 2. + - `test_output_is_not_interleaved`: two modules that each print 200 numbered lines with sleeps → each module's lines are contiguous under its header. + - `test_limit_is_respected`: `-j 2`, four modules that append start/stop times to a shared file → never more than 2 overlap. +- [ ] **Step 2: Run** `python3 -m unittest -v test_unit_runner` in `test/unit` → FAIL (script missing). +- [ ] **Step 3: Implement** the script; in `test-unit` replace the discover line with `@$(TEST_ENV) python3 scripts/run-unit-tests.py -j $(TEST_THREADS) test/unit`. +- [ ] **Step 4: Run, expect PASS**; then `make test-unit` three times in a row: the Python total is 971 plus the new tests each time, no FAIL. Put the wall time before and after in the commit message. +- [ ] **Step 5: Commit** — `test: make test-unit runs the Python test modules in parallel`. + +### Task 6: `make test-menu` + +**Files:** Modify `Makefile` (next to `test-unit`), `test/integration/test-make-targets.sh`, `CLAUDE.md` (Commands). + +**Interfaces:** +- Produces: variable `MENU_CARGO_TEST` (the `cargo test` line of `test-unit`, shared by both targets); target `test-menu` (no prerequisites) in `$(MENU_DIR)`: `cargo fmt --check`, `CARGO_BUILD_JOBS=$(BUILD_JOBS) $(MEMCAP) cargo clippy --locked --all-targets -j $(BUILD_JOBS) -- -D warnings`, then `$(MENU_CARGO_TEST)`. + +- [ ] **Step 1: Check today's state:** `cargo clippy --all-targets -- -D warnings` and `cargo fmt --check` in `src/thinfloe-menu`. If clippy warns, drop `-D warnings` from the target and list the warnings in the PR body's follow-up list; do not fix them here. +- [ ] **Step 2: Failing test** in `test-make-targets.sh`: `dry test-menu` contains `cargo fmt --check`, `cargo clippy`, `cargo test` and no `podman`, `export-rootfs.sh`, `unittest` → "test-menu needs no image". +- [ ] **Step 3: Implement**; `test-unit` uses `$(MENU_CARGO_TEST)` too. +- [ ] **Step 4: Run** the test, then `make test-menu` → cargo lib tests 1241 passed / 6 ignored (or the current count), exit 0. +- [ ] **Step 5:** CLAUDE.md Commands: `make test-menu # fmt, clippy and cargo test of the menu; no image`. Commit — `test: make test-menu checks the menu crate without an image`. + +### Task 7: Menu test fixtures with `..Default::default()` + +**Files:** Modify the test modules of `src/thinfloe-menu/src/nettab.rs` (~694), `menu.rs` (~3534), `ui.rs` (~4053, ~4611, ~5670), and any other test-only struct literal the sweep finds. No production code (`netdetail.rs:65`, `app.rs:674` stay as they are). + +- [ ] **Step 1: Sweep.** In the `#[cfg(test)]` modules and `tests/`, list the struct literals that name every field of a struct with `Default` (start with `netdetail::Details`, `IfDetail`, `timetab::TimeStatus`, then the structs of `graphics`, `status/`, `config.rs`). The list goes in the commit message. +- [ ] **Step 2: Prove the problem:** add a temporary field `pub probe: u8` to `Details`; `cargo test --no-run` → compile errors at the listed literals. Remove the field. +- [ ] **Step 3: Rewrite** each listed literal to name only the fields the test asserts on, plus `..Default::default()`. Add `#[derive(Default)]` only where every field has one and a default value means something; else leave the struct alone and say so in the list. +- [ ] **Step 4: Verify:** the probe field again → `cargo test --no-run` compiles; remove it; `cargo clippy --all-targets` clean, `cargo test` the same pass count as before. +- [ ] **Step 5: Commit** — `test(menu): fixtures name only the fields they test`. + +### Task 8: Workflow rules in CLAUDE.md + +**Files:** Modify `CLAUDE.md` (new `### Workflow` under "Rules that are easy to break", before `### Build`; Commands). + +- [ ] **Step 1: Write** three short rules from spec parts 1-3, in the file's style: the `round/` integration branch and one PR; the gates (the full set before the first push; after a rebase with no conflict outside `CHANGES.md` only `make test-menu`; after a code conflict the full set again); the review loop (one review, one fix wave, one scoped re-review, the rest to the PR body's follow-up list unless a real bug; per branch, before it joins the round). Commands: `make qemu-smoke… PREBUILT=1` with one line on what it needs. +- [ ] **Step 2: Check** that `test-public-content.sh` passes (CLAUDE.md is public). +- [ ] **Step 3: Commit** — `docs: workflow rules for rounds, gates and reviews`. +- [ ] **Step 4: Branch gates and review:** `make test-unit`, `make test`, `make test-runtime`, `make qemu-smoke PREBUILT=1` after a fresh `make image`; one whole-branch review, one fix wave, one scoped re-review. Then PR 2 is ready for the owner's merge. diff --git a/docs/superpowers/specs/2026-11-21-dev-pace-design.md b/docs/superpowers/specs/2026-11-21-dev-pace-design.md new file mode 100644 index 00000000..5f0db33b --- /dev/null +++ b/docs/superpowers/specs/2026-11-21-dev-pace-design.md @@ -0,0 +1,194 @@ +# Dev pace: faster rounds from branch to main + +Date: 2026-11-21. Owner approved all six proposals on 2026-11-21; the +controller made the calls marked "controller's call". + +## Intent + +A feature round (several branches, 2026-11-20) cost about four hours of +waiting between the first push and the last merge. The tests themselves are +not slow; the time goes into merging PRs one after the other (every merge +makes the next PR out of date, and `local / image` takes 29-33 min per +run), repeated local gates after each rebase, and review loops that fix +every minor finding. The goal is a shorter round without weaker checks on +what reaches `main`. + +## Goals + +1. One CI run per round instead of one per branch. +2. Fewer local gate runs: the full set once, a short set after a rebase. +3. A shorter `local / image` job. +4. Faster local tests, and a menu-only target that needs no image build. +5. Tests that do not break when another branch adds a struct field. +6. A review loop with a fixed end. + +## Non-goals + +- No change to the repo-infra ruleset standard: `strict` up-to-date checks + stay on, no merge queue (controller's call: both are standard-wide + changes, and an integration branch makes `strict` cheap). +- No change to what the tests check, and no test removed. The smoke tests + and `test-runtime` still run on every PR. +- No change to `release-build-local.yml` in this round (it runs the same + smoke tests; it can follow the CI split later if that works). +- `CHANGES.d/` fragments: this waits for oposs/repo-infra#172. Nothing in + thinfloe until that lands. + +## Anti-goals + +- A green `ci-passed` on a build that did not run every test. The + `if: always()` logic in `ci.yml` and the list of jobs in `ci-passed`'s + `needs:` must cover every new job. +- A CI job that silently rebuilds the container instead of using the one + built in the job before it (then the split costs time instead of saving + it, and tests may run on a different build). +- More than 4 cores used locally. + +## Measurements (run 44822629521, `local / image`, 32 min) + +| Step | Time | +|------|------| +| runner setup | 0:13 | +| `test-unit` inside `make test` (also run by `local / unit`) | ~4:45 | +| container build (labwc 1:20, menu 3:00, plymouth art 0:10, rootfs 2:50) | ~7:40 | +| rootfs export, mksquashfs | ~2:20 | +| integration tests | ~2:00 | +| `make image` | 0:11 | +| nine smoke tests, in sequence (upgrade 3:49, two-sticks 1:18, write 1:14) | ~9:40 | +| `test-runtime` | 4:57 | + +## The parts + +### 1. Integration branch per round (practice, in CLAUDE.md) + +For a round with more than one feature branch: the branches are rebased in +turn onto one `round/` branch, the full local gates run once on that +branch, and one PR goes to `main`. A branch that is ready alone and has no +round open still goes as its own PR. The rebase merge onto `main` keeps +every branch's commits. + +### 2. Gate rules (CLAUDE.md, and memory) + +- Before the first push of a branch: the full local gates (`make test-unit`, + `make test`, `make test-runtime` where the change touches the session or + the menu, and the smoke tests the change touches). +- After a rebase with no conflict outside `CHANGES.md`: only + `cargo clippy --all-targets` and `cargo test` in `src/thinfloe-menu`; CI + does the rest. +- After a rebase with a code conflict: the full gates again. + +### 3. Review rule (CLAUDE.md, and memory) + +One whole-branch review, one fix wave, one scoped re-review of the fixes. +After that, a finding goes to a follow-up list in the PR body unless it is +a real bug (wrong behaviour, data loss, security, a broken constraint). +This replaces the old rule "fix every minor". + +Controller's call for a round: the loop runs per feature branch, before +the branch joins the round branch (the reviews of several branches can run +at the same time, and each diff stays one coherent change). The round +branch gets no second whole review; only conflict resolutions made during +the rebases are checked. The follow-up lists go into the round PR's body, +one heading per branch; the controller offers them to the owner after the +merge, as with any other noticed defect. + +### 4. CI: no duplicate unit run + +`make test` depends on `test-unit`. Add a target (controller's call: +`test-integration`, the current `test` recipe without the `test-unit` +prerequisite; `test` becomes `test-unit test-integration`) and let the +image job call `make test-integration`. Locally `make test` stays as it +is. Expected saving: about 4:45. + +### 5. CI: smoke tests and runtime tests in parallel jobs + +The `image` job builds the container, exports the rootfs, runs the +integration tests and `make image`, uploads `artifacts/rootfs/` and +`artifacts/images/` as an actions artifact, and then runs `test-runtime` +itself. The smoke tests run in parallel jobs that download the artifact. +Controller's call: `test-runtime` stays in the `image` job because it is +the only step that needs the `thinfloe-ubuntu` podman image (as the base +of the runtime-test image); moving it would mean `podman save`/`load` of a +multi-GB image, which eats the saving. The smoke tests need only files. +Grouping, by measured time: (a) `qemu-smoke-upgrade`; (b) +`qemu-smoke-two-sticks`, `-write`, `-grow`; (c) `qemu-smoke`, `-netboot`, +`-addons`, `-timezone`, `-timezone-netboot`. The builder may change the +groups after measuring. + +The hand-over is the hard part: + +- `make` in the smoke jobs must not build the container or export the + rootfs again (there is no podman image there). The stamp + `.ubuntu-container` holds the image ID and an input hash; the smoke + targets need a way to run on the downloaded files without their build + prerequisites. How is left to the builder. The guard is a check in each + smoke job that fails if a `thinfloe-ubuntu` container build or a rootfs + export starts (not any `podman build`: `test-runtime` builds its own + test image legitimately), so a silent rebuild turns the job red. +- `ci-passed` in `ci.yml` lists `local` only; the new jobs live inside + `ci-local.yml`, so a failure in any of them fails `local`. Check that + this holds for jobs with `needs:` inside a called workflow. +- The failure log upload (`artifacts/images/*.log*`) moves to each smoke + job. + +Measurement (controller's call): part 4 goes to `main` first as its own +PR; its CI runs on `main` give the baseline. Part 5 is judged on two CI +runs of its PR against two baseline runs, `local` time from the first job +start to the last job end. Expected: about 32 → 27 min after part 4, about +27 → 19 min after part 5 (the smoke chain of 9:40 becomes the longest +group, ~4 min, plus ~1.5 min for setup and download, and it overlaps +`test-runtime`). Keep part 5 if the mean drops by at least 4 min; else +revert it and record the numbers in `docs/release-ci.md`. + +### 6. CI: podman layer cache (not in this round) + +Controller's call: not built in this round, whatever the numbers. If +`local` still takes more than 20 min after parts 4 and 5, +`docs/release-ci.md` records the measured time and the recommendation +(actions cache of the podman storage, within the 10 GB repo limit, or a +registry cache, which needs `packages: write` in a repo-infra caller and so +goes through repo-infra first). + +### 7. Local: parallel Python unit tests + +`make test-unit` runs the 8 `test_*.py` modules of `test/unit/` with at most +4 processes at once (one `python3 -m unittest -v ` per module, no +new dependency), keeps each module's output together, and fails if any +module fails. Controller's call: a small script under `scripts/` with +`$(TEST_THREADS)` as the process count. `cargo test` follows the Python +tests as today. + +### 8. Local: `make test-menu` + +`cargo fmt --check`, `cargo clippy --all-targets -- -D warnings` (with the +flags the project uses today, if they differ) and `cargo test` in +`src/thinfloe-menu`, with `BUILD_JOBS` and `TEST_THREADS`. No container, no +image. `test-runtime` needs the rootfs and stays out. + +### 9. Test fixtures with `..Default::default()` + +Struct literals in the menu's tests that list every field (the case that +broke: `netdetail::Details` and its `time` field) use +`..Default::default()` for the fields the test does not care about. Where a +struct has no `Default`, add a derive if every field has one; if a field +cannot have a sensible default, leave that struct alone. Production code is +not changed. Scope: the test modules and `tests/`; the builder lists which +structs it touched. + +## Docs + +- CLAUDE.md: parts 1-3 as short rules under "Rules that are easy to + break" or a new short "Workflow" section; the new make targets in + "Commands". +- `docs/release-ci.md`: the new CI split and the measured times, before and + after. +- CHANGES.md: no entry for parts 1-3 and 9 (nothing users see). Parts 4-8 + change only developer targets; the controller's call is no entry either, + since CHANGES.md readers are users and administrators. + +## Left to the builder + +- How the downstream jobs avoid the rebuild (section 5). +- The exact smoke grouping after the first measured run. +- The parallel runner script's name and output format. +- Which fixture structs get `..Default::default()`. diff --git a/test/integration/test-make-targets.sh b/test/integration/test-make-targets.sh new file mode 100755 index 00000000..76d0f9f3 --- /dev/null +++ b/test/integration/test-make-targets.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Offline tests for the shape of the test targets in the Makefile. +# +# CI's image job runs `make test-integration`; the unit tests have a job of +# their own, so the image job must not run them a second time. `make test` +# still runs both. +# +# Runs `make -n` on the real Makefile in a scratch tree with a stub container +# runtime; nothing is built. +# +# Usage: test/integration/test-make-targets.sh + +set -uo pipefail + +REPO="$(cd "$(dirname "$0")/../.." && pwd)" +# shellcheck source=test/integration/lib/scratch-tree.sh +source "$REPO/test/integration/lib/scratch-tree.sh" + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +fail=0 +ok() { echo "ok - $1"; } +bad() { echo "FAIL - $1"; fail=1; } + +RUNTIME="$WORK/podman" +cat >"$RUNTIME" <<'STUB' +#!/bin/bash +case "$1 ${2:-}" in + "image inspect") echo sha256:aaaa ;; + *) exit 125 ;; +esac +STUB +chmod +x "$RUNTIME" + +TREE="$WORK/tree" +mkdir -p "$TREE/containers/image-builder" "$TREE/containers/ubuntu/patches" "$TREE/containers/ubuntu/slim" \ + "$TREE/src/thinfloe-menu/src" "$TREE/artifacts/rootfs" "$TREE/artifacts/images" "$TREE/configs" "$TREE/scripts/lib" +cp "$REPO/Makefile" "$TREE/Makefile" +tf_guard_tree "$TREE" "$WORK" +inputs=(RELEASE containers/ubuntu/Dockerfile containers/image-builder/Dockerfile src/thinfloe-menu/src/main.rs + configs/syslinux.cfg scripts/make-bootable-image.sh scripts/export-rootfs.sh scripts/lib/common.sh) +for f in "${inputs[@]}"; do : >"$TREE/$f"; done +echo 4.2.0 >"$TREE/RELEASE" + +# Prints what `make -n [VAR=val...]` would run in the scratch tree. +dry() { + env -u MAKEFLAGS -u MAKELEVEL make -n -C "$TREE" CONTAINER_RUNTIME="$RUNTIME" \ + TF_VERSION=4.2.0-dev+abc "$@" 2>&1 +} + +out=$(dry test-integration) +if grep -q 'test-firmware.sh' <<<"$out" && ! grep -qE 'unittest|cargo test' <<<"$out"; then + ok "test-integration runs no unit tests" +else + bad "test-integration runs no unit tests" +fi + +out=$(dry test) +if grep -q 'unittest' <<<"$out" && grep -q 'cargo test' <<<"$out" && grep -q 'test-firmware.sh' <<<"$out"; then + ok "make test still runs the unit and the integration tests" +else + bad "make test still runs the unit and the integration tests" +fi + +exit $fail