From 4bac0caed1a11323f6fd908be5ec16b4ecb45a7b Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Tue, 10 Nov 2026 05:55:15 +0530 Subject: [PATCH] ci: shard the dev checks in the Python 3.14 test parts The whole dev/tests suite ran inside 3.14 part 2 beside two other shards and set the length of every merge-queue run: 76-97 s, against 60 s or less for every other job (10 merge_group runs, 2026-11-09/10). Inside that job the dev suite took 57-87 s; the engine and audit-tools shards ended by 59 s. dev/test_shard.py now knows the dev suite, and test-part shards it (TEST_SHARDED_SUITES). Shard 1 is test_implement_eval alone (38 s locally); shard 2 holds the other 20 modules (14 s), so the small whole targets (test-implement-tools, verify-rules) move to part 2. The 3.11 test-dev job, coverage-part and coverage-combine are unchanged; coverage.py never measured dev/. Locally, on 4 CPUs with Python 3.14, the slower part drops from 53 s to 38 s, with the same test counts per suite. Roadmap-Step: CS6 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 7 ++++--- Makefile | 25 ++++++++++++++++--------- dev/test_shard.py | 4 +++- dev/test_shard_weights.json | 23 +++++++++++++++++++++++ dev/tests/test_ci_parity.py | 27 ++++++++++++++++++++------- dev/tests/test_test_shard.py | 2 +- docs/reference/branch-protection.md | 5 +++-- docs/reference/dev-tooling.md | 2 +- 8 files changed, 71 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18aad898..d85cad17 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,9 +126,10 @@ jobs: version-file: dev/tools/pyproject.toml - run: make test-dev - # Every suite on the newest Python, in parts with one shard of each sharded suite (`make test-part`; - # TEST_PART_ in the Makefile). Each part runs its targets at once, as `make test` runs its suites, so this job also proves the - # parallel run on every pull request. The nightly workflow runs the unsharded `make test`. + # Every suite on the newest Python, in parts with one shard of each suite in TEST_SHARDED_SUITES, the dev checks + # included (`make test-part`; TEST_PART_ in the Makefile). Each part runs its targets at once, as `make test` runs + # its suites, so this job also proves the parallel run on every pull request. The nightly workflow runs the + # unsharded `make test`. test-py314: name: test (part ${{ matrix.shard }}, Python 3.14) runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index e0416ccd..9ed26de4 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ endif ifndef TEST_SUITES_SUBMAKE .NOTPARALLEL: endif -.PHONY: help setup format format-check format\:check lint typecheck test $(TEST_SUITES) test-engine-shard test-audit-tools-shard test-part coverage coverage-engine coverage-engine-shard coverage-audit-tools-shard coverage-part coverage-audit-tools coverage-implement-tools coverage-combine verify-rules crosscheck recipe-lint masters-check update-masters check security security-secrets security-deps update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-prose docs-mermaid changelog-fix workflow-lint commit-lint spell spell-fix +.PHONY: help setup format format-check format\:check lint typecheck test $(TEST_SUITES) test-engine-shard test-audit-tools-shard test-dev-shard test-part coverage coverage-engine coverage-engine-shard coverage-audit-tools-shard coverage-part coverage-audit-tools coverage-implement-tools coverage-combine verify-rules crosscheck recipe-lint masters-check update-masters check security security-secrets security-deps update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-prose docs-mermaid changelog-fix workflow-lint commit-lint spell spell-fix help: @echo "verbs: format format:check (format-check) lint typecheck test check security" @@ -161,9 +161,12 @@ test-dev: # CI splits the slow suites (SHARDED_SUITES) into TEST_SHARDS parts that run in separate jobs; SHARD= picks one # (1 to TEST_SHARDS). dev/test_shard.py finds the modules at run time and balances them by # dev/test_shard_weights.json; every module is in exactly one shard, and a shard that selects no test fails. `test` -# and `coverage` never shard. +# and `coverage` never shard. TEST_SHARDED_SUITES are the suites that test-part shards: SHARDED_SUITES, plus the dev +# checks, which coverage.py does not measure. The dev checks are sharded only in the Python 3.14 parts, where the whole +# suite set the length of the run (76-97 s, against 60 s or less for every other job; measured 2026-11-10). TEST_SHARDS := 2 SHARDED_SUITES := engine audit-tools +TEST_SHARDED_SUITES := engine audit-tools dev test_shard = $(CURDIR)/dev/test_shard.py --suite $(1) --shard $(SHARD)/$(TEST_SHARDS) require_shard = @test -n "$(SHARD)" || { echo "$@: set SHARD to a number from 1 to $(TEST_SHARDS)" >&2; exit 1; } @@ -175,15 +178,19 @@ test-audit-tools-shard: $(require_shard) $(PYTHON) $(call test_shard,audit-tools) -# Every suite of `test` in TEST_SHARDS parts, for CI's Python 3.14 jobs: part is shard of each sharded suite -# plus the targets in TEST_PART_, run at once as `test` runs its suites. test-dev takes about as long as an engine -# shard, so it shares a part with no other whole suite. dev/tests/test_ci_parity.py fails when the parts leave out a -# suite of `test`. -TEST_PART_1 := test-implement-tools verify-rules -TEST_PART_2 := test-dev +test-dev-shard: + $(require_shard) + $(PYTHON) $(call test_shard,dev) + +# Every suite of `test` in TEST_SHARDS parts, for CI's Python 3.14 jobs: part is shard of each suite in +# TEST_SHARDED_SUITES plus the targets in TEST_PART_, run at once as `test` runs its suites. The dev checks' first +# shard is test_implement_eval alone (about 40 s), so the small whole targets go with the second. +# dev/tests/test_ci_parity.py fails when the parts leave out a suite of `test`. +TEST_PART_1 := +TEST_PART_2 := test-implement-tools verify-rules test-part: $(require_shard) - $(MAKE) --no-print-directory TEST_SUITES_SUBMAKE=1 $(TEST_JOBS) $(SHARDED_SUITES:%=test-%-shard) $(TEST_PART_$(SHARD)) + $(MAKE) --no-print-directory TEST_SUITES_SUBMAKE=1 $(TEST_JOBS) $(TEST_SHARDED_SUITES:%=test-%-shard) $(TEST_PART_$(SHARD)) # Runs the engine's suite and the tools' suites under coverage.py and fails when the total drops below the floor in # .coveragerc (`fail_under`). Not part of `check`; needs uv. Each coverage- writes its own data file diff --git a/dev/test_shard.py b/dev/test_shard.py index 33e095eb..7a18e78b 100644 --- a/dev/test_shard.py +++ b/dev/test_shard.py @@ -4,7 +4,8 @@ """Runs one shard of a unittest suite, so CI can split a slow suite over jobs that run at once. ``python3 dev/test_shard.py --suite engine --shard 1/2`` finds the suite's test modules at run time, assigns them -to shards by the weights in ``dev/test_shard_weights.json`` and runs the modules of one shard. Each module is loaded +to shards by the weights in ``dev/test_shard_weights.json`` and runs the modules of one shard. The suites are the +engine's, the audit tools' and the dev checks' (``SUITES``). Each module is loaded as ``python3 -m unittest discover`` loads it, from the same working directory and ``sys.path``. A module with no weight gets ``DEFAULT_WEIGHT``, so a new module is always in exactly one shard. A shard that selects no test exits 1, so a shard cannot pass by running nothing. @@ -38,6 +39,7 @@ SUITES = { "engine": (REPO / "audit" / "engine", "tests"), "audit-tools": (REPO, "audit/tools/tests"), + "dev": (REPO, "dev/tests"), } #: The weight of a module that the weights file does not name yet (seconds). DEFAULT_WEIGHT = 1.0 diff --git a/dev/test_shard_weights.json b/dev/test_shard_weights.json index af06adf2..8ac83b89 100644 --- a/dev/test_shard_weights.json +++ b/dev/test_shard_weights.json @@ -21,6 +21,29 @@ "test_summary": 0.0, "test_validate_report": 0.4 }, + "dev": { + "test_build_masters": 0.2, + "test_build_skills": 1.9, + "test_ci_parity": 0.0, + "test_commit_lint": 0.1, + "test_eval_expected_fixtures": 0.1, + "test_eval_score": 0.2, + "test_implement_eval": 38.3, + "test_implement_eval_cli": 10.2, + "test_mcp_config": 0.0, + "test_mcp_config_tool": 0.0, + "test_mermaid_check": 0.0, + "test_pointer_skills": 0.0, + "test_prose_check": 0.0, + "test_prose_engine_parity": 0.1, + "test_pyright_config": 0.0, + "test_quality_baseline": 0.0, + "test_recipe_lint": 1.4, + "test_release_config": 0.0, + "test_release_provenance": 0.0, + "test_test_shard": 0.0, + "test_tool_pins": 0.1 + }, "engine": { "test_agt_findings": 3.1, "test_agt_pack": 2.9, diff --git a/dev/tests/test_ci_parity.py b/dev/tests/test_ci_parity.py index 362b64fd..5cce3ee5 100644 --- a/dev/tests/test_ci_parity.py +++ b/dev/tests/test_ci_parity.py @@ -7,7 +7,8 @@ they run at once. These tests fail when a step of ``check`` (or a suite of ``test``) has no CI job, so a step added to ``check`` cannot silently drop out of the gate that decides a pull request. -CI also splits the slow suites into shards (``SHARDED_SUITES``, ``TEST_SHARDS``, ``dev/test_shard.py``). These tests fail when a +CI also splits the slow suites into shards (``SHARDED_SUITES``, ``TEST_SHARDED_SUITES``, ``TEST_SHARDS``, +``dev/test_shard.py``). These tests fail when a matrix misses a shard, when a shard leaves the coverage environment, or when a verb (``test``, ``coverage``) starts to shard. The nightly workflow runs the verbs whole. """ @@ -85,6 +86,7 @@ def setUp(self) -> None: self.suites = make_variable(self.makefile, "TEST_SUITES") self.coverage_suites = make_variable(self.makefile, "COVERAGE_SUITES") self.sharded = make_variable(self.makefile, "SHARDED_SUITES") + self.test_sharded = make_variable(self.makefile, "TEST_SHARDED_SUITES") self.count = int(make_variable(self.makefile, "TEST_SHARDS")[0]) def test_every_step_of_check_has_a_ci_job(self) -> None: @@ -96,12 +98,17 @@ def runs(self, target: str, targets: set[str]) -> bool: """``target`` runs whole, or in shards (test_shard_matrices_list_every_shard checks that none is missing).""" return target in targets or f"{target}-shard" in targets + def sharded_by(self, kind: str) -> tuple[str, list[str]]: + """The variable that names the suites ``make -part`` shards, and its suites.""" + return ("TEST_SHARDED_SUITES", self.test_sharded) if kind == "test" else ("SHARDED_SUITES", self.sharded) + def parts(self, kind: str) -> set[str]: """Every target that ``make -part`` runs over its TEST_SHARDS parts: each sharded suite's shard target, plus ``_PART_``.""" recipe = make_recipe(self.makefile, f"{kind}-part") - self.assertIn(f"$(SHARDED_SUITES:%={kind}-%-shard) $({kind.upper()}_PART_$(SHARD))", recipe) - targets = {f"{kind}-{suite}-shard" for suite in self.sharded} + variable, suites = self.sharded_by(kind) + self.assertIn(f"$({variable}:%={kind}-%-shard) $({kind.upper()}_PART_$(SHARD))", recipe) + targets = {f"{kind}-{suite}-shard" for suite in suites} for shard in range(1, self.count + 1): targets.update(make_variable(self.makefile, f"{kind.upper()}_PART_{shard}")) return targets @@ -197,7 +204,7 @@ def test_shard_matrices_list_every_shard(self) -> None: # A job that runs a shard target must run it once for each of the TEST_SHARDS shards, or a part of a suite # never runs. Each shard reads its number from the job's SHARD variable. shard_targets = {"test-part", "coverage-part"} | { - f"{kind}-{suite}-shard" for kind in ("test", "coverage") for suite in self.sharded + f"{kind}-{suite}-shard" for kind in ("test", "coverage") for suite in self.sharded_by(kind)[1] } sharded = [] for job, block in ci_jobs(CI.read_text(encoding="utf-8")).items(): @@ -210,8 +217,10 @@ def test_shard_matrices_list_every_shard(self) -> None: self.assertEqual(sorted(sharded), ["test-coverage", "test-py314"], "both Pythons run the suites in shards") def test_the_makefile_and_the_runner_shard_the_same_suites(self) -> None: - self.assertEqual(sorted(self.sharded), sorted(test_shard.SUITES)) - for suite in self.sharded: + # test-part shards every suite the runner knows; coverage-part shards a subset (coverage.py skips dev/). + self.assertEqual(sorted(self.test_sharded), sorted(test_shard.SUITES)) + self.assertLessEqual(set(self.sharded), set(self.test_sharded)) + for suite in self.test_sharded: with self.subTest(suite=suite): self.assertIn(f"test-{suite}", self.suites) self.assertIn(f"$(call test_shard,{suite})", make_recipe(self.makefile, f"test-{suite}-shard")) @@ -227,7 +236,11 @@ def test_a_coverage_shard_runs_in_the_coverage_environment(self) -> None: self.assertIn(f"$(call test_shard,{suite})", recipe) def test_the_verbs_never_shard(self) -> None: - rules = ["test", "coverage", *(f"{kind}-{suite}" for kind in ("test", "coverage") for suite in self.sharded)] + rules = [ + "test", + "coverage", + *(f"{kind}-{suite}" for kind in ("test", "coverage") for suite in self.sharded_by(kind)[1]), + ] for rule in rules: with self.subTest(rule=rule): recipe = make_recipe(self.makefile, rule) + " ".join(make_rule(self.makefile, rule)) diff --git a/dev/tests/test_test_shard.py b/dev/tests/test_test_shard.py index 509204e4..a5b152ab 100644 --- a/dev/tests/test_test_shard.py +++ b/dev/tests/test_test_shard.py @@ -33,7 +33,7 @@ def suites(self) -> list[tuple[str, list[str], dict[str, float]]]: ] def test_every_suite_has_modules_and_weights(self) -> None: - self.assertEqual(sorted(test_shard.SUITES), ["audit-tools", "engine"]) + self.assertEqual(sorted(test_shard.SUITES), ["audit-tools", "dev", "engine"]) for name, modules, weights in self.suites(): with self.subTest(suite=name): self.assertGreater(len(modules), 10) diff --git a/docs/reference/branch-protection.md b/docs/reference/branch-protection.md index ecb0f3f9..0873a92c 100644 --- a/docs/reference/branch-protection.md +++ b/docs/reference/branch-protection.md @@ -80,7 +80,8 @@ request, on every merge group of the `dev` merge queue and on every push to `dev jobs and fails unless each one succeeded. The jobs run on Ubuntu (issue #82 tracks macOS and Windows). The first five run the steps of `make check` at once; `dev/tests/test_ci_parity.py` fails when a step of `check` has no job. The engine's and the audit tools' suites run in shards (`TEST_SHARDS` and `SHARDED_SUITES` in the `Makefile`, split by -`dev/test_shard.py`); the parity test also fails when a matrix misses a shard or a part misses a suite: +`dev/test_shard.py`). On Python 3.14 the dev checks run in shards too (`TEST_SHARDED_SUITES`). The parity test also +fails when a matrix misses a shard or a part misses a suite: | Job | What it runs | | --- | --- | @@ -88,7 +89,7 @@ engine's and the audit tools' suites run in shards (`TEST_SHARDS` and `SHARDED_S | `crosscheck` | `make crosscheck` on Python 3.11 and 3.14 | | `test-coverage` | `make coverage-part`, one job per shard: a shard of the engine's and of the audit tools' suites, and the implement tools in part 1, under coverage.py on Python 3.11; each part runs its targets at once | | `test-dev` | `make test-dev` on Python 3.11 | -| `test-py314` | `make test-part`, one job per shard: every suite of `make test` and `verify-rules` on Python 3.14; each part runs its targets at once | +| `test-py314` | `make test-part`, one job per shard: every suite of `make test` and `verify-rules` on Python 3.14, with a shard of the engine's, the audit tools' and the dev checks' suites in each part; each part runs its targets at once | | `coverage` | `make coverage-combine SHARDED=1`: merges the data files (one per shard of a sharded suite) and fails below the floor in `.coveragerc` | | `security-secrets` | `make security-secrets`: the secrets scan, the first part of `make security`; no Go | | `security-deps` | `make security-deps`: the dependency audit (osv-scanner), the second part of `make security` | diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 31b27aa6..09326e37 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -13,7 +13,7 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | --- | --- | --- | --- | --- | | ruff | Python formatting and lint errors (rules in `ruff.toml`) | `make format`, `make format:check`, `make lint` | `static` | `dev/tools/pyproject.toml` | | mypy | Python type errors, strict mode (`mypy.ini`) | `make typecheck` | `static` | `dev/tools/pyproject.toml` | -| unittest | Failing tests: the engine, the audit tools, the implement tools, the dev checks | `make test` (on Linux, its four suites at once; CI splits the engine's and the audit tools' suites into shards with `dev/test_shard.py`) | `test-coverage`, `test-dev`, `test-py314`; unsharded in `nightly.yml` | Python standard library | +| unittest | Failing tests: the engine, the audit tools, the implement tools, the dev checks | `make test` (on Linux, its four suites at once; CI splits the engine's and the audit tools' suites into shards with `dev/test_shard.py`, and the dev checks too on Python 3.14) | `test-coverage`, `test-dev`, `test-py314`; unsharded in `nightly.yml` | Python standard library | | coverage.py | Test coverage below the floor in `.coveragerc` | `make coverage` (in CI: `coverage-part`, then `coverage-combine SHARDED=1`) | `test-coverage`, `coverage`; unsharded in `nightly.yml` | `dev/tools/pyproject.toml` | | jsonschema | The tools' schema validator disagrees with the reference package | `make crosscheck` | `crosscheck` | `dev/tools/pyproject.toml` | | `dev/recipe_lint.py` | Broken links, bad `SKILL.md` frontmatter, a recipe `SKILL.md` that does not link the shared user prompt, oversize files, unknown tools, check IDs and requirement IDs | `make recipe-lint` | `static` | This repository |