From 0ccd527015468f70ba2221040e42f9a2b1b4709d Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Tue, 10 Nov 2026 05:59:00 +0530 Subject: [PATCH 01/11] fix(eval): regenerate the eval-expected files with the current paths The notes in 80 of the files named the old engine path brownfield-recipes/deterministic-audit/, and every file named the generator as tools/eval/. The generator and its test now name dev/eval/ and the real output folder, and all files are regenerated. Roadmap-Step: 1.1 Co-Authored-By: Claude Opus 5.5 --- .../expected/fixtures/agt-01-marker-words-in-prose.json | 2 +- dev/eval/expected/fixtures/agt-01-mis-cased.json | 2 +- dev/eval/expected/fixtures/agt-01-missing.json | 2 +- dev/eval/expected/fixtures/agt-01-shadowed.json | 2 +- dev/eval/expected/fixtures/agt-01-stub.json | 2 +- dev/eval/expected/fixtures/agt-02-uncovered.json | 2 +- dev/eval/expected/fixtures/agt-03-incomplete.json | 2 +- dev/eval/expected/fixtures/agt-04-missing.json | 2 +- dev/eval/expected/fixtures/agt-05-ok.json | 2 +- .../expected/fixtures/agt-05-verbs-named-not-resolved.json | 2 +- dev/eval/expected/fixtures/agt-06-thin.json | 2 +- dev/eval/expected/fixtures/agt-07-missing-deny.json | 2 +- dev/eval/expected/fixtures/agt-08-competitor.json | 2 +- dev/eval/expected/fixtures/agt-09-no-release.json | 2 +- .../expected/fixtures/agt-09-ruleset-branch-unnamed.json | 2 +- dev/eval/expected/fixtures/agt-10-deferred.json | 2 +- dev/eval/expected/fixtures/agt-12-no-rules.json | 2 +- dev/eval/expected/fixtures/base-01-no-baseline.json | 2 +- dev/eval/expected/fixtures/base-02-no-count.json | 2 +- dev/eval/expected/fixtures/base-03-no-ratchet.json | 2 +- dev/eval/expected/fixtures/base-04-no-coverage-floor.json | 2 +- dev/eval/expected/fixtures/base-ok.json | 2 +- dev/eval/expected/fixtures/ci-01-no-pr-trigger.json | 2 +- dev/eval/expected/fixtures/ci-02-parity-break.json | 2 +- dev/eval/expected/fixtures/ci-03-ci-only-step.json | 2 +- dev/eval/expected/fixtures/ci-05-ci-only-lint.json | 2 +- dev/eval/expected/fixtures/ci-05-no-workflow-lint.json | 2 +- dev/eval/expected/fixtures/ci-ok-gh.json | 2 +- dev/eval/expected/fixtures/cmd-01-make-parse-error.json | 2 +- dev/eval/expected/fixtures/cmd-01-maven-no-formatter.json | 2 +- dev/eval/expected/fixtures/cmd-01-missing-verb.json | 2 +- dev/eval/expected/fixtures/cmd-02-weak-check.json | 2 +- dev/eval/expected/fixtures/cmd-03-divergent.json | 2 +- dev/eval/expected/fixtures/cmd-04-write-in-check.json | 2 +- dev/eval/expected/fixtures/cmd-ok-gradle.json | 2 +- dev/eval/expected/fixtures/cmd-ok-make-alias.json | 2 +- dev/eval/expected/fixtures/cmd-ok-make-prereqs.json | 2 +- dev/eval/expected/fixtures/cmd-ok-make.json | 2 +- dev/eval/expected/fixtures/cmd-ok-maven.json | 2 +- dev/eval/expected/fixtures/cmd-ok-npm.json | 2 +- dev/eval/expected/fixtures/cmd-ok-pyproject-poe.json | 2 +- .../expected/fixtures/cmd-ok-root-make-nested-template.json | 2 +- dev/eval/expected/fixtures/con-01-names-existing-paths.json | 2 +- dev/eval/expected/fixtures/con-01-names-missing-paths.json | 2 +- dev/eval/expected/fixtures/con-01-names-no-path.json | 2 +- .../expected/fixtures/con-02-no-prohibitions-section.json | 2 +- dev/eval/expected/fixtures/con-02-no-test-directory.json | 2 +- dev/eval/expected/fixtures/con-02-partly-protected.json | 2 +- dev/eval/expected/fixtures/con-02-protected.json | 2 +- dev/eval/expected/fixtures/con-02-specs-are-not-tests.json | 2 +- dev/eval/expected/fixtures/con-02-unprotected-tests.json | 2 +- .../expected/fixtures/con-03-artefacts-not-ignored.json | 2 +- dev/eval/expected/fixtures/doc-01-empty.json | 2 +- dev/eval/expected/fixtures/doc-02-no-style-guide.json | 2 +- dev/eval/expected/fixtures/doc-03-tracked-artefacts.json | 2 +- dev/eval/expected/fixtures/doc-04-adr-unindexed.json | 2 +- dev/eval/expected/fixtures/doc-05-long-sentences.json | 2 +- dev/eval/expected/fixtures/doc-nav-ok.json | 2 +- dev/eval/expected/fixtures/exec-01-nested-go-unpinned.json | 2 +- dev/eval/expected/fixtures/exec-01-python-source-only.json | 2 +- dev/eval/expected/fixtures/exec-01-unpinned.json | 2 +- dev/eval/expected/fixtures/exec-02-nested-lockfile.json | 2 +- dev/eval/expected/fixtures/exec-02-no-lockfile.json | 2 +- dev/eval/expected/fixtures/exec-03-no-setup.json | 2 +- dev/eval/expected/fixtures/exec-04-missing-env.json | 2 +- dev/eval/expected/fixtures/exec-05-generated.json | 2 +- dev/eval/expected/fixtures/exec-06-broken-command.json | 2 +- dev/eval/expected/fixtures/exec-07-dual-lint.json | 2 +- dev/eval/expected/fixtures/exec-07-dual-python-types.json | 2 +- dev/eval/expected/fixtures/exec-08-duplicate-pin.json | 2 +- dev/eval/expected/fixtures/exec-08-inline-version.json | 2 +- dev/eval/expected/fixtures/exec-ok.json | 2 +- dev/eval/expected/fixtures/governed-minimal.json | 2 +- dev/eval/expected/fixtures/hyg-01-placeholder-readme.json | 2 +- dev/eval/expected/fixtures/hyg-09-target-branch-trap.json | 2 +- dev/eval/expected/fixtures/hyg-09-uncovered-ecosystem.json | 2 +- dev/eval/expected/fixtures/hyg-10-no-license.json | 2 +- .../expected/fixtures/hyg-11-no-source-branch-check.json | 2 +- dev/eval/expected/fixtures/hyg-minimal.json | 2 +- dev/eval/expected/fixtures/idx-01-no-mcp-config.json | 2 +- dev/eval/expected/fixtures/idx-02-no-index.json | 2 +- dev/eval/expected/fixtures/ignore-ok-nested-gitignore.json | 2 +- dev/eval/expected/fixtures/nav-01-no-manifest.json | 2 +- dev/eval/expected/fixtures/nav-02-no-entry.json | 2 +- dev/eval/expected/fixtures/nav-03-red-flags.json | 2 +- dev/eval/expected/fixtures/sec-01-excluded-env.json | 2 +- dev/eval/expected/fixtures/sec-01-ok.json | 2 +- dev/eval/expected/fixtures/sec-01-tracked-env.json | 2 +- dev/eval/expected/fixtures/sec-02-env-reference.json | 2 +- .../expected/fixtures/sec-02-expression-assignment.json | 2 +- dev/eval/expected/fixtures/sec-02-key-shaped.json | 2 +- dev/eval/expected/fixtures/sec-03-ok-negated-exception.json | 2 +- dev/eval/expected/fixtures/sec-03-thin-ignore.json | 2 +- dev/eval/expected/fixtures/tool-01-no-formatter.json | 2 +- dev/eval/expected/fixtures/tool-02-no-linter.json | 2 +- dev/eval/expected/fixtures/tool-03-nonstrict-ts.json | 2 +- dev/eval/expected/fixtures/tool-04-local-only-audit.json | 2 +- .../expected/fixtures/tool-04-make-security-no-audit.json | 2 +- dev/eval/expected/fixtures/tool-04-no-audit-verb.json | 2 +- dev/eval/expected/fixtures/tool-05-unwired-hooks.json | 2 +- dev/eval/expected/fixtures/tool-06-ci-only-commit-rule.json | 2 +- .../expected/fixtures/tool-06-local-only-commit-rule.json | 2 +- dev/eval/expected/fixtures/tool-06-no-commit-rule.json | 2 +- dev/eval/expected/fixtures/tool-07-no-branch-guard.json | 2 +- dev/eval/expected/fixtures/tool-ok-node.json | 2 +- dev/eval/expected/fixtures/tst-01-no-suite.json | 2 +- dev/eval/expected/fixtures/tst-03-skip-markers.json | 2 +- dev/eval/expected/fixtures/tst-04-no-fastpath.json | 2 +- dev/eval/expected/fixtures/tst-05-no-coverage.json | 2 +- dev/eval/generate_fixture_expected.py | 6 +++--- dev/tests/test_eval_expected_fixtures.py | 2 +- 111 files changed, 113 insertions(+), 113 deletions(-) diff --git a/dev/eval/expected/fixtures/agt-01-marker-words-in-prose.json b/dev/eval/expected/fixtures/agt-01-marker-words-in-prose.json index c3f18a19..6d2735f4 100644 --- a/dev/eval/expected/fixtures/agt-01-marker-words-in-prose.json +++ b/dev/eval/expected/fixtures/agt-01-marker-words-in-prose.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-01-marker-words-in-prose", "repo_sha": "33089f531b04c89bc06532ec7a252b99d793427f", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-01-marker-words-in-prose' (brownfield-recipes/deterministic-audit/tests/fixtures/agt-01-marker-words-in-prose), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-01-marker-words-in-prose' (audit/engine/tests/fixtures/agt-01-marker-words-in-prose), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-01", diff --git a/dev/eval/expected/fixtures/agt-01-mis-cased.json b/dev/eval/expected/fixtures/agt-01-mis-cased.json index 55039af5..a6b54ead 100644 --- a/dev/eval/expected/fixtures/agt-01-mis-cased.json +++ b/dev/eval/expected/fixtures/agt-01-mis-cased.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-01-mis-cased", "repo_sha": "8e72730efd1a1e60b481fe3b0a3756cf235d7822", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-01-mis-cased' (brownfield-recipes/deterministic-audit/tests/fixtures/agt-01-mis-cased), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-01-mis-cased' (audit/engine/tests/fixtures/agt-01-mis-cased), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-01", diff --git a/dev/eval/expected/fixtures/agt-01-missing.json b/dev/eval/expected/fixtures/agt-01-missing.json index 4ad612b1..721b9afa 100644 --- a/dev/eval/expected/fixtures/agt-01-missing.json +++ b/dev/eval/expected/fixtures/agt-01-missing.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-01-missing", "repo_sha": "5c282d79af1a0f6307d0648c7cc6e433a3028a78", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-01-missing' (brownfield-recipes/deterministic-audit/tests/fixtures/agt-01-missing), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-01-missing' (audit/engine/tests/fixtures/agt-01-missing), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-01", diff --git a/dev/eval/expected/fixtures/agt-01-shadowed.json b/dev/eval/expected/fixtures/agt-01-shadowed.json index 8bbc571d..f8d89f5a 100644 --- a/dev/eval/expected/fixtures/agt-01-shadowed.json +++ b/dev/eval/expected/fixtures/agt-01-shadowed.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-01-shadowed", "repo_sha": "9481d3d954b43bb6ed0fb6fbed20d53db3a803b1", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-01-shadowed' (brownfield-recipes/deterministic-audit/tests/fixtures/agt-01-shadowed), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-01-shadowed' (audit/engine/tests/fixtures/agt-01-shadowed), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-01", diff --git a/dev/eval/expected/fixtures/agt-01-stub.json b/dev/eval/expected/fixtures/agt-01-stub.json index fcb646c4..bb48bb28 100644 --- a/dev/eval/expected/fixtures/agt-01-stub.json +++ b/dev/eval/expected/fixtures/agt-01-stub.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-01-stub", "repo_sha": "551547e71c80877e832d8c5769b1865850ebc4fe", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-01-stub' (brownfield-recipes/deterministic-audit/tests/fixtures/agt-01-stub), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-01-stub' (audit/engine/tests/fixtures/agt-01-stub), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-01", diff --git a/dev/eval/expected/fixtures/agt-02-uncovered.json b/dev/eval/expected/fixtures/agt-02-uncovered.json index b99e142e..207d5f4f 100644 --- a/dev/eval/expected/fixtures/agt-02-uncovered.json +++ b/dev/eval/expected/fixtures/agt-02-uncovered.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-02-uncovered", "repo_sha": "75b3ee1501e78f6afbb06d0ac149dc13f41fc1f1", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-02-uncovered' (brownfield-recipes/deterministic-audit/tests/fixtures/agt-02-uncovered), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-02-uncovered' (audit/engine/tests/fixtures/agt-02-uncovered), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-02", diff --git a/dev/eval/expected/fixtures/agt-03-incomplete.json b/dev/eval/expected/fixtures/agt-03-incomplete.json index aa4605ac..050736eb 100644 --- a/dev/eval/expected/fixtures/agt-03-incomplete.json +++ b/dev/eval/expected/fixtures/agt-03-incomplete.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-03-incomplete", "repo_sha": "35d6235d53a93cce4383aeef400453954dd64e28", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-03-incomplete' (audit/engine/tests/fixtures/agt-03-incomplete), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-03-incomplete' (audit/engine/tests/fixtures/agt-03-incomplete), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-03", diff --git a/dev/eval/expected/fixtures/agt-04-missing.json b/dev/eval/expected/fixtures/agt-04-missing.json index 81aea269..cc2f1def 100644 --- a/dev/eval/expected/fixtures/agt-04-missing.json +++ b/dev/eval/expected/fixtures/agt-04-missing.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-04-missing", "repo_sha": "85bb5f62204524385bf0592e8c6c6681fbb5788a", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-04-missing' (audit/engine/tests/fixtures/agt-04-missing), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-04-missing' (audit/engine/tests/fixtures/agt-04-missing), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-04", diff --git a/dev/eval/expected/fixtures/agt-05-ok.json b/dev/eval/expected/fixtures/agt-05-ok.json index 657ac36f..80075ecc 100644 --- a/dev/eval/expected/fixtures/agt-05-ok.json +++ b/dev/eval/expected/fixtures/agt-05-ok.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-05-ok", "repo_sha": "288d6457903db3435b654b6d5981051eee981aba", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-05-ok' (brownfield-recipes/deterministic-audit/tests/fixtures/agt-05-ok), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-05-ok' (audit/engine/tests/fixtures/agt-05-ok), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-05", diff --git a/dev/eval/expected/fixtures/agt-05-verbs-named-not-resolved.json b/dev/eval/expected/fixtures/agt-05-verbs-named-not-resolved.json index af080b9b..5b90608c 100644 --- a/dev/eval/expected/fixtures/agt-05-verbs-named-not-resolved.json +++ b/dev/eval/expected/fixtures/agt-05-verbs-named-not-resolved.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-05-verbs-named-not-resolved", "repo_sha": "2b4ee01616a57fc977c251fa39cf8f0c53b81df6", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-05-verbs-named-not-resolved' (brownfield-recipes/deterministic-audit/tests/fixtures/agt-05-verbs-named-not-resolved), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-05-verbs-named-not-resolved' (audit/engine/tests/fixtures/agt-05-verbs-named-not-resolved), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-05", diff --git a/dev/eval/expected/fixtures/agt-06-thin.json b/dev/eval/expected/fixtures/agt-06-thin.json index 14a259b1..e1e62e5a 100644 --- a/dev/eval/expected/fixtures/agt-06-thin.json +++ b/dev/eval/expected/fixtures/agt-06-thin.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-06-thin", "repo_sha": "62addf544f38ceb86119b69fc00eff3652855b5b", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-06-thin' (audit/engine/tests/fixtures/agt-06-thin), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-06-thin' (audit/engine/tests/fixtures/agt-06-thin), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-06", diff --git a/dev/eval/expected/fixtures/agt-07-missing-deny.json b/dev/eval/expected/fixtures/agt-07-missing-deny.json index e5ca3860..7e3fc513 100644 --- a/dev/eval/expected/fixtures/agt-07-missing-deny.json +++ b/dev/eval/expected/fixtures/agt-07-missing-deny.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-07-missing-deny", "repo_sha": "57e5e534ef0afaf0aa568c3d642ae43c1fcb4010", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-07-missing-deny' (audit/engine/tests/fixtures/agt-07-missing-deny), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-07-missing-deny' (audit/engine/tests/fixtures/agt-07-missing-deny), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-07", diff --git a/dev/eval/expected/fixtures/agt-08-competitor.json b/dev/eval/expected/fixtures/agt-08-competitor.json index 9b9d7b5a..7814e081 100644 --- a/dev/eval/expected/fixtures/agt-08-competitor.json +++ b/dev/eval/expected/fixtures/agt-08-competitor.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-08-competitor", "repo_sha": "7fdb3ebd100382052c2c9c919841faea83f1ef0b", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-08-competitor' (brownfield-recipes/deterministic-audit/tests/fixtures/agt-08-competitor), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-08-competitor' (audit/engine/tests/fixtures/agt-08-competitor), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-08", diff --git a/dev/eval/expected/fixtures/agt-09-no-release.json b/dev/eval/expected/fixtures/agt-09-no-release.json index 20c6b6cb..4ab4c7cb 100644 --- a/dev/eval/expected/fixtures/agt-09-no-release.json +++ b/dev/eval/expected/fixtures/agt-09-no-release.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-09-no-release", "repo_sha": "8255596daac385b8c6ddb2fbca3db55898b92ceb", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-09-no-release' (audit/engine/tests/fixtures/agt-09-no-release), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-09-no-release' (audit/engine/tests/fixtures/agt-09-no-release), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-09", diff --git a/dev/eval/expected/fixtures/agt-09-ruleset-branch-unnamed.json b/dev/eval/expected/fixtures/agt-09-ruleset-branch-unnamed.json index 1635e869..73952466 100644 --- a/dev/eval/expected/fixtures/agt-09-ruleset-branch-unnamed.json +++ b/dev/eval/expected/fixtures/agt-09-ruleset-branch-unnamed.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-09-ruleset-branch-unnamed", "repo_sha": "6720857e865a131947fdee09b6df6262c8768d8f", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-09-ruleset-branch-unnamed' (audit/engine/tests/fixtures/agt-09-ruleset-branch-unnamed), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-09-ruleset-branch-unnamed' (audit/engine/tests/fixtures/agt-09-ruleset-branch-unnamed), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-09", diff --git a/dev/eval/expected/fixtures/agt-10-deferred.json b/dev/eval/expected/fixtures/agt-10-deferred.json index b5507708..d709ef56 100644 --- a/dev/eval/expected/fixtures/agt-10-deferred.json +++ b/dev/eval/expected/fixtures/agt-10-deferred.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-10-deferred", "repo_sha": "9c9f549c6b00d8dac5f18b64aff96b31737326ba", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-10-deferred' (brownfield-recipes/deterministic-audit/tests/fixtures/agt-10-deferred), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-10-deferred' (audit/engine/tests/fixtures/agt-10-deferred), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-10", diff --git a/dev/eval/expected/fixtures/agt-12-no-rules.json b/dev/eval/expected/fixtures/agt-12-no-rules.json index 2be7bcab..6d8dc76e 100644 --- a/dev/eval/expected/fixtures/agt-12-no-rules.json +++ b/dev/eval/expected/fixtures/agt-12-no-rules.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "agt-12-no-rules", "repo_sha": "d6d7ec8fd78d9a8c4a72b0a327932d49d5df01ae", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-12-no-rules' (audit/engine/tests/fixtures/agt-12-no-rules), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'agt-12-no-rules' (audit/engine/tests/fixtures/agt-12-no-rules), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-09", diff --git a/dev/eval/expected/fixtures/base-01-no-baseline.json b/dev/eval/expected/fixtures/base-01-no-baseline.json index f7252cba..265ff60b 100644 --- a/dev/eval/expected/fixtures/base-01-no-baseline.json +++ b/dev/eval/expected/fixtures/base-01-no-baseline.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "base-01-no-baseline", "repo_sha": "afcb6e0b540ac7c7541e29bc660e67fc2f18ade9", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'base-01-no-baseline' (brownfield-recipes/deterministic-audit/tests/fixtures/base-01-no-baseline), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'base-01-no-baseline' (audit/engine/tests/fixtures/base-01-no-baseline), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "BASE-01", diff --git a/dev/eval/expected/fixtures/base-02-no-count.json b/dev/eval/expected/fixtures/base-02-no-count.json index b8b662f0..e54c9e4b 100644 --- a/dev/eval/expected/fixtures/base-02-no-count.json +++ b/dev/eval/expected/fixtures/base-02-no-count.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "base-02-no-count", "repo_sha": "4e88bb048da1dbef7988f153031d90bc81ce6ae4", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'base-02-no-count' (brownfield-recipes/deterministic-audit/tests/fixtures/base-02-no-count), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'base-02-no-count' (audit/engine/tests/fixtures/base-02-no-count), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "BASE-02", diff --git a/dev/eval/expected/fixtures/base-03-no-ratchet.json b/dev/eval/expected/fixtures/base-03-no-ratchet.json index 75f9a13e..0d607c92 100644 --- a/dev/eval/expected/fixtures/base-03-no-ratchet.json +++ b/dev/eval/expected/fixtures/base-03-no-ratchet.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "base-03-no-ratchet", "repo_sha": "f19773802cacb04b70634876e67eaf98c7560fbb", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'base-03-no-ratchet' (brownfield-recipes/deterministic-audit/tests/fixtures/base-03-no-ratchet), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'base-03-no-ratchet' (audit/engine/tests/fixtures/base-03-no-ratchet), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "BASE-03", diff --git a/dev/eval/expected/fixtures/base-04-no-coverage-floor.json b/dev/eval/expected/fixtures/base-04-no-coverage-floor.json index 8eead1fe..1dd30ec4 100644 --- a/dev/eval/expected/fixtures/base-04-no-coverage-floor.json +++ b/dev/eval/expected/fixtures/base-04-no-coverage-floor.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "base-04-no-coverage-floor", "repo_sha": "ae2bb01bae3d7e21aa957be41eb61512e33b40e8", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'base-04-no-coverage-floor' (brownfield-recipes/deterministic-audit/tests/fixtures/base-04-no-coverage-floor), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'base-04-no-coverage-floor' (audit/engine/tests/fixtures/base-04-no-coverage-floor), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "BASE-04", diff --git a/dev/eval/expected/fixtures/base-ok.json b/dev/eval/expected/fixtures/base-ok.json index d0df7218..ba973bc2 100644 --- a/dev/eval/expected/fixtures/base-ok.json +++ b/dev/eval/expected/fixtures/base-ok.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "base-ok", "repo_sha": "8f963e4a82b91a9dfec055ddeecbbdc5287ba361", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'base-ok' (brownfield-recipes/deterministic-audit/tests/fixtures/base-ok), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'base-ok' (audit/engine/tests/fixtures/base-ok), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "BASE-01", diff --git a/dev/eval/expected/fixtures/ci-01-no-pr-trigger.json b/dev/eval/expected/fixtures/ci-01-no-pr-trigger.json index e5714456..a032effc 100644 --- a/dev/eval/expected/fixtures/ci-01-no-pr-trigger.json +++ b/dev/eval/expected/fixtures/ci-01-no-pr-trigger.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "ci-01-no-pr-trigger", "repo_sha": "478e463754690702307d993254f646e25f81ba4f", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-01-no-pr-trigger' (audit/engine/tests/fixtures/ci-01-no-pr-trigger), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-01-no-pr-trigger' (audit/engine/tests/fixtures/ci-01-no-pr-trigger), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CI-01", diff --git a/dev/eval/expected/fixtures/ci-02-parity-break.json b/dev/eval/expected/fixtures/ci-02-parity-break.json index e6afe83b..781bd230 100644 --- a/dev/eval/expected/fixtures/ci-02-parity-break.json +++ b/dev/eval/expected/fixtures/ci-02-parity-break.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "ci-02-parity-break", "repo_sha": "9b64c9236d97edf457bf81a1ddfb9a26caf43d8a", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-02-parity-break' (audit/engine/tests/fixtures/ci-02-parity-break), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-02-parity-break' (audit/engine/tests/fixtures/ci-02-parity-break), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CI-02", diff --git a/dev/eval/expected/fixtures/ci-03-ci-only-step.json b/dev/eval/expected/fixtures/ci-03-ci-only-step.json index 2f8b9691..db69bd70 100644 --- a/dev/eval/expected/fixtures/ci-03-ci-only-step.json +++ b/dev/eval/expected/fixtures/ci-03-ci-only-step.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "ci-03-ci-only-step", "repo_sha": "be0baaaf44f34d1e694b1899d6d72df3d2e70dee", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-03-ci-only-step' (audit/engine/tests/fixtures/ci-03-ci-only-step), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-03-ci-only-step' (audit/engine/tests/fixtures/ci-03-ci-only-step), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CI-03", diff --git a/dev/eval/expected/fixtures/ci-05-ci-only-lint.json b/dev/eval/expected/fixtures/ci-05-ci-only-lint.json index a6e6b6bd..2f8efe7f 100644 --- a/dev/eval/expected/fixtures/ci-05-ci-only-lint.json +++ b/dev/eval/expected/fixtures/ci-05-ci-only-lint.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "ci-05-ci-only-lint", "repo_sha": "1e83b2a77847db141437468c274eec9a55857711", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-05-ci-only-lint' (audit/engine/tests/fixtures/ci-05-ci-only-lint), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-05-ci-only-lint' (audit/engine/tests/fixtures/ci-05-ci-only-lint), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CI-05", diff --git a/dev/eval/expected/fixtures/ci-05-no-workflow-lint.json b/dev/eval/expected/fixtures/ci-05-no-workflow-lint.json index 24e69d94..0f341ab1 100644 --- a/dev/eval/expected/fixtures/ci-05-no-workflow-lint.json +++ b/dev/eval/expected/fixtures/ci-05-no-workflow-lint.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "ci-05-no-workflow-lint", "repo_sha": "a3361285b063a4b4771f2c2ecbc8235dd618be60", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-05-no-workflow-lint' (audit/engine/tests/fixtures/ci-05-no-workflow-lint), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-05-no-workflow-lint' (audit/engine/tests/fixtures/ci-05-no-workflow-lint), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CI-05", diff --git a/dev/eval/expected/fixtures/ci-ok-gh.json b/dev/eval/expected/fixtures/ci-ok-gh.json index 93fe94f0..047ec854 100644 --- a/dev/eval/expected/fixtures/ci-ok-gh.json +++ b/dev/eval/expected/fixtures/ci-ok-gh.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "ci-ok-gh", "repo_sha": "c507095f93c759de05601eca66a5f1ee4ad56685", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-ok-gh' (audit/engine/tests/fixtures/ci-ok-gh), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ci-ok-gh' (audit/engine/tests/fixtures/ci-ok-gh), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CI-01", diff --git a/dev/eval/expected/fixtures/cmd-01-make-parse-error.json b/dev/eval/expected/fixtures/cmd-01-make-parse-error.json index 732d1a38..728f3447 100644 --- a/dev/eval/expected/fixtures/cmd-01-make-parse-error.json +++ b/dev/eval/expected/fixtures/cmd-01-make-parse-error.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-01-make-parse-error", "repo_sha": "618e256caa9ff4eacd535e74b694a47ef8c5faf1", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-01-make-parse-error' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-01-make-parse-error), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-01-make-parse-error' (audit/engine/tests/fixtures/cmd-01-make-parse-error), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-01", diff --git a/dev/eval/expected/fixtures/cmd-01-maven-no-formatter.json b/dev/eval/expected/fixtures/cmd-01-maven-no-formatter.json index bc50e445..1e67e84f 100644 --- a/dev/eval/expected/fixtures/cmd-01-maven-no-formatter.json +++ b/dev/eval/expected/fixtures/cmd-01-maven-no-formatter.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-01-maven-no-formatter", "repo_sha": "6178674a3c32c42da9f6c502e990f0eb85dfcfc6", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-01-maven-no-formatter' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-01-maven-no-formatter), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-01-maven-no-formatter' (audit/engine/tests/fixtures/cmd-01-maven-no-formatter), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-01", diff --git a/dev/eval/expected/fixtures/cmd-01-missing-verb.json b/dev/eval/expected/fixtures/cmd-01-missing-verb.json index a8e01a30..4e4292b3 100644 --- a/dev/eval/expected/fixtures/cmd-01-missing-verb.json +++ b/dev/eval/expected/fixtures/cmd-01-missing-verb.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-01-missing-verb", "repo_sha": "30bb1a8af368fa54fdb95038879558726aef73cb", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-01-missing-verb' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-01-missing-verb), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-01-missing-verb' (audit/engine/tests/fixtures/cmd-01-missing-verb), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-01", diff --git a/dev/eval/expected/fixtures/cmd-02-weak-check.json b/dev/eval/expected/fixtures/cmd-02-weak-check.json index faeef5db..412a2275 100644 --- a/dev/eval/expected/fixtures/cmd-02-weak-check.json +++ b/dev/eval/expected/fixtures/cmd-02-weak-check.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-02-weak-check", "repo_sha": "aa923291dde8047813a8fdadf93526a85fc53510", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-02-weak-check' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-02-weak-check), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-02-weak-check' (audit/engine/tests/fixtures/cmd-02-weak-check), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-02", diff --git a/dev/eval/expected/fixtures/cmd-03-divergent.json b/dev/eval/expected/fixtures/cmd-03-divergent.json index a064e8f3..d6ae7727 100644 --- a/dev/eval/expected/fixtures/cmd-03-divergent.json +++ b/dev/eval/expected/fixtures/cmd-03-divergent.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-03-divergent", "repo_sha": "d09905f0bfcbce2780e87b5df4e348c7126d4b33", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-03-divergent' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-03-divergent), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-03-divergent' (audit/engine/tests/fixtures/cmd-03-divergent), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-03", diff --git a/dev/eval/expected/fixtures/cmd-04-write-in-check.json b/dev/eval/expected/fixtures/cmd-04-write-in-check.json index 13dcf30a..be1442c3 100644 --- a/dev/eval/expected/fixtures/cmd-04-write-in-check.json +++ b/dev/eval/expected/fixtures/cmd-04-write-in-check.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-04-write-in-check", "repo_sha": "19217ed5bb394c973b29afc28c03a6af36bf84cb", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-04-write-in-check' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-04-write-in-check), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-04-write-in-check' (audit/engine/tests/fixtures/cmd-04-write-in-check), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-04", diff --git a/dev/eval/expected/fixtures/cmd-ok-gradle.json b/dev/eval/expected/fixtures/cmd-ok-gradle.json index e830b866..35be1597 100644 --- a/dev/eval/expected/fixtures/cmd-ok-gradle.json +++ b/dev/eval/expected/fixtures/cmd-ok-gradle.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-ok-gradle", "repo_sha": "fc2b6ed3248cfa1ba6b42b9cf4df42fe1c3993ab", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-gradle' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-ok-gradle), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-gradle' (audit/engine/tests/fixtures/cmd-ok-gradle), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-01", diff --git a/dev/eval/expected/fixtures/cmd-ok-make-alias.json b/dev/eval/expected/fixtures/cmd-ok-make-alias.json index e806b1a7..f30b2ac6 100644 --- a/dev/eval/expected/fixtures/cmd-ok-make-alias.json +++ b/dev/eval/expected/fixtures/cmd-ok-make-alias.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-ok-make-alias", "repo_sha": "1ec1a9d828070d48bbc0e831016ea640b5099dce", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-make-alias' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-ok-make-alias), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-make-alias' (audit/engine/tests/fixtures/cmd-ok-make-alias), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-01", diff --git a/dev/eval/expected/fixtures/cmd-ok-make-prereqs.json b/dev/eval/expected/fixtures/cmd-ok-make-prereqs.json index 98b01207..231de341 100644 --- a/dev/eval/expected/fixtures/cmd-ok-make-prereqs.json +++ b/dev/eval/expected/fixtures/cmd-ok-make-prereqs.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-ok-make-prereqs", "repo_sha": "bdfb5306967c0d3622dc06ef79747053b3ab3218", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-make-prereqs' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-ok-make-prereqs), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-make-prereqs' (audit/engine/tests/fixtures/cmd-ok-make-prereqs), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-01", diff --git a/dev/eval/expected/fixtures/cmd-ok-make.json b/dev/eval/expected/fixtures/cmd-ok-make.json index 3e7445ed..241bd6b7 100644 --- a/dev/eval/expected/fixtures/cmd-ok-make.json +++ b/dev/eval/expected/fixtures/cmd-ok-make.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-ok-make", "repo_sha": "e38ee5b385e3a0770e98421ef91da20d4d15f0f9", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-make' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-ok-make), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-make' (audit/engine/tests/fixtures/cmd-ok-make), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-01", diff --git a/dev/eval/expected/fixtures/cmd-ok-maven.json b/dev/eval/expected/fixtures/cmd-ok-maven.json index 3d5d6631..ddc4a8e1 100644 --- a/dev/eval/expected/fixtures/cmd-ok-maven.json +++ b/dev/eval/expected/fixtures/cmd-ok-maven.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-ok-maven", "repo_sha": "ea234ed74d61e43266f95f3ccba65102484ebb0a", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-maven' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-ok-maven), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-maven' (audit/engine/tests/fixtures/cmd-ok-maven), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-01", diff --git a/dev/eval/expected/fixtures/cmd-ok-npm.json b/dev/eval/expected/fixtures/cmd-ok-npm.json index ea37457e..234c3e93 100644 --- a/dev/eval/expected/fixtures/cmd-ok-npm.json +++ b/dev/eval/expected/fixtures/cmd-ok-npm.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-ok-npm", "repo_sha": "db11a81e1ece50eab652f5cc1af843d9236ff354", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-npm' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-ok-npm), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-npm' (audit/engine/tests/fixtures/cmd-ok-npm), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-01", diff --git a/dev/eval/expected/fixtures/cmd-ok-pyproject-poe.json b/dev/eval/expected/fixtures/cmd-ok-pyproject-poe.json index e83f9e75..e3f72c4a 100644 --- a/dev/eval/expected/fixtures/cmd-ok-pyproject-poe.json +++ b/dev/eval/expected/fixtures/cmd-ok-pyproject-poe.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-ok-pyproject-poe", "repo_sha": "b3c22d05b4da44738521f39783f5757b3892f3c7", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-pyproject-poe' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-ok-pyproject-poe), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-pyproject-poe' (audit/engine/tests/fixtures/cmd-ok-pyproject-poe), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-01", diff --git a/dev/eval/expected/fixtures/cmd-ok-root-make-nested-template.json b/dev/eval/expected/fixtures/cmd-ok-root-make-nested-template.json index e82d2e17..63a7fffe 100644 --- a/dev/eval/expected/fixtures/cmd-ok-root-make-nested-template.json +++ b/dev/eval/expected/fixtures/cmd-ok-root-make-nested-template.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "cmd-ok-root-make-nested-template", "repo_sha": "b875afd5bdf8ef891a5c26e9b216980fa0af6e76", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-root-make-nested-template' (brownfield-recipes/deterministic-audit/tests/fixtures/cmd-ok-root-make-nested-template), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'cmd-ok-root-make-nested-template' (audit/engine/tests/fixtures/cmd-ok-root-make-nested-template), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CMD-01", diff --git a/dev/eval/expected/fixtures/con-01-names-existing-paths.json b/dev/eval/expected/fixtures/con-01-names-existing-paths.json index ec86da27..71091e14 100644 --- a/dev/eval/expected/fixtures/con-01-names-existing-paths.json +++ b/dev/eval/expected/fixtures/con-01-names-existing-paths.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "con-01-names-existing-paths", "repo_sha": "4a5f91a47e6737eb859d4642c61a0b0e6b6acd24", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-01-names-existing-paths' (brownfield-recipes/deterministic-audit/tests/fixtures/con-01-names-existing-paths), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-01-names-existing-paths' (audit/engine/tests/fixtures/con-01-names-existing-paths), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CON-01", diff --git a/dev/eval/expected/fixtures/con-01-names-missing-paths.json b/dev/eval/expected/fixtures/con-01-names-missing-paths.json index 75452d17..5980589f 100644 --- a/dev/eval/expected/fixtures/con-01-names-missing-paths.json +++ b/dev/eval/expected/fixtures/con-01-names-missing-paths.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "con-01-names-missing-paths", "repo_sha": "6d878a4f688cf10d012adbea9c9f7157940f026a", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-01-names-missing-paths' (brownfield-recipes/deterministic-audit/tests/fixtures/con-01-names-missing-paths), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-01-names-missing-paths' (audit/engine/tests/fixtures/con-01-names-missing-paths), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CON-01", diff --git a/dev/eval/expected/fixtures/con-01-names-no-path.json b/dev/eval/expected/fixtures/con-01-names-no-path.json index 3661308c..283f4c5a 100644 --- a/dev/eval/expected/fixtures/con-01-names-no-path.json +++ b/dev/eval/expected/fixtures/con-01-names-no-path.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "con-01-names-no-path", "repo_sha": "a7bc2fab15dd90bf2e42d059b9a9895759790832", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-01-names-no-path' (brownfield-recipes/deterministic-audit/tests/fixtures/con-01-names-no-path), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-01-names-no-path' (audit/engine/tests/fixtures/con-01-names-no-path), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CON-01", diff --git a/dev/eval/expected/fixtures/con-02-no-prohibitions-section.json b/dev/eval/expected/fixtures/con-02-no-prohibitions-section.json index 5f2b9e98..0f76338d 100644 --- a/dev/eval/expected/fixtures/con-02-no-prohibitions-section.json +++ b/dev/eval/expected/fixtures/con-02-no-prohibitions-section.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "con-02-no-prohibitions-section", "repo_sha": "741eabf30dcf55fe5eafe89df3c4ced1f69d696f", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-02-no-prohibitions-section' (brownfield-recipes/deterministic-audit/tests/fixtures/con-02-no-prohibitions-section), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-02-no-prohibitions-section' (audit/engine/tests/fixtures/con-02-no-prohibitions-section), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CON-01", diff --git a/dev/eval/expected/fixtures/con-02-no-test-directory.json b/dev/eval/expected/fixtures/con-02-no-test-directory.json index 6c2d5cd1..46f7a5ef 100644 --- a/dev/eval/expected/fixtures/con-02-no-test-directory.json +++ b/dev/eval/expected/fixtures/con-02-no-test-directory.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "con-02-no-test-directory", "repo_sha": "306767c97b7b77f3eed8e2a027721a722542257f", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-02-no-test-directory' (brownfield-recipes/deterministic-audit/tests/fixtures/con-02-no-test-directory), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-02-no-test-directory' (audit/engine/tests/fixtures/con-02-no-test-directory), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CON-02", diff --git a/dev/eval/expected/fixtures/con-02-partly-protected.json b/dev/eval/expected/fixtures/con-02-partly-protected.json index b3d7d9e1..fbd39057 100644 --- a/dev/eval/expected/fixtures/con-02-partly-protected.json +++ b/dev/eval/expected/fixtures/con-02-partly-protected.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "con-02-partly-protected", "repo_sha": "bdb5528d433a5c01bcbad357e62245990d2e25d0", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-02-partly-protected' (brownfield-recipes/deterministic-audit/tests/fixtures/con-02-partly-protected), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-02-partly-protected' (audit/engine/tests/fixtures/con-02-partly-protected), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CON-02", diff --git a/dev/eval/expected/fixtures/con-02-protected.json b/dev/eval/expected/fixtures/con-02-protected.json index ecd86223..3261e813 100644 --- a/dev/eval/expected/fixtures/con-02-protected.json +++ b/dev/eval/expected/fixtures/con-02-protected.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "con-02-protected", "repo_sha": "7bc3a49aed50315ff101f79f597425c0fa7184c3", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-02-protected' (brownfield-recipes/deterministic-audit/tests/fixtures/con-02-protected), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-02-protected' (audit/engine/tests/fixtures/con-02-protected), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CON-01", diff --git a/dev/eval/expected/fixtures/con-02-specs-are-not-tests.json b/dev/eval/expected/fixtures/con-02-specs-are-not-tests.json index 52101175..8e0064f0 100644 --- a/dev/eval/expected/fixtures/con-02-specs-are-not-tests.json +++ b/dev/eval/expected/fixtures/con-02-specs-are-not-tests.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "con-02-specs-are-not-tests", "repo_sha": "5dd3e2bc71f495b7ba755c0bd9c1d1218fca14e6", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-02-specs-are-not-tests' (brownfield-recipes/deterministic-audit/tests/fixtures/con-02-specs-are-not-tests), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-02-specs-are-not-tests' (audit/engine/tests/fixtures/con-02-specs-are-not-tests), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CON-02", diff --git a/dev/eval/expected/fixtures/con-02-unprotected-tests.json b/dev/eval/expected/fixtures/con-02-unprotected-tests.json index c13dbaa8..bc5b590c 100644 --- a/dev/eval/expected/fixtures/con-02-unprotected-tests.json +++ b/dev/eval/expected/fixtures/con-02-unprotected-tests.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "con-02-unprotected-tests", "repo_sha": "88e278cb807b67fd4f57ee6aaec89d9a3a361ff1", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-02-unprotected-tests' (brownfield-recipes/deterministic-audit/tests/fixtures/con-02-unprotected-tests), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-02-unprotected-tests' (audit/engine/tests/fixtures/con-02-unprotected-tests), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CON-01", diff --git a/dev/eval/expected/fixtures/con-03-artefacts-not-ignored.json b/dev/eval/expected/fixtures/con-03-artefacts-not-ignored.json index 260ec33a..d2bc8797 100644 --- a/dev/eval/expected/fixtures/con-03-artefacts-not-ignored.json +++ b/dev/eval/expected/fixtures/con-03-artefacts-not-ignored.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "con-03-artefacts-not-ignored", "repo_sha": "3a7e79388df11673b009ed27fe13ff4c5aa2b3e4", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-03-artefacts-not-ignored' (brownfield-recipes/deterministic-audit/tests/fixtures/con-03-artefacts-not-ignored), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'con-03-artefacts-not-ignored' (audit/engine/tests/fixtures/con-03-artefacts-not-ignored), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CON-03", diff --git a/dev/eval/expected/fixtures/doc-01-empty.json b/dev/eval/expected/fixtures/doc-01-empty.json index f5cdea1b..68f26fff 100644 --- a/dev/eval/expected/fixtures/doc-01-empty.json +++ b/dev/eval/expected/fixtures/doc-01-empty.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "doc-01-empty", "repo_sha": "681e05e4d3dd8e3794999523858a432085da8225", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'doc-01-empty' (brownfield-recipes/deterministic-audit/tests/fixtures/doc-01-empty), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'doc-01-empty' (audit/engine/tests/fixtures/doc-01-empty), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "DOC-01", diff --git a/dev/eval/expected/fixtures/doc-02-no-style-guide.json b/dev/eval/expected/fixtures/doc-02-no-style-guide.json index e3ceccf6..bd7ad67b 100644 --- a/dev/eval/expected/fixtures/doc-02-no-style-guide.json +++ b/dev/eval/expected/fixtures/doc-02-no-style-guide.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "doc-02-no-style-guide", "repo_sha": "0b5a781f5b8ca7fdab2a6ceb3971a83407b8d4e9", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'doc-02-no-style-guide' (brownfield-recipes/deterministic-audit/tests/fixtures/doc-02-no-style-guide), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'doc-02-no-style-guide' (audit/engine/tests/fixtures/doc-02-no-style-guide), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "DOC-02", diff --git a/dev/eval/expected/fixtures/doc-03-tracked-artefacts.json b/dev/eval/expected/fixtures/doc-03-tracked-artefacts.json index 2fde30b7..502f6248 100644 --- a/dev/eval/expected/fixtures/doc-03-tracked-artefacts.json +++ b/dev/eval/expected/fixtures/doc-03-tracked-artefacts.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "doc-03-tracked-artefacts", "repo_sha": "31614613f8692990ec9a49b67f06f2e33ebd3b8d", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'doc-03-tracked-artefacts' (brownfield-recipes/deterministic-audit/tests/fixtures/doc-03-tracked-artefacts), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'doc-03-tracked-artefacts' (audit/engine/tests/fixtures/doc-03-tracked-artefacts), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "DOC-03", diff --git a/dev/eval/expected/fixtures/doc-04-adr-unindexed.json b/dev/eval/expected/fixtures/doc-04-adr-unindexed.json index 5dc26e5f..7a47465a 100644 --- a/dev/eval/expected/fixtures/doc-04-adr-unindexed.json +++ b/dev/eval/expected/fixtures/doc-04-adr-unindexed.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "doc-04-adr-unindexed", "repo_sha": "88aa1126c6ba90c41694f49fa31e8fa57b5efb07", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'doc-04-adr-unindexed' (brownfield-recipes/deterministic-audit/tests/fixtures/doc-04-adr-unindexed), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'doc-04-adr-unindexed' (audit/engine/tests/fixtures/doc-04-adr-unindexed), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "DOC-04", diff --git a/dev/eval/expected/fixtures/doc-05-long-sentences.json b/dev/eval/expected/fixtures/doc-05-long-sentences.json index 542d7110..946e03a1 100644 --- a/dev/eval/expected/fixtures/doc-05-long-sentences.json +++ b/dev/eval/expected/fixtures/doc-05-long-sentences.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "doc-05-long-sentences", "repo_sha": "0fb4d86585029b2877fdfbc594d31744450a83c1", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'doc-05-long-sentences' (audit/engine/tests/fixtures/doc-05-long-sentences), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'doc-05-long-sentences' (audit/engine/tests/fixtures/doc-05-long-sentences), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "DOC-01", diff --git a/dev/eval/expected/fixtures/doc-nav-ok.json b/dev/eval/expected/fixtures/doc-nav-ok.json index db9883b9..b0a96803 100644 --- a/dev/eval/expected/fixtures/doc-nav-ok.json +++ b/dev/eval/expected/fixtures/doc-nav-ok.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "doc-nav-ok", "repo_sha": "416dde6e6e737b5bd35b82bae3dd46997813c367", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'doc-nav-ok' (audit/engine/tests/fixtures/doc-nav-ok), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'doc-nav-ok' (audit/engine/tests/fixtures/doc-nav-ok), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "DOC-01", diff --git a/dev/eval/expected/fixtures/exec-01-nested-go-unpinned.json b/dev/eval/expected/fixtures/exec-01-nested-go-unpinned.json index 483c0617..fad2bae3 100644 --- a/dev/eval/expected/fixtures/exec-01-nested-go-unpinned.json +++ b/dev/eval/expected/fixtures/exec-01-nested-go-unpinned.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-01-nested-go-unpinned", "repo_sha": "98bfd25d8e383f460498ed8c42d6ac40c7405c0f", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-01-nested-go-unpinned' (audit/engine/tests/fixtures/exec-01-nested-go-unpinned), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-01-nested-go-unpinned' (audit/engine/tests/fixtures/exec-01-nested-go-unpinned), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-01", diff --git a/dev/eval/expected/fixtures/exec-01-python-source-only.json b/dev/eval/expected/fixtures/exec-01-python-source-only.json index 397e9da8..3393c0dc 100644 --- a/dev/eval/expected/fixtures/exec-01-python-source-only.json +++ b/dev/eval/expected/fixtures/exec-01-python-source-only.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-01-python-source-only", "repo_sha": "c2cbe442b19235054a581e73f9f3cf69d6e40a25", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-01-python-source-only' (brownfield-recipes/deterministic-audit/tests/fixtures/exec-01-python-source-only), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-01-python-source-only' (audit/engine/tests/fixtures/exec-01-python-source-only), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-01", diff --git a/dev/eval/expected/fixtures/exec-01-unpinned.json b/dev/eval/expected/fixtures/exec-01-unpinned.json index 69dada48..372bd70a 100644 --- a/dev/eval/expected/fixtures/exec-01-unpinned.json +++ b/dev/eval/expected/fixtures/exec-01-unpinned.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-01-unpinned", "repo_sha": "df9b5dcca5b92aa02a3eccb30b171ea0459b8c6a", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-01-unpinned' (brownfield-recipes/deterministic-audit/tests/fixtures/exec-01-unpinned), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-01-unpinned' (audit/engine/tests/fixtures/exec-01-unpinned), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-01", diff --git a/dev/eval/expected/fixtures/exec-02-nested-lockfile.json b/dev/eval/expected/fixtures/exec-02-nested-lockfile.json index 0886a726..75af16c8 100644 --- a/dev/eval/expected/fixtures/exec-02-nested-lockfile.json +++ b/dev/eval/expected/fixtures/exec-02-nested-lockfile.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-02-nested-lockfile", "repo_sha": "bfa392bcbfd724e64a4e9016bb0ff4bc4e30481d", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-02-nested-lockfile' (brownfield-recipes/deterministic-audit/tests/fixtures/exec-02-nested-lockfile), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-02-nested-lockfile' (audit/engine/tests/fixtures/exec-02-nested-lockfile), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-02", diff --git a/dev/eval/expected/fixtures/exec-02-no-lockfile.json b/dev/eval/expected/fixtures/exec-02-no-lockfile.json index 19f920e4..436dc5cf 100644 --- a/dev/eval/expected/fixtures/exec-02-no-lockfile.json +++ b/dev/eval/expected/fixtures/exec-02-no-lockfile.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-02-no-lockfile", "repo_sha": "25a0680e1f63cc4834e9db62afa131b5664cd56e", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-02-no-lockfile' (brownfield-recipes/deterministic-audit/tests/fixtures/exec-02-no-lockfile), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-02-no-lockfile' (audit/engine/tests/fixtures/exec-02-no-lockfile), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-02", diff --git a/dev/eval/expected/fixtures/exec-03-no-setup.json b/dev/eval/expected/fixtures/exec-03-no-setup.json index 92144159..dfdc20d9 100644 --- a/dev/eval/expected/fixtures/exec-03-no-setup.json +++ b/dev/eval/expected/fixtures/exec-03-no-setup.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-03-no-setup", "repo_sha": "4fde2e638713a38a3a7fb38ff443e6339e07b8e0", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-03-no-setup' (brownfield-recipes/deterministic-audit/tests/fixtures/exec-03-no-setup), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-03-no-setup' (audit/engine/tests/fixtures/exec-03-no-setup), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-03", diff --git a/dev/eval/expected/fixtures/exec-04-missing-env.json b/dev/eval/expected/fixtures/exec-04-missing-env.json index 271f08ce..7573e745 100644 --- a/dev/eval/expected/fixtures/exec-04-missing-env.json +++ b/dev/eval/expected/fixtures/exec-04-missing-env.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-04-missing-env", "repo_sha": "431f8779aa027c177f00671ba85ca3809537de3a", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-04-missing-env' (brownfield-recipes/deterministic-audit/tests/fixtures/exec-04-missing-env), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-04-missing-env' (audit/engine/tests/fixtures/exec-04-missing-env), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-04", diff --git a/dev/eval/expected/fixtures/exec-05-generated.json b/dev/eval/expected/fixtures/exec-05-generated.json index 96d6dc9f..e4a8837e 100644 --- a/dev/eval/expected/fixtures/exec-05-generated.json +++ b/dev/eval/expected/fixtures/exec-05-generated.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-05-generated", "repo_sha": "e70c65a4916dfe416432f68427532ca7dc6016a4", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-05-generated' (brownfield-recipes/deterministic-audit/tests/fixtures/exec-05-generated), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-05-generated' (audit/engine/tests/fixtures/exec-05-generated), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-05", diff --git a/dev/eval/expected/fixtures/exec-06-broken-command.json b/dev/eval/expected/fixtures/exec-06-broken-command.json index b5782a1a..15962e04 100644 --- a/dev/eval/expected/fixtures/exec-06-broken-command.json +++ b/dev/eval/expected/fixtures/exec-06-broken-command.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-06-broken-command", "repo_sha": "82da21567dd798a176889435f7c6bce93f4accaa", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-06-broken-command' (brownfield-recipes/deterministic-audit/tests/fixtures/exec-06-broken-command), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-06-broken-command' (audit/engine/tests/fixtures/exec-06-broken-command), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-06", diff --git a/dev/eval/expected/fixtures/exec-07-dual-lint.json b/dev/eval/expected/fixtures/exec-07-dual-lint.json index 1916f8cd..01550011 100644 --- a/dev/eval/expected/fixtures/exec-07-dual-lint.json +++ b/dev/eval/expected/fixtures/exec-07-dual-lint.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-07-dual-lint", "repo_sha": "10098781ae3680622185593c5baa3af0cf4a55a9", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-07-dual-lint' (brownfield-recipes/deterministic-audit/tests/fixtures/exec-07-dual-lint), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-07-dual-lint' (audit/engine/tests/fixtures/exec-07-dual-lint), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-07", diff --git a/dev/eval/expected/fixtures/exec-07-dual-python-types.json b/dev/eval/expected/fixtures/exec-07-dual-python-types.json index bb72b797..38df82d8 100644 --- a/dev/eval/expected/fixtures/exec-07-dual-python-types.json +++ b/dev/eval/expected/fixtures/exec-07-dual-python-types.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-07-dual-python-types", "repo_sha": "2ab9fb2080528acf934ebf51823fb5c1b68daed5", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-07-dual-python-types' (audit/engine/tests/fixtures/exec-07-dual-python-types), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-07-dual-python-types' (audit/engine/tests/fixtures/exec-07-dual-python-types), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-07", diff --git a/dev/eval/expected/fixtures/exec-08-duplicate-pin.json b/dev/eval/expected/fixtures/exec-08-duplicate-pin.json index 1d3a8989..cd8dfed3 100644 --- a/dev/eval/expected/fixtures/exec-08-duplicate-pin.json +++ b/dev/eval/expected/fixtures/exec-08-duplicate-pin.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-08-duplicate-pin", "repo_sha": "5439f014bd0bb8ea2eaaefef84c673c97685d8e9", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-08-duplicate-pin' (audit/engine/tests/fixtures/exec-08-duplicate-pin), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-08-duplicate-pin' (audit/engine/tests/fixtures/exec-08-duplicate-pin), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-08", diff --git a/dev/eval/expected/fixtures/exec-08-inline-version.json b/dev/eval/expected/fixtures/exec-08-inline-version.json index 18224e53..7eb5aff1 100644 --- a/dev/eval/expected/fixtures/exec-08-inline-version.json +++ b/dev/eval/expected/fixtures/exec-08-inline-version.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-08-inline-version", "repo_sha": "b21b3e039c094238c38017b89781af73d0f77b75", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-08-inline-version' (audit/engine/tests/fixtures/exec-08-inline-version), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-08-inline-version' (audit/engine/tests/fixtures/exec-08-inline-version), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-08", diff --git a/dev/eval/expected/fixtures/exec-ok.json b/dev/eval/expected/fixtures/exec-ok.json index 16873b05..d7cf38d6 100644 --- a/dev/eval/expected/fixtures/exec-ok.json +++ b/dev/eval/expected/fixtures/exec-ok.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "exec-ok", "repo_sha": "2ec6ea554e3c31a122700f4354b1e81926e80fff", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-ok' (brownfield-recipes/deterministic-audit/tests/fixtures/exec-ok), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-ok' (audit/engine/tests/fixtures/exec-ok), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "EXEC-01", diff --git a/dev/eval/expected/fixtures/governed-minimal.json b/dev/eval/expected/fixtures/governed-minimal.json index 241c71af..9d7484cc 100644 --- a/dev/eval/expected/fixtures/governed-minimal.json +++ b/dev/eval/expected/fixtures/governed-minimal.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "governed-minimal", "repo_sha": "55cea7482f664f50ad8f3dc0c6e794ebb7572ae4", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'governed-minimal' (audit/engine/tests/fixtures/governed-minimal), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'governed-minimal' (audit/engine/tests/fixtures/governed-minimal), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "AGT-01", diff --git a/dev/eval/expected/fixtures/hyg-01-placeholder-readme.json b/dev/eval/expected/fixtures/hyg-01-placeholder-readme.json index 2a598c04..20b7639c 100644 --- a/dev/eval/expected/fixtures/hyg-01-placeholder-readme.json +++ b/dev/eval/expected/fixtures/hyg-01-placeholder-readme.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "hyg-01-placeholder-readme", "repo_sha": "050d86dac44cc11766132b4e7141cad5ee3b60e7", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-01-placeholder-readme' (brownfield-recipes/deterministic-audit/tests/fixtures/hyg-01-placeholder-readme), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-01-placeholder-readme' (audit/engine/tests/fixtures/hyg-01-placeholder-readme), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "HYG-01", diff --git a/dev/eval/expected/fixtures/hyg-09-target-branch-trap.json b/dev/eval/expected/fixtures/hyg-09-target-branch-trap.json index 75a7cf9f..bf6a4312 100644 --- a/dev/eval/expected/fixtures/hyg-09-target-branch-trap.json +++ b/dev/eval/expected/fixtures/hyg-09-target-branch-trap.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "hyg-09-target-branch-trap", "repo_sha": "8d6da8e873c18b65ead44b1e2397ff3dc12a5ef5", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-09-target-branch-trap' (audit/engine/tests/fixtures/hyg-09-target-branch-trap), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-09-target-branch-trap' (audit/engine/tests/fixtures/hyg-09-target-branch-trap), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "HYG-09", diff --git a/dev/eval/expected/fixtures/hyg-09-uncovered-ecosystem.json b/dev/eval/expected/fixtures/hyg-09-uncovered-ecosystem.json index 81f25b5c..623df6fc 100644 --- a/dev/eval/expected/fixtures/hyg-09-uncovered-ecosystem.json +++ b/dev/eval/expected/fixtures/hyg-09-uncovered-ecosystem.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "hyg-09-uncovered-ecosystem", "repo_sha": "17bc601cdf67a32857dc021e17d6c1baf6f52ce8", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-09-uncovered-ecosystem' (audit/engine/tests/fixtures/hyg-09-uncovered-ecosystem), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-09-uncovered-ecosystem' (audit/engine/tests/fixtures/hyg-09-uncovered-ecosystem), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "HYG-09", diff --git a/dev/eval/expected/fixtures/hyg-10-no-license.json b/dev/eval/expected/fixtures/hyg-10-no-license.json index e56c19aa..e964f268 100644 --- a/dev/eval/expected/fixtures/hyg-10-no-license.json +++ b/dev/eval/expected/fixtures/hyg-10-no-license.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "hyg-10-no-license", "repo_sha": "4605a91cab56418003ee26e21351d66ddb1a3bea", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-10-no-license' (brownfield-recipes/deterministic-audit/tests/fixtures/hyg-10-no-license), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-10-no-license' (audit/engine/tests/fixtures/hyg-10-no-license), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "HYG-01", diff --git a/dev/eval/expected/fixtures/hyg-11-no-source-branch-check.json b/dev/eval/expected/fixtures/hyg-11-no-source-branch-check.json index 8ed3054c..8a8674ca 100644 --- a/dev/eval/expected/fixtures/hyg-11-no-source-branch-check.json +++ b/dev/eval/expected/fixtures/hyg-11-no-source-branch-check.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "hyg-11-no-source-branch-check", "repo_sha": "bae1ed487572451d50a5360de7020e0e6031e6d0", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-11-no-source-branch-check' (audit/engine/tests/fixtures/hyg-11-no-source-branch-check), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-11-no-source-branch-check' (audit/engine/tests/fixtures/hyg-11-no-source-branch-check), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "HYG-11", diff --git a/dev/eval/expected/fixtures/hyg-minimal.json b/dev/eval/expected/fixtures/hyg-minimal.json index 7fbdb6e5..28fb15cc 100644 --- a/dev/eval/expected/fixtures/hyg-minimal.json +++ b/dev/eval/expected/fixtures/hyg-minimal.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "hyg-minimal", "repo_sha": "388bc1400e8e1056af97140be30fb042c39757de", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-minimal' (brownfield-recipes/deterministic-audit/tests/fixtures/hyg-minimal), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'hyg-minimal' (audit/engine/tests/fixtures/hyg-minimal), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "HYG-01", diff --git a/dev/eval/expected/fixtures/idx-01-no-mcp-config.json b/dev/eval/expected/fixtures/idx-01-no-mcp-config.json index 9b520bfa..8df41c34 100644 --- a/dev/eval/expected/fixtures/idx-01-no-mcp-config.json +++ b/dev/eval/expected/fixtures/idx-01-no-mcp-config.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "idx-01-no-mcp-config", "repo_sha": "fd2100346a87aa48d5c196ea715d7d3412cc329a", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'idx-01-no-mcp-config' (brownfield-recipes/deterministic-audit/tests/fixtures/idx-01-no-mcp-config), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'idx-01-no-mcp-config' (audit/engine/tests/fixtures/idx-01-no-mcp-config), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "IDX-01", diff --git a/dev/eval/expected/fixtures/idx-02-no-index.json b/dev/eval/expected/fixtures/idx-02-no-index.json index 698d2f39..b0ea8c30 100644 --- a/dev/eval/expected/fixtures/idx-02-no-index.json +++ b/dev/eval/expected/fixtures/idx-02-no-index.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "idx-02-no-index", "repo_sha": "c8e5cb9ce1a0029589b983957e1f9fc5715aa1e9", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'idx-02-no-index' (brownfield-recipes/deterministic-audit/tests/fixtures/idx-02-no-index), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'idx-02-no-index' (audit/engine/tests/fixtures/idx-02-no-index), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "IDX-02", diff --git a/dev/eval/expected/fixtures/ignore-ok-nested-gitignore.json b/dev/eval/expected/fixtures/ignore-ok-nested-gitignore.json index ebeaec3f..a34e1117 100644 --- a/dev/eval/expected/fixtures/ignore-ok-nested-gitignore.json +++ b/dev/eval/expected/fixtures/ignore-ok-nested-gitignore.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "ignore-ok-nested-gitignore", "repo_sha": "22bd81d14845cd97fa1ccd57cd5dcfddff4fb6cc", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ignore-ok-nested-gitignore' (brownfield-recipes/deterministic-audit/tests/fixtures/ignore-ok-nested-gitignore), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'ignore-ok-nested-gitignore' (audit/engine/tests/fixtures/ignore-ok-nested-gitignore), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "CON-03", diff --git a/dev/eval/expected/fixtures/nav-01-no-manifest.json b/dev/eval/expected/fixtures/nav-01-no-manifest.json index 249b14e0..953edd93 100644 --- a/dev/eval/expected/fixtures/nav-01-no-manifest.json +++ b/dev/eval/expected/fixtures/nav-01-no-manifest.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "nav-01-no-manifest", "repo_sha": "ed0957a48cddf9cd1a8197effc2342e00f44b1c2", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'nav-01-no-manifest' (brownfield-recipes/deterministic-audit/tests/fixtures/nav-01-no-manifest), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'nav-01-no-manifest' (audit/engine/tests/fixtures/nav-01-no-manifest), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "NAV-01", diff --git a/dev/eval/expected/fixtures/nav-02-no-entry.json b/dev/eval/expected/fixtures/nav-02-no-entry.json index 9140b813..1e03dbc0 100644 --- a/dev/eval/expected/fixtures/nav-02-no-entry.json +++ b/dev/eval/expected/fixtures/nav-02-no-entry.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "nav-02-no-entry", "repo_sha": "2e90b2c98552a9636133e78063e5add5fbd64f5b", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'nav-02-no-entry' (brownfield-recipes/deterministic-audit/tests/fixtures/nav-02-no-entry), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'nav-02-no-entry' (audit/engine/tests/fixtures/nav-02-no-entry), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "NAV-02", diff --git a/dev/eval/expected/fixtures/nav-03-red-flags.json b/dev/eval/expected/fixtures/nav-03-red-flags.json index 7d4f9840..af9c5efb 100644 --- a/dev/eval/expected/fixtures/nav-03-red-flags.json +++ b/dev/eval/expected/fixtures/nav-03-red-flags.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "nav-03-red-flags", "repo_sha": "d98072167a9f6b4229a400c8073f0850286a2bf7", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'nav-03-red-flags' (brownfield-recipes/deterministic-audit/tests/fixtures/nav-03-red-flags), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'nav-03-red-flags' (audit/engine/tests/fixtures/nav-03-red-flags), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "NAV-03", diff --git a/dev/eval/expected/fixtures/sec-01-excluded-env.json b/dev/eval/expected/fixtures/sec-01-excluded-env.json index f499c555..3bc61357 100644 --- a/dev/eval/expected/fixtures/sec-01-excluded-env.json +++ b/dev/eval/expected/fixtures/sec-01-excluded-env.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "sec-01-excluded-env", "repo_sha": "b691529e2d1a241487f9552d6e928334c25bcfec", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-01-excluded-env' (brownfield-recipes/deterministic-audit/tests/fixtures/sec-01-excluded-env), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-01-excluded-env' (audit/engine/tests/fixtures/sec-01-excluded-env), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "SEC-01", diff --git a/dev/eval/expected/fixtures/sec-01-ok.json b/dev/eval/expected/fixtures/sec-01-ok.json index b2f4874f..9f181d31 100644 --- a/dev/eval/expected/fixtures/sec-01-ok.json +++ b/dev/eval/expected/fixtures/sec-01-ok.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "sec-01-ok", "repo_sha": "9c52b12fbbc9884b49714a900cf307521ab8f648", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-01-ok' (brownfield-recipes/deterministic-audit/tests/fixtures/sec-01-ok), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-01-ok' (audit/engine/tests/fixtures/sec-01-ok), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "SEC-01", diff --git a/dev/eval/expected/fixtures/sec-01-tracked-env.json b/dev/eval/expected/fixtures/sec-01-tracked-env.json index de13139e..3ea9f38e 100644 --- a/dev/eval/expected/fixtures/sec-01-tracked-env.json +++ b/dev/eval/expected/fixtures/sec-01-tracked-env.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "sec-01-tracked-env", "repo_sha": "626cebf389a6f4a4b6389f811d2d9a2374f421ee", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-01-tracked-env' (brownfield-recipes/deterministic-audit/tests/fixtures/sec-01-tracked-env), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-01-tracked-env' (audit/engine/tests/fixtures/sec-01-tracked-env), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "SEC-01", diff --git a/dev/eval/expected/fixtures/sec-02-env-reference.json b/dev/eval/expected/fixtures/sec-02-env-reference.json index 81c28ded..8e48892a 100644 --- a/dev/eval/expected/fixtures/sec-02-env-reference.json +++ b/dev/eval/expected/fixtures/sec-02-env-reference.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "sec-02-env-reference", "repo_sha": "e336ab13e76fe214c81af461aedcb0db06bff0b2", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-02-env-reference' (brownfield-recipes/deterministic-audit/tests/fixtures/sec-02-env-reference), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-02-env-reference' (audit/engine/tests/fixtures/sec-02-env-reference), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "SEC-02", diff --git a/dev/eval/expected/fixtures/sec-02-expression-assignment.json b/dev/eval/expected/fixtures/sec-02-expression-assignment.json index afadfa58..85503778 100644 --- a/dev/eval/expected/fixtures/sec-02-expression-assignment.json +++ b/dev/eval/expected/fixtures/sec-02-expression-assignment.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "sec-02-expression-assignment", "repo_sha": "367549b7ec8050bfb1ece519848d7388a71c9847", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-02-expression-assignment' (brownfield-recipes/deterministic-audit/tests/fixtures/sec-02-expression-assignment), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-02-expression-assignment' (audit/engine/tests/fixtures/sec-02-expression-assignment), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "SEC-02", diff --git a/dev/eval/expected/fixtures/sec-02-key-shaped.json b/dev/eval/expected/fixtures/sec-02-key-shaped.json index d6638ac5..9d98849c 100644 --- a/dev/eval/expected/fixtures/sec-02-key-shaped.json +++ b/dev/eval/expected/fixtures/sec-02-key-shaped.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "sec-02-key-shaped", "repo_sha": "2048a415b12d6de6ee2c33998be0016e89115864", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-02-key-shaped' (brownfield-recipes/deterministic-audit/tests/fixtures/sec-02-key-shaped), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-02-key-shaped' (audit/engine/tests/fixtures/sec-02-key-shaped), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "SEC-02", diff --git a/dev/eval/expected/fixtures/sec-03-ok-negated-exception.json b/dev/eval/expected/fixtures/sec-03-ok-negated-exception.json index 5bea0634..d70f12e2 100644 --- a/dev/eval/expected/fixtures/sec-03-ok-negated-exception.json +++ b/dev/eval/expected/fixtures/sec-03-ok-negated-exception.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "sec-03-ok-negated-exception", "repo_sha": "4f115065877710d0f32b97e3d97e4c3049f76a9e", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-03-ok-negated-exception' (brownfield-recipes/deterministic-audit/tests/fixtures/sec-03-ok-negated-exception), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-03-ok-negated-exception' (audit/engine/tests/fixtures/sec-03-ok-negated-exception), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "SEC-03", diff --git a/dev/eval/expected/fixtures/sec-03-thin-ignore.json b/dev/eval/expected/fixtures/sec-03-thin-ignore.json index 24845b7a..ab62b247 100644 --- a/dev/eval/expected/fixtures/sec-03-thin-ignore.json +++ b/dev/eval/expected/fixtures/sec-03-thin-ignore.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "sec-03-thin-ignore", "repo_sha": "f843f7a5c1eec32aa9d9d07cbf07fb33418e8378", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-03-thin-ignore' (brownfield-recipes/deterministic-audit/tests/fixtures/sec-03-thin-ignore), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'sec-03-thin-ignore' (audit/engine/tests/fixtures/sec-03-thin-ignore), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "SEC-03", diff --git a/dev/eval/expected/fixtures/tool-01-no-formatter.json b/dev/eval/expected/fixtures/tool-01-no-formatter.json index 8a28ad68..aecbe5e0 100644 --- a/dev/eval/expected/fixtures/tool-01-no-formatter.json +++ b/dev/eval/expected/fixtures/tool-01-no-formatter.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tool-01-no-formatter", "repo_sha": "9b93ccbb58497d240d22adcfe12e76d95dd9e254", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-01-no-formatter' (brownfield-recipes/deterministic-audit/tests/fixtures/tool-01-no-formatter), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-01-no-formatter' (audit/engine/tests/fixtures/tool-01-no-formatter), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TOOL-01", diff --git a/dev/eval/expected/fixtures/tool-02-no-linter.json b/dev/eval/expected/fixtures/tool-02-no-linter.json index a839973e..2c18d1ee 100644 --- a/dev/eval/expected/fixtures/tool-02-no-linter.json +++ b/dev/eval/expected/fixtures/tool-02-no-linter.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tool-02-no-linter", "repo_sha": "986ccd8cb71b97dba3193cf5924089304866826a", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-02-no-linter' (brownfield-recipes/deterministic-audit/tests/fixtures/tool-02-no-linter), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-02-no-linter' (audit/engine/tests/fixtures/tool-02-no-linter), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TOOL-02", diff --git a/dev/eval/expected/fixtures/tool-03-nonstrict-ts.json b/dev/eval/expected/fixtures/tool-03-nonstrict-ts.json index b8d39d64..e159d412 100644 --- a/dev/eval/expected/fixtures/tool-03-nonstrict-ts.json +++ b/dev/eval/expected/fixtures/tool-03-nonstrict-ts.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tool-03-nonstrict-ts", "repo_sha": "931b7940c85b9e555aace385a473e7510d07716d", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-03-nonstrict-ts' (brownfield-recipes/deterministic-audit/tests/fixtures/tool-03-nonstrict-ts), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-03-nonstrict-ts' (audit/engine/tests/fixtures/tool-03-nonstrict-ts), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TOOL-03", diff --git a/dev/eval/expected/fixtures/tool-04-local-only-audit.json b/dev/eval/expected/fixtures/tool-04-local-only-audit.json index 2363d02c..55feee7b 100644 --- a/dev/eval/expected/fixtures/tool-04-local-only-audit.json +++ b/dev/eval/expected/fixtures/tool-04-local-only-audit.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tool-04-local-only-audit", "repo_sha": "26cd480995f7ef5e8024b42aebbd3549dd328f0d", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-04-local-only-audit' (audit/engine/tests/fixtures/tool-04-local-only-audit), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-04-local-only-audit' (audit/engine/tests/fixtures/tool-04-local-only-audit), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TOOL-04", diff --git a/dev/eval/expected/fixtures/tool-04-make-security-no-audit.json b/dev/eval/expected/fixtures/tool-04-make-security-no-audit.json index 146743eb..407281c2 100644 --- a/dev/eval/expected/fixtures/tool-04-make-security-no-audit.json +++ b/dev/eval/expected/fixtures/tool-04-make-security-no-audit.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tool-04-make-security-no-audit", "repo_sha": "e1eb4fe8cfb1960a1c6b35694cea65c5ae68d511", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-04-make-security-no-audit' (audit/engine/tests/fixtures/tool-04-make-security-no-audit), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-04-make-security-no-audit' (audit/engine/tests/fixtures/tool-04-make-security-no-audit), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TOOL-04", diff --git a/dev/eval/expected/fixtures/tool-04-no-audit-verb.json b/dev/eval/expected/fixtures/tool-04-no-audit-verb.json index 6abfb80a..96667ed7 100644 --- a/dev/eval/expected/fixtures/tool-04-no-audit-verb.json +++ b/dev/eval/expected/fixtures/tool-04-no-audit-verb.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tool-04-no-audit-verb", "repo_sha": "caa504f1117e25aaba947b45e8ab8a1b59dc465d", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-04-no-audit-verb' (brownfield-recipes/deterministic-audit/tests/fixtures/tool-04-no-audit-verb), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-04-no-audit-verb' (audit/engine/tests/fixtures/tool-04-no-audit-verb), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TOOL-04", diff --git a/dev/eval/expected/fixtures/tool-05-unwired-hooks.json b/dev/eval/expected/fixtures/tool-05-unwired-hooks.json index a2ec8fa1..f4836b5f 100644 --- a/dev/eval/expected/fixtures/tool-05-unwired-hooks.json +++ b/dev/eval/expected/fixtures/tool-05-unwired-hooks.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tool-05-unwired-hooks", "repo_sha": "de151de74aac5b45193845eab7b7da8b8a57992c", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-05-unwired-hooks' (brownfield-recipes/deterministic-audit/tests/fixtures/tool-05-unwired-hooks), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-05-unwired-hooks' (audit/engine/tests/fixtures/tool-05-unwired-hooks), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TOOL-05", diff --git a/dev/eval/expected/fixtures/tool-06-ci-only-commit-rule.json b/dev/eval/expected/fixtures/tool-06-ci-only-commit-rule.json index e9d38919..a1eddf79 100644 --- a/dev/eval/expected/fixtures/tool-06-ci-only-commit-rule.json +++ b/dev/eval/expected/fixtures/tool-06-ci-only-commit-rule.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tool-06-ci-only-commit-rule", "repo_sha": "77b64665fce027f489e1d7fe56fe10468e245347", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-06-ci-only-commit-rule' (audit/engine/tests/fixtures/tool-06-ci-only-commit-rule), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-06-ci-only-commit-rule' (audit/engine/tests/fixtures/tool-06-ci-only-commit-rule), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TOOL-06", diff --git a/dev/eval/expected/fixtures/tool-06-local-only-commit-rule.json b/dev/eval/expected/fixtures/tool-06-local-only-commit-rule.json index 24bd1ac3..f9002932 100644 --- a/dev/eval/expected/fixtures/tool-06-local-only-commit-rule.json +++ b/dev/eval/expected/fixtures/tool-06-local-only-commit-rule.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tool-06-local-only-commit-rule", "repo_sha": "504396dc259eb5cce4d045ba697c3362fac34836", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-06-local-only-commit-rule' (audit/engine/tests/fixtures/tool-06-local-only-commit-rule), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-06-local-only-commit-rule' (audit/engine/tests/fixtures/tool-06-local-only-commit-rule), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TOOL-06", diff --git a/dev/eval/expected/fixtures/tool-06-no-commit-rule.json b/dev/eval/expected/fixtures/tool-06-no-commit-rule.json index b0ba17e0..331cb849 100644 --- a/dev/eval/expected/fixtures/tool-06-no-commit-rule.json +++ b/dev/eval/expected/fixtures/tool-06-no-commit-rule.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tool-06-no-commit-rule", "repo_sha": "ae747f912c2af69a6d614185be19d0d40a670fed", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-06-no-commit-rule' (brownfield-recipes/deterministic-audit/tests/fixtures/tool-06-no-commit-rule), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-06-no-commit-rule' (audit/engine/tests/fixtures/tool-06-no-commit-rule), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TOOL-06", diff --git a/dev/eval/expected/fixtures/tool-07-no-branch-guard.json b/dev/eval/expected/fixtures/tool-07-no-branch-guard.json index 798c50eb..6d9dc05c 100644 --- a/dev/eval/expected/fixtures/tool-07-no-branch-guard.json +++ b/dev/eval/expected/fixtures/tool-07-no-branch-guard.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tool-07-no-branch-guard", "repo_sha": "d013a7d5b41ba648441fa237cf200a7604cf483b", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-07-no-branch-guard' (audit/engine/tests/fixtures/tool-07-no-branch-guard), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-07-no-branch-guard' (audit/engine/tests/fixtures/tool-07-no-branch-guard), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TOOL-05", diff --git a/dev/eval/expected/fixtures/tool-ok-node.json b/dev/eval/expected/fixtures/tool-ok-node.json index 965e7e9e..63295857 100644 --- a/dev/eval/expected/fixtures/tool-ok-node.json +++ b/dev/eval/expected/fixtures/tool-ok-node.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tool-ok-node", "repo_sha": "08cce29b42a499cac1659dc548df9a2436b6f720", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-ok-node' (audit/engine/tests/fixtures/tool-ok-node), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-ok-node' (audit/engine/tests/fixtures/tool-ok-node), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TOOL-01", diff --git a/dev/eval/expected/fixtures/tst-01-no-suite.json b/dev/eval/expected/fixtures/tst-01-no-suite.json index 34c62f35..9b36da5e 100644 --- a/dev/eval/expected/fixtures/tst-01-no-suite.json +++ b/dev/eval/expected/fixtures/tst-01-no-suite.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tst-01-no-suite", "repo_sha": "8a31fa210fb0e0fa535cf9b06b305c347fb92127", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tst-01-no-suite' (brownfield-recipes/deterministic-audit/tests/fixtures/tst-01-no-suite), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tst-01-no-suite' (audit/engine/tests/fixtures/tst-01-no-suite), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TST-01", diff --git a/dev/eval/expected/fixtures/tst-03-skip-markers.json b/dev/eval/expected/fixtures/tst-03-skip-markers.json index f2ed7bcf..0aca8c2d 100644 --- a/dev/eval/expected/fixtures/tst-03-skip-markers.json +++ b/dev/eval/expected/fixtures/tst-03-skip-markers.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tst-03-skip-markers", "repo_sha": "fe8a54db1728884d20e61ad5db78ff66bcdab160", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tst-03-skip-markers' (brownfield-recipes/deterministic-audit/tests/fixtures/tst-03-skip-markers), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tst-03-skip-markers' (audit/engine/tests/fixtures/tst-03-skip-markers), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TST-03", diff --git a/dev/eval/expected/fixtures/tst-04-no-fastpath.json b/dev/eval/expected/fixtures/tst-04-no-fastpath.json index c8d9a021..c296fa15 100644 --- a/dev/eval/expected/fixtures/tst-04-no-fastpath.json +++ b/dev/eval/expected/fixtures/tst-04-no-fastpath.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tst-04-no-fastpath", "repo_sha": "4beaa6e81cb5d8e0ffe5e9de978f5935a3ba9c5d", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tst-04-no-fastpath' (brownfield-recipes/deterministic-audit/tests/fixtures/tst-04-no-fastpath), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tst-04-no-fastpath' (audit/engine/tests/fixtures/tst-04-no-fastpath), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TST-04", diff --git a/dev/eval/expected/fixtures/tst-05-no-coverage.json b/dev/eval/expected/fixtures/tst-05-no-coverage.json index cd5f7aca..2fafd9f8 100644 --- a/dev/eval/expected/fixtures/tst-05-no-coverage.json +++ b/dev/eval/expected/fixtures/tst-05-no-coverage.json @@ -2,7 +2,7 @@ "expected_version": "0.1", "repo_label": "tst-05-no-coverage", "repo_sha": "9b746cef9ebb838ab93db505a2bc686de5714314", - "notes": "Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tst-05-no-coverage' (brownfield-recipes/deterministic-audit/tests/fixtures/tst-05-no-coverage), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tst-05-no-coverage' (audit/engine/tests/fixtures/tst-05-no-coverage), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", "checks": [ { "id": "TST-05", diff --git a/dev/eval/generate_fixture_expected.py b/dev/eval/generate_fixture_expected.py index 823ef8a0..cdbb30aa 100644 --- a/dev/eval/generate_fixture_expected.py +++ b/dev/eval/generate_fixture_expected.py @@ -14,8 +14,8 @@ 3. asserts the draft's verdict for every check the matrix names for that fixture agrees with the matrix (the "engine baseline") — a disagreement is a defect in this generator or in git-ifying a fixture that was not built to be one, and stops the run rather than writing a wrong ground truth; -4. writes ``schema/examples/eval-expected/fixtures/.json``, matching - ``schema/eval-expected.schema.json``. +4. writes ``dev/eval/expected/fixtures/.json``, matching + ``audit/schema/eval-expected.schema.json``. A fixture that needs an isolated ``HOME`` (``NEEDS_ISOLATED_ENV``) or a forced-tracked path despite an ignore rule (``FORCE_ADD``) is prepared the same way ``test_fixture_matrix.py`` prepares it for its own run. @@ -161,7 +161,7 @@ def build_expected(name: str, row: dict[str, str], draft: dict[str, Any], sha: s "repo_label": name, "repo_sha": sha, "notes": ( - f"Generated by tools/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row " + f"Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row " f"for '{name}' (audit/engine/tests/fixtures/{name}), a one-defect " "engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is " "reproducible." diff --git a/dev/tests/test_eval_expected_fixtures.py b/dev/tests/test_eval_expected_fixtures.py index 9ba66628..4134954a 100644 --- a/dev/tests/test_eval_expected_fixtures.py +++ b/dev/tests/test_eval_expected_fixtures.py @@ -3,7 +3,7 @@ """The fixture corpus's expected files agree with the engine's own matrix (S4.2.1). -``tools/eval/generate_fixture_expected.py`` wrote one ``schema/examples/eval-expected/fixtures/.json`` +``dev/eval/generate_fixture_expected.py`` wrote one ``dev/eval/expected/fixtures/.json`` per row of the engine's ``tests/test_fixture_matrix.py`` (``MATRIX``), running the real engine on a throwaway, deterministically-committed copy of each fixture. Running the engine 89 times is too slow for this suite, so this test does not regenerate anything — it loads the committed files and checks them against the matrix and From d25eae4558695621aa9d7cd0489a860c795e780d Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Tue, 10 Nov 2026 06:09:14 +0530 Subject: [PATCH 02/11] ci: run the eval-expected check in make check make eval-expected-check runs generate_fixture_expected.py --check, which compares the committed eval-expected files with a fresh engine run (a few seconds). It joins make check and the CI static job, so a stale file can no longer reach dev unseen. The docs that list the parts of check say so. Roadmap-Step: 1.1 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- AGENTS.md | 3 ++- Makefile | 12 +++++++++--- dev/eval/generate_fixture_expected.py | 6 +++--- dev/tests/test_eval_expected_fixtures.py | 7 ++++--- docs/how-to/run-the-evals.md | 3 ++- docs/reference/branch-protection.md | 2 +- docs/reference/dev-tooling.md | 5 +++-- 8 files changed, 25 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d85cad17..265f9981 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,7 +53,7 @@ jobs: with: go-version-file: dev/tools/actionlint/go.mod cache-dependency-path: dev/tools/actionlint/go.sum - - run: make format-check typecheck verify-rules recipe-lint masters-check workflow-lint spell + - run: make format-check typecheck verify-rules recipe-lint masters-check eval-expected-check workflow-lint spell crosscheck: name: crosscheck (Python ${{ matrix.python }}) diff --git a/AGENTS.md b/AGENTS.md index d3345ae7..131f0b63 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,8 @@ make lint # read-only lint make typecheck # mypy, strict, on audit/tools, implement/tools and dev, and the engine's build.py make test # the engine's, the audit tools', the implement tools' and the dev checks' suites (at once on Linux) make check # THE gate: format:check, typecheck, test, the engine's --verify-rules, the schema - # cross-check, recipe-lint, masters-check, workflow-lint (actionlint, zizmor, check-jsonschema), spell (typos) + # cross-check, recipe-lint, masters-check, eval-expected-check, workflow-lint (actionlint, + # zizmor, check-jsonschema), spell (typos) make security # secrets scan, then the dependency audit (osv-scanner; needs Go and the network) make commit-lint # Conventional Commits subjects on this branch (RANGE=origin/dev..HEAD); CI runs it on each PR make update-masters # regenerate docs/reference/audit-master.md and implement-master.md (check runs masters-check) diff --git a/Makefile b/Makefile index 9ed26de4..66f0e682 100644 --- a/Makefile +++ b/Makefile @@ -107,13 +107,13 @@ endif ifndef TEST_SUITES_SUBMAKE .NOTPARALLEL: endif -.PHONY: help setup format format-check format\:check lint typecheck test $(TEST_SUITES) test-engine-shard test-audit-tools-shard test-dev-shard test-part coverage coverage-engine coverage-engine-shard coverage-audit-tools-shard coverage-part coverage-audit-tools coverage-implement-tools coverage-combine verify-rules crosscheck recipe-lint masters-check update-masters check security security-secrets security-deps update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-prose docs-mermaid changelog-fix workflow-lint commit-lint spell spell-fix +.PHONY: help setup format format-check format\:check lint typecheck test $(TEST_SUITES) test-engine-shard test-audit-tools-shard test-dev-shard test-part coverage coverage-engine coverage-engine-shard coverage-audit-tools-shard coverage-part coverage-audit-tools coverage-implement-tools coverage-combine verify-rules crosscheck recipe-lint masters-check eval-expected-check update-masters check security security-secrets security-deps update-secrets-baseline self-audit update-audit-baseline validate-skills docs-lint docs-lint-all docs-prose docs-mermaid changelog-fix workflow-lint commit-lint spell spell-fix help: @echo "verbs: format format:check (format-check) lint typecheck test check security" @echo "parts: test-engine test-audit-tools test-implement-tools test-dev coverage-engine coverage-audit-tools coverage-implement-tools coverage-combine" @echo "ci: test-part SHARD= coverage-part SHARD= coverage-combine SHARDED=1 (and test--shard, coverage--shard)" - @echo "extra: setup (checks prerequisites, installs nothing) coverage self-audit update-audit-baseline verify-rules crosscheck recipe-lint masters-check update-masters workflow-lint commit-lint spell spell-fix update-secrets-baseline validate-skills" + @echo "extra: setup (checks prerequisites, installs nothing) coverage self-audit update-audit-baseline verify-rules crosscheck recipe-lint masters-check eval-expected-check update-masters workflow-lint commit-lint spell spell-fix update-secrets-baseline validate-skills" @echo "docs: docs-lint docs-lint-all docs-prose [FILES=...] docs-mermaid FILES=... changelog-fix" # There is nothing to install into the project: the runtime code is standard-library Python and the dev tools run @@ -300,6 +300,12 @@ masters-check: update-masters: $(PYTHON) dev/build_masters.py --write +# The eval-expected files (dev/eval/expected/fixtures/) are generated from the engine's fixture matrix by running the +# engine on each fixture (a few seconds). eval-expected-check fails when one is stale; run the script without --check +# to rewrite them. +eval-expected-check: + $(PYTHON) dev/eval/generate_fixture_expected.py --check + # Lints the GitHub Actions workflows and checks .github/dependabot.yml. All three tools run offline. # Read-only: report each misspelling with its file and line. `make spell-fix` writes the suggested fixes; review them. spell: @@ -320,7 +326,7 @@ RANGE ?= origin/dev..HEAD commit-lint: $(PYTHON) dev/commit_lint.py --range $(RANGE) -check: format-check typecheck test verify-rules crosscheck recipe-lint masters-check workflow-lint spell +check: format-check typecheck test verify-rules crosscheck recipe-lint masters-check eval-expected-check workflow-lint spell # No third-party runtime dependencies exist to audit, so the gate is a secrets scan. The baseline # lists the deliberate key-shaped strings in the fixtures and tests; a new hit fails the scan. diff --git a/dev/eval/generate_fixture_expected.py b/dev/eval/generate_fixture_expected.py index cdbb30aa..56da84a0 100644 --- a/dev/eval/generate_fixture_expected.py +++ b/dev/eval/generate_fixture_expected.py @@ -21,9 +21,9 @@ rule (``FORCE_ADD``) is prepared the same way ``test_fixture_matrix.py`` prepares it for its own run. ``--check`` compares the already-written files against a fresh generation without rewriting them, and exits -1 on any difference. It is not run by ``make check`` or ``make test`` — regenerating means running the -engine 69 times — so ``tools/tests/test_eval_expected_fixtures.py`` instead loads the committed files and -checks them against the matrix directly, with no engine run. Standard library only. +1 on any difference. ``make check`` runs it as ``make eval-expected-check`` (a few seconds), so a stale file fails +the gate. ``dev/tests/test_eval_expected_fixtures.py`` also loads the committed files and checks them against the +matrix directly, with no engine run. Standard library only. """ from __future__ import annotations diff --git a/dev/tests/test_eval_expected_fixtures.py b/dev/tests/test_eval_expected_fixtures.py index 4134954a..00a99719 100644 --- a/dev/tests/test_eval_expected_fixtures.py +++ b/dev/tests/test_eval_expected_fixtures.py @@ -5,9 +5,10 @@ ``dev/eval/generate_fixture_expected.py`` wrote one ``dev/eval/expected/fixtures/.json`` per row of the engine's ``tests/test_fixture_matrix.py`` (``MATRIX``), running the real engine on a throwaway, -deterministically-committed copy of each fixture. Running the engine 89 times is too slow for this suite, so -this test does not regenerate anything — it loads the committed files and checks them against the matrix and -the expected-results schema directly. Regenerate with the script above after a matrix change. Standard +deterministically-committed copy of each fixture. This test does not regenerate anything: it loads the +committed files and checks them against the matrix and the expected-results schema directly. +``make eval-expected-check`` (part of ``make check``) compares them with a fresh run. Regenerate with the script +above after a matrix change. Standard library only. """ diff --git a/docs/how-to/run-the-evals.md b/docs/how-to/run-the-evals.md index f365cf82..98da1927 100644 --- a/docs/how-to/run-the-evals.md +++ b/docs/how-to/run-the-evals.md @@ -32,7 +32,8 @@ The corpus is two kinds of repository (DP §11, review R-07): of them — each fails exactly one catalogue check by construction, so the expected verdict is known without a human re-deriving it. Their expected files are already generated and committed at `dev/eval/expected/fixtures/.json`; regenerate with - `dev/eval/generate_fixture_expected.py` after a matrix change (S4.2.1). + `dev/eval/generate_fixture_expected.py` after a matrix change (S4.2.1). `make check` fails when one is stale + (`make eval-expected-check`). - **Real repositories**, hand-picked for stack coverage and for the properties DP §11 asks the corpus to exercise. The set holds a repository that already passes, a monorepo and a tiny repository. It also holds one with no tests or CI, and one with a planted hostile file (an injection test). diff --git a/docs/reference/branch-protection.md b/docs/reference/branch-protection.md index 0873a92c..1e6d4644 100644 --- a/docs/reference/branch-protection.md +++ b/docs/reference/branch-protection.md @@ -85,7 +85,7 @@ fails when a matrix misses a shard or a part misses a suite: | Job | What it runs | | --- | --- | -| `static` | `make format-check typecheck verify-rules recipe-lint masters-check workflow-lint spell` | +| `static` | `make format-check typecheck verify-rules recipe-lint masters-check eval-expected-check workflow-lint spell` | | `crosscheck` | `make crosscheck` on Python 3.11 and 3.14 | | `test-coverage` | `make coverage-part`, one job per shard: a shard of the engine's and of the audit tools' suites, and the implement tools in part 1, under coverage.py on Python 3.11; each part runs its targets at once | | `test-dev` | `make test-dev` on Python 3.11 | diff --git a/docs/reference/dev-tooling.md b/docs/reference/dev-tooling.md index 09326e37..d20e7322 100644 --- a/docs/reference/dev-tooling.md +++ b/docs/reference/dev-tooling.md @@ -18,6 +18,7 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | jsonschema | The tools' schema validator disagrees with the reference package | `make crosscheck` | `crosscheck` | `dev/tools/pyproject.toml` | | `dev/recipe_lint.py` | Broken links, bad `SKILL.md` frontmatter, a recipe `SKILL.md` that does not link the shared user prompt, oversize files, unknown tools, check IDs and requirement IDs | `make recipe-lint` | `static` | This repository | | `dev/build_masters.py` | An [audit master list](audit-master.md) or [implement master list](implement-master.md) that no longer matches the catalogue, the check files or the playbooks | `make masters-check`; `make update-masters` rewrites both pages | `static` | This repository | +| `dev/eval/generate_fixture_expected.py` | An eval-expected file (`dev/eval/expected/fixtures/`) that no longer matches a fresh engine run on its fixture | `make eval-expected-check`; the script without `--check` rewrites the files | `static` | This repository | | actionlint | Workflow mistakes: syntax, expressions, action inputs, shellcheck on `run:` blocks | `make workflow-lint` | `static` | `dev/tools/actionlint/go.mod` | | zizmor | Workflow and Dependabot security: injection, broad permissions, unpinned actions, cooldown. Runs at the pedantic persona, so a permission without a comment or a missing concurrency limit fails too | `make workflow-lint` | `static` | `dev/tools/pyproject.toml` | | typos | Misspelled words in the tracked text, code identifiers included (word list: [`_typos.toml`](../../_typos.toml)) ([Spell check](#spell-check)) | `make spell`; `make spell-fix` writes the fixes | `static` | `dev/tools/pyproject.toml` | @@ -39,8 +40,8 @@ and where its version is pinned. The rules for the gates are in [`AGENTS.md`](.. | Go | Builds and runs actionlint and osv-scanner through `go tool` | — | `check`, `security` | the `go` line of each Go tool module | `make check` runs `format:check`, `typecheck`, `test`, the engine's rule check, `crosscheck`, `recipe-lint`, -`masters-check`, `workflow-lint` and `spell`, in sequence. CI's `CI Gate` passes only when the `check`, `security`, -`coverage`, `self-audit`, `docs` and `commits` jobs pass ([Branch protection](branch-protection.md#the-required-checks)). +`masters-check`, `eval-expected-check`, `workflow-lint` and `spell`, in sequence. CI's `CI Gate` passes only when +the `check`, `security`, `coverage`, `self-audit`, `docs` and `commits` jobs pass ([Branch protection](branch-protection.md#the-required-checks)). ## Spell check From 518207775a3653fa2ba531672b5d9728eead3526 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Tue, 10 Nov 2026 06:21:27 +0530 Subject: [PATCH 03/11] docs(spec): let any verb run the dependency scan in REQ-SH-030 REQ-SH-030 said the security verb MUST run the dependency scan, while TOOL-04 passes a scan under any verb. The owner chose the check's reading: a verb MUST run the scan and it SHOULD be security. Spec decision 0003 records why; no verdict moves. Roadmap-Step: 3.1 Co-Authored-By: Claude Opus 5.5 --- docs/architecture/adrs/README.md | 1 + spec/README.md | 2 +- spec/decisions/0003-dependency-scan-verb.md | 42 +++++++++++++++++++++ spec/readiness-requirements.md | 3 +- 4 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 spec/decisions/0003-dependency-scan-verb.md diff --git a/docs/architecture/adrs/README.md b/docs/architecture/adrs/README.md index 249eb3f0..93570a58 100644 --- a/docs/architecture/adrs/README.md +++ b/docs/architecture/adrs/README.md @@ -23,6 +23,7 @@ The root router is [`docs/README.md`](../../README.md). | --- | --- | --- | --- | | [0001](../../../spec/decisions/0001-constraint-artifact.md) | Use the prohibitions section as the deny list | Accepted | 2026-10-10 | | [0002](../../../spec/decisions/0002-lint-in-the-gate.md) | Lint is gated through `format:check` | Accepted | 2026-11-05 | +| [0003](../../../spec/decisions/0003-dependency-scan-verb.md) | Any verb may run the dependency scan; `security` is the preferred one | Accepted | 2026-11-10 | ## Audit recipe and tools: `audit/docs/architecture/adrs/` diff --git a/spec/README.md b/spec/README.md index c90af8f7..f8bb25da 100644 --- a/spec/README.md +++ b/spec/README.md @@ -79,7 +79,7 @@ Where the framework documents leave a question open, the answer is recorded in ` - [command-surface.md](command-surface.md): the verbs every repository exposes and how each runner spells them. - [readiness-summary.md](readiness-summary.md): the headline of a readiness report: the Phase-1 score, the blockers remaining and the unattested list. - [traceability.md](traceability.md): which requirements each audit-engine check verifies, which requirements have no check, and which go beyond the framework documents. -- [decisions/](decisions/0001-constraint-artifact.md): decision records. 0001 settles what the allow/deny artifact is; 0002 settles where lint is gated. +- [decisions/](decisions/0001-constraint-artifact.md): decision records. 0001 settles what the allow/deny artifact is; 0002 settles where lint is gated; 0003 settles which verb runs the dependency scan. ### Planned diff --git a/spec/decisions/0003-dependency-scan-verb.md b/spec/decisions/0003-dependency-scan-verb.md new file mode 100644 index 00000000..1ea605ac --- /dev/null +++ b/spec/decisions/0003-dependency-scan-verb.md @@ -0,0 +1,42 @@ +# 0003. Any verb may run the dependency scan; `security` is the preferred one + +* **Status:** Accepted +* **Date:** 2026-11-10 +* **Last reviewed:** 2026-11-10 +* **Deciders:** K. C. Ramakrishna (v0.3.5.1 question 3.1, accepted 2026-11-10) + +--- + +## Context + +[REQ-SH-030](../readiness-requirements.md#req-sh-030--dependency-vulnerability-scan) said "the `security` verb +MUST run a dependency vulnerability scan". The `TOOL-04` check gives `PASS` when any verb runs the scan, for example +a `check` that runs `osv-scanner`. A repository that scans inside `check` is as safe for an agent as one with a +`security` verb: the risk is an unscanned dependency, not the name of the verb. The requirement and the check +disagreed, and the requirement is normative, so one of them had to change. + +## Decision + +1. **A verb MUST run a dependency vulnerability scan.** Any verb of the [command surface](../command-surface.md) + counts. +2. **The verb SHOULD be `security`.** It is the verb the command surface names for this gate. +3. **CI SHOULD run the same scan**, as before. + +`TOOL-04` already reads the requirement this way, so no verdict moves. + +## Alternatives Considered + +| Alternative | Why it was rejected | +|---|---| +| Keep MUST on `security`; `TOOL-04` gives `PARTIAL` when only another verb runs the scan | Moves verdicts on repositories that are safe today, for a naming rule. Needs an engine change and a ruleset bump. | + +## Consequences + +### Positive + +* The requirement and `TOOL-04` agree; no engine change. +* A repository that scans inside `check` or `test` is not marked down for the verb's name. + +### Negative / Trade-offs + +* A repository may have no `security` verb at all and still meet the MUST. The SHOULD keeps the name visible. diff --git a/spec/readiness-requirements.md b/spec/readiness-requirements.md index af41be23..9d1b4732 100644 --- a/spec/readiness-requirements.md +++ b/spec/readiness-requirements.md @@ -276,7 +276,8 @@ Where the language has a type checker, the repository MUST configure it in stric - **Checks:** TOOL-03 ### REQ-SH-030 — Dependency vulnerability scan -The `security` verb MUST run a dependency vulnerability scan, and CI SHOULD run the same scan. +A verb MUST run a dependency vulnerability scan, and it SHOULD be the `security` verb. CI SHOULD run the same scan; +see [decision 0003](decisions/0003-dependency-scan-verb.md). - **Phase:** 1 - **Verification:** automated From 6a98a5dd6cd7acba2a95c728867d9bbe50d4a3cf Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Tue, 10 Nov 2026 06:18:31 +0530 Subject: [PATCH 04/11] fix(skills): resolve every link the audit bundle keeps The audit bundle rewrote only the link to the shared prompt. It now uses the same rewrite as the implement bundle. A link to a carried file points at its place in the bundle. Any other link becomes plain text naming the framework path. The bundle also carries the Tools CLI sections for its five tools, so the SKILL.md tool links resolve. Both copies of the schema docs are rewritten for their own folder. A new test holds every link in the bundle resolvable. Roadmap-Step: 1.3 Co-Authored-By: Claude Opus 5.5 --- dev/build_skills.py | 169 ++++++++++++++++++++++----------- dev/tests/test_build_skills.py | 80 ++++++++++++---- docs/reference/tools-cli.md | 15 ++- 3 files changed, 187 insertions(+), 77 deletions(-) diff --git a/dev/build_skills.py b/dev/build_skills.py index 97771da4..d17aa0ab 100644 --- a/dev/build_skills.py +++ b/dev/build_skills.py @@ -22,8 +22,8 @@ (:func:`stamp_frontmatter`), so an installed bundle says which release it is. - ``references/``: the check files (``audit/checks/``), the stack profiles (``audit/stacks/``), the shared prompt on how to talk to the user (``audit/prompts/talking-to-the-user.md``, which the implement bundle carries too) and - the schema docs (``audit/schema/*.schema.json``, ``SCHEMA.md``, ``LICENSING.md``) — read by the model, not run. - ``SKILL.md``'s link to the prompt is rewritten to its place under ``references/prompts/``. + the schema docs (``audit/schema/*.schema.json``, ``SCHEMA.md``, ``LICENSING.md``), and the parts of the Tools CLI + reference that cover the audit tools (:data:`AUDIT_TOOLS_CLI_SECTIONS`) — read by the model, not run. - ``scripts/``: the five tools the recipe's Tools table calls (``facts.py``, ``engine.py``, ``report.py``, ``validate_report.py``, ``render_report.py``, from ``audit/tools/``) and the private modules they import; the schema files and ``catalogue.json`` those tools read at a fixed path next to themselves @@ -32,12 +32,7 @@ and ``audit.pyz``, the engine's single-file build, for a person who wants to run it directly with no Python path setup at all. -Some of the audit ``SKILL.md``'s own relative links point outside what that bundle carries (the Tools CLI -reference, ``spec/command-surface.md``, ``docs/framework/phased-approach.md`` from the check files) — found, -not fixed, for the audit bundle. - -**agentic-readiness-implement** carries the implement recipe, a ``LICENSE`` like the audit bundle's, and resolves -every link it keeps: +**agentic-readiness-implement** carries the implement recipe and a ``LICENSE`` like the audit bundle's: - ``SKILL.md``: ``implement/SKILL.md``, its ``name`` rewritten from ``implement`` to the bundle's name, and the same ``metadata.version`` added. @@ -53,9 +48,10 @@ agentic-readiness-audit skill installed in the same skills folder, so the two skills share one engine copy (``validate_report.py`` reads the engine's credential patterns). -Every carried Markdown file is rewritten in two ways. A command ``python3 audit/tools/.py`` or +Both bundles resolve every link they keep. Every carried Markdown file is rewritten in two ways. A command ``python3 audit/tools/.py`` or ``python3 implement/tools/.py`` becomes the path from the skill's own folder -(``python3 scripts/tools/.py``; ``engine.py`` points at the audit skill beside it). A relative link to a +(``python3 scripts/tools/.py``; in the implement bundle ``engine.py`` points at the audit skill beside it). +A relative link to a carried file is rewritten to that file's place in the bundle; a link to anything else becomes plain text naming the framework path, so no link leaves the bundle. @@ -143,9 +139,32 @@ #: a person pastes it into a harness; the skill does not read it. AUDIT_PROMPT_FILES = ("talking-to-the-user.md",) -#: `audit/SKILL.md` links each carried prompt relative to itself, with or without a heading anchor; in the bundle -#: the prompt sits under `references/`. -AUDIT_SKILL_LINK_REWRITES = tuple((f"](prompts/{name}", f"](references/prompts/{name}") for name in AUDIT_PROMPT_FILES) +#: The level-2 sections of the Tools CLI reference the audit bundle carries, in page order: the five tools of +#: `audit/SKILL.md`'s Tools table and the contract every tool shares. +AUDIT_TOOLS_CLI_SECTIONS = ( + "Every tool", + "engine.py", + "validate_report.py", + "report.py", + "render_report.py", + "facts.py", +) + +#: The source of each carried Markdown file or folder of the audit bundle and its place in the bundle, as +#: :data:`IMPLEMENT_REFERENCES` is for the implement bundle. The first place of a source is the one links point at; +#: the schema docs also sit beside the schema files in `scripts/schema/`, rewritten for that place. +AUDIT_REFERENCES = ( + ("audit/SKILL.md", "SKILL.md"), + ("audit/checks", "references/checks"), + ("audit/stacks", "references/stacks"), + *((f"audit/prompts/{name}", f"references/prompts/{name}") for name in AUDIT_PROMPT_FILES), + *((f"audit/schema/{name}", f"references/schema/{name}") for name in SCHEMA_FILES if name.endswith(".md")), + ("docs/reference/tools-cli.md", "references/tools-cli.md"), + *((f"audit/schema/{name}", f"scripts/schema/{name}") for name in SCHEMA_FILES if name.endswith(".md")), +) + +#: The audit sources name a tool by its file name alone; a command written for a checkout still gets the bundle path. +AUDIT_COMMAND_REWRITES = (("python3 audit/tools/", "python3 scripts/tools/"),) SKILL_MD_SOURCE_NAME = "name: audit\n" SKILL_MD_BUNDLE_NAME = f"name: {BUNDLE_NAME}\n" @@ -187,26 +206,40 @@ def stamp_frontmatter(text: str, source: str, source_name: str, bundle_name: str return frontmatter + text[end:] -def _write_skill_md(bundle_dir: Path) -> None: - text = (AUDIT / "SKILL.md").read_text(encoding="utf-8") - for old, new in AUDIT_SKILL_LINK_REWRITES: - text = text.replace(old, new) - stamped = stamp_frontmatter(text, "audit/SKILL.md", SKILL_MD_SOURCE_NAME, SKILL_MD_BUNDLE_NAME, release_version()) - (bundle_dir / "SKILL.md").write_text(stamped, encoding="utf-8") +def _audit_carried_other() -> dict[Path, str]: + """The audit bundle's carried files that are not Markdown: tools, schema files, the catalogue, the licence.""" + carried = {(AUDIT_TOOLS / name).resolve(): f"scripts/tools/{name}" for name in TOOL_FILES} + for name in SCHEMA_FILES: + if not name.endswith(".md"): + carried[(AUDIT_SCHEMA / name).resolve()] = f"references/schema/{name}" + carried[(AUDIT / "catalogue.json").resolve()] = "scripts/catalogue.json" + carried[LICENSE_CODE.resolve()] = "LICENSE" + return carried + + +def _write_audit_markdown(bundle_dir: Path) -> None: + """Write every carried Markdown file of the audit bundle, its commands and links rewritten for the bundle.""" + markdown = _carried_markdown(AUDIT_REFERENCES) + carried = {**_audit_carried_other(), **markdown} + texts: dict[str, tuple[Path, str]] = {} + for source, dest in _reference_places(AUDIT_REFERENCES): + text = source.read_text(encoding="utf-8") + if dest == "SKILL.md": + text = stamp_frontmatter( + text, "audit/SKILL.md", SKILL_MD_SOURCE_NAME, SKILL_MD_BUNDLE_NAME, release_version() + ) + elif dest == "references/tools-cli.md": + text = extract_sections(text, AUDIT_TOOLS_CLI_SECTIONS) + texts[dest] = (source, text) + _write_rewritten(bundle_dir, texts, carried, AUDIT_COMMAND_REWRITES) def _write_references(bundle_dir: Path) -> None: - references = bundle_dir / "references" - _copy_tree(AUDIT / "checks", references / "checks") - _copy_tree(AUDIT / "stacks", references / "stacks") - prompts_dir = references / "prompts" - prompts_dir.mkdir(parents=True) - for name in AUDIT_PROMPT_FILES: - shutil.copy2(AUDIT / "prompts" / name, prompts_dir / name) - schema_dir = references / "schema" + schema_dir = bundle_dir / "references" / "schema" schema_dir.mkdir(parents=True) for name in SCHEMA_FILES: - shutil.copy2(AUDIT_SCHEMA / name, schema_dir / name) + if not name.endswith(".md"): + shutil.copy2(AUDIT_SCHEMA / name, schema_dir / name) def _build_pyz(dest: Path) -> None: @@ -230,7 +263,8 @@ def _write_scripts(bundle_dir: Path) -> None: schema_dir = scripts / "schema" schema_dir.mkdir() for name in SCHEMA_FILES: - shutil.copy2(AUDIT_SCHEMA / name, schema_dir / name) + if not name.endswith(".md"): + shutil.copy2(AUDIT_SCHEMA / name, schema_dir / name) shutil.copy2(AUDIT / "catalogue.json", scripts / "catalogue.json") _copy_tree(ENGINE / "audit", scripts / "engine" / "audit") _build_pyz(scripts / "audit.pyz") @@ -243,9 +277,9 @@ def build_bundle(out_dir: Path) -> Path: shutil.rmtree(bundle_dir) bundle_dir.mkdir(parents=True) _write_license(bundle_dir) - _write_skill_md(bundle_dir) _write_references(bundle_dir) _write_scripts(bundle_dir) + _write_audit_markdown(bundle_dir) return bundle_dir @@ -363,19 +397,42 @@ def load_engine(): _LEVEL2 = re.compile(r"^## (.+?)\s*$") -def _carried_markdown() -> dict[Path, str]: - """Every carried Markdown source file (absolute) and its path in the bundle (POSIX, relative).""" - carried: dict[Path, str] = {} - for source, dest in IMPLEMENT_REFERENCES: +def _reference_places(references: Sequence[tuple[str, str]]) -> list[tuple[Path, str]]: + """Every carried Markdown source file (absolute) and each of its places in the bundle (POSIX, relative).""" + places: list[tuple[Path, str]] = [] + for source, dest in references: path = REPO / source if path.is_dir(): - for file in sorted(path.rglob("*.md")): - carried[file.resolve()] = f"{dest}/{file.relative_to(path).as_posix()}" + places.extend( + (file.resolve(), f"{dest}/{file.relative_to(path).as_posix()}") for file in sorted(path.rglob("*.md")) + ) else: - carried[path.resolve()] = dest + places.append((path.resolve(), dest)) + return places + + +def _carried_markdown(references: Sequence[tuple[str, str]]) -> dict[Path, str]: + """Every carried Markdown source file and the place in the bundle a link to it points at: its first place.""" + carried: dict[Path, str] = {} + for source, dest in _reference_places(references): + carried.setdefault(source, dest) return carried +def _write_rewritten( + bundle_dir: Path, + texts: dict[str, tuple[Path, str]], + carried: dict[Path, str], + commands: Sequence[tuple[str, str]], +) -> None: + """Write each text at its bundle place `dest`, its tool commands and relative links rewritten for that place.""" + slugs = {dest: _heading_slugs(text) for dest, (_, text) in texts.items()} + for dest, (source, text) in texts.items(): + target = bundle_dir / dest + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(rewrite_text(text, source, dest, carried, slugs, commands), encoding="utf-8") + + def _carried_other() -> dict[Path, str]: """The carried files that are not Markdown: tools and schema files, which links may point at too.""" carried = {(IMPLEMENT_TOOLS / name).resolve(): f"scripts/tools/{name}" for name in IMPLEMENT_OWN_TOOL_FILES} @@ -388,8 +445,9 @@ def _carried_other() -> dict[Path, str]: return carried -def extract_sections(text: str, titles: Sequence[str]) -> str: - """The page's title line, a note, and only the level-2 sections named in `titles`, in page order.""" +def extract_sections(text: str, titles: Sequence[str], note_tail: str = "") -> str: + """The page's title line, a note (ending with `note_tail`), and only the level-2 sections named in `titles`, + in page order.""" lines = text.splitlines(keepends=True) sections: dict[str, list[str]] = {} current: str | None = None @@ -407,23 +465,30 @@ def extract_sections(text: str, titles: Sequence[str]) -> str: if missing: raise _common.ToolError(f"docs/reference/tools-cli.md has no section {', '.join(missing)}") title = lines[0] if lines and lines[0].startswith("# ") else "# Tools CLI Reference\n" - note = ( - "\nThe sections of the framework's Tools CLI reference that cover the tools this skill carries. The " - f"engine's own section is in the {BUNDLE_NAME} skill.\n\n" + note = "\nThe sections of the framework's Tools CLI reference that cover the tools this skill carries." + ( + f" {note_tail}" if note_tail else "" ) + note += "\n\n" body = "".join("".join(sections[t]).rstrip("\n") + "\n\n" for t in titles) return title + note + body.rstrip("\n") + "\n" def _heading_slugs(text: str) -> set[str]: - import recipe_lint # a sibling tool; imported here so the audit-only path does not need it + import recipe_lint # a sibling tool, imported where it is used return recipe_lint.heading_slugs(Path("x.md"), text.splitlines()) -def rewrite_text(text: str, source: Path, dest: str, carried: dict[Path, str], slugs: dict[str, set[str]]) -> str: +def rewrite_text( + text: str, + source: Path, + dest: str, + carried: dict[Path, str], + slugs: dict[str, set[str]], + commands: Sequence[tuple[str, str]] = COMMAND_REWRITES, +) -> str: """Rewrite one carried file's tool commands and relative links for its place `dest` in the bundle.""" - for old, new in COMMAND_REWRITES: + for old, new in commands: text = text.replace(old, new) out: list[str] = [] fenced = False @@ -470,23 +535,21 @@ def build_implement_bundle(out_dir: Path) -> Path: shutil.rmtree(bundle_dir) bundle_dir.mkdir(parents=True) _write_license(bundle_dir) - markdown = _carried_markdown() + markdown = _carried_markdown(IMPLEMENT_REFERENCES) carried = {**_carried_other(), **markdown} texts: dict[str, tuple[Path, str]] = {} - for source, dest in markdown.items(): + for source, dest in _reference_places(IMPLEMENT_REFERENCES): text = source.read_text(encoding="utf-8") if dest == "SKILL.md": text = stamp_frontmatter( text, "implement/SKILL.md", IMPLEMENT_SKILL_SOURCE_NAME, IMPLEMENT_SKILL_BUNDLE_NAME, release_version() ) elif dest == "references/tools-cli.md": - text = extract_sections(text, TOOLS_CLI_SECTIONS) + text = extract_sections( + text, TOOLS_CLI_SECTIONS, f"The engine's own section is in the {BUNDLE_NAME} skill." + ) texts[dest] = (source, text) - slugs = {dest: _heading_slugs(text) for dest, (_, text) in texts.items()} - for dest, (source, text) in texts.items(): - target = bundle_dir / dest - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(rewrite_text(text, source, dest, carried, slugs), encoding="utf-8") + _write_rewritten(bundle_dir, texts, carried, COMMAND_REWRITES) for name in IMPLEMENT_SCHEMA_FILES: shutil.copy2(AUDIT_SCHEMA / name, bundle_dir / "references" / "schema" / name) tools_dir = bundle_dir / "scripts" / "tools" diff --git a/dev/tests/test_build_skills.py b/dev/tests/test_build_skills.py index e4e91c61..60ca28c9 100644 --- a/dev/tests/test_build_skills.py +++ b/dev/tests/test_build_skills.py @@ -3,8 +3,8 @@ """The skill bundler: each bundle has its three parts (`SKILL.md`, `references/`, `scripts/`) and a `LICENSE`, the frontmatter name is rewritten for the bundle, and two builds from the same source tree agree file for file. -The implement bundle also keeps no framework tool path and no link that leaves it, and its tools run against the -audit bundle installed beside it. +Neither bundle keeps a framework tool path or a link that leaves it, and the implement bundle's tools run against +the audit bundle installed beside it. Builds for real, into a temporary directory — there is no fixture to fake this against, since the whole point is copying this repository's own files; a build that can run only against a double would not prove the @@ -172,6 +172,66 @@ def test_rebuilding_into_the_same_out_dir_replaces_the_bundle_whole(self) -> Non self.assertFalse(stray.exists()) +def assert_links_resolve_inside(test: unittest.TestCase, bundle: Path) -> None: + """Every relative link in every Markdown file of `bundle` names an existing file inside it, and a heading there.""" + root = bundle.resolve() + for file in sorted(bundle.rglob("*.md")): + text = file.read_text(encoding="utf-8") + for target in links_outside_fences(text): + if re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*:", target): + continue + where = f"{file.relative_to(bundle)} -> {target}" + path_part, _, fragment = target.partition("#") + resolved = (file.parent / path_part).resolve() if path_part else file.resolve() + test.assertTrue(resolved == root or root in resolved.parents, f"leaves the bundle: {where}") + test.assertTrue(resolved.exists(), f"does not resolve: {where}") + if fragment and resolved.suffix == ".md": + lines = resolved.read_text(encoding="utf-8").splitlines() + test.assertIn(fragment, recipe_lint.heading_slugs(resolved, lines), f"no such heading: {where}") + + +class AuditBundleLinkTests(unittest.TestCase): + _tmp: "tempfile.TemporaryDirectory[str]" + bundle: Path + + @classmethod + def setUpClass(cls) -> None: + cls._tmp = tempfile.TemporaryDirectory() + cls.bundle = bs.build_bundle(Path(cls._tmp.name)) + + @classmethod + def tearDownClass(cls) -> None: + cls._tmp.cleanup() + + def test_every_link_resolves_inside_the_bundle(self) -> None: + assert_links_resolve_inside(self, self.bundle) + + def test_a_link_to_a_file_the_bundle_lacks_becomes_plain_text(self) -> None: + text = (self.bundle / "SKILL.md").read_text(encoding="utf-8") + self.assertIn("(framework: `spec/command-surface.md", text) + self.assertNotIn("](../", text) + + def test_skill_md_links_the_carried_tools_cli_sections(self) -> None: + text = (self.bundle / "SKILL.md").read_text(encoding="utf-8") + self.assertIn("](references/tools-cli.md#enginepy)", text) + reference = (self.bundle / "references" / "tools-cli.md").read_text(encoding="utf-8") + headings = [line[3:] for line in reference.splitlines() if line.startswith("## ")] + self.assertEqual(headings, list(bs.AUDIT_TOOLS_CLI_SECTIONS)) + self.assertNotIn("engine's own section", reference) + + def test_no_framework_tool_command_is_left(self) -> None: + for file in sorted(self.bundle.rglob("*.md")): + text = file.read_text(encoding="utf-8") + for prefix in SOURCE_TOOL_PREFIXES: + self.assertNotIn(prefix, text, file.relative_to(self.bundle)) + + def test_both_schema_doc_copies_are_rewritten_for_their_place(self) -> None: + references = (self.bundle / "references" / "schema" / "SCHEMA.md").read_text(encoding="utf-8") + scripts = (self.bundle / "scripts" / "schema" / "SCHEMA.md").read_text(encoding="utf-8") + self.assertIn("](../../scripts/tools/facts.py)", references) + self.assertIn("](../tools/facts.py)", scripts) + + def links_outside_fences(text: str) -> list[str]: targets: list[str] = [] fenced = False @@ -262,25 +322,13 @@ def test_no_framework_tool_command_is_left(self) -> None: self.assertIn(f"python3 ../{bs.BUNDLE_NAME}/scripts/tools/engine.py", skill) def test_every_link_resolves_inside_the_bundle(self) -> None: - root = self.bundle.resolve() - for file in self.markdown_files(): - text = file.read_text(encoding="utf-8") - for target in links_outside_fences(text): - if re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*:", target): - continue - where = f"{file.relative_to(self.bundle)} -> {target}" - path_part, _, fragment = target.partition("#") - resolved = (file.parent / path_part).resolve() if path_part else file.resolve() - self.assertTrue(resolved == root or root in resolved.parents, f"leaves the bundle: {where}") - self.assertTrue(resolved.exists(), f"does not resolve: {where}") - if fragment and resolved.suffix == ".md": - lines = resolved.read_text(encoding="utf-8").splitlines() - self.assertIn(fragment, recipe_lint.heading_slugs(resolved, lines), f"no such heading: {where}") + assert_links_resolve_inside(self, self.bundle) def test_tools_cli_reference_keeps_only_the_carried_tools(self) -> None: text = (self.bundle / "references" / "tools-cli.md").read_text(encoding="utf-8") headings = [line[3:] for line in text.splitlines() if line.startswith("## ")] self.assertEqual(headings, list(bs.TOOLS_CLI_SECTIONS)) + self.assertIn(f"The engine's own section is in the {bs.BUNDLE_NAME} skill.", text) def test_extract_sections_refuses_a_missing_section(self) -> None: with self.assertRaises(_common.ToolError): diff --git a/docs/reference/tools-cli.md b/docs/reference/tools-cli.md index b4fd2c99..bfa5ae18 100644 --- a/docs/reference/tools-cli.md +++ b/docs/reference/tools-cli.md @@ -797,12 +797,11 @@ bundle is replaced whole on every run, so a stale file from an earlier build can - `references/`: read by the model; - `scripts/`: run by it. -- **`agentic-readiness-audit`**: `references/` holds the check files, the stack profiles, the shared prompt on - how to talk to the user and the schema docs. `scripts/` holds the tools the recipe calls. Beside them are the - schema files and the `catalogue.json` they read at a fixed path, and the engine's `audit` package, one folder - level up as `engine.py` expects. `audit.pyz` is the engine's single-file build. The link to the shared prompt is - rewritten to its place in the bundle. Some other links in its `SKILL.md` point outside what the bundle carries - (found, not fixed; `skills-ref` validates only the frontmatter and the directory-name match). +- **`agentic-readiness-audit`**: `references/` holds the check files, the stack profiles and the schema docs. It + also holds the shared prompt on how to talk to the user and the sections of this page for the tools it carries. + `scripts/` holds the tools the recipe calls. Beside them are the schema files and the `catalogue.json` they read + at a fixed path. The engine's `audit` package sits one folder level up, as `engine.py` expects. `audit.pyz` is + the engine's single-file build. Its Markdown files are rewritten the same way as the implement bundle's, below. - **`agentic-readiness-implement`**: `references/` holds the state contract, the playbooks, the kickoff prompt and the shared prompt on how to talk to the user. It also holds the check files, the stack profiles, ADR 0003, `spec/command-surface.md` and the spec decisions, the state schema and its docs, and the sections of this page @@ -810,9 +809,9 @@ bundle is replaced whole on every run, so a stale file from an earlier build can `validate_report.py` with their private modules, the schema files, `catalogue.json`, and an `engine.py` shim. The engine is not copied. The shim loads `engine.py` from the `agentic-readiness-audit` skill installed in the same skills folder, so both skills run one engine. A tool that needs the engine without it exits 1 and names the - missing skill. Every carried Markdown file is rewritten in three ways: + missing skill. In both bundles, every carried Markdown file is rewritten in three ways: - a `python3 audit/tools/.py` command becomes the path from the skill's own folder - (`python3 scripts/tools/.py`, or the audit skill's path for `engine.py`); + (`python3 scripts/tools/.py`; in the implement bundle, the audit skill's path for `engine.py`); - a link to a carried file points at its place in the bundle; - any other relative link becomes plain text that names the framework path. No link leaves the bundle. From 71b4e616f1ea8d48746c1c68e6ccc49a5b81cd08 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Tue, 10 Nov 2026 06:28:04 +0530 Subject: [PATCH 05/11] fix(engine): TOOL-07 needs a guard that exits non-zero A hook that read the branch and only printed a warning passed TOOL-07, though the commit went ahead. A guard that reads the branch must now also exit non-zero (exit 1 to 255, process.exit(1), sys.exit(1) or a bare false) in its lefthook command block or hook script; comments are skipped. no-commit-to-branch always stops the commit. No wired guard that exits is PARTIAL, and branch coverage is read only on the guards that exit. Ruleset 2026-11-10.3; no catalogue bump. One-defect fixture tool-07-guard-no-exit (a lefthook guard that only warns, PARTIAL) with its eval file, six in-process tests, rubric and changelog. Self-audit: TOOL-07 still PASS (the lefthook guard exits 1). Roadmap-Step: 4.4 Co-Authored-By: Claude Opus 5.5 --- audit/checks/05-tooling.md | 11 +++-- audit/engine/audit/__init__.py | 2 +- audit/engine/audit/rules/checks/tool.py | 44 +++++++++++++++--- audit/engine/docs/reference/changelog.md | 1 + audit/engine/tests/fixtures/README.md | 1 + .../tool-07-guard-no-exit/lefthook.yml | 10 +++++ .../packs/tool/test_tool07_branch_guard.py | 45 +++++++++++++++++++ audit/engine/tests/test_fixture_matrix.py | 1 + .../fixtures/tool-07-guard-no-exit.json | 20 +++++++++ 9 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 audit/engine/tests/fixtures/tool-07-guard-no-exit/lefthook.yml create mode 100644 dev/eval/expected/fixtures/tool-07-guard-no-exit.json diff --git a/audit/checks/05-tooling.md b/audit/checks/05-tooling.md index 0a53460a..30ae55e5 100644 --- a/audit/checks/05-tooling.md +++ b/audit/checks/05-tooling.md @@ -263,8 +263,11 @@ Engine `TOOL-07`. A guard is pre-commit's `no-commit-to-branch` hook ( globs are not required. With no declared rules, any guard counts. A `.husky/` guard is wired when a `package.json` script runs `husky`; another hook directory is wired when `core.hooksPath` is set. Distrust a `FAIL` when the hook calls a script that holds the guard - (`.husky/pre-commit` running `node scripts/check-branch.js`) — read the script. The engine - does not read whether the guard exits non-zero. + (`.husky/pre-commit` running `node scripts/check-branch.js`) — read the script. A guard + that reads the branch must also exit non-zero (`exit 1`, `process.exit(1)`, a bare `false`) + in its lefthook command block or anywhere in its hook script; `no-commit-to-branch` always + does. The exit is not tied to the branch test, so read a script whose only `exit 1` is for + another reason. Method automated Look for `.pre-commit-config.yaml` for `no-commit-to-branch` and its `args`; the `pre-commit` and `commit-msg` stages of `lefthook.yml`; the hook scripts named `pre-commit` and `commit-msg`; @@ -272,7 +275,9 @@ Look for `.pre-commit-config.yaml` for `no-commit-to-branch` and its `ar Rubric PASS: a guard in a wired hook chain names every branch the declared rules protect, or any guard is wired and no rules are declared. PARTIAL: a guard exists but its hook chain is not wired (an unreferenced `.husky/`, a - `.githooks/` with no `core.hooksPath`); or a wired guard does not name a protected branch. + `.githooks/` with no `core.hooksPath`); or no wired guard exits non-zero (it reads the + branch, prints a warning and lets the commit go ahead); or a wired guard does not name a + protected branch. FAIL: a hook chain exists (wired or configured) and no hook in it is a guard. NOT_APPLICABLE: never. UNKNOWN when: no hook chain exists at all; `TOOL-05` owns that finding. diff --git a/audit/engine/audit/__init__.py b/audit/engine/audit/__init__.py index 1d54cb9a..3a84610e 100644 --- a/audit/engine/audit/__init__.py +++ b/audit/engine/audit/__init__.py @@ -24,7 +24,7 @@ #: Bumped whenever a rule pack, a check, or the instruction-variant table changes. #: Recorded in every report's provenance block so a report can be attributed to an #: exact catalogue revision (docs: Scan Engine & Tiering, § 7 Determinism). -__ruleset_revision__ = "2026-11-10.2" +__ruleset_revision__ = "2026-11-10.3" #: Highest framework phase this build evaluates. Phase 1 only in v1. __phase_ceiling__ = 1 diff --git a/audit/engine/audit/rules/checks/tool.py b/audit/engine/audit/rules/checks/tool.py index e9897a02..120b3728 100644 --- a/audit/engine/audit/rules/checks/tool.py +++ b/audit/engine/audit/rules/checks/tool.py @@ -46,9 +46,12 @@ `pre-commit`/`commit-msg` hook script that reads the current branch (`git symbolic-ref ... HEAD`, `git branch --show-current`, `git rev-parse --abbrev-ref HEAD`). The branches it must name are the literal ones the declared rules protect (:func:`audit.rules.checks._common.declared_branch_rules`); with none declared, any - guard counts. A guard in a wired chain that names them all ⇒ `PASS`; a guard in a chain that is not wired, or one + guard counts. A guard that reads the branch must also exit non-zero (``exit 1``, ``process.exit(1)``, a bare + ``false``) in its command block or hook script; ``no-commit-to-branch`` always does. A stopping guard in a wired + chain that names them all ⇒ `PASS`; a guard in a chain that is not wired, a guard with no non-zero exit, or one that misses a protected branch ⇒ `PARTIAL`; hooks but no guard ⇒ `FAIL`; no hook at all ⇒ `UNKNOWN`, which is - `TOOL-05`'s finding. Whether the guard exits non-zero, and a guard in a script the hook calls, are not read. + `TOOL-05`'s finding. The exit is read anywhere in a hook script, not tied to the branch test; a guard in a script + the hook calls is not read. Deliberate limits, recorded rather than hidden. **Config detection is filename-based** against the documented set below; a config that lives somewhere unexpected is not seen. **Verb resolution is @@ -1034,6 +1037,12 @@ def check_tool06(*, spec, target, inventory, stack, components, session) -> Chec #: without ``--pattern`` (``args.branch or ('master', 'main')`` in its source, checked at v6.0.0). _NO_COMMIT_DEFAULTS = ("main", "master") +#: A command that ends the hook with a non-zero status, so the commit stops: ``exit 1`` (any 1 to 255), +#: ``process.exit(1)`` or ``sys.exit(1)`` in a script hook, or a bare ``false`` as a command. +_NONZERO_EXIT_RE = re.compile( + r"\bexit\s+[1-9]|\bexit\s*\(\s*[1-9]|(?:^|[;&|{(]|\bthen|\belse|\bdo)\s*false\s*(?:$|[;&|})])", re.MULTILINE +) + _ARG_SEP = r"[\s,='\"\-]+" _BRANCH_ARG_RE = re.compile(r"(?:--branch|\B-b)" + _ARG_SEP + r"([A-Za-z0-9_][\w./-]*)") _PATTERN_ARG_RE = re.compile(r"(?:--pattern|\B-p)" + _ARG_SEP + r"([^\s,'\"\]]+)") @@ -1042,12 +1051,15 @@ def check_tool06(*, spec, target, inventory, stack, components, session) -> Chec class _Guard: """One guard: where it is, whether its hook chain is wired, and which branches it names.""" - def __init__(self, rel: str, line: int, wired: bool, text: str, patterns: tuple[str, ...] = ()) -> None: + def __init__( + self, rel: str, line: int, wired: bool, text: str, patterns: tuple[str, ...] = (), *, exits: bool = True + ) -> None: self.rel = rel self.line = line self.wired = wired self.text = text self.patterns = patterns + self.exits = exits def covers(self, branch: str) -> bool: for pattern in self.patterns: @@ -1099,7 +1111,9 @@ def _branch_read_guards(text: str, rel: str, wired: bool, *, whole_file: bool, s continue if staged and _stage(lines, index) not in _GUARD_HOOK_NAMES: continue - return [_Guard(rel, index + 1, wired, text if whole_file else _block(lines, index))] + guard_text = text if whole_file else _block(lines, index) + uncommented = "\n".join(line for line in guard_text.splitlines() if not line.lstrip().startswith("#")) + return [_Guard(rel, index + 1, wired, guard_text, exits=bool(_NONZERO_EXIT_RE.search(uncommented)))] return [] @@ -1166,7 +1180,25 @@ def check_tool07(*, spec, target, inventory, stack, components, session) -> Chec ], ) - missing = [branch for branch in protected if not any(guard.covers(branch) for guard in wired_guards)] + stopping = [guard for guard in wired_guards if guard.exits] + if not stopping: + return _outcome( + spec, + Verdict.PARTIAL, + f"branch guard at {where} reads the branch but never exits non-zero", + [ + _finding( + spec, + "rely on the branch guard to stop a commit on a protected branch", + f"the guard at {where} reads the branch but has no non-zero exit, so the commit goes ahead", + Verdict.PARTIAL, + [Evidence(guard.rel, line=guard.line) for guard in wired_guards], + "End the guard with `exit 1` when the branch is protected, so the hook stops the commit.", + ) + ], + ) + where = ", ".join(f"{guard.rel}:{guard.line}" for guard in stopping) + missing = [branch for branch in protected if not any(guard.covers(branch) for guard in stopping)] if missing: return _outcome( spec, @@ -1178,7 +1210,7 @@ def check_tool07(*, spec, target, inventory, stack, components, session) -> Chec f"learn before a commit that {', '.join(missing)} is protected", f"the guard at {where} does not name {', '.join(missing)}", Verdict.PARTIAL, - [Evidence(guard.rel, line=guard.line) for guard in wired_guards], + [Evidence(guard.rel, line=guard.line) for guard in stopping], f"Add {', '.join(missing)} to the guard's branch list.", ) ], diff --git a/audit/engine/docs/reference/changelog.md b/audit/engine/docs/reference/changelog.md index be7e9834..1fa1b215 100644 --- a/audit/engine/docs/reference/changelog.md +++ b/audit/engine/docs/reference/changelog.md @@ -13,6 +13,7 @@ the record of how; git has the rest. | Revision | Date | Scoreable checks | What it is | |---|---|---|---| +| `2026-11-10.3` | 2026-11-10 | 60 | v0.3.5.1 item 4.4: `TOOL-07` (local guard against commits to a protected branch, COSMETIC) reads whether a guard that reads the branch also exits non-zero: `exit 1` to `exit 255`, `process.exit(1)` or `sys.exit(1)`, or a bare `false`, in its lefthook command block or anywhere in its hook script, with comments skipped. `no-commit-to-branch` always stops the commit. No wired guard that exits ⇒ `PARTIAL`; the branch coverage is read only on the guards that exit. A repository whose only guard prints a warning drops from `PASS` to `PARTIAL` | | `2026-11-10.2` | 2026-11-10 | 60 | v0.3.4 T2: new check `DOC-05` (docs are readable, COSMETIC). It measures the prose of the Markdown pages under `docs/` with the three plain-language rules of the framework's prose check: the share of sentences over 25 words (20 in a numbered step), vague words per 1,000 words, and explanation or tutorial pages without a Mermaid block. Every pass level (10%, 1, none) ⇒ `PASS`; over 25% long sentences ⇒ `FAIL`; else `PARTIAL`; under 10 sentences, or prose that is not English ⇒ `UNKNOWN`. Every repository with English docs outside a pass level gains a `DOC-05` finding | | `2026-11-10.1` | 2026-11-10 | 59 | v0.3.4 T1: new check `AGT-12` (communication rules stated, COSMETIC). It reads the entry instruction file for a section headed for writing to people (`communication`, `how to write`, `reporting` and the like) and for a Markdown file it names whose name contains `communicat`. The rules must name the reader, how to report and how to ask for a decision: all three ⇒ `PASS`; one or two missing, or a named file that does not exist ⇒ `PARTIAL`; no rules ⇒ `FAIL`; no instruction file ⇒ `UNKNOWN` (`AGT-01`'s finding). Every repository without such rules gains an `AGT-12` finding | | `2026-11-09.11` | 2026-11-09 | 58 | v0.3.3 C12.0: `EXEC-07` (competing configs, DEGRADER) counts type configs per language, `types (TypeScript)` and `types (Python)`, so `tsconfig.json` beside `mypy.ini` no longer competes. A `pyrightconfig.json` beside one mypy config, when the `typecheck` verb runs mypy and not pyright (directly or through the root Makefile), is a language-server mirror and is named in the summary. A repository with that mirror rises from `FAIL` to `PASS`; the `types` concern's summary label changes | diff --git a/audit/engine/tests/fixtures/README.md b/audit/engine/tests/fixtures/README.md index 2bd60d75..7b01117c 100644 --- a/audit/engine/tests/fixtures/README.md +++ b/audit/engine/tests/fixtures/README.md @@ -100,6 +100,7 @@ fixtures/ tool-06-local-only-commit-rule/ commit-msg hook, no CI step -> TOOL-06 PARTIAL tool-06-ci-only-commit-rule/ PR-title check in CI, no hook -> TOOL-06 PARTIAL tool-07-no-branch-guard/ wired lefthook, no protected-branch guard -> TOOL-07 FAIL + tool-07-guard-no-exit/ wired lefthook guard reads the branch but only warns, no non-zero exit -> TOOL-07 PARTIAL tool-ok-node/ the smallest node repository passing every TOOL check (hook and CI step) ci-01-no-pr-trigger/ workflow without a pull-request trigger -> CI-01 PARTIAL ci-02-parity-break/ CI runs a framework verb with no local script -> CI-02 FAIL diff --git a/audit/engine/tests/fixtures/tool-07-guard-no-exit/lefthook.yml b/audit/engine/tests/fixtures/tool-07-guard-no-exit/lefthook.yml new file mode 100644 index 00000000..22f106cb --- /dev/null +++ b/audit/engine/tests/fixtures/tool-07-guard-no-exit/lefthook.yml @@ -0,0 +1,10 @@ +# The hook chain is wired (lefthook) and a command reads the branch, but it only warns: no non-zero exit, so the +# commit goes ahead on a protected branch. +pre-commit: + commands: + protected-branch: + run: >- + branch="$(git symbolic-ref --quiet --short HEAD || true)"; + case "$branch" in main|master) echo "You are committing on $branch." >&2;; esac + lint: + run: make lint diff --git a/audit/engine/tests/packs/tool/test_tool07_branch_guard.py b/audit/engine/tests/packs/tool/test_tool07_branch_guard.py index 5d23c244..a588caf6 100644 --- a/audit/engine/tests/packs/tool/test_tool07_branch_guard.py +++ b/audit/engine/tests/packs/tool/test_tool07_branch_guard.py @@ -184,6 +184,51 @@ def test_githooks_guard_misses_a_protected_branch(self): self.assertEqual(check["verdict"], "PARTIAL") self.assertIn("does not name dev", check["summary"]) + # -- the guard must stop the commit ------------------------------------------------------------------------- + + def test_a_guard_that_only_warns_is_partial(self): + lefthook = LEFTHOOK_GUARD.replace("exit 1;;", "echo protected >&2;;") + check = self.check({"lefthook.yml": lefthook, **RULESETS}) + self.assertEqual(check["verdict"], "PARTIAL") + self.assertIn("never exits non-zero", check["summary"]) + self.assertIn("so the commit goes ahead", check["findings"][0]["statement"]) + + def test_a_commented_out_exit_does_not_stop_the_commit(self): + script = SHELL_GUARD.replace(" exit 1", " # exit 1") + check = self.check({"package.json": PACKAGE_WITH_HUSKY, ".husky/pre-commit": script}) + self.assertEqual(check["verdict"], "PARTIAL") + + def test_a_bare_false_stops_the_commit(self): + script = SHELL_GUARD.replace(" exit 1", " false") + check = self.check({"package.json": PACKAGE_WITH_HUSKY, ".husky/pre-commit": script}) + self.assertEqual(check["verdict"], "PASS") + + def test_process_exit_in_a_node_hook_stops_the_commit(self): + script = ( + "#!/usr/bin/env node\n" + "const branch = require('child_process').execSync('git branch --show-current').toString().trim();\n" + "if (branch === 'main') { console.error('protected'); process.exit(1); }\n" + ) + check = self.check({"package.json": PACKAGE_WITH_HUSKY, ".husky/pre-commit": script}) + self.assertEqual(check["verdict"], "PASS") + + def test_one_stopping_guard_is_enough_and_is_the_evidence(self): + warn = "#!/bin/sh\ngit branch --show-current\n" + makefile = "setup:\n\tgit config core.hooksPath .githooks\n" + files = {"Makefile": makefile, ".githooks/commit-msg": warn, ".githooks/pre-commit": SHELL_GUARD} + check = self.check(files) + self.assertEqual(check["verdict"], "PASS") + self.assertEqual(check["summary"], "branch guard at .githooks/pre-commit:2") + + def test_the_coverage_finding_cites_only_stopping_guards(self): + warn = "#!/bin/sh\ngit branch --show-current\necho dev main\n" + makefile = "setup:\n\tgit config core.hooksPath .githooks\n" + files = {"Makefile": makefile, ".githooks/commit-msg": warn, ".githooks/pre-commit": SHELL_GUARD, **RULESETS} + check = self.check(files) + self.assertEqual(check["verdict"], "PARTIAL") + self.assertIn("does not name dev", check["summary"]) + self.assertNotIn("commit-msg", check["summary"]) + # -- no guard ----------------------------------------------------------------------------------------------- def test_hooks_without_a_guard_fail(self): diff --git a/audit/engine/tests/test_fixture_matrix.py b/audit/engine/tests/test_fixture_matrix.py index d2f3dc62..0d899af6 100644 --- a/audit/engine/tests/test_fixture_matrix.py +++ b/audit/engine/tests/test_fixture_matrix.py @@ -109,6 +109,7 @@ "tool-06-local-only-commit-rule": {"TOOL-06": "PARTIAL"}, "tool-06-ci-only-commit-rule": {"TOOL-06": "PARTIAL"}, "tool-07-no-branch-guard": {"TOOL-05": "PASS", "TOOL-07": "FAIL"}, + "tool-07-guard-no-exit": {"TOOL-05": "PASS", "TOOL-07": "PARTIAL"}, "tool-ok-node": { "TOOL-01": "PASS", "TOOL-02": "PASS", diff --git a/dev/eval/expected/fixtures/tool-07-guard-no-exit.json b/dev/eval/expected/fixtures/tool-07-guard-no-exit.json new file mode 100644 index 00000000..54f68798 --- /dev/null +++ b/dev/eval/expected/fixtures/tool-07-guard-no-exit.json @@ -0,0 +1,20 @@ +{ + "expected_version": "0.1", + "repo_label": "tool-07-guard-no-exit", + "repo_sha": "457ee85b35aef1cd95bc8ae3d22937e8cb4efd8f", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'tool-07-guard-no-exit' (audit/engine/tests/fixtures/tool-07-guard-no-exit), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "checks": [ + { + "id": "TOOL-05", + "acceptable_verdicts": [ + "PASS" + ] + }, + { + "id": "TOOL-07", + "acceptable_verdicts": [ + "PARTIAL" + ] + } + ] +} From 54c10e030c14a56edef057d6f82e554eeda5e946 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Tue, 10 Nov 2026 06:39:18 +0530 Subject: [PATCH 06/11] fix(engine): add the tool-07-guard-no-exit eval file to the secrets baseline The eval file of the new TOOL-07 fixture holds the fixture's fixed repo_sha, which detect-secrets reads as a high-entropy string. The gates of the 4.4 commit ran before the new file was staged, so make security did not scan it. The baseline now lists it, like the other eval files. Roadmap-Step: 4.4 Co-Authored-By: Claude Opus 5.5 --- .secrets.baseline | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.secrets.baseline b/.secrets.baseline index b96197b8..2d86f9d3 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1501,6 +1501,15 @@ "line_number": 4 } ], + "dev/eval/expected/fixtures/tool-07-guard-no-exit.json": [ + { + "type": "Hex High Entropy String", + "filename": "dev/eval/expected/fixtures/tool-07-guard-no-exit.json", + "hashed_secret": "4be49be8288daa370ddfcab7d155c5f709175e59", + "is_verified": false, + "line_number": 4 + } + ], "dev/eval/expected/fixtures/tool-07-no-branch-guard.json": [ { "type": "Hex High Entropy String", @@ -1597,5 +1606,5 @@ } ] }, - "generated_at": "2026-11-09T23:33:03Z" + "generated_at": "2026-11-10T01:04:19Z" } From 86cab41d9f27e5c4a6191e45f1ecbc1855816c10 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Tue, 10 Nov 2026 06:44:24 +0530 Subject: [PATCH 07/11] fix(engine): EXEC-01 reads every module of a pinned ecosystem One pinned nested module passed EXEC-01 even when a sibling module had no pin. The check now reads every module manifest (package.json, pyproject.toml, go.mod) of a pinned ecosystem. A module is reached by a pin in its own manifest, or by a pin file or (node, python) a pinning manifest in its folder or a parent folder; a parent go.mod does not reach a nested Go module. A module no pin reaches is PARTIAL and is named in the evidence. Two tests that asserted PASS for exactly this gap now assert PARTIAL; their single-module cases are kept. Ruleset 2026-11-10.4; no catalogue bump. One-defect fixture exec-01-sibling-module-unpinned with its eval file and baseline entry, ten in-process tests, rubric, audit master list and changelog. Self-audit: EXEC-01 still PASS. Roadmap-Step: 4.5 Co-Authored-By: Claude Opus 5.5 --- .secrets.baseline | 11 ++- audit/checks/09-execution-determinism.md | 15 ++-- audit/engine/audit/__init__.py | 2 +- audit/engine/audit/rules/checks/execution.py | 86 ++++++++++++++++++- audit/engine/docs/reference/changelog.md | 1 + audit/engine/tests/fixtures/README.md | 1 + .../exec-01-sibling-module-unpinned/README.md | 10 +++ .../tools/fmt/go.mod | 3 + .../tools/lint/go.mod | 1 + .../packs/exec/test_exec01_nested_pins.py | 85 +++++++++++++++++- audit/engine/tests/test_fixture_matrix.py | 1 + .../exec-01-sibling-module-unpinned.json | 14 +++ docs/reference/audit-master.md | 3 +- 13 files changed, 218 insertions(+), 15 deletions(-) create mode 100644 audit/engine/tests/fixtures/exec-01-sibling-module-unpinned/README.md create mode 100644 audit/engine/tests/fixtures/exec-01-sibling-module-unpinned/tools/fmt/go.mod create mode 100644 audit/engine/tests/fixtures/exec-01-sibling-module-unpinned/tools/lint/go.mod create mode 100644 dev/eval/expected/fixtures/exec-01-sibling-module-unpinned.json diff --git a/.secrets.baseline b/.secrets.baseline index 2d86f9d3..70613d57 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1114,6 +1114,15 @@ "line_number": 4 } ], + "dev/eval/expected/fixtures/exec-01-sibling-module-unpinned.json": [ + { + "type": "Hex High Entropy String", + "filename": "dev/eval/expected/fixtures/exec-01-sibling-module-unpinned.json", + "hashed_secret": "44c3dcd5a88e7040f1a3420ecd1bae542cd62db1", + "is_verified": false, + "line_number": 4 + } + ], "dev/eval/expected/fixtures/exec-01-unpinned.json": [ { "type": "Hex High Entropy String", @@ -1606,5 +1615,5 @@ } ] }, - "generated_at": "2026-11-10T01:04:19Z" + "generated_at": "2026-11-10T01:09:28Z" } diff --git a/audit/checks/09-execution-determinism.md b/audit/checks/09-execution-determinism.md index 09d7459c..08bcba06 100644 --- a/audit/checks/09-execution-determinism.md +++ b/audit/checks/09-execution-determinism.md @@ -35,8 +35,11 @@ Method automated Look for Each detected pinnable ecosystem's own pin file or declaration, named above; absent any of them, `.tool-versions` for a matching line. Rubric PASS: every detected pinnable ecosystem has at least one of its own pin files, or a - matching `.tool-versions` line, or (node/python/go) the manifest-level constraint. - PARTIAL: not used by this check — any one ecosystem left unpinned fails the whole check. + matching `.tool-versions` line, or (node/python/go) the manifest-level constraint, and a + pin reaches every module manifest of each pinned ecosystem. + PARTIAL: every ecosystem is pinned, but a module manifest of a pinned ecosystem has no pin + of its own and no pin in its folder or a parent folder. An ecosystem left unpinned still + fails the whole check. FAIL: at least one detected pinnable ecosystem has none of its pin evidence. NOT_APPLICABLE: never. UNKNOWN when: no ecosystem from the pinnable set (node, python, jvm, dotnet, rust, ruby, @@ -51,9 +54,11 @@ Remediation `pin-execution-environment` Pitfalls A monorepo where only one of several ecosystems is pinned still reads `FAIL` overall — the engine fails the whole check on the first unpinned ecosystem, so a `FAIL` summary naming one ecosystem does not mean the others are also unpinned; read which ecosystem - the finding actually names before writing remediation. The reverse holds within one - ecosystem: one pinned nested module passes it even when a sibling module has no pin, so - read each module's manifest before you accept a `PASS` in a monorepo. + the finding actually names before writing remediation. Within one ecosystem, each module + manifest (`package.json`, `pyproject.toml`, `go.mod`) must be reached by a pin: its own + manifest, or a pin file or (node, python) a pinning manifest in its folder or a parent + folder; one unreached module reads `PARTIAL` and is named in the evidence. A nested + `.tool-versions` is not read, and a parent `go.mod` does not pin a nested Go module. ## EXEC-02 — Lockfile committed and consistent diff --git a/audit/engine/audit/__init__.py b/audit/engine/audit/__init__.py index 3a84610e..b8a2bbc9 100644 --- a/audit/engine/audit/__init__.py +++ b/audit/engine/audit/__init__.py @@ -24,7 +24,7 @@ #: Bumped whenever a rule pack, a check, or the instruction-variant table changes. #: Recorded in every report's provenance block so a report can be attributed to an #: exact catalogue revision (docs: Scan Engine & Tiering, § 7 Determinism). -__ruleset_revision__ = "2026-11-10.3" +__ruleset_revision__ = "2026-11-10.4" #: Highest framework phase this build evaluates. Phase 1 only in v1. __phase_ceiling__ = 1 diff --git a/audit/engine/audit/rules/checks/execution.py b/audit/engine/audit/rules/checks/execution.py index efdd0d9e..163efb33 100644 --- a/audit/engine/audit/rules/checks/execution.py +++ b/audit/engine/audit/rules/checks/execution.py @@ -12,8 +12,9 @@ * **EXEC-01** — a toolchain pin appropriate to each **detected** ecosystem (``.nvmrc``, ``engines``, ``.python-version``/``requires-python``, ``gradlew``/``mvnw``, ``global.json``, ``rust-toolchain.toml``, the ``go`` directive). Pin files and manifests count at any depth, so a nested - ``go.mod``'s ``go`` directive, a nested ``requires-python`` or a nested ``engines.node`` pins the ecosystem; one - pin is enough. No recognised ecosystem ⇒ ``UNKNOWN``; an ecosystem with no pin in the table (php) is not + ``go.mod``'s ``go`` directive, a nested ``requires-python`` or a nested ``engines.node`` pins the ecosystem. A + pinned ecosystem with a module manifest that no pin reaches (:func:`_unpinned_modules`: not its own manifest, nor + a pin file or a node/python pinning manifest in its folder or a parent) ⇒ ``PARTIAL``. No recognised ecosystem ⇒ ``UNKNOWN``; an ecosystem with no pin in the table (php) is not asserted against. * **EXEC-02** — the ecosystem's lockfile is **tracked** (``audit.ignore.tracked_files``) and its declared dependencies agree with the manifest. ``tracked_files`` returning ``None`` (not a git @@ -168,18 +169,94 @@ def _pin_evidence(inv: Inventory, ecosystem: str) -> list[str]: return sorted(dict.fromkeys(evidence)) +#: The manifest that makes a module of the ecosystem, and what in it pins the toolchain. +_MODULE_MANIFESTS = {"node": "package.json", "python": "pyproject.toml", "go": "go.mod"} + +#: At most this many unpinned modules are named in a summary; the finding's evidence lists them all. +_MODULES_NAMED = 5 + + +def _dir_of(rel: str) -> str: + return rel.rsplit("/", 1)[0] if "/" in rel else "" + + +def _manifest_pins(inv: Inventory, ecosystem: str, rel: str) -> bool: + text = inv.read(rel) or "" + if ecosystem == "node": + try: + engines = json.loads(text or "{}").get("engines") + except (json.JSONDecodeError, AttributeError): + return False + return isinstance(engines, dict) and bool(engines.get("node")) + if ecosystem == "python": + return re.search(r"(?m)^\s*requires-python\s*=", text) is not None + return re.search(r"(?m)^\s*go\s+\d", text) is not None + + +def _unpinned_modules(inv: Inventory, ecosystem: str) -> list[str]: + """The module manifests of ``ecosystem`` that no pin reaches. A pin reaches a module when the module's own + manifest pins, or when its folder or a parent folder holds a pin file (``.nvmrc``, ``.python-version``, a + ``.tool-versions`` line) or, for node and python, a pinning manifest (a workspace root's ``engines.node`` or + ``requires-python``). A Go module is its own toolchain unit, so a parent ``go.mod`` does not reach it.""" + name = _MODULE_MANIFESTS.get(ecosystem) + if name is None: + return [] + manifests = _manifests(inv, name) + pin_dirs: set[str] = set() + pin_names = set(_PIN_FILES.get(ecosystem, ())) + pattern = _TOOL_VERSIONS.get(ecosystem) + for entry in inv.files: + base = entry.rel.rsplit("/", 1)[-1] + if base in pin_names or ( + base == ".tool-versions" and pattern is not None and pattern.search(inv.read(entry.rel) or "") + ): + pin_dirs.add(_dir_of(entry.rel)) + pinning = {rel for rel in manifests if _manifest_pins(inv, ecosystem, rel)} + if ecosystem != "go": + pin_dirs |= {_dir_of(rel) for rel in pinning} + + def reached(folder: str) -> bool: + return any(pin == "" or folder == pin or folder.startswith(pin + "/") for pin in pin_dirs) + + return [rel for rel in manifests if rel not in pinning and not reached(_dir_of(rel))] + + def check_exec01(*, spec, target, inventory, stack, components, session) -> CheckOutcome: ecosystems = sorted(stack.ecosystems & _PINNABLE) if not ecosystems: return _unknown(spec, "no recognised ecosystem, so no toolchain pin can be chosen") findings: list[Finding] = [] + partial: list[Finding] = [] pinned: list[str] = [] unpinned: list[str] = [] + gaps: list[str] = [] for ecosystem in ecosystems: evidence = _pin_evidence(inventory, ecosystem) if evidence: pinned.append(f"{ecosystem} ({', '.join(evidence)})") + modules = _unpinned_modules(inventory, ecosystem) + if modules: + named = ", ".join(modules[:_MODULES_NAMED]) + more = f" and {len(modules) - _MODULES_NAMED} more" if len(modules) > _MODULES_NAMED else "" + gaps.append(f"{ecosystem} ({named}{more})") + partial.append( + Finding( + check=spec.id, + severity=spec.severity, + phase=spec.phase, + verdict=Verdict.PARTIAL, + statement=statement( + f"reproduce the {ecosystem} toolchain in every module", + f"{len(modules)} {ecosystem} module(s) carry no pin and sit under no pinned folder", + ), + remediation=f"Pin each {ecosystem} module: a pin in its own manifest, or a " + f"{', '.join(_PIN_FILES.get(ecosystem) or ('.tool-versions',))} in its folder or a " + "parent folder.", + path=ecosystem, + evidence=[Evidence(rel) for rel in modules], + ) + ) continue unpinned.append(ecosystem) findings.append( @@ -200,7 +277,10 @@ def check_exec01(*, spec, target, inventory, stack, components, session) -> Chec ) if findings: - return _outcome(spec, Verdict.FAIL, f"unpinned: {', '.join(unpinned)}", findings) + return _outcome(spec, Verdict.FAIL, f"unpinned: {', '.join(unpinned)}", findings + partial) + if partial: + summary = "pinned: " + "; ".join(pinned) + "; modules no pin reaches: " + "; ".join(gaps) + return _outcome(spec, Verdict.PARTIAL, summary, partial) return _outcome(spec, Verdict.PASS, "pinned: " + "; ".join(pinned)) diff --git a/audit/engine/docs/reference/changelog.md b/audit/engine/docs/reference/changelog.md index 1fa1b215..dbbae71f 100644 --- a/audit/engine/docs/reference/changelog.md +++ b/audit/engine/docs/reference/changelog.md @@ -13,6 +13,7 @@ the record of how; git has the rest. | Revision | Date | Scoreable checks | What it is | |---|---|---|---| +| `2026-11-10.4` | 2026-11-10 | 60 | v0.3.5.1 item 4.5: `EXEC-01` (toolchain pinned, BLOCKER) reads every module manifest (`package.json`, `pyproject.toml`, `go.mod`) of a pinned ecosystem. A module is reached by a pin in its own manifest, or by a pin file (`.nvmrc`, `.python-version`, a root `.tool-versions` line) or, for node and python, a pinning manifest in its folder or a parent folder; a parent `go.mod` does not reach a nested Go module. A module no pin reaches ⇒ `PARTIAL`, with every such module in the evidence. A monorepo where one pinned module covered an unpinned sibling drops from `PASS` to `PARTIAL` | | `2026-11-10.3` | 2026-11-10 | 60 | v0.3.5.1 item 4.4: `TOOL-07` (local guard against commits to a protected branch, COSMETIC) reads whether a guard that reads the branch also exits non-zero: `exit 1` to `exit 255`, `process.exit(1)` or `sys.exit(1)`, or a bare `false`, in its lefthook command block or anywhere in its hook script, with comments skipped. `no-commit-to-branch` always stops the commit. No wired guard that exits ⇒ `PARTIAL`; the branch coverage is read only on the guards that exit. A repository whose only guard prints a warning drops from `PASS` to `PARTIAL` | | `2026-11-10.2` | 2026-11-10 | 60 | v0.3.4 T2: new check `DOC-05` (docs are readable, COSMETIC). It measures the prose of the Markdown pages under `docs/` with the three plain-language rules of the framework's prose check: the share of sentences over 25 words (20 in a numbered step), vague words per 1,000 words, and explanation or tutorial pages without a Mermaid block. Every pass level (10%, 1, none) ⇒ `PASS`; over 25% long sentences ⇒ `FAIL`; else `PARTIAL`; under 10 sentences, or prose that is not English ⇒ `UNKNOWN`. Every repository with English docs outside a pass level gains a `DOC-05` finding | | `2026-11-10.1` | 2026-11-10 | 59 | v0.3.4 T1: new check `AGT-12` (communication rules stated, COSMETIC). It reads the entry instruction file for a section headed for writing to people (`communication`, `how to write`, `reporting` and the like) and for a Markdown file it names whose name contains `communicat`. The rules must name the reader, how to report and how to ask for a decision: all three ⇒ `PASS`; one or two missing, or a named file that does not exist ⇒ `PARTIAL`; no rules ⇒ `FAIL`; no instruction file ⇒ `UNKNOWN` (`AGT-01`'s finding). Every repository without such rules gains an `AGT-12` finding | diff --git a/audit/engine/tests/fixtures/README.md b/audit/engine/tests/fixtures/README.md index 7b01117c..72431e3f 100644 --- a/audit/engine/tests/fixtures/README.md +++ b/audit/engine/tests/fixtures/README.md @@ -67,6 +67,7 @@ fixtures/ sec-01-ok/ .env present but untracked + ignored (needs git) -> SEC-01…03 PASS exec-01-unpinned/ node project, no toolchain pin -> EXEC-01 FAIL exec-01-nested-go-unpinned/ a nested go.mod with no go directive -> EXEC-01 FAIL + exec-01-sibling-module-unpinned/ one go.mod pins Go, its sibling does not -> EXEC-01 PARTIAL exec-01-python-source-only/ a standard-library-only Python tool, no manifest, no pin -> EXEC-01 FAIL, EXEC-02 UNKNOWN exec-02-no-lockfile/ node project, no lockfile (needs git) -> EXEC-02 FAIL exec-02-nested-lockfile/ a tracked lockfile in `packages/app`, none at the root (needs git) -> EXEC-02 PARTIAL diff --git a/audit/engine/tests/fixtures/exec-01-sibling-module-unpinned/README.md b/audit/engine/tests/fixtures/exec-01-sibling-module-unpinned/README.md new file mode 100644 index 00000000..80a898e2 --- /dev/null +++ b/audit/engine/tests/fixtures/exec-01-sibling-module-unpinned/README.md @@ -0,0 +1,10 @@ +# exec-01-sibling-module-unpinned + +Two Go tool modules. `tools/fmt/go.mod` pins Go with a `go` directive; its sibling `tools/lint/go.mod` has none. +Run the command below from each module's folder. + +## Development + +```bash +go mod download +``` diff --git a/audit/engine/tests/fixtures/exec-01-sibling-module-unpinned/tools/fmt/go.mod b/audit/engine/tests/fixtures/exec-01-sibling-module-unpinned/tools/fmt/go.mod new file mode 100644 index 00000000..378b09fe --- /dev/null +++ b/audit/engine/tests/fixtures/exec-01-sibling-module-unpinned/tools/fmt/go.mod @@ -0,0 +1,3 @@ +module example.com/fmt + +go 1.24 diff --git a/audit/engine/tests/fixtures/exec-01-sibling-module-unpinned/tools/lint/go.mod b/audit/engine/tests/fixtures/exec-01-sibling-module-unpinned/tools/lint/go.mod new file mode 100644 index 00000000..662f3f8b --- /dev/null +++ b/audit/engine/tests/fixtures/exec-01-sibling-module-unpinned/tools/lint/go.mod @@ -0,0 +1 @@ +module example.com/lint diff --git a/audit/engine/tests/packs/exec/test_exec01_nested_pins.py b/audit/engine/tests/packs/exec/test_exec01_nested_pins.py index a635fb50..9f212723 100644 --- a/audit/engine/tests/packs/exec/test_exec01_nested_pins.py +++ b/audit/engine/tests/packs/exec/test_exec01_nested_pins.py @@ -6,6 +6,9 @@ A repository whose only Go module is `tools/osv/go.mod` pins Go with that file's `go` directive. Before v0.3.3 C13 the check found the nested `go.mod` (the stack was detected) but read only a root `go.mod`, so it failed a pinned module. The same held for a nested `requires-python` and a nested `engines.node`. + +Since v0.3.5.1 item 4.5 (ruleset 2026-11-10.4) one pinned module no longer covers its siblings: a module manifest +that no pin reaches (its own manifest, or a pin in its folder or a parent folder) makes the check `PARTIAL`. """ from __future__ import annotations @@ -45,7 +48,11 @@ def check(self, files: dict[str, str]) -> dict: components=None, session=None, ) - return {"verdict": outcome.verdict.value, "summary": outcome.summary} + return { + "verdict": outcome.verdict.value, + "summary": outcome.summary, + "findings": [finding.to_dict() for finding in outcome.findings], + } def test_nested_go_directive_pins_go(self): check = self.check({"tools/osv/go.mod": "module example.com/osv\n\ngo 1.24\n"}) @@ -57,15 +64,79 @@ def test_nested_go_mod_without_a_directive_fails(self): self.assertEqual(check["verdict"], "FAIL") self.assertIn("unpinned: go", check["summary"]) - def test_one_pinned_module_is_enough(self): + def test_one_pinned_module_does_not_cover_an_unpinned_sibling(self): check = self.check( { "tools/a/go.mod": "module example.com/a\n", "tools/b/go.mod": "module example.com/b\n\ngo 1.24\n", } ) + self.assertEqual(check["verdict"], "PARTIAL") + self.assertIn("pinned: go (tools/b/go.mod (go directive))", check["summary"]) + self.assertIn("modules no pin reaches: go (tools/a/go.mod)", check["summary"]) + self.assertEqual([e["path"] for e in check["findings"][0]["evidence"]], ["tools/a/go.mod"]) + + def test_a_parent_go_mod_does_not_pin_a_nested_module(self): + check = self.check({"go.mod": "module example.com/root\n\ngo 1.23\n", "tools/a/go.mod": "module a\n"}) + self.assertEqual(check["verdict"], "PARTIAL") + + def test_every_module_pinned_passes(self): + check = self.check( + { + "tools/a/go.mod": "module example.com/a\n\ngo 1.24\n", + "tools/b/go.mod": "module example.com/b\n\ngo 1.24\n", + } + ) self.assertEqual(check["verdict"], "PASS") - self.assertIn("tools/b/go.mod", check["summary"]) + + def test_a_root_pin_file_reaches_every_module(self): + files = { + ".python-version": "3.12\n", + "pyproject.toml": '[project]\nname = "root"\n', + "tools/x/pyproject.toml": '[project]\nname = "x"\n', + } + self.assertEqual(self.check(files)["verdict"], "PASS") + + def test_a_pin_file_reaches_the_modules_below_it(self): + files = {"apps/.nvmrc": "22\n", "apps/web/package.json": json.dumps({"name": "web"})} + self.assertEqual(self.check(files)["verdict"], "PASS") + + def test_a_root_tool_versions_line_reaches_every_module(self): + files = {".tool-versions": "nodejs 22.1.0\n", "apps/web/package.json": json.dumps({"name": "web"})} + self.assertEqual(self.check(files)["verdict"], "PASS") + + def test_a_workspace_root_pin_reaches_its_members(self): + root = json.dumps({"name": "root", "engines": {"node": ">=22"}, "workspaces": ["apps/*"]}) + files = {"package.json": root, "apps/web/package.json": json.dumps({"name": "web"})} + self.assertEqual(self.check(files)["verdict"], "PASS") + + def test_a_pin_in_a_sibling_folder_does_not_reach(self): + files = { + "apps/api/.nvmrc": "22\n", + "apps/api/package.json": json.dumps({"name": "api"}), + "apps/web/package.json": json.dumps({"name": "web"}), + } + check = self.check(files) + self.assertEqual(check["verdict"], "PARTIAL") + self.assertIn("node (apps/web/package.json)", check["summary"]) + + def test_a_long_list_of_modules_is_cut_in_the_summary(self): + files = {f"m{i}/go.mod": f"module m{i}\n" for i in range(7)} + files["pinned/go.mod"] = "module p\n\ngo 1.24\n" + check = self.check(files) + self.assertEqual(check["verdict"], "PARTIAL") + self.assertIn("and 2 more", check["summary"]) + self.assertEqual(len(check["findings"][0]["evidence"]), 7) + + def test_a_fail_keeps_the_partial_findings_of_other_ecosystems(self): + files = { + "tools/a/go.mod": "module a\n", + "tools/b/go.mod": "module b\n\ngo 1.24\n", + "web/package.json": json.dumps({"name": "web"}), + } + check = self.check(files) + self.assertEqual(check["verdict"], "FAIL") + self.assertEqual(sorted(f["verdict"] for f in check["findings"]), ["FAIL", "PARTIAL"]) def test_the_shallowest_pinning_manifest_is_the_evidence(self): check = self.check( @@ -84,10 +155,16 @@ def test_nested_requires_python_pins_python(self): def test_nested_engines_pins_node(self): manifest = json.dumps({"name": "web", "engines": {"node": ">=22"}}) - check = self.check({"package.json": json.dumps({"name": "root"}), "apps/web/package.json": manifest}) + check = self.check({"apps/web/package.json": manifest}) self.assertEqual(check["verdict"], "PASS") self.assertIn("apps/web/package.json (engines.node)", check["summary"]) + def test_a_pinned_member_does_not_pin_an_unpinned_root(self): + manifest = json.dumps({"name": "web", "engines": {"node": ">=22"}}) + check = self.check({"package.json": json.dumps({"name": "root"}), "apps/web/package.json": manifest}) + self.assertEqual(check["verdict"], "PARTIAL") + self.assertIn("node (package.json)", check["summary"]) + def test_nested_manifest_without_a_pin_still_fails(self): check = self.check({"apps/web/package.json": json.dumps({"name": "web"})}) self.assertEqual(check["verdict"], "FAIL") diff --git a/audit/engine/tests/test_fixture_matrix.py b/audit/engine/tests/test_fixture_matrix.py index 0d899af6..dc8f0673 100644 --- a/audit/engine/tests/test_fixture_matrix.py +++ b/audit/engine/tests/test_fixture_matrix.py @@ -145,6 +145,7 @@ # -- EXEC --------------------------------------------------------------------------------- "exec-01-unpinned": {"EXEC-01": "FAIL"}, "exec-01-nested-go-unpinned": {"EXEC-01": "FAIL"}, + "exec-01-sibling-module-unpinned": {"EXEC-01": "PARTIAL"}, "exec-01-python-source-only": {"EXEC-01": "FAIL", "EXEC-02": "UNKNOWN"}, "exec-02-no-lockfile": {"EXEC-02": "FAIL"}, "exec-02-nested-lockfile": {"EXEC-02": "PARTIAL"}, diff --git a/dev/eval/expected/fixtures/exec-01-sibling-module-unpinned.json b/dev/eval/expected/fixtures/exec-01-sibling-module-unpinned.json new file mode 100644 index 00000000..95391178 --- /dev/null +++ b/dev/eval/expected/fixtures/exec-01-sibling-module-unpinned.json @@ -0,0 +1,14 @@ +{ + "expected_version": "0.1", + "repo_label": "exec-01-sibling-module-unpinned", + "repo_sha": "51d43e485efbcc01f5d783bb5139292823937a0d", + "notes": "Generated by dev/eval/generate_fixture_expected.py from test_fixture_matrix.py's MATRIX row for 'exec-01-sibling-module-unpinned' (audit/engine/tests/fixtures/exec-01-sibling-module-unpinned), a one-defect engine fixture committed into a throwaway git repository with a fixed commit so repo_sha is reproducible.", + "checks": [ + { + "id": "EXEC-01", + "acceptable_verdicts": [ + "PARTIAL" + ] + } + ] +} diff --git a/docs/reference/audit-master.md b/docs/reference/audit-master.md index 017adf60..8d853906 100644 --- a/docs/reference/audit-master.md +++ b/docs/reference/audit-master.md @@ -570,7 +570,8 @@ Does the same command give the same result on a fresh machine, without a human i [`pin-execution-environment`](implement-master.md#pin-execution-environment) - **Passes when:** every detected pinnable ecosystem has at least one of its own pin files, or a matching - `.tool-versions` line, or (node/python/go) the manifest-level constraint. + `.tool-versions` line, or (node/python/go) the manifest-level constraint, and a pin reaches every module manifest of + each pinned ecosystem. - **Why it matters:** An agent's runtime must be the repository's, not whatever happens to be on the host — otherwise the gate passes locally and fails in CI, or an agent reproduces nothing ([greenfield/README.md § File Tree — What Gets Created](../../greenfield/README.md#file-tree--what-gets-created)). From af0b6a1e9ea2d78326561f6faf0e61e6f7d09478 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Tue, 10 Nov 2026 06:49:47 +0530 Subject: [PATCH 08/11] docs(implement): write the Renovate security-only form from Renovate's docs Step 8 of add-repo-hygiene told the agent to confirm a guessed Renovate form. Renovate's docs give a preset for it: security:only-security-updates turns off other updates with matchPackageNames ["*"] and turns on vulnerabilityAlerts and osvVulnerabilityAlerts. The step now writes that preset with minimumReleaseAge "7 days", says vulnerabilityAlerts works on GitHub only, and names the osv.dev limits on other platforms. Roadmap-Step: 2.2 Co-Authored-By: Claude Opus 5.5 --- implement/playbooks/add-repo-hygiene.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/implement/playbooks/add-repo-hygiene.md b/implement/playbooks/add-repo-hygiene.md index 798ff3b2..75a681d1 100644 --- a/implement/playbooks/add-repo-hygiene.md +++ b/implement/playbooks/add-repo-hygiene.md @@ -277,10 +277,21 @@ the policy files, and for nothing else: (version updates off; security updates are a repository setting and an owner action), an `ignore` for `version-update:semver-major` (a major stays an advisory alert), a `security-fixes` group with `applies-to: security-updates`, and `cooldown: default-days: 7`. `security-and-minor` drops the limit and - keeps the major `ignore`; `all` drops both. For Renovate, write the same policy with `vulnerabilityAlerts`, - a `packageRules` entry with `enabled: false` for other updates, and `minimumReleaseAge: "7 days"`; confirm - the exact form in Renovate's configuration documentation before writing it, and cite the page in the pull - request body. The cooldown matches zizmor's `dependabot-cooldown` rule (CI-05). + keeps the major `ignore`; `all` drops both. The cooldown matches zizmor's `dependabot-cooldown` rule (CI-05). + For Renovate, `security-only` extends the `security:only-security-updates` preset and adds + `minimumReleaseAge: "7 days"`: + + ```json + { "extends": ["config:recommended", "security:only-security-updates"], "minimumReleaseAge": "7 days" } + ``` + + The preset turns off every other update (`packageRules` with `matchPackageNames: ["*"]` and + `enabled: false`). It turns on `vulnerabilityAlerts` and `osvVulnerabilityAlerts`. `vulnerabilityAlerts` + reads GitHub's alerts, so it works on GitHub only. On GitLab, Azure DevOps and Bitbucket, the security pull + requests come from `osvVulnerabilityAlerts` (osv.dev). Renovate marks it experimental, and it covers direct + dependencies only; say so in the pull request body. Source: Renovate's docs, "Security presets" + (`docs.renovatebot.com/presets-security/`) and the `vulnerabilityAlerts` and `osvVulnerabilityAlerts` + options (`docs.renovatebot.com/configuration-options/`). - **Owner action.** Record "turn on Dependabot security updates in the repository settings" as a note on the item: an agent can neither read nor set it. - When HYG-09 passed and has no item, name it in the pull request body and change nothing. From 18ec3b3f58c48d8bd30a7591abd1b29447736839 Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Tue, 10 Nov 2026 06:55:01 +0530 Subject: [PATCH 09/11] docs(implement): confirm the GitLab variables of the source-branch job The playbook called the GitLab variable names assumed. GitLab's predefined variables list CI_PIPELINE_SOURCE, CI_MERGE_REQUEST_SOURCE_BRANCH_NAME and CI_MERGE_REQUEST_TARGET_BRANCH_NAME with the meaning the job uses. The step now cites the page, says the branch names exist only in merge request pipelines, and warns that a pipeline with only .pre jobs does not run. Roadmap-Step: 2.3 Co-Authored-By: Claude Opus 5.5 --- implement/playbooks/add-repo-hygiene.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/implement/playbooks/add-repo-hygiene.md b/implement/playbooks/add-repo-hygiene.md index 75a681d1..40b64954 100644 --- a/implement/playbooks/add-repo-hygiene.md +++ b/implement/playbooks/add-repo-hygiene.md @@ -241,8 +241,10 @@ the policy files, and for nothing else: fi ``` - GitLab CI (the owner also turns on "Pipelines must succeed"). The variable names are assumed from GitLab's - predefined variables; confirm them in GitLab's documentation before you write the job: + GitLab CI (the owner also turns on "Pipelines must succeed"). The variables are GitLab's predefined ones + (`docs.gitlab.com/ci/variables/predefined_variables/`); the two branch names exist only in merge request + pipelines. GitLab does not run a pipeline that holds only `.pre` jobs, so the file needs a job in another + stage: ```yaml check-source-branch: From 9db8c0b325cab76cc281af01ed46a8e49d1d6fed Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Tue, 10 Nov 2026 07:04:42 +0530 Subject: [PATCH 10/11] docs(implement): add the Dependabot pre-commit ecosystem to step 8 GitHub's list of supported ecosystems has pre-commit: Dependabot updates the rev: lines of .pre-commit-config.yaml, with version updates only and no security updates. Step 8 of add-repo-hygiene now adds a pre-commit entry when the file exists and the policy is not security-only. The coverage bullet is rewrapped, and its 45-word sentence is split. Roadmap-Step: 2.4 Co-Authored-By: Claude Opus 5.5 --- implement/playbooks/add-repo-hygiene.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/implement/playbooks/add-repo-hygiene.md b/implement/playbooks/add-repo-hygiene.md index 40b64954..cdfe25d8 100644 --- a/implement/playbooks/add-repo-hygiene.md +++ b/implement/playbooks/add-repo-hygiene.md @@ -267,10 +267,12 @@ the policy files, and for nothing else: - **Tool.** Keep the tool the repository already uses. With none: Dependabot on GitHub, Renovate on GitLab, Azure DevOps or Bitbucket, where Dependabot does not run. - **Coverage.** One entry per ecosystem the audit detected from a manifest, with the `package-ecosystem` the - stack profile names (Node `npm`, Python `pip` or `uv`, Java `maven` or `gradle`, Go `gomod`, Rust `cargo`, - .NET `nuget`, Ruby `bundler`, PHP `composer`), plus `github-actions` when the repository has GitHub - workflows. Set `directory` (or `directories`) to where each manifest lives. Renovate needs no entry per - ecosystem; remove a detected one from `enabledManagers` only when the owner asks. + stack profile names (Node `npm`, Python `pip` or `uv`, Java `maven` or `gradle`, Go `gomod`, Rust `cargo`, .NET + `nuget`, Ruby `bundler`, PHP `composer`). Add `github-actions` when the repository has GitHub workflows. With a + `.pre-commit-config.yaml` and a policy other than `security-only`, add a `pre-commit` entry: Dependabot updates + the hooks' `rev:` lines. It has no security updates for this ecosystem (GitHub's "Supported ecosystems and + repositories" page). Set `directory` (or `directories`) to where each manifest lives. Renovate needs no entry + per ecosystem; remove a detected one from `enabledManagers` only when the owner asks. - **The `target-branch` trap.** Dependabot drops an entry's `ignore` and `groups` for security updates when the entry sets `target-branch`; security pull requests always go to the default branch. If `target-branch` names the default branch, delete the line. If it names another branch, do not change it: ask the owner whether to From ddf8b05d1e64eaa670239af84abfb7d4a52549cf Mon Sep 17 00:00:00 2001 From: "K. C. Ramakrishna" Date: Tue, 10 Nov 2026 07:31:45 +0530 Subject: [PATCH 11/11] fix(implement): store commit ids in the state as 12 digits progress.py wrote the framework commits and the source reports' git_sha as 12 digits, but copied --commit and --merged-sha as given, so a full 40-digit id could land in the state, where secret scanners read it as a token. Both are now cut to 12 digits; a shorter id (7 to 11) is kept. The schema still accepts 7 to 40, so its version does not change. state.md, the Tools CLI reference and two new tests say so. Roadmap-Step: 1.2 Co-Authored-By: Claude Opus 5.5 --- docs/reference/tools-cli.md | 4 ++-- implement/state.md | 2 +- implement/tools/progress.py | 4 ++-- implement/tools/tests/test_progress_transitions.py | 13 +++++++++++++ 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/reference/tools-cli.md b/docs/reference/tools-cli.md index bfa5ae18..2ba2d976 100644 --- a/docs/reference/tools-cli.md +++ b/docs/reference/tools-cli.md @@ -887,7 +887,7 @@ python3 implement/tools/progress.py --describe | `--framework-rev` | `init`: the framework commit to record, cut to 12 digits. Default: the framework checkout's `HEAD`, or null outside one. | | `--label` | Every command but `init`: which state to use when the folder holds more than one. | | `FINDING STATUS` | `set-status`: the item and the status to move it to. The move must be in the state diagram of the [state contract](../../implement/state.md#item-statuses). `skipped`, `deferred` and the moves back out of them have their own commands. | -| `--commit` | `set-status`: a commit SHA to add to the item (repeatable). | +| `--commit` | `set-status`: a commit SHA to add to the item (repeatable), 7 to 40 digits, stored cut to 12. | | `--verification-command`, `--verification-result` | `set-status`: the verification and what it gave; given together. `done` needs one. | | `--debt`, `--clear-debt` | `set-status`: record debt (repeatable; same `what` replaces) or drop it. `partial` needs debt; `done` needs none. | | `--reason` | The reason. Required by `failed`, `blocked`, `skip`, `defer` and `pref`. | @@ -896,7 +896,7 @@ python3 implement/tools/progress.py --describe | `--by` | Who decided, `owner` or `agent` (default `agent`). `skip`, `unskip`, `defer`, `undefer` and `pref` refuse anything but `owner`: the agent relays the owner's answer; the tool cannot see who is typing. | | `--phase` | Every write: the step recorded in the log entry and on the item or preference. | | `--render` | Every write: also replace the Markdown view. Without it the tool warns that the view is out of date. | -| `--status`, `--branch`, `--pr`, `--merged-sha` | `theme`: the theme's new status (it only moves forward) and the data that status needs: a branch, then a PR, then the merge SHA. | +| `--status`, `--branch`, `--pr`, `--merged-sha` | `theme`: the theme's new status (it only moves forward) and the data that status needs: a branch, then a PR, then the merge SHA (stored cut to 12 digits). | | `--json` | `show`: print the state itself instead of the summary. Long lists are cut to fit the output and `truncated` is set; the log keeps its newest entries. | **Outputs** diff --git a/implement/state.md b/implement/state.md index 6999e5ba..a2d03f1f 100644 --- a/implement/state.md +++ b/implement/state.md @@ -156,4 +156,4 @@ The schema subset cannot express these rules, so `progress.py` checks them on ev The tools write the framework commits (`framework.revision`, `last_framework.revision`) and each source report's `git_sha` as 12 lower-case hex digits, as the report does: a full id has the shape secret scanners read as a token. -`commits` and `merged_sha` are copied as given, 7 to 40 digits. +`progress.py` cuts `commits` and `merged_sha` to 12 digits too; a shorter id given (7 to 11 digits) is kept as it is. diff --git a/implement/tools/progress.py b/implement/tools/progress.py index 0a2e899b..0f0d31e9 100644 --- a/implement/tools/progress.py +++ b/implement/tools/progress.py @@ -410,7 +410,7 @@ def _sha(text: str) -> str: def _sha_option(text: str) -> str: if not re.fullmatch(r"[0-9a-f]{7,40}", text): raise _common.UsageError(f"--merged-sha must be 7 to 40 lower-case hex digits, not {text!r}") - return text + return short_sha(text) or text def run_init(args: Any) -> _common.Result: @@ -852,7 +852,7 @@ def parse_debt(text: str) -> dict[str, Any]: def _commit(text: str) -> str: if not re.fullmatch(r"[0-9a-f]{7,40}", text): raise _refuse(f"--commit must be 7 to 40 lower-case hex digits, not {text!r}") - return text + return short_sha(text) or text def find_item(state: dict[str, Any], finding: str) -> dict[str, Any]: diff --git a/implement/tools/tests/test_progress_transitions.py b/implement/tools/tests/test_progress_transitions.py index 7aadcf16..cba089a9 100644 --- a/implement/tools/tests/test_progress_transitions.py +++ b/implement/tools/tests/test_progress_transitions.py @@ -191,6 +191,13 @@ def test_commits_accumulate_without_repeats_and_verification_is_kept(self) -> No self.assertEqual(item["notes"], "retry") self.assertEqual(item["reason"], "y") + def test_full_commit_ids_are_stored_cut_to_12_digits(self) -> None: + f = self.to_in_progress() + full = (SHA * 4)[:40] + self.ok("set-status", f, "failed", "--reason", "x", "--commit", full, "--commit", "abcdef1") + self.ok("set-status", f, "in_progress", "--commit", full, "--note", "same commit again") + self.assertEqual(self.item(f)["commits"], [SHA, "abcdef1"]) + def test_leaving_failed_clears_the_reason(self) -> None: f = self.to_in_progress() self.ok("set-status", f, "blocked", "--reason", "waits") @@ -427,6 +434,12 @@ def test_a_theme_walks_forward(self) -> None: ) # fmt: skip self.assertIn("theme T1 pr_open to merged", self.state()["log"][-1]["event"]) + def test_a_full_merge_sha_is_stored_cut_to_12_digits(self) -> None: + self.to_in_progress() + full = (SHA2 * 4)[:40] + self.ok("theme", "T1", "--status", "merged", "--branch", "b", "--pr", "1", "--merged-sha", full) + self.assertEqual(self.theme()["merged_sha"], SHA2) + def test_a_theme_may_skip_a_step_when_it_has_the_data(self) -> None: self.ok("theme", "T2", "--status", "pr_open", "--branch", "b", "--pr", "3") self.assertEqual(self.theme("T2")["status"], "pr_open")